package file import ( "context" "errors" "fmt" "os" "path/filepath" "regexp" "strings" "sync" "sync/atomic" "syscall" "time" "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/logwisp/internal/core" "github.com/lixenwraith/logwisp/internal/plugin" "github.com/lixenwraith/logwisp/internal/session" "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) // init registers the component in plugin factory func init() { if err := plugin.RegisterSource("file", NewFileSourcePlugin); err != nil { panic(fmt.Sprintf("failed to register file source: %v", err)) } } // FileSource monitors log files and tails them type FileSource struct { // Plugin identity and session management id string proxy *session.Proxy session *session.Session // Configuration config *config.FileSourceOptions // lw's stdout and stderr when they are files: a console sink writes there, // so following them would feed lw its own output. ownSkipped is per path // already reported, touched only by monitorLoop. own []os.FileInfo ownSkipped map[string]bool scanErr string // the last scan's error, "" once it read; monitorLoop's // Application subscribers []chan core.LogEntry watchers map[string]*fileWatcher logger *log.Logger // Runtime mu sync.RWMutex ctx context.Context cancel context.CancelFunc wg sync.WaitGroup // Statistics totalEntries atomic.Uint64 droppedEntries atomic.Uint64 startTime time.Time lastEntryTime atomic.Value // time.Time } // NewFileSourcePlugin creates a file source through plugin factory func NewFileSourcePlugin( id string, configMap map[string]any, logger *log.Logger, proxy *session.Proxy, ) (source.Source, error) { opts, err := config.Decode[config.FileSourceOptions]("source", "file", configMap) if err != nil { return nil, err } fs := &FileSource{ id: id, proxy: proxy, config: opts, subscribers: make([]chan core.LogEntry, 0), watchers: make(map[string]*fileWatcher), ownSkipped: make(map[string]bool), logger: logger, } for _, f := range []*os.File{os.Stdout, os.Stderr} { if info, err := f.Stat(); err == nil && info.Mode().IsRegular() { fs.own = append(fs.own, info) } } fs.lastEntryTime.Store(time.Time{}) fs.session = proxy.CreateSession( fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Pattern), map[string]any{ "instance_id": id, "type": "file", "directory": opts.Directory, "pattern": opts.Pattern, }, ) fs.logger.Info("msg", "File source initialized", "component", "file_source", "instance_id", id, "directory", opts.Directory, "pattern", opts.Pattern, "raw", opts.Raw, "from", opts.From) return fs, nil } // Capabilities returns supported capabilities func (fs *FileSource) Capabilities() []core.Capability { return []core.Capability{ core.CapSessionAware, // Tracks sessions per file core.CapMultiSession, // Multiple file sessions } } // Subscribe returns a channel for receiving log entries func (fs *FileSource) Subscribe() <-chan core.LogEntry { fs.mu.Lock() defer fs.mu.Unlock() ch := make(chan core.LogEntry, 1000) fs.subscribers = append(fs.subscribers, ch) return ch } // Start begins the file monitoring loop func (fs *FileSource) Start() error { fs.ctx, fs.cancel = context.WithCancel(context.Background()) fs.startTime = time.Now() fs.wg.Add(1) go fs.monitorLoop() fs.logger.Info("msg", "File source started", "component", "file_source", "path", fs.config.Directory, "pattern", fs.config.Pattern, "check_interval_ms", fs.config.CheckIntervalMS) return nil } // Stop gracefully shuts down the file source and all file watchers func (fs *FileSource) Stop() { if fs.cancel != nil { fs.cancel() } fs.wg.Wait() fs.proxy.RemoveSession(fs.session.ID) fs.mu.Lock() for _, w := range fs.watchers { w.stop() } for _, ch := range fs.subscribers { close(ch) } fs.mu.Unlock() fs.logger.Info("msg", "File source stopped", "component", "file_source", "instance_id", fs.id, "path", fs.config.Directory) } // GetStats returns the source's statistics, including active watchers. func (fs *FileSource) GetStats() source.SourceStats { lastEntry, _ := fs.lastEntryTime.Load().(time.Time) fs.mu.RLock() watcherCount := int64(len(fs.watchers)) details := make(map[string]any) // Add watcher details watchers := make([]map[string]any, 0, watcherCount) for _, w := range fs.watchers { info := w.getInfo() watchers = append(watchers, map[string]any{ "directory": info.Directory, "size": info.Size, "position": info.Position, "entries_read": info.EntriesRead, "rotations": info.Rotations, "last_read": info.LastReadTime, }) } details["watchers"] = watchers details["active_watchers"] = watcherCount fs.mu.RUnlock() return source.SourceStats{ ID: fs.id, Type: "file", TotalEntries: fs.totalEntries.Load(), DroppedEntries: fs.droppedEntries.Load(), StartTime: fs.startTime, LastEntryTime: lastEntry, Details: details, } } // monitorLoop periodically scans path for new or changed files. func (fs *FileSource) monitorLoop() { defer fs.wg.Done() fs.checkTargets() ticker := time.NewTicker(time.Duration(fs.config.CheckIntervalMS) * time.Millisecond) defer ticker.Stop() for { select { case <-fs.ctx.Done(): return case <-ticker.C: fs.checkTargets() } } } // checkTargets finds matching files and ensures watchers are running for them. func (fs *FileSource) checkTargets() { files, err := fs.scanFile() fs.noteScan(err) if err != nil { return } for _, file := range files { fs.ensureWatcher(file) } fs.cleanupWatchers() } // noteScan logs when the directory cannot be read and when it can again, not // every check: a missing one at info, as it may yet be made func (fs *FileSource) noteScan(err error) { switch { case !changed(&fs.scanErr, err): case err == nil: fs.logger.Info("msg", "Directory readable", "component", "file_source", "instance_id", fs.id, "directory", fs.config.Directory) case errors.Is(err, os.ErrNotExist): fs.logger.Info("msg", "Waiting for directory", "component", "file_source", "instance_id", fs.id, "directory", fs.config.Directory) default: fs.logger.Warn("msg", "Cannot read directory", "component", "file_source", "instance_id", fs.id, "directory", fs.config.Directory, "error", err) } } // changed reports whether err differs from the last check's, kept in last // ("" for none), so a failure is logged as it starts or changes func changed(last *string, err error) bool { text := "" if err != nil { text = err.Error() } was := *last *last = text return text != was } // ensureWatcher creates and starts a new file watcher if one doesn't exist for the given path. func (fs *FileSource) ensureWatcher(path string) { fs.mu.Lock() defer fs.mu.Unlock() if _, exists := fs.watchers[path]; exists { return } w := newFileWatcher(path, fs.config.Raw, fs.config.From == "start", fs.publish, fs.logger) // A rotation renames the file out from under its watcher, so the same inode // reappears here under the archive name. Resume where it was left: from the // start would re-emit every record the file has already delivered. if position, ok := fs.readPosition(path); ok { w.position = position } fs.watchers[path] = w fs.logger.Debug("msg", "Created file watcher", "component", "file_source", "path", path) fs.wg.Add(1) go func() { defer fs.wg.Done() if err := w.watch(fs.ctx); err != nil { fs.logger.Debug("msg", "Watcher cancelled", "component", "file_source", "path", path) } fs.removeWatcher(path, w) }() } // readPosition reports how far a running watcher has read the file now at path. // Callers hold fs.mu. func (fs *FileSource) readPosition(path string) (int64, bool) { info, err := os.Stat(path) if err != nil { return 0, false } stat, ok := info.Sys().(*syscall.Stat_t) if !ok { return 0, false } for _, w := range fs.watchers { if position, ok := w.readTo(stat.Ino); ok { return position, true } } return 0, false } // removeWatcher removes only the watcher that finished. A deleted file can be // recreated before its old watcher observes stop; in that case ensureWatcher // has already installed a replacement under the same path, which must survive. func (fs *FileSource) removeWatcher(path string, watcher *fileWatcher) { fs.mu.Lock() if fs.watchers[path] == watcher { delete(fs.watchers, path) } fs.mu.Unlock() } // cleanupWatchers stops and removes watchers for files that no longer exist. func (fs *FileSource) cleanupWatchers() { fs.mu.Lock() defer fs.mu.Unlock() for path, w := range fs.watchers { if _, err := os.Stat(path); os.IsNotExist(err) { w.stop() delete(fs.watchers, path) fs.logger.Debug("msg", "Cleaned up watcher for non-existent file", "component", "file_source", "path", path) } } } // publish sends a log entry to all subscribers. func (fs *FileSource) publish(entry core.LogEntry) { fs.mu.RLock() defer fs.mu.RUnlock() fs.totalEntries.Add(1) fs.lastEntryTime.Store(entry.Time) for _, ch := range fs.subscribers { select { case ch <- entry: default: fs.droppedEntries.Add(1) fs.logger.Debug("msg", "Dropped log entry - subscriber buffer full", "component", "file_source") } } } // scanFile finds all files in the configured path that match the pattern. func (fs *FileSource) scanFile() ([]string, error) { entries, err := os.ReadDir(fs.config.Directory) if err != nil { return nil, err } // Convert glob pattern to regex regexPattern := globToRegex(fs.config.Pattern) re, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("invalid pattern regex: %w", err) } var files []string for _, entry := range entries { if entry.IsDir() { continue } path := filepath.Join(fs.config.Directory, entry.Name()) if re.MatchString(entry.Name()) && !fs.isOwn(path) { files = append(files, path) } } return files, nil } // isOwn reports whether path is lw's own stdout or stderr, warning once per path func (fs *FileSource) isOwn(path string) bool { if len(fs.own) == 0 { return false } info, err := os.Stat(path) if err != nil { return false } for _, own := range fs.own { if os.SameFile(info, own) { if !fs.ownSkipped[path] { fs.ownSkipped[path] = true fs.logger.Warn("msg", "Not following lw's own output", "component", "file_source", "instance_id", fs.id, "path", path) } return true } } return false } // globToRegex converts a simple glob pattern to a regular expression. func globToRegex(glob string) string { regex := regexp.QuoteMeta(glob) regex = strings.ReplaceAll(regex, `\*`, `.*`) regex = strings.ReplaceAll(regex, `\?`, `.`) return "^" + regex + "$" }