v0.13.1 folder restructure, test script added, format adapter async fix

2026-07-17 09:09:00 -04:00
parent 87e57784da
commit b8dd591b4b
49 changed files with 402 additions and 206 deletions
@@ -0,0 +1,364 @@
+package file
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSource("file", NewFileSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register file source: %v", err))
+	}
+}
+
+// FileSource monitors log files and tails them
+type FileSource struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Configuration
+	config *config.FileSourceOptions
+
+	// Application
+	subscribers []chan core.LogEntry
+	watchers    map[string]*fileWatcher
+	logger      *log.Logger
+
+	// Runtime
+	mu     sync.RWMutex
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+
+	// Statistics
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	startTime      time.Time
+	lastEntryTime  atomic.Value // time.Time
+}
+
+const (
+	DefaultFileSourcePattern         = "*"
+	DefaultFileSourceCheckIntervalMS = 100
+	MinFileSourceCheckIntervalMS     = 10
+)
+
+// NewFileSourcePlugin creates a file source through plugin factory
+func NewFileSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	opts := &config.FileSourceOptions{}
+
+	// Use lconfig to scan map into struct (overriding defaults)
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	// Validate and apply defaults
+	if err := lconfig.NonEmpty(opts.Directory); err != nil {
+		return nil, fmt.Errorf("directory: %w", err)
+	}
+
+	if opts.Pattern == "" {
+		opts.Pattern = DefaultFileSourcePattern
+	}
+	if opts.CheckIntervalMS <= 0 {
+		opts.CheckIntervalMS = DefaultFileSourceCheckIntervalMS
+	} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
+		return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
+	}
+
+	// Create and return plugin instance
+	fs := &FileSource{
+		id:          id,
+		proxy:       proxy,
+		config:      opts,
+		subscribers: make([]chan core.LogEntry, 0),
+		watchers:    make(map[string]*fileWatcher),
+		logger:      logger,
+	}
+	fs.lastEntryTime.Store(time.Time{})
+
+	fs.session = proxy.CreateSession(
+		fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Pattern),
+		map[string]any{
+			"instance_id": id,
+			"type":        "file",
+			"directory":   opts.Directory,
+			"pattern":     opts.Pattern,
+		},
+	)
+
+	fs.logger.Info("msg", "File source initialized",
+		"component", "file_source",
+		"instance_id", id,
+		"directory", opts.Directory,
+		"pattern", opts.Pattern)
+
+	return fs, nil
+}
+
+// Capabilities returns supported capabilities
+func (fs *FileSource) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware, // Tracks sessions per file
+		core.CapMultiSession, // Multiple file sessions
+	}
+}
+
+// Subscribe returns a channel for receiving log entries
+func (fs *FileSource) Subscribe() <-chan core.LogEntry {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	ch := make(chan core.LogEntry, 1000)
+	fs.subscribers = append(fs.subscribers, ch)
+	return ch
+}
+
+// Start begins the file monitoring loop
+func (fs *FileSource) Start() error {
+	fs.ctx, fs.cancel = context.WithCancel(context.Background())
+	fs.startTime = time.Now()
+	fs.wg.Add(1)
+	go fs.monitorLoop()
+
+	fs.logger.Info("msg", "File source started",
+		"component", "File_source",
+		"path", fs.config.Directory,
+		"pattern", fs.config.Pattern,
+		"check_interval_ms", fs.config.CheckIntervalMS)
+	return nil
+}
+
+// Stop gracefully shuts down the file source and all file watchers
+func (fs *FileSource) Stop() {
+	if fs.cancel != nil {
+		fs.cancel()
+	}
+	fs.wg.Wait()
+
+	fs.proxy.RemoveSession(fs.session.ID)
+
+	fs.mu.Lock()
+	for _, w := range fs.watchers {
+		w.stop()
+	}
+	for _, ch := range fs.subscribers {
+		close(ch)
+	}
+	fs.mu.Unlock()
+
+	fs.logger.Info("msg", "File source stopped",
+		"component", "file_source",
+		"instance_id", fs.id,
+		"path", fs.config.Directory)
+}
+
+// GetStats returns the source's statistics, including active watchers.
+func (fs *FileSource) GetStats() source.SourceStats {
+	lastEntry, _ := fs.lastEntryTime.Load().(time.Time)
+
+	fs.mu.RLock()
+	watcherCount := int64(len(fs.watchers))
+	details := make(map[string]any)
+
+	// Add watcher details
+	watchers := make([]map[string]any, 0, watcherCount)
+	for _, w := range fs.watchers {
+		info := w.getInfo()
+		watchers = append(watchers, map[string]any{
+			"directory":    info.Directory,
+			"size":         info.Size,
+			"position":     info.Position,
+			"entries_read": info.EntriesRead,
+			"rotations":    info.Rotations,
+			"last_read":    info.LastReadTime,
+		})
+	}
+	details["watchers"] = watchers
+	details["active_watchers"] = watcherCount
+	fs.mu.RUnlock()
+
+	return source.SourceStats{
+		ID:             fs.id,
+		Type:           "file",
+		TotalEntries:   fs.totalEntries.Load(),
+		DroppedEntries: fs.droppedEntries.Load(),
+		StartTime:      fs.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        details,
+	}
+}
+
+// monitorLoop periodically scans path for new or changed files.
+func (fs *FileSource) monitorLoop() {
+	defer fs.wg.Done()
+
+	fs.checkTargets()
+
+	ticker := time.NewTicker(time.Duration(fs.config.CheckIntervalMS) * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-fs.ctx.Done():
+			return
+		case <-ticker.C:
+			fs.checkTargets()
+		}
+	}
+}
+
+// checkTargets finds matching files and ensures watchers are running for them.
+func (fs *FileSource) checkTargets() {
+	files, err := fs.scanFile()
+	if err != nil {
+		fs.logger.Warn("msg", "Failed to scan file",
+			"component", "file_source",
+			"path", fs.config.Directory,
+			"pattern", fs.config.Pattern,
+			"error", err)
+		return
+	}
+
+	for _, file := range files {
+		fs.ensureWatcher(file)
+	}
+
+	fs.cleanupWatchers()
+}
+
+// ensureWatcher creates and starts a new file watcher if one doesn't exist for the given path.
+func (fs *FileSource) ensureWatcher(path string) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	if _, exists := fs.watchers[path]; exists {
+		return
+	}
+
+	w := newFileWatcher(path, fs.publish, fs.logger)
+	fs.watchers[path] = w
+
+	fs.logger.Debug("msg", "Created file watcher",
+		"component", "file_source",
+		"path", path)
+
+	fs.wg.Add(1)
+	go func() {
+		defer fs.wg.Done()
+		if err := w.watch(fs.ctx); err != nil {
+			if errors.Is(err, context.Canceled) {
+				fs.logger.Debug("msg", "Watcher cancelled",
+					"component", "file_source",
+					"path", path)
+			} else {
+				fs.logger.Error("msg", "Watcher failed",
+					"component", "file_source",
+					"path", path,
+					"error", err)
+			}
+		}
+
+		fs.mu.Lock()
+		delete(fs.watchers, path)
+		fs.mu.Unlock()
+	}()
+}
+
+// cleanupWatchers stops and removes watchers for files that no longer exist.
+func (fs *FileSource) cleanupWatchers() {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	for path, w := range fs.watchers {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			w.stop()
+			delete(fs.watchers, path)
+			fs.logger.Debug("msg", "Cleaned up watcher for non-existent file",
+				"component", "file_source",
+				"path", path)
+		}
+	}
+}
+
+// publish sends a log entry to all subscribers.
+func (fs *FileSource) publish(entry core.LogEntry) {
+	fs.mu.RLock()
+	defer fs.mu.RUnlock()
+
+	fs.totalEntries.Add(1)
+	fs.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range fs.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			fs.droppedEntries.Add(1)
+			fs.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+				"component", "file_source")
+		}
+	}
+}
+
+// scanFile finds all files in the configured path that match the pattern.
+func (fs *FileSource) scanFile() ([]string, error) {
+	entries, err := os.ReadDir(fs.config.Directory)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert glob pattern to regex
+	regexPattern := globToRegex(fs.config.Pattern)
+	re, err := regexp.Compile(regexPattern)
+	if err != nil {
+		return nil, fmt.Errorf("invalid pattern regex: %w", err)
+	}
+
+	var files []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		name := entry.Name()
+		if re.MatchString(name) {
+			files = append(files, filepath.Join(fs.config.Directory, name))
+		}
+	}
+
+	return files, nil
+}
+
+// globToRegex converts a simple glob pattern to a regular expression.
+func globToRegex(glob string) string {
+	regex := regexp.QuoteMeta(glob)
+	regex = strings.ReplaceAll(regex, `\*`, `.*`)
+	regex = strings.ReplaceAll(regex, `\?`, `.`)
+	return "^" + regex + "$"
+}
+
@@ -0,0 +1,383 @@
+package file
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"logwisp/internal/core"
+	"logwisp/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// WatcherInfo contains snapshot information about a file watcher's state
+type WatcherInfo struct {
+	Directory    string
+	Size         int64
+	Position     int64
+	ModTime      time.Time
+	EntriesRead  uint64
+	LastReadTime time.Time
+	Rotations    int64
+}
+
+// fileWatcher tails a single file, handles rotations, and sends new lines to a callback
+type fileWatcher struct {
+	directory    string
+	callback     func(core.LogEntry)
+	position     int64
+	size         int64
+	inode        uint64
+	modTime      time.Time
+	mu           sync.Mutex
+	stopped      bool
+	rotationSeq  int64
+	entriesRead  atomic.Uint64
+	lastReadTime atomic.Value // time.Time
+	logger       *log.Logger
+}
+
+// newFileWatcher creates a new watcher for a specific file path
+func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
+	w := &fileWatcher{
+		directory: directory,
+		callback:  callback,
+		position:  -1,
+		logger:    logger,
+	}
+	w.lastReadTime.Store(time.Time{})
+	return w
+}
+
+// watch starts the main monitoring loop for the file
+func (w *fileWatcher) watch(ctx context.Context) error {
+	if err := w.seekToEnd(); err != nil {
+		return fmt.Errorf("seekToEnd failed: %w", err)
+	}
+
+	ticker := time.NewTicker(core.FileWatcherPollInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			if w.isStopped() {
+				return fmt.Errorf("watcher stopped")
+			}
+			if err := w.checkFile(); err != nil {
+				// Log error but continue watching
+				w.logger.Warn("msg", "checkFile error", "error", err)
+			}
+		}
+	}
+}
+
+// stop signals the watcher to terminate its loop
+func (w *fileWatcher) stop() {
+	w.mu.Lock()
+	w.stopped = true
+	w.mu.Unlock()
+}
+
+// getInfo returns a snapshot of the watcher's current statistics
+func (w *fileWatcher) getInfo() WatcherInfo {
+	w.mu.Lock()
+	info := WatcherInfo{
+		Directory:   w.directory,
+		Size:        w.size,
+		Position:    w.position,
+		ModTime:     w.modTime,
+		EntriesRead: w.entriesRead.Load(),
+		Rotations:   w.rotationSeq,
+	}
+	w.mu.Unlock()
+
+	if lastRead, ok := w.lastReadTime.Load().(time.Time); ok {
+		info.LastReadTime = lastRead
+	}
+
+	return info
+}
+
+// checkFile examines the file for changes, rotations, or new content
+func (w *fileWatcher) checkFile() error {
+	file, err := os.Open(w.directory)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// File doesn't exist yet, keep watching
+			return nil
+		}
+		w.logger.Error("msg", "Failed to open file for checking",
+			"component", "file_watcher",
+			"directory", w.directory,
+			"error", err)
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		w.logger.Error("msg", "Failed to stat file",
+			"component", "file_watcher",
+			"directory", w.directory,
+			"error", err)
+		return err
+	}
+
+	w.mu.Lock()
+	oldPos := w.position
+	oldSize := w.size
+	oldInode := w.inode
+	oldModTime := w.modTime
+	w.mu.Unlock()
+
+	currentSize := info.Size()
+	currentModTime := info.ModTime()
+	var currentInode uint64
+
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		currentInode = stat.Ino
+	}
+
+	// Handle first time seeing a file that didn't exist before
+	if oldInode == 0 && currentInode != 0 {
+		// File just appeared, don't treat as rotation
+		w.mu.Lock()
+		w.inode = currentInode
+		w.size = currentSize
+		w.modTime = currentModTime
+		// Position stays at 0 for new files
+		w.mu.Unlock()
+		// Don't return here - continue to read content
+	}
+
+	// Check for rotation
+	rotated := false
+	rotationReason := ""
+	startPos := oldPos
+
+	// Rotation detection
+	if currentSize < oldSize {
+		// File was truncated
+		rotated = true
+		rotationReason = "size decrease"
+	} else if currentModTime.Before(oldModTime) && currentSize <= oldSize {
+		// Modification time went backwards (logrotate behavior)
+		rotated = true
+		rotationReason = "modification time reset"
+	} else if oldPos > currentSize+1024 {
+		// Our position is way beyond file size
+		rotated = true
+		rotationReason = "position beyond file size"
+	} else if oldInode != 0 && currentInode != 0 && currentInode != oldInode {
+		// Inode changed - distinguish between rotation and atomic save
+		if currentSize == 0 {
+			// Empty file with new inode = likely rotation
+			rotated = true
+			rotationReason = "inode change with empty file"
+		} else if currentSize < oldPos {
+			// New file is smaller than our position = rotation
+			rotated = true
+			rotationReason = "inode change with size less than position"
+		} else {
+			// Inode changed but file has content and size >= position
+			// This is likely an atomic save by an editor
+			// Update inode but keep position
+			w.mu.Lock()
+			w.inode = currentInode
+			w.mu.Unlock()
+
+			w.logger.Debug("msg", "Atomic file update detected",
+				"component", "file_watcher",
+				"directory", w.directory,
+				"old_inode", oldInode,
+				"new_inode", currentInode,
+				"position", oldPos,
+				"size", currentSize)
+		}
+	}
+
+	if rotated {
+		startPos = 0
+		w.mu.Lock()
+		w.rotationSeq++
+		seq := w.rotationSeq
+		w.inode = currentInode
+		w.position = 0 // Reset position on rotation
+		w.mu.Unlock()
+
+		w.callback(core.LogEntry{
+			Time:    time.Now(),
+			Source:  filepath.Base(w.directory),
+			Level:   "INFO",
+			Message: fmt.Sprintf("Log rotation detected (#%d): %s", seq, rotationReason),
+		})
+
+		w.logger.Info("msg", "Log rotation detected",
+			"component", "file_watcher",
+			"directory", w.directory,
+			"sequence", seq,
+			"reason", rotationReason)
+	}
+
+	// Read if there's new content OR if we need to continue from position
+	if currentSize > startPos {
+		if _, err := file.Seek(startPos, io.SeekStart); err != nil {
+			return err
+		}
+
+		scanner := bufio.NewScanner(file)
+		scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
+
+		for scanner.Scan() {
+			line := scanner.Text()
+			if line == "" {
+				continue
+			}
+
+			rawSize := int64(len(line))
+			entry := w.parseLine(line)
+			entry.RawSize = rawSize
+
+			w.callback(entry)
+			w.entriesRead.Add(1)
+			w.lastReadTime.Store(time.Now())
+		}
+
+		if err := scanner.Err(); err != nil {
+			w.logger.Error("msg", "Scanner error while reading file",
+				"component", "file_watcher",
+				"directory", w.directory,
+				"position", startPos,
+				"error", err)
+			return err
+		}
+
+		// Update position after successful read
+		currentPos, err := file.Seek(0, io.SeekCurrent)
+		if err != nil {
+			// Log error but don't fail - best effort position tracking
+			w.logger.Warn("msg", "Failed to get file position", "error", err)
+			// Use size as fallback position
+			currentPos = currentSize
+		}
+
+		w.mu.Lock()
+		w.position = currentPos
+		w.size = currentSize
+		w.modTime = currentModTime
+		if !rotated && currentInode != 0 {
+			w.inode = currentInode
+		}
+		w.mu.Unlock()
+
+		return scanner.Err()
+	}
+
+	// Update metadata even if no new content
+	w.mu.Lock()
+	w.size = currentSize
+	w.modTime = currentModTime
+	if currentInode != 0 {
+		w.inode = currentInode
+	}
+	w.mu.Unlock()
+
+	return nil
+}
+
+// seekToEnd sets the initial read position to the end of the file
+func (w *fileWatcher) seekToEnd() error {
+	file, err := os.Open(w.directory)
+	if err != nil {
+		if os.IsNotExist(err) {
+			w.mu.Lock()
+			w.position = 0
+			w.size = 0
+			w.modTime = time.Now()
+			w.inode = 0
+			w.mu.Unlock()
+			return nil
+		}
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	// Keep existing position (including 0)
+	// First time initialization seeks to the end of the file
+	if w.position == -1 {
+		pos, err := file.Seek(0, io.SeekEnd)
+		if err != nil {
+			return err
+		}
+		w.position = pos
+	}
+
+	w.size = info.Size()
+	w.modTime = info.ModTime()
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		w.inode = stat.Ino
+	}
+
+	return nil
+}
+
+// isStopped checks if the watcher has been instructed to stop
+func (w *fileWatcher) isStopped() bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.stopped
+}
+
+// parseLine attempts to parse a line as JSON, falling back to plain text
+func (w *fileWatcher) parseLine(line string) core.LogEntry {
+	var jsonLog struct {
+		Time    string          `json:"time"`
+		Level   string          `json:"level"`
+		Message string          `json:"msg"`
+		Fields  json.RawMessage `json:"fields"`
+	}
+
+	if err := json.Unmarshal([]byte(line), &jsonLog); err == nil {
+		timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time)
+		if err != nil {
+			timestamp = time.Now()
+		}
+
+		return core.LogEntry{
+			Time:    timestamp,
+			Source:  filepath.Base(w.directory),
+			Level:   jsonLog.Level,
+			Message: jsonLog.Message,
+			Fields:  jsonLog.Fields,
+		}
+	}
+
+	level := source.ExtractLogLevel(line)
+
+	return core.LogEntry{
+		Time:    time.Now(),
+		Source:  filepath.Base(w.directory),
+		Level:   level,
+		Message: line,
+	}
+}