v0.13.1 folder restructure, test script added, format adapter async fix

2026-07-17 09:09:00 -04:00
parent a48514a2eb
commit e96ede32ae
49 changed files with 402 additions and 206 deletions
@@ -0,0 +1,192 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	_ "logwisp/internal/source/console"
+	_ "logwisp/internal/source/file"
+	_ "logwisp/internal/source/httpchain"
+	_ "logwisp/internal/source/null"
+	_ "logwisp/internal/source/random"
+	_ "logwisp/internal/source/tcpchain"
+
+	_ "logwisp/internal/sink/console"
+	_ "logwisp/internal/sink/file"
+	_ "logwisp/internal/sink/http"
+	_ "logwisp/internal/sink/httpchain"
+	_ "logwisp/internal/sink/null"
+	_ "logwisp/internal/sink/tcp"
+	_ "logwisp/internal/sink/tcpchain"
+
+	"logwisp/internal/config"
+	"logwisp/internal/service"
+	"logwisp/internal/version"
+
+	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/log/sanitizer"
+)
+
+// bootstrapInitial handles initial service startup with status reporter
+func bootstrapInitial(ctx context.Context, cfg *config.Config) (*service.Service, context.CancelFunc, error) {
+	svc, err := bootstrapService(ctx, cfg)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to bootstrap service: %w", err)
+	}
+
+	if err := svc.Start(); err != nil {
+		return nil, nil, fmt.Errorf("failed to start service pipelines: %w", err)
+	}
+
+	var statusCancel context.CancelFunc
+	if cfg.StatusReporter {
+		statusCancel = startStatusReporter(ctx, svc)
+	}
+
+	return svc, statusCancel, nil
+}
+
+// handleReload orchestrates the entire hot-reload process including status reporter lifecycle
+func handleReload(ctx context.Context, oldSvc *service.Service, statusCancel context.CancelFunc) (*service.Service, *config.Config, context.CancelFunc, error) {
+	logger.Info("msg", "Starting configuration hot reload")
+
+	// Get updated config from the lixenwraith/config manager
+	lcfg := config.GetConfigManager()
+	if lcfg == nil {
+		err := fmt.Errorf("config manager not available for reload")
+		logger.Error("msg", "Reload failed", "error", err)
+		return nil, nil, nil, err
+	}
+
+	updatedCfgStruct, err := lcfg.AsStruct()
+	if err != nil {
+		logger.Error("msg", "Failed to get updated config for reload", "error", err, "action", "keeping current configuration")
+		return nil, nil, nil, err
+	}
+	newCfg := updatedCfgStruct.(*config.Config)
+
+	// Bootstrap a new service to ensure it's valid before touching the old one
+	logger.Debug("msg", "Bootstrapping new service with updated config")
+	newService, err := bootstrapService(ctx, newCfg)
+	if err != nil {
+		logger.Error("msg", "Failed to bootstrap new service, keeping old service running", "error", err)
+		return nil, nil, nil, err
+	}
+
+	// Gracefully shut down the old service
+	if oldSvc != nil {
+		logger.Info("msg", "Shutting down old service before activating new one")
+		oldSvc.Shutdown()
+	}
+
+	// Start the new service
+	if err := newService.Start(); err != nil {
+		logger.Error("msg", "Failed to start new service pipelines after reload. The application may be in a non-functional state.", "error", err)
+		return nil, nil, nil, fmt.Errorf("failed to start new service: %w", err)
+	}
+
+	// Manage status reporter lifecycle
+	if statusCancel != nil {
+		statusCancel()
+	}
+
+	var newStatusCancel context.CancelFunc
+	if newCfg.StatusReporter {
+		newStatusCancel = startStatusReporter(ctx, newService)
+	}
+
+	logger.Info("msg", "Configuration hot reload completed successfully")
+	return newService, newCfg, newStatusCancel, nil
+}
+
+// bootstrapService creates and initializes the main log transport service and its pipelines
+func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service, error) {
+	// Create service with logger dependency injection
+	svc, err := service.NewService(ctx, cfg, logger)
+	if err != nil {
+		logger.Error("msg", "Failed to initialize service",
+			"component", "bootstrap",
+		)
+		return nil, err
+	}
+
+	logger.Info("msg", "LogWisp started",
+		"version", version.Short(),
+	)
+
+	return svc, nil
+}
+
+// initializeLogger sets up the global logger based on the application's configuration
+func initializeLogger(cfg *config.Config) error {
+	logger = log.NewLogger()
+	logCfg := log.DefaultConfig()
+
+	if cfg.Quiet {
+		// In quiet mode, disable ALL logging output
+		logCfg.Level = 255 // A level that disables all output
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = false
+		return logger.ApplyConfig(logCfg)
+	}
+
+	// Determine log level
+	levelValue, err := log.Level(cfg.Logging.Level)
+	if err != nil {
+		return fmt.Errorf("invalid log level: %w", err)
+	}
+	logCfg.Level = levelValue
+
+	// Configure log format
+	if cfg.Logging.Format != "" {
+		logCfg.Format = cfg.Logging.Format
+	}
+	if cfg.Logging.Sanitization != "" {
+		logCfg.Sanitization = sanitizer.PolicyPreset(cfg.Logging.Sanitization)
+	}
+
+	// Configure based on output mode
+	switch cfg.Logging.Output {
+	case "none":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = false
+	case "stdout":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "stdout"
+	case "stderr":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "stderr"
+	case "split":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "split"
+	case "file":
+		logCfg.EnableFile = true
+		logCfg.EnableConsole = false
+		configureFileLogging(logCfg, cfg)
+	case "all":
+		logCfg.EnableFile = true
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "split"
+		configureFileLogging(logCfg, cfg)
+	default:
+		return fmt.Errorf("invalid log output mode: %s", cfg.Logging.Output)
+	}
+
+	return logger.ApplyConfig(logCfg)
+}
+
+// configureFileLogging sets up file-based logging parameters from the configuration
+func configureFileLogging(logCfg *log.Config, cfg *config.Config) {
+	if cfg.Logging.File != nil {
+		logCfg.Directory = cfg.Logging.File.Directory
+		logCfg.Name = cfg.Logging.File.Name
+		logCfg.MaxSizeKB = cfg.Logging.File.MaxSizeMB * 1000
+		logCfg.MaxTotalSizeKB = cfg.Logging.File.MaxTotalSizeMB * 1000
+		if cfg.Logging.File.RetentionHours > 0 {
+			logCfg.RetentionPeriodHrs = cfg.Logging.File.RetentionHours
+		}
+	}
+}
@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"logwisp/internal/version"
+	"os"
+)
+
+// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths.
+const helpText = `LogWisp %s - log collection, processing, and distribution
+
+Usage:
+  logwisp [options]
+  logwisp help | -h | --help
+  logwisp --version
+
+Any configuration key is settable as a flag using its TOML path:
+  --<path>=<value>              e.g. --logging.level=debug
+
+Common options:
+  -c, --config <path>           Configuration file (default: ./logwisp.toml)
+      --quiet                   Suppress console output
+      --status_reporter=<bool>  Periodic status logging (default: true)
+      --auto_reload=<bool>      Config hot reload on file change (default: false)
+
+Logging:
+      --logging.output=<mode>   file|stdout|stderr|split|all|none
+      --logging.level=<level>   debug|info|warn|error
+      --logging.file.directory=<path>
+      --logging.console.target=<target>   stdout|stderr|split
+
+Pipelines (N = 0-based index):
+      --pipelines.N.name=<name>
+      --pipelines.N.plugin_sources.N.type=<type>   file|console|random|null
+      --pipelines.N.plugin_sinks.N.type=<type>     console|file|http|tcp|null
+      --pipelines.N.flow.filters.N.patterns='["ERROR","WARN"]'
+
+Environment:
+  LOGWISP_<PATH>                Config path, '.' -> '_', uppercase
+                                e.g. LOGWISP_LOGGING_LEVEL=debug
+  LOGWISP_CONFIG_FILE           Configuration file path
+  LOGWISP_CONFIG_DIR            Configuration directory
+
+Signals:
+  SIGINT, SIGTERM               Graceful shutdown
+  SIGHUP, SIGUSR1               Reload configuration
+
+Exit codes:
+  0  success
+  1  general error
+  2  configuration file not found
+`
+
+// handleHelp prints usage and exits if a help request is present in args
+func handleHelp(args []string) {
+	if len(args) > 0 && args[0] == "help" {
+		printHelp()
+	}
+	for _, arg := range args {
+		if arg == "--" {
+			break // end of flags
+		}
+		if arg == "-h" || arg == "--help" {
+			printHelp()
+		}
+	}
+}
+
+// printHelp writes usage to stdout and exits with success
+func printHelp() {
+	fmt.Printf(helpText, version.Short())
+	os.Exit(0)
+}
@@ -0,0 +1,156 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/version"
+
+	"github.com/lixenwraith/log"
+)
+
+// logger is the global logger instance for the application
+var logger *log.Logger
+
+// main is the entry point for the LogWisp application
+func main() {
+	// --- 1. Initial setup ---
+	// Emulates nohup
+	signal.Ignore(syscall.SIGHUP)
+
+	// Help handled before config parsing; loader has no help flag.
+	// Also the future dispatch point for subcommands (tls, etc.)
+	handleHelp(os.Args[1:])
+
+	// Load configuration with automatic CLI parsing
+	cfg, err := config.Load(os.Args[1:])
+	if err != nil {
+		if strings.Contains(err.Error(), "not found") && cfg != nil && cfg.ConfigFile != "" {
+			fmt.Fprintf(os.Stderr, "Error: Config file not found: %s\n", cfg.ConfigFile)
+			os.Exit(2)
+		}
+		fmt.Fprintf(os.Stderr, "Error: Failed to load config: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Initialize output handler
+	InitOutputHandler(cfg.Quiet)
+
+	// Handle version
+	if cfg.ShowVersion {
+		fmt.Println(version.String())
+		os.Exit(0)
+	}
+
+	// Initialize logger instance and apply configuration
+	if err := initializeLogger(cfg); err != nil {
+		FatalError(1, "Failed to initialize logger: %v\n", err)
+	}
+	defer shutdownLogger()
+
+	// Start the logger
+	if err := logger.Start(); err != nil {
+		FatalError(1, "Failed to start logger: %v\n", err)
+	}
+
+	// Log startup information
+	logger.Info("msg", "LogWisp starting",
+		"version", version.String(),
+		"config_file", cfg.ConfigFile,
+		"log_output", cfg.Logging.Output,
+		"status_reporter", cfg.StatusReporter,
+		"auto_reload", cfg.ConfigAutoReload)
+
+	// Create context for shutdown
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// --- 2. Bootstrap initial service ---
+	svc, statusReporterCancel, err := bootstrapInitial(ctx, cfg)
+	if err != nil {
+		logger.Error("msg", "Failed to initialize service", "error", err)
+		os.Exit(1)
+	}
+
+	// --- 3. Setup signals and shutdown ---
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGUSR1)
+
+	var configChanges <-chan string
+	lcfg := config.GetConfigManager()
+	if cfg.ConfigAutoReload && lcfg != nil {
+		configChanges = lcfg.Watch()
+		logger.Info("msg", "Config auto-reload enabled", "config_file", cfg.ConfigFile)
+	} else {
+		logger.Info("msg", "Config auto-reload disabled")
+	}
+
+	// Service shutdown sequence
+	defer func() {
+		logger.Info("msg", "Shutdown initiated")
+		if statusReporterCancel != nil {
+			statusReporterCancel()
+		}
+		if svc != nil {
+			svc.Shutdown()
+		}
+		if lcfg != nil {
+			lcfg.StopAutoUpdate()
+		}
+		logger.Info("msg", "Shutdown complete")
+		// Deferred logger shutdown will run after this
+	}()
+
+	// --- 4. Main Application Event Loop ---
+	logger.Info("msg", "Application started, waiting for signals or config changes")
+	for {
+		select {
+		case sig := <-sigChan:
+			if sig == syscall.SIGHUP || sig == syscall.SIGUSR1 {
+				logger.Info("msg", "Reload signal received, triggering manual reload", "signal", sig)
+				newSvc, newCfg, newStatusCancel, err := handleReload(ctx, svc, statusReporterCancel)
+				if err == nil {
+					svc = newSvc
+					cfg = newCfg
+					statusReporterCancel = newStatusCancel
+				}
+			} else {
+				logger.Info("msg", "Shutdown signal received", "signal", sig)
+				cancel() // Trigger service shutdown via context
+			}
+
+		case event, ok := <-configChanges:
+			if !ok {
+				logger.Warn("msg", "Configuration watch channel closed, disabling auto-reload")
+				configChanges = nil // Stop selecting on this channel
+				continue
+			}
+			logger.Info("msg", "Configuration file change detected, triggering reload", "event", event)
+			newSvc, newCfg, newStatusCancel, err := handleReload(ctx, svc, statusReporterCancel)
+			if err == nil {
+				svc = newSvc
+				cfg = newCfg
+				statusReporterCancel = newStatusCancel
+			}
+
+		case <-ctx.Done():
+			return // Exit the loop and trigger deferred shutdown
+		}
+	}
+}
+
+// shutdownLogger gracefully shuts down the global logger.
+func shutdownLogger() {
+	if logger != nil {
+		if err := logger.Shutdown(core.LoggerShutdownTimeout); err != nil {
+			// Best effort - can't log the shutdown error
+			Error("Logger shutdown error: %v\n", err)
+		}
+	}
+}
@@ -0,0 +1,93 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sync"
+)
+
+// OutputHandler manages all application output, respecting the global quiet mode
+type OutputHandler struct {
+	quiet  bool
+	mu     sync.RWMutex
+	stdout io.Writer
+	stderr io.Writer
+}
+
+// output is the global instance of the OutputHandler
+var output *OutputHandler
+
+// InitOutputHandler initializes the global output handler
+func InitOutputHandler(quiet bool) {
+	output = &OutputHandler{
+		quiet:  quiet,
+		stdout: os.Stdout,
+		stderr: os.Stderr,
+	}
+}
+
+// Print writes to stdout
+func Print(format string, args ...any) {
+	if output != nil {
+		output.Print(format, args...)
+	}
+}
+
+// Error writes to stderr
+func Error(format string, args ...any) {
+	if output != nil {
+		output.Error(format, args...)
+	}
+}
+
+// FatalError writes to stderr and exits the application
+func FatalError(code int, format string, args ...any) {
+	if output != nil {
+		output.FatalError(code, format, args...)
+	} else {
+		// Fallback if handler not initialized
+		fmt.Fprintf(os.Stderr, format, args...)
+		os.Exit(code)
+	}
+}
+
+// Print writes a formatted string to stdout if not in quiet mode
+func (o *OutputHandler) Print(format string, args ...any) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+
+	if !o.quiet {
+		fmt.Fprintf(o.stdout, format, args...)
+	}
+}
+
+// Error writes a formatted string to stderr if not in quiet mode
+func (o *OutputHandler) Error(format string, args ...any) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+
+	if !o.quiet {
+		fmt.Fprintf(o.stderr, format, args...)
+	}
+}
+
+// FatalError writes a formatted string to stderr and exits with the given code.
+func (o *OutputHandler) FatalError(code int, format string, args ...any) {
+	o.Error(format, args...)
+	os.Exit(code)
+}
+
+// IsQuiet returns the current quiet mode status.
+func (o *OutputHandler) IsQuiet() bool {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.quiet
+}
+
+// SetQuiet updates the quiet mode status.
+func (o *OutputHandler) SetQuiet(quiet bool) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.quiet = quiet
+}
@@ -0,0 +1,148 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"logwisp/internal/service"
+)
+
+// startStatusReporter starts a new status reporter for a service and returns its cancel function.
+func startStatusReporter(ctx context.Context, svc *service.Service) context.CancelFunc {
+	reporterCtx, cancel := context.WithCancel(ctx)
+	go statusReporter(svc, reporterCtx)
+	logger.Debug("msg", "Started status reporter")
+	return cancel
+}
+
+// statusReporter periodically logs the health and statistics of the service
+func statusReporter(service *service.Service, ctx context.Context) {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if service == nil {
+				logger.Warn("msg", "Status reporter: service is nil",
+					"component", "status_reporter")
+				return
+			}
+
+			// Safely get stats with recovery
+			func() {
+				defer func() {
+					if r := recover(); r != nil {
+						logger.Error("msg", "Panic in status reporter",
+							"component", "status_reporter",
+							"panic", r)
+					}
+				}()
+
+				stats := service.GetGlobalStats()
+				totalPipelines, ok := stats["total_pipelines"].(int)
+				if !ok || totalPipelines == 0 {
+					logger.Warn("msg", "No active pipelines in status report",
+						"component", "status_reporter")
+					return
+				}
+
+				// Log service-level summary
+				logger.Debug("msg", "Status report",
+					"component", "status_reporter",
+					"active_pipelines", totalPipelines,
+					"time", time.Now().Format("15:04:05"))
+
+				// Log each pipeline's stats recursively
+				if pipelines, ok := stats["pipelines"].(map[string]any); ok {
+					for name, pipelineStats := range pipelines {
+						logStats("Pipeline status", name, pipelineStats)
+					}
+				}
+			}()
+		}
+	}
+}
+
+// logStats recursively logs statistics with automatic field extraction
+func logStats(msg string, name string, stats any) {
+	// Build base log fields
+	fields := []any{
+		"msg", msg,
+		"name", name,
+	}
+
+	// Extract and flatten important metrics from stats map
+	if statsMap, ok := stats.(map[string]any); ok {
+		// Add scalar values directly
+		for key, value := range statsMap {
+			switch v := value.(type) {
+			case string, bool, int, int64, uint64, float64:
+				fields = append(fields, key, v)
+			case time.Time:
+				if !v.IsZero() {
+					fields = append(fields, key, v.Format(time.RFC3339))
+				}
+			case map[string]any:
+				// For nested maps, log summary counts if they contain arrays/maps
+				if count := getItemCount(v); count > 0 {
+					fields = append(fields, fmt.Sprintf("%s_count", key), count)
+				}
+			case []any, []map[string]any:
+				// For arrays, just log the count
+				fields = append(fields, fmt.Sprintf("%s_count", key), getArrayLength(value))
+			}
+		}
+
+		// Log the flattened stats
+		logger.Debug(fields...)
+
+		// Recursively log nested structures with detail
+		for key, value := range statsMap {
+			switch v := value.(type) {
+			case map[string]any:
+				// Log nested component stats
+				if key == "flow" || key == "rate_limiter" || key == "filters" {
+					logStats(fmt.Sprintf("%s %s", name, key), key, v)
+				}
+			case []map[string]any:
+				// Log array items (sources, sinks, filters)
+				for i, item := range v {
+					if itemName, ok := item["id"].(string); ok {
+						logStats(fmt.Sprintf("%s %s", name, key), itemName, item)
+					} else {
+						logStats(fmt.Sprintf("%s %s", name, key), fmt.Sprintf("%s[%d]", key, i), item)
+					}
+				}
+			}
+		}
+	}
+}
+
+// getItemCount returns the count of items in a map (for nested structures)
+func getItemCount(m map[string]any) int {
+	for _, v := range m {
+		switch v.(type) {
+		case []any:
+			return len(v.([]any))
+		case []map[string]any:
+			return len(v.([]map[string]any))
+		}
+	}
+	return 0
+}
+
+// getArrayLength safely gets the length of various array types
+func getArrayLength(v any) int {
+	switch arr := v.(type) {
+	case []any:
+		return len(arr)
+	case []map[string]any:
+		return len(arr)
+	default:
+		return 0
+	}
+}