v0.13.1 folder restructure, test script added, format adapter async fix

This commit is contained in:
2026-07-17 09:09:00 -04:00
parent 87e57784da
commit b8dd591b4b
49 changed files with 402 additions and 206 deletions
+325
View File
@@ -0,0 +1,325 @@
package config
// --- LogWisp Configuration Options ---
// Config is the top-level configuration structure for the LogWisp application
type Config struct {
// Top-level flags for application control
ShowVersion bool `toml:"version"`
Quiet bool `toml:"quiet"`
// Runtime behavior flags
StatusReporter bool `toml:"status_reporter"`
ConfigAutoReload bool `toml:"auto_reload"`
// Configuration file path
ConfigFile string `toml:"config_file"`
// Existing fields
Logging *LogConfig `toml:"logging"`
Pipelines []PipelineConfig `toml:"pipelines"`
}
// --- Logging Options ---
// LogConfig represents the logging configuration for the LogWisp application itself
type LogConfig struct {
// Output mode: "file", "stdout", "stderr", "split", "all", "none"
Output string `toml:"output"`
// Log level: "debug", "info", "warn", "error"
Level string `toml:"level"`
// Format: "raw", "txt", "json"
Format string `toml:"format"`
// Sanitization policy for console output
Sanitization string `toml:"sanitization"`
// File output settings (when Output includes "file" or "all")
File *LogFileConfig `toml:"file"`
// Console output settings
Console *LogConsoleConfig `toml:"console"`
}
// LogFileConfig defines settings for file-based application logging
type LogFileConfig struct {
// Directory for log files
Directory string `toml:"directory"`
// Base name for log files
Name string `toml:"name"`
// Maximum size per log file in MB
MaxSizeMB int64 `toml:"max_size_mb"`
// Maximum total size of all logs in MB
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
// Log retention in hours (0 = disabled)
RetentionHours float64 `toml:"retention_hours"`
}
// LogConsoleConfig defines settings for console-based application logging
type LogConsoleConfig struct {
// Target for console output: "stdout", "stderr"
Target string `toml:"target"`
}
// --- Pipeline ---
// PipelineConfig defines a complete data flow from sources to sinks
type PipelineConfig struct {
Name string `toml:"name"`
Flow *FlowConfig `toml:"flow"`
PluginSources []PluginSourceConfig `toml:"plugin_sources,omitempty"`
PluginSinks []PluginSinkConfig `toml:"plugin_sinks,omitempty"`
}
// --- Flow ---
// FlowConfig consolidates all processing stages between sources and sinks
type FlowConfig struct {
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
RateLimit *RateLimitConfig `toml:"rate_limit"`
Filters []FilterConfig `toml:"filters"`
Format *FormatConfig `toml:"format"`
}
// --- Heartbeat Options ---
// HeartbeatConfig defines settings for periodic keep-alive or status messages
type HeartbeatConfig struct {
Enabled bool `toml:"enabled"`
IntervalMS int64 `toml:"interval_ms"`
IncludeTimestamp bool `toml:"include_timestamp"`
IncludeStats bool `toml:"include_stats"`
Format string `toml:"format"`
}
// --- Formatter Options ---
// FormatConfig is a polymorphic struct representing log entry formatting options
type FormatConfig struct {
Type string `toml:"type"` // "json", "txt", "raw"
Flags int64 `toml:"flags"`
TimestampFormat string `toml:"timestamp_format"`
SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell"
}
// --- Rate Limit Options ---
// RateLimitPolicy defines the action to take when a rate limit is exceeded
type RateLimitPolicy int
const (
// PolicyPass allows all logs through, effectively disabling the limiter
PolicyPass RateLimitPolicy = iota
// PolicyDrop drops logs that exceed the rate limit
PolicyDrop
)
// RateLimitConfig defines the configuration for pipeline-level rate limiting
type RateLimitConfig struct {
// Rate is the number of log entries allowed per second. Default: 0 (disabled)
Rate float64 `toml:"rate"`
// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate
Burst float64 `toml:"burst"`
// Policy defines the action to take when the limit is exceeded. "pass" or "drop"
Policy string `toml:"policy"`
// MaxEntrySizeBytes is the maximum allowed size for a single log entry. 0 = no limit
MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes"`
}
// --- Filter Options ---
// FilterType represents the filter's behavior (include or exclude)
type FilterType string
const (
// FilterTypeInclude specifies that only matching logs will pass
FilterTypeInclude FilterType = "include" // Whitelist - only matching logs pass
// FilterTypeExclude specifies that matching logs will be dropped
FilterTypeExclude FilterType = "exclude" // Blacklist - matching logs are dropped
)
// FilterLogic represents how multiple filter patterns are combined
type FilterLogic string
const (
// FilterLogicOr specifies that a match on any pattern is sufficient
FilterLogicOr FilterLogic = "or" // Match any pattern
// FilterLogicAnd specifies that all patterns must match
FilterLogicAnd FilterLogic = "and" // Match all patterns
)
// FilterConfig represents the configuration for a single filter
type FilterConfig struct {
Type FilterType `toml:"type"`
Logic FilterLogic `toml:"logic"`
Patterns []string `toml:"patterns"`
}
// --- Source Options ---
// PluginSourceConfig represents a source plugin instance configuration
type PluginSourceConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
}
// // SourceConfig is a polymorphic struct representing a single data source
// type SourceConfig struct {
// Type string `toml:"type"`
//
// // Polymorphic - only one populated based on type
// File *FileSourceOptions `toml:"file,omitempty"`
// Console *ConsoleSourceOptions `toml:"console,omitempty"`
// }
// NullSourceOptions defines settings for a null source (no configuration needed)
type NullSourceOptions struct{}
// RandomSourceOptions defines settings for a random log generator source
type RandomSourceOptions struct {
IntervalMS int64 `toml:"interval_ms"`
JitterMS int64 `toml:"jitter_ms"`
Format string `toml:"format"`
Length int64 `toml:"length"`
Special bool `toml:"special"`
}
// FileSourceOptions defines settings for a file-based source
type FileSourceOptions struct {
Directory string `toml:"directory"`
Pattern string `toml:"pattern"` // glob pattern
CheckIntervalMS int64 `toml:"check_interval_ms"`
}
// ConsoleSourceOptions defines settings for a stdin-based source
type ConsoleSourceOptions struct {
BufferSize int64 `toml:"buffer_size"`
}
// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting
// NDJSON entries from upstream logwisp tcp_chain sinks
type TCPChainSourceOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // per-connection idle deadline, 0 = none
HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
// Future: TLS/auth options
}
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
// NDJSON batches from upstream logwisp http_chain sinks
type HTTPChainSourceOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
IngestPath string `toml:"ingest_path"`
BufferSize int64 `toml:"buffer_size"`
MaxBodyBytes int64 `toml:"max_body_bytes"` // per-request cap
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
// Future: TLS/auth options
}
// --- Sink Options ---
// PluginSinkConfig represents a sink plugin instance configuration
type PluginSinkConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
}
// // SinkConfig is a polymorphic struct representing a single data sink
// type SinkConfig struct {
// Type string `toml:"type"`
//
// // Polymorphic - only one populated based on type
// Console *ConsoleSinkOptions `toml:"console,omitempty"`
// File *FileSinkOptions `toml:"file,omitempty"`
// }
// NullSinkOptions defines settings for a null sink (no configuration needed)
type NullSinkOptions struct{}
// ConsoleSinkOptions defines settings for a console-based sink
type ConsoleSinkOptions struct {
Target string `toml:"target"` // "stdout", "stderr"
BufferSize int64 `toml:"buffer_size"`
}
// FileSinkOptions defines settings for a file-based sink
type FileSinkOptions struct {
Directory string `toml:"directory"`
Name string `toml:"name"`
MaxSizeMB int64 `toml:"max_size_mb"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
MinDiskFreeMB int64 `toml:"min_disk_free_mb"`
RetentionHours float64 `toml:"retention_hours"`
BufferSize int64 `toml:"buffer_size"`
FlushIntervalMs int64 `toml:"flush_interval_ms"`
}
// TCPSinkOptions defines settings for a TCP server sink
type TCPSinkOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
WriteTimeout int64 `toml:"write_timeout_ms"`
KeepAlivePeriod int64 `toml:"keep_alive_period_ms"`
KeepAlive bool `toml:"keep_alive"`
}
// HTTPSinkOptions defines settings for an HTTP SSE server sink
type HTTPSinkOptions struct {
StreamPath string `toml:"stream_path"`
StatusPath string `toml:"status_path"`
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
WriteTimeout int64 `toml:"write_timeout_ms"`
}
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
// entries to a downstream logwisp tcp_chain source
type TCPChainSinkOptions struct {
Node string `toml:"node"` // origin label, default: os.Hostname()
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
DialTimeoutMS int64 `toml:"dial_timeout_ms"`
WriteTimeoutMS int64 `toml:"write_timeout_ms"`
BackoffMinMS int64 `toml:"backoff_min_ms"`
BackoffMaxMS int64 `toml:"backoff_max_ms"`
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
KeepAlive bool `toml:"keep_alive"`
// Future: TLS/auth options
}
// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting
// NDJSON batches to a downstream logwisp http_chain source
type HTTPChainSinkOptions struct {
Node string `toml:"node"` // origin label, default: os.Hostname()
Host string `toml:"host"`
Port int64 `toml:"port"`
IngestPath string `toml:"ingest_path"`
BufferSize int64 `toml:"buffer_size"`
MaxBatchCount int64 `toml:"max_batch_count"`
MaxBatchBytes int64 `toml:"max_batch_bytes"`
FlushIntervalMS int64 `toml:"flush_interval_ms"`
RequestTimeoutMS int64 `toml:"request_timeout_ms"` // covers dial + write + response
BackoffMinMS int64 `toml:"backoff_min_ms"`
BackoffMaxMS int64 `toml:"backoff_max_ms"`
// Future: TLS/auth options
}
+205
View File
@@ -0,0 +1,205 @@
package config
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"logwisp/internal/core"
lconfig "github.com/lixenwraith/config"
)
// configManager holds the global instance of the configuration manager
var configManager *lconfig.Config
// Load is the single entry point for loading all application configuration
func Load(args []string) (*Config, error) {
configPath, isExplicit := resolveConfigPath(args)
// Build configuration with all sources
// Create target config instance that will be populated
finalConfig := &Config{}
// Builder handles loading, populating the target struct, and validation
cfg, err := lconfig.NewBuilder().
WithTarget(finalConfig). // Typed target struct
WithDefaults(defaults()). // Default values
WithSources(
lconfig.SourceCLI,
lconfig.SourceEnv,
lconfig.SourceFile,
lconfig.SourceDefault,
).
WithEnvTransform(customEnvTransform). // Convert '.' to '_' in env separation
WithEnvPrefix("LOGWISP_"). // Environment variable prefix
WithArgs(args). // Command-line arguments
WithFile(configPath). // TOML config file
WithFileFormat("toml"). // Explicit format
WithTypedValidator(ValidateConfig). // Centralized validation
WithSecurityOptions(lconfig.SecurityOptions{
PreventPathTraversal: true,
MaxFileSize: 10 * 1024 * 1024, // 10MB max config
}).
Build()
if err != nil {
// Handle file not found errors - maintain existing behavior
if errors.Is(err, lconfig.ErrConfigNotFound) {
if isExplicit {
// Return empty config with file path
finalConfig.ConfigFile = configPath
return finalConfig, fmt.Errorf("config file not found: %s", configPath)
}
// If the default config file is not found, it's not an error, default/cli/env will be used
} else {
return nil, fmt.Errorf("failed to load or validate config: %w", err)
}
}
// Store the config file path for hot reload
finalConfig.ConfigFile = configPath
// Store the manager for hot reload
configManager = cfg
// Surface typo'd flags (e.g. --status-reporter vs --status_reporter);
// pre-logger phase, stderr only, suppressed in quiet mode
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet {
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
}
// Start watcher if auto-reload is enabled
if finalConfig.ConfigAutoReload {
watchOpts := lconfig.WatchOptions{
PollInterval: core.ReloadWatchPollInterval,
Debounce: core.ReloadWatchDebounce,
ReloadTimeout: core.ReloadWatchTimeout,
VerifyPermissions: true,
}
cfg.AutoUpdateWithOptions(watchOpts)
}
return finalConfig, nil
}
// GetConfigManager returns the global configuration manager instance for hot-reloading
func GetConfigManager() *lconfig.Config {
return configManager
}
// defaults provides the default configuration values for the application
func defaults() *Config {
return &Config{
// Top-level flag defaults
ShowVersion: false,
Quiet: false,
// Runtime behavior defaults
StatusReporter: true,
ConfigAutoReload: false,
// Existing defaults
Logging: &LogConfig{
Output: "stdout",
Level: "info",
Format: "txt",
File: &LogFileConfig{
Directory: "./log",
Name: "logwisp",
MaxSizeMB: 100,
MaxTotalSizeMB: 1000,
RetentionHours: 168, // 7 days
},
Console: &LogConsoleConfig{
Target: "stdout",
},
},
Pipelines: []PipelineConfig{
{
Name: "default_pipeline",
Flow: &FlowConfig{
RateLimit: &RateLimitConfig{
Rate: 5,
Burst: 10,
Policy: "drop",
MaxEntrySizeBytes: 65536,
},
Format: &FormatConfig{
Type: "json",
SanitizerPolicy: "json",
},
},
PluginSources: []PluginSourceConfig{
{
ID: "default_source",
Type: "random",
Config: map[string]any{
"special": true,
},
// Config: &FileSourceOptions{
// Directory: "./",
// Pattern: "*.log",
// CheckIntervalMS: int64(100),
},
},
PluginSinks: []PluginSinkConfig{
{
ID: "default_sink",
Type: "console",
Config: map[string]any{
"target": "stdout",
"buffer_size": 100,
},
},
},
},
},
}
}
// resolveConfigPath determines the configuration file path based on CLI args, env vars, and default locations
func resolveConfigPath(args []string) (path string, isExplicit bool) {
// 1. Check for --config flag in command-line arguments (highest precedence)
for i, arg := range args {
if arg == "-c" {
return args[i+1], true
}
if strings.HasPrefix(arg, "--config=") {
return strings.TrimPrefix(arg, "--config="), true
}
}
// 2. Check environment variables
if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" {
path = configFile
if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
path = filepath.Join(configDir, configFile)
}
return path, true
}
if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
return filepath.Join(configDir, "logwisp.toml"), true
}
// 3. Check default user config location
if homeDir, err := os.UserHomeDir(); err == nil {
configPath := filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml")
if _, err := os.Stat(configPath); err == nil {
return configPath, false // Found a default, but not explicitly set by user
}
}
// 4. Fallback to default in current directory
return "logwisp.toml", false
}
// customEnvTransform converts TOML-style config paths (e.g., logging.level) to environment variable format (LOGGING_LEVEL)
func customEnvTransform(path string) string {
env := strings.ReplaceAll(path, ".", "_")
env = strings.ToUpper(env)
// env = "LOGWISP_" + env // already added by WithEnvPrefix
return env
}
+83
View File
@@ -0,0 +1,83 @@
package config
import (
"fmt"
lconfig "github.com/lixenwraith/config"
)
// ValidateConfig validates top-level structure only
// Value range validation is delegated to component constructors
func ValidateConfig(cfg *Config) error {
if cfg == nil {
return fmt.Errorf("config is nil")
}
if len(cfg.Pipelines) == 0 {
return fmt.Errorf("no pipelines configured")
}
// Reject duplicate pipeline names (service map is keyed by name)
names := make(map[string]struct{}, len(cfg.Pipelines))
for i, p := range cfg.Pipelines {
if _, dup := names[p.Name]; dup {
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
}
names[p.Name] = struct{}{}
}
if err := validateLogConfig(cfg.Logging); err != nil {
return fmt.Errorf("logging: %w", err)
}
for i, p := range cfg.Pipelines {
if err := lconfig.NonEmpty(p.Name); err != nil {
return fmt.Errorf("pipeline[%d].name: %w", i, err)
}
if len(p.PluginSources) == 0 {
return fmt.Errorf("pipeline[%d]: no sources defined", i)
}
if len(p.PluginSinks) == 0 {
return fmt.Errorf("pipeline[%d]: no sinks defined", i)
}
}
return nil
}
// validateLogConfig validates application logging settings
func validateLogConfig(cfg *LogConfig) error {
if cfg == nil {
return nil
}
validateOutput := lconfig.OneOf("file", "stdout", "stderr", "split", "all", "none")
if err := validateOutput(cfg.Output); err != nil {
return fmt.Errorf("output: %w", err)
}
validateLevel := lconfig.OneOf("debug", "info", "warn", "error")
if err := validateLevel(cfg.Level); err != nil {
return fmt.Errorf("level: %w", err)
}
if cfg.Format != "" {
if err := lconfig.OneOf("raw", "txt", "json")(cfg.Format); err != nil {
return fmt.Errorf("format: %w", err)
}
}
if cfg.Sanitization != "" {
if err := lconfig.OneOf("raw", "json", "txt", "shell")(cfg.Sanitization); err != nil {
return fmt.Errorf("sanitization: %w", err)
}
}
if cfg.Console != nil {
validateTarget := lconfig.OneOf("stdout", "stderr", "split")
if err := validateTarget(cfg.Console.Target); err != nil {
return fmt.Errorf("console.target: %w", err)
}
}
return nil
}