v0.19.0 use updated config and toml packages, dep update, minor fixes
Test / test (push) Canceled after 0s

This commit is contained in:
lixen committed 2026-09-29 00:20:53 -04:00
1 parent b8982e4e44
commit cb0da23070
22 files changed
+692 -232

No files matched your search

+2 -2
View File
@@ -12,8 +12,8 @@ type Config struct {
StatusReporter bool `toml:"status_reporter"`
ConfigAutoReload bool `toml:"auto_reload"`
// Configuration file path
ConfigFile string `toml:"config_file"`
// Selected file path is runtime metadata, never a configurable override.
ConfigFile string `toml:"-"`
// Existing fields
Logging *LogConfig `toml:"logging"`
+93 -91
View File
@@ -12,84 +12,85 @@ import (
lconfig "github.com/lixenwraith/config"
)
// configManager holds the global instance of the configuration manager
var configManager *lconfig.Config
// ErrConfigNotFound identifies an explicitly requested missing configuration file.
var ErrConfigNotFound = lconfig.ErrConfigNotFound
// Load is the single entry point for loading all application configuration
func Load(args []string) (*Config, error) {
configPath, isExplicit := resolveConfigPath(args)
// Build configuration with all sources
// Manager owns the configuration sources and watcher for one application instance.
// Its snapshots are detached from each other and from the running service.
type Manager struct {
config *lconfig.Config
path string
}
// Create target config instance that will be populated
finalConfig := &Config{}
// Builder handles loading, populating the target struct, and validation
// Load reads the startup sources and validates the initial configuration.
// Watching starts only when Watch is called after successful service startup.
func Load(args []string) (*Manager, error) {
configPath, isExplicit, configArgs, err := resolveConfigPath(args)
if err != nil {
return nil, err
}
initial := defaults()
cfg, err := lconfig.NewBuilder().
WithTarget(finalConfig). // Typed target struct
WithDefaults(defaults()). // Default values
WithSources(
lconfig.SourceCLI,
lconfig.SourceEnv,
lconfig.SourceFile,
lconfig.SourceDefault,
).
WithEnvTransform(customEnvTransform). // Convert '.' to '_' in env separation
WithEnvPrefix("LOGWISP_"). // Environment variable prefix
WithArgs(args). // Command-line arguments
WithFile(configPath). // TOML config file
WithFileFormat("toml"). // Explicit format
WithTypedValidator(ValidateConfig). // Centralized validation
WithTarget(initial).
WithEnvPrefix("LOGWISP_").
WithArgs(configArgs).
WithFile(configPath).
WithTypedValidator(ValidateConfig).
WithSecurityOptions(lconfig.SecurityOptions{
PreventPathTraversal: true,
MaxFileSize: 10 * 1024 * 1024, // 10MB max config
MaxFileSize: 10 * 1024 * 1024,
}).
Build()
if err != nil {
// Handle file not found errors - maintain existing behavior
if errors.Is(err, lconfig.ErrConfigNotFound) {
if isExplicit {
// Return empty config with file path
finalConfig.ConfigFile = configPath
return finalConfig, fmt.Errorf("config file not found: %s", configPath)
}
// If the default config file is not found, it's not an error, default/cli/env will be used
} else {
if !errors.Is(err, lconfig.ErrConfigNotFound) {
return nil, fmt.Errorf("failed to load or validate config: %w", err)
}
if isExplicit {
return nil, fmt.Errorf("config file %q: %w", configPath, err)
}
// A missing discovered default still permits valid CLI/env/default values.
}
// Store the config file path for hot reload
finalConfig.ConfigFile = configPath
// Store the manager for hot reload
configManager = cfg
// Surface typo'd flags (e.g. --status-reporter vs --status_reporter);
// pre-logger phase, stderr only, suppressed in quiet mode
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet {
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !initial.Quiet {
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
}
return &Manager{config: cfg, path: configPath}, nil
}
// Start watcher if auto-reload is enabled
if finalConfig.ConfigAutoReload {
watchOpts := lconfig.WatchOptions{
PollInterval: core.ReloadWatchPollInterval,
Debounce: core.ReloadWatchDebounce,
ReloadTimeout: core.ReloadWatchTimeout,
VerifyPermissions: true,
}
cfg.AutoUpdateWithOptions(watchOpts)
// Snapshot validates a detached candidate before it is used to build a service.
// Builder validators only run during Load, so every reload needs this check too.
func (m *Manager) Snapshot() (*Config, error) {
value, err := m.config.AsStruct()
if err != nil {
return nil, fmt.Errorf("decode configuration: %w", err)
}
return finalConfig, nil
cfg := value.(*Config)
cfg.ConfigFile = m.path
if err := ValidateConfig(cfg); err != nil {
return nil, fmt.Errorf("validate configuration: %w", err)
}
return cfg, nil
}
// GetConfigManager returns the global configuration manager instance for hot-reloading
func GetConfigManager() *lconfig.Config {
return configManager
// Reload rereads the selected file, retaining the startup CLI/environment sources.
// Signals must call this even when automatic watching is disabled.
func (m *Manager) Reload() (*Config, error) {
if err := m.config.LoadFile(m.path); err != nil {
return nil, fmt.Errorf("reload %q: %w", m.path, err)
}
return m.Snapshot()
}
// Watch subscribes with logwisp's polling and debounce settings.
func (m *Manager) Watch() <-chan string {
opts := lconfig.DefaultWatchOptions()
opts.PollInterval = core.ReloadWatchPollInterval
opts.Debounce = core.ReloadWatchDebounce
opts.ReloadTimeout = core.ReloadWatchTimeout
return m.config.WatchWithOptions(opts)
}
func (m *Manager) Close() { m.config.StopAutoUpdate() }
// defaults provides the default configuration values for the application
func defaults() *Config {
return &Config{
@@ -139,10 +140,6 @@ func defaults() *Config {
Config: map[string]any{
"special": true,
},
// Config: &FileSourceOptions{
// Directory: "./",
// Pattern: "*.log",
// CheckIntervalMS: int64(100),
},
},
PluginSinks: []PluginSinkConfig{
@@ -160,46 +157,51 @@ func defaults() *Config {
}
}
// resolveConfigPath determines the configuration file path based on CLI args, env vars, and default locations
func resolveConfigPath(args []string) (path string, isExplicit bool) {
// 1. Check for --config flag in command-line arguments (highest precedence)
for i, arg := range args {
if arg == "-c" {
return args[i+1], true
// resolveConfigPath consumes file-selection flags before schema CLI parsing.
// The last selection wins, and -- terminates option handling.
func resolveConfigPath(args []string) (path string, isExplicit bool, remaining []string, err error) {
remaining = make([]string, 0, len(args))
for i := 0; i < len(args); i++ {
arg := args[i]
if arg == "--" {
remaining = append(remaining, args[i:]...)
break
}
if strings.HasPrefix(arg, "--config=") {
return strings.TrimPrefix(arg, "--config="), true
switch {
case arg == "-c" || arg == "--config":
if i+1 == len(args) || args[i+1] == "" || strings.HasPrefix(args[i+1], "-") {
return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg)
}
i++
path, isExplicit = args[i], true
case strings.HasPrefix(arg, "--config=") || strings.HasPrefix(arg, "-c="):
_, path, _ = strings.Cut(arg, "=")
if path == "" {
return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg)
}
isExplicit = true
default:
remaining = append(remaining, arg)
}
}
// 2. Check environment variables
if isExplicit {
return path, true, remaining, nil
}
if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" {
path = configFile
if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
path = filepath.Join(configDir, configFile)
}
return path, true
return path, true, remaining, nil
}
if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
return filepath.Join(configDir, "logwisp.toml"), true
return filepath.Join(configDir, "logwisp.toml"), true, remaining, nil
}
// 3. Check default user config location
if homeDir, err := os.UserHomeDir(); err == nil {
configPath := filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml")
if _, err := os.Stat(configPath); err == nil {
return configPath, false // Found a default, but not explicitly set by user
path = filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml")
if _, err := os.Stat(path); err == nil {
return path, false, remaining, nil
}
}
// 4. Fallback to default in current directory
return "logwisp.toml", false
}
// customEnvTransform converts TOML-style config paths (e.g., logging.level) to environment variable format (LOGGING_LEVEL)
func customEnvTransform(path string) string {
env := strings.ReplaceAll(path, ".", "_")
env = strings.ToUpper(env)
// env = "LOGWISP_" + env // already added by WithEnvPrefix
return env
return "logwisp.toml", false, remaining, nil
}
+217
View File
@@ -0,0 +1,217 @@
package config
import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
lconfig "github.com/lixenwraith/config"
)
func isolateConfig(t *testing.T) {
t.Helper()
for _, entry := range os.Environ() {
key, value, _ := strings.Cut(entry, "=")
if strings.HasPrefix(key, "LOGWISP_") {
t.Setenv(key, value)
if err := os.Unsetenv(key); err != nil {
t.Fatal(err)
}
}
}
dir := t.TempDir()
t.Setenv("HOME", dir)
t.Chdir(dir)
}
func writeConfig(t *testing.T, path, contents string) {
t.Helper()
if err := os.WriteFile(path, []byte(contents), 0600); err != nil {
t.Fatal(err)
}
}
func TestResolveConfigArguments(t *testing.T) {
isolateConfig(t)
for _, args := range [][]string{{"-c", "chosen.toml"}, {"--config", "chosen.toml"}, {"--config=chosen.toml"}, {"-c=chosen.toml"}, {"-c", "old.toml", "--config=chosen.toml"}} {
path, explicit, remaining, err := resolveConfigPath(append(args, "--quiet"))
if err != nil || path != "chosen.toml" || !explicit || !reflect.DeepEqual(remaining, []string{"--quiet"}) {
t.Fatalf("%v: %q %v %v %v", args, path, explicit, remaining, err)
}
}
for _, args := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}} {
if _, _, _, err := resolveConfigPath(args); err == nil {
t.Fatalf("missing path accepted: %v", args)
}
}
args := []string{"--", "-c", "ignored.toml"}
path, explicit, remaining, err := resolveConfigPath(args)
if err != nil || explicit || path != "logwisp.toml" || !reflect.DeepEqual(remaining, args) {
t.Fatalf("terminator ignored: %q %v %v %v", path, explicit, remaining, err)
}
t.Setenv("LOGWISP_CONFIG_DIR", "configs")
t.Setenv("LOGWISP_CONFIG_FILE", "env.toml")
path, explicit, _, err = resolveConfigPath(nil)
if err != nil || !explicit || path != filepath.Join("configs", "env.toml") {
t.Fatalf("environment path: %q %v %v", path, explicit, err)
}
}
func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
isolateConfig(t)
writeConfig(t, "selected.toml", "config_file = \"ignored.toml\"\n[logging]\nlevel = \"error\"\n")
t.Setenv("LOGGING_LEVEL", "invalid-bare-variable")
t.Setenv("LOGWISP_LOGGING_LEVEL", "debug")
m, err := Load([]string{"--config", "selected.toml", "--logging.level=warn", "--quiet"})
if err != nil {
t.Fatal(err)
}
defer m.Close()
if unknown := m.config.UnknownCLIKeys(); len(unknown) != 0 {
t.Fatalf("path flags reached schema parser: %v", unknown)
}
first, err := m.Snapshot()
if err != nil || first.Logging.Level != "warn" || first.ConfigFile != "selected.toml" {
t.Fatalf("snapshot: %+v %v", first, err)
}
first.Logging.Level = "mutated"
first.Pipelines[0].PluginSources[0].Config["special"] = false
second, err := m.Snapshot()
if err != nil || second.Logging.Level != "warn" || second.Pipelines[0].PluginSources[0].Config["special"] != true {
t.Fatalf("snapshot aliases another result: %+v %v", second, err)
}
env, err := Load([]string{"-c=selected.toml"})
if err != nil {
t.Fatal(err)
}
defer env.Close()
fromEnv, err := env.Snapshot()
if err != nil || fromEnv.Logging.Level != "debug" {
t.Fatalf("prefixed environment ignored: %+v %v", fromEnv, err)
}
}
func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) {
isolateConfig(t)
m, err := Load(nil)
if err != nil {
t.Fatal(err)
}
m.Close()
if _, err := Load([]string{"-c", "missing.toml"}); !errors.Is(err, ErrConfigNotFound) {
t.Fatalf("missing explicit path: %v", err)
}
if _, err := Load([]string{"--logging.file.max_size_mb=1.5"}); err == nil || errors.Is(err, ErrConfigNotFound) {
t.Fatalf("invalid CLI hidden by missing default: %v", err)
}
t.Setenv("LOGWISP_STATUS_REPORTER", "invalid")
if _, err := Load(nil); err == nil || errors.Is(err, ErrConfigNotFound) {
t.Fatalf("invalid environment hidden by missing default: %v", err)
}
}
func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
isolateConfig(t)
writeConfig(t, "reload.toml", "status_reporter = true\n")
m, err := Load([]string{"-c", "reload.toml"})
if err != nil {
t.Fatal(err)
}
defer m.Close()
old, err := m.Snapshot()
if err != nil {
t.Fatal(err)
}
writeConfig(t, "reload.toml", "status_reporter = false\n")
next, err := m.Reload()
if err != nil || next.StatusReporter || !old.StatusReporter || next.ConfigFile != old.ConfigFile {
t.Fatalf("disk reload/snapshot ownership: %+v %v", next, err)
}
// Builder validators do not run on LoadFile; Snapshot must run them again.
writeConfig(t, "reload.toml", "pipelines = []\n")
if _, err := m.Reload(); err == nil || !strings.Contains(err.Error(), "no pipelines") {
t.Fatalf("semantically invalid reload accepted: %v", err)
}
if len(next.Pipelines) == 0 || !old.StatusReporter {
t.Fatal("invalid edit changed a published snapshot")
}
writeConfig(t, "reload.toml", "[logging.file]\nmax_size_mb = 1.5\n")
if _, err := m.Reload(); err == nil {
t.Fatal("fractional integer accepted")
}
writeConfig(t, "reload.toml", "# remove the status override\n")
recovered, err := m.Reload()
if err != nil || !recovered.StatusReporter {
t.Fatalf("override removal/recovery: %+v %v", recovered, err)
}
}
func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) {
isolateConfig(t)
writeConfig(t, "watched.toml", "auto_reload = true\n")
m, err := Load([]string{"-c", "watched.toml"})
if err != nil {
t.Fatal(err)
}
defer m.Close()
if m.config.IsWatching() {
t.Fatal("watching started before service startup")
}
changes := m.Watch()
if !m.config.IsWatching() {
t.Fatal("subscription did not start watching")
}
m.Close()
select {
case _, ok := <-changes:
if ok {
t.Fatal("watch channel still open")
}
case <-time.After(time.Second):
t.Fatal("watcher did not close")
}
}
func TestNestedPluginOptionsKeepDefaultsAndNumericTypes(t *testing.T) {
isolateConfig(t)
writeConfig(t, "plugins.toml", `
[[pipelines]]
name = "nested"
[[pipelines.plugin_sources]]
id = "in"
type = "null"
[[pipelines.plugin_sinks]]
id = "out"
type = "tcp"
[pipelines.plugin_sinks.config]
port = 9000
write_timeout_ms = 9007199254740993
[pipelines.plugin_sinks.config.tls]
enabled = false
server_name = "relay.example"
[pipelines.plugin_sinks.config.auth]
type = "none"
allow = ["one", "two"]
`)
m, err := Load([]string{"-c", "plugins.toml"})
if err != nil {
t.Fatal(err)
}
defer m.Close()
cfg, err := m.Snapshot()
if err != nil {
t.Fatal(err)
}
opts := TCPSinkOptions{Host: "127.0.0.1", BufferSize: 1000}
if err := lconfig.ScanMap(cfg.Pipelines[0].PluginSinks[0].Config, &opts); err != nil {
t.Fatal(err)
}
if opts.Host != "127.0.0.1" || opts.BufferSize != 1000 || opts.Port != 9000 || opts.WriteTimeoutMS != 9007199254740993 ||
opts.TLS == nil || opts.TLS.ServerName != "relay.example" || opts.Auth == nil || !reflect.DeepEqual(opts.Auth.Allow, []string{"one", "two"}) {
t.Fatalf("plugin defaults, nested tables or integer precision lost: %+v", opts)
}
}
+2
View File
@@ -76,6 +76,7 @@ func NewPipeline(
flowProcessor, err := flow.NewFlow(cfg.Flow, logger)
if err != nil {
// If flow fails, stop session manager
pipelineCancel()
sessionManager.Stop()
return nil, fmt.Errorf("failed to create flow processor: %w", err)
}
@@ -84,6 +85,7 @@ func NewPipeline(
// Initialize sources and sinks
if err := pipeline.initializeComponents(); err != nil {
pipelineCancel()
sessionManager.Stop()
return nil, err
}
+18 -8
View File
@@ -14,12 +14,13 @@ import (
// Service manages a collection of log processing pipelines
type Service struct {
pipelines map[string]*pipeline.Pipeline
mu sync.RWMutex
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
logger *log.Logger
pipelines map[string]*pipeline.Pipeline
mu sync.RWMutex
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
logger *log.Logger
shutdownOnce sync.Once
}
// NewService creates a new, empty service
@@ -49,9 +50,14 @@ func NewService(ctx context.Context, cfg *config.Config, logger *log.Logger) (*S
}
}
if errs != nil {
// A rejected reload must not retain prepared pipelines or their workers.
svc.Shutdown()
return nil, errs
}
logger.Info("msg", "Service initialization completed", "pipelines", len(svc.pipelines))
return svc, errs
return svc, nil
}
// Start starts all or specific pipelines
@@ -168,6 +174,10 @@ func (svc *Service) RemovePipeline(name string) error {
// Shutdown gracefully stops all pipelines managed by the service
func (svc *Service) Shutdown() {
svc.shutdownOnce.Do(svc.shutdown)
}
func (svc *Service) shutdown() {
svc.logger.Info("msg", "Service shutdown initiated")
svc.mu.Lock()
@@ -209,4 +219,4 @@ func (svc *Service) GetGlobalStats() map[string]any {
}
return stats
}
}
+3 -5
View File
@@ -29,10 +29,8 @@ func (p *Proxy) CreateSession(remoteAddr string, metadata map[string]any) *Sessi
metadata["instance_id"] = p.instanceID
// Create session with instance-scoped source
session := p.manager.CreateSession(remoteAddr, p.instanceID, metadata)
session.InstanceID = p.instanceID
return session
// Set identity before publishing the session to concurrent snapshot readers.
return p.manager.createSession(remoteAddr, p.instanceID, p.instanceID, metadata)
}
// GetSession retrieves a session if it belongs to this instance
@@ -79,4 +77,4 @@ func (p *Proxy) UpdateActivity(sessionID string) bool {
// GetInstanceID returns the instance ID this proxy is bound to
func (p *Proxy) GetInstanceID() string {
return p.instanceID
}
}
+20 -6
View File
@@ -4,6 +4,7 @@ import (
"crypto/rand"
"encoding/base64"
"fmt"
"maps"
"sync"
"time"
@@ -59,7 +60,12 @@ func NewManager(maxIdleTime time.Duration) *Manager {
// CreateSession creates and stores a new session for a connection
func (m *Manager) CreateSession(remoteAddr string, source string, metadata map[string]any) *Session {
return m.createSession(remoteAddr, source, "", metadata)
}
func (m *Manager) createSession(remoteAddr, source, instanceID string, metadata map[string]any) *Session {
session := &Session{
InstanceID: instanceID,
ID: generateSessionID(),
RemoteAddr: remoteAddr,
CreatedAt: time.Now(),
@@ -127,7 +133,7 @@ func (m *Manager) GetActiveSessions() []*Session {
sessions := make([]*Session, 0, len(m.sessions))
for _, session := range m.sessions {
sessions = append(sessions, session)
sessions = append(sessions, snapshotSession(session))
}
return sessions
}
@@ -139,7 +145,7 @@ func (m *Manager) GetSessionCount() int {
return len(m.sessions)
}
// GetSessionsBySource returns all sessions matching a specific source type
// GetSessionsBySource returns snapshots of sessions matching a specific source type.
func (m *Manager) GetSessionsBySource(source string) []*Session {
m.mu.RLock()
defer m.mu.RUnlock()
@@ -147,13 +153,13 @@ func (m *Manager) GetSessionsBySource(source string) []*Session {
var sessions []*Session
for _, session := range m.sessions {
if session.Source == source {
sessions = append(sessions, session)
sessions = append(sessions, snapshotSession(session))
}
}
return sessions
}
// GetActiveSessionsBySource returns all active sessions for a given source
// GetActiveSessionsBySource returns snapshots of active sessions for a given source.
func (m *Manager) GetActiveSessionsBySource(source string) []*Session {
m.mu.RLock()
defer m.mu.RUnlock()
@@ -163,12 +169,20 @@ func (m *Manager) GetActiveSessionsBySource(source string) []*Session {
for _, session := range m.sessions {
if session.Source == source && now.Sub(session.LastActivity) < m.maxIdleTime {
sessions = append(sessions, session)
sessions = append(sessions, snapshotSession(session))
}
}
return sessions
}
// Call under m.mu. Metadata values are immutable after session creation; copy
// the top-level map so readers cannot add or remove entries in stored metadata.
func snapshotSession(session *Session) *Session {
snapshot := *session
snapshot.Metadata = maps.Clone(session.Metadata)
return &snapshot
}
// GetStats returns statistics about the session manager
func (m *Manager) GetStats() map[string]any {
m.mu.RLock()
@@ -291,4 +305,4 @@ func generateSessionID() string {
return fmt.Sprintf("session_%d", time.Now().UnixNano())
}
return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(b)
}
}
+45
View File
@@ -0,0 +1,45 @@
package session
import (
"sync"
"testing"
"time"
)
func TestSessionSnapshotsDoNotRaceActivityUpdates(t *testing.T) {
m := NewManager(time.Minute)
defer m.Stop()
proxy := NewProxy(m, "source")
stored := proxy.CreateSession("remote", map[string]any{"label": "original"})
readers := []func() []*Session{
m.GetActiveSessions,
func() []*Session { return m.GetSessionsBySource("source") },
func() []*Session { return m.GetActiveSessionsBySource("source") },
}
var wg sync.WaitGroup
wg.Go(func() {
for range 1000 {
proxy.UpdateActivity(stored.ID)
}
})
for _, read := range readers {
for range 100 {
snapshots := read()
if len(snapshots) != 1 || snapshots[0] == stored || snapshots[0].InstanceID != "source" {
t.Errorf("invalid snapshot: %+v", snapshots)
break
}
snapshot := snapshots[0]
before := snapshot.LastActivity
proxy.UpdateActivity(stored.ID)
if !snapshot.LastActivity.Equal(before) {
t.Error("activity update changed an earlier snapshot")
}
snapshot.Metadata["label"] = "changed"
}
}
wg.Wait()
if stored.Metadata["label"] != "original" {
t.Fatal("snapshot metadata aliases the stored map")
}
}