lixen/logwisp
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.11.0 external formatter and sanitizer integrated, refactored

Browse Source
This commit is contained in:
lixen
2025-12-10 08:37:26 -05:00
parent 46a436baa0
commit 61d0269dcf
44 changed files with 1623 additions and 1501 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internal/config/config.go
+41 -54
View File
@@ -1,4 +1,3 @@
// FILE: logwisp/src/internal/config/config.go
package config
// --- LogWisp Configuration Options ---
@@ -6,13 +5,12 @@ package config
// Config is the top-level configuration structure for the LogWisp application
type Config struct {
// Top-level flags for application control
Background bool `toml:"background"`
ShowVersion bool `toml:"version"`
Quiet bool `toml:"quiet"`
// Runtime behavior flags
DisableStatusReporter bool `toml:"disable_status_reporter"`
ConfigAutoReload bool `toml:"config_auto_reload"`
StatusReporter bool `toml:"status_reporter"`
ConfigAutoReload bool `toml:"auto_reload"`
// Internal flag indicating demonized child process (DO NOT SET IN CONFIG FILE)
BackgroundDaemon bool
@@ -35,6 +33,12 @@ type LogConfig struct {
// Log level: "debug", "info", "warn", "error"
Level string `toml:"level"`
// Format: "raw", "txt", "json"
Format string `toml:"format"`
// Sanitization policy for console output
Sanitization string `toml:"sanitization"`
// File output settings (when Output includes "file" or "all")
File *LogFileConfig `toml:"file"`
@@ -64,9 +68,6 @@ type LogFileConfig struct {
type LogConsoleConfig struct {
// Target for console output: "stdout", "stderr"
Target string `toml:"target"`
// Format: "txt" or "json"
Format string `toml:"format"`
}
// --- Pipeline ---
@@ -76,11 +77,6 @@ type PipelineConfig struct {
Name string `toml:"name"`
Flow *FlowConfig `toml:"flow"`
// CHANGED: Legacy configs for backward compatibility
Sources []SourceConfig `toml:"sources,omitempty"`
Sinks []SinkConfig `toml:"sinks,omitempty"`
// CHANGED: New plugin-based configs
PluginSources []PluginSourceConfig `toml:"plugin_sources,omitempty"`
PluginSinks []PluginSinkConfig `toml:"plugin_sinks,omitempty"`
}
@@ -110,34 +106,10 @@ type HeartbeatConfig struct {
// FormatConfig is a polymorphic struct representing log entry formatting options
type FormatConfig struct {
// Format configuration - polymorphic like sources/sinks
Type string `toml:"type"` // "json", "txt", "raw"
// Only one will be populated based on format type
JSONFormatOptions *JSONFormatterOptions `toml:"json,omitempty"`
TxtFormatOptions *TxtFormatterOptions `toml:"txt,omitempty"`
RawFormatOptions *RawFormatterOptions `toml:"raw,omitempty"`
}
// JSONFormatterOptions defines settings for the JSON formatter
type JSONFormatterOptions struct {
Pretty bool `toml:"pretty"`
TimestampField string `toml:"timestamp_field"`
LevelField string `toml:"level_field"`
MessageField string `toml:"message_field"`
SourceField string `toml:"source_field"`
}
// TxtFormatterOptions defines settings for the text template formatter
type TxtFormatterOptions struct {
Template string `toml:"template"`
Type string `toml:"type"` // "json", "txt", "raw"
Flags int64 `toml:"flags"`
TimestampFormat string `toml:"timestamp_format"`
Colorize bool `toml:"colorize"` // TODO: Implement
}
// RawFormatterOptions defines settings for the raw pass-through formatter
type RawFormatterOptions struct {
AddNewLine bool `toml:"add_new_line"`
SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell"
}
// --- Rate Limit Options ---
@@ -200,16 +172,28 @@ type PluginSourceConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"`
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
}
// SourceConfig is a polymorphic struct representing a single data source
type SourceConfig struct {
Type string `toml:"type"`
// // SourceConfig is a polymorphic struct representing a single data source
// type SourceConfig struct {
// Type string `toml:"type"`
//
// // Polymorphic - only one populated based on type
// File *FileSourceOptions `toml:"file,omitempty"`
// Console *ConsoleSourceOptions `toml:"console,omitempty"`
// }
// Polymorphic - only one populated based on type
File *FileSourceOptions `toml:"file,omitempty"`
Console *ConsoleSourceOptions `toml:"console,omitempty"`
// NullSourceOptions defines settings for a null source (no configuration needed)
type NullSourceOptions struct{}
// RandomSourceOptions defines settings for a random log generator source
type RandomSourceOptions struct {
IntervalMS int64 `toml:"interval_ms"`
JitterMS int64 `toml:"jitter_ms"`
Format string `toml:"format"`
Length int64 `toml:"length"`
Special bool `toml:"special"`
}
// FileSourceOptions defines settings for a file-based source
@@ -232,17 +216,20 @@ type PluginSinkConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"`
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
}
// SinkConfig is a polymorphic struct representing a single data sink
type SinkConfig struct {
Type string `toml:"type"`
// // SinkConfig is a polymorphic struct representing a single data sink
// type SinkConfig struct {
// Type string `toml:"type"`
//
// // Polymorphic - only one populated based on type
// Console *ConsoleSinkOptions `toml:"console,omitempty"`
// File *FileSinkOptions `toml:"file,omitempty"`
// }
// Polymorphic - only one populated based on type
Console *ConsoleSinkOptions `toml:"console,omitempty"`
File *FileSinkOptions `toml:"file,omitempty"`
}
// NullSinkOptions defines settings for a null sink (no configuration needed)
type NullSinkOptions struct{}
// ConsoleSinkOptions defines settings for a console-based sink
type ConsoleSinkOptions struct {
src/internal/config/loader.go
+34 -17
View File
@@ -1,4 +1,3 @@
// FILE: logwisp/src/internal/config/loader.go
package config
import (
@@ -8,6 +7,8 @@ import (
"path/filepath"
"strings"
"logwisp/src/internal/core"
lconfig "github.com/lixenwraith/config"
)
@@ -48,7 +49,9 @@ func Load(args []string) (*Config, error) {
// Handle file not found errors - maintain existing behavior
if errors.Is(err, lconfig.ErrConfigNotFound) {
if isExplicit {
return nil, fmt.Errorf("config file not found: %s", configPath)
// Return empty config with file path
finalConfig.ConfigFile = configPath
return finalConfig, fmt.Errorf("config file not found: %s", configPath)
}
// If the default config file is not found, it's not an error, default/cli/env will be used
} else {
@@ -62,6 +65,17 @@ func Load(args []string) (*Config, error) {
// Store the manager for hot reload
configManager = cfg
// Start watcher if auto-reload is enabled
if finalConfig.ConfigAutoReload {
watchOpts := lconfig.WatchOptions{
PollInterval: core.ReloadWatchPollInterval,
Debounce: core.ReloadWatchDebounce,
ReloadTimeout: core.ReloadWatchTimeout,
VerifyPermissions: true,
}
cfg.AutoUpdateWithOptions(watchOpts)
}
return finalConfig, nil
}
@@ -74,13 +88,12 @@ func GetConfigManager() *lconfig.Config {
func defaults() *Config {
return &Config{
// Top-level flag defaults
Background: false,
ShowVersion: false,
Quiet: false,
// Runtime behavior defaults
DisableStatusReporter: false,
ConfigAutoReload: false,
StatusReporter: true,
ConfigAutoReload: false,
// Child process indicator
BackgroundDaemon: false,
@@ -98,28 +111,32 @@ func defaults() *Config {
},
Console: &LogConsoleConfig{
Target: "stdout",
Format: "txt",
},
},
Pipelines: []PipelineConfig{
{
Name: "default",
Sources: []SourceConfig{
Name: "default_pipeline",
Flow: &FlowConfig{},
PluginSources: []PluginSourceConfig{
{
Type: "file",
File: &FileSourceOptions{
Directory: "./",
Pattern: "*.log",
CheckIntervalMS: int64(100),
ID: "default_source",
Type: "random",
Config: map[string]any{
"special": true,
},
// Config: &FileSourceOptions{
// Directory: "./",
// Pattern: "*.log",
// CheckIntervalMS: int64(100),
},
},
Sinks: []SinkConfig{
PluginSinks: []PluginSinkConfig{
{
ID: "default_sink",
Type: "console",
Console: &ConsoleSinkOptions{
Target: "stdout",
BufferSize: 100,
Config: map[string]any{
"target": "stdout",
"buffer_size": 100,
},
},
},
src/internal/config/validation.go
+8 -210
View File
@@ -1,4 +1,3 @@
// FILE: logwisp/src/internal/config/validation.go
package config
import (
@@ -6,7 +5,6 @@ import (
"path/filepath"
"regexp"
"strings"
"time"
lconfig "github.com/lixenwraith/config"
)
@@ -25,15 +23,15 @@ func ValidateConfig(cfg *Config) error {
return fmt.Errorf("logging config: %w", err)
}
// Track used ports across all pipelines
allPorts := make(map[int64]string)
pipelineNames := make(map[string]bool)
// // Track used ports across all pipelines
// allPorts := make(map[int64]string)
// pipelineNames := make(map[string]bool)
for i, pipeline := range cfg.Pipelines {
if err := validatePipeline(i, &pipeline, pipelineNames, allPorts); err != nil {
return err
}
}
// for i, pipeline := range cfg.Pipelines {
// if err := validatePipeline(i, &pipeline, pipelineNames, allPorts); err != nil {
// return err
// }
// }
return nil
}
@@ -62,206 +60,6 @@ func validateLogConfig(cfg *LogConfig) error {
if !validTargets[cfg.Console.Target] {
return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
}
validFormats := map[string]bool{
"txt": true, "json": true, "": true,
}
if !validFormats[cfg.Console.Format] {
return fmt.Errorf("invalid console format: %s", cfg.Console.Format)
}
}
return nil
}
// validatePipeline validates a single pipeline's configuration
func validatePipeline(index int, p *PipelineConfig, pipelineNames map[string]bool, allPorts map[int64]string) error {
// Validate pipeline name
if err := lconfig.NonEmpty(p.Name); err != nil {
return fmt.Errorf("pipeline %d: missing name", index)
}
if pipelineNames[p.Name] {
return fmt.Errorf("pipeline %d: duplicate name '%s'", index, p.Name)
}
pipelineNames[p.Name] = true
// Must have at least one source
if len(p.Sources) == 0 {
return fmt.Errorf("pipeline '%s': no sources specified", p.Name)
}
// Validate each source
for j, source := range p.Sources {
if err := validateSourceConfig(p.Name, j, &source); err != nil {
return err
}
}
// Validate flow configuration
if p.Flow != nil {
// Validate rate limit if present
if p.Flow.RateLimit != nil {
if err := validateRateLimit(p.Name, p.Flow.RateLimit); err != nil {
return err
}
}
// Validate filters
for j, filter := range p.Flow.Filters {
if err := validateFilter(p.Name, j, &filter); err != nil {
return err
}
}
// Validate formatter configuration
if err := validateFormatterConfig(p); err != nil {
return fmt.Errorf("pipeline '%s': %w", p.Name, err)
}
}
// Must have at least one sink
if len(p.Sinks) == 0 {
return fmt.Errorf("pipeline '%s': no sinks specified", p.Name)
}
// Validate each sink
for j, sink := range p.Sinks {
if err := validateSinkConfig(p.Name, j, &sink, allPorts); err != nil {
return err
}
}
return nil
}
// validateSourceConfig validates a polymorphic source configuration
func validateSourceConfig(pipelineName string, index int, s *SourceConfig) error {
if err := lconfig.NonEmpty(s.Type); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: missing type", pipelineName, index)
}
// Count how many source configs are populated
populated := 0
var populatedType string
if s.File != nil {
populated++
populatedType = "file"
}
if s.Console != nil {
populated++
populatedType = "console"
}
if populated == 0 {
return fmt.Errorf("pipeline '%s' source[%d]: no configuration provided for type '%s'",
pipelineName, index, s.Type)
}
if populated > 1 {
return fmt.Errorf("pipeline '%s' source[%d]: multiple configurations provided, only one allowed",
pipelineName, index)
}
if populatedType != s.Type {
return fmt.Errorf("pipeline '%s' source[%d]: type mismatch - type is '%s' but config is for '%s'",
pipelineName, index, s.Type, populatedType)
}
// Validate specific source type
switch s.Type {
case "file":
return validateFileSource(pipelineName, index, s.File)
case "console":
return validateConsoleSource(pipelineName, index, s.Console)
default:
return fmt.Errorf("pipeline '%s' source[%d]: unknown type '%s'", pipelineName, index, s.Type)
}
}
// validateSinkConfig validates a polymorphic sink configuration
func validateSinkConfig(pipelineName string, index int, s *SinkConfig, allPorts map[int64]string) error {
if err := lconfig.NonEmpty(s.Type); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: missing type", pipelineName, index)
}
// Count populated sink configs
populated := 0
var populatedType string
if s.Console != nil {
populated++
populatedType = "console"
}
if s.File != nil {
populated++
populatedType = "file"
}
if populated == 0 {
return fmt.Errorf("pipeline '%s' sink[%d]: no configuration provided for type '%s'",
pipelineName, index, s.Type)
}
if populated > 1 {
return fmt.Errorf("pipeline '%s' sink[%d]: multiple configurations provided, only one allowed",
pipelineName, index)
}
if populatedType != s.Type {
return fmt.Errorf("pipeline '%s' sink[%d]: type mismatch - type is '%s' but config is for '%s'",
pipelineName, index, s.Type, populatedType)
}
// Validate specific sink type
switch s.Type {
case "console":
return validateConsoleSink(pipelineName, index, s.Console)
case "file":
return validateFileSink(pipelineName, index, s.File)
default:
return fmt.Errorf("pipeline '%s' sink[%d]: unknown type '%s'", pipelineName, index, s.Type)
}
}
// validateFormatterConfig validates formatter configuration
func validateFormatterConfig(p *PipelineConfig) error {
if p.Flow.Format == nil {
p.Flow.Format = &FormatConfig{
Type: "raw",
RawFormatOptions: &RawFormatterOptions{AddNewLine: true},
}
} else if p.Flow.Format.Type == "" {
p.Flow.Format.Type = "raw" // Default
}
switch p.Flow.Format.Type {
case "raw":
if p.Flow.Format.RawFormatOptions == nil {
p.Flow.Format.RawFormatOptions = &RawFormatterOptions{}
}
case "txt":
if p.Flow.Format.TxtFormatOptions == nil {
p.Flow.Format.TxtFormatOptions = &TxtFormatterOptions{}
}
// Default template format
templateStr := "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}{{ if .Fields }} {{.Fields}}{{ end }}"
if p.Flow.Format.TxtFormatOptions.Template != "" {
p.Flow.Format.TxtFormatOptions.Template = templateStr
}
// Default timestamp format
timestampFormat := time.RFC3339
if p.Flow.Format.TxtFormatOptions.TimestampFormat != "" {
p.Flow.Format.TxtFormatOptions.TimestampFormat = timestampFormat
}
case "json":
if p.Flow.Format.JSONFormatOptions == nil {
p.Flow.Format.JSONFormatOptions = &JSONFormatterOptions{}
}
}
return nil