lixen/logwisp
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.10.0 flow and plugin structure, networking and commands removed, dirty

Browse Source
This commit is contained in:
lixen
2025-11-11 16:42:09 -05:00
parent d38908e0f1
commit 46a436baa0
57 changed files with 2637 additions and 7301 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/internal/config/config.go
+150 -267
View File
@@ -3,7 +3,7 @@ package config
// --- LogWisp Configuration Options ---
// Config is the top-level configuration structure for the LogWisp application.
// Config is the top-level configuration structure for the LogWisp application
type Config struct {
// Top-level flags for application control
Background bool `toml:"background"`
@@ -27,7 +27,7 @@ type Config struct {
// --- Logging Options ---
// LogConfig represents the logging configuration for the LogWisp application itself.
// LogConfig represents the logging configuration for the LogWisp application itself
type LogConfig struct {
// Output mode: "file", "stdout", "stderr", "split", "all", "none"
Output string `toml:"output"`
@@ -42,7 +42,7 @@ type LogConfig struct {
Console *LogConsoleConfig `toml:"console"`
}
// LogFileConfig defines settings for file-based application logging.
// LogFileConfig defines settings for file-based application logging
type LogFileConfig struct {
// Directory for log files
Directory string `toml:"directory"`
@@ -60,74 +60,44 @@ type LogFileConfig struct {
RetentionHours float64 `toml:"retention_hours"`
}
// LogConsoleConfig defines settings for console-based application logging.
// LogConsoleConfig defines settings for console-based application logging
type LogConsoleConfig struct {
// Target for console output: "stdout", "stderr", "split"
// "split": info/debug to stdout, warn/error to stderr
// Target for console output: "stdout", "stderr"
Target string `toml:"target"`
// Format: "txt" or "json"
Format string `toml:"format"`
}
// --- Pipeline Options ---
// --- Pipeline ---
// PipelineConfig defines a complete data flow from sources to sinks.
// PipelineConfig defines a complete data flow from sources to sinks
type PipelineConfig struct {
Name string `toml:"name"`
Sources []SourceConfig `toml:"sources"`
Name string `toml:"name"`
Flow *FlowConfig `toml:"flow"`
// CHANGED: Legacy configs for backward compatibility
Sources []SourceConfig `toml:"sources,omitempty"`
Sinks []SinkConfig `toml:"sinks,omitempty"`
// CHANGED: New plugin-based configs
PluginSources []PluginSourceConfig `toml:"plugin_sources,omitempty"`
PluginSinks []PluginSinkConfig `toml:"plugin_sinks,omitempty"`
}
// --- Flow ---
// FlowConfig consolidates all processing stages between sources and sinks
type FlowConfig struct {
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
RateLimit *RateLimitConfig `toml:"rate_limit"`
Filters []FilterConfig `toml:"filters"`
Format *FormatConfig `toml:"format"`
Sinks []SinkConfig `toml:"sinks"`
}
// Common configuration structs used across components
// --- Heartbeat Options ---
// ACLConfig defines network-level access control and rate limiting rules.
type ACLConfig struct {
Enabled bool `toml:"enabled"`
RequestsPerSecond float64 `toml:"requests_per_second"`
BurstSize int64 `toml:"burst_size"`
ResponseMessage string `toml:"response_message"`
ResponseCode int64 `toml:"response_code"` // Default: 429
MaxConnectionsPerIP int64 `toml:"max_connections_per_ip"`
MaxConnectionsTotal int64 `toml:"max_connections_total"`
IPWhitelist []string `toml:"ip_whitelist"`
IPBlacklist []string `toml:"ip_blacklist"`
}
// TLSServerConfig defines TLS settings for a server (HTTP Source, HTTP Sink).
type TLSServerConfig struct {
Enabled bool `toml:"enabled"`
CertFile string `toml:"cert_file"` // Server's certificate file.
KeyFile string `toml:"key_file"` // Server's private key file.
ClientAuth bool `toml:"client_auth"` // Enable/disable mTLS.
ClientCAFile string `toml:"client_ca_file"` // CA for verifying client certificates.
VerifyClientCert bool `toml:"verify_client_cert"` // Require and verify client certs.
// Common TLS settings
MinVersion string `toml:"min_version"` // "TLS1.2", "TLS1.3"
MaxVersion string `toml:"max_version"`
CipherSuites string `toml:"cipher_suites"`
}
// TLSClientConfig defines TLS settings for a client (HTTP Client Sink).
type TLSClientConfig struct {
Enabled bool `toml:"enabled"`
ServerCAFile string `toml:"server_ca_file"` // CA for verifying the remote server's certificate.
ClientCertFile string `toml:"client_cert_file"` // Client's certificate for mTLS.
ClientKeyFile string `toml:"client_key_file"` // Client's private key for mTLS.
ServerName string `toml:"server_name"` // For server certificate validation (SNI).
InsecureSkipVerify bool `toml:"insecure_skip_verify"` // Skip server verification, Use with caution.
// Common TLS settings
MinVersion string `toml:"min_version"`
MaxVersion string `toml:"max_version"`
CipherSuites string `toml:"cipher_suites"`
}
// HeartbeatConfig defines settings for periodic keep-alive or status messages.
// HeartbeatConfig defines settings for periodic keep-alive or status messages
type HeartbeatConfig struct {
Enabled bool `toml:"enabled"`
IntervalMS int64 `toml:"interval_ms"`
@@ -136,211 +106,9 @@ type HeartbeatConfig struct {
Format string `toml:"format"`
}
// TODO: Future implementation
// ClientAuthConfig defines settings for client-side authentication.
type ClientAuthConfig struct {
Type string `toml:"type"` // "none"
}
// --- Source Options ---
// SourceConfig is a polymorphic struct representing a single data source.
type SourceConfig struct {
Type string `toml:"type"`
// Polymorphic - only one populated based on type
File *FileSourceOptions `toml:"file,omitempty"`
Console *ConsoleSourceOptions `toml:"console,omitempty"`
HTTP *HTTPSourceOptions `toml:"http,omitempty"`
TCP *TCPSourceOptions `toml:"tcp,omitempty"`
}
// FileSourceOptions defines settings for a file-based source.
type FileSourceOptions struct {
Directory string `toml:"directory"`
Pattern string `toml:"pattern"` // glob pattern
CheckIntervalMS int64 `toml:"check_interval_ms"`
Recursive bool `toml:"recursive"` // TODO: implement logic
}
// ConsoleSourceOptions defines settings for a stdin-based source.
type ConsoleSourceOptions struct {
BufferSize int64 `toml:"buffer_size"`
}
// HTTPSourceOptions defines settings for an HTTP server source.
type HTTPSourceOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
IngestPath string `toml:"ingest_path"`
BufferSize int64 `toml:"buffer_size"`
MaxRequestBodySize int64 `toml:"max_body_size"`
ReadTimeout int64 `toml:"read_timeout_ms"`
WriteTimeout int64 `toml:"write_timeout_ms"`
ACL *ACLConfig `toml:"acl"`
TLS *TLSServerConfig `toml:"tls"`
Auth *ServerAuthConfig `toml:"auth"`
}
// TCPSourceOptions defines settings for a TCP server source.
type TCPSourceOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
ReadTimeout int64 `toml:"read_timeout_ms"`
KeepAlive bool `toml:"keep_alive"`
KeepAlivePeriod int64 `toml:"keep_alive_period_ms"`
ACL *ACLConfig `toml:"acl"`
Auth *ServerAuthConfig `toml:"auth"`
}
// --- Sink Options ---
// SinkConfig is a polymorphic struct representing a single data sink.
type SinkConfig struct {
Type string `toml:"type"`
// Polymorphic - only one populated based on type
Console *ConsoleSinkOptions `toml:"console,omitempty"`
File *FileSinkOptions `toml:"file,omitempty"`
HTTP *HTTPSinkOptions `toml:"http,omitempty"`
TCP *TCPSinkOptions `toml:"tcp,omitempty"`
HTTPClient *HTTPClientSinkOptions `toml:"http_client,omitempty"`
TCPClient *TCPClientSinkOptions `toml:"tcp_client,omitempty"`
}
// ConsoleSinkOptions defines settings for a console-based sink.
type ConsoleSinkOptions struct {
Target string `toml:"target"` // "stdout", "stderr", "split"
Colorize bool `toml:"colorize"`
BufferSize int64 `toml:"buffer_size"`
}
// FileSinkOptions defines settings for a file-based sink.
type FileSinkOptions struct {
Directory string `toml:"directory"`
Name string `toml:"name"`
MaxSizeMB int64 `toml:"max_size_mb"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
MinDiskFreeMB int64 `toml:"min_disk_free_mb"`
RetentionHours float64 `toml:"retention_hours"`
BufferSize int64 `toml:"buffer_size"`
FlushInterval int64 `toml:"flush_interval_ms"`
}
// HTTPSinkOptions defines settings for an HTTP server sink.
type HTTPSinkOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
StreamPath string `toml:"stream_path"`
StatusPath string `toml:"status_path"`
BufferSize int64 `toml:"buffer_size"`
WriteTimeout int64 `toml:"write_timeout_ms"`
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
ACL *ACLConfig `toml:"acl"`
TLS *TLSServerConfig `toml:"tls"`
Auth *ServerAuthConfig `toml:"auth"`
}
// TCPSinkOptions defines settings for a TCP server sink.
type TCPSinkOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
WriteTimeout int64 `toml:"write_timeout_ms"`
KeepAlive bool `toml:"keep_alive"`
KeepAlivePeriod int64 `toml:"keep_alive_period_ms"`
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
ACL *ACLConfig `toml:"acl"`
Auth *ServerAuthConfig `toml:"auth"`
}
// HTTPClientSinkOptions defines settings for an HTTP client sink.
type HTTPClientSinkOptions struct {
URL string `toml:"url"`
BufferSize int64 `toml:"buffer_size"`
BatchSize int64 `toml:"batch_size"`
BatchDelayMS int64 `toml:"batch_delay_ms"`
Timeout int64 `toml:"timeout_seconds"`
MaxRetries int64 `toml:"max_retries"`
RetryDelayMS int64 `toml:"retry_delay_ms"`
RetryBackoff float64 `toml:"retry_backoff"`
InsecureSkipVerify bool `toml:"insecure_skip_verify"`
TLS *TLSClientConfig `toml:"tls"`
Auth *ClientAuthConfig `toml:"auth"`
}
// TCPClientSinkOptions defines settings for a TCP client sink.
type TCPClientSinkOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
DialTimeout int64 `toml:"dial_timeout_seconds"`
WriteTimeout int64 `toml:"write_timeout_seconds"`
ReadTimeout int64 `toml:"read_timeout_seconds"`
KeepAlive int64 `toml:"keep_alive_seconds"`
ReconnectDelayMS int64 `toml:"reconnect_delay_ms"`
MaxReconnectDelayMS int64 `toml:"max_reconnect_delay_ms"`
ReconnectBackoff float64 `toml:"reconnect_backoff"`
Auth *ClientAuthConfig `toml:"auth"`
}
// --- Rate Limit Options ---
// RateLimitPolicy defines the action to take when a rate limit is exceeded.
type RateLimitPolicy int
const (
// PolicyPass allows all logs through, effectively disabling the limiter.
PolicyPass RateLimitPolicy = iota
// PolicyDrop drops logs that exceed the rate limit.
PolicyDrop
)
// RateLimitConfig defines the configuration for pipeline-level rate limiting.
type RateLimitConfig struct {
// Rate is the number of log entries allowed per second. Default: 0 (disabled).
Rate float64 `toml:"rate"`
// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate.
Burst float64 `toml:"burst"`
// Policy defines the action to take when the limit is exceeded. "pass" or "drop".
Policy string `toml:"policy"`
// MaxEntrySizeBytes is the maximum allowed size for a single log entry. 0 = no limit.
MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes"`
}
// --- Filter Options ---
// FilterType represents the filter's behavior (include or exclude).
type FilterType string
const (
// FilterTypeInclude specifies that only matching logs will pass.
FilterTypeInclude FilterType = "include" // Whitelist - only matching logs pass
// FilterTypeExclude specifies that matching logs will be dropped.
FilterTypeExclude FilterType = "exclude" // Blacklist - matching logs are dropped
)
// FilterLogic represents how multiple filter patterns are combined.
type FilterLogic string
const (
// FilterLogicOr specifies that a match on any pattern is sufficient.
FilterLogicOr FilterLogic = "or" // Match any pattern
// FilterLogicAnd specifies that all patterns must match.
FilterLogicAnd FilterLogic = "and" // Match all patterns
)
// FilterConfig represents the configuration for a single filter.
type FilterConfig struct {
Type FilterType `toml:"type"`
Logic FilterLogic `toml:"logic"`
Patterns []string `toml:"patterns"`
}
// --- Formatter Options ---
// FormatConfig is a polymorphic struct representing log entry formatting options.
// FormatConfig is a polymorphic struct representing log entry formatting options
type FormatConfig struct {
// Format configuration - polymorphic like sources/sinks
Type string `toml:"type"` // "json", "txt", "raw"
@@ -351,7 +119,7 @@ type FormatConfig struct {
RawFormatOptions *RawFormatterOptions `toml:"raw,omitempty"`
}
// JSONFormatterOptions defines settings for the JSON formatter.
// JSONFormatterOptions defines settings for the JSON formatter
type JSONFormatterOptions struct {
Pretty bool `toml:"pretty"`
TimestampField string `toml:"timestamp_field"`
@@ -360,21 +128,136 @@ type JSONFormatterOptions struct {
SourceField string `toml:"source_field"`
}
// TxtFormatterOptions defines settings for the text template formatter.
// TxtFormatterOptions defines settings for the text template formatter
type TxtFormatterOptions struct {
Template string `toml:"template"`
TimestampFormat string `toml:"timestamp_format"`
Colorize bool `toml:"colorize"` // TODO: Implement
}
// RawFormatterOptions defines settings for the raw pass-through formatter.
// RawFormatterOptions defines settings for the raw pass-through formatter
type RawFormatterOptions struct {
AddNewLine bool `toml:"add_new_line"`
}
// --- Server-side Auth (for sources) ---
// --- Rate Limit Options ---
// TODO: future implementation
// ServerAuthConfig defines settings for server-side authentication.
type ServerAuthConfig struct {
Type string `toml:"type"` // "none"
// RateLimitPolicy defines the action to take when a rate limit is exceeded
type RateLimitPolicy int
const (
// PolicyPass allows all logs through, effectively disabling the limiter
PolicyPass RateLimitPolicy = iota
// PolicyDrop drops logs that exceed the rate limit
PolicyDrop
)
// RateLimitConfig defines the configuration for pipeline-level rate limiting
type RateLimitConfig struct {
// Rate is the number of log entries allowed per second. Default: 0 (disabled)
Rate float64 `toml:"rate"`
// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate
Burst float64 `toml:"burst"`
// Policy defines the action to take when the limit is exceeded. "pass" or "drop"
Policy string `toml:"policy"`
// MaxEntrySizeBytes is the maximum allowed size for a single log entry. 0 = no limit
MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes"`
}
// --- Filter Options ---
// FilterType represents the filter's behavior (include or exclude)
type FilterType string
const (
// FilterTypeInclude specifies that only matching logs will pass
FilterTypeInclude FilterType = "include" // Whitelist - only matching logs pass
// FilterTypeExclude specifies that matching logs will be dropped
FilterTypeExclude FilterType = "exclude" // Blacklist - matching logs are dropped
)
// FilterLogic represents how multiple filter patterns are combined
type FilterLogic string
const (
// FilterLogicOr specifies that a match on any pattern is sufficient
FilterLogicOr FilterLogic = "or" // Match any pattern
// FilterLogicAnd specifies that all patterns must match
FilterLogicAnd FilterLogic = "and" // Match all patterns
)
// FilterConfig represents the configuration for a single filter
type FilterConfig struct {
Type FilterType `toml:"type"`
Logic FilterLogic `toml:"logic"`
Patterns []string `toml:"patterns"`
}
// --- Source Options ---
// PluginSourceConfig represents a source plugin instance configuration
type PluginSourceConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"`
}
// SourceConfig is a polymorphic struct representing a single data source
type SourceConfig struct {
Type string `toml:"type"`
// Polymorphic - only one populated based on type
File *FileSourceOptions `toml:"file,omitempty"`
Console *ConsoleSourceOptions `toml:"console,omitempty"`
}
// FileSourceOptions defines settings for a file-based source
type FileSourceOptions struct {
Directory string `toml:"directory"`
Pattern string `toml:"pattern"` // glob pattern
CheckIntervalMS int64 `toml:"check_interval_ms"`
Recursive bool `toml:"recursive"` // TODO: implement logic
}
// ConsoleSourceOptions defines settings for a stdin-based source
type ConsoleSourceOptions struct {
BufferSize int64 `toml:"buffer_size"`
}
// --- Sink Options ---
// PluginSinkConfig represents a sink plugin instance configuration
type PluginSinkConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"`
}
// SinkConfig is a polymorphic struct representing a single data sink
type SinkConfig struct {
Type string `toml:"type"`
// Polymorphic - only one populated based on type
Console *ConsoleSinkOptions `toml:"console,omitempty"`
File *FileSinkOptions `toml:"file,omitempty"`
}
// ConsoleSinkOptions defines settings for a console-based sink
type ConsoleSinkOptions struct {
Target string `toml:"target"` // "stdout", "stderr"
BufferSize int64 `toml:"buffer_size"`
}
// FileSinkOptions defines settings for a file-based sink
type FileSinkOptions struct {
Directory string `toml:"directory"`
Name string `toml:"name"`
MaxSizeMB int64 `toml:"max_size_mb"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
MinDiskFreeMB int64 `toml:"min_disk_free_mb"`
RetentionHours float64 `toml:"retention_hours"`
BufferSize int64 `toml:"buffer_size"`
FlushIntervalMs int64 `toml:"flush_interval_ms"`
}
src/internal/config/loader.go
+6 -7
View File
@@ -11,10 +11,10 @@ import (
lconfig "github.com/lixenwraith/config"
)
// configManager holds the global instance of the configuration manager.
// configManager holds the global instance of the configuration manager
var configManager *lconfig.Config
// Load is the single entry point for loading all application configuration.
// Load is the single entry point for loading all application configuration
func Load(args []string) (*Config, error) {
configPath, isExplicit := resolveConfigPath(args)
// Build configuration with all sources
@@ -65,12 +65,12 @@ func Load(args []string) (*Config, error) {
return finalConfig, nil
}
// GetConfigManager returns the global configuration manager instance for hot-reloading.
// GetConfigManager returns the global configuration manager instance for hot-reloading
func GetConfigManager() *lconfig.Config {
return configManager
}
// defaults provides the default configuration values for the application.
// defaults provides the default configuration values for the application
func defaults() *Config {
return &Config{
// Top-level flag defaults
@@ -119,7 +119,6 @@ func defaults() *Config {
Type: "console",
Console: &ConsoleSinkOptions{
Target: "stdout",
Colorize: false,
BufferSize: 100,
},
},
@@ -129,7 +128,7 @@ func defaults() *Config {
}
}
// resolveConfigPath determines the configuration file path based on CLI args, env vars, and default locations.
// resolveConfigPath determines the configuration file path based on CLI args, env vars, and default locations
func resolveConfigPath(args []string) (path string, isExplicit bool) {
// 1. Check for --config flag in command-line arguments (highest precedence)
for i, arg := range args {
@@ -165,7 +164,7 @@ func resolveConfigPath(args []string) (path string, isExplicit bool) {
return "logwisp.toml", false
}
// customEnvTransform converts TOML-style config paths (e.g., logging.level) to environment variable format (LOGGING_LEVEL).
// customEnvTransform converts TOML-style config paths (e.g., logging.level) to environment variable format (LOGGING_LEVEL)
func customEnvTransform(path string) string {
env := strings.ReplaceAll(path, ".", "_")
env = strings.ToUpper(env)
src/internal/config/validation.go
+50 -436
View File
@@ -3,7 +3,6 @@ package config
import (
"fmt"
"net/url"
"path/filepath"
"regexp"
"strings"
@@ -12,7 +11,7 @@ import (
lconfig "github.com/lixenwraith/config"
)
// ValidateConfig is the centralized validator for the entire configuration structure.
// ValidateConfig is the centralized validator for the entire configuration structure
func ValidateConfig(cfg *Config) error {
if cfg == nil {
return fmt.Errorf("config is nil")
@@ -39,7 +38,7 @@ func ValidateConfig(cfg *Config) error {
return nil
}
// validateLogConfig validates the application's own logging settings.
// validateLogConfig validates the application's own logging settings
func validateLogConfig(cfg *LogConfig) error {
validOutputs := map[string]bool{
"file": true, "stdout": true, "stderr": true,
@@ -75,7 +74,7 @@ func validateLogConfig(cfg *LogConfig) error {
return nil
}
// validatePipeline validates a single pipeline's configuration.
// validatePipeline validates a single pipeline's configuration
func validatePipeline(index int, p *PipelineConfig, pipelineNames map[string]bool, allPorts map[int64]string) error {
// Validate pipeline name
if err := lconfig.NonEmpty(p.Name); err != nil {
@@ -99,23 +98,28 @@ func validatePipeline(index int, p *PipelineConfig, pipelineNames map[string]boo
}
}
// Validate rate limit if present
if p.RateLimit != nil {
if err := validateRateLimit(p.Name, p.RateLimit); err != nil {
return err
}
}
// Validate flow configuration
if p.Flow != nil {
// Validate filters
for j, filter := range p.Filters {
if err := validateFilter(p.Name, j, &filter); err != nil {
return err
// Validate rate limit if present
if p.Flow.RateLimit != nil {
if err := validateRateLimit(p.Name, p.Flow.RateLimit); err != nil {
return err
}
}
// Validate filters
for j, filter := range p.Flow.Filters {
if err := validateFilter(p.Name, j, &filter); err != nil {
return err
}
}
// Validate formatter configuration
if err := validateFormatterConfig(p); err != nil {
return fmt.Errorf("pipeline '%s': %w", p.Name, err)
}
}
// Validate formatter configuration
if err := validateFormatterConfig(p); err != nil {
return fmt.Errorf("pipeline '%s': %w", p.Name, err)
}
// Must have at least one sink
@@ -133,7 +137,7 @@ func validatePipeline(index int, p *PipelineConfig, pipelineNames map[string]boo
return nil
}
// validateSourceConfig validates a polymorphic source configuration.
// validateSourceConfig validates a polymorphic source configuration
func validateSourceConfig(pipelineName string, index int, s *SourceConfig) error {
if err := lconfig.NonEmpty(s.Type); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: missing type", pipelineName, index)
@@ -151,14 +155,6 @@ func validateSourceConfig(pipelineName string, index int, s *SourceConfig) error
populated++
populatedType = "console"
}
if s.HTTP != nil {
populated++
populatedType = "http"
}
if s.TCP != nil {
populated++
populatedType = "tcp"
}
if populated == 0 {
return fmt.Errorf("pipeline '%s' source[%d]: no configuration provided for type '%s'",
@@ -176,19 +172,15 @@ func validateSourceConfig(pipelineName string, index int, s *SourceConfig) error
// Validate specific source type
switch s.Type {
case "file":
return validateDirectorySource(pipelineName, index, s.File)
return validateFileSource(pipelineName, index, s.File)
case "console":
return validateConsoleSource(pipelineName, index, s.Console)
case "http":
return validateHTTPSource(pipelineName, index, s.HTTP)
case "tcp":
return validateTCPSource(pipelineName, index, s.TCP)
default:
return fmt.Errorf("pipeline '%s' source[%d]: unknown type '%s'", pipelineName, index, s.Type)
}
}
// validateSinkConfig validates a polymorphic sink configuration.
// validateSinkConfig validates a polymorphic sink configuration
func validateSinkConfig(pipelineName string, index int, s *SinkConfig, allPorts map[int64]string) error {
if err := lconfig.NonEmpty(s.Type); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: missing type", pipelineName, index)
@@ -206,22 +198,6 @@ func validateSinkConfig(pipelineName string, index int, s *SinkConfig, allPorts
populated++
populatedType = "file"
}
if s.HTTP != nil {
populated++
populatedType = "http"
}
if s.TCP != nil {
populated++
populatedType = "tcp"
}
if s.HTTPClient != nil {
populated++
populatedType = "http_client"
}
if s.TCPClient != nil {
populated++
populatedType = "tcp_client"
}
if populated == 0 {
return fmt.Errorf("pipeline '%s' sink[%d]: no configuration provided for type '%s'",
@@ -242,14 +218,6 @@ func validateSinkConfig(pipelineName string, index int, s *SinkConfig, allPorts
return validateConsoleSink(pipelineName, index, s.Console)
case "file":
return validateFileSink(pipelineName, index, s.File)
case "http":
return validateHTTPSink(pipelineName, index, s.HTTP, allPorts)
case "tcp":
return validateTCPSink(pipelineName, index, s.TCP, allPorts)
case "http_client":
return validateHTTPClientSink(pipelineName, index, s.HTTPClient)
case "tcp_client":
return validateTCPClientSink(pipelineName, index, s.TCPClient)
default:
return fmt.Errorf("pipeline '%s' sink[%d]: unknown type '%s'", pipelineName, index, s.Type)
}
@@ -257,48 +225,49 @@ func validateSinkConfig(pipelineName string, index int, s *SinkConfig, allPorts
// validateFormatterConfig validates formatter configuration
func validateFormatterConfig(p *PipelineConfig) error {
if p.Format == nil {
p.Format = &FormatConfig{
Type: "raw",
if p.Flow.Format == nil {
p.Flow.Format = &FormatConfig{
Type: "raw",
RawFormatOptions: &RawFormatterOptions{AddNewLine: true},
}
} else if p.Format.Type == "" {
p.Format.Type = "raw" // Default
} else if p.Flow.Format.Type == "" {
p.Flow.Format.Type = "raw" // Default
}
switch p.Format.Type {
switch p.Flow.Format.Type {
case "raw":
if p.Format.RawFormatOptions == nil {
p.Format.RawFormatOptions = &RawFormatterOptions{}
if p.Flow.Format.RawFormatOptions == nil {
p.Flow.Format.RawFormatOptions = &RawFormatterOptions{}
}
case "txt":
if p.Format.TxtFormatOptions == nil {
p.Format.TxtFormatOptions = &TxtFormatterOptions{}
if p.Flow.Format.TxtFormatOptions == nil {
p.Flow.Format.TxtFormatOptions = &TxtFormatterOptions{}
}
// Default template format
templateStr := "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}{{ if .Fields }} {{.Fields}}{{ end }}"
if p.Format.TxtFormatOptions.Template != "" {
p.Format.TxtFormatOptions.Template = templateStr
if p.Flow.Format.TxtFormatOptions.Template != "" {
p.Flow.Format.TxtFormatOptions.Template = templateStr
}
// Default timestamp format
timestampFormat := time.RFC3339
if p.Format.TxtFormatOptions.TimestampFormat != "" {
p.Format.TxtFormatOptions.TimestampFormat = timestampFormat
if p.Flow.Format.TxtFormatOptions.TimestampFormat != "" {
p.Flow.Format.TxtFormatOptions.TimestampFormat = timestampFormat
}
case "json":
if p.Format.JSONFormatOptions == nil {
p.Format.JSONFormatOptions = &JSONFormatterOptions{}
if p.Flow.Format.JSONFormatOptions == nil {
p.Flow.Format.JSONFormatOptions = &JSONFormatterOptions{}
}
}
return nil
}
// validateRateLimit validates the pipeline-level rate limit settings.
// validateRateLimit validates the pipeline-level rate limit settings
func validateRateLimit(pipelineName string, cfg *RateLimitConfig) error {
if cfg == nil {
return nil
@@ -328,7 +297,7 @@ func validateRateLimit(pipelineName string, cfg *RateLimitConfig) error {
return nil
}
// validateFilter validates a single filter's configuration.
// validateFilter validates a single filter's configuration
func validateFilter(pipelineName string, filterIndex int, cfg *FilterConfig) error {
// Validate filter type
switch cfg.Type {
@@ -364,8 +333,8 @@ func validateFilter(pipelineName string, filterIndex int, cfg *FilterConfig) err
return nil
}
// validateDirectorySource validates the settings for a directory source.
func validateDirectorySource(pipelineName string, index int, opts *FileSourceOptions) error {
// validateFileSource validates the settings for a directory source
func validateFileSource(pipelineName string, index int, opts *FileSourceOptions) error {
if err := lconfig.NonEmpty(opts.Directory); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: directory requires 'path'", pipelineName, index)
} else {
@@ -401,7 +370,7 @@ func validateDirectorySource(pipelineName string, index int, opts *FileSourceOpt
return nil
}
// validateConsoleSource validates the settings for a console source.
// validateConsoleSource validates the settings for a console source
func validateConsoleSource(pipelineName string, index int, opts *ConsoleSourceOptions) error {
if opts.BufferSize < 0 {
return fmt.Errorf("pipeline '%s' source[%d]: buffer_size must be positive", pipelineName, index)
@@ -411,111 +380,7 @@ func validateConsoleSource(pipelineName string, index int, opts *ConsoleSourceOp
return nil
}
// validateHTTPSource validates the settings for an HTTP source.
func validateHTTPSource(pipelineName string, index int, opts *HTTPSourceOptions) error {
// Validate port
if err := lconfig.Port(opts.Port); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
}
// Set defaults
if opts.Host == "" {
opts.Host = "0.0.0.0"
}
if opts.IngestPath == "" {
opts.IngestPath = "/ingest"
}
if opts.MaxRequestBodySize <= 0 {
opts.MaxRequestBodySize = 10 * 1024 * 1024 // 10MB default
}
if opts.ReadTimeout <= 0 {
opts.ReadTimeout = 5000 // 5 seconds
}
if opts.WriteTimeout <= 0 {
opts.WriteTimeout = 5000 // 5 seconds
}
// Validate host if specified
if opts.Host != "" && opts.Host != "0.0.0.0" {
if err := lconfig.IPAddress(opts.Host); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
}
}
// Validate paths
if !strings.HasPrefix(opts.IngestPath, "/") {
return fmt.Errorf("pipeline '%s' source[%d]: ingest_path must start with /", pipelineName, index)
}
// Validate auth configuration
validHTTPSourceAuthTypes := map[string]bool{"basic": true, "token": true, "mtls": true}
if opts.Auth != nil && opts.Auth.Type != "none" && opts.Auth.Type != "" {
if !validHTTPSourceAuthTypes[opts.Auth.Type] {
return fmt.Errorf("pipeline '%s' source[%d]: %s is not a valid auth type",
pipelineName, index, opts.Auth.Type)
}
// All non-none auth types require TLS for HTTP
if opts.TLS == nil || !opts.TLS.Enabled {
return fmt.Errorf("pipeline '%s' source[%d]: %s auth requires TLS to be enabled",
pipelineName, index, opts.Auth.Type)
}
}
// Validate nested configs
if opts.ACL != nil {
if err := validateACL(pipelineName, fmt.Sprintf("source[%d]", index), opts.ACL); err != nil {
return err
}
}
if opts.TLS != nil {
if err := validateTLSServer(pipelineName, fmt.Sprintf("source[%d]", index), opts.TLS); err != nil {
return err
}
}
return nil
}
// validateTCPSource validates the settings for a TCP source.
func validateTCPSource(pipelineName string, index int, opts *TCPSourceOptions) error {
// Validate port
if err := lconfig.Port(opts.Port); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
}
// Set defaults
if opts.Host == "" {
opts.Host = "0.0.0.0"
}
if opts.ReadTimeout <= 0 {
opts.ReadTimeout = 5000 // 5 seconds
}
if !opts.KeepAlive {
opts.KeepAlive = true // Default enabled
}
if opts.KeepAlivePeriod <= 0 {
opts.KeepAlivePeriod = 30000 // 30 seconds
}
// Validate host if specified
if opts.Host != "" && opts.Host != "0.0.0.0" {
if err := lconfig.IPAddress(opts.Host); err != nil {
return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
}
}
// Validate ACL if present
if opts.ACL != nil {
if err := validateACL(pipelineName, fmt.Sprintf("source[%d]", index), opts.ACL); err != nil {
return err
}
}
return nil
}
// validateConsoleSink validates the settings for a console sink.
// validateConsoleSink validates the settings for a console sink
func validateConsoleSink(pipelineName string, index int, opts *ConsoleSinkOptions) error {
if opts.BufferSize < 1 {
return fmt.Errorf("pipeline '%s' sink[%d]: buffer_size must be positive", pipelineName, index)
@@ -523,7 +388,7 @@ func validateConsoleSink(pipelineName string, index int, opts *ConsoleSinkOption
return nil
}
// validateFileSink validates the settings for a file sink.
// validateFileSink validates the settings for a file sink
func validateFileSink(pipelineName string, index int, opts *FileSinkOptions) error {
if err := lconfig.NonEmpty(opts.Directory); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: file requires 'directory'", pipelineName, index)
@@ -557,258 +422,7 @@ func validateFileSink(pipelineName string, index int, opts *FileSinkOptions) err
return nil
}
// validateHTTPSink validates the settings for an HTTP sink.
func validateHTTPSink(pipelineName string, index int, opts *HTTPSinkOptions, allPorts map[int64]string) error {
// Validate port
if err := lconfig.Port(opts.Port); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
}
// Check port conflicts
if existing, exists := allPorts[opts.Port]; exists {
return fmt.Errorf("pipeline '%s' sink[%d]: port %d already used by %s",
pipelineName, index, opts.Port, existing)
}
allPorts[opts.Port] = fmt.Sprintf("%s-http[%d]", pipelineName, index)
// Validate host if specified
if opts.Host != "" {
if err := lconfig.IPAddress(opts.Host); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
}
}
// Validate paths
if !strings.HasPrefix(opts.StreamPath, "/") {
return fmt.Errorf("pipeline '%s' sink[%d]: stream_path must start with /", pipelineName, index)
}
if !strings.HasPrefix(opts.StatusPath, "/") {
return fmt.Errorf("pipeline '%s' sink[%d]: status_path must start with /", pipelineName, index)
}
// Validate buffer
if opts.BufferSize < 1 {
return fmt.Errorf("pipeline '%s' sink[%d]: buffer_size must be positive", pipelineName, index)
}
// Validate nested configs
if opts.Heartbeat != nil {
if err := validateHeartbeat(pipelineName, fmt.Sprintf("sink[%d]", index), opts.Heartbeat); err != nil {
return err
}
}
if opts.ACL != nil {
if err := validateACL(pipelineName, fmt.Sprintf("sink[%d]", index), opts.ACL); err != nil {
return err
}
}
if opts.TLS != nil {
if err := validateTLSServer(pipelineName, fmt.Sprintf("sink[%d]", index), opts.TLS); err != nil {
return err
}
}
return nil
}
// validateTCPSink validates the settings for a TCP sink.
func validateTCPSink(pipelineName string, index int, opts *TCPSinkOptions, allPorts map[int64]string) error {
// Validate port
if err := lconfig.Port(opts.Port); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
}
// Check port conflicts
if existing, exists := allPorts[opts.Port]; exists {
return fmt.Errorf("pipeline '%s' sink[%d]: port %d already used by %s",
pipelineName, index, opts.Port, existing)
}
allPorts[opts.Port] = fmt.Sprintf("%s-tcp[%d]", pipelineName, index)
// Validate host if specified
if opts.Host != "" {
if err := lconfig.IPAddress(opts.Host); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
}
}
// Validate buffer
if opts.BufferSize < 1 {
return fmt.Errorf("pipeline '%s' sink[%d]: buffer_size must be positive", pipelineName, index)
}
// Validate nested configs
if opts.Heartbeat != nil {
if err := validateHeartbeat(pipelineName, fmt.Sprintf("sink[%d]", index), opts.Heartbeat); err != nil {
return err
}
}
if opts.ACL != nil {
if err := validateACL(pipelineName, fmt.Sprintf("sink[%d]", index), opts.ACL); err != nil {
return err
}
}
return nil
}
// validateHTTPClientSink validates the settings for an HTTP client sink.
func validateHTTPClientSink(pipelineName string, index int, opts *HTTPClientSinkOptions) error {
// Validate URL
if err := lconfig.NonEmpty(opts.URL); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: http_client requires 'url'", pipelineName, index)
}
parsedURL, err := url.Parse(opts.URL)
if err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: invalid URL: %w", pipelineName, index, err)
}
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
return fmt.Errorf("pipeline '%s' sink[%d]: URL must use http or https scheme", pipelineName, index)
}
// Set defaults for unspecified fields
if opts.BufferSize <= 0 {
opts.BufferSize = 1000
}
if opts.BatchSize <= 0 {
opts.BatchSize = 100
}
if opts.BatchDelayMS <= 0 {
opts.BatchDelayMS = 1000 // 1 second in ms
}
if opts.Timeout <= 0 {
opts.Timeout = 30 // 30 seconds
}
if opts.MaxRetries < 0 {
opts.MaxRetries = 3
}
if opts.RetryDelayMS <= 0 {
opts.RetryDelayMS = 1000 // 1 second in ms
}
if opts.RetryBackoff < 1.0 {
opts.RetryBackoff = 2.0
}
// Validate TLS config if present
if opts.TLS != nil {
if err := validateTLSClient(pipelineName, fmt.Sprintf("sink[%d]", index), opts.TLS); err != nil {
return err
}
}
return nil
}
// validateTCPClientSink validates the settings for a TCP client sink.
func validateTCPClientSink(pipelineName string, index int, opts *TCPClientSinkOptions) error {
// Validate host and port
if err := lconfig.NonEmpty(opts.Host); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: tcp_client requires 'host'", pipelineName, index)
}
if err := lconfig.Port(opts.Port); err != nil {
return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
}
// Set defaults
if opts.BufferSize <= 0 {
opts.BufferSize = 1000
}
if opts.DialTimeout <= 0 {
opts.DialTimeout = 10
}
if opts.WriteTimeout <= 0 {
opts.WriteTimeout = 30 // 30 seconds
}
if opts.ReadTimeout <= 0 {
opts.ReadTimeout = 10 // 10 seconds
}
if opts.KeepAlive <= 0 {
opts.KeepAlive = 30 // 30 seconds
}
if opts.ReconnectDelayMS <= 0 {
opts.ReconnectDelayMS = 1000 // 1 second in ms
}
if opts.MaxReconnectDelayMS <= 0 {
opts.MaxReconnectDelayMS = 30000 // 30 seconds in ms
}
if opts.ReconnectBackoff < 1.0 {
opts.ReconnectBackoff = 1.5
}
return nil
}
// validateACL validates nested ACLConfig settings.
func validateACL(pipelineName, location string, nl *ACLConfig) error {
if !nl.Enabled {
return nil // Skip validation if disabled
}
if nl.MaxConnectionsPerIP < 0 {
return fmt.Errorf("pipeline '%s' %s: max_connections_per_ip cannot be negative", pipelineName, location)
}
if nl.MaxConnectionsTotal < 0 {
return fmt.Errorf("pipeline '%s' %s: max_connections_total cannot be negative", pipelineName, location)
}
if nl.MaxConnectionsTotal < nl.MaxConnectionsPerIP && nl.MaxConnectionsTotal != 0 {
return fmt.Errorf("pipeline '%s' %s: max_connections_total cannot be less than max_connections_per_ip", pipelineName, location)
}
if nl.BurstSize < 0 {
return fmt.Errorf("pipeline '%s' %s: burst_size cannot be negative", pipelineName, location)
}
return nil
}
// validateTLSServer validates the new TLSServerConfig struct.
func validateTLSServer(pipelineName, location string, tls *TLSServerConfig) error {
if !tls.Enabled {
return nil // Skip validation if disabled
}
// If TLS is enabled for a server, cert and key files are mandatory.
if tls.CertFile == "" || tls.KeyFile == "" {
return fmt.Errorf("pipeline '%s' %s: TLS enabled requires both cert_file and key_file", pipelineName, location)
}
// If mTLS (ClientAuth) is enabled, a client CA file is mandatory.
if tls.ClientAuth && tls.ClientCAFile == "" {
return fmt.Errorf("pipeline '%s' %s: client_auth is enabled, which requires a client_ca_file", pipelineName, location)
}
return nil
}
// validateTLSClient validates the new TLSClientConfig struct.
func validateTLSClient(pipelineName, location string, tls *TLSClientConfig) error {
if !tls.Enabled {
return nil // Skip validation if disabled
}
// If verification is not skipped, a server CA file must be provided.
if !tls.InsecureSkipVerify && tls.ServerCAFile == "" {
return fmt.Errorf("pipeline '%s' %s: TLS verification is enabled (insecure_skip_verify=false) but server_ca_file is not provided", pipelineName, location)
}
// For client mTLS, both the cert and key must be provided together.
if (tls.ClientCertFile != "" && tls.ClientKeyFile == "") || (tls.ClientCertFile == "" && tls.ClientKeyFile != "") {
return fmt.Errorf("pipeline '%s' %s: for client mTLS, both client_cert_file and client_key_file must be provided", pipelineName, location)
}
return nil
}
// validateHeartbeat validates nested HeartbeatConfig settings.
// validateHeartbeat validates nested HeartbeatConfig settings
func validateHeartbeat(pipelineName, location string, hb *HeartbeatConfig) error {
if !hb.Enabled {
return nil // Skip validation if disabled