v0.3.3 pipeline rate limiter added

src/internal/config/pipeline.go

@@ -17,6 +17,9 @@ type PipelineConfig struct {
 	// Data sources for this pipeline
 	Sources []SourceConfig `toml:"sources"`
 
+	// Rate limiting
+	RateLimit *RateLimitConfig `toml:"rate_limit"`
+
 	// Filter configuration
 	Filters []FilterConfig `toml:"filters"`
 
@@ -37,7 +40,7 @@ type SourceConfig struct {
 
 	// Placeholder for future source-side rate limiting
 	// This will be used for features like aggregation and summarization
-	RateLimit *RateLimitConfig `toml:"rate_limit"`
+	NetLimit *NetLimitConfig `toml:"net_limit"`
 }
 
 // SinkConfig represents an output destination
@@ -187,9 +190,9 @@ func validateSink(pipelineName string, sinkIndex int, cfg *SinkConfig, allPorts
 			}
 		}
 
-		// Validate rate limit if present
-		if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
-			if err := validateRateLimitOptions("HTTP", pipelineName, sinkIndex, rl); err != nil {
+		// Validate net limit if present
+		if rl, ok := cfg.Options["net_limit"].(map[string]any); ok {
+			if err := validateNetLimitOptions("HTTP", pipelineName, sinkIndex, rl); err != nil {
 				return err
 			}
 		}
@@ -231,9 +234,9 @@ func validateSink(pipelineName string, sinkIndex int, cfg *SinkConfig, allPorts
 			}
 		}
 
-		// Validate rate limit if present
-		if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
-			if err := validateRateLimitOptions("TCP", pipelineName, sinkIndex, rl); err != nil {
+		// Validate net limit if present
+		if rl, ok := cfg.Options["net_limit"].(map[string]any); ok {
+			if err := validateNetLimitOptions("TCP", pipelineName, sinkIndex, rl); err != nil {
 				return err
 			}
 		}

src/internal/config/ratelimit.go

@@ -0,0 +1,52 @@
+// FILE: src/internal/config/ratelimit.go
+package config
+
+import (
+	"fmt"
+	"strings"
+)
+
+// RateLimitPolicy defines the action to take when a rate limit is exceeded.
+type RateLimitPolicy int
+
+const (
+	// PolicyPass allows all logs through, effectively disabling the limiter.
+	PolicyPass RateLimitPolicy = iota
+	// PolicyDrop drops logs that exceed the rate limit.
+	PolicyDrop
+)
+
+// RateLimitConfig defines the configuration for pipeline-level rate limiting.
+type RateLimitConfig struct {
+	// Rate is the number of log entries allowed per second. Default: 0 (disabled).
+	Rate float64 `toml:"rate"`
+	// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate.
+	Burst float64 `toml:"burst"`
+	// Policy defines the action to take when the limit is exceeded. "pass" or "drop".
+	Policy string `toml:"policy"`
+}
+
+func validateRateLimit(pipelineName string, cfg *RateLimitConfig) error {
+	if cfg == nil {
+		return nil
+	}
+
+	if cfg.Rate < 0 {
+		return fmt.Errorf("pipeline '%s': rate limit rate cannot be negative", pipelineName)
+	}
+
+	if cfg.Burst < 0 {
+		return fmt.Errorf("pipeline '%s': rate limit burst cannot be negative", pipelineName)
+	}
+
+	// Validate policy
+	switch strings.ToLower(cfg.Policy) {
+	case "", "pass", "drop":
+		// Valid policies
+	default:
+		return fmt.Errorf("pipeline '%s': invalid rate limit policy '%s' (must be 'pass' or 'drop')",
+			pipelineName, cfg.Policy)
+	}
+
+	return nil
+}

src/internal/config/server.go

@@ -11,8 +11,8 @@ type TCPConfig struct {
 	// SSL/TLS Configuration
 	SSL *SSLConfig `toml:"ssl"`
 
-	// Rate limiting
-	RateLimit *RateLimitConfig `toml:"rate_limit"`
+	// Net limiting
+	NetLimit *NetLimitConfig `toml:"net_limit"`
 
 	// Heartbeat
 	Heartbeat HeartbeatConfig `toml:"heartbeat"`
@@ -30,8 +30,8 @@ type HTTPConfig struct {
 	// SSL/TLS Configuration
 	SSL *SSLConfig `toml:"ssl"`
 
-	// Rate limiting
-	RateLimit *RateLimitConfig `toml:"rate_limit"`
+	// Nate limiting
+	NetLimit *NetLimitConfig `toml:"net_limit"`
 
 	// Heartbeat
 	Heartbeat HeartbeatConfig `toml:"heartbeat"`
@@ -45,8 +45,8 @@ type HeartbeatConfig struct {
 	Format           string `toml:"format"` // "comment" or "json"
 }
 
-type RateLimitConfig struct {
-	// Enable rate limiting
+type NetLimitConfig struct {
+	// Enable net limiting
 	Enabled bool `toml:"enabled"`
 
 	// Requests per second per client
@@ -55,12 +55,12 @@ type RateLimitConfig struct {
 	// Burst size (token bucket)
 	BurstSize int `toml:"burst_size"`
 
-	// Rate limit by: "ip", "user", "token", "global"
+	// Net limit by: "ip", "user", "token", "global"
 	LimitBy string `toml:"limit_by"`
 
-	// Response when rate limited
+	// Response when net limited
 	ResponseCode    int    `toml:"response_code"`    // Default: 429
-	ResponseMessage string `toml:"response_message"` // Default: "Rate limit exceeded"
+	ResponseMessage string `toml:"response_message"` // Default: "Net limit exceeded"
 
 	// Connection limits
 	MaxConnectionsPerIP int `toml:"max_connections_per_ip"`
@@ -85,7 +85,7 @@ func validateHeartbeatOptions(serverType, pipelineName string, sinkIndex int, hb
 	return nil
 }
 
-func validateRateLimitOptions(serverType, pipelineName string, sinkIndex int, rl map[string]any) error {
+func validateNetLimitOptions(serverType, pipelineName string, sinkIndex int, rl map[string]any) error {
 	if enabled, ok := rl["enabled"].(bool); !ok || !enabled {
 		return nil
 	}

src/internal/config/validation.go

@@ -48,6 +48,11 @@ func (c *Config) validate() error {
 			}
 		}
 
+		// Validate rate limit if present
+		if err := validateRateLimit(pipeline.Name, pipeline.RateLimit); err != nil {
+			return err
+		}
+
 		// Validate filters
 		for j, filterCfg := range pipeline.Filters {
 			if err := validateFilter(pipeline.Name, j, &filterCfg); err != nil {