v0.2.0 restructured to pipeline architecture, dirty

src/internal/config/auth.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/auth.go
 package config
 
+import "fmt"
+
 type AuthConfig struct {
 	// Authentication type: "none", "basic", "bearer", "mtls"
 	Type string `toml:"type"`
@@ -53,4 +55,25 @@ type JWTConfig struct {
 
 	// Expected audience
 	Audience string `toml:"audience"`
+}
+
+func validateAuth(pipelineName string, auth *AuthConfig) error {
+	if auth == nil {
+		return nil
+	}
+
+	validTypes := map[string]bool{"none": true, "basic": true, "bearer": true, "mtls": true}
+	if !validTypes[auth.Type] {
+		return fmt.Errorf("pipeline '%s': invalid auth type: %s", pipelineName, auth.Type)
+	}
+
+	if auth.Type == "basic" && auth.BasicAuth == nil {
+		return fmt.Errorf("pipeline '%s': basic auth type specified but config missing", pipelineName)
+	}
+
+	if auth.Type == "bearer" && auth.BearerAuth == nil {
+		return fmt.Errorf("pipeline '%s': bearer auth type specified but config missing", pipelineName)
+	}
+
+	return nil
 }

src/internal/config/config.go

@@ -5,10 +5,33 @@ type Config struct {
 	// Logging configuration
 	Logging *LogConfig `toml:"logging"`
 
-	// Stream configurations
-	Streams []StreamConfig `toml:"streams"`
+	// Pipeline configurations
+	Pipelines []PipelineConfig `toml:"pipelines"`
 }
 
-type MonitorConfig struct {
-	CheckIntervalMs int `toml:"check_interval_ms"`
+// Helper functions to handle type conversions from any
+func toInt(v any) (int, bool) {
+	switch val := v.(type) {
+	case int:
+		return val, true
+	case int64:
+		return int(val), true
+	case float64:
+		return int(val), true
+	default:
+		return 0, false
+	}
+}
+
+func toFloat(v any) (float64, bool) {
+	switch val := v.(type) {
+	case float64:
+		return val, true
+	case int:
+		return float64(val), true
+	case int64:
+		return float64(val), true
+	default:
+		return 0, false
+	}
 }

src/internal/config/filter.go

@@ -0,0 +1,44 @@
+// FILE: src/internal/config/filter.go
+package config
+
+import (
+	"fmt"
+	"regexp"
+
+	"logwisp/src/internal/filter"
+)
+
+func validateFilter(pipelineName string, filterIndex int, cfg *filter.Config) error {
+	// Validate filter type
+	switch cfg.Type {
+	case filter.TypeInclude, filter.TypeExclude, "":
+		// Valid types
+	default:
+		return fmt.Errorf("pipeline '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
+			pipelineName, filterIndex, cfg.Type)
+	}
+
+	// Validate filter logic
+	switch cfg.Logic {
+	case filter.LogicOr, filter.LogicAnd, "":
+		// Valid logic
+	default:
+		return fmt.Errorf("pipeline '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
+			pipelineName, filterIndex, cfg.Logic)
+	}
+
+	// Empty patterns is valid - passes everything
+	if len(cfg.Patterns) == 0 {
+		return nil
+	}
+
+	// Validate regex patterns
+	for i, pattern := range cfg.Patterns {
+		if _, err := regexp.Compile(pattern); err != nil {
+			return fmt.Errorf("pipeline '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
+				pipelineName, filterIndex, i, pattern, err)
+		}
+	}
+
+	return nil
+}

src/internal/config/loader.go

@@ -13,27 +13,35 @@ import (
 func defaults() *Config {
 	return &Config{
 		Logging: DefaultLogConfig(),
-		Streams: []StreamConfig{
+		Pipelines: []PipelineConfig{
 			{
 				Name: "default",
-				Monitor: &StreamMonitorConfig{
-					CheckIntervalMs: 100,
-					Targets: []MonitorTarget{
-						{Path: "./", Pattern: "*.log", IsFile: false},
+				Sources: []SourceConfig{
+					{
+						Type: "directory",
+						Options: map[string]any{
+							"path":              "./",
+							"pattern":           "*.log",
+							"check_interval_ms": 100,
+						},
 					},
 				},
-				HTTPServer: &HTTPConfig{
-					Enabled:    true,
-					Port:       8080,
-					BufferSize: 1000,
-					StreamPath: "/transport",
-					StatusPath: "/status",
-					Heartbeat: HeartbeatConfig{
-						Enabled:          true,
-						IntervalSeconds:  30,
-						IncludeTimestamp: true,
-						IncludeStats:     false,
-						Format:           "comment",
+				Sinks: []SinkConfig{
+					{
+						Type: "http",
+						Options: map[string]any{
+							"port":        8080,
+							"buffer_size": 1000,
+							"stream_path": "/transport",
+							"status_path": "/status",
+							"heartbeat": map[string]any{
+								"enabled":           true,
+								"interval_seconds":  30,
+								"include_timestamp": true,
+								"include_stats":     false,
+								"format":            "comment",
+							},
+						},
 					},
 				},
 			},

src/internal/config/logging.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/logging.go
 package config
 
+import "fmt"
+
 // LogConfig represents logging configuration for LogWisp
 type LogConfig struct {
 	// Output mode: "file", "stdout", "stderr", "both", "none"
@@ -59,4 +61,32 @@ func DefaultLogConfig() *LogConfig {
 			Format: "txt",
 		},
 	}
+}
+
+func validateLogConfig(cfg *LogConfig) error {
+	validOutputs := map[string]bool{
+		"file": true, "stdout": true, "stderr": true,
+		"both": true, "none": true,
+	}
+	if !validOutputs[cfg.Output] {
+		return fmt.Errorf("invalid log output mode: %s", cfg.Output)
+	}
+
+	validLevels := map[string]bool{
+		"debug": true, "info": true, "warn": true, "error": true,
+	}
+	if !validLevels[cfg.Level] {
+		return fmt.Errorf("invalid log level: %s", cfg.Level)
+	}
+
+	if cfg.Console != nil {
+		validTargets := map[string]bool{
+			"stdout": true, "stderr": true, "split": true,
+		}
+		if !validTargets[cfg.Console.Target] {
+			return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
+		}
+	}
+
+	return nil
 }

src/internal/config/pipeline.go

@@ -0,0 +1,276 @@
+// FILE: src/internal/config/pipeline.go
+package config
+
+import (
+	"fmt"
+	"logwisp/src/internal/filter"
+	"path/filepath"
+	"strings"
+)
+
+// PipelineConfig represents a data processing pipeline
+type PipelineConfig struct {
+	// Pipeline identifier (used in logs and metrics)
+	Name string `toml:"name"`
+
+	// Data sources for this pipeline
+	Sources []SourceConfig `toml:"sources"`
+
+	// Filter configuration
+	Filters []filter.Config `toml:"filters"`
+
+	// Output sinks for this pipeline
+	Sinks []SinkConfig `toml:"sinks"`
+
+	// Authentication/Authorization (applies to network sinks)
+	Auth *AuthConfig `toml:"auth"`
+}
+
+// SourceConfig represents an input data source
+type SourceConfig struct {
+	// Source type: "directory", "file", "stdin", etc.
+	Type string `toml:"type"`
+
+	// Type-specific configuration options
+	Options map[string]any `toml:"options"`
+
+	// Placeholder for future source-side rate limiting
+	// This will be used for features like aggregation and summarization
+	RateLimit *RateLimitConfig `toml:"rate_limit"`
+}
+
+// SinkConfig represents an output destination
+type SinkConfig struct {
+	// Sink type: "http", "tcp", "file", "stdout", "stderr"
+	Type string `toml:"type"`
+
+	// Type-specific configuration options
+	Options map[string]any `toml:"options"`
+}
+
+func validateSource(pipelineName string, sourceIndex int, cfg *SourceConfig) error {
+	if cfg.Type == "" {
+		return fmt.Errorf("pipeline '%s' source[%d]: missing type", pipelineName, sourceIndex)
+	}
+
+	switch cfg.Type {
+	case "directory":
+		// Validate directory source options
+		path, ok := cfg.Options["path"].(string)
+		if !ok || path == "" {
+			return fmt.Errorf("pipeline '%s' source[%d]: directory source requires 'path' option",
+				pipelineName, sourceIndex)
+		}
+
+		// Check for directory traversal
+		if strings.Contains(path, "..") {
+			return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal",
+				pipelineName, sourceIndex)
+		}
+
+		// Validate pattern if provided
+		if pattern, ok := cfg.Options["pattern"].(string); ok && pattern != "" {
+			// Try to compile as glob pattern (will be converted to regex internally)
+			if strings.Count(pattern, "*") == 0 && strings.Count(pattern, "?") == 0 {
+				// If no wildcards, ensure it's a valid filename
+				if filepath.Base(pattern) != pattern {
+					return fmt.Errorf("pipeline '%s' source[%d]: pattern contains path separators",
+						pipelineName, sourceIndex)
+				}
+			}
+		}
+
+		// Validate check interval if provided
+		if interval, ok := cfg.Options["check_interval_ms"]; ok {
+			if intVal, ok := toInt(interval); ok {
+				if intVal < 10 {
+					return fmt.Errorf("pipeline '%s' source[%d]: check interval too small: %d ms (min: 10ms)",
+						pipelineName, sourceIndex, intVal)
+				}
+			} else {
+				return fmt.Errorf("pipeline '%s' source[%d]: invalid check_interval_ms type",
+					pipelineName, sourceIndex)
+			}
+		}
+
+	case "file":
+		// Validate file source options
+		path, ok := cfg.Options["path"].(string)
+		if !ok || path == "" {
+			return fmt.Errorf("pipeline '%s' source[%d]: file source requires 'path' option",
+				pipelineName, sourceIndex)
+		}
+
+		// Check for directory traversal
+		if strings.Contains(path, "..") {
+			return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal",
+				pipelineName, sourceIndex)
+		}
+
+	case "stdin":
+		// No specific validation needed for stdin
+
+	default:
+		return fmt.Errorf("pipeline '%s' source[%d]: unknown source type '%s'",
+			pipelineName, sourceIndex, cfg.Type)
+	}
+
+	// Note: RateLimit field is ignored for now as it's a placeholder
+	return nil
+}
+
+func validateSink(pipelineName string, sinkIndex int, cfg *SinkConfig, allPorts map[int]string) error {
+	if cfg.Type == "" {
+		return fmt.Errorf("pipeline '%s' sink[%d]: missing type", pipelineName, sinkIndex)
+	}
+
+	switch cfg.Type {
+	case "http":
+		// Extract and validate HTTP configuration
+		port, ok := toInt(cfg.Options["port"])
+		if !ok || port < 1 || port > 65535 {
+			return fmt.Errorf("pipeline '%s' sink[%d]: invalid or missing HTTP port",
+				pipelineName, sinkIndex)
+		}
+
+		// Check port conflicts
+		if existing, exists := allPorts[port]; exists {
+			return fmt.Errorf("pipeline '%s' sink[%d]: HTTP port %d already used by %s",
+				pipelineName, sinkIndex, port, existing)
+		}
+		allPorts[port] = fmt.Sprintf("%s-http[%d]", pipelineName, sinkIndex)
+
+		// Validate buffer size
+		if bufSize, ok := toInt(cfg.Options["buffer_size"]); ok {
+			if bufSize < 1 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: HTTP buffer size must be positive: %d",
+					pipelineName, sinkIndex, bufSize)
+			}
+		}
+
+		// Validate paths if provided
+		if streamPath, ok := cfg.Options["stream_path"].(string); ok {
+			if !strings.HasPrefix(streamPath, "/") {
+				return fmt.Errorf("pipeline '%s' sink[%d]: stream path must start with /: %s",
+					pipelineName, sinkIndex, streamPath)
+			}
+		}
+
+		if statusPath, ok := cfg.Options["status_path"].(string); ok {
+			if !strings.HasPrefix(statusPath, "/") {
+				return fmt.Errorf("pipeline '%s' sink[%d]: status path must start with /: %s",
+					pipelineName, sinkIndex, statusPath)
+			}
+		}
+
+		// Validate heartbeat if present
+		if hb, ok := cfg.Options["heartbeat"].(map[string]any); ok {
+			if err := validateHeartbeatOptions("HTTP", pipelineName, sinkIndex, hb); err != nil {
+				return err
+			}
+		}
+
+		// Validate SSL if present
+		if ssl, ok := cfg.Options["ssl"].(map[string]any); ok {
+			if err := validateSSLOptions("HTTP", pipelineName, sinkIndex, ssl); err != nil {
+				return err
+			}
+		}
+
+		// Validate rate limit if present
+		if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
+			if err := validateRateLimitOptions("HTTP", pipelineName, sinkIndex, rl); err != nil {
+				return err
+			}
+		}
+
+	case "tcp":
+		// Extract and validate TCP configuration
+		port, ok := toInt(cfg.Options["port"])
+		if !ok || port < 1 || port > 65535 {
+			return fmt.Errorf("pipeline '%s' sink[%d]: invalid or missing TCP port",
+				pipelineName, sinkIndex)
+		}
+
+		// Check port conflicts
+		if existing, exists := allPorts[port]; exists {
+			return fmt.Errorf("pipeline '%s' sink[%d]: TCP port %d already used by %s",
+				pipelineName, sinkIndex, port, existing)
+		}
+		allPorts[port] = fmt.Sprintf("%s-tcp[%d]", pipelineName, sinkIndex)
+
+		// Validate buffer size
+		if bufSize, ok := toInt(cfg.Options["buffer_size"]); ok {
+			if bufSize < 1 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: TCP buffer size must be positive: %d",
+					pipelineName, sinkIndex, bufSize)
+			}
+		}
+
+		// Validate heartbeat if present
+		if hb, ok := cfg.Options["heartbeat"].(map[string]any); ok {
+			if err := validateHeartbeatOptions("TCP", pipelineName, sinkIndex, hb); err != nil {
+				return err
+			}
+		}
+
+		// Validate SSL if present
+		if ssl, ok := cfg.Options["ssl"].(map[string]any); ok {
+			if err := validateSSLOptions("TCP", pipelineName, sinkIndex, ssl); err != nil {
+				return err
+			}
+		}
+
+		// Validate rate limit if present
+		if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
+			if err := validateRateLimitOptions("TCP", pipelineName, sinkIndex, rl); err != nil {
+				return err
+			}
+		}
+
+	case "file":
+		// Validate file sink options
+		directory, ok := cfg.Options["directory"].(string)
+		if !ok || directory == "" {
+			return fmt.Errorf("pipeline '%s' sink[%d]: file sink requires 'directory' option",
+				pipelineName, sinkIndex)
+		}
+
+		name, ok := cfg.Options["name"].(string)
+		if !ok || name == "" {
+			return fmt.Errorf("pipeline '%s' sink[%d]: file sink requires 'name' option",
+				pipelineName, sinkIndex)
+		}
+
+		// Validate numeric options
+		if maxSize, ok := toInt(cfg.Options["max_size_mb"]); ok {
+			if maxSize < 1 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: max_size_mb must be positive: %d",
+					pipelineName, sinkIndex, maxSize)
+			}
+		}
+
+		if maxTotalSize, ok := toInt(cfg.Options["max_total_size_mb"]); ok {
+			if maxTotalSize < 0 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: max_total_size_mb cannot be negative: %d",
+					pipelineName, sinkIndex, maxTotalSize)
+			}
+		}
+
+		if retention, ok := toFloat(cfg.Options["retention_hours"]); ok {
+			if retention < 0 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: retention_hours cannot be negative: %f",
+					pipelineName, sinkIndex, retention)
+			}
+		}
+
+	case "stdout", "stderr":
+		// No specific validation needed for console sinks
+
+	default:
+		return fmt.Errorf("pipeline '%s' sink[%d]: unknown sink type '%s'",
+			pipelineName, sinkIndex, cfg.Type)
+	}
+
+	return nil
+}

src/internal/config/server.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/server.go
 package config
 
+import "fmt"
+
 type TCPConfig struct {
 	Enabled    bool `toml:"enabled"`
 	Port       int  `toml:"port"`
@@ -63,4 +65,72 @@ type RateLimitConfig struct {
 	// Connection limits
 	MaxConnectionsPerIP int `toml:"max_connections_per_ip"`
 	MaxTotalConnections int `toml:"max_total_connections"`
+}
+
+func validateHeartbeatOptions(serverType, pipelineName string, sinkIndex int, hb map[string]any) error {
+	if enabled, ok := hb["enabled"].(bool); ok && enabled {
+		interval, ok := toInt(hb["interval_seconds"])
+		if !ok || interval < 1 {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: heartbeat interval must be positive",
+				pipelineName, sinkIndex, serverType)
+		}
+
+		if format, ok := hb["format"].(string); ok {
+			if format != "json" && format != "comment" {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: heartbeat format must be 'json' or 'comment': %s",
+					pipelineName, sinkIndex, serverType, format)
+			}
+		}
+	}
+	return nil
+}
+
+func validateRateLimitOptions(serverType, pipelineName string, sinkIndex int, rl map[string]any) error {
+	if enabled, ok := rl["enabled"].(bool); !ok || !enabled {
+		return nil
+	}
+
+	// Validate requests per second
+	rps, ok := toFloat(rl["requests_per_second"])
+	if !ok || rps <= 0 {
+		return fmt.Errorf("pipeline '%s' sink[%d] %s: requests_per_second must be positive",
+			pipelineName, sinkIndex, serverType)
+	}
+
+	// Validate burst size
+	burst, ok := toInt(rl["burst_size"])
+	if !ok || burst < 1 {
+		return fmt.Errorf("pipeline '%s' sink[%d] %s: burst_size must be at least 1",
+			pipelineName, sinkIndex, serverType)
+	}
+
+	// Validate limit_by
+	if limitBy, ok := rl["limit_by"].(string); ok && limitBy != "" {
+		validLimitBy := map[string]bool{"ip": true, "global": true}
+		if !validLimitBy[limitBy] {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid limit_by value: %s (must be 'ip' or 'global')",
+				pipelineName, sinkIndex, serverType, limitBy)
+		}
+	}
+
+	// Validate response code
+	if respCode, ok := toInt(rl["response_code"]); ok {
+		if respCode > 0 && (respCode < 400 || respCode >= 600) {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: response_code must be 4xx or 5xx: %d",
+				pipelineName, sinkIndex, serverType, respCode)
+		}
+	}
+
+	// Validate connection limits
+	maxPerIP, perIPOk := toInt(rl["max_connections_per_ip"])
+	maxTotal, totalOk := toInt(rl["max_total_connections"])
+
+	if perIPOk && totalOk && maxPerIP > 0 && maxTotal > 0 {
+		if maxPerIP > maxTotal {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: max_connections_per_ip (%d) cannot exceed max_total_connections (%d)",
+				pipelineName, sinkIndex, serverType, maxPerIP, maxTotal)
+		}
+	}
+
+	return nil
 }

src/internal/config/ssl.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/ssl.go
 package config
 
+import "fmt"
+
 type SSLConfig struct {
 	Enabled  bool   `toml:"enabled"`
 	CertFile string `toml:"cert_file"`
@@ -17,4 +19,39 @@ type SSLConfig struct {
 
 	// Cipher suites (comma-separated list)
 	CipherSuites string `toml:"cipher_suites"`
+}
+
+func validateSSLOptions(serverType, pipelineName string, sinkIndex int, ssl map[string]any) error {
+	if enabled, ok := ssl["enabled"].(bool); ok && enabled {
+		certFile, certOk := ssl["cert_file"].(string)
+		keyFile, keyOk := ssl["key_file"].(string)
+
+		if !certOk || certFile == "" || !keyOk || keyFile == "" {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: SSL enabled but cert/key files not specified",
+				pipelineName, sinkIndex, serverType)
+		}
+
+		if clientAuth, ok := ssl["client_auth"].(bool); ok && clientAuth {
+			if caFile, ok := ssl["client_ca_file"].(string); !ok || caFile == "" {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: client auth enabled but CA file not specified",
+					pipelineName, sinkIndex, serverType)
+			}
+		}
+
+		// Validate TLS versions
+		validVersions := map[string]bool{"TLS1.0": true, "TLS1.1": true, "TLS1.2": true, "TLS1.3": true}
+		if minVer, ok := ssl["min_version"].(string); ok && minVer != "" {
+			if !validVersions[minVer] {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid min TLS version: %s",
+					pipelineName, sinkIndex, serverType, minVer)
+			}
+		}
+		if maxVer, ok := ssl["max_version"].(string); ok && maxVer != "" {
+			if !validVersions[maxVer] {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid max TLS version: %s",
+					pipelineName, sinkIndex, serverType, maxVer)
+			}
+		}
+	}
+	return nil
 }

src/internal/config/stream.go

@@ -1,49 +0,0 @@
-// FILE: src/internal/config/transport.go
-package config
-
-import (
-	"logwisp/src/internal/filter"
-)
-
-type StreamConfig struct {
-	// Stream identifier (used in logs and metrics)
-	Name string `toml:"name"`
-
-	// Monitor configuration for this transport
-	Monitor *StreamMonitorConfig `toml:"monitor"`
-
-	// Filter configuration
-	Filters []filter.Config `toml:"filters"`
-
-	// Server configurations
-	TCPServer  *TCPConfig  `toml:"tcpserver"`
-	HTTPServer *HTTPConfig `toml:"httpserver"`
-
-	// Authentication/Authorization
-	Auth *AuthConfig `toml:"auth"`
-}
-
-type StreamMonitorConfig struct {
-	CheckIntervalMs int             `toml:"check_interval_ms"`
-	Targets         []MonitorTarget `toml:"targets"`
-}
-
-type MonitorTarget struct {
-	Path    string `toml:"path"`
-	Pattern string `toml:"pattern"`
-	IsFile  bool   `toml:"is_file"`
-}
-
-func (s *StreamConfig) GetTargets(defaultTargets []MonitorTarget) []MonitorTarget {
-	if s.Monitor != nil && len(s.Monitor.Targets) > 0 {
-		return s.Monitor.Targets
-	}
-	return nil
-}
-
-func (s *StreamConfig) GetCheckInterval(defaultInterval int) int {
-	if s.Monitor != nil && s.Monitor.CheckIntervalMs > 0 {
-		return s.Monitor.CheckIntervalMs
-	}
-	return defaultInterval
-}

src/internal/config/validation.go

@@ -3,309 +3,67 @@ package config
 
 import (
 	"fmt"
-	"regexp"
-	"strings"
-
-	"logwisp/src/internal/filter"
 )
 
 func (c *Config) validate() error {
-	if len(c.Streams) == 0 {
-		return fmt.Errorf("no streams configured")
+	if len(c.Pipelines) == 0 {
+		return fmt.Errorf("no pipelines configured")
 	}
 
 	if err := validateLogConfig(c.Logging); err != nil {
 		return fmt.Errorf("logging config: %w", err)
 	}
 
-	// Validate each transport
-	streamNames := make(map[string]bool)
-	streamPorts := make(map[int]string)
+	// Track used ports across all pipelines
+	allPorts := make(map[int]string)
+	pipelineNames := make(map[string]bool)
 
-	for i, stream := range c.Streams {
-		if stream.Name == "" {
-			return fmt.Errorf("transport %d: missing name", i)
+	for i, pipeline := range c.Pipelines {
+		if pipeline.Name == "" {
+			return fmt.Errorf("pipeline %d: missing name", i)
 		}
 
-		if streamNames[stream.Name] {
-			return fmt.Errorf("transport %d: duplicate name '%s'", i, stream.Name)
+		if pipelineNames[pipeline.Name] {
+			return fmt.Errorf("pipeline %d: duplicate name '%s'", i, pipeline.Name)
 		}
-		streamNames[stream.Name] = true
+		pipelineNames[pipeline.Name] = true
 
-		// Stream must have monitor config with targets
-		if stream.Monitor == nil || len(stream.Monitor.Targets) == 0 {
-			return fmt.Errorf("transport '%s': no monitor targets specified", stream.Name)
+		// Pipeline must have at least one source
+		if len(pipeline.Sources) == 0 {
+			return fmt.Errorf("pipeline '%s': no sources specified", pipeline.Name)
 		}
 
-		// Validate check interval
-		if stream.Monitor.CheckIntervalMs < 10 {
-			return fmt.Errorf("transport '%s': check interval too small: %d ms (min: 10ms)",
-				stream.Name, stream.Monitor.CheckIntervalMs)
-		}
-
-		// Validate targets
-		for j, target := range stream.Monitor.Targets {
-			if target.Path == "" {
-				return fmt.Errorf("transport '%s' target %d: empty path", stream.Name, j)
-			}
-			if strings.Contains(target.Path, "..") {
-				return fmt.Errorf("transport '%s' target %d: path contains directory traversal", stream.Name, j)
+		// Validate sources
+		for j, source := range pipeline.Sources {
+			if err := validateSource(pipeline.Name, j, &source); err != nil {
+				return err
 			}
 		}
 
 		// Validate filters
-		for j, filterCfg := range stream.Filters {
-			if err := validateFilter(stream.Name, j, &filterCfg); err != nil {
+		for j, filterCfg := range pipeline.Filters {
+			if err := validateFilter(pipeline.Name, j, &filterCfg); err != nil {
 				return err
 			}
 		}
 
-		// Validate TCP server
-		if stream.TCPServer != nil && stream.TCPServer.Enabled {
-			if stream.TCPServer.Port < 1 || stream.TCPServer.Port > 65535 {
-				return fmt.Errorf("transport '%s': invalid TCP port: %d", stream.Name, stream.TCPServer.Port)
-			}
-			if existing, exists := streamPorts[stream.TCPServer.Port]; exists {
-				return fmt.Errorf("transport '%s': TCP port %d already used by transport '%s'",
-					stream.Name, stream.TCPServer.Port, existing)
-			}
-			streamPorts[stream.TCPServer.Port] = stream.Name + "-tcp"
-
-			if stream.TCPServer.BufferSize < 1 {
-				return fmt.Errorf("transport '%s': TCP buffer size must be positive: %d",
-					stream.Name, stream.TCPServer.BufferSize)
-			}
-
-			if err := validateHeartbeat("TCP", stream.Name, &stream.TCPServer.Heartbeat); err != nil {
-				return err
-			}
-
-			if err := validateSSL("TCP", stream.Name, stream.TCPServer.SSL); err != nil {
-				return err
-			}
-
-			if err := validateRateLimit("TCP", stream.Name, stream.TCPServer.RateLimit); err != nil {
-				return err
-			}
+		// Pipeline must have at least one sink
+		if len(pipeline.Sinks) == 0 {
+			return fmt.Errorf("pipeline '%s': no sinks specified", pipeline.Name)
 		}
 
-		// Validate HTTP server
-		if stream.HTTPServer != nil && stream.HTTPServer.Enabled {
-			if stream.HTTPServer.Port < 1 || stream.HTTPServer.Port > 65535 {
-				return fmt.Errorf("transport '%s': invalid HTTP port: %d", stream.Name, stream.HTTPServer.Port)
-			}
-			if existing, exists := streamPorts[stream.HTTPServer.Port]; exists {
-				return fmt.Errorf("transport '%s': HTTP port %d already used by transport '%s'",
-					stream.Name, stream.HTTPServer.Port, existing)
-			}
-			streamPorts[stream.HTTPServer.Port] = stream.Name + "-http"
-
-			if stream.HTTPServer.BufferSize < 1 {
-				return fmt.Errorf("transport '%s': HTTP buffer size must be positive: %d",
-					stream.Name, stream.HTTPServer.BufferSize)
-			}
-
-			// Validate paths
-			if stream.HTTPServer.StreamPath == "" {
-				stream.HTTPServer.StreamPath = "/transport"
-			}
-			if stream.HTTPServer.StatusPath == "" {
-				stream.HTTPServer.StatusPath = "/status"
-			}
-			if !strings.HasPrefix(stream.HTTPServer.StreamPath, "/") {
-				return fmt.Errorf("transport '%s': transport path must start with /: %s",
-					stream.Name, stream.HTTPServer.StreamPath)
-			}
-			if !strings.HasPrefix(stream.HTTPServer.StatusPath, "/") {
-				return fmt.Errorf("transport '%s': status path must start with /: %s",
-					stream.Name, stream.HTTPServer.StatusPath)
-			}
-
-			if err := validateHeartbeat("HTTP", stream.Name, &stream.HTTPServer.Heartbeat); err != nil {
+		// Validate sinks and check for port conflicts
+		for j, sink := range pipeline.Sinks {
+			if err := validateSink(pipeline.Name, j, &sink, allPorts); err != nil {
 				return err
 			}
-
-			if err := validateSSL("HTTP", stream.Name, stream.HTTPServer.SSL); err != nil {
-				return err
-			}
-
-			if err := validateRateLimit("HTTP", stream.Name, stream.HTTPServer.RateLimit); err != nil {
-				return err
-			}
-		}
-
-		// At least one server must be enabled
-		tcpEnabled := stream.TCPServer != nil && stream.TCPServer.Enabled
-		httpEnabled := stream.HTTPServer != nil && stream.HTTPServer.Enabled
-		if !tcpEnabled && !httpEnabled {
-			return fmt.Errorf("transport '%s': no servers enabled", stream.Name)
 		}
 
 		// Validate auth if present
-		if err := validateAuth(stream.Name, stream.Auth); err != nil {
+		if err := validateAuth(pipeline.Name, pipeline.Auth); err != nil {
 			return err
 		}
 	}
 
-	return nil
-}
-
-func validateHeartbeat(serverType, streamName string, hb *HeartbeatConfig) error {
-	if hb.Enabled {
-		if hb.IntervalSeconds < 1 {
-			return fmt.Errorf("transport '%s' %s: heartbeat interval must be positive: %d",
-				streamName, serverType, hb.IntervalSeconds)
-		}
-		if hb.Format != "json" && hb.Format != "comment" {
-			return fmt.Errorf("transport '%s' %s: heartbeat format must be 'json' or 'comment': %s",
-				streamName, serverType, hb.Format)
-		}
-	}
-	return nil
-}
-
-func validateSSL(serverType, streamName string, ssl *SSLConfig) error {
-	if ssl != nil && ssl.Enabled {
-		if ssl.CertFile == "" || ssl.KeyFile == "" {
-			return fmt.Errorf("transport '%s' %s: SSL enabled but cert/key files not specified",
-				streamName, serverType)
-		}
-
-		if ssl.ClientAuth && ssl.ClientCAFile == "" {
-			return fmt.Errorf("transport '%s' %s: client auth enabled but CA file not specified",
-				streamName, serverType)
-		}
-
-		// Validate TLS versions
-		validVersions := map[string]bool{"TLS1.0": true, "TLS1.1": true, "TLS1.2": true, "TLS1.3": true}
-		if ssl.MinVersion != "" && !validVersions[ssl.MinVersion] {
-			return fmt.Errorf("transport '%s' %s: invalid min TLS version: %s",
-				streamName, serverType, ssl.MinVersion)
-		}
-		if ssl.MaxVersion != "" && !validVersions[ssl.MaxVersion] {
-			return fmt.Errorf("transport '%s' %s: invalid max TLS version: %s",
-				streamName, serverType, ssl.MaxVersion)
-		}
-	}
-	return nil
-}
-
-func validateAuth(streamName string, auth *AuthConfig) error {
-	if auth == nil {
-		return nil
-	}
-
-	validTypes := map[string]bool{"none": true, "basic": true, "bearer": true, "mtls": true}
-	if !validTypes[auth.Type] {
-		return fmt.Errorf("transport '%s': invalid auth type: %s", streamName, auth.Type)
-	}
-
-	if auth.Type == "basic" && auth.BasicAuth == nil {
-		return fmt.Errorf("transport '%s': basic auth type specified but config missing", streamName)
-	}
-
-	if auth.Type == "bearer" && auth.BearerAuth == nil {
-		return fmt.Errorf("transport '%s': bearer auth type specified but config missing", streamName)
-	}
-
-	return nil
-}
-
-func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error {
-	if rl == nil || !rl.Enabled {
-		return nil
-	}
-
-	if rl.RequestsPerSecond <= 0 {
-		return fmt.Errorf("transport '%s' %s: requests_per_second must be positive: %f",
-			streamName, serverType, rl.RequestsPerSecond)
-	}
-
-	if rl.BurstSize < 1 {
-		return fmt.Errorf("transport '%s' %s: burst_size must be at least 1: %d",
-			streamName, serverType, rl.BurstSize)
-	}
-
-	validLimitBy := map[string]bool{"ip": true, "global": true, "": true}
-	if !validLimitBy[rl.LimitBy] {
-		return fmt.Errorf("transport '%s' %s: invalid limit_by value: %s (must be 'ip' or 'global')",
-			streamName, serverType, rl.LimitBy)
-	}
-
-	if rl.ResponseCode > 0 && (rl.ResponseCode < 400 || rl.ResponseCode >= 600) {
-		return fmt.Errorf("transport '%s' %s: response_code must be 4xx or 5xx: %d",
-			streamName, serverType, rl.ResponseCode)
-	}
-
-	if rl.MaxConnectionsPerIP > 0 && rl.MaxTotalConnections > 0 {
-		if rl.MaxConnectionsPerIP > rl.MaxTotalConnections {
-			return fmt.Errorf("stream '%s' %s: max_connections_per_ip (%d) cannot exceed max_total_connections (%d)",
-				streamName, serverType, rl.MaxConnectionsPerIP, rl.MaxTotalConnections)
-		}
-	}
-
-	return nil
-}
-
-func validateFilter(streamName string, filterIndex int, cfg *filter.Config) error {
-	// Validate filter type
-	switch cfg.Type {
-	case filter.TypeInclude, filter.TypeExclude, "":
-		// Valid types
-	default:
-		return fmt.Errorf("transport '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
-			streamName, filterIndex, cfg.Type)
-	}
-
-	// Validate filter logic
-	switch cfg.Logic {
-	case filter.LogicOr, filter.LogicAnd, "":
-		// Valid logic
-	default:
-		return fmt.Errorf("transport '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
-			streamName, filterIndex, cfg.Logic)
-	}
-
-	// Empty patterns is valid - passes everything
-	if len(cfg.Patterns) == 0 {
-		return nil
-	}
-
-	// Validate regex patterns
-	for i, pattern := range cfg.Patterns {
-		if _, err := regexp.Compile(pattern); err != nil {
-			return fmt.Errorf("transport '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
-				streamName, filterIndex, i, pattern, err)
-		}
-	}
-
-	return nil
-}
-
-func validateLogConfig(cfg *LogConfig) error {
-	validOutputs := map[string]bool{
-		"file": true, "stdout": true, "stderr": true,
-		"both": true, "none": true,
-	}
-	if !validOutputs[cfg.Output] {
-		return fmt.Errorf("invalid log output mode: %s", cfg.Output)
-	}
-
-	validLevels := map[string]bool{
-		"debug": true, "info": true, "warn": true, "error": true,
-	}
-	if !validLevels[cfg.Level] {
-		return fmt.Errorf("invalid log level: %s", cfg.Level)
-	}
-
-	if cfg.Console != nil {
-		validTargets := map[string]bool{
-			"stdout": true, "stderr": true, "split": true,
-		}
-		if !validTargets[cfg.Console.Target] {
-			return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
-		}
-	}
-
 	return nil
 }