v0.18.0 raw and json formatter improvements, dockerfile, go bump to 1.27.1

.gitignore

@@ -1,11 +1,17 @@
 .idea
-data
+data/
-dev
+dev/
-log
+log/
-logs
+logs/
-cert
+cert/
-bin
+bin/
-script
+script/
-build
+build/
 *.log
 *.toml
+!config/*.toml
+build.sh
+catalog.txt
+combined.txt
+test/run/
+test/run-mtls/

Dockerfile

@@ -0,0 +1,40 @@
+# Builder pin and go.mod directive are the same patch release deliberately;
+# an older builder reports the mismatch only after downloading the module graph.
+ARG GO_VERSION=1.27.1
+
+FROM docker.io/library/golang:${GO_VERSION}-alpine AS build
+
+# Git supplies Go's VCS build information; only /out/logwisp crosses stages.
+RUN apk add --no-cache git
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
+ARG VERSION=dev
+ARG REVISION=unknown
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
+    go build -trimpath \
+      -ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION}" \
+      -o /out/logwisp ./cmd/logwisp
+
+FROM scratch
+
+ARG VERSION=dev
+ARG REVISION=unknown
+
+LABEL org.opencontainers.image.title="logwisp" \
+      org.opencontainers.image.description="Log transport: sources, flow, sinks" \
+      org.opencontainers.image.source="https://github.com/lixenwraith/logwisp" \
+      org.opencontainers.image.revision="${REVISION}" \
+      org.opencontainers.image.version="${VERSION}" \
+      org.opencontainers.image.licenses="BSD-3-Clause"
+
+COPY --from=build /out/logwisp /logwisp
+
+# Numeric identity is required in scratch and satisfies a restricted pod spec.
+USER 65532:65532
+
+ENTRYPOINT ["/logwisp"]

Makefile

@@ -11,9 +11,9 @@ BUILD_TIME != date -u '+%Y-%m-%d_%H:%M:%S'
 # Go build variables
 GO = go
 GOFLAGS =
-LDFLAGS = -X 'logwisp/src/internal/version.Version=$(VERSION)' \
+LDFLAGS = -X 'logwisp/internal/version.Version=$(VERSION)' \
-          -X 'logwisp/src/internal/version.GitCommit=$(GIT_COMMIT)' \
+          -X 'logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \
-          -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'
+          -X 'logwisp/internal/version.BuildTime=$(BUILD_TIME)'
 
 # Installation directories
 PREFIX ?= /usr/local
@@ -25,7 +25,7 @@ all: build
 # Build the binary
 build:
 	mkdir -p $(BUILD_DIR)
-	$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./src/cmd/logwisp
+	$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/logwisp
 
 # Install the binary
 install: build
@@ -41,7 +41,7 @@ clean:
 
 # Development build with race detector
 dev:
-	$(GO) build $(GOFLAGS) -race -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./src/cmd/logwisp
+	$(GO) build $(GOFLAGS) -race -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/logwisp
 
 # Show current version
 version:

README.md

@@ -6,7 +6,7 @@
     <td>
       <h1>LogWisp</h1>
       <p>
-        <a href="https://golang.org"><img src="https://img.shields.io/badge/Go-1.25-00ADD8?style=flat&logo=go" alt="Go"></a>
+        <a href="https://golang.org"><img src="https://img.shields.io/badge/Go-1.26-00ADD8?style=flat&logo=go" alt="Go"></a>
         <a href="https://opensource.org/licenses/BSD-3-Clause"><img src="https://img.shields.io/badge/License-BSD_3--Clause-blue.svg" alt="License"></a>
         <a href="doc/"><img src="https://img.shields.io/badge/Docs-Available-green.svg" alt="Documentation"></a>
       </p>
@@ -16,78 +16,130 @@
 
 # LogWisp
 
-A high-performance, pipeline-based log transport and processing system built in Go. LogWisp provides flexible log collection, filtering, formatting, and distribution with enterprise-grade security and reliability features.
+A pipeline-based log transport and processing system written in Go. LogWisp
+collects log entries from files, stdin, or other LogWisp nodes; rate-limits,
+filters, and formats them; and distributes them to files, consoles, live network
+streams, or downstream LogWisp nodes.
 
 ## Features
 
-### Core Capabilities
+### Pipeline
-- **Pipeline Architecture**: Independent processing pipelines with source(s) → filter → format → sink(s) flow
-- **Multiple Input Sources**: Directory monitoring, stdin, HTTP, TCP
-- **Flexible Output Sinks**: Console, file, HTTP SSE, TCP streaming, HTTP/TCP forwarding
-- **Real-time Processing**: Sub-millisecond latency with configurable buffering
-- **Hot Configuration Reload**: Update pipelines without service restart
 
-### Data Processing
+- **Independent pipelines**, each `sources → flow → sinks`, running concurrently
-- **Pattern-based Filtering**: Chainable include/exclude filters with regex support
+  in one process
-- **Multiple Formatters**: Raw, JSON, and template-based text formatting
+- **Fan-in and fan-out**: many sources and many sinks per pipeline
-- **Rate Limiting**: Pipeline rate control
+- **Never blocks**: a stalled sink drops its own events and is counted, rather
+  than stalling the pipeline or its sibling sinks
+- **Hot reload** via `SIGHUP`/`SIGUSR1` or a config file watch, with the new
+  configuration validated before the old service is torn down
 
-### Security & Reliability
+### Inputs
-- **Authentication**: Basic, token, and mTLS support for HTTPS, and SCRAM for TCP
-- **TLS Encryption**: TLS 1.2/1.3 support for HTTP connections
-- **Access Control**: IP whitelisting/blacklisting, connection limits
-- **Automatic Reconnection**: Resilient client connections with exponential backoff
-- **File Rotation**: Size-based rotation with retention policies
 
-### Operational Features
+`file` (directory tail with rotation detection and JSON line parsing),
-- **Status Monitoring**: Real-time statistics and health endpoints
+`console` (stdin), `random` (synthetic generator), `null`, and the chain ingest
-- **Signal Handling**: Graceful shutdown and configuration reload via signals
+listeners `tcp_chain` and `http_chain`.
-- **Background Mode**: Daemon operation with proper signal handling
+
-- **Quiet Mode**: Silent operation for automated deployments
+### Outputs
+
+`console`, `file` (rotating with retention), `http` (Server-Sent Events plus a
+JSON status endpoint), `tcp` (broadcast server), `null`, and the chain
+forwarders `tcp_chain` and `http_chain`.
+
+### Processing
+
+- **Filters**: chainable include/exclude RE2 patterns with `or`/`and` logic
+- **Formatters**: `raw`, `txt`, and `json` with selectable sanitizer policies
+- **Rate limiting**: token bucket with an optional per-entry size cap
+- **Heartbeats**: flow-level keep-alive entries that reach every sink
+
+### Chaining
+
+Multi-node topologies over a versioned protocol. Chain links carry the
+**structured entry**, not the formatted text, so a relay can filter and reformat
+as if the entries were local. Entries keep a `node` label identifying their
+origin across any number of hops. Chain sinks reconnect automatically with
+exponential backoff and jitter.
+
+### Transport security and authentication
+
+- TLS 1.2/1.3 on every network source and sink, listener and dialer alike
+- Mutual TLS: listeners can require and verify client certificates; dialers can
+  present a client identity
+- Authorization by certificate identity: an `auth` block admits named peers
+  (exact or RE2) rather than everything the CA issued, gates the `http` sink's
+  stream and status endpoints, and lets a dialer pin the server it talks to
+- Node binding: a chain source can label entries from the sender's certificate
+  instead of from what the sender claims, so origin attribution is not forgeable
+
+See [Security](doc/security.md) for configuration and the exact boundary, and
+the [mTLS authentication design](doc/mtls-auth-plan.md) for the rationale and
+what is deliberately left out. Password, token, and SCRAM authentication were
+removed during the restructure and are not currently available.
 
 ## Documentation
 
-Available in `doc/` directory.
+| Document | Contents |
+|----------|----------|
+| [Installation](doc/installation.md) | Building, installing, running as a service |
+| [Architecture](doc/architecture.md) | Component model, data flow, concurrency, back-pressure |
+| [Configuration](doc/configuration.md) | TOML structure, precedence, environment and CLI overrides |
+| [Sources](doc/sources.md) | Every input plugin and its options |
+| [Sinks](doc/sinks.md) | Every output plugin and its options |
+| [Filters](doc/filters.md) | Pattern-based inclusion and exclusion |
+| [Formatters](doc/formatters.md) | Output shaping and sanitization |
+| [Chaining](doc/chaining.md) | Multi-node topologies and the chain wire protocol |
+| [Networking](doc/networking.md) | Listeners, dialers, timeouts, connection limits |
+| [Security](doc/security.md) | TLS, mTLS, and peer authorization; threat model and current limits |
+| [mTLS Authentication](doc/mtls-auth-plan.md) | Design and rationale for certificate-based authorization |
+| [CLI](doc/cli.md) | Flags, signals, exit codes |
+| [Operations](doc/operations.md) | Running, monitoring, tuning, troubleshooting |
 
-- [Installation Guide](doc/installation.md) - Platform setup and service configuration
+A fully annotated configuration covering every option ships as
-- [Architecture Overview](doc/architecture.md) - System design and component interaction
+[`config/logwisp.toml`](config/logwisp.toml).
-- [Configuration Reference](doc/configuration.md) - TOML structure and configuration methods
-- [Input Sources](doc/sources.md) - Available source types and configurations
-- [Output Sinks](doc/sinks.md) - Sink types and output options
-- [Filters](doc/filters.md) - Pattern-based log filtering
-- [Formatters](doc/formatters.md) - Log formatting and transformation
-- [Authentication](doc/authentication.md) - Security configurations and auth methods
-- [Networking](doc/networking.md) - TLS, rate limiting, and network features
-- [Command Line Interface](doc/cli.md) - CLI flags and subcommands
-- [Operations Guide](doc/operations.md) - Running and maintaining LogWisp
 
 ## Quick Start
 
-Install LogWisp and create a basic configuration:
+```bash
+make
+```
 
 ```toml
+# logwisp.toml
 [[pipelines]]
 name = "default"
 
-[[pipelines.sources]]
+[pipelines.flow.format]
-type = "directory"
+type = "json"
-[pipelines.sources.directory]
+sanitizer_policy = "json"
-path = "./"
+
+[[pipelines.plugin_sources]]
+id = "app_logs"
+type = "file"
+[pipelines.plugin_sources.config]
+directory = "/var/log/myapp"
 pattern = "*.log"
 
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
+id = "stdout"
 type = "console"
-[pipelines.sinks.console]
+[pipelines.plugin_sinks.config]
 target = "stdout"
 ```
 
-Run with: `logwisp -c config.toml`
+```bash
+logwisp -c logwisp.toml
+```
+
+Running with no configuration file starts a self-demonstrating pipeline: a
+synthetic generator writing JSON to stdout.
 
 ## System Requirements
 
-- **Operating Systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
+- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
 - **Architecture**: amd64
-- **Go Version**: 1.25+ (for building from source)
+- **Go**: 1.26+ to build from source
+
+Network sources and sinks bind and dial over IPv4 only.
 
 ## License
 

cmd/logwisp/bootstrap.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	_ "logwisp/internal/source/console"
+	_ "logwisp/internal/source/file"
+	_ "logwisp/internal/source/httpchain"
+	_ "logwisp/internal/source/null"
+	_ "logwisp/internal/source/random"
+	_ "logwisp/internal/source/tcpchain"
+
+	_ "logwisp/internal/sink/console"
+	_ "logwisp/internal/sink/file"
+	_ "logwisp/internal/sink/http"
+	_ "logwisp/internal/sink/httpchain"
+	_ "logwisp/internal/sink/null"
+	_ "logwisp/internal/sink/tcp"
+	_ "logwisp/internal/sink/tcpchain"
+
+	"logwisp/internal/config"
+	"logwisp/internal/service"
+	"logwisp/internal/version"
+
+	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/log/sanitizer"
+)
+
+// bootstrapInitial handles initial service startup with status reporter
+func bootstrapInitial(ctx context.Context, cfg *config.Config) (*service.Service, context.CancelFunc, error) {
+	svc, err := bootstrapService(ctx, cfg)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to bootstrap service: %w", err)
+	}
+
+	if err := svc.Start(); err != nil {
+		return nil, nil, fmt.Errorf("failed to start service pipelines: %w", err)
+	}
+
+	var statusCancel context.CancelFunc
+	if cfg.StatusReporter {
+		statusCancel = startStatusReporter(ctx, svc)
+	}
+
+	return svc, statusCancel, nil
+}
+
+// handleReload orchestrates the entire hot-reload process including status reporter lifecycle
+func handleReload(ctx context.Context, oldSvc *service.Service, statusCancel context.CancelFunc) (*service.Service, *config.Config, context.CancelFunc, error) {
+	logger.Info("msg", "Starting configuration hot reload")
+
+	// Get updated config from the lixenwraith/config manager
+	lcfg := config.GetConfigManager()
+	if lcfg == nil {
+		err := fmt.Errorf("config manager not available for reload")
+		logger.Error("msg", "Reload failed", "error", err)
+		return nil, nil, nil, err
+	}
+
+	updatedCfgStruct, err := lcfg.AsStruct()
+	if err != nil {
+		logger.Error("msg", "Failed to get updated config for reload", "error", err, "action", "keeping current configuration")
+		return nil, nil, nil, err
+	}
+	newCfg := updatedCfgStruct.(*config.Config)
+
+	// Bootstrap a new service to ensure it's valid before touching the old one
+	logger.Debug("msg", "Bootstrapping new service with updated config")
+	newService, err := bootstrapService(ctx, newCfg)
+	if err != nil {
+		logger.Error("msg", "Failed to bootstrap new service, keeping old service running", "error", err)
+		return nil, nil, nil, err
+	}
+
+	// Gracefully shut down the old service
+	if oldSvc != nil {
+		logger.Info("msg", "Shutting down old service before activating new one")
+		oldSvc.Shutdown()
+	}
+
+	// Start the new service
+	if err := newService.Start(); err != nil {
+		logger.Error("msg", "Failed to start new service pipelines after reload. The application may be in a non-functional state.", "error", err)
+		return nil, nil, nil, fmt.Errorf("failed to start new service: %w", err)
+	}
+
+	// Manage status reporter lifecycle
+	if statusCancel != nil {
+		statusCancel()
+	}
+
+	var newStatusCancel context.CancelFunc
+	if newCfg.StatusReporter {
+		newStatusCancel = startStatusReporter(ctx, newService)
+	}
+
+	logger.Info("msg", "Configuration hot reload completed successfully")
+	return newService, newCfg, newStatusCancel, nil
+}
+
+// bootstrapService creates and initializes the main log transport service and its pipelines
+func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service, error) {
+	// Create service with logger dependency injection
+	svc, err := service.NewService(ctx, cfg, logger)
+	if err != nil {
+		logger.Error("msg", "Failed to initialize service",
+			"component", "bootstrap",
+		)
+		return nil, err
+	}
+
+	logger.Info("msg", "LogWisp started",
+		"version", version.Short(),
+	)
+
+	return svc, nil
+}
+
+// initializeLogger sets up the global logger based on the application's configuration
+func initializeLogger(cfg *config.Config) error {
+	logger = log.NewLogger()
+	logCfg := log.DefaultConfig()
+
+	if cfg.Quiet {
+		// In quiet mode, disable ALL logging output
+		logCfg.Level = 255 // A level that disables all output
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = false
+		return logger.ApplyConfig(logCfg)
+	}
+
+	// Determine log level
+	levelValue, err := log.Level(cfg.Logging.Level)
+	if err != nil {
+		return fmt.Errorf("invalid log level: %w", err)
+	}
+	logCfg.Level = levelValue
+
+	// Configure log format
+	if cfg.Logging.Format != "" {
+		logCfg.Format = cfg.Logging.Format
+	}
+	if cfg.Logging.Sanitization != "" {
+		logCfg.Sanitization = sanitizer.PolicyPreset(cfg.Logging.Sanitization)
+	}
+
+	// Configure based on output mode
+	switch cfg.Logging.Output {
+	case "none":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = false
+	case "stdout":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "stdout"
+	case "stderr":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "stderr"
+	case "split":
+		logCfg.EnableFile = false
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "split"
+	case "file":
+		logCfg.EnableFile = true
+		logCfg.EnableConsole = false
+		configureFileLogging(logCfg, cfg)
+	case "all":
+		logCfg.EnableFile = true
+		logCfg.EnableConsole = true
+		logCfg.ConsoleTarget = "split"
+		configureFileLogging(logCfg, cfg)
+	default:
+		return fmt.Errorf("invalid log output mode: %s", cfg.Logging.Output)
+	}
+
+	return logger.ApplyConfig(logCfg)
+}
+
+// configureFileLogging sets up file-based logging parameters from the configuration
+func configureFileLogging(logCfg *log.Config, cfg *config.Config) {
+	if cfg.Logging.File != nil {
+		logCfg.Directory = cfg.Logging.File.Directory
+		logCfg.Name = cfg.Logging.File.Name
+		logCfg.MaxSizeKB = cfg.Logging.File.MaxSizeMB * 1000
+		logCfg.MaxTotalSizeKB = cfg.Logging.File.MaxTotalSizeMB * 1000
+		if cfg.Logging.File.RetentionHours > 0 {
+			logCfg.RetentionPeriodHrs = cfg.Logging.File.RetentionHours
+		}
+	}
+}

cmd/logwisp/help.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"logwisp/internal/version"
+	"os"
+)
+
+// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths.
+const helpText = `LogWisp %s - log collection, processing, and distribution
+
+Usage:
+  logwisp [options]
+  logwisp help | -h | --help
+  logwisp --version
+
+Any configuration key is settable as a flag using its TOML path:
+  --<path>=<value>              e.g. --logging.level=debug
+
+Common options:
+  -c, --config <path>           Configuration file (default: ./logwisp.toml)
+      --quiet                   Suppress console output
+      --status_reporter=<bool>  Periodic status logging (default: true)
+      --auto_reload=<bool>      Config hot reload on file change (default: false)
+
+Logging:
+      --logging.output=<mode>   file|stdout|stderr|split|all|none
+      --logging.level=<level>   debug|info|warn|error
+      --logging.file.directory=<path>
+      --logging.console.target=<target>   stdout|stderr|split
+
+Pipelines (N = 0-based index):
+      --pipelines.N.name=<name>
+      --pipelines.N.plugin_sources.N.type=<type>   file|console|random|null
+      --pipelines.N.plugin_sinks.N.type=<type>     console|file|http|tcp|null
+      --pipelines.N.flow.filters.N.patterns='["ERROR","WARN"]'
+
+Environment:
+  LOGWISP_<PATH>                Config path, '.' -> '_', uppercase
+                                e.g. LOGWISP_LOGGING_LEVEL=debug
+  LOGWISP_CONFIG_FILE           Configuration file path
+  LOGWISP_CONFIG_DIR            Configuration directory
+
+Signals:
+  SIGINT, SIGTERM               Graceful shutdown
+  SIGHUP, SIGUSR1               Reload configuration
+
+Exit codes:
+  0  success
+  1  general error
+  2  configuration file not found
+`
+
+// handleHelp prints usage and exits if a help request is present in args
+func handleHelp(args []string) {
+	if len(args) > 0 && args[0] == "help" {
+		printHelp()
+	}
+	for _, arg := range args {
+		if arg == "--" {
+			break // end of flags
+		}
+		if arg == "-h" || arg == "--help" {
+			printHelp()
+		}
+	}
+}
+
+// printHelp writes usage to stdout and exits with success
+func printHelp() {
+	fmt.Printf(helpText, version.Short())
+	os.Exit(0)
+}

cmd/logwisp/main.go

@@ -0,0 +1,157 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/version"
+
+	"github.com/lixenwraith/log"
+)
+
+// logger is the global logger instance for the application
+var logger *log.Logger
+
+// main is the entry point for the LogWisp application
+func main() {
+	// --- 1. Initial setup ---
+	// Emulates nohup
+	signal.Ignore(syscall.SIGHUP)
+
+	// Help handled before config parsing; loader has no help flag.
+	// Also the future dispatch point for subcommands (tls, etc.)
+	handleHelp(os.Args[1:])
+
+	// Load configuration with automatic CLI parsing
+	cfg, err := config.Load(os.Args[1:])
+	if err != nil {
+		if strings.Contains(err.Error(), "not found") && cfg != nil && cfg.ConfigFile != "" {
+			fmt.Fprintf(os.Stderr, "Error: Config file not found: %s\n", cfg.ConfigFile)
+			os.Exit(2)
+		}
+		fmt.Fprintf(os.Stderr, "Error: Failed to load config: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Initialize output handler
+	InitOutputHandler(cfg.Quiet)
+
+	// Handle version
+	if cfg.ShowVersion {
+		fmt.Println(version.String())
+		os.Exit(0)
+	}
+
+	// Initialize logger instance and apply configuration
+	if err := initializeLogger(cfg); err != nil {
+		FatalError(1, "Failed to initialize logger: %v\n", err)
+	}
+	defer shutdownLogger()
+
+	// Start the logger
+	if err := logger.Start(); err != nil {
+		FatalError(1, "Failed to start logger: %v\n", err)
+	}
+
+	// Log startup information
+	logger.Info("msg", "LogWisp starting",
+		"version", version.String(),
+		"config_file", cfg.ConfigFile,
+		"log_output", cfg.Logging.Output,
+		"status_reporter", cfg.StatusReporter,
+		"auto_reload", cfg.ConfigAutoReload)
+
+	// Create context for shutdown
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// --- 2. Bootstrap initial service ---
+	svc, statusReporterCancel, err := bootstrapInitial(ctx, cfg)
+	if err != nil {
+		logger.Error("msg", "Failed to initialize service", "error", err)
+		shutdownLogger()
+		os.Exit(1)
+	}
+
+	// --- 3. Setup signals and shutdown ---
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGUSR1)
+
+	var configChanges <-chan string
+	lcfg := config.GetConfigManager()
+	if cfg.ConfigAutoReload && lcfg != nil {
+		configChanges = lcfg.Watch()
+		logger.Info("msg", "Config auto-reload enabled", "config_file", cfg.ConfigFile)
+	} else {
+		logger.Info("msg", "Config auto-reload disabled")
+	}
+
+	// Service shutdown sequence
+	defer func() {
+		logger.Info("msg", "Shutdown initiated")
+		if statusReporterCancel != nil {
+			statusReporterCancel()
+		}
+		if svc != nil {
+			svc.Shutdown()
+		}
+		if lcfg != nil {
+			lcfg.StopAutoUpdate()
+		}
+		logger.Info("msg", "Shutdown complete")
+		// Deferred logger shutdown will run after this
+	}()
+
+	// --- 4. Main Application Event Loop ---
+	logger.Info("msg", "Application started, waiting for signals or config changes")
+	for {
+		select {
+		case sig := <-sigChan:
+			if sig == syscall.SIGHUP || sig == syscall.SIGUSR1 {
+				logger.Info("msg", "Reload signal received, triggering manual reload", "signal", sig)
+				newSvc, newCfg, newStatusCancel, err := handleReload(ctx, svc, statusReporterCancel)
+				if err == nil {
+					svc = newSvc
+					cfg = newCfg
+					statusReporterCancel = newStatusCancel
+				}
+			} else {
+				logger.Info("msg", "Shutdown signal received", "signal", sig)
+				cancel() // Trigger service shutdown via context
+			}
+
+		case event, ok := <-configChanges:
+			if !ok {
+				logger.Warn("msg", "Configuration watch channel closed, disabling auto-reload")
+				configChanges = nil // Stop selecting on this channel
+				continue
+			}
+			logger.Info("msg", "Configuration file change detected, triggering reload", "event", event)
+			newSvc, newCfg, newStatusCancel, err := handleReload(ctx, svc, statusReporterCancel)
+			if err == nil {
+				svc = newSvc
+				cfg = newCfg
+				statusReporterCancel = newStatusCancel
+			}
+
+		case <-ctx.Done():
+			return // Exit the loop and trigger deferred shutdown
+		}
+	}
+}
+
+// shutdownLogger gracefully shuts down the global logger.
+func shutdownLogger() {
+	if logger != nil {
+		if err := logger.Shutdown(core.LoggerShutdownTimeout); err != nil {
+			// Best effort - can't log the shutdown error
+			Error("Logger shutdown error: %v\n", err)
+		}
+	}
+}

cmd/logwisp/output.go

@@ -1,4 +1,3 @@
-// FILE: logwisp/src/cmd/logwisp/output.go
 package main
 
 import (
@@ -8,7 +7,7 @@ import (
 	"sync"
 )
 
-// Manages all application output respecting quiet mode
+// OutputHandler manages all application output, respecting the global quiet mode
 type OutputHandler struct {
 	quiet  bool
 	mu     sync.RWMutex
@@ -16,10 +15,10 @@ type OutputHandler struct {
 	stderr io.Writer
 }
 
-// Global output handler instance
+// output is the global instance of the OutputHandler
 var output *OutputHandler
 
-// Initializes the global output handler
+// InitOutputHandler initializes the global output handler
 func InitOutputHandler(quiet bool) {
 	output = &OutputHandler{
 		quiet:  quiet,
@@ -28,59 +27,21 @@ func InitOutputHandler(quiet bool) {
 	}
 }
 
-// Writes to stdout if not in quiet mode
+// Print writes to stdout
-func (o *OutputHandler) Print(format string, args ...any) {
-	o.mu.RLock()
-	defer o.mu.RUnlock()
-
-	if !o.quiet {
-		fmt.Fprintf(o.stdout, format, args...)
-	}
-}
-
-// Writes to stderr if not in quiet mode
-func (o *OutputHandler) Error(format string, args ...any) {
-	o.mu.RLock()
-	defer o.mu.RUnlock()
-
-	if !o.quiet {
-		fmt.Fprintf(o.stderr, format, args...)
-	}
-}
-
-// Writes to stderr and exits (respects quiet mode)
-func (o *OutputHandler) FatalError(code int, format string, args ...any) {
-	o.Error(format, args...)
-	os.Exit(code)
-}
-
-// Returns the current quiet mode status
-func (o *OutputHandler) IsQuiet() bool {
-	o.mu.RLock()
-	defer o.mu.RUnlock()
-	return o.quiet
-}
-
-// Updates quiet mode (useful for testing)
-func (o *OutputHandler) SetQuiet(quiet bool) {
-	o.mu.Lock()
-	defer o.mu.Unlock()
-	o.quiet = quiet
-}
-
-// Helper functions for global output handler
 func Print(format string, args ...any) {
 	if output != nil {
 		output.Print(format, args...)
 	}
 }
 
+// Error writes to stderr
 func Error(format string, args ...any) {
 	if output != nil {
 		output.Error(format, args...)
 	}
 }
 
+// FatalError writes to stderr and exits the application
 func FatalError(code int, format string, args ...any) {
 	if output != nil {
 		output.FatalError(code, format, args...)
@@ -90,3 +51,43 @@ func FatalError(code int, format string, args ...any) {
 		os.Exit(code)
 	}
 }
+
+// Print writes a formatted string to stdout if not in quiet mode
+func (o *OutputHandler) Print(format string, args ...any) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+
+	if !o.quiet {
+		fmt.Fprintf(o.stdout, format, args...)
+	}
+}
+
+// Error writes a formatted string to stderr if not in quiet mode
+func (o *OutputHandler) Error(format string, args ...any) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+
+	if !o.quiet {
+		fmt.Fprintf(o.stderr, format, args...)
+	}
+}
+
+// FatalError writes a formatted string to stderr and exits with the given code.
+func (o *OutputHandler) FatalError(code int, format string, args ...any) {
+	o.Error(format, args...)
+	os.Exit(code)
+}
+
+// IsQuiet returns the current quiet mode status.
+func (o *OutputHandler) IsQuiet() bool {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+	return o.quiet
+}
+
+// SetQuiet updates the quiet mode status.
+func (o *OutputHandler) SetQuiet(quiet bool) {
+	o.mu.Lock()
+	defer o.mu.Unlock()
+	o.quiet = quiet
+}

cmd/logwisp/status.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"logwisp/internal/service"
+)
+
+// startStatusReporter starts a new status reporter for a service and returns its cancel function.
+func startStatusReporter(ctx context.Context, svc *service.Service) context.CancelFunc {
+	reporterCtx, cancel := context.WithCancel(ctx)
+	go statusReporter(svc, reporterCtx)
+	logger.Debug("msg", "Started status reporter")
+	return cancel
+}
+
+// statusReporter periodically logs the health and statistics of the service
+func statusReporter(service *service.Service, ctx context.Context) {
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if service == nil {
+				logger.Warn("msg", "Status reporter: service is nil",
+					"component", "status_reporter")
+				return
+			}
+
+			// Safely get stats with recovery
+			func() {
+				defer func() {
+					if r := recover(); r != nil {
+						logger.Error("msg", "Panic in status reporter",
+							"component", "status_reporter",
+							"panic", r)
+					}
+				}()
+
+				stats := service.GetGlobalStats()
+				totalPipelines, ok := stats["total_pipelines"].(int)
+				if !ok || totalPipelines == 0 {
+					logger.Warn("msg", "No active pipelines in status report",
+						"component", "status_reporter")
+					return
+				}
+
+				// Log service-level summary
+				logger.Debug("msg", "Status report",
+					"component", "status_reporter",
+					"active_pipelines", totalPipelines,
+					"time", time.Now().Format("15:04:05"))
+
+				// Log each pipeline's stats recursively
+				if pipelines, ok := stats["pipelines"].(map[string]any); ok {
+					for name, pipelineStats := range pipelines {
+						logStats("Pipeline status", name, pipelineStats)
+					}
+				}
+			}()
+		}
+	}
+}
+
+// logStats recursively logs statistics with automatic field extraction
+func logStats(msg string, name string, stats any) {
+	// Build base log fields
+	fields := []any{
+		"msg", msg,
+		"name", name,
+	}
+
+	// Extract and flatten important metrics from stats map
+	if statsMap, ok := stats.(map[string]any); ok {
+		// Add scalar values directly
+		for key, value := range statsMap {
+			switch v := value.(type) {
+			case string, bool, int, int64, uint64, float64:
+				fields = append(fields, key, v)
+			case time.Time:
+				if !v.IsZero() {
+					fields = append(fields, key, v.Format(time.RFC3339))
+				}
+			case map[string]any:
+				// For nested maps, log summary counts if they contain arrays/maps
+				if count := getItemCount(v); count > 0 {
+					fields = append(fields, fmt.Sprintf("%s_count", key), count)
+				}
+			case []any, []map[string]any:
+				// For arrays, just log the count
+				fields = append(fields, fmt.Sprintf("%s_count", key), getArrayLength(value))
+			}
+		}
+
+		// Log the flattened stats
+		logger.Debug(fields...)
+
+		// Recursively log nested structures with detail
+		for key, value := range statsMap {
+			switch v := value.(type) {
+			case map[string]any:
+				// Log nested component stats
+				if key == "flow" || key == "rate_limiter" || key == "filters" {
+					logStats(fmt.Sprintf("%s %s", name, key), key, v)
+				}
+			case []map[string]any:
+				// Log array items (sources, sinks, filters)
+				for i, item := range v {
+					if itemName, ok := item["id"].(string); ok {
+						logStats(fmt.Sprintf("%s %s", name, key), itemName, item)
+					} else {
+						logStats(fmt.Sprintf("%s %s", name, key), fmt.Sprintf("%s[%d]", key, i), item)
+					}
+				}
+			}
+		}
+	}
+}
+
+// getItemCount returns the count of items in a map (for nested structures)
+func getItemCount(m map[string]any) int {
+	for _, v := range m {
+		switch v.(type) {
+		case []any:
+			return len(v.([]any))
+		case []map[string]any:
+			return len(v.([]map[string]any))
+		}
+	}
+	return 0
+}
+
+// getArrayLength safely gets the length of various array types
+func getArrayLength(v any) int {
+	switch arr := v.(type) {
+	case []any:
+		return len(arr)
+	case []map[string]any:
+		return len(arr)
+	default:
+		return 0
+	}
+}

config/logwisp.toml

@@ -0,0 +1,367 @@
+###############################################################################
+### LogWisp Configuration
+### Default location: ~/.config/logwisp/logwisp.toml
+### Precedence: CLI flags > Environment > File > Defaults
+###
+### Commented values are the built-in defaults unless marked "example".
+### Uncommenting a default is a no-op.
+###
+### NOTE: password/token/SCRAM authentication, network access control (ACL),
+### http/tcp ingest sources, and http_client/tcp_client sinks were removed in
+### the restructure and are not available in this version. TLS and mTLS ARE
+### available on every network source and sink; see the [...tls] blocks below,
+### and the [...auth] blocks to authorize peers by certificate identity.
+###
+### Environment overrides are currently read WITHOUT the LOGWISP_ prefix
+### (QUIET, LOGGING_LEVEL, ...). LOGWISP_CONFIG_FILE and LOGWISP_CONFIG_DIR
+### are the exceptions and do carry it. Array-indexed paths such as
+### pipelines.0.name cannot be set from the CLI or environment at all.
+###############################################################################
+
+###############################################################################
+### Global Settings
+###############################################################################
+
+quiet = false                           # Suppress console output
+status_reporter = true                  # Periodic status logging (30s, DEBUG level)
+auto_reload = false                     # Config auto-reload on file change
+
+###############################################################################
+### Logging (LogWisp's internal operational logging)
+###############################################################################
+
+[logging]
+output = "stdout"                       # file|stdout|stderr|split|all|none
+level = "info"                          # debug|info|warn|error
+# format = "txt"                        # raw|txt|json
+# sanitization = ""                     # raw|json|txt|shell (empty = logger default)
+
+# [logging.file]                        # Used when output is "file" or "all"
+# directory = "./log"
+# name = "logwisp"
+# max_size_mb = 100
+# max_total_size_mb = 1000
+# retention_hours = 168.0               # 7 days
+
+## Validated but NOT applied: the console destination comes from `output` above.
+# [logging.console]
+# target = "stdout"                     # stdout|stderr|split
+
+###############################################################################
+### Pipelines
+### Each pipeline: plugin_sources -> flow (rate_limit|filters|format) -> plugin_sinks
+### Names must be unique. 1+ source and 1+ sink required.
+###############################################################################
+
+[[pipelines]]
+name = "default"
+
+###============================================================================
+### Flow (processing between sources and sinks)
+###============================================================================
+
+## policy="pass" short-circuits the size check too: enforcing a size cap needs
+## rate > 0 AND policy = "drop".
+# [pipelines.flow.rate_limit]
+# rate = 0.0                            # Entries/second (0 = limiter disabled)
+# burst = 0.0                           # Burst capacity (defaults to rate)
+# policy = "pass"                       # pass|drop
+# max_entry_size_bytes = 0              # 0 = unlimited
+
+## Filters: sequential include/exclude chain, matched against "<source> <level> <message>"
+# [[pipelines.flow.filters]]
+# type = "include"                      # include|exclude
+# logic = "or"                          # or|and
+# patterns = [".*ERROR.*", ".*WARN.*"]  # example; RE2 syntax
+
+# [pipelines.flow.format]
+# type = "raw"                          # raw|json|txt
+# flags = 0                             # Formatter flags (0 = defaults per type)
+# timestamp_format = ""                 # Go time layout (formatter default if empty)
+# sanitizer_policy = ""                 # raw|json|txt|shell (defaults per type)
+
+## Flow-level heartbeat (fan-out to all sinks, traverses chain links)
+# [pipelines.flow.heartbeat]
+# enabled = false
+# interval_ms = 1000                    # Minimum 100
+# include_timestamp = false
+# include_stats = false
+# format = "txt"                        # txt|json|raw ("comment" is rejected)
+
+###============================================================================
+### TLS (shared shape; applies to every network source and sink)
+###
+### Listeners  (tcp/http sinks, tcp_chain/http_chain sources):
+###   cert_file + key_file are REQUIRED; client_auth + client_ca_file enable mTLS.
+### Dialers    (tcp_chain/http_chain sinks):
+###   ca_file + server_name verify the server; cert_file + key_file present a
+###   client identity for mTLS.
+###
+### enabled = false      Master switch
+### cert_file = ""       Local certificate
+### key_file = ""        Private key for cert_file (set together)
+### client_auth = false  Listener: require and verify a client certificate
+### client_ca_file = ""  Listener: CA bundle verifying client certs
+### ca_file = ""         Dialer: CA bundle verifying the server (empty = system)
+### server_name = ""     Dialer: SNI / name to verify (empty = configured host)
+### insecure_skip_verify = false   Dialer: disable verification (never in prod)
+### min_version = "1.3"  "1.2" or "1.3". No max_version, no cipher_suites.
+###
+### TLS alone is a CA membership check: ANY certificate the CA signed is
+### accepted. Add an [...auth] block to decide WHICH of them may connect.
+###============================================================================
+
+###============================================================================
+### AUTH (shared shape; sits beside [...tls] on every network source and sink)
+###
+### Listeners  (tcp/http sinks, tcp_chain/http_chain sources):
+###   authorize the client certificate. type = "mtls" REQUIRES tls.enabled and
+###   tls.client_auth. On the http sink it gates stream_path AND status_path.
+### Chain sources additionally bind the node label to the identity.
+### Dialers    (tcp_chain/http_chain sinks):
+###   pin the server identity. type = "mtls" REQUIRES tls.enabled and forbids
+###   tls.insecure_skip_verify.
+###
+### type = "none"        none | mtls
+### identity = "cn"      cn | san_dns | san_uri | san_email
+### allow = []           Exact identities. Empty allow AND allow_patterns
+###                      admits any identity the CA vouches for (logged WARN).
+### allow_patterns = []  RE2 patterns; anchor them yourself (^...$)
+### node_binding = ""    Chain sources only, default "force" under mtls:
+###                        none   - trust_node governs, as before
+###                        assert - declared label must equal the identity;
+###                                 per-entry node labels still follow trust_node
+###                        force  - label AND every entry take the identity
+###                      Overrides trust_node. See doc/security.md.
+###============================================================================
+
+###============================================================================
+### Sources (1+ required)
+###============================================================================
+
+## Null source (testing)
+# [[pipelines.plugin_sources]]
+# id = "null_in"
+# type = "null"
+
+[[pipelines.plugin_sources]]
+id = "default_source"
+type = "file"
+[pipelines.plugin_sources.config]
+directory = "./"                        # Directory to monitor (required, not recursive)
+pattern = "*.log"                       # Glob pattern (* and ? only)
+## Tailing an already-open file polls at a fixed 100ms, regardless of this value.
+check_interval_ms = 100                 # Directory rescan interval (min 10)
+## raw = true never parses a line; with format type "raw" the file is relayed byte for byte.
+raw = false                             # Keep the whole line as the message
+from = "end"                            # "end" or "start" of a newly discovered file
+
+## Console source (stdin, single instance per pipeline)
+# [[pipelines.plugin_sources]]
+# id = "console_in"
+# type = "console"
+# [pipelines.plugin_sources.config]
+# buffer_size = 1000
+
+## Random source (testing; special=true exercises sanitizer policies)
+# [[pipelines.plugin_sources]]
+# id = "random_in"
+# type = "random"
+# [pipelines.plugin_sources.config]
+# interval_ms = 500
+# jitter_ms = 0                         # Clamped to interval_ms
+# format = "txt"                        # raw|txt|json
+# length = 20
+# special = false
+
+## TCP chain source (stdlib listener; receives NDJSON from upstream tcp_chain sinks)
+## Topology: jail [file source -> tcp_chain sink] -> host [tcp_chain source -> aggregate sink]
+# [[pipelines.plugin_sources]]
+# id = "chain_in"
+# type = "tcp_chain"
+# [pipelines.plugin_sources.config]
+# host = "0.0.0.0"                      # IPv4 only
+# port = 9440                           # Required
+# buffer_size = 1000
+# max_connections = 0                   # 0 = unlimited
+# read_timeout_ms = 0                   # idle deadline, 0 = none
+# hello_timeout_ms = 10000              # Protocol preamble deadline
+# trust_node = true                     # false: label entries by remote address
+# [pipelines.plugin_sources.config.tls]
+# enabled = true                        # example: mTLS listener
+# cert_file = "/etc/logwisp/tls/server.crt"
+# key_file = "/etc/logwisp/tls/server.key"
+# client_auth = true
+# client_ca_file = "/etc/logwisp/tls/client-ca.crt"
+# min_version = "1.3"
+# [pipelines.plugin_sources.config.auth]  # authorize senders by certificate
+# type = "none"                         # none | mtls; mtls requires client_auth
+# identity = "cn"                       # cn | san_dns | san_uri | san_email
+# allow = []                            # exact identities; empty = any the CA issued
+# allow_patterns = []                   # RE2, anchor them yourself
+# node_binding = "force"                # none | assert | force; overrides trust_node
+
+## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks)
+# [[pipelines.plugin_sources]]
+# id = "hchain_in"
+# type = "http_chain"
+# [pipelines.plugin_sources.config]
+# host = "0.0.0.0"
+# port = 9441                           # Required
+# ingest_path = "/ingest"               # Must start with "/"
+# buffer_size = 1000
+# max_body_bytes = 8388608              # per-request cap (8 MiB)
+# read_timeout_ms = 30000
+# trust_node = true                     # false: label entries by remote address
+# [pipelines.plugin_sources.config.tls]
+# enabled = true                        # example: mTLS listener
+# cert_file = "/etc/logwisp/tls/server.crt"
+# key_file = "/etc/logwisp/tls/server.key"
+# client_auth = true
+# client_ca_file = "/etc/logwisp/tls/client-ca.crt"
+# [pipelines.plugin_sources.config.auth]  # authorize senders by certificate
+# type = "none"                         # none | mtls; mtls requires client_auth
+# identity = "cn"                       # cn | san_dns | san_uri | san_email
+# allow = []                            # exact identities; empty = any the CA issued
+# allow_patterns = []                   # RE2, anchor them yourself
+# node_binding = "force"                # none | assert | force; overrides trust_node
+
+###============================================================================
+### Sinks (1+ required, fan-out)
+###============================================================================
+
+## Null sink (testing)
+# [[pipelines.plugin_sinks]]
+# id = "null_out"
+# type = "null"
+
+[[pipelines.plugin_sinks]]
+id = "default_sink"
+type = "console"
+[pipelines.plugin_sinks.config]
+target = "stdout"                       # stdout|stderr ("split" NOT supported)
+# buffer_size = 1000
+
+## File sink (rotating)
+# [[pipelines.plugin_sinks]]
+# id = "file_out"
+# type = "file"
+# [pipelines.plugin_sinks.config]
+# directory = "./logs"                  # Required
+# name = "output"                       # Required
+# max_size_mb = 100
+# max_total_size_mb = 1000
+# min_disk_free_mb = 0                  # 0 = no floor (only negatives become 100)
+# retention_hours = 168.0
+# buffer_size = 1000
+# flush_interval_ms = 100
+
+## HTTP sink (SSE streaming server + JSON status endpoint; IPv4 clients only)
+## Both endpoints are UNAUTHENTICATED and the stream sends
+## Access-Control-Allow-Origin: *. Bind to a trusted interface.
+# [[pipelines.plugin_sinks]]
+# id = "http_out"
+# type = "http"
+# [pipelines.plugin_sinks.config]
+# host = "0.0.0.0"
+# port = 8081                           # Required
+# stream_path = "/stream"               # Must start with "/"
+# status_path = "/status"               # Must differ from stream_path
+# buffer_size = 1000                    # Sink input queue
+# client_buffer_size = 256              # Per-client send queue
+# write_timeout_ms = 0                  # Per-event deadline, 0 = none
+# max_connections = 0                   # 0 = unlimited
+# [pipelines.plugin_sinks.config.tls]
+# enabled = true                        # example
+# cert_file = "/etc/logwisp/tls/server.crt"
+# key_file = "/etc/logwisp/tls/server.key"
+# client_auth = false
+# client_ca_file = ""
+# [pipelines.plugin_sinks.config.auth]  # gates BOTH stream_path and status_path
+# type = "none"                         # none | mtls; mtls requires client_auth
+# identity = "cn"                       # cn | san_dns | san_uri | san_email
+# allow = []                            # exact identities; empty = any the CA issued
+# allow_patterns = []                   # RE2, anchor them yourself
+
+## TCP sink (streaming server, IPv4 clients only)
+# [[pipelines.plugin_sinks]]
+# id = "tcp_out"
+# type = "tcp"
+# [pipelines.plugin_sinks.config]
+# host = "0.0.0.0"
+# port = 9090                           # Required
+# buffer_size = 1000
+# client_buffer_size = 256
+# write_timeout_ms = 5000               # Missed deadline disconnects the client
+# keep_alive = true
+# keep_alive_period_ms = 30000
+# max_connections = 0
+# [pipelines.plugin_sinks.config.tls]
+# enabled = true                        # example
+# cert_file = "/etc/logwisp/tls/server.crt"
+# key_file = "/etc/logwisp/tls/server.key"
+# client_auth = true
+# client_ca_file = "/etc/logwisp/tls/client-ca.crt"
+# [pipelines.plugin_sinks.config.auth]  # authorize stream readers
+# type = "none"                         # none | mtls; mtls requires client_auth
+# identity = "cn"                       # cn | san_dns | san_uri | san_email
+# allow = []                            # exact identities; empty = any the CA issued
+# allow_patterns = []                   # RE2, anchor them yourself
+
+## TCP chain sink (stdlib client; forwards to downstream tcp_chain source)
+## Do NOT point a chain sink at a chain source in the SAME pipeline: entries
+## loop back in and amplify without bound.
+# [[pipelines.plugin_sinks]]
+# id = "chain_out"
+# type = "tcp_chain"
+# [pipelines.plugin_sinks.config]
+# host = "10.0.0.1"                     # Required
+# port = 9440                           # Required
+# node = ""                             # origin label, default hostname; preserved across hops
+# buffer_size = 1000
+# dial_timeout_ms = 5000
+# write_timeout_ms = 5000
+# backoff_min_ms = 500
+# backoff_max_ms = 30000
+# keep_alive = true
+# keep_alive_period_ms = 30000
+# [pipelines.plugin_sinks.config.tls]
+# enabled = true                        # example: mTLS dialer
+# ca_file = "/etc/logwisp/tls/ca.crt"
+# server_name = ""
+# insecure_skip_verify = false
+# cert_file = "/etc/logwisp/tls/client.crt"
+# key_file = "/etc/logwisp/tls/client.key"
+# min_version = "1.3"
+# [pipelines.plugin_sinks.config.auth]  # pin the downstream server's identity
+# type = "none"                         # none | mtls; mtls requires tls.enabled
+# identity = "cn"                       # cn | san_dns | san_uri | san_email
+# allow = []                            # exact identities; empty = any the CA issued
+# allow_patterns = []                   # RE2, anchor them yourself
+
+## HTTP chain sink (stdlib client; batched NDJSON POST to downstream http_chain source)
+# [[pipelines.plugin_sinks]]
+# id = "hchain_out"
+# type = "http_chain"
+# [pipelines.plugin_sinks.config]
+# host = "10.0.0.1"                     # Required
+# port = 9441                           # Required
+# ingest_path = "/ingest"
+# node = ""                             # origin label, default hostname; preserved across hops
+# buffer_size = 1000
+# max_batch_count = 100
+# max_batch_bytes = 1048576
+# flush_interval_ms = 1000
+# request_timeout_ms = 10000            # Covers dial + write + response
+# backoff_min_ms = 500
+# backoff_max_ms = 30000
+# [pipelines.plugin_sinks.config.tls]
+# enabled = true                        # example: mTLS dialer
+# ca_file = "/etc/logwisp/tls/ca.crt"
+# cert_file = "/etc/logwisp/tls/client.crt"
+# key_file = "/etc/logwisp/tls/client.key"
+# [pipelines.plugin_sinks.config.auth]  # pin the downstream server's identity
+# type = "none"                         # none | mtls; mtls requires tls.enabled
+# identity = "cn"                       # cn | san_dns | san_uri | san_email
+# allow = []                            # exact identities; empty = any the CA issued
+# allow_patterns = []                   # RE2, anchor them yourself

config/logwisp.toml.defaults

@@ -1,408 +0,0 @@
-###############################################################################
-### LogWisp Configuration
-### Default location: ~/.config/logwisp/logwisp.toml
-### Configuration Precedence: CLI flags > Environment > File > Defaults
-### Default values shown - uncommented lines represent active configuration
-###############################################################################
-
-###############################################################################
-### Global Settings
-###############################################################################
-
-background = false                    # Run as daemon
-quiet = false                         # Suppress console output
-disable_status_reporter = false       # Disable periodic status logging
-config_auto_reload = false            # Reload config on file change
-
-###############################################################################
-### Logging Configuration
-###############################################################################
-
-[logging]
-output = "stdout"                     # file|stdout|stderr|split|all|none
-level = "info"                        # debug|info|warn|error
-
-# [logging.file]
-# directory = "./log"                 # Log directory path
-# name = "logwisp"                    # Base filename
-# max_size_mb = 100                   # Rotation threshold
-# max_total_size_mb = 1000            # Total size limit
-# retention_hours = 168.0             # Delete logs older than (7 days)
-
-[logging.console]
-target = "stdout"                     # stdout|stderr|split
-format = "txt"                        # txt|json
-
-###############################################################################
-### Pipeline Configuration
-###############################################################################
-
-[[pipelines]]
-name = "default"                      # Pipeline identifier
-
-###============================================================================
-### Rate Limiting (Pipeline-level)
-###============================================================================
-
-# [pipelines.rate_limit]
-# rate = 1000.0                        # Entries per second (0=disabled)
-# burst = 2000.0                       # Burst capacity (defaults to rate)
-# policy = "drop"                      # pass|drop
-# max_entry_size_bytes = 0             # Max entry size (0=unlimited)
-
-###============================================================================
-### Filters
-###============================================================================
-
-### ⚠️ Example: Include only ERROR and WARN logs
-## [[pipelines.filters]]
-## type = "include"                     # include|exclude
-## logic = "or"                         # or|and
-## patterns = [".*ERROR.*", ".*WARN.*"]
-
-### ⚠️ Example: Exclude debug logs
-## [[pipelines.filters]]
-## type = "exclude"
-## patterns = [".*DEBUG.*"]
-
-###============================================================================
-### Format Configuration
-###============================================================================
-
-# [pipelines.format]
-# type = "raw"                         # json|txt|raw
-
-### Raw formatter options (default)
-# [pipelines.format.raw]
-# add_new_line = true                  # Add newline to messages
-
-### JSON formatter options
-# [pipelines.format.json]
-# pretty = false                       # Pretty print JSON
-# timestamp_field = "timestamp"        # Field name for timestamp
-# level_field = "level"                # Field name for log level
-# message_field = "message"            # Field name for message
-# source_field = "source"              # Field name for source
-
-### Text formatter options
-# [pipelines.format.txt]
-# template = "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}"
-# timestamp_format = "2006-01-02T15:04:05.000Z07:00"   # Go time format string
-
-###============================================================================
-### Sources (Input Sources)
-###============================================================================
-
-###----------------------------------------------------------------------------
-### Directory Source (Active Default)
-[[pipelines.sources]]
-type = "directory"
-
-[pipelines.sources.directory]
-path = "./"                           # Watch directory
-pattern = "*.log"                     # File pattern (glob)
-check_interval_ms = 100               # Poll interval
-recursive = false                     # Scan subdirectories
-
-###----------------------------------------------------------------------------
-### Stdin Source
-# [[pipelines.sources]]
-# type = "stdin"
-
-# [pipelines.sources.stdin]
-# buffer_size = 1000                  # Internal buffer size
-
-###----------------------------------------------------------------------------
-### HTTP Source (Receives via POST)
-# [[pipelines.sources]]
-# type = "http"
-
-# [pipelines.sources.http]
-# host = "0.0.0.0"                    # Listen address
-# port = 8081                         # Listen port
-# ingest_path = "/ingest"             # Ingest endpoint
-# buffer_size = 1000                  # Internal buffer size
-# max_body_size = 1048576             # Max request body (1MB)
-# read_timeout_ms = 10000             # Read timeout
-# write_timeout_ms = 10000            # Write timeout
-
-### TLS configuration
-# [pipelines.sources.http.tls]
-# enabled = false
-# cert_file = "/path/to/cert.pem"
-# key_file = "/path/to/key.pem"
-# ca_file = "/path/to/ca.pem"
-# min_version = "TLS1.2"              # TLS1.2|TLS1.3
-# client_auth = false                 # Require client certs
-# client_ca_file = "/path/to/ca.pem"  # CA to validate client certs
-# verify_client_cert = true           # Require valid client cert
-
-### ⚠️ Example: TLS configuration to enable auth)
-## [pipelines.sources.http.tls]
-## enabled = true                      # MUST be true for auth
-## cert_file = "/path/to/server.pem"
-## key_file = "/path/to/server.key"
-
-### Network limiting (access control)
-# [pipelines.sources.http.net_limit]
-# enabled = false
-# max_connections_per_ip = 10
-# max_connections_total = 100
-# requests_per_second = 100.0         # Rate limit per client
-# burst_size = 200                    # Token bucket burst
-# response_code = 429                 # HTTP rate limit response code
-# response_message = "Rate limit exceeded"
-# ip_whitelist = []
-# ip_blacklist = []
-
-### Authentication (validates clients)
-### ☢ SECURITY: HTTP auth REQUIRES TLS to be enabled
-# [pipelines.sources.http.auth]
-# type = "none"                       # none|basic|token|mtls (NO scram)
-# realm = "LogWisp"                   # For basic auth
-
-### Basic auth users
-# [[pipelines.sources.http.auth.basic.users]]
-# username = "admin"
-# password_hash = "$argon2..."        # Argon2 hash
-
-### Token auth tokens
-# [pipelines.sources.http.auth.token]
-# tokens = ["token1", "token2"]
-
-###----------------------------------------------------------------------------
-### TCP Source (Receives logs via TCP Client Sink)
-# [[pipelines.sources]]
-# type = "tcp"
-
-# [pipelines.sources.tcp]
-# host = "0.0.0.0"                    # Listen address
-# port = 9091                         # Listen port
-# buffer_size = 1000                  # Internal buffer size
-# read_timeout_ms = 10000             # Read timeout
-# keep_alive = true                   # Enable TCP keep-alive
-# keep_alive_period_ms = 30000        # Keep-alive interval
-
-### ☣ WARNING: TCP has NO TLS support (gnet limitation)
-### Use HTTP with TLS for encrypted transport
-
-### Network limiting (access control)
-# [pipelines.sources.tcp.net_limit]
-# enabled = false
-# max_connections_per_ip = 10
-# max_connections_total = 100
-# requests_per_second = 100.0
-# burst_size = 200
-# ip_whitelist = []
-# ip_blacklist = []
-
-### Authentication
-# [pipelines.sources.tcp.auth]
-# type = "none"                       # none|scram ONLY (no basic/token/mtls)
-
-### SCRAM auth users for TCP Source
-# [[pipelines.sources.tcp.auth.scram.users]]
-# username = "user1"
-# stored_key = "base64..."            # Pre-computed SCRAM keys
-# server_key = "base64..."
-# salt = "base64..."
-# argon_time = 3
-# argon_memory = 65536
-# argon_threads = 4
-
-###============================================================================
-### Sinks (Output Destinations)
-###============================================================================
-
-###----------------------------------------------------------------------------
-### Console Sink (Active Default)
-[[pipelines.sinks]]
-type = "console"
-
-[pipelines.sinks.console]
-target = "stdout"                     # stdout|stderr|split
-colorize = false                      # Enable colored output
-buffer_size = 100                     # Internal buffer size
-
-###----------------------------------------------------------------------------
-### File Sink
-# [[pipelines.sinks]]
-# type = "file"
-
-# [pipelines.sinks.file]
-# directory = "./logs"                # Output directory
-# name = "output"                     # Base filename
-# max_size_mb = 100                   # Rotation threshold
-# max_total_size_mb = 1000            # Total size limit
-# min_disk_free_mb = 500              # Minimum free disk space
-# retention_hours = 168.0             # Delete logs older than (7 days)
-# buffer_size = 1000                  # Internal buffer size
-# flush_interval_ms = 1000            # Force flush interval
-
-###----------------------------------------------------------------------------
-### HTTP Sink (SSE streaming to browser/HTTP client)
-# [[pipelines.sinks]]
-# type = "http"
-
-# [pipelines.sinks.http]
-# host = "0.0.0.0"                    # Listen address
-# port = 8080                         # Listen port
-# stream_path = "/stream"             # SSE stream endpoint
-# status_path = "/status"             # Status endpoint
-# buffer_size = 1000                  # Internal buffer size
-# max_connections = 100               # Max concurrent clients
-# read_timeout_ms = 10000             # Read timeout
-# write_timeout_ms = 10000            # Write timeout
-
-### Heartbeat configuration (keeps SSE alive)
-# [pipelines.sinks.http.heartbeat]
-# enabled = true
-# interval_ms = 30000                 # 30 seconds
-# include_timestamp = true
-# include_stats = false
-# format = "comment"                  # comment|event|json
-
-### TLS configuration
-# [pipelines.sinks.http.tls]
-# enabled = false
-# cert_file = "/path/to/cert.pem"
-# key_file = "/path/to/key.pem"
-# ca_file = "/path/to/ca.pem"
-# min_version = "TLS1.2"              # TLS1.2|TLS1.3
-# client_auth = false                 # Require client certs
-
-### ⚠️ Example: HTTP Client Sink → HTTP Source with mTLS
-## HTTP Source with mTLS:
-## [pipelines.sources.http.tls]
-## enabled = true
-## cert_file = "/path/to/server.pem"
-## key_file = "/path/to/server.key"
-## client_auth = true                  # Enable client cert verification
-## client_ca_file = "/path/to/ca.pem"
-
-## HTTP Client with client cert:
-## [pipelines.sinks.http_client.tls]
-## enabled = true
-## cert_file = "/path/to/client.pem"  # Client certificate
-## key_file = "/path/to/client.key"
-
-### Network limiting (access control)
-# [pipelines.sinks.http.net_limit]
-# enabled = false
-# max_connections_per_ip = 10
-# max_connections_total = 100
-# ip_whitelist = ["192.168.1.0/24"]
-# ip_blacklist = []
-
-### Authentication (for clients)
-### ☢ SECURITY: HTTP auth REQUIRES TLS to be enabled
-# [pipelines.sinks.http.auth]
-# type = "none"                       # none|basic|bearer|mtls
-
-###----------------------------------------------------------------------------
-### TCP Sink (Server - accepts connections from TCP clients)
-# [[pipelines.sinks]]
-# type = "tcp"
-
-# [pipelines.sinks.tcp]
-# host = "0.0.0.0"                    # Listen address
-# port = 9090                         # Listen port
-# buffer_size = 1000                  # Internal buffer size
-# max_connections = 100               # Max concurrent clients
-# keep_alive = true                   # Enable TCP keep-alive
-# keep_alive_period_ms = 30000        # Keep-alive interval
-
-### Heartbeat configuration
-# [pipelines.sinks.tcp.heartbeat]
-# enabled = false
-# interval_ms = 30000
-# include_timestamp = true
-# include_stats = false
-# format = "json"                     # json|txt
-
-### ☣ WARNING: TCP has NO TLS support (gnet limitation)
-### Use HTTP with TLS for encrypted transport
-
-### Network limiting
-# [pipelines.sinks.tcp.net_limit]
-# enabled = false
-# max_connections_per_ip = 10
-# max_connections_total = 100
-# ip_whitelist = []
-# ip_blacklist = []
-
-### ☣ WARNING: TCP Sink has NO AUTH support (aimed for debugging)
-### Use HTTP with TLS for encrypted transport
-
-###----------------------------------------------------------------------------
-### HTTP Client Sink (POST to HTTP Source endpoint)
-# [[pipelines.sinks]]
-# type = "http_client"
-
-# [pipelines.sinks.http_client]
-# url = "https://logs.example.com/ingest"
-# buffer_size = 1000
-# batch_size = 100                    # Logs per request
-# batch_delay_ms = 1000               # Max wait before sending
-# timeout_seconds = 30                # Request timeout
-# max_retries = 3                     # Retry attempts
-# retry_delay_ms = 1000               # Initial retry delay
-# retry_backoff = 2.0                 # Exponential backoff
-# insecure_skip_verify = false        # Skip TLS verification
-
-### TLS configuration
-# [pipelines.sinks.http_client.tls]
-# enabled = false
-# server_name = "logs.example.com"    # For verification
-# skip_verify = false                 # Skip verification
-# cert_file = "/path/to/client.pem"   # Client cert for mTLS
-# key_file = "/path/to/client.key"    # Client key for mTLS
-
-### ⚠️ Example: HTTP Client Sink → HTTP Source with mTLS
-## HTTP Source with mTLS:
-## [pipelines.sources.http.tls]
-## enabled = true
-## cert_file = "/path/to/server.pem"
-## key_file = "/path/to/server.key"
-## client_auth = true                  # Enable client cert verification
-## client_ca_file = "/path/to/ca.pem"
-
-## HTTP Client with client cert:
-## [pipelines.sinks.http_client.tls]
-## enabled = true
-## cert_file = "/path/to/client.pem"  # Client certificate
-## key_file = "/path/to/client.key"
-
-### Client authentication
-### ☢ SECURITY: HTTP auth REQUIRES TLS to be enabled
-# [pipelines.sinks.http_client.auth]
-# type = "none"                       # none|basic|token|mtls (NO scram)
-# # token = "your-token"              # For token auth
-# # username = "user"                 # For basic auth
-# # password = "pass"                 # For basic auth
-
-###----------------------------------------------------------------------------
-### TCP Client Sink (Connect to TCP Source server)
-# [[pipelines.sinks]]
-# type = "tcp_client"
-
-## [pipelines.sinks.tcp_client]
-# host = "logs.example.com"           # Target host
-# port = 9090                         # Target port
-# buffer_size = 1000                  # Internal buffer size
-# dial_timeout = 10                   # Connection timeout (seconds)
-# write_timeout = 30                  # Write timeout (seconds)
-# read_timeout = 10                   # Read timeout (seconds)
-# keep_alive = 30                     # TCP keep-alive (seconds)
-# reconnect_delay_ms = 1000           # Initial reconnect delay
-# max_reconnect_delay_ms = 30000      # Max reconnect delay
-# reconnect_backoff = 1.5             # Exponential backoff
-
-### ☣ WARNING: TCP has NO TLS support (gnet limitation)
-### Use HTTP with TLS for encrypted transport
-
-### Client authentication
-# [pipelines.sinks.tcp_client.auth]
-# type = "none"                       # none|scram ONLY (no basic/token/mtls)
-# # username = "user"                 # For SCRAM auth
-# # password = "pass"                 # For SCRAM auth

doc/README.md

@@ -1,76 +1,108 @@
-# LogWisp
+# LogWisp Documentation
 
-A high-performance, pipeline-based log transport and processing system built in Go. LogWisp provides flexible log collection, filtering, formatting, and distribution with security and reliability features.
+LogWisp is a pipeline-based log transport and processing system written in Go.
+It collects log entries from files, stdin, or other LogWisp nodes; rate-limits,
+filters, and formats them; and distributes them to files, consoles, live network
+streams, or downstream LogWisp nodes.
 
-## Features
+## Documentation Map
 
-### Core Capabilities
+| Document | Contents |
-- **Pipeline Architecture**: Independent processing pipelines with source(s) → filter → format → sink(s) flow
+|----------|----------|
-- **Multiple Input Sources**: Directory monitoring, stdin, HTTP, TCP
+| [Installation](installation.md) | Building, installing, and running as a service |
-- **Flexible Output Sinks**: Console, file, HTTP SSE, TCP streaming, HTTP/TCP forwarding
+| [Architecture](architecture.md) | Component model, data flow, concurrency, back-pressure |
-- **Real-time Processing**: Sub-millisecond latency with configurable buffering
+| [Configuration](configuration.md) | TOML structure, precedence, environment and CLI overrides |
-- **Hot Configuration Reload**: Update pipelines without service restart
+| [Sources](sources.md) | Every input plugin and its options |
+| [Sinks](sinks.md) | Every output plugin and its options |
+| [Filters](filters.md) | Pattern-based inclusion and exclusion |
+| [Formatters](formatters.md) | Output shaping and sanitization |
+| [Chaining](chaining.md) | Multi-node topologies and the chain wire protocol |
+| [Networking](networking.md) | Listeners, dialers, timeouts, connection limits |
+| [Security](security.md) | TLS, mTLS, and peer authorization; threat model and current limits |
+| [mTLS Authentication](mtls-auth-plan.md) | Design and rationale for certificate-based authorization |
+| [CLI](cli.md) | Flags, signals, exit codes |
+| [Operations](operations.md) | Running, monitoring, tuning, troubleshooting |
 
-### Data Processing
+A fully annotated configuration covering every option lives at
-- **Pattern-based Filtering**: Chainable include/exclude filters with regex support
+[`config/logwisp.toml`](../config/logwisp.toml).
-- **Multiple Formatters**: Raw, JSON, and template-based text formatting
-- **Rate Limiting**: Pipeline rate controls
 
-### Security & Reliability  
+## Capabilities
-- **Authentication**: Basic, token, SCRAM, and mTLS support
-- **TLS Encryption**: Full TLS 1.2/1.3 support for HTTP connections
-- **Access Control**: IP whitelisting/blacklisting, connection limits
-- **Automatic Reconnection**: Resilient client connections with exponential backoff
-- **File Rotation**: Size-based rotation with retention policies
 
-### Operational Features
+### Pipeline
-- **Status Monitoring**: Real-time statistics and health endpoints
-- **Signal Handling**: Graceful shutdown and configuration reload via signals
-- **Background Mode**: Daemon operation with proper signal handling
-- **Quiet Mode**: Silent operation for automated deployments
 
-## Documentation
+- Independent named pipelines, each `sources → flow → sinks`
+- Fan-in (many sources per pipeline) and fan-out (many sinks per pipeline)
+- Non-blocking sink dispatch: a stalled sink drops its own events and never
+  stalls the pipeline or its sibling sinks
+- Hot reload of pipeline configuration via `SIGHUP`/`SIGUSR1` or a file watch
 
-- [Installation Guide](installation.md) - Platform setup and service configuration
+### Inputs
-- [Architecture Overview](architecture.md) - System design and component interaction  
+
-- [Configuration Reference](configuration.md) - TOML structure and configuration methods
+`file` (directory tail with rotation detection), `console` (stdin),
-- [Input Sources](sources.md) - Available source types and configurations
+`random` (synthetic generator), `null`, and the chain ingest listeners
-- [Output Sinks](sinks.md) - Sink types and output options
+`tcp_chain` and `http_chain`.
-- [Filters](filters.md) - Pattern-based log filtering
+
-- [Formatters](formatters.md) - Log formatting and transformation
+### Outputs
-- [Authentication](authentication.md) - Security configurations and auth methods
+
-- [Networking](networking.md) - TLS, rate limiting, and network features
+`console`, `file` (rotating), `http` (Server-Sent Events plus a JSON status
-- [Command Line Interface](cli.md) - CLI flags and subcommands
+endpoint), `tcp` (broadcast server), `null`, and the chain forwarders
-- [Operations Guide](operations.md) - Running and maintaining LogWisp
+`tcp_chain` and `http_chain`.
+
+### Processing
+
+- Token-bucket rate limiting with an optional per-entry size cap
+- Chainable include/exclude regex filters with `or`/`and` logic
+- `raw`, `txt`, and `json` formatting with selectable sanitizer policies
+- Optional flow-level heartbeat entries
+
+### Transport security and authentication
+
+- TLS 1.2/1.3 on every network source and sink, listener and dialer alike
+- Mutual TLS: listeners can require and verify client certificates; dialers can
+  present a client identity
+- Authorization by certificate identity, per listener: named peers rather than
+  everything the CA issued, with the `http` sink's endpoints gated too
+- Node binding, so a chain source labels entries from the sender's certificate
+  rather than from what the sender claims
+
+See [Security](security.md) for what each layer does and does not give you.
 
 ## Quick Start
 
-Install LogWisp and create a basic configuration:
-
 ```toml
 [[pipelines]]
 name = "default"
 
-[[pipelines.sources]]
+[pipelines.flow.format]
-type = "directory"
+type = "json"
-[pipelines.sources.directory]
+sanitizer_policy = "json"
-path = "./"
+
+[[pipelines.plugin_sources]]
+id = "app_logs"
+type = "file"
+[pipelines.plugin_sources.config]
+directory = "/var/log/myapp"
 pattern = "*.log"
 
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
+id = "stdout"
 type = "console"
-[pipelines.sinks.console]
+[pipelines.plugin_sinks.config]
 target = "stdout"
 ```
 
-Run with: `logwisp -c config.toml`
+```bash
+logwisp -c config.toml
+```
 
 ## System Requirements
 
-- **Operating Systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
+- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
 - **Architecture**: amd64
-- **Go Version**: 1.25+ (for building from source)
+- **Go**: 1.26+ to build from source
+
+Network sources and sinks bind and dial over IPv4 only.
 
 ## License
 
-BSD 3-Clause License
+BSD 3-Clause.

doc/architecture.md

@@ -1,168 +1,198 @@
 # Architecture Overview
 
-LogWisp implements a pipeline-based architecture for flexible log processing and distribution.
+LogWisp moves log entries through independent pipelines. Everything else —
+plugins, sessions, TLS, statistics — hangs off that spine.
 
-## Core Concepts
+## Component Hierarchy
-
-### Pipeline Model
-
-Each pipeline operates independently with a source → filter → format → sink flow. Multiple pipelines can run concurrently within a single LogWisp instance, each processing different log streams with unique configurations.
-
-### Component Hierarchy
 
 ```
-Service (Main Process)
+main
-├── Pipeline 1
+└── Service
-│   ├── Sources (1 or more)
+    ├── Pipeline "app"
-│   ├── Rate Limiter (optional)
+    │   ├── Registry           instance tracking, single-instance enforcement
-│   ├── Filter Chain (optional)
+    │   ├── Session Manager    per-pipeline connection/session bookkeeping
-│   ├── Formatter (optional)
+    │   ├── Sources[]          plugin instances, keyed by id
-│   └── Sinks (1 or more)
+    │   ├── Flow
-├── Pipeline 2
+    │   │   ├── Rate Limiter   optional, token bucket
-│   └── [Same structure]
+    │   │   ├── Filter Chain   optional, ordered
-└── Status Reporter (optional)
+    │   │   ├── Formatter      raw | txt | json, with sanitizer
+    │   │   └── Heartbeat      optional generator
+    │   └── Sinks[]            plugin instances, keyed by id
+    ├── Pipeline "audit"
+    │   └── ...
+    └── Status Reporter        optional, 30s interval
 ```
 
+Package map:
+
+| Package | Responsibility |
+|---------|----------------|
+| `cmd/logwisp` | Entry point, help, logger bootstrap, signal loop, status reporter |
+| `internal/config` | Typed config schema, loading, top-level validation |
+| `internal/service` | Owns the pipeline set; start, stop, shutdown, global stats |
+| `internal/pipeline` | Pipeline runtime and per-pipeline plugin registry |
+| `internal/flow` | Rate limiter, filter chain invocation, formatting, heartbeat |
+| `internal/filter` | Regex filter and filter chain |
+| `internal/format` | Adapter over `lixenwraith/log` formatter + sanitizer |
+| `internal/source/*` | Source plugins |
+| `internal/sink/*` | Sink plugins |
+| `internal/plugin` | Global factory registry populated by plugin `init()` |
+| `internal/chain` | Chain wire protocol: hello preamble, entry codec, backoff |
+| `internal/tlsx` | The single seam between `TLSOptions` and `crypto/tls` |
+| `internal/session` | Session manager and per-instance proxy |
+| `internal/core` | Shared types (`LogEntry`, `TransportEvent`), capabilities, constants |
+| `internal/tokenbucket` | Rate limiter primitive |
+| `internal/sanitize` | Standalone hex-escaping helpers |
+
+## Plugin Registration
+
+Every plugin registers itself in an `init()` function, and
+`cmd/logwisp/bootstrap.go` blank-imports each package to trigger those
+`init()`s. Adding a plugin therefore means writing the package, calling
+`plugin.RegisterSource` / `plugin.RegisterSink`, and adding one blank import.
+
+Registration may attach metadata. The `console` source declares
+`MaxInstances: 1`, because a process has only one stdin; the per-pipeline
+registry rejects a second instance of any such type.
+
 ## Data Flow
 
-### Processing Stages
+### Entry lifecycle
 
-1. **Source Stage**: Sources monitor inputs and generate log entries
+1. **Source** produces a `core.LogEntry` and publishes it to every subscriber
-2. **Rate Limiting**: Optional pipeline-level rate control
+   channel it has handed out. Publication is non-blocking: a full subscriber
-3. **Filtering**: Pattern-based inclusion/exclusion
+   channel increments the source's `dropped_entries` counter.
-4. **Formatting**: Transform entries to desired output format
+2. **Flow** applies, in order: rate limit → filter chain → formatter. A drop at
-5. **Distribution**: Fan-out to multiple sinks
+   any stage ends the entry's life and increments `flow.total_dropped`.
+3. The formatter output becomes a `core.TransportEvent`, which carries both the
+   formatted `Payload` and the original structured `Entry`.
+4. **Dispatch** sends the event to every sink's input channel with a
+   non-blocking send.
 
-### Entry Lifecycle
+`LogEntry` fields:
 
-Log entries flow through the pipeline as `core.LogEntry` structures containing:
+| Field | Purpose |
-- **Time**: Entry timestamp
+|-------|---------|
-- **Level**: Log level (DEBUG, INFO, WARN, ERROR)
+| `Time` | Entry timestamp |
-- **Source**: Origin identifier
+| `Node` | Origin node label for chained topologies; stamped at the first hop, preserved by relays |
-- **Message**: Log content
+| `Source` | Origin identifier within the node (filename, plugin id, …) |
-- **Fields**: Additional metadata (JSON)
+| `Level` | `DEBUG`/`INFO`/`WARN`/`ERROR`/`TRACE`, when detected |
-- **RawSize**: Original entry size
+| `Message` | Log content |
+| `Fields` | Optional structured metadata as raw JSON |
+| `RawSize` | Original byte size, used by the entry-size cap |
 
-### Buffering Strategy
+Carrying `Entry` alongside `Payload` is what makes chain sinks
+format-independent: a `tcp_chain` or `http_chain` sink re-serializes the
+structured entry rather than shipping whatever text the local formatter chose.
 
-Each component maintains internal buffers to handle burst traffic:
+### Back-pressure and drops
-- Sources: Configurable buffer size (default 1000 entries)
-- Sinks: Independent buffers per sink
-- Network components: Additional TCP/HTTP buffers
 
-## Component Types
+There is exactly one drop policy and it is not configurable: **never block**.
 
-### Sources (Input)
+| Stage | Full-buffer behaviour | Counter |
+|-------|----------------------|---------|
+| Source → subscriber | Drop the entry | source `dropped_entries` |
+| Flow | Drop on rate limit, filter, or format error | `flow.total_dropped` |
+| Pipeline → sink | Drop for that sink only | pipeline `total_dropped_by_sink` |
+| TCP/HTTP sink → client queue | Drop for that client only | sink `dropped_writes` |
 
-- **Directory Source**: File system monitoring with rotation detection
+The `tcp_chain` sink is the one deliberate exception. It holds a line across
-- **Stdin Source**: Standard input processing
+reconnects until it is written or the process shuts down, so a downstream
-- **HTTP Source**: REST endpoint for log ingestion
+outage propagates backwards as a full input buffer and surfaces as
-- **TCP Source**: Raw TCP socket listener
+`total_dropped_by_sink` on the pipeline rather than as silent data loss inside
-
+the sink. The `http_chain` sink retries a batch with backoff, and drops it only
-### Sinks (Output)
+on a non-retryable response or on shutdown (`dropped_batches`).
-
-- **Console Sink**: stdout/stderr output
-- **File Sink**: Rotating file writer
-- **HTTP Sink**: Server-Sent Events (SSE) streaming
-- **TCP Sink**: TCP server for client connections
-- **HTTP Client Sink**: Forward to remote HTTP endpoints
-- **TCP Client Sink**: Forward to remote TCP servers
-
-### Processing Components
-
-- **Rate Limiter**: Token bucket algorithm for flow control
-- **Filter Chain**: Sequential pattern matching
-- **Formatters**: Raw, JSON, or template-based text transformation
 
 ## Concurrency Model
 
-### Goroutine Architecture
+- One goroutine per source drains that source's subscription and feeds the flow.
+- The flow's formatter holds a mutex; the underlying formatter reuses an
+  internal buffer and is not goroutine-safe.
+- Each network sink runs one broadcast/broker goroutine plus, per connection, a
+  writer goroutine (and for TCP, a reader goroutine that exists only to detect
+  disconnects and refresh session activity).
+- Chain sinks run a single run-loop goroutine that exclusively owns the
+  connection or the pending batch, so no locking is needed around either.
+- Statistics are atomics; configuration and registries use RW mutexes;
+  shutdown is context cancellation plus wait groups.
 
-- Each source runs in dedicated goroutines for monitoring
+### Shutdown ordering
-- Sinks operate independently with their own processing loops
-- Network listeners use optimized event loops (gnet for TCP)
-- Pipeline processing uses channel-based communication
 
-### Synchronization
+`Pipeline.Stop` is deliberately ordered so in-flight data drains:
 
-- Atomic counters for statistics
+1. Stop all sources concurrently; each closes its subscriber channels.
-- Read-write mutexes for configuration access
+2. Wait for the run loop, which ends when every subscription channel closes.
-- Context-based cancellation for graceful shutdown
+3. Stop all sinks concurrently.
-- Wait groups for coordinated startup/shutdown
 
 ## Network Architecture
 
-### Connection Patterns
+All listeners bind `tcp4` and all dialers dial `tcp4`. IPv6 clients cannot
+connect; this is deliberate, not an oversight.
 
-**Chaining Design**:
+| Plugin | Role | Protocol |
-- TCP Client Sink → TCP Source: Direct TCP forwarding
+|--------|------|----------|
-- HTTP Client Sink → HTTP Source: HTTP-based forwarding
+| `tcp` sink | Listener | Raw broadcast of formatted payloads |
+| `http` sink | Listener | HTTP/1.1 SSE; HTTP/2 negotiated via ALPN when TLS is on |
+| `tcp_chain` source | Listener | Chain protocol, persistent NDJSON stream |
+| `http_chain` source | Listener | Chain protocol, NDJSON batches over POST |
+| `tcp_chain` sink | Dialer | Chain protocol, persistent stream, auto-reconnect |
+| `http_chain` sink | Dialer | Chain protocol, batched POST with retry |
 
-**Monitoring Design**:
+TLS is built in exactly one place, `internal/tlsx`, which exposes
-- TCP Sink: Debugging interface
+`Server(opts)` for listeners and `Client(opts, host)` for dialers. See
-- HTTP Sink: Browser-based live monitoring
+[Security](security.md).
 
-### Protocol Support
+## Sessions
 
-- HTTP/1.1 and HTTP/2 for HTTP connections
+Each pipeline owns a `session.Manager`. Plugins receive a `session.Proxy`
-- Raw TCP with optional SCRAM authentication
+scoped to their instance id, so one plugin cannot see or remove another's
-- TLS 1.2/1.3 for HTTPS connections (HTTP only)
+sessions. A session records the remote address, creation and last-activity
-- Server-Sent Events for real-time streaming
+timestamps, and metadata — including `tls` and `tls_peer_cn` for TLS peers, and
+`auth_method` / `auth_identity` for authorized ones.
+
+Idle sessions are reaped every 5 minutes against a 30-minute idle limit. The
+HTTP sink's broker treats a vanished session as an eviction signal and closes
+the corresponding SSE client.
+
+Authorization decisions do not read session metadata — they are made from the
+handshake by `internal/authz`, at the point of connection or request, and their
+outcome is *recorded* in the session. That ordering matters: a session exists
+only for a peer that was already admitted. See the
+[mTLS authentication design](mtls-auth-plan.md).
+
+## Configuration Reload
+
+Reload (signal or file watch) rebuilds the entire service:
+
+1. Re-read the config through the config manager.
+2. Build a **new** service from it. If construction fails, the old service keeps
+   running untouched.
+3. Shut the old service down, start the new one, and restart the status
+   reporter if it is enabled.
+
+Because this is a full rebuild, listening sockets close and reopen and all
+clients are disconnected. Application logging is configured once at startup and
+is **not** re-applied on reload.
 
 ## Resource Management
 
-### Memory Management
+- Every buffer is bounded; the drop-not-block policy keeps memory flat under
+  load.
+- Network sinks and chain sources accept a `max_connections` cap. Admission is
+  a load-then-check, so a burst can over-admit by roughly one connection.
+- Chain listeners bound a single line at `core.MaxLogEntryBytes` (1 MiB); an
+  oversized line is a protocol violation and terminates the connection.
+- The `http_chain` source caps each request body at `max_body_bytes`.
+- File sinks rotate on size, cap total rotated size, and honour a retention
+  window.
 
-- Bounded buffers prevent unbounded growth
+## Performance Notes
-- Automatic garbage collection via Go runtime
-- Connection limits prevent resource exhaustion
 
-### File Management
+- In-memory entry processing is sub-millisecond; the formatter mutex is the
-
+  only shared serialization point in the hot path.
-- Automatic rotation based on size thresholds
+- File tailing detects new content within roughly 100 ms (fixed poll), while
-- Retention policies for old log files
+  `check_interval_ms` governs how quickly a *newly created* file is noticed.
-- Minimum disk space checks before writing
+- `http_chain` trades latency for efficiency: entries wait up to
-
+  `flush_interval_ms` (default 1 s) before a batch is sent.
-### Connection Management
+- Scale out with more pipelines per process, more sinks per pipeline, or more
-
+  nodes chained together.
-- Per-IP connection limits
-- Global connection caps
-- Automatic reconnection with exponential backoff
-- Keep-alive for persistent connections
-
-## Reliability Features
-
-### Fault Tolerance
-
-- Panic recovery in pipeline processing
-- Independent pipeline operation
-- Automatic source restart on failure
-- Sink failure isolation
-
-### Data Integrity
-
-- Entry validation at ingestion
-- Size limits for entries and batches
-- Duplicate detection in file monitoring
-- Position tracking for file reads
-
-## Performance Characteristics
-
-### Throughput
-
-- Pipeline rate limiting: Configurable (default 1000 entries/second)
-- Network throughput: Limited by network and sink capacity
-- File monitoring: Sub-second detection (default 100ms interval)
-
-### Latency
-
-- Entry processing: Sub-millisecond in-memory
-- Network forwarding: Depends on batch configuration
-- File detection: Configurable check interval
-
-### Scalability
-
-- Horizontal: Multiple LogWisp instances with different configurations
-- Vertical: Multiple pipelines per instance
-- Fan-out: Multiple sinks per pipeline
-- Fan-in: Multiple sources per pipeline

doc/authentication.md

@@ -1,237 +0,0 @@
-# Authentication
-
-LogWisp supports multiple authentication methods for securing network connections.
-
-## Authentication Methods
-
-### Overview
-
-| Method | HTTP Source | HTTP Sink | HTTP Client | TCP Source | TCP Client | TCP Sink |
-|--------|------------|-----------|-------------|------------|------------|----------|
-| None | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
-| Basic | ✓ (TLS req) | ✓ (TLS req) | ✓ (TLS req) | ✗ | ✗ | ✗ |
-| Token | ✓ (TLS req) | ✓ (TLS req) | ✓ (TLS req) | ✗ | ✗ | ✗ |
-| SCRAM | ✗ | ✗ | ✗ | ✓ | ✓ | ✗ |
-| mTLS | ✓ | ✓ | ✓ | ✗ | ✗ | ✗ |
-
-**Important Notes:**
-- HTTP authentication **requires** TLS to be enabled
-- TCP connections are **always** unencrypted
-- TCP Sink has **no** authentication (debugging only)
-
-## Basic Authentication
-
-HTTP/HTTPS connections with username/password.
-
-### Configuration
-
-```toml
-[pipelines.sources.http.auth]
-type = "basic"
-realm = "LogWisp"
-
-[[pipelines.sources.http.auth.basic.users]]
-username = "admin"
-password_hash = "$argon2id$v=19$m=65536,t=3,p=2$..."
-```
-
-### Generating Credentials
-
-Use the `auth` command:
-```bash
-logwisp auth -u admin -b
-```
-
-Output includes:
-- Argon2id password hash for configuration
-- TOML configuration snippet
-
-### Password Hash Format
-
-LogWisp uses Argon2id with parameters:
-- Memory: 65536 KB
-- Iterations: 3
-- Parallelism: 2
-- Salt: Random 16 bytes
-
-## Token Authentication
-
-Bearer token authentication for HTTP/HTTPS.
-
-### Configuration
-
-```toml
-[pipelines.sources.http.auth]
-type = "token"
-
-[pipelines.sources.http.auth.token]
-tokens = ["token1", "token2", "token3"]
-```
-
-### Generating Tokens
-
-```bash
-logwisp auth -k -l 32
-```
-
-Generates:
-- Base64-encoded token
-- Hex-encoded token
-- Configuration snippet
-
-### Token Usage
-
-Include in requests:
-```
-Authorization: Bearer <token>
-```
-
-## SCRAM Authentication
-
-Secure Challenge-Response for TCP connections.
-
-### Configuration
-
-```toml
-[pipelines.sources.tcp.auth]
-type = "scram"
-
-[[pipelines.sources.tcp.auth.scram.users]]
-username = "tcpuser"
-stored_key = "base64..."
-server_key = "base64..."
-salt = "base64..."
-argon_time = 3
-argon_memory = 65536
-argon_threads = 4
-```
-
-### Generating SCRAM Credentials
-
-```bash
-logwisp auth -u tcpuser -s
-```
-
-### SCRAM Features
-
-- Argon2-SCRAM-SHA256 algorithm
-- Challenge-response mechanism
-- No password transmission
-- Replay attack protection
-- Works over unencrypted connections
-
-## mTLS (Mutual TLS)
-
-Certificate-based authentication for HTTPS.
-
-### Server Configuration
-
-```toml
-[pipelines.sources.http.tls]
-enabled = true
-cert_file = "/path/to/server.pem"
-key_file = "/path/to/server.key"
-client_auth = true
-client_ca_file = "/path/to/ca.pem"
-verify_client_cert = true
-
-[pipelines.sources.http.auth]
-type = "mtls"
-```
-
-### Client Configuration
-
-```toml
-[pipelines.sinks.http_client.tls]
-enabled = true
-cert_file = "/path/to/client.pem"
-key_file = "/path/to/client.key"
-
-[pipelines.sinks.http_client.auth]
-type = "mtls"
-```
-
-### Certificate Generation
-
-Use the `tls` command:
-```bash
-# Generate CA
-logwisp tls -ca -o ca
-
-# Generate server certificate
-logwisp tls -server -ca-cert ca.pem -ca-key ca.key -host localhost -o server
-
-# Generate client certificate
-logwisp tls -client -ca-cert ca.pem -ca-key ca.key -o client
-```
-
-## Authentication Command
-
-### Usage
-
-```bash
-logwisp auth [options]
-```
-
-### Options
-
-| Flag | Description |
-|------|-------------|
-| `-u, --user` | Username for credential generation |
-| `-p, --password` | Password (prompts if not provided) |
-| `-b, --basic` | Generate basic auth (HTTP/HTTPS) |
-| `-s, --scram` | Generate SCRAM auth (TCP) |
-| `-k, --token` | Generate bearer token |
-| `-l, --length` | Token length in bytes (default: 32) |
-
-### Security Best Practices
-
-1. **Always use TLS** for HTTP authentication
-2. **Never hardcode passwords** in configuration
-3. **Use strong passwords** (minimum 12 characters)
-4. **Rotate tokens regularly**
-5. **Limit user permissions** to minimum required
-6. **Store password hashes only**, never plaintext
-7. **Use unique credentials** per service/user
-
-## Access Control Lists
-
-Combine authentication with IP-based access control:
-
-```toml
-[pipelines.sources.http.net_limit]
-enabled = true
-ip_whitelist = ["192.168.1.0/24", "10.0.0.0/8"]
-ip_blacklist = ["192.168.1.100"]
-```
-
-Priority order:
-1. Blacklist (checked first, immediate deny)
-2. Whitelist (if configured, must match)
-3. Authentication (if configured)
-
-## Credential Storage
-
-### Configuration File
-
-Store hashes in TOML:
-```toml
-[[pipelines.sources.http.auth.basic.users]]
-username = "admin"
-password_hash = "$argon2id$..."
-```
-
-### Environment Variables
-
-Override via environment:
-```bash
-export LOGWISP_PIPELINES_0_SOURCES_0_HTTP_AUTH_BASIC_USERS_0_USERNAME=admin
-export LOGWISP_PIPELINES_0_SOURCES_0_HTTP_AUTH_BASIC_USERS_0_PASSWORD_HASH='$argon2id$...'
-```
-
-### External Files
-
-Future support planned for:
-- External user databases
-- LDAP/AD integration
-- OAuth2/OIDC providers

doc/chaining.md

@@ -0,0 +1,236 @@
+# Chaining
+
+Chaining links LogWisp nodes together. An edge node forwards its entries to a
+relay or collector, which can filter, reformat, fan out, or forward them again.
+Unlike the `tcp` and `http` sinks — which emit *formatted text* for humans and
+generic clients — chain links carry the **structured entry**, so downstream
+nodes can filter and reformat as if the entries were local.
+
+## Topology
+
+```
+  edge-01                       relay                        consumers
+  ┌───────────────┐             ┌────────────────────┐       ┌──────────────┐
+  │ file source   │             │ tcp_chain source   │       │ browser (SSE)│
+  │      ↓        │  TCP/TLS    │        ↓           │  ───► │ nc / telnet  │
+  │ tcp_chain sink├────────────►│      flow          │       │ archive file │
+  └───────────────┘   :15801    │        ↓           │       └──────────────┘
+                                │ http sink, tcp sink│
+  edge-02                       │ file sink          │
+  ┌───────────────┐             │ http_chain sink ───┼──►  upstream collector
+  │ file source   │  HTTP/TLS   │                    │
+  │      ↓        ├────────────►│ http_chain source  │
+  │ http_chain sink│   :15802   └────────────────────┘
+  └───────────────┘
+```
+
+Both chain sources can feed a single pipeline (fan-in) whose sinks then fan the
+merged stream out. `test/chain-test.sh` builds the two-independent-pipelines
+variant; `test/chain-aggregate-test.sh` builds the fan-in variant.
+
+## Node Identity
+
+Chained entries carry a `node` label identifying where they originated.
+
+- A chain **sink** stamps `node` on any entry that does not already have one.
+  The label comes from the `node` option, defaulting to `os.Hostname()`.
+- A chain **source** either honours the sender's label or overrides it,
+  according to `trust_node`:
+
+| `trust_node` | Behaviour |
+|--------------|-----------|
+| `true` (default) | Keep the label the sender declared; fall back to the remote address when absent |
+| `false` | Always overwrite with the sender's remote address |
+
+Relays preserve `node`, so a label survives any number of hops and identifies
+the original producer rather than the last relay.
+
+Under mTLS the source can instead bind the label to the sender's certificate,
+which overrides `trust_node` entirely:
+
+| `auth.node_binding` | Connection label | Per-entry `node` field |
+|---------------------|------------------|------------------------|
+| `none` | `trust_node` governs | `trust_node` governs |
+| `assert` | Must equal the certificate identity, or the peer is rejected | `trust_node` governs |
+| `force` (default under `mtls`) | The certificate identity | Overwritten with the identity |
+
+Pick `force` at an ingest boundary you do not trust — it is the only setting
+where a compromised edge cannot mislabel its entries, including through the
+per-entry `node` field. Pick `assert` on a relay-to-relay hop, where the relay
+should prove its own identity but the origin labels it forwards must survive.
+See [Security](security.md#node-binding).
+
+Formatters render node identity as a syslog-style prefix on the source field:
+`edge-01/app.log`. In JSON output the node therefore appears inside the source
+field, not as a separate top-level key.
+
+> `trust_node = true` with no `auth` block means any peer the CA vouches for can
+> claim **any** node label, including one belonging to another host. On an
+> untrusted network set `auth.type = "mtls"` with `node_binding = "force"`;
+> `trust_node = false` is the fallback when certificates are not an option.
+
+## Wire Protocol
+
+Protocol version: **1**. Both transports carry the same canonical entry
+encoding, and differ only in how the preamble and framing are expressed.
+
+### TCP transport
+
+A persistent connection carrying newline-delimited JSON.
+
+1. The dialer connects and, under TLS, completes the handshake.
+2. The dialer immediately writes the hello preamble as one JSON line:
+
+   ```json
+   {"logwisp":1,"node":"edge-01"}
+   ```
+
+3. The listener reads that line within `hello_timeout_ms` and rejects the
+   connection if it is malformed or declares a different protocol version.
+4. Every subsequent line is one JSON-encoded `LogEntry`.
+
+Line size is bounded at 1 MiB. An oversized line is a protocol violation and
+terminates the connection, because the scanner cannot resynchronize afterwards.
+
+### HTTP transport
+
+Batches of NDJSON delivered by `POST`, with the preamble expressed as headers.
+
+| Header | Direction | Meaning |
+|--------|-----------|---------|
+| `X-Logwisp-Protocol` | request | Protocol version; must be `1` |
+| `X-Logwisp-Node` | request | Origin node label |
+| `Content-Type` | request | `application/x-ndjson` |
+| `X-Logwisp-Accepted` | response | Number of entries ingested |
+
+Responses: `204` on success, `400` for a bad protocol version or a malformed
+body, `413` when the body cap is exceeded, `405` for a non-`POST` method.
+
+### Entry encoding
+
+```json
+{
+  "time": "2026-01-02T15:04:05.123456789Z",
+  "node": "edge-01",
+  "source": "app.log",
+  "level": "ERROR",
+  "message": "connection refused",
+  "fields": {"attempt": 3}
+}
+```
+
+`node`, `level`, and `fields` are omitted when empty. A missing `time` is filled
+in at ingest.
+
+## Delivery Semantics
+
+| Transport | Guarantee | Failure behaviour |
+|-----------|-----------|-------------------|
+| `tcp_chain` | Per-line, held across reconnects | Retries with exponential backoff plus ±20 % jitter until written or shutdown; back-pressure appears upstream as `total_dropped_by_sink` |
+| `http_chain` | At-least-once per batch | Retries transport errors, `408`, `429`, `5xx`; drops on any other non-2xx (`dropped_batches`) |
+
+`http_chain` batches can be delivered twice when a successful request's response
+is lost. There is no de-duplication downstream; design your consumers to
+tolerate it, or use `tcp_chain` where each line is written once per successful
+write.
+
+Neither transport persists anything to disk. Entries buffered in memory during
+an outage are lost if the process exits.
+
+## Worked Example
+
+**Edge node** — tail files, forward over mTLS:
+
+```toml
+[[pipelines]]
+name = "edge"
+
+[[pipelines.plugin_sources]]
+id = "app"
+type = "file"
+[pipelines.plugin_sources.config]
+directory = "/var/log/myapp"
+pattern = "*.log"
+
+[[pipelines.plugin_sinks]]
+id = "forward"
+type = "tcp_chain"
+[pipelines.plugin_sinks.config]
+host = "relay.internal"
+port = 15801
+node = "edge-01"
+[pipelines.plugin_sinks.config.tls]
+enabled   = true
+ca_file   = "/etc/logwisp/tls/ca.crt"
+cert_file = "/etc/logwisp/tls/edge-01.crt"
+key_file  = "/etc/logwisp/tls/edge-01.key"
+[pipelines.plugin_sinks.config.auth]
+type  = "mtls"
+allow = ["relay.internal"]      # pin the relay, not just its hostname
+```
+
+**Relay** — ingest, keep errors only, archive and stream:
+
+```toml
+[[pipelines]]
+name = "relay"
+
+[[pipelines.flow.filters]]
+type = "include"
+patterns = ["ERROR", "FATAL"]
+
+[pipelines.flow.format]
+type = "json"
+sanitizer_policy = "json"
+
+[[pipelines.plugin_sources]]
+id = "ingest"
+type = "tcp_chain"
+[pipelines.plugin_sources.config]
+host = "0.0.0.0"
+port = 15801
+[pipelines.plugin_sources.config.tls]
+enabled        = true
+cert_file      = "/etc/logwisp/tls/relay.crt"
+key_file       = "/etc/logwisp/tls/relay.key"
+client_auth    = true
+client_ca_file = "/etc/logwisp/tls/ca.crt"
+[pipelines.plugin_sources.config.auth]
+type         = "mtls"
+allow        = ["edge-01", "edge-02"]
+node_binding = "force"          # entries are labelled from the certificate
+
+[[pipelines.plugin_sinks]]
+id = "archive"
+type = "file"
+[pipelines.plugin_sinks.config]
+directory = "/var/log/logwisp"
+name = "errors"
+
+[[pipelines.plugin_sinks]]
+id = "live"
+type = "http"
+[pipelines.plugin_sinks.config]
+host = "127.0.0.1"
+port = 8080
+```
+
+Entries arriving on this relay are labelled `edge-01` or `edge-02` because that
+is what their certificates say, regardless of the `node` each edge configured.
+`test/mtls-chain-test.sh` builds exactly this shape against a throwaway PKI.
+
+## Operational Notes
+
+- **Formatting is a relay decision.** Because chain links carry structured
+  entries, the edge node's `flow.format` affects only its own local sinks. Set
+  the output shape on the node that owns the human-facing sink.
+- **Filtering early saves bandwidth.** A filter on the edge drops entries before
+  they cross the network; a filter on the relay is easier to change centrally.
+- **Rate limits are per pipeline.** An edge limit protects the link; a relay
+  limit protects the relay from a noisy edge.
+- **Heartbeats traverse chain links** as ordinary structured entries and keep
+  otherwise-idle links and their sessions warm.
+- **Ports** used by the bundled test scripts: `15801` tcp_chain ingest, `15802`
+  http_chain ingest, `15803` tcp sink, `15804` http sink.
+- **Use `127.0.0.1`, not `localhost`**, when testing locally: all listeners and
+  dialers are IPv4-only, and `localhost` may resolve to `::1`.

doc/cli.md

@@ -1,260 +1,184 @@
 # Command Line Interface
 
-LogWisp CLI reference for commands and options.
+```
-
-## Synopsis
-
-```bash
-logwisp [command] [options]
 logwisp [options]
-```
+logwisp help | -h | --help
-
-## Commands
-
-### Main Commands
-
-| Command | Description |
-|---------|-------------|
-| `auth` | Generate authentication credentials |
-| `tls` | Generate TLS certificates |
-| `version` | Display version information |
-| `help` | Show help information |
-
-### auth Command
-
-Generate authentication credentials.
-
-```bash
-logwisp auth [options]
-```
-
-**Options:**
-
-| Flag | Description | Default |
-|------|-------------|---------|
-| `-u, --user` | Username | Required for password auth |
-| `-p, --password` | Password | Prompts if not provided |
-| `-b, --basic` | Generate basic auth | - |
-| `-s, --scram` | Generate SCRAM auth | - |
-| `-k, --token` | Generate bearer token | - |
-| `-l, --length` | Token length in bytes | 32 |
-
-### tls Command
-
-Generate TLS certificates.
-
-```bash
-logwisp tls [options]
-```
-
-**Options:**
-
-| Flag | Description | Default |
-|------|-------------|---------|
-| `-ca` | Generate CA certificate | - |
-| `-server` | Generate server certificate | - |
-| `-client` | Generate client certificate | - |
-| `-host` | Comma-separated hosts/IPs | localhost |
-| `-o` | Output file prefix | Required |
-| `-ca-cert` | CA certificate file | Required for server/client |
-| `-ca-key` | CA key file | Required for server/client |
-| `-days` | Certificate validity days | 365 |
-
-### version Command
-
-Display version information.
-
-```bash
-logwisp version
-logwisp -v
 logwisp --version
 ```
 
-Output includes:
+LogWisp has no subcommands. Earlier releases shipped `logwisp auth` and
-- Version number
+`logwisp tls` for credential and certificate generation; both were removed
-- Build date
+during the restructure. Use `openssl` or your PKI tooling instead — see
-- Git commit hash
+[Security](security.md).
-- Go version
 
-## Global Options
+## Options
 
-### Configuration Options
+Any scalar configuration key is settable as a flag using its TOML path:
+
+```
+--<path>=<value>        e.g. --logging.level=debug
+--<path> <value>        e.g. --logging.level debug
+--<path>                bare flag, means true
+```
+
+### Common
 
 | Flag | Description | Default |
 |------|-------------|---------|
-| `-c, --config` | Configuration file path | `./logwisp.toml` |
+| `-c <path>` | Configuration file | `./logwisp.toml` |
-| `-b, --background` | Run as daemon | false |
+| `--config=<path>` | Configuration file (equals form only) | `./logwisp.toml` |
-| `-q, --quiet` | Suppress console output | false |
+| `--quiet` | Suppress all application output | `false` |
-| `--disable-status-reporter` | Disable status logging | false |
+| `--status_reporter=<bool>` | Periodic status logging | `true` |
-| `--config-auto-reload` | Enable config hot reload | false |
+| `--auto_reload=<bool>` | Reload config when the file changes | `false` |
+| `--version` | Print version and exit | — |
+| `-h`, `--help`, `help` | Print usage and exit | — |
 
-### Logging Options
+> `--config <path>` with a space is not recognized. The path resolver
+> understands `-c <path>` and `--config=<path>` only; the space form is treated
+> as an unknown flag, warned about, and ignored, after which LogWisp silently
+> falls back to `./logwisp.toml`.
+>
+> `-c` as the final argument, with no path after it, crashes with an index
+> panic rather than reporting a usage error.
 
-| Flag | Description | Values |
+### Logging
-|------|-------------|--------|
-| `--logging.output` | Log output mode | file, stdout, stderr, split, all, none |
-| `--logging.level` | Log level | debug, info, warn, error |
-| `--logging.file.directory` | Log directory | Path |
-| `--logging.file.name` | Log filename | String |
-| `--logging.file.max_size_mb` | Max file size | Integer |
-| `--logging.file.max_total_size_mb` | Total size limit | Integer |
-| `--logging.file.retention_hours` | Retention period | Float |
-| `--logging.console.target` | Console target | stdout, stderr, split |
-| `--logging.console.format` | Output format | txt, json |
 
-### Pipeline Options
+| Flag | Values |
+|------|--------|
+| `--logging.output` | `file`, `stdout`, `stderr`, `split`, `all`, `none` |
+| `--logging.level` | `debug`, `info`, `warn`, `error` |
+| `--logging.format` | `raw`, `txt`, `json` |
+| `--logging.sanitization` | `raw`, `json`, `txt`, `shell` |
+| `--logging.file.directory` | path |
+| `--logging.file.name` | string |
+| `--logging.file.max_size_mb` | integer |
+| `--logging.file.max_total_size_mb` | integer |
+| `--logging.file.retention_hours` | float |
 
-Configure pipelines via CLI (N = array index, 0-based).
+`--logging.console.target` is accepted but has no effect; the console
+destination is derived from `--logging.output`.
 
-**Pipeline Configuration:**
+### Pipelines
 
-| Flag | Description |
+Pipelines, sources, sinks, and filters **cannot** be configured from the command
-|------|-------------|
+line. Array-indexed paths such as `--pipelines.0.name=app` or
-| `--pipelines.N.name` | Pipeline name |
+`--pipelines.0.plugin_sinks.0.type=null` are reported as unrecognized and
-| `--pipelines.N.sources.N.type` | Source type |
+ignored:
-| `--pipelines.N.filters.N.type` | Filter type |
-| `--pipelines.N.sinks.N.type` | Sink type |
 
-## Flag Formats
+```
-
+Warning: unrecognized flags ignored: [pipelines.0.name]
-### Boolean Flags
-
-```bash
-logwisp --quiet
-logwisp --quiet=true
-logwisp --quiet=false
 ```
 
-### String Flags
+Use a configuration file. Older documentation described CLI pipeline overrides
-
+that the current loader does not implement.
-```bash
-logwisp --config /etc/logwisp/config.toml
-logwisp -c config.toml
-```
-
-### Nested Configuration
-
-```bash
-logwisp --logging.level=debug
-logwisp --pipelines.0.name=myapp
-logwisp --pipelines.0.sources.0.type=stdin
-```
-
-### Array Values (JSON)
-
-```bash
-logwisp --pipelines.0.filters.0.patterns='["ERROR","WARN"]'
-```
 
 ## Environment Variables
 
-All flags can be set via environment:
+Configuration paths map to environment variables by replacing `.` with `_` and
+uppercasing:
 
 ```bash
-export LOGWISP_QUIET=true
+export QUIET=true
-export LOGWISP_LOGGING_LEVEL=debug
+export LOGGING_LEVEL=debug
-export LOGWISP_PIPELINES_0_NAME=myapp
+export LOGGING_FILE_DIRECTORY=/var/log/logwisp
 ```
 
-## Configuration Precedence
+> The `LOGWISP_` prefix is **not** currently applied to these — see
+> [Configuration](configuration.md#environment-variables). Bare names like
+> `QUIET` are what LogWisp actually reads, which is worth knowing both to make
+> overrides work and to avoid accidental collisions.
 
-1. Command-line flags (highest)
+The two variables that do carry the prefix are read directly by the path
+resolver:
+
+| Variable | Effect |
+|----------|--------|
+| `LOGWISP_CONFIG_FILE` | Configuration file path; joined onto `LOGWISP_CONFIG_DIR` when both are set |
+| `LOGWISP_CONFIG_DIR` | Configuration directory; alone, implies `<dir>/logwisp.toml` |
+
+As with flags, array elements cannot be set this way.
+
+## Precedence
+
+1. Command-line flags
 2. Environment variables
 3. Configuration file
-4. Built-in defaults (lowest)
+4. Built-in defaults
 
-## Exit Codes
+## Signals
-
-| Code | Description |
-|------|-------------|
-| 0 | Success |
-| 1 | General error |
-| 2 | Configuration file not found |
-| 137 | SIGKILL received |
-
-## Signal Handling
 
 | Signal | Action |
 |--------|--------|
-| SIGINT (Ctrl+C) | Graceful shutdown |
+| `SIGINT` | Graceful shutdown |
-| SIGTERM | Graceful shutdown |
+| `SIGTERM` | Graceful shutdown |
-| SIGHUP | Reload configuration |
+| `SIGHUP` | Reload configuration |
-| SIGUSR1 | Reload configuration |
+| `SIGUSR1` | Reload configuration |
-| SIGKILL | Immediate termination |
+
+`SIGHUP` is ignored during startup, before the signal handler is installed, so
+LogWisp survives a terminal hang-up like `nohup`. Once running, it triggers a
+reload rather than terminating.
+
+Reload rebuilds the whole service. A configuration error leaves the running
+service untouched; see [Configuration](configuration.md#hot-reload).
+
+## Exit Codes
+
+| Code | Meaning |
+|------|---------|
+| `0` | Clean shutdown, or `--version` / `--help` |
+| `1` | General error: config load or validation failure, logger init failure, service bootstrap failure |
+| `2` | Explicitly requested configuration file not found |
+
+Exit code 2 applies only when the file was named explicitly (`-c`,
+`--config=`, or the `LOGWISP_CONFIG_*` variables). A missing discovered default
+is not an error, and LogWisp starts on built-in defaults.
+
+## Built-in Defaults
+
+With no configuration file present, LogWisp runs one pipeline named
+`default_pipeline`: a `random` source with `special = true`, JSON formatting,
+a rate limit of 5 entries/second with a burst of 10 and `policy = "drop"`, and a
+`console` sink on stdout. It is a self-demonstrating idle mode, not a useful
+production configuration.
+
+Note that as soon as your file defines `[[pipelines]]`, that entire default
+pipeline — rate limit included — is replaced rather than merged.
 
 ## Usage Patterns
 
-### Development Mode
+**Development**
 
 ```bash
-# Verbose logging to console
+# verbose, everything to stderr
-logwisp --logging.output=stderr --logging.level=debug
+logwisp -c dev.toml --logging.output=stderr --logging.level=debug
 
-# Quick test with stdin
+# no config at all: synthetic generator to stdout
-logwisp --pipelines.0.sources.0.type=stdin --pipelines.0.sinks.0.type=console
+logwisp
 ```
 
-### Production Deployment
+**Configuration check**
 
 ```bash
-# Background with file logging
+# starts the service; a config error exits non-zero before any pipeline runs
-logwisp --background --config /etc/logwisp/prod.toml --logging.output=file
+logwisp -c /etc/logwisp/logwisp.toml --logging.level=debug
-
-# Systemd service
-ExecStart=/usr/local/bin/logwisp --config /etc/logwisp/config.toml
 ```
 
-### Debugging
+There is no dry-run or validate-only mode. The closest approximation is starting
+with debug logging and stopping once the pipelines report as started.
+
+**Production**
 
 ```bash
-# Check configuration
+logwisp -c /etc/logwisp/logwisp.toml --logging.output=file
-logwisp --config test.toml --logging.level=debug --disable-status-reporter
-
-# Dry run (verify config only)
-logwisp --config test.toml --quiet
 ```
 
-### Quick Commands
+Run under a supervisor (systemd, rc.d) rather than backgrounding it — there is
+no `--background` flag; earlier releases had one and it was removed. See
+[Installation](installation.md).
+
+**Reload**
 
 ```bash
-# Generate admin password
+kill -HUP  $(pidof logwisp)
-logwisp auth -u admin -b
+kill -USR1 $(pidof logwisp)
-
-# Create self-signed certs
-logwisp tls -server -host localhost -o server
-
-# Check version
-logwisp version
 ```
-
-## Help System
-
-### General Help
-
-```bash
-logwisp --help
-logwisp -h
-logwisp help
-```
-
-### Command Help
-
-```bash
-logwisp auth --help
-logwisp tls --help
-logwisp help auth
-```
-
-## Special Flags
-
-### Internal Flags
-
-These flags are for internal use:
-- `--background-daemon`: Child process indicator
-- `--config-save-on-exit`: Save config on shutdown
-
-### Hidden Behaviors
-
-- SIGHUP ignored by default (nohup behavior)
-- Automatic panic recovery in pipelines
-- Resource cleanup on shutdown

doc/configuration.md

@@ -1,198 +1,285 @@
 # Configuration Reference
 
-LogWisp configuration uses TOML format with flexible override mechanisms.
+LogWisp is configured with TOML. A complete annotated file listing every option
+and its default ships as [`config/logwisp.toml`](../config/logwisp.toml).
 
 ## Configuration Precedence
 
-Configuration sources are evaluated in order:
+Sources are merged in this order, highest priority first:
-1. **Command-line flags** (highest priority)
+
-2. **Environment variables**
+1. Command-line flags
-3. **Configuration file**
+2. Environment variables
-4. **Built-in defaults** (lowest priority)
+3. Configuration file
+4. Built-in defaults
+
+The `pipelines` array is replaced wholesale, not merged: as soon as your file
+defines `[[pipelines]]`, the built-in default pipeline (and its default rate
+limit and formatter) disappears entirely.
 
 ## File Location
 
-LogWisp searches for configuration in order:
+The path is resolved before any other configuration is read:
-1. Path specified via `--config` flag
+
-2. Path from `LOGWISP_CONFIG_FILE` environment variable
+1. `-c <path>` on the command line
-3. `~/.config/logwisp/logwisp.toml`
+2. `--config=<path>` on the command line
-4. `./logwisp.toml` in current directory
+3. `$LOGWISP_CONFIG_FILE`, joined onto `$LOGWISP_CONFIG_DIR` when both are set
+4. `$LOGWISP_CONFIG_DIR/logwisp.toml`
+5. `~/.config/logwisp/logwisp.toml`, if it exists
+6. `./logwisp.toml`
+
+Missing file behaviour differs by how it was chosen. An explicitly requested
+file that does not exist is a fatal error (exit code 2); a missing discovered
+default is not an error, and LogWisp starts on built-in defaults.
+
+> `--config <path>` with a space is **not** recognized as a config path. It is
+> parsed as an unknown flag, warned about, and ignored — LogWisp then silently
+> falls back to `./logwisp.toml`. Use `-c <path>` or `--config=<path>`.
 
 ## Global Settings
 
-Top-level configuration options:
-
 | Setting | Type | Default | Description |
 |---------|------|---------|-------------|
-| `background` | bool | false | Run as daemon process |
+| `quiet` | bool | `false` | Disable all application logging and console diagnostics |
-| `quiet` | bool | false | Suppress console output |
+| `status_reporter` | bool | `true` | Emit a periodic status report every 30 s at DEBUG level |
-| `disable_status_reporter` | bool | false | Disable periodic status logging |
+| `auto_reload` | bool | `false` | Watch the config file and reload pipelines on change |
-| `config_auto_reload` | bool | false | Enable file watch for auto-reload |
 
-## Logging Configuration
+`--version` prints version information and exits; it is not a persistent
+setting.
 
-LogWisp's internal operational logging:
+Note that `status_reporter` writes at DEBUG level, so it produces nothing unless
+`logging.level = "debug"`.
+
+## Application Logging
+
+This configures LogWisp's own operational log, not the log data it transports.
 
 ```toml
 [logging]
-output = "stdout"  # file|stdout|stderr|split|all|none
+output = "stdout"      # file | stdout | stderr | split | all | none
-level = "info"     # debug|info|warn|error
+level  = "info"        # debug | info | warn | error
+format = "txt"         # raw | txt | json
+# sanitization = ""    # raw | json | txt | shell
 
 [logging.file]
-directory = "./log"
+directory         = "./log"
-name = "logwisp"
+name              = "logwisp"
-max_size_mb = 100
+max_size_mb       = 100
 max_total_size_mb = 1000
-retention_hours = 168.0
+retention_hours   = 168.0
-
-[logging.console]
-target = "stdout"  # stdout|stderr|split
-format = "txt"     # txt|json
 ```
 
-### Output Modes
+### Output modes
 
-- **file**: Write to log files only
+| Mode | Behaviour |
-- **stdout**: Write to standard output
+|------|-----------|
-- **stderr**: Write to standard error
+| `file` | Files only |
-- **split**: INFO/DEBUG to stdout, WARN/ERROR to stderr
+| `stdout` | Standard output only |
-- **all**: Write to both file and console
+| `stderr` | Standard error only |
-- **none**: Disable all logging
+| `split` | DEBUG/INFO to stdout, WARN/ERROR to stderr |
+| `all` | Files plus split console |
+| `none` | No application logging |
+
+`[logging.file]` applies only to the `file` and `all` modes.
+
+> `[logging.console].target` is accepted and validated (`stdout`, `stderr`,
+> `split`) but **not applied**. The console destination is derived from
+> `logging.output`. The key is retained for compatibility; setting it has no
+> effect.
+
+`quiet = true` overrides every logging setting and disables both file and
+console output.
 
 ## Pipeline Configuration
 
-Each `[[pipelines]]` section defines an independent processing pipeline:
-
 ```toml
 [[pipelines]]
-name = "pipeline-name"
+name = "app"                       # required, unique across pipelines
 
-# Rate limiting (optional)
+# --- flow: everything between sources and sinks ---
-[pipelines.rate_limit]
+[pipelines.flow.rate_limit]
-rate = 1000.0
+rate                 = 1000.0
-burst = 2000.0
+burst                = 2000.0
-policy = "drop"  # pass|drop
+policy               = "drop"
-max_entry_size_bytes = 0  # 0=unlimited
+max_entry_size_bytes = 65536
 
-# Format configuration (optional)
+[[pipelines.flow.filters]]
-[pipelines.format]
+type     = "include"
-type = "json"  # raw|json|txt
+logic    = "or"
-
-# Sources (required, 1+)
-[[pipelines.sources]]
-type = "directory"
-# ... source-specific config
-
-# Filters (optional)
-[[pipelines.filters]]
-type = "include"
-logic = "or"
 patterns = ["ERROR", "WARN"]
 
-# Sinks (required, 1+)
+[pipelines.flow.format]
-[[pipelines.sinks]]
+type             = "json"
+sanitizer_policy = "json"
+
+[pipelines.flow.heartbeat]
+enabled     = true
+interval_ms = 30000
+
+# --- sources: one or more ---
+[[pipelines.plugin_sources]]
+id   = "app_logs"                  # unique within the pipeline
+type = "file"
+[pipelines.plugin_sources.config]
+directory = "/var/log/myapp"
+
+# --- sinks: one or more ---
+[[pipelines.plugin_sinks]]
+id   = "sse"
 type = "http"
-# ... sink-specific config
+[pipelines.plugin_sinks.config]
+port = 8080
 ```
 
+Every source and sink is a plugin instance with three keys:
+
+| Key | Meaning |
+|-----|---------|
+| `id` | Instance identifier, unique within the pipeline; appears in logs and stats |
+| `type` | Registered plugin type |
+| `config` | Plugin-specific table; see [Sources](sources.md) and [Sinks](sinks.md) |
+
+`config_file` is reserved on both structures for a future include mechanism and
+is not implemented.
+
+### Flow stages
+
+| Block | Optional | Reference |
+|-------|----------|-----------|
+| `flow.rate_limit` | yes | below |
+| `flow.filters` | yes | [Filters](filters.md) |
+| `flow.format` | yes (defaults to `raw`) | [Formatters](formatters.md) |
+| `flow.heartbeat` | yes | below |
+
+#### Rate limiting
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `rate` | float | `0` | Entries per second; `<= 0` disables the limiter entirely |
+| `burst` | float | `rate` | Token bucket capacity |
+| `policy` | string | `pass` | `pass` allows everything through, `drop` discards over-limit entries |
+| `max_entry_size_bytes` | int | `0` | Per-entry byte cap; `0` = unlimited |
+
+Two behaviours are easy to trip over:
+
+- The limiter is constructed only when `rate > 0`. With `rate = 0`,
+  `max_entry_size_bytes` is never enforced.
+- `policy = "pass"` short-circuits the whole check, including the size cap.
+  To enforce a size cap you need `rate > 0` **and** `policy = "drop"`.
+
+#### Heartbeat
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `enabled` | bool | `false` | Enable heartbeat generation |
+| `interval_ms` | int | `1000` | Interval; minimum `100` |
+| `include_timestamp` | bool | `false` | `false` formats with level only, no timestamp |
+| `include_stats` | bool | `false` | Attach `beat_count` and measured `interval_ms` as fields |
+| `format` | string | `txt` | `txt`, `json`, or `raw` |
+
+Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They
+are generated after the flow's filter and rate-limit stages, so filters do not
+suppress them, and they reach every sink in the pipeline.
+
+> `format = "comment"` (SSE comment framing) appears in older documentation and
+> in a code path in the generator, but the validator rejects it and the pipeline
+> fails to start. Use `txt`, `json`, or `raw`.
+
 ## Environment Variables
 
-All configuration options support environment variable overrides:
+Environment overrides are derived from the TOML path: `.` becomes `_` and the
+result is uppercased.
 
-### Naming Convention
+| TOML path | Environment variable |
-
-- Prefix: `LOGWISP_`
-- Path separator: `_` (underscore)
-- Array indices: Numeric suffix (0-based)
-- Case: UPPERCASE
-
-### Mapping Examples
-
-| TOML Path | Environment Variable |
 |-----------|---------------------|
-| `quiet` | `LOGWISP_QUIET` |
+| `quiet` | `QUIET` |
-| `logging.level` | `LOGWISP_LOGGING_LEVEL` |
+| `status_reporter` | `STATUS_REPORTER` |
-| `pipelines[0].name` | `LOGWISP_PIPELINES_0_NAME` |
+| `logging.level` | `LOGGING_LEVEL` |
-| `pipelines[0].sources[0].type` | `LOGWISP_PIPELINES_0_SOURCES_0_TYPE` |
+| `logging.file.directory` | `LOGGING_FILE_DIRECTORY` |
+
+> **The `LOGWISP_` prefix is not currently applied.** The configuration loader
+> requests it, but supplying a custom path-to-variable transform replaces the
+> prefixing step rather than composing with it, so LogWisp reads bare
+> `QUIET`, `LOGGING_LEVEL`, and so on from the environment. Treat this as
+> current behaviour to be aware of — bare names like `QUIET` can collide with
+> unrelated variables — rather than as intended design.
+>
+> The two exceptions are `LOGWISP_CONFIG_FILE` and `LOGWISP_CONFIG_DIR`, which
+> are read directly by the path resolver and **do** carry the prefix.
+
+Only scalar paths that exist in the configuration schema can be set this way.
+Array elements cannot: `PIPELINES_0_NAME` has no effect.
 
 ## Command-Line Overrides
 
-All configuration options can be overridden via CLI flags:
+Any scalar configuration path is settable as a flag using its TOML path:
 
 ```bash
-logwisp --quiet \
+logwisp --logging.level=debug --status_reporter=false
-  --logging.level=debug \
+logwisp --logging.level debug          # space form also works
-  --pipelines.0.name=myapp \
+logwisp --quiet                        # bare flag means true
-  --pipelines.0.sources.0.type=stdin
 ```
 
-## Configuration Validation
+Unrecognized flags are reported on stderr before the logger exists and are then
+ignored:
 
-LogWisp validates configuration at startup:
+```
-- Required fields presence
+Warning: unrecognized flags ignored: [pipelines.0.name]
-- Type correctness
+```
-- Port conflicts
+
-- Path accessibility
+> Array-indexed paths are **not** settable from the command line.
-- Pattern compilation
+> `--pipelines.0.name=x`, `--pipelines.0.plugin_sinks.0.type=null`, and similar
-- Network address formats
+> flags are reported as unrecognized and ignored. Pipelines, sources, sinks, and
+> filters can only be defined in the configuration file. Older documentation
+> claimed otherwise.
+
+## Validation
+
+Startup validation is intentionally split.
+
+`internal/config` validates only global structure:
+
+- at least one pipeline
+- unique, non-empty pipeline names
+- at least one source and one sink per pipeline
+- `logging.output`, `logging.level`, `logging.format`, `logging.sanitization`,
+  and `logging.console.target` enum membership
+
+Everything else is validated by the plugin constructor that owns it — port
+range, required paths, path prefixes, enum values, regex compilation, TLS file
+loading. A failure there aborts pipeline construction with a message naming the
+pipeline, plugin id, and offending key.
+
+There is **no** cross-pipeline port-conflict detection. Two sinks bound to the
+same port fail at listener bind time, when the pipeline starts.
 
 ## Hot Reload
 
-Enable configuration hot reload:
-
 ```toml
-config_auto_reload = true
+auto_reload = true
 ```
 
-Or via command line:
+or send `SIGHUP` / `SIGUSR1`.
-```bash
-logwisp --config-auto-reload
-```
 
-Reload triggers:
+Reload rebuilds the whole service: a new service is constructed from the new
-- File modification detection
+configuration first, and only if that succeeds is the old one shut down. A
-- SIGHUP or SIGUSR1 signals
+configuration error therefore leaves the running service untouched.
 
-Reloadable items:
+| Reloaded | Not reloaded |
-- Pipeline configurations
+|----------|--------------|
-- Sources and sinks
+| Pipelines, sources, sinks | `logging.*` (applied once at startup) |
-- Filters and formatters
+| Filters, formatters, rate limits, heartbeats | `quiet` |
-- Rate limits
+| `status_reporter` | `auto_reload` (the watcher is not restarted) |
 
-Non-reloadable (requires restart):
+Because the rebuild is total, listeners close and reopen and every connected
-- Logging configuration
+client is disconnected. Chain sinks reconnect on their own backoff schedule.
-- Background mode
-- Global settings
 
-## Default Configuration
+## Type Reference
 
-Minimal working configuration:
+| TOML type | Go type | Command-line / environment form |
-
+|-----------|---------|-------------------------------|
-```toml
+| String | `string` | Plain text |
-[[pipelines]]
+| Integer | `int64` | Decimal string |
-name = "default"
+| Float | `float64` | Decimal string |
-
+| Boolean | `bool` | `true` / `false`, or a bare flag for `true` |
-[[pipelines.sources]]
+| Array | `[]T` | Not settable outside the file |
-type = "directory"
+| Table | struct | Nested path with `.` (flags) or `_` (environment) |
-[pipelines.sources.directory]
-path = "./"
-pattern = "*.log"
-
-[[pipelines.sinks]]
-type = "console"
-[pipelines.sinks.console]
-target = "stdout"
-```
-
-## Configuration Schema
-
-### Type Reference
-
-| TOML Type | Go Type | Environment Format |
-|-----------|---------|-------------------|
-| String | string | Plain text |
-| Integer | int64 | Numeric string |
-| Float | float64 | Decimal string |
-| Boolean | bool | true/false |
-| Array | []T | JSON array string |
-| Table | struct | Nested with `_` |

doc/filters.md

@@ -1,185 +1,192 @@
 # Filters
 
-LogWisp filters control which log entries pass through the pipeline using pattern matching.
+Filters decide which entries continue through a pipeline. They run in the flow,
-
+after rate limiting and before formatting.
-## Filter Types
-
-### Include Filter
-
-Only entries matching patterns pass through.
 
 ```toml
-[[pipelines.filters]]
+[[pipelines.flow.filters]]
-type = "include"
+type     = "include"
-logic = "or"  # or|and
+logic    = "or"
-patterns = [
+patterns = ["ERROR", "WARN"]
-    "ERROR",
-    "WARN",
-    "CRITICAL"
-]
 ```
 
-### Exclude Filter
+## Options
-
-Entries matching patterns are dropped.
-
-```toml
-[[pipelines.filters]]
-type = "exclude"
-patterns = [
-    "DEBUG",
-    "TRACE",
-    "health-check"
-]
-```
-
-## Configuration Options
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `type` | string | Required | Filter type (include/exclude) |
+| `type` | string | `include` | `include` (only matches pass) or `exclude` (matches are dropped) |
-| `logic` | string | "or" | Pattern matching logic (or/and) |
+| `logic` | string | `or` | `or` (any pattern matches) or `and` (every pattern matches) |
-| `patterns` | []string | Required | Pattern list |
+| `patterns` | []string | `[]` | Go RE2 regular expressions |
+
+A filter with no patterns passes everything. Invalid patterns fail at startup
+with the filter index and the offending pattern in the message.
+
+## What Gets Matched
+
+Patterns are matched against a single string assembled from the entry:
+
+```
+"<source> <level> <message>"
+```
+
+Empty parts are omitted, so an entry with no detected level matches
+`"<source> <message>"`. This means a pattern can target the source name or the
+level as easily as the message body:
+
+| Pattern | Matches |
+|---------|---------|
+| `"^app\\.log "` | Entries whose source is `app.log` |
+| `"ERROR"` | Level `ERROR`, or the word `ERROR` anywhere in the message |
+
+The structured `fields` payload is **not** part of the match text.
+
+For entries that arrived over a chain link, the `source` used here is the bare
+source — the `node/source` prefix is applied later, by the formatter — so
+filtering by originating node requires matching on the message, or filtering on
+the node that produces the entries.
+
+## Filter Types
+
+### include
+
+Only matching entries pass. Everything else is dropped.
+
+```toml
+[[pipelines.flow.filters]]
+type     = "include"
+patterns = ["ERROR", "WARN", "FATAL"]
+```
+
+### exclude
+
+Matching entries are dropped. Everything else passes.
+
+```toml
+[[pipelines.flow.filters]]
+type     = "exclude"
+patterns = ["/healthz", "TRACE"]
+```
+
+## Logic
+
+### or (default)
+
+```toml
+logic    = "or"
+patterns = ["ERROR", "WARN"]
+# passes: "ERROR in module"      "WARN: low memory"
+# blocks: "INFO: started"
+```
+
+### and
+
+```toml
+logic    = "and"
+patterns = ["database", "ERROR"]
+# passes: "ERROR: database connection failed"
+# blocks: "ERROR: file not found"
+```
+
+With `logic = "and"` on an `exclude` filter, an entry is dropped only when it
+matches *every* pattern.
+
+## Filter Chains
+
+Filters are evaluated in declaration order and an entry must survive all of
+them. The first filter to reject an entry ends its life; later filters never see
+it.
+
+```toml
+# 1. keep only production traffic
+[[pipelines.flow.filters]]
+type     = "include"
+patterns = ["prod-", "production"]
+
+# 2. of that, keep only failures
+[[pipelines.flow.filters]]
+type     = "include"
+patterns = ["ERROR", "EXCEPTION", "FATAL"]
+
+# 3. minus known noise
+[[pipelines.flow.filters]]
+type     = "exclude"
+patterns = ["ECONNRESET", "broken pipe"]
+```
+
+Order matters for cost, not for correctness: put the most selective filter first
+so later ones evaluate fewer entries.
 
 ## Pattern Syntax
 
-Patterns support regular expression syntax:
+Go's RE2 syntax. No backreferences and no lookaround — RE2 guarantees linear
+time, which is exactly what you want in a log hot path.
 
-### Basic Patterns
+| Need | Pattern |
-- **Literal match**: `"ERROR"` - matches "ERROR" anywhere
+|------|---------|
-- **Case-insensitive**: `"(?i)error"` - matches "error", "ERROR", "Error"
+| Literal substring | `ERROR` |
-- **Word boundary**: `"\\berror\\b"` - matches whole word only
+| Case-insensitive | `(?i)error` |
+| Whole word | `\\berror\\b` |
+| Alternation | `ERROR\|WARN\|FATAL` |
+| Character class | `[0-9]{3}` |
+| Anchors | `^ERROR`, `ERROR$` |
+| Any characters | `.*exception.*` |
 
-### Advanced Patterns
+Remember that TOML basic strings process escapes, so a regex backslash needs
-- **Alternation**: `"ERROR|WARN|FATAL"`
+doubling: `"\\berror\\b"`. TOML literal strings avoid the issue:
-- **Character classes**: `"[0-9]{3}"`
+`'\berror\b'`.
-- **Wildcards**: `".*exception.*"`
-- **Line anchors**: `"^ERROR"` (start), `"ERROR$"` (end)
 
-### Special Characters
+Anchors apply to the assembled match text, which begins with the source name —
-Escape special regex characters with backslash:
+so `^ERROR` will not match an entry whose source is non-empty. Use
-- `.` → `\\.`
+`\\bERROR\\b` instead unless you mean to anchor on the source.
-- `*` → `\\*`
-- `[` → `\\[`
-- `(` → `\\(`
 
-## Filter Logic
+## Common Recipes
 
-### OR Logic (default)
+**Severity floor**
-Entry passes if ANY pattern matches:
-```toml
-logic = "or"
-patterns = ["ERROR", "WARN"]
-# Passes: "ERROR in module", "WARN: low memory"
-# Blocks: "INFO: started"
-```
-
-### AND Logic
-Entry passes only if ALL patterns match:
-```toml
-logic = "and"  
-patterns = ["database", "ERROR"]
-# Passes: "ERROR: database connection failed"
-# Blocks: "ERROR: file not found"
-```
-
-## Filter Chain
-
-Multiple filters execute sequentially:
 
 ```toml
-# First filter: Include errors and warnings
+[[pipelines.flow.filters]]
-[[pipelines.filters]]
+type     = "include"
-type = "include"
+patterns = ["ERROR", "FATAL", "CRITICAL"]
-patterns = ["ERROR", "WARN"]
-
-# Second filter: Exclude test environments
-[[pipelines.filters]]
-type = "exclude"
-patterns = ["test-env", "staging"]
 ```
 
-Processing order:
+**Noise reduction**
-1. Entry arrives from source
-2. Include filter evaluates
-3. If passed, exclude filter evaluates
-4. If passed all filters, entry continues to sink
 
-## Performance Considerations
-
-### Pattern Compilation
-- Patterns compile once at startup
-- Invalid patterns cause startup failure
-- Complex patterns may impact performance
-
-### Optimization Tips
-- Place most selective filters first
-- Use simple patterns when possible
-- Combine related patterns with alternation
-- Avoid excessive wildcards (`.*`)
-
-## Filter Statistics
-
-Filters track:
-- Total entries evaluated
-- Entries passed
-- Entries blocked
-- Processing time per pattern
-
-## Common Use Cases
-
-### Log Level Filtering
 ```toml
-[[pipelines.filters]]
+[[pipelines.flow.filters]]
-type = "include"
+type     = "exclude"
-patterns = ["ERROR", "WARN", "FATAL", "CRITICAL"]
+patterns = ["/healthz", "/metrics", "\\bping\\b"]
 ```
 
-### Application Filtering
+**Secret suppression** — see [Security](security.md); filters are the only
+redaction mechanism LogWisp currently offers.
+
 ```toml
-[[pipelines.filters]]
+[[pipelines.flow.filters]]
-type = "include"
+type     = "exclude"
-patterns = ["app1", "app2", "app3"]
+patterns = ["password", "api[_-]?key", "authorization", "bearer ", "secret", "token"]
 ```
 
-### Noise Reduction
+Note this drops the whole entry, it does not redact part of it.
-```toml
-[[pipelines.filters]]
-type = "exclude"
-patterns = [
-    "health-check",
-    "ping",
-    "/metrics",
-    "heartbeat"
-]
-```
 
-### Security Filtering
+**Per-application routing** — run one pipeline per application, each with its
-```toml
+own include filter, rather than trying to route inside one pipeline. Sinks fan
-[[pipelines.filters]]
+out to *all* sinks in a pipeline; there is no conditional routing.
-type = "exclude"
-patterns = [
-    "password",
-    "token",
-    "api[_-]key",
-    "secret"
-]
-```
 
-### Multi-stage Filtering
+## Statistics
-```toml
-# Include production logs
-[[pipelines.filters]]
-type = "include"
-patterns = ["prod-", "production"]
 
-# Include only errors
+Each filter reports `type`, `logic`, `pattern_count`, `total_processed`,
-[[pipelines.filters]]
+`total_matched`, and `total_dropped`. The chain reports `filter_count`,
-type = "include"
+`total_processed`, and `total_passed`; the pipeline derives
-patterns = ["ERROR", "EXCEPTION", "FATAL"]
+`total_filtered` as the difference.
 
-# Exclude known issues
+## Performance
-[[pipelines.filters]]
+
-type = "exclude"
+Patterns compile once at startup. Every entry that reaches the filter stage is
-patterns = ["ECONNRESET", "broken pipe"]
+evaluated against every filter until one rejects it, so cost scales with the
-```
+number of patterns and their complexity. Prefer literal substrings and simple
+alternations over broad `.*` wildcards.
+
+Filters log at DEBUG on every entry — pattern text, match results, and the
+final decision. That is invaluable when a filter is not behaving as expected and
+very expensive in production; keep `logging.level` at `info` or higher on a busy
+pipeline.

doc/formatters.md

@@ -1,215 +1,183 @@
 # Formatters
 
-LogWisp formatters transform log entries before output to sinks.
+The formatter is the last flow stage. It turns a `core.LogEntry` into the byte
-
+payload that sinks write, applying a sanitizer policy on the way.
-## Formatter Types
-
-### Raw Formatter
-
-Outputs the log message as-is with optional newline.
 
 ```toml
-[pipelines.format]
+[pipelines.flow.format]
+type             = "json"
+sanitizer_policy = "json"
+flags            = 0
+timestamp_format = ""
+```
+
+One formatter serves the whole pipeline. Sinks receive an identical payload;
+there is no per-sink formatting. When you need two shapes of the same data, run
+two pipelines, or chain to a node that formats differently.
+
+Omitting `[pipelines.flow.format]` entirely selects `raw`.
+
+## Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `type` | string | `raw` | `raw`, `txt` (alias `text`), or `json` |
+| `sanitizer_policy` | string | derived from `type` | `raw`, `txt`, `json`, or `shell` |
+| `flags` | int64 | `0` | Bitmask override; `0` selects a per-type default |
+| `timestamp_format` | string | formatter default | Go reference layout, e.g. `"2006-01-02T15:04:05Z07:00"` |
+
+## Types
+
+### raw
+
+Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches
+the sink exactly as the source produced it, with no timestamp, level or source
+prefix added. An entry that also carries `fields` gets the fields JSON appended
+verbatim after a single space — `raw` never drops data and never re-encodes it.
+
+```toml
+[pipelines.flow.format]
 type = "raw"
-
-[pipelines.format.raw]
-add_new_line = true
 ```
 
-**Configuration Options:**
+Fastest option, and the right one when you are relaying text that is already in
+its final form. Note that it also bypasses sanitization, so control characters
+in the source data reach your sinks intact.
 
-| Option | Type | Default | Description |
+Byte-exact transport needs a source that does not split the line: the `console`
-|--------|------|---------|-------------|
+source, or the `file` source with `raw = true`. Both put the whole line —
-| `add_new_line` | bool | true | Append newline to messages |
+newline included — in the message and leave `fields` empty. The `file` source's
+JSON branch splits a line into message and fields, so `raw` reassembles it as
+`<msg> <fields>` rather than reproducing the original object.
 
-### JSON Formatter
+### txt
 
-Produces structured JSON output.
+Human-readable line output with a timestamp and level.
 
 ```toml
-[pipelines.format]
+[pipelines.flow.format]
-type = "json"
+type             = "txt"
-
+sanitizer_policy = "txt"
-[pipelines.format.json]
+timestamp_format = "2006-01-02 15:04:05"
-pretty = false
-timestamp_field = "timestamp"
-level_field = "level"
-message_field = "message"
-source_field = "source"
 ```
 
-**Configuration Options:**
+### json
 
-| Option | Type | Default | Description |
+Structured output, the natural choice for downstream ingestion.
-|--------|------|---------|-------------|
+
-| `pretty` | bool | false | Pretty print JSON |
+```toml
-| `timestamp_field` | string | "timestamp" | Field name for timestamp |
+[pipelines.flow.format]
-| `level_field` | string | "level" | Field name for log level |
+type             = "json"
-| `message_field` | string | "message" | Field name for message |
+sanitizer_policy = "json"
-| `source_field` | string | "source" | Field name for source |
+```
+
+Output has the shape:
 
-**Output Structure:**
 ```json
-{
+{"time":"2026-01-02T15:04:05.123Z","level":"ERROR","trace":"edge-01/app.log","fields":["connection refused"]}
-  "timestamp": "2024-01-01T12:00:00Z",
-  "level": "ERROR",
-  "source": "app",
-  "message": "Connection failed"
-}
 ```
 
-### Text Formatter
+The exact key names and structure come from the `lixenwraith/log` formatter, not
+from LogWisp; they are stable for a given dependency version but are not part of
+LogWisp's own configuration surface.
 
-Template-based text formatting.
+## Flags
 
-```toml
+`flags` is a bitmask passed to the underlying formatter. Leave it at `0` unless
-[pipelines.format]
+you need to override the defaults.
-type = "txt"
 
-[pipelines.format.txt]
+| Value | Name | Effect |
-template = "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}"
+|-------|------|--------|
-timestamp_format = "2006-01-02T15:04:05.000Z07:00"
+| `1` | Raw | Bypass formatting and sanitization entirely |
+| `2` | ShowTimestamp | Emit the timestamp |
+| `4` | ShowLevel | Emit the level |
+| `8` | StructuredJSON | Render attached fields as a JSON object |
+| `16` | NoTimestamp | Suppress the timestamp |
+| `32` | NoLevel | Suppress the level |
+
+With `flags = 0` the formatter selects `1` for `type = "raw"` and `6`
+(timestamp + level) for every other type. `8` is added automatically whenever an
+entry carries parseable `fields` and `1` is not set; `1` always wins.
+
+Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp
+only, no level.
+
+## Sanitizer Policies
+
+The sanitizer runs before serialization and neutralizes control characters that
+would otherwise break framing or reach a terminal.
+
+| Policy | Behaviour | Use with |
+|--------|-----------|----------|
+| `raw` | No-op passthrough | `type = "raw"` where you control the data |
+| `txt` | Escapes non-printable characters | File and console sinks |
+| `json` | Escapes control characters for safe JSON embedding | `type = "json"`, chain links |
+| `shell` | Strips shell metacharacters, whitespace, and control characters | Data that will be passed to a command |
+
+When `sanitizer_policy` is omitted, the policy is derived from `type`: `json`
+for `json`, `txt` for `txt`/`text`, and `raw` for anything else — so the safe
+pairing is the default.
+
+> `shell` strips dangerous characters but is **not** sufficient to make a string
+> safe for shell construction. Pass arguments through `exec` argv instead of
+> building command lines.
+
+To see a policy working, point a pipeline at the `random` source with
+`special = true`, which injects control bytes and multi-byte Unicode into every
+message.
+
+## Node Identity in Output
+
+Entries that arrived over a chain link carry a `Node` label. The formatter
+renders it as a syslog-style prefix on the source field:
+
+```
+edge-01/app.log
 ```
 
-**Configuration Options:**
+Entries with no node label show the bare source. Node identity therefore appears
+*inside* the source field rather than as a separate output key — worth knowing
+when writing downstream parsers or grep patterns.
 
-| Option | Type | Default | Description |
+## Structured Fields
-|--------|------|---------|-------------|
-| `template` | string | See below | Go template string |
-| `timestamp_format` | string | RFC3339 | Go time format string |
 
-**Default Template:**
+When an entry carries `Fields` (raw JSON) and `FlagRaw` is not set, the
-```
+formatter parses it and switches to structured rendering by adding the
-[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}{{ if .Fields }} {{.Fields}}{{ end }}
+`StructuredJSON` flag automatically. Fields reach a pipeline in two ways: from
-```
+the `file` source when a tailed line parses as JSON with a `fields` key, and
+from the heartbeat generator when `include_stats = true`.
 
-## Template Functions
+## Choosing a Configuration
 
-Available functions in text templates:
+| Goal | Configuration |
+|------|---------------|
+| Maximum throughput, data already formatted | `type = "raw"` |
+| Human reading in a terminal or file | `type = "txt"`, `sanitizer_policy = "txt"` |
+| Downstream ingestion (Loki, Elasticsearch, jq) | `type = "json"`, `sanitizer_policy = "json"` |
+| Compact console output | `type = "txt"`, `flags = 4` |
+| Untrusted log content | never `raw`; pick `txt` or `json` and set the matching policy |
 
-| Function | Description | Example |
+## Formatting and Chain Links
-|----------|-------------|---------|
-| `FmtTime` | Format timestamp | `{{.Timestamp \| FmtTime}}` |
-| `ToUpper` | Convert to uppercase | `{{.Level \| ToUpper}}` |
-| `ToLower` | Convert to lowercase | `{{.Source \| ToLower}}` |
-| `TrimSpace` | Remove whitespace | `{{.Message \| TrimSpace}}` |
 
-## Template Variables
+Chain sinks (`tcp_chain`, `http_chain`) do **not** ship the formatted payload.
+They re-serialize the structured entry into the canonical chain encoding, which
+makes them independent of the local formatter.
 
-Available variables in templates:
+The practical consequence: setting `flow.format` on an edge node changes only
+that node's own local sinks. The output shape seen by a human or a downstream
+system is decided on the node that owns the sink they read.
 
-| Variable | Type | Description |
+If an event ever reaches a chain sink without a structured entry, the sink wraps
-|----------|------|-------------|
+the formatted payload into a synthetic entry and counts it in `synthesized`.
-| `.Timestamp` | time.Time | Entry timestamp |
+A non-zero `synthesized` count means something upstream lost structure.
-| `.Level` | string | Log level |
-| `.Source` | string | Source identifier |
-| `.Message` | string | Log message |
-| `.Fields` | string | Additional fields (JSON) |
 
-## Time Format Strings
+## Performance
 
-Common Go time format patterns:
+Relative cost, cheapest first: `raw` (passthrough) → `txt` (line assembly) →
+`json` (serialization). Sanitization adds a scan of the message; the `raw`
+policy skips it.
 
-| Pattern | Example Output |
+The formatter holds a mutex because the underlying implementation reuses an
-|---------|---------------|
+internal buffer and is not goroutine-safe. It is the only shared serialization
-| `2006-01-02T15:04:05Z07:00` | 2024-01-02T15:04:05Z |
+point in the hot path, and the reason a single pipeline formats entries one at
-| `2006-01-02 15:04:05` | 2024-01-02 15:04:05 |
+a time.
-| `Jan 2 15:04:05` | Jan 2 15:04:05 |
-| `15:04:05.000` | 15:04:05.123 |
-| `2006/01/02` | 2024/01/02 |
-
-## Format Selection
-
-### Default Behavior
-
-If no formatter specified:
-- **HTTP/TCP sinks**: JSON format
-- **Console/File sinks**: Raw format
-- **Client sinks**: JSON format
-
-### Per-Pipeline Configuration
-
-Each pipeline can have its own formatter:
-
-```toml
-[[pipelines]]
-name = "json-pipeline"
-[pipelines.format]
-type = "json"
-
-[[pipelines]]
-name = "text-pipeline"
-[pipelines.format]
-type = "txt"
-```
-
-## Message Processing
-
-### JSON Message Handling
-
-When using JSON formatter with JSON log messages:
-1. Attempts to parse message as JSON
-2. Merges fields with LogWisp metadata
-3. LogWisp fields take precedence
-4. Falls back to string if parsing fails
-
-### Field Preservation
-
-LogWisp metadata always includes:
-- Timestamp (from source or current time)
-- Level (detected or default)
-- Source (origin identifier)
-- Message (original content)
-
-## Performance Characteristics
-
-### Formatter Performance
-
-Relative performance (fastest to slowest):
-1. **Raw**: Direct passthrough
-2. **Text**: Template execution
-3. **JSON**: Serialization
-4. **JSON (pretty)**: Formatted serialization
-
-### Optimization Tips
-
-- Use raw format for high throughput
-- Cache template compilation (automatic)
-- Minimize template complexity
-- Avoid pretty JSON in production
-
-## Common Configurations
-
-### Structured Logging
-```toml
-[pipelines.format]
-type = "json"
-[pipelines.format.json]
-pretty = false
-```
-
-### Human-Readable Logs
-```toml
-[pipelines.format]
-type = "txt"
-[pipelines.format.txt]
-template = "{{.Timestamp | FmtTime}} [{{.Level}}] {{.Message}}"
-timestamp_format = "15:04:05"
-```
-
-### Syslog Format
-```toml
-[pipelines.format]
-type = "txt"
-[pipelines.format.txt]
-template = "{{.Timestamp | FmtTime}} {{.Source}} {{.Level}}: {{.Message}}"
-timestamp_format = "Jan 2 15:04:05"
-```
-
-### Minimal Output
-```toml
-[pipelines.format]
-type = "txt"
-[pipelines.format.txt]
-template = "{{.Message}}"
-```

doc/installation.md

@@ -1,49 +1,80 @@
 # Installation Guide
 
-LogWisp installation and service configuration for Linux and FreeBSD systems.
+## Requirements
 
-## Installation Methods
+- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
+- **Architecture**: amd64
+- **Go**: 1.27.1 or newer, to build from source
 
-### Pre-built Binaries
+## Building from Source
-
-Download the latest release binary for your platform and install to `/usr/local/bin`:
 
 ```bash
-# Linux amd64
+git clone https://github.com/lixenwraith/logwisp.git
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
-chmod +x logwisp-linux-amd64
-sudo mv logwisp-linux-amd64 /usr/local/bin/logwisp
-
-# FreeBSD amd64  
-fetch https://github.com/yourusername/logwisp/releases/latest/download/logwisp-freebsd-amd64
-chmod +x logwisp-freebsd-amd64
-sudo mv logwisp-freebsd-amd64 /usr/local/bin/logwisp
-```
-
-### Building from Source
-
-Requires Go 1.24 or newer:
-
-```bash
-git clone https://github.com/yourusername/logwisp.git
 cd logwisp
-go build -o logwisp ./src/cmd/logwisp
+make
-sudo install -m 755 logwisp /usr/local/bin/
+sudo make install          # installs to $PREFIX/bin, default /usr/local/bin
 ```
 
-### Go Install Method
+The Makefile works with both GNU make and BSD make. Targets:
 
-Install directly using Go (version information will not be embedded):
+| Target | Effect |
+|--------|--------|
+| `make` / `make build` | Build `bin/logwisp` with version metadata |
+| `make dev` | Build with the race detector enabled |
+| `make install` | Install the binary to `$(PREFIX)/bin` (default `/usr/local`) |
+| `make uninstall` | Intended to remove the installed binary — currently broken: it expands to `$(BINDIR)/bin/logwisp` instead of `$(BINDIR)/logwisp`, so it removes nothing. Delete the binary by hand |
+| `make clean` | Remove the built binary |
+| `make version` | Print the version, commit, and build time that would be embedded |
+
+Version, commit hash, and build time are injected via `-ldflags` from `git
+describe` and `git rev-parse`. A plain `go build` produces a working binary that
+reports `dev` for all three:
 
 ```bash
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+go build -o bin/logwisp ./cmd/logwisp
 ```
 
-## Service Configuration
+`go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with
+the same loss of version metadata.
+
+## Container Image
+
+The root `Dockerfile` builds the same package into `scratch` under UID 65532,
+static and stripped. There is no shell and no config in the image: mount one and
+name it, as the binary has no daemon mode and no built-in defaults worth running.
+
+```bash
+REV=$(git rev-parse HEAD)
+docker build -t "logwisp:$(git rev-parse --short HEAD)" \
+  --build-arg VERSION="$(git describe --tags --always)" \
+  --build-arg REVISION="$REV" .
+docker run --rm -v /etc/logwisp:/etc/logwisp:ro logwisp:... -c /etc/logwisp/logwisp.toml
+```
+
+Sinks that listen (`http`, `tcp`) need their ports published; the read-only
+root filesystem and dropped capabilities a restricted runtime imposes are all
+compatible with it, provided a `file` sink's directory is writable by 65532.
+
+## Configuration
+
+Copy the annotated reference configuration and edit it:
+
+```bash
+sudo mkdir -p /etc/logwisp
+sudo cp config/logwisp.toml /etc/logwisp/logwisp.toml
+```
+
+LogWisp searches, in order: `-c <path>`, `--config=<path>`,
+`$LOGWISP_CONFIG_DIR`/`$LOGWISP_CONFIG_FILE`, `~/.config/logwisp/logwisp.toml`,
+`./logwisp.toml`. See [Configuration](configuration.md).
+
+## Running as a Service
+
+LogWisp has no daemon mode; run it in the foreground under a supervisor.
 
 ### Linux (systemd)
 
-Create systemd service file `/etc/systemd/system/logwisp.service`:
+`/etc/systemd/system/logwisp.service`:
 
 ```ini
 [Unit]
@@ -55,30 +86,47 @@ Type=simple
 User=logwisp
 Group=logwisp
 ExecStart=/usr/local/bin/logwisp -c /etc/logwisp/logwisp.toml
+ExecReload=/bin/kill -HUP $MAINPID
 Restart=on-failure
 RestartSec=10
+WorkingDirectory=/var/lib/logwisp
 StandardOutput=journal
 StandardError=journal
-WorkingDirectory=/var/lib/logwisp
+
+# Hardening
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths=/var/log/logwisp /var/lib/logwisp
 
 [Install]
 WantedBy=multi-user.target
 ```
 
-Setup service user and directories:
+`ExecReload` gives you `systemctl reload logwisp` for configuration and
+certificate rotation without dropping the process.
+
+If a pipeline binds a port below 1024, add
+`AmbientCapabilities=CAP_NET_BIND_SERVICE` rather than running as root.
+
+Setup:
 
 ```bash
-sudo useradd -r -s /bin/false logwisp
+sudo useradd -r -s /usr/sbin/nologin logwisp
 sudo mkdir -p /etc/logwisp /var/lib/logwisp /var/log/logwisp
 sudo chown logwisp:logwisp /var/lib/logwisp /var/log/logwisp
 sudo systemctl daemon-reload
-sudo systemctl enable logwisp
+sudo systemctl enable --now logwisp
-sudo systemctl start logwisp
 ```
 
+The service account needs **read** access to every directory a `file` source
+watches and **write** access to every directory a `file` sink or
+`logging.file` writes to.
+
 ### FreeBSD (rc.d)
 
-Create rc script `/usr/local/etc/rc.d/logwisp`:
+`/usr/local/etc/rc.d/logwisp`:
 
 ```sh
 #!/bin/sh
@@ -92,8 +140,9 @@ Create rc script `/usr/local/etc/rc.d/logwisp`:
 name="logwisp"
 rcvar="${name}_enable"
 pidfile="/var/run/${name}.pid"
-command="/usr/local/bin/logwisp"
+procname="/usr/local/bin/logwisp"
-command_args="-c /usr/local/etc/logwisp/logwisp.toml"
+command="/usr/sbin/daemon"
+command_args="-p ${pidfile} -f ${procname} -c /usr/local/etc/logwisp/logwisp.toml"
 
 load_rc_config $name
 : ${logwisp_enable:="NO"}
@@ -101,7 +150,7 @@ load_rc_config $name
 run_rc_command "$1"
 ```
 
-Setup service:
+Setup:
 
 ```bash
 sudo chmod +x /usr/local/etc/rc.d/logwisp
@@ -112,45 +161,59 @@ sudo sysrc logwisp_enable="YES"
 sudo service logwisp start
 ```
 
-## Directory Structure
+## Directory Layout
-
-Standard installation directories:
 
 | Purpose | Linux | FreeBSD |
 |---------|-------|---------|
 | Binary | `/usr/local/bin/logwisp` | `/usr/local/bin/logwisp` |
 | Configuration | `/etc/logwisp/` | `/usr/local/etc/logwisp/` |
-| Working Directory | `/var/lib/logwisp/` | `/var/db/logwisp/` |
+| TLS material | `/etc/logwisp/tls/` | `/usr/local/etc/logwisp/tls/` |
-| Log Files | `/var/log/logwisp/` | `/var/log/logwisp/` |
+| Working directory | `/var/lib/logwisp/` | `/var/db/logwisp/` |
-| PID File | `/var/run/logwisp.pid` | `/var/run/logwisp.pid` |
+| Application logs | `/var/log/logwisp/` | `/var/log/logwisp/` |
 
-## Post-Installation Verification
+Key files should be mode `0600` and owned by the service account.
 
-Verify the installation:
+## Verification
 
 ```bash
-# Check version
+logwisp --version
-logwisp version
 
-# Test configuration
+# start in the foreground with debug logging and watch pipelines come up
-logwisp -c /etc/logwisp/logwisp.toml --disable-status-reporter
+logwisp -c /etc/logwisp/logwisp.toml --logging.level=debug --logging.output=stderr
 
-# Check service status (Linux)
+sudo systemctl status logwisp      # Linux
-sudo systemctl status logwisp
+sudo service logwisp status        # FreeBSD
-
-# Check service status (FreeBSD)
-sudo service logwisp status
 ```
 
-## Uninstallation
+Expect `Created source instance`, `Created sink instance`, and
+`Starting pipeline` for each configured pipeline. There is no validate-only
+mode; see [Operations](operations.md#checking-a-configuration).
+
+## Test Scripts
+
+End-to-end scripts under `test/` run against a local build:
+
+```bash
+make
+./test/chain-test.sh --auto             # two independent relay pipelines
+./test/chain-aggregate-test.sh --auto   # fan-in: both edges into one pipeline
+./test/mtls-chain-test.sh --auto        # the same fan-in under mTLS
+./test/passthrough-test.sh              # file source relays a wide envelope intact
+```
+
+Without `--auto` the chain scripts run the relay in the foreground for
+interactive inspection. They need bash 5+, coreutils, and curl, and they bind
+ports 15801–15804. The pass-through test binds nothing. Generated configuration
+and logs land in `test/run/`.
+
+## Uninstall
 
 ### Linux
 
 ```bash
-sudo systemctl stop logwisp
+sudo systemctl disable --now logwisp
-sudo systemctl disable logwisp
+sudo rm /usr/local/bin/logwisp /etc/systemd/system/logwisp.service
-sudo rm /usr/local/bin/logwisp
+sudo systemctl daemon-reload
-sudo rm /etc/systemd/system/logwisp.service
 sudo rm -rf /etc/logwisp /var/lib/logwisp /var/log/logwisp
 sudo userdel logwisp
 ```
@@ -160,8 +223,7 @@ sudo userdel logwisp
 ```bash
 sudo service logwisp stop
 sudo sysrc -x logwisp_enable
-sudo rm /usr/local/bin/logwisp
+sudo rm /usr/local/bin/logwisp /usr/local/etc/rc.d/logwisp
-sudo rm /usr/local/etc/rc.d/logwisp
 sudo rm -rf /usr/local/etc/logwisp /var/db/logwisp /var/log/logwisp
 sudo pw userdel logwisp
 ```

doc/mtls-auth-plan.md

@@ -0,0 +1,373 @@
+# mTLS as Authentication
+
+**Status:** implemented. Phases 1–3 of the original proposal, plus dialer-side
+server identity pinning from phase 4, are in the tree and covered by
+`test/mtls-chain-test.sh`. The remaining phase-4 items are listed under
+[Not Implemented](#not-implemented).
+
+**Scope:** turn transport-level mutual TLS into an authentication and
+authorization mechanism.
+
+## Problem
+
+LogWisp already did mutual TLS at the transport layer. A listener with
+`client_auth = true` refused any peer that could not present a certificate
+chaining to `client_ca_file`, and `internal/tlsx` extracted the peer's Common
+Name into session metadata as `tls_peer_cn`.
+
+Nothing read it back. The result was a CA-wide membership check with no notion
+of *which* peer connected:
+
+1. **No per-identity authorization.** Every certificate the CA issued was
+   equivalent. There was no way to say "only `edge-01` and `edge-02` may write
+   to this ingest port", so one CA could not serve several trust domains, and
+   withdrawing one peer meant rotating the CA bundle for all of them.
+2. **Node labels were unauthenticated.** With `trust_node = true` (the default)
+   a peer declares its own `node` label in the chain hello or the
+   `X-Logwisp-Node` header. Any certificate holder could claim any label,
+   including another host's, and every downstream consumer would attribute
+   those entries accordingly. The only defence, `trust_node = false`, replaced
+   the label with a remote address — unforgeable, but useless for identifying a
+   host behind NAT or a load balancer.
+3. **The `http` sink had no authentication at all**, even with TLS on. Its
+   stream and status endpoints were readable by anyone who could reach the port.
+
+Password, token, and SCRAM authentication were removed during the plugin/flow
+restructure and the move to standard-library networking. Certificates are the
+one credential the transport already carries, which made mTLS the cheapest path
+back to authenticated peers.
+
+## Goals
+
+- Authorize peers by certificate identity, per listener. ✅
+- Bind the chain `node` label to the authenticated identity, so origin
+  attribution is trustworthy. ✅
+- Gate the `http` sink's endpoints on client certificates. ✅
+- Make identity visible in sessions, statistics, and logs. ✅
+- Change nothing for existing configurations that omit the new block. ✅
+
+## Non-Goals
+
+- Reviving password, token, or SCRAM authentication. The reserved hooks
+  (`chain.Hello.Auth`, the `Authorization` header comment in the `http_chain`
+  sink, the "Future: password auth block" comments in the network options
+  structs) stay reserved.
+- IP allow/deny lists and per-peer rate limits. Related, but a separate feature
+  with its own config surface.
+- OCSP. See [Revocation](#revocation) for what is done instead.
+- Authorization *within* a stream — the unit of decision is a connection (TCP)
+  or a request (HTTP), never an individual entry.
+
+## Design
+
+### Identity
+
+The authenticated identity is a single string derived from the peer's verified
+leaf certificate. Because `tls.RequireAndVerifyClientCert` has already validated
+the chain, signature, and validity window by the time we look, extraction is
+pure field selection — `tlsx.PeerIdentity`.
+
+| `identity` mode | Source | Notes |
+|-----------------|--------|-------|
+| `cn` (default) | `Subject.CommonName` | Matches the existing `tls_peer_cn` metadata |
+| `san_dns` | first `DNSNames` entry | Preferred for host identities |
+| `san_uri` | first `URIs` entry | SPIFFE-style IDs |
+| `san_email` | first `EmailAddresses` entry | Operator identities |
+
+An empty identity is a rejection, not an empty match: a certificate with no
+usable identity field cannot satisfy any policy.
+
+Identities are not secrets, so ordinary string comparison is used; there is no
+timing side channel worth defending here.
+
+### Configuration
+
+An `auth` table sits beside `tls` in every network plugin's `config`. Keeping it
+separate from `tls` matters: TLS answers "is this channel private and does the
+peer chain to a CA", auth answers "may *this* peer do *this*", and a later
+non-certificate method can reuse the block.
+
+```toml
+[pipelines.plugin_sources.config.auth]
+type           = "mtls"                  # none (default) | mtls
+identity       = "cn"                    # cn | san_dns | san_uri | san_email
+allow          = ["edge-01", "edge-02"]  # exact identities
+allow_patterns = ["^edge-\\d{2}$"]       # RE2, anchored by the author
+node_binding   = "force"                 # none | assert | force
+```
+
+| Option | Type | Default | Meaning |
+|--------|------|---------|---------|
+| `type` | string | `none` | `none` preserves pre-auth behaviour exactly; `mtls` enables the policy |
+| `identity` | string | `cn` | Which certificate field is the identity |
+| `allow` | []string | `[]` | Exact identity matches |
+| `allow_patterns` | []string | `[]` | RE2 patterns matched against the identity |
+| `node_binding` | string | `force` when `type = "mtls"` | Chain sources only; see below |
+
+Empty `allow` **and** empty `allow_patterns` under `type = "mtls"` means "any
+identity the CA vouches for" — that is, the pre-auth behaviour, but with the
+identity now recorded and node binding available. It is a deliberate, documented
+default rather than a silent deny-all, and the plugin logs a WARN at startup
+saying so.
+
+`node_binding` applies only to the chain sources, where a `node` label is
+declared. Setting it on any other plugin is a configuration error.
+
+| Value | Connection label | Per-entry `node` field |
+|-------|------------------|------------------------|
+| `none` | `trust_node` governs, as before | `trust_node` governs |
+| `assert` | Must equal the identity; a mismatch or an omission is rejected | `trust_node` governs |
+| `force` | The declared label is ignored and the identity is used | Overwritten with the identity |
+
+The split between `assert` and `force` is what makes both worth having:
+
+- **`force`** is for an ingest boundary that does not trust its peer. Every
+  entry is relabeled, so a compromised edge cannot smuggle a foreign origin
+  through the per-entry `node` field either. It is the default under
+  `type = "mtls"` because it is the only setting where a misconfigured or
+  hostile edge cannot mislabel its entries.
+- **`assert`** is for a relay-to-relay hop. The relay must prove *its own*
+  identity — a mismatch is loud rather than silently corrected — but the entries
+  it forwards keep the origin labels stamped at the first hop, so multi-hop
+  attribution survives.
+
+`node_binding` overrides `trust_node`; when binding is active the constructor
+logs that `trust_node` is being ignored.
+
+Dialer-side plugins (`tcp_chain` and `http_chain` sinks) accept the same block
+to pin the *server's* identity beyond hostname verification. There
+`node_binding` does not apply, and `tls.insecure_skip_verify` is rejected:
+identity read from an unverified chain is a claim, not a fact.
+
+### Validation
+
+At plugin construction, before anything binds:
+
+- `type = "mtls"` on a listener requires `tls.enabled = true` and
+  `tls.client_auth = true`; on a dialer it requires `tls.enabled = true` and
+  forbids `tls.insecure_skip_verify`. Silently accepting an auth policy the
+  transport cannot enforce is the failure mode worth designing out.
+- `identity` must be one of the four modes.
+- Every entry in `allow_patterns` must compile.
+- `node_binding` must be one of the three values, and must be absent or `none`
+  outside the chain sources.
+
+Errors follow existing style: `auth: type "mtls" requires tls.client_auth`.
+
+### The `internal/authz` package
+
+```go
+package authz
+
+// Policy is the compiled form of config.AuthOptions.
+type Policy struct { /* role, identity mode, exact set, patterns, binding, counters */ }
+
+// Role selects the validation and behavior appropriate to the call site.
+const ( RoleListener Role = iota; RoleChainListener; RoleDialer )
+
+// New compiles a policy. Returns (nil, nil) when auth is disabled, matching
+// the tlsx.Server / tlsx.Client convention. tlsOpts is the sibling `tls`
+// block, so an unenforceable policy fails here rather than at run time.
+func New(o *config.AuthOptions, tlsOpts *config.TLSOptions, role Role) (*Policy, error)
+
+// Identity is the outcome of a successful authorization.
+type Identity struct {
+    Name   string // the selected certificate field
+    Method string // "mtls"
+}
+
+// Apply stamps an identity onto session metadata.
+func (id Identity) Apply(meta map[string]any)
+
+// Authorize extracts and checks the peer identity from a completed handshake.
+func (p *Policy) Authorize(cs *tls.ConnectionState) (Identity, error)
+
+// VerifyConnection is assignable to tls.Config.VerifyConnection on a dialer.
+func (p *Policy) VerifyConnection(cs tls.ConnectionState) error
+
+// ResolveNode applies node_binding to the label a peer declared.
+func (p *Policy) ResolveNode(declared, fallback string, trustNode bool, id Identity) (string, error)
+
+// TrustsEntryNode reports whether per-entry node labels survive the policy.
+func (p *Policy) TrustsEntryNode(trustNode bool) bool
+
+// Stats reports counters for the sink/source stats map.
+func (p *Policy) Stats() map[string]any
+```
+
+This mirrors `internal/tlsx`: one small package that is the single seam between
+declarative config and a cross-cutting concern. `New` returns `(nil, nil)` for
+the disabled case and **every method tolerates a nil receiver**, so a call site
+reads identically whether or not auth is configured — no nil checks, no branch
+on config:
+
+```go
+id, err := s.auth.Authorize(tlsState)   // nil policy: (zero Identity, nil)
+if err != nil { /* reject */ }
+```
+
+### Enforcement Points
+
+**`tcp_chain` source** (`internal/source/tcpchain/tcpchain.go`, `handleConn`)
+
+Handshake → **authorize** → read hello → `ResolveNode` → create session. The
+authorization sits between the handshake and the hello read, so an unauthorized
+peer never gets a preamble parsed on its behalf. `chain.DecodeEntry` is then
+called with `auth.TrustsEntryNode(trust_node)` rather than `trust_node` itself.
+
+**`http_chain` source** (`internal/source/httpchain/httpchain.go`, `handleIngest`)
+
+Per request, from `r.TLS`, before the body is read — an unauthorized sender does
+not get to stream `max_body_bytes` into the process. Rejection is `403`,
+distinct from the `400` used for protocol errors, so a sender can tell "you are
+not allowed" from "your batch was malformed". `ResolveNode` then governs the
+`X-Logwisp-Node` header exactly as it governs the TCP hello. The session cache
+key includes the identity, so two peers sharing a remote address never share a
+session.
+
+**`tcp` sink** (`internal/sink/tcp/tcp.go`, `handleConn`)
+
+After the explicit handshake, before the session is created and the client is
+registered — so an unauthorized peer never appears in the client map and never
+receives a broadcast.
+
+**`http` sink** (`internal/sink/http/http.go`, `authMiddleware`)
+
+A middleware around the mux covers both the stream and the status endpoint with
+one wrapper and keeps the handlers themselves unaware of authorization. The
+authorized identity is passed down through the request context for session
+metadata. Rejections are `403` with no body detail — the status endpoint leaks
+host, port, and throughput counters, so a rejection should not leak policy shape
+on top of that.
+
+**`tcp_chain` / `http_chain` sinks** (dialers)
+
+The policy is installed as `tls.Config.VerifyConnection`, which runs after the
+standard chain and hostname checks. A server whose identity the policy rejects
+fails the handshake itself rather than the first write, and the chain sink's
+existing backoff loop handles it as any other connect failure.
+
+### Capabilities
+
+`core.CapAuth` now means "this plugin authorizes peers" — it is derived from the
+policy, not from `tlsConfig.ClientAuth`. Transport-level mTLS without a policy
+still reports `CapTLS`, the `mtls=true` field on the startup log line, and the
+`tls` statistic.
+
+`Pipeline.initSourceCapabilities` and `initSinkCapabilities` treat this as a
+cross-cutting check: a plugin advertising `CapAuth` without `CapTLS` is a
+contradiction and fails pipeline construction rather than starting.
+
+### Observability
+
+Every authorization decision is visible, because a silent deny is
+indistinguishable from a network fault at 3am.
+
+- **Session metadata** gains `auth_method` and `auth_identity` alongside the
+  existing `tls` and `tls_peer_cn`.
+- **Statistics** gain `auth`, `auth_identity` (the mode), `auth_unrestricted`,
+  `auth_allowed`, `auth_rejected`, and — on chain sources — `node_binding`, in
+  the `details` map of every affected source and sink. They surface in the
+  status reporter and in the `http` sink's status endpoint.
+- **Logs** record a WARN per rejection with the remote address and the reason.
+  The startup line carries a rendered policy summary
+  (`auth="mtls identity=cn allow=[1 exact, 0 pattern(s)] node_binding=force"`),
+  a WARN when the allow list is empty, and an INFO when node binding overrides
+  `trust_node`.
+
+### Revocation
+
+Certificate revocation is handled by the allow-list rather than by CRL or OCSP:
+
+1. Remove the identity from `allow` / `allow_patterns`.
+2. `kill -HUP`.
+
+The reload path rebuilds every pipeline, so the policy takes effect on the next
+connection and existing connections are dropped by the rebuild itself. This is
+one moving part instead of three, it needs no network calls on the handshake
+path, and it is exact — no window between revocation and the next CRL
+publication.
+
+## Compatibility
+
+No configuration breaks. Omitting the `auth` block, or setting `type = "none"`,
+reproduces the previous behaviour exactly: `Policy` is nil, every call site
+short-circuits, and `trust_node` continues to govern node labels.
+
+The one behavioural note for adopters: turning on `type = "mtls"` defaults
+`node_binding` to `force`, so entries from a peer whose certificate identity
+differs from its configured `node` label will be relabelled. That is the point
+of the feature, but it moves data between labels in a dashboard, so plan for it.
+Use `node_binding = "assert"` on relay-to-relay hops where upstream origin
+labels must survive.
+
+## Verification
+
+`test/mtls-chain-test.sh` builds a full PKI with `openssl` and exercises both
+target topologies end to end:
+
+```
+./test/mtls-chain-test.sh --auto
+```
+
+Scenario 1 — chained instances, client authenticating with mTLS:
+
+- an authorized edge (`edge-01`) delivers entries through both the `tcp_chain`
+  and `http_chain` ingest ports into a file sink
+- `node_binding = "force"` overrides the label the sender configured
+- an identity outside the allow list (`edge-99`) is refused, even while claiming
+  to be `edge-01`
+- a peer presenting no certificate fails the handshake
+- a dialer that pins a server identity the relay does not hold refuses to
+  connect, even though the server certificate chains to the trusted CA
+
+Scenario 2 — a viewer client reading a streaming sink over mTLS:
+
+- an authorized viewer streams from the `tcp` sink and from the `http` sink's
+  SSE endpoint, and reads `/status`
+- a CA-valid but unauthorized viewer gets nothing from the `tcp` sink and `403`
+  from both `http` sink endpoints
+- a client with no certificate fails the handshake
+- the status endpoint reports the policy and its rejection count
+
+`test/chain-test.sh` and `test/chain-aggregate-test.sh` continue to pass
+unchanged, which is the regression check for the auth-disabled path.
+
+## Not Implemented
+
+The remaining phase-4 items, in rough order of value:
+
+1. **CRL file support** alongside `client_ca_file`, re-read on reload, for
+   operators with existing CRL infrastructure. The allow-list covers the same
+   ground with fewer moving parts, so this is only worth doing for a fleet whose
+   revocation already flows through a CRL.
+2. **Certificate expiry warnings** at startup and on reload — a leaf expiring
+   inside 30 days logged at WARN. Nothing warns today; expiry shows up as a
+   handshake failure.
+3. **Per-identity rate limits.** The natural follow-on now that identity exists,
+   and the natural home for the per-IP limiting that was also removed. Kept out
+   of scope here so this feature stayed reviewable.
+4. **Per-client identity in the `http` sink's status output.** The endpoint
+   reports the policy and counters, but not which identities are currently
+   connected; session metadata has the data.
+5. **A list of `identity` modes** (try `san_uri`, fall back to `cn`) for
+   heterogeneous PKI. A single mode is simpler and covers a uniform CA.
+
+## Decisions Taken
+
+Four questions were left open by the proposal. What was chosen, and why:
+
+1. **An empty allow-list allows rather than denies.** Deny-by-default is the
+   safer instinct, but `type = "mtls"` with no list is a legitimate
+   configuration — "any peer this CA issued, but bind the node labels" — and
+   node binding alone is worth enabling without enumerating every node.
+   Erroring on it would force operators to list their whole fleet to get
+   trustworthy attribution. The compromise is a WARN at startup naming the
+   condition and the fix.
+2. **`identity` takes a single mode.** Simpler, and a uniform CA is the common
+   case. Listed above as a possible extension.
+3. **Per-identity rate limits are out of scope**, as proposed.
+4. **`assert` rejects rather than warns and corrects.** A certificate/config
+   mismatch under `assert` is an outage, which is the point: `force` is the
+   forgiving option and it is the default, so an operator reaches for `assert`
+   precisely when they want the mismatch to be loud.

doc/networking.md

@@ -1,289 +1,208 @@
 # Networking
 
-Network configuration for LogWisp connections, including TLS, rate limiting, and access control.
+Everything LogWisp does over a socket, and the knobs that shape it. For
+certificates and trust see [Security](security.md); for multi-node topologies
+see [Chaining](chaining.md).
 
-## TLS Configuration
+## Address Family
 
-### TLS Support Matrix
+**All listeners bind `tcp4` and all dialers dial `tcp4`.** IPv6 is not
+supported, deliberately. An IPv6 client cannot connect and will simply see a
+connection failure.
 
-| Component | TLS Support | Notes |
+When testing locally use `127.0.0.1`, not `localhost` — the latter may resolve
-|-----------|-------------|-------|
+to `::1` and appear as an unexplained connection refusal.
-| HTTP Source | ✓ | Full TLS 1.2/1.3 |
-| HTTP Sink | ✓ | Full TLS 1.2/1.3 |
-| HTTP Client | ✓ | Client certificates |
-| TCP Source | ✗ | No encryption |
-| TCP Sink | ✗ | No encryption |
-| TCP Client | ✗ | No encryption |
 
-### Server TLS Configuration
+## Network Plugins
 
-```toml
+| Plugin | Role | Protocol | Purpose |
-[pipelines.sources.http.tls]
+|--------|------|----------|---------|
-enabled = true
+| `tcp` sink | Listener | Raw stream | Broadcast formatted payloads to clients |
-cert_file = "/path/to/server.pem"
+| `http` sink | Listener | HTTP SSE | Browser-friendly live stream plus status JSON |
-key_file = "/path/to/server.key"
+| `tcp_chain` source | Listener | Chain v1 | Ingest a persistent NDJSON stream |
-ca_file = "/path/to/ca.pem"
+| `http_chain` source | Listener | Chain v1 | Ingest NDJSON batches over POST |
-min_version = "TLS1.2"  # TLS1.2|TLS1.3
+| `tcp_chain` sink | Dialer | Chain v1 | Forward entries over a persistent connection |
-client_auth = false
+| `http_chain` sink | Dialer | Chain v1 | Forward entries as batched POSTs |
-client_ca_file = "/path/to/client-ca.pem"
+
-verify_client_cert = true
+There is no port registry and no default port: `port` is required on every
+network plugin. There is also no cross-pipeline conflict detection — two sinks
+on the same port fail at bind time when the pipeline starts:
+
+```
+ERROR msg="Failed to start sink" error="tcp sink bind 0.0.0.0:9090: listen tcp4 0.0.0.0:9090: bind: address already in use"
 ```
 
-### Client TLS Configuration
+## Timeouts
+
+Every network plugin exposes the deadlines relevant to its role. Zero means "no
+deadline" wherever the table says so.
+
+| Plugin | Option | Default | Bounds |
+|--------|--------|---------|--------|
+| `tcp` sink | `write_timeout_ms` | `5000` | One write to one client; a miss disconnects that client |
+| `http` sink | `write_timeout_ms` | `0` (none) | One SSE event write |
+| `tcp_chain` source | `hello_timeout_ms` | `10000` | Reading the protocol preamble |
+| `tcp_chain` source | `read_timeout_ms` | `0` (none) | Idle time between entries |
+| `http_chain` source | `read_timeout_ms` | `30000` | Reading a whole request body |
+| `tcp_chain` sink | `dial_timeout_ms` | `5000` | TCP connect |
+| `tcp_chain` sink | `write_timeout_ms` | `5000` | One line write |
+| `http_chain` sink | `request_timeout_ms` | `10000` | Dial plus write plus response |
+
+Fixed, non-configurable bounds:
+
+| Bound | Value | Applies to |
+|-------|-------|------------|
+| TLS handshake | 10 s | All TLS listeners and dialers |
+| HTTP read-header timeout | 10 s | `http` sink, `http_chain` source |
+| HTTP server shutdown grace | 2 s | `http` sink, `http_chain` source |
+| Max single entry line | 1 MiB | Chain listeners |
+
+The `http` sink deliberately leaves the server's `WriteTimeout` unset, since it
+would terminate long-lived SSE streams; per-event deadlines come from
+`write_timeout_ms` instead.
+
+## Connection Limits
+
+`max_connections` caps concurrent connections on the `tcp` sink, the `http`
+sink, and the `tcp_chain` source. `0` means unlimited.
+
+- Admission is a load-then-check, so a burst can over-admit by roughly one
+  connection. This is accepted, not a bug to work around.
+- On the `tcp` sink and `tcp_chain` source the count is taken at accept, so it
+  bounds concurrent TLS handshakes as well as established sessions.
+- Over-limit connections are closed immediately and counted in `rejected_conns`
+  (TCP) or `rejected_clients` (HTTP, which first answers `503`).
+
+The `http_chain` source has no connection cap; it bounds work with
+`max_body_bytes` and `read_timeout_ms` instead.
+
+There is **no** per-IP limiting and no IP allow/deny list. `flow.rate_limit` is
+a pipeline-wide entry rate limit, not a network-level one — it cannot
+distinguish or throttle an individual peer.
+
+## Keep-Alive
+
+TCP keep-alive is available on the `tcp` sink (for accepted connections) and the
+`tcp_chain` sink (for its outbound connection):
 
 ```toml
-[pipelines.sinks.http_client.tls]
+keep_alive           = true
-enabled = true
+keep_alive_period_ms = 30000
-server_name = "logs.example.com"
-skip_verify = false
-cert_file = "/path/to/client.pem"  # For mTLS
-key_file = "/path/to/client.key"   # For mTLS
 ```
 
-### TLS Certificate Generation
+This is kernel-level keep-alive; it detects a dead peer but does not keep an
+application-level stream flowing. For that, use a heartbeat.
 
-Using the `tls` command:
+## Heartbeats
 
-```bash
+Heartbeats are a **flow-level** feature, not a per-sink one. Enabling one
-# Generate CA certificate
+injects a synthetic entry into the pipeline at a fixed interval; it reaches
-logwisp tls -ca -o myca
+every sink and traverses chain links as an ordinary structured entry.
-
-# Generate server certificate
-logwisp tls -server -ca-cert myca.pem -ca-key myca.key -host localhost,server.example.com -o server
-
-# Generate client certificate
-logwisp tls -client -ca-cert myca.pem -ca-key myca.key -o client
-```
-
-Command options:
-
-| Flag | Description |
-|------|-------------|
-| `-ca` | Generate CA certificate |
-| `-server` | Generate server certificate |
-| `-client` | Generate client certificate |
-| `-host` | Comma-separated hostnames/IPs |
-| `-o` | Output file prefix |
-| `-days` | Certificate validity (default: 365) |
-
-## Network Rate Limiting
-
-### Configuration Options
 
 ```toml
-[pipelines.sources.http.net_limit]
+[pipelines.flow.heartbeat]
-enabled = true
+enabled           = true
-max_connections_per_ip = 10
+interval_ms       = 30000
-max_connections_total = 100
-requests_per_second = 100.0
-burst_size = 200
-response_code = 429
-response_message = "Rate limit exceeded"
-ip_whitelist = ["192.168.1.0/24"]
-ip_blacklist = ["10.0.0.0/8"]
-```
-
-### Rate Limiting Parameters
-
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `enabled` | bool | Enable rate limiting |
-| `max_connections_per_ip` | int | Per-IP connection limit |
-| `max_connections_total` | int | Global connection limit |
-| `requests_per_second` | float | Request rate limit |
-| `burst_size` | int | Token bucket burst capacity |
-| `response_code` | int | HTTP response code when limited |
-| `response_message` | string | Response message when limited |
-
-### IP Access Control
-
-**Whitelist**: Only specified IPs/networks allowed
-```toml
-ip_whitelist = [
-    "192.168.1.0/24",  # Local network
-    "10.0.0.0/8",      # Private network
-    "203.0.113.5"      # Specific IP
-]
-```
-
-**Blacklist**: Specified IPs/networks denied
-```toml
-ip_blacklist = [
-    "192.168.1.100",   # Blocked host
-    "10.0.0.0/16"      # Blocked subnet
-]
-```
-
-Processing order:
-1. Blacklist (immediate deny if matched)
-2. Whitelist (must match if configured)
-3. Rate limiting
-4. Authentication
-
-## Connection Management
-
-### TCP Keep-Alive
-
-```toml
-[pipelines.sources.tcp]
-keep_alive = true
-keep_alive_period_ms = 30000  # 30 seconds
-```
-
-Benefits:
-- Detect dead connections
-- Prevent connection timeout
-- Maintain NAT mappings
-
-### Connection Timeouts
-
-```toml
-[pipelines.sources.http]
-read_timeout_ms = 10000   # 10 seconds
-write_timeout_ms = 10000  # 10 seconds
-
-[pipelines.sinks.tcp_client]
-dial_timeout = 10         # Connection timeout
-write_timeout = 30        # Write timeout
-read_timeout = 10         # Read timeout
-```
-
-### Connection Limits
-
-Global limits:
-```toml
-max_connections = 100     # Total concurrent connections
-```
-
-Per-IP limits:
-```toml
-max_connections_per_ip = 10
-```
-
-## Heartbeat Configuration
-
-Keep connections alive with periodic heartbeats:
-
-### HTTP Sink Heartbeat
-
-```toml
-[pipelines.sinks.http.heartbeat]
-enabled = true
-interval_ms = 30000
 include_timestamp = true
-include_stats = false
+include_stats     = false
-format = "comment"  # comment|event|json
+format            = "txt"      # txt | json | raw
 ```
 
-Formats:
+Use it to keep idle SSE clients, TCP clients, and chain links from being
-- **comment**: SSE comment (`: heartbeat`)
+reaped by intermediate NAT or proxy timeouts, and to make an idle pipeline
-- **event**: SSE event with data
+visibly alive.
-- **json**: JSON-formatted heartbeat
 
-### TCP Sink Heartbeat
+> `format = "comment"` (SSE `:` comment framing) is rejected by validation
+> despite appearing in older documentation and in a still-present code branch.
+> A pipeline configured with it fails to start.
+
+## Reconnection
+
+Chain sinks reconnect on their own. Both use exponential backoff between
+`backoff_min_ms` and `backoff_max_ms` with ±20 % jitter, and both are
+interruptible by shutdown.
 
 ```toml
-[pipelines.sinks.tcp.heartbeat]
+backoff_min_ms = 500
-enabled = true
+backoff_max_ms = 30000
-interval_ms = 30000
-include_timestamp = true
-include_stats = false
-format = "json"  # json|txt
 ```
 
-## Network Protocols
+The connection is established lazily, so an edge node starts cleanly even when
+its relay is down and connects as soon as the relay appears. Reconnect counts
+are reported in the sink's `reconnects` statistic.
 
-### HTTP/HTTPS
+Server-side sinks (`tcp`, `http`) do not reconnect; clients are expected to
+retry. Browsers reconnect SSE streams automatically.
 
-- HTTP/1.1 and HTTP/2 support
+## Protocol Details
-- Persistent connections
-- Chunked transfer encoding
-- Server-Sent Events (SSE)
 
-### TCP
+**HTTP sink (SSE)** — HTTP/1.1 in plaintext; HTTP/2 is negotiated via ALPN when
+TLS is enabled. Only `GET` is routed to the stream and status paths. Each event
+is framed as one `data:` line per newline in the payload, so multi-line entries
+stream intact. Response headers set `Cache-Control: no-cache`,
+`X-Accel-Buffering: no`, and `Access-Control-Allow-Origin: *`.
 
-- Raw TCP sockets
+**TCP sink** — raw payload bytes, no framing added by the sink. Whether entries
-- Newline-delimited protocol
+are newline-delimited depends on the formatter.
-- Binary-safe transmission
-- No encryption available
 
-## Port Configuration
+**Chain transports** — see [Chaining](chaining.md) for the hello preamble,
-
+headers, and entry encoding.
-### Default Ports
-
-| Service | Default Port | Protocol |
-|---------|--------------|----------|
-| HTTP Source | 8081 | HTTP/HTTPS |
-| HTTP Sink | 8080 | HTTP/HTTPS |
-| TCP Source | 9091 | TCP |
-| TCP Sink | 9090 | TCP |
-
-### Port Conflict Prevention
-
-LogWisp validates port usage at startup:
-- Detects port conflicts across pipelines
-- Prevents duplicate bindings
-- Suggests alternative ports
-
-## Network Security
-
-### Best Practices
-
-1. **Use TLS for HTTP** connections when possible
-2. **Implement rate limiting** to prevent DoS
-3. **Configure IP whitelists** for restricted access
-4. **Enable authentication** for all network endpoints
-5. **Use non-standard ports** to reduce scanning exposure
-6. **Monitor connection metrics** for anomalies
-7. **Set appropriate timeouts** to prevent resource exhaustion
-
-### Security Warnings
-
-- TCP connections are **always unencrypted**
-- HTTP Basic/Token auth **requires TLS**
-- Avoid `skip_verify` in production
-- Never expose unauthenticated endpoints publicly
-
-## Load Balancing
-
-### Client-Side Load Balancing
-
-Configure multiple endpoints (future feature):
-```toml
-[[pipelines.sinks.http_client]]
-urls = [
-    "https://log1.example.com/ingest",
-    "https://log2.example.com/ingest"
-]
-strategy = "round-robin"  # round-robin|random|least-conn
-```
-
-### Server-Side Considerations
-
-- Use reverse proxy for load distribution
-- Configure session affinity if needed
-- Monitor individual instance health
 
 ## Troubleshooting
 
-### Common Issues
+**Connection refused**
+- Confirm the pipeline started; a bind failure is logged at ERROR.
+- Confirm you are dialing IPv4. `localhost` may resolve to `::1`.
+- Check the port is not already bound by another pipeline in the same process.
 
-**Connection Refused**
+**TLS handshake failure**
-- Check firewall rules
+- `client didn't provide a certificate` — the listener has `client_auth = true`
-- Verify service is running
+  and the dialer has no `cert_file`/`key_file`.
-- Confirm correct port/host
+- `certificate signed by unknown authority` — the dialer's `ca_file` does not
+  contain the issuer of the server certificate, or the listener's
+  `client_ca_file` does not contain the issuer of the client certificate.
+- `certificate is not valid for any names` / SAN mismatch — the dialed `host`
+  is not covered by the server certificate's SANs; set `server_name`.
+- `protocol version not supported` — one side is pinned to `min_version = "1.3"`
+  and the other cannot negotiate it.
+- Handshake failures appear as WARN with the remote address, and increment
+  `tls_handshake_errors`.
 
-**TLS Handshake Failure**
+**Rejected after a successful handshake**
-- Verify certificate validity
+- `auth: identity "..." is not allowed` — the certificate is valid and chains to
-- Check certificate chain
+  the CA, but the identity is not in `auth.allow` / `auth.allow_patterns`. On
-- Confirm TLS versions match
+  TCP the connection is closed; on HTTP the answer is `403`.
+- `auth: peer certificate carries no <mode> identity` — `auth.identity` names a
+  field the certificate does not populate, e.g. `san_dns` on a CN-only leaf.
+- `auth: node_binding "assert": declared node "..." does not match identity` —
+  the sender's `node` option and its certificate disagree. Fix one, or use
+  `node_binding = "force"` to let the certificate win silently.
+- On a dialer, the same message inside `Chain connect failed` means the
+  *server* was refused: its certificate identity is not in the sink's
+  `auth.allow`.
+- Rejections appear as WARN and increment `auth_rejected`.
 
-**Rate Limit Exceeded**
+**Entries not arriving over a chain link**
-- Adjust rate limit parameters
+- Check the sink's `connected` statistic and its `reconnects` count.
-- Add IP to whitelist
+- Check the source's `auth_rejected` — an allow-list miss looks exactly like a
-- Implement client-side throttling
+  network fault from the sender's side.
+- Check the source's `parse_errors` — a version skew shows up here.
+- On `http_chain`, remember entries wait up to `flush_interval_ms` before a
+  batch is sent.
 
-**Connection Timeout**
+**Entries arriving under an unexpected node label**
-- Increase timeout values
+- `auth.node_binding` defaults to `force` when `auth.type = "mtls"`, which
-- Check network latency
+  relabels every entry with the sender's certificate identity. If a dashboard
-- Verify keep-alive settings
+  suddenly shows a different label, that is why. Use `node_binding = "assert"`
+  to keep upstream origin labels on relay-to-relay hops, or `"none"` to leave
+  `trust_node` in charge.
+
+**Clients connect but see nothing**
+- The pipeline may be filtering everything out; check `flow.filters` stats.
+- The rate limiter may be dropping everything; check `rate_limiter` stats.
+- Nothing may be arriving from the sources; check source `total_entries`.
+
+**Entries missing under load**
+- Compare `dropped_writes` (per-client queue full — raise
+  `client_buffer_size`) against `total_dropped_by_sink` (sink input queue full
+  — raise `buffer_size` or reduce sink latency).

doc/operations.md

@@ -1,358 +1,308 @@
 # Operations Guide
 
-Running, monitoring, and maintaining LogWisp in production.
+Running, monitoring, and maintaining LogWisp.
 
-## Starting LogWisp
+## Starting
-
-### Manual Start
 
 ```bash
-# Foreground with default config
+# foreground, explicit config
+logwisp -c /etc/logwisp/logwisp.toml
+
+# no config: built-in demo pipeline (random source -> stdout)
 logwisp
-
-# Background mode
-logwisp --background
-
-# With specific configuration
-logwisp --config /etc/logwisp/production.toml
 ```
 
-### Service Management
+There is no built-in daemon mode. Run LogWisp in the foreground under a
+supervisor — systemd, rc.d, or a container runtime — which is where restart,
+log capture, and resource limits belong. See [Installation](installation.md).
+
+**systemd**
 
-**Linux (systemd):**
 ```bash
 sudo systemctl start logwisp
-sudo systemctl stop logwisp
-sudo systemctl restart logwisp
 sudo systemctl status logwisp
+sudo journalctl -u logwisp -f
 ```
 
-**FreeBSD (rc.d):**
+**FreeBSD rc.d**
+
 ```bash
 sudo service logwisp start
-sudo service logwisp stop
-sudo service logwisp restart
 sudo service logwisp status
 ```
 
-## Configuration Management
+## Configuration Changes
 
-### Hot Reload
+### Hot reload
 
-Enable automatic configuration reload:
 ```toml
-config_auto_reload = true
+auto_reload = true
 ```
 
-Or via command line:
+or send a signal:
-```bash
-logwisp --config-auto-reload
-```
 
-Trigger manual reload:
 ```bash
 kill -HUP $(pidof logwisp)
-# or
-kill -USR1 $(pidof logwisp)
 ```
 
-### Configuration Validation
+Reload constructs a new service from the new configuration **before** tearing
+the old one down, so a broken configuration leaves the running service intact
+and logs the failure:
+
+```
+ERROR msg="Failed to bootstrap new service, keeping old service running" error=...
+```
+
+What reload does *not* do:
+
+- Re-apply `logging.*`; application logging is configured once at startup.
+- Preserve connections. Listeners close and reopen, and every SSE, TCP, and
+  chain client is disconnected. Chain sinks reconnect on their own backoff;
+  browsers reconnect SSE automatically; raw TCP consumers must retry themselves.
+- Reload certificates without a reload — certificate files are read at plugin
+  construction, so rotation requires `SIGHUP`.
+
+Plan reloads on a busy relay the way you would plan a restart.
+
+### Checking a configuration
+
+There is no validate-only mode. To check a file, start it with debug logging and
+watch for pipeline startup:
 
-Test configuration without starting:
 ```bash
-logwisp --config test.toml --quiet --disable-status-reporter
+logwisp -c candidate.toml --logging.level=debug --logging.output=stderr
 ```
 
-Check for errors:
+Success looks like `Created source instance`, `Created sink instance`, and
-- Port conflicts
+`Starting pipeline` for each pipeline. Failures name the pipeline and the
-- Invalid patterns
+offending key:
-- Missing required fields
+
-- File permissions
+```
+ERROR msg="Failed to create pipeline" pipeline=app error="failed to create sink out: port: must be 1-65535, got 0"
+```
+
+Remember that most validation lives in plugin constructors, so a config only
+proves itself when the pipeline is actually built.
 
 ## Monitoring
 
-### Status Reporter
+### Status reporter
 
-Built-in periodic status logging (30-second intervals):
+Enabled by default, every 30 seconds. It logs at **DEBUG**, so it produces
+nothing unless `logging.level = "debug"` — a common surprise.
 
-```
-[INFO] Status report active_pipelines=2 time=15:04:05
-[INFO] Pipeline status pipeline=app entries_processed=10523
-[INFO] Pipeline status pipeline=system entries_processed=5231
-```
-
-Disable if not needed:
 ```toml
-disable_status_reporter = true
+status_reporter = true
+
+[logging]
+level = "debug"
 ```
 
-### HTTP Status Endpoint
+It emits a service summary and then walks each pipeline, flattening scalar
+statistics into log fields and recursing into flow, rate limiter, filter,
+source, and sink stats.
+
+Disable with `status_reporter = false`.
+
+### HTTP status endpoint
+
+When a pipeline has an `http` sink:
 
-When using HTTP sink:
 ```bash
-curl http://localhost:8080/status | jq .
+curl -s http://127.0.0.1:8080/status | jq .
 ```
 
-Response structure:
 ```json
 {
-  "uptime": "2h15m30s",
+  "service": "LogWisp",
-  "pipelines": {
+  "version": "v0.16.0",
-    "default": {
+  "instance_id": "sse",
-      "sources": 1,
+  "server": {
-      "sinks": 2,
+    "type": "http",
-      "processed": 15234,
+    "host": "0.0.0.0",
-      "filtered": 523,
+    "port": 8080,
-      "dropped": 12
+    "tls": false,
-    }
+    "active_clients": 3,
+    "buffer_size": 1000,
+    "uptime_seconds": 8130
+  },
+  "endpoints": { "stream": "/stream", "status": "/status" },
+  "statistics": {
+    "total_processed": 15234,
+    "dropped_writes": 12,
+    "rejected_clients": 0
   }
 }
 ```
 
-### Metrics Collection
+This endpoint is scoped to one sink, not to the whole process, and it is
+**unauthenticated**. Bind it to a trusted interface.
 
-Track via logs:
+### Metrics worth watching
-- Total entries processed
+
-- Entries filtered
+| Metric | Where | Meaning if rising |
-- Entries dropped
+|--------|-------|-------------------|
-- Active connections
+| `dropped_entries` | source | Downstream cannot keep up with the source |
-- Buffer utilization
+| `total_dropped` | flow | Rate limit or filters are discarding entries (often intended) |
+| `total_dropped_by_sink` | pipeline | A sink's input queue is full |
+| `dropped_writes` | tcp/http sink | A specific client is too slow |
+| `rejected_conns` / `rejected_clients` | tcp/http sink, tcp_chain source | `max_connections` is being hit |
+| `tls_handshake_errors` | tcp sink, tcp_chain source | Certificate or version mismatch, or scanning |
+| `parse_errors` | chain source | Protocol or version skew upstream |
+| `reconnects` | chain sink | Unstable link or a flapping downstream |
+| `dropped_batches` | http_chain sink | Downstream rejecting batches permanently |
+| `synthesized` | chain sink | Events reaching the sink without structure |
 
 ## Log Management
 
-### LogWisp's Operational Logs
+LogWisp's own operational log:
-
-Configuration for LogWisp's own logs:
 
 ```toml
 [logging]
 output = "file"
-level = "info"
+level  = "info"
 
 [logging.file]
-directory = "/var/log/logwisp"
+directory         = "/var/log/logwisp"
-name = "logwisp"
+name              = "logwisp"
-max_size_mb = 100
+max_size_mb       = 100
-retention_hours = 168
+max_total_size_mb = 1000
+retention_hours   = 168.0
 ```
 
-### Log Rotation
+Rotation is automatic on size, with a total-size cap and a retention window.
+There is no signal to reopen log files, so do not move files out from under
+LogWisp and expect it to reattach — let it rotate, or restart it.
 
-Automatic rotation based on:
+Production level: `info`, or `warn` on a busy relay. Avoid `debug` under load:
-- File size threshold
+the filter stage logs several lines per entry evaluated.
-- Total size limit
-- Retention period
-
-Manual rotation:
-```bash
-# Move current log
-mv /var/log/logwisp/logwisp.log /var/log/logwisp/logwisp.log.1
-# Send signal to reopen
-kill -USR1 $(pidof logwisp)
-```
-
-### Log Levels
-
-Operational log levels:
-- **debug**: Detailed debugging information
-- **info**: General operational messages
-- **warn**: Warning conditions
-- **error**: Error conditions
-
-Production recommendation: `info` or `warn`
 
 ## Performance Tuning
 
-### Buffer Sizing
+### Buffers
 
-Adjust buffers based on load:
+Raise `buffer_size` when `total_dropped_by_sink` is climbing but the sink itself
+is healthy — that is a burst-absorption problem. Raise `client_buffer_size` when
+`dropped_writes` is climbing for network sinks; that is a slow-consumer problem,
+and a bigger buffer only buys time.
 
 ```toml
-# High-volume source
+[pipelines.plugin_sinks.config]
-[[pipelines.sources]]
+buffer_size        = 5000
-type = "http"
+client_buffer_size = 1024
-[pipelines.sources.http]
-buffer_size = 5000  # Increase for burst traffic
-
-# Slow consumer sink
-[[pipelines.sinks]]
-type = "http_client"
-[pipelines.sinks.http_client]
-buffer_size = 10000  # Larger buffer for slow endpoints
-batch_size = 500     # Larger batches
 ```
 
-### Rate Limiting
+### Rate limiting
-
-Protect against overload:
 
 ```toml
-[pipelines.rate_limit]
+[pipelines.flow.rate_limit]
-rate = 1000.0        # Entries per second
+rate                 = 1000.0
-burst = 2000.0       # Burst capacity
+burst                = 2000.0
-policy = "drop"      # Drop excess entries
+policy               = "drop"
+max_entry_size_bytes = 65536
 ```
 
-### Connection Limits
+Two behaviours to keep in mind: the limiter does not exist at all when
+`rate <= 0`, and `policy = "pass"` short-circuits the size cap as well as the
+rate check. Enforcing `max_entry_size_bytes` therefore requires `rate > 0` and
+`policy = "drop"`.
 
-Prevent resource exhaustion:
+### Formatting
 
-```toml
+`raw` is the cheapest and skips sanitization; `json` costs the most. The
-[pipelines.sources.http.net_limit]
+formatter serializes on a mutex, so it is the one shared bottleneck in a
-max_connections_total = 1000
+pipeline — splitting work across pipelines parallelizes it.
-max_connections_per_ip = 50
+
-```
+### Chain batching
+
+`http_chain` trades latency for efficiency. Lower `flush_interval_ms` for
+freshness, raise `max_batch_count` and `max_batch_bytes` for throughput. Use
+`tcp_chain` when per-entry latency matters.
 
 ## Troubleshooting
 
-### Common Issues
+**Nothing appears at the sink**
 
-**High Memory Usage**
+Walk the pipeline in order and read the counters: source `total_entries` (is
-- Check buffer sizes
+anything being produced?), flow `total_dropped` (filters or rate limit?),
-- Monitor goroutine count
+pipeline `total_dropped_by_sink` (sink backed up?), sink `total_processed`.
-- Review retention settings
 
-**Dropped Entries**
+**File source reads nothing**
-- Increase buffer sizes
-- Add rate limiting
-- Check sink performance
 
-**Connection Errors**
+- The watcher seeks to end-of-file on start; only content appended afterwards is
-- Verify network connectivity
+  read. Positions are in memory, so a restart re-seeks to end and anything
-- Check firewall rules
+  written during the downtime is lost. `from = "start"` reads each file whole
-- Review TLS certificates
+  instead, and replays it on every restart.
+- `pattern` is a filename glob with `*` and `?` only, and matching is not
+  recursive.
+- `check_interval_ms` governs how quickly a *new file* is noticed; tailing an
+  open file polls at a fixed 100 ms.
 
-### Debug Mode
+**High memory use**
 
-Enable detailed logging:
+Buffers are bounded, so unbounded growth almost always means many buffers:
-```bash
+count sinks × `buffer_size`, plus clients × `client_buffer_size`. A `tcp_chain`
-logwisp --logging.level=debug --logging.output=stderr
+sink blocked on an unreachable downstream also holds its full input queue.
-```
 
-### Health Checks
+**Chain link not delivering**
 
-Implement external monitoring:
+Check `connected` and `reconnects` on the sink, `parse_errors` on the source,
-```bash
+and remember `http_chain` waits up to `flush_interval_ms`. For TLS problems see
-#!/bin/bash
+[Networking](networking.md#troubleshooting).
-# Health check script
-if ! curl -sf http://localhost:8080/status > /dev/null; then
-  echo "LogWisp health check failed"
-  exit 1
-fi
-```
 
-## Backup and Recovery
+**Environment variable override has no effect**
 
-### Configuration Backup
+LogWisp currently reads these **without** the `LOGWISP_` prefix — `QUIET`,
-
+`LOGGING_LEVEL`, and so on. Array-indexed paths cannot be set from the
-```bash
+environment or the command line at all.
-# Backup configuration
-cp /etc/logwisp/logwisp.toml /backup/logwisp-$(date +%Y%m%d).toml
-
-# Version control
-git add /etc/logwisp/
-git commit -m "LogWisp config update"
-```
-
-### State Recovery
-
-LogWisp maintains minimal state:
-- File read positions (automatic)
-- Connection state (automatic)
-
-Recovery after crash:
-1. Service automatically restarts (systemd/rc.d)
-2. File sources resume from last position
-3. Network sources accept new connections
-4. Clients reconnect automatically
 
 ## Security Operations
 
-### Certificate Management
+**Certificate rotation**
 
-Monitor certificate expiration:
 ```bash
-openssl x509 -in /path/to/cert.pem -noout -enddate
+openssl x509 -in /etc/logwisp/tls/relay.crt -noout -enddate
 ```
 
-Rotate certificates:
+Certificates load at plugin construction, so rotation is: write the new files,
-1. Generate new certificates
+then `kill -HUP`. Automate the expiry check; nothing in LogWisp warns you.
-2. Update configuration
-3. Reload service (SIGHUP)
 
-### Credential Rotation
+**Access review**
 
-Update authentication:
+With `tls` alone, any certificate signed by the configured `client_ca_file` is
-```bash
+accepted, so "access review" means reviewing what your CA has issued. Add an
-# Generate new credentials
+`auth` block with an explicit `allow` list and the review becomes the config
-logwisp auth -u admin -b
+file itself: the identities listed there are the ones that can connect, and
+removing one plus a `SIGHUP` is the revocation path. Authorized identities are
+recorded in session metadata as `auth_identity`; rejections are counted in
+`auth_rejected` and logged at WARN. See
+[Security](security.md#the-auth-block).
 
-# Update configuration
+**Secret leakage**
-vim /etc/logwisp/logwisp.toml
 
-# Reload service
+Filters are the only redaction mechanism, and they drop whole entries rather
-kill -HUP $(pidof logwisp)
+than masking parts of them. See [Filters](filters.md#common-recipes).
-```
-
-### Access Auditing
-
-Monitor access patterns:
-- Review connection logs
-- Track authentication failures
-- Monitor rate limit hits
 
 ## Maintenance
 
-### Planned Maintenance
+**Upgrades**
 
-1. Notify users of maintenance window
+1. Read the changelog for configuration-schema changes.
-2. Stop accepting new connections
+2. Start the new binary against the current configuration in a scratch
-3. Drain existing connections
+   environment.
-4. Perform maintenance
+3. Stop the old process, install the new binary, start it.
-5. Restart service
+4. Confirm each pipeline started and that counters are advancing.
 
-### Upgrade Process
+**Backup**
 
-1. Download new version
+Configuration files and TLS material are the only durable state worth backing
-2. Test with current configuration
+up. LogWisp keeps no persistent runtime state: file read positions live in
-3. Stop old version
+memory, connections are re-established on restart, and in-flight entries are
-4. Install new version
+lost.
-5. Start service
-6. Verify operation
 
-### Cleanup Tasks
+**Redundancy**
 
-Regular maintenance:
+Because there is no persistence, availability comes from topology, not from
-- Remove old log files
+LogWisp itself. Give each edge two chain sinks pointing at two relays if you
-- Clean temporary files
+need to survive a relay outage, and accept that this duplicates entries
-- Verify disk space
+downstream.
-- Update documentation
-
-## Disaster Recovery
-
-### Backup Strategy
-
-- Configuration files: Daily
-- TLS certificates: After generation
-- Authentication credentials: Secure storage
-
-### Recovery Procedures
-
-Service failure:
-1. Check service status
-2. Review error logs
-3. Verify configuration
-4. Restart service
-
-Data loss:
-1. Restore configuration from backup
-2. Regenerate certificates if needed
-3. Recreate authentication credentials
-4. Restart service
-
-### Business Continuity
-
-- Run multiple instances for redundancy
-- Use load balancer for distribution
-- Implement monitoring alerts
-- Document recovery procedures

doc/security.md

@@ -0,0 +1,395 @@
+# Security
+
+This page covers LogWisp's transport security: what it protects, how to
+configure it, and — equally important — what it does not yet do.
+
+## Current State
+
+| Capability | Status |
+|------------|--------|
+| TLS 1.2 / 1.3 on all network sources and sinks | Implemented |
+| Server certificate verification by dialers | Implemented |
+| Mutual TLS (client certificate required and verified) | Implemented at the transport layer |
+| Peer identity recorded per session | Implemented |
+| Authorization from peer identity (allow-lists, node binding) | Implemented — see [The Auth Block](#the-auth-block) |
+| Authentication on the `http` sink's stream and status endpoints | Implemented, via the auth block |
+| Server identity pinning by dialers | Implemented, via the auth block |
+| Certificate revocation lists (CRL) or OCSP | **Not implemented** — revoke by editing the allow-list |
+| Password, token, or SCRAM authentication | **Removed**; not currently available |
+| IP allow/deny lists, per-IP connection or request limits | **Not implemented** |
+
+Earlier releases carried basic-auth, bearer-token, and SCRAM authentication.
+Those were removed during the move to the plugin/flow architecture and the
+switch to standard-library networking. Certificates are the one credential the
+transport still carries, so they are what authentication is built on: the `tls`
+block establishes that a peer chains to your CA, and the `auth` block decides
+which peers that CA vouches for may actually do what.
+
+## The TLS Block
+
+One option shape serves both roles, so the configuration reads the same
+wherever it appears. Which keys matter depends on whether the plugin listens or
+dials.
+
+```toml
+[pipelines.plugin_sources.config.tls]     # or plugin_sinks.config.tls
+enabled              = false
+cert_file            = ""
+key_file             = ""
+client_auth          = false
+client_ca_file       = ""
+ca_file              = ""
+server_name          = ""
+insecure_skip_verify = false
+min_version          = "1.3"
+```
+
+| Option | Role | Default | Description |
+|--------|------|---------|-------------|
+| `enabled` | both | `false` | Master switch; when false the whole block is ignored |
+| `cert_file` | both | — | Local certificate. **Required** for listeners; optional client identity for dialers |
+| `key_file` | both | — | Private key for `cert_file`. Must be set together with it |
+| `client_auth` | listener | `false` | Require and verify a client certificate (mTLS) |
+| `client_ca_file` | listener | — | CA bundle used to verify client certificates. **Required** when `client_auth` is true |
+| `ca_file` | dialer | system store | CA bundle used to verify the server certificate |
+| `server_name` | dialer | the configured `host` | SNI and certificate name to verify against |
+| `insecure_skip_verify` | dialer | `false` | Disable server verification |
+| `min_version` | both | `"1.3"` | `"1.2"` or `"1.3"` |
+
+**Roles by plugin:**
+
+| Plugin | Role | Keys that apply |
+|--------|------|-----------------|
+| `tcp` sink, `http` sink | Listener | `cert_file`, `key_file`, `client_auth`, `client_ca_file`, `min_version` |
+| `tcp_chain` source, `http_chain` source | Listener | same as above |
+| `tcp_chain` sink, `http_chain` sink | Dialer | `ca_file`, `server_name`, `insecure_skip_verify`, `cert_file`, `key_file`, `min_version` |
+
+> `min_version` takes `"1.2"` or `"1.3"`. The older `"TLS1.2"` spelling from
+> pre-restructure releases is rejected. There is no `max_version` and no
+> `cipher_suites` option; TLS 1.3 suites are not configurable in Go, and the
+> 1.2 defaults are the standard library's.
+
+### Validation
+
+Misconfiguration fails at plugin construction, before the pipeline starts:
+
+- a listener with `enabled = true` and no `cert_file`/`key_file`
+- `client_auth = true` with no `client_ca_file`
+- a dialer with only one of `cert_file` / `key_file`
+- a certificate or key that will not load, or a CA file containing no
+  certificates
+- a `min_version` that is neither `"1.2"` nor `"1.3"`
+
+## The Auth Block
+
+TLS answers "is this channel private, and does the peer chain to a CA". Auth
+answers "may *this* peer do *this*". They are separate blocks because they are
+separate questions, and because a later non-certificate method should be able to
+reuse the second one.
+
+```toml
+[pipelines.plugin_sources.config.auth]    # or plugin_sinks.config.auth
+type           = "none"                   # none | mtls
+identity       = "cn"                     # cn | san_dns | san_uri | san_email
+allow          = []
+allow_patterns = []
+node_binding   = "force"                  # chain sources only
+```
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `type` | string | `none` | `none` ignores the whole block; `mtls` authorizes by certificate identity |
+| `identity` | string | `cn` | Which certificate field carries the identity |
+| `allow` | []string | `[]` | Exact identities to admit |
+| `allow_patterns` | []string | `[]` | RE2 patterns matched against the identity; anchor them yourself |
+| `node_binding` | string | `force` under `mtls` | Chain sources only: `none`, `assert`, or `force` |
+
+**Roles by plugin:**
+
+| Plugin | Role | Decides |
+|--------|------|---------|
+| `tcp_chain` source, `http_chain` source | Listener | Which senders may ingest, and what node label their entries carry |
+| `tcp` sink, `http` sink | Listener | Which clients may read the stream (and, on `http`, the status endpoint) |
+| `tcp_chain` sink, `http_chain` sink | Dialer | Which server identity to accept, beyond hostname verification |
+
+### Identity
+
+The identity is one string pulled from the peer's verified leaf certificate.
+The handshake has already checked the chain, signature, and validity window, so
+this is pure field selection.
+
+| Mode | Source | Typical use |
+|------|--------|-------------|
+| `cn` (default) | `Subject.CommonName` | Matches the existing `tls_peer_cn` metadata |
+| `san_dns` | first DNS SAN | Host identities |
+| `san_uri` | first URI SAN | SPIFFE-style IDs |
+| `san_email` | first email SAN | Operator identities |
+
+A certificate with no usable value in the chosen field is rejected. An empty
+identity is a refusal, not an empty match.
+
+### The allow list
+
+`allow` is an exact-match set; `allow_patterns` holds RE2 patterns. An identity
+passes if it appears in either.
+
+Leaving **both** empty under `type = "mtls"` admits any identity the CA vouches
+for. That is deliberate — it is how you enable node binding without enumerating
+a whole fleet — but it is announced rather than silent:
+
+```
+WARN msg="Auth policy admits any identity the configured CA vouches for"
+     component=tcp_chain_source instance_id=in_tcp
+     hint="set auth.allow or auth.allow_patterns to authorize named peers"
+```
+
+Anchor your patterns. `allow_patterns = ["edge-\\d{2}"]` matches
+`evil-edge-01-impostor`; `["^edge-\\d{2}$"]` does not.
+
+### Node binding
+
+`node_binding` applies only to the chain sources, and it overrides `trust_node`.
+
+| Value | Connection label | Per-entry `node` field |
+|-------|------------------|------------------------|
+| `none` | `trust_node` governs | `trust_node` governs |
+| `assert` | Must equal the identity; a mismatch or an omission is rejected | `trust_node` governs |
+| `force` | Ignored; the identity is used | Overwritten with the identity |
+
+Use **`force`** on an ingest boundary you do not trust. Every entry is
+relabelled, so a compromised edge cannot smuggle a foreign origin through the
+per-entry `node` field either. It is the default under `type = "mtls"`.
+
+Use **`assert`** on a relay-to-relay hop. The relay must prove its own identity —
+a mismatch fails loudly instead of being silently corrected — but the entries it
+forwards keep the origin labels stamped at the first hop, so multi-hop
+attribution survives.
+
+When binding is active the source says so at startup:
+
+```
+INFO msg="Node labels bound to peer identity; trust_node is ignored"
+     component=tcp_chain_source node_binding=force trust_node=true
+```
+
+### Dialer-side pinning
+
+On a chain sink, the same block pins the *server's* identity. Hostname
+verification already proves the server holds a certificate valid for the address
+you dialed; pinning additionally requires that certificate to name an identity
+you listed.
+
+```toml
+[pipelines.plugin_sinks.config.auth]
+type  = "mtls"
+allow = ["relay.internal"]
+```
+
+The check runs as part of the handshake, so a server the policy rejects never
+receives an entry — the sink's normal backoff loop handles it like any other
+connect failure. `insecure_skip_verify` is refused alongside `type = "mtls"`:
+an identity read from an unverified chain is a claim, not a fact.
+
+### Validation
+
+Misconfiguration fails at plugin construction, before the pipeline starts:
+
+- `type = "mtls"` on a listener without `tls.enabled` **and** `tls.client_auth`
+- `type = "mtls"` on a dialer without `tls.enabled`, or with
+  `tls.insecure_skip_verify`
+- an `identity` that is not one of the four modes
+- an `allow_patterns` entry that does not compile
+- a `node_binding` that is not one of the three values, or one set on a plugin
+  that has no node concept
+
+Errors read like `auth: type "mtls" requires tls.client_auth`.
+
+## Enabling mTLS
+
+### 1. Generate a CA and certificates
+
+LogWisp no longer ships a certificate-generation subcommand; the `logwisp tls`
+command was removed with the rest of the CLI restructure. Use `openssl`,
+`cfssl`, `step-cli`, or your existing PKI.
+
+```bash
+# CA
+openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \
+  -keyout ca.key -out ca.crt -subj "/CN=LogWisp CA"
+
+# Relay (server) certificate — SAN must match how clients address it
+openssl req -newkey rsa:2048 -nodes -keyout relay.key -out relay.csr \
+  -subj "/CN=relay.internal"
+openssl x509 -req -in relay.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
+  -out relay.crt -days 825 \
+  -extfile <(printf "subjectAltName=DNS:relay.internal\nextendedKeyUsage=serverAuth")
+
+# Edge (client) certificate — CN identifies the node
+openssl req -newkey rsa:2048 -nodes -keyout edge-01.key -out edge-01.csr \
+  -subj "/CN=edge-01"
+openssl x509 -req -in edge-01.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
+  -out edge-01.crt -days 825 \
+  -extfile <(printf "extendedKeyUsage=clientAuth")
+```
+
+The server certificate's SAN must cover the address clients dial. Dialers seed
+`ServerName` from the configured `host`, so an IP literal in `host` requires an
+IP SAN, and a DNS name requires a DNS SAN. Override with `server_name` when the
+dialed address and the certificate name legitimately differ.
+
+### 2. Configure the listener
+
+```toml
+[pipelines.plugin_sources.config.tls]
+enabled        = true
+cert_file      = "/etc/logwisp/tls/relay.crt"
+key_file       = "/etc/logwisp/tls/relay.key"
+client_auth    = true
+client_ca_file = "/etc/logwisp/tls/ca.crt"
+min_version    = "1.3"
+
+[pipelines.plugin_sources.config.auth]
+type         = "mtls"
+allow        = ["edge-01", "edge-02"]
+node_binding = "force"
+```
+
+Without the `auth` block the listener accepts every certificate the CA issued.
+With it, only `edge-01` and `edge-02` may ingest, and their entries are labelled
+from their certificates rather than from whatever they declare.
+
+### 3. Configure the dialer
+
+```toml
+[pipelines.plugin_sinks.config.tls]
+enabled   = true
+ca_file   = "/etc/logwisp/tls/ca.crt"
+cert_file = "/etc/logwisp/tls/edge-01.crt"
+key_file  = "/etc/logwisp/tls/edge-01.key"
+min_version = "1.3"
+
+[pipelines.plugin_sinks.config.auth]
+type  = "mtls"
+allow = ["relay.internal"]
+```
+
+### 4. Verify
+
+Startup logs report the transport flags and the compiled policy:
+
+```
+INFO msg="TCP chain source initialized" ... tls=true mtls=true
+     auth="mtls identity=cn allow=[2 exact, 0 pattern(s)] node_binding=force"
+INFO msg="TCP chain sink initialized"   ... tls=true mtls=true
+     auth="mtls identity=cn allow=[1 exact, 0 pattern(s)] node_binding=none"
+```
+
+A client that presents no certificate is refused during the handshake:
+
+```
+WARN msg="TLS handshake failed" component=tcp_chain_source
+     remote_addr=127.0.0.1:53840 error="tls: client didn't provide a certificate"
+```
+
+A client whose certificate is valid but whose identity is not authorized gets
+past the handshake and is refused by the policy:
+
+```
+WARN msg="Connection rejected by auth policy" component=tcp_chain_source
+     remote_addr=127.0.0.1:33946 error="auth: identity \"edge-99\" is not allowed"
+```
+
+Handshake failures are counted in `tls_handshake_errors`; policy rejections in
+`auth_rejected`. Both appear in the status reporter and in the `http` sink's
+status endpoint. Accepted peers are recorded in session metadata as
+`auth_method` and `auth_identity`.
+
+`test/mtls-chain-test.sh` builds a throwaway PKI and exercises the whole surface
+end to end — run it with `--auto` to see each guarantee asserted.
+
+## What Each Layer Enforces
+
+**`tls` with `client_auth = true`** — a membership check. The peer holds a
+certificate chaining to `client_ca_file`, within its validity window, and holds
+the matching private key. An attacker without a CA-issued certificate cannot
+connect at all. What it does *not* decide is which CA-issued certificate: every
+one is equivalent at this layer.
+
+**`auth` with `type = "mtls"`** — an identity check, per listener:
+
+- only the identities you list may connect, so one CA can serve several trust
+  domains and a single peer can be withdrawn without touching the others
+- the chain `node` label is bound to the certificate, so a compromised edge
+  cannot attribute its entries to another host
+- the `http` sink's stream and status endpoints stop being open to anyone who
+  can reach the port
+
+**Revocation** is the allow-list, not a CRL. Remove the identity from `allow` /
+`allow_patterns` and send `SIGHUP`: the reload rebuilds every pipeline, so the
+change takes effect on the next connection and existing ones are dropped by the
+rebuild. No network call on the handshake path, and no window between revocation
+and the next CRL publication. See
+[mtls-auth-plan.md](mtls-auth-plan.md#not-implemented) for what CRL support
+would add.
+
+## Surfaces Without Access Control
+
+An `auth` block closes each of these. Without one, bind them to a trusted
+interface or front them with an authenticating proxy.
+
+| Surface | Exposure when `auth.type = "none"` |
+|---------|-----------------------------------|
+| `http` sink `stream_path` | Full log stream, with `Access-Control-Allow-Origin: *`, so any browser origin can read it |
+| `http` sink `status_path` | Host, port, TLS flag, uptime, client counts, throughput counters |
+| `tcp` sink | Full log stream to any client that connects |
+| `tcp_chain` / `http_chain` source | Ingest from any peer the CA vouches for, under any node label it claims |
+
+`max_connections` bounds concurrency on all of them but does not distinguish
+callers.
+
+Note that `auth` requires `client_auth = true`, which requires TLS. There is no
+way to authenticate a plaintext listener.
+
+## Operational Guidance
+
+**Certificates**
+
+- Use a dedicated CA for LogWisp so its trust decisions stay independent.
+- Keep leaf lifetimes short (90–825 days) and automate renewal.
+- Key files should be `0600` and owned by the service account.
+- Rotation requires a reload (`SIGHUP`), because certificates are loaded once at
+  plugin construction; there is no on-disk watch for certificate files.
+- Check expiry yourself: `openssl x509 -in relay.crt -noout -enddate`. Nothing
+  warns before a certificate lapses; it surfaces as a handshake failure.
+- Keep the identity field you authorize on stable across rotations. Reissuing a
+  leaf with a different CN silently drops the peer out of the allow list.
+
+**Deployment**
+
+- Prefer `min_version = "1.3"`. Drop to `"1.2"` only for a peer that genuinely
+  cannot do 1.3.
+- Never enable `insecure_skip_verify` outside a lab; it disables server
+  verification entirely and makes the connection trivially interceptable.
+- Bind listeners to specific interfaces rather than `0.0.0.0` where you can.
+- On any ingest port reachable from a network you do not fully control, set
+  `auth.type = "mtls"` with an explicit `allow` list. `trust_node = false` is the
+  fallback when certificates are not an option; it is unforgeable but labels
+  entries by remote address, which is useless behind NAT or a load balancer.
+- Run LogWisp as an unprivileged user with write access only to its own log and
+  configuration directories.
+
+**Log content**
+
+Logs routinely contain secrets that were never meant to leave the host. Filters
+are the available tool:
+
+```toml
+[[pipelines.flow.filters]]
+type = "exclude"
+patterns = ["password", "api[_-]?key", "authorization", "bearer ", "secret"]
+```
+
+Choose a sanitizer policy that matches the sink — `json` for JSON output,
+`txt` for files and consoles — so control characters in log data cannot break
+framing or inject terminal escapes downstream. See
+[Formatters](formatters.md).

doc/sinks.md

@@ -1,293 +1,365 @@
 # Output Sinks
 
-LogWisp sinks deliver processed log entries to various destinations.
+Sinks consume `core.TransportEvent` values — a formatted `Payload` plus the
+original structured `Entry` — and deliver them somewhere. Each sink is declared
+as a `[[pipelines.plugin_sinks]]` entry with an `id`, a `type`, and a
+type-specific `config` table.
 
-## Sink Types
+Registered types: `console`, `file`, `http`, `tcp`, `null`, `tcp_chain`,
+`http_chain`.
 
-### Console Sink
+Dispatch into a sink is non-blocking. A sink whose input queue is full drops the
+event *for itself only* and the pipeline counts it in `total_dropped_by_sink`;
+sibling sinks are unaffected.
 
-Output to stdout/stderr.
+---
+
+## console
+
+Writes formatted payloads to stdout or stderr.
 
 ```toml
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
+id   = "stdout"
 type = "console"
-
+[pipelines.plugin_sinks.config]
-[pipelines.sinks.console]
+target      = "stdout"
-target = "stdout"  # stdout|stderr|split
+buffer_size = 1000
-colorize = false
-buffer_size = 100
 ```
 
-**Configuration Options:**
-
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `target` | string | "stdout" | Output target (stdout/stderr/split) |
+| `target` | string | `stdout` | `stdout` or `stderr` |
-| `colorize` | bool | false | Enable colored output |
+| `buffer_size` | int | `1000` | Sink input queue depth |
-| `buffer_size` | int | 100 | Internal buffer size |
 
-**Target Modes:**
+> `split` is **not** a valid target for this sink and is rejected at startup.
-- **stdout**: All output to standard output
+> Level-based splitting exists only for LogWisp's own application log
-- **stderr**: All output to standard error
+> (`logging.output = "split"`).
-- **split**: INFO/DEBUG to stdout, WARN/ERROR to stderr
 
-### File Sink
+Payloads are written verbatim; the sink adds no framing. Whether entries are
+newline-terminated is decided by the formatter.
 
-Write logs to rotating files.
+---
+
+## file
+
+Rotating file writer.
 
 ```toml
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
+id   = "archive"
 type = "file"
-
+[pipelines.plugin_sinks.config]
-[pipelines.sinks.file]
+directory         = "/var/log/logwisp"
-directory = "./logs"
+name              = "output"
-name = "output"
+max_size_mb       = 100
-max_size_mb = 100
 max_total_size_mb = 1000
-min_disk_free_mb = 500
+min_disk_free_mb  = 0
-retention_hours = 168.0
+retention_hours   = 168.0
-buffer_size = 1000
+buffer_size       = 1000
-flush_interval_ms = 1000
+flush_interval_ms = 100
 ```
 
-**Configuration Options:**
-
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `directory` | string | Required | Output directory |
+| `directory` | string | **required** | Output directory |
-| `name` | string | Required | Base filename |
+| `name` | string | **required** | Base filename |
-| `max_size_mb` | int | 100 | Rotation threshold |
+| `max_size_mb` | int | `100` | Rotate when the active file reaches this size |
-| `max_total_size_mb` | int | 1000 | Total size limit |
+| `max_total_size_mb` | int | `1000` | Cap across all rotated files |
-| `min_disk_free_mb` | int | 500 | Minimum free disk space |
+| `min_disk_free_mb` | int | `0` | Free-space floor before writing; `0` = no floor |
-| `retention_hours` | float | 168 | Delete files older than |
+| `retention_hours` | float | `168.0` | Delete rotated files older than this |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `buffer_size` | int | `1000` | Sink input queue depth |
-| `flush_interval_ms` | int | 1000 | Force flush interval |
+| `flush_interval_ms` | int | `100` | Forced flush interval |
 
-**Features:**
+> `min_disk_free_mb` has an unusual default. The constructor replaces only
-- Automatic rotation on size
+> *negative* values with `100`; leaving the key unset yields `0`, which means no
-- Retention management
+> free-space floor. Set it explicitly if you want one.
-- Disk space monitoring
-- Periodic flushing
 
-### HTTP Sink
+The sink drives an internal writer configured for raw output with timestamps and
+levels disabled, so what lands on disk is exactly the formatted payload.
 
-SSE (Server-Sent Events) streaming server.
+---
+
+## null
+
+Discards everything, counting entries and bytes. Useful for benchmarking a
+source or flow in isolation.
 
 ```toml
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
+id   = "discard"
+type = "null"
+```
+
+No options. The input queue is fixed at 1000.
+
+---
+
+## http
+
+Server-Sent Events stream plus a JSON status endpoint.
+
+```toml
+[[pipelines.plugin_sinks]]
+id   = "sse"
 type = "http"
+[pipelines.plugin_sinks.config]
+host               = "0.0.0.0"
+port               = 8080
+stream_path        = "/stream"
+status_path        = "/status"
+buffer_size        = 1000
+client_buffer_size = 256
+write_timeout_ms   = 0
+max_connections    = 0
 
-[pipelines.sinks.http]
+[pipelines.plugin_sinks.config.tls]
-host = "0.0.0.0"
+enabled        = true
-port = 8080
+cert_file      = "/etc/logwisp/tls/server.crt"
-stream_path = "/stream"
+key_file       = "/etc/logwisp/tls/server.key"
-status_path = "/status"
+client_auth    = true
-buffer_size = 1000
+client_ca_file = "/etc/logwisp/tls/client-ca.crt"
-max_connections = 100
+
-read_timeout_ms = 10000
+[pipelines.plugin_sinks.config.auth]
-write_timeout_ms = 10000
+type  = "mtls"
+allow = ["viewer-01"]
 ```
 
-**Configuration Options:**
-
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `host` | string | "0.0.0.0" | Bind address |
+| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
-| `port` | int | Required | Listen port |
+| `port` | int | **required** | Listen port |
-| `stream_path` | string | "/stream" | SSE stream endpoint |
+| `stream_path` | string | `/stream` | SSE endpoint; must start with `/` |
-| `status_path` | string | "/status" | Status endpoint |
+| `status_path` | string | `/status` | Status endpoint; must start with `/` and differ from `stream_path` |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `buffer_size` | int | `1000` | Sink input queue depth |
-| `max_connections` | int | 100 | Maximum concurrent clients |
+| `client_buffer_size` | int | `256` | Per-client send queue depth |
-| `read_timeout_ms` | int | 10000 | Read timeout |
+| `write_timeout_ms` | int | `0` | Per-event write deadline; `0` = none |
-| `write_timeout_ms` | int | 10000 | Write timeout |
+| `max_connections` | int | `0` | Concurrent stream cap; `0` = unlimited |
+| `tls` | table | — | Listener TLS; see [Security](security.md) |
+| `auth` | table | — | Client authorization; see [Security](security.md#the-auth-block) |
 
-**Heartbeat Configuration:**
+**Behaviour**
+
+- Only `GET` is routed to either path; anything else gets `405`.
+- With an `auth` block, one middleware gates **both** endpoints: an
+  unauthorized client gets `403` with no body detail, and the rejection is
+  logged at WARN and counted in `auth_rejected`. The authorized identity is
+  recorded in the client's session as `auth_method` / `auth_identity`.
+- On connect the client receives an `event: connected` frame carrying its
+  client id, session id, sink instance id, endpoint paths, and buffer size.
+- Payloads are framed per the SSE spec, one `data:` line per newline in the
+  payload, so multi-line entries stream correctly.
+- The server sets no `WriteTimeout` (that would kill long-lived streams);
+  per-write deadlines come from `write_timeout_ms` via `http.ResponseController`.
+- A client whose send queue is full has that event dropped
+  (`dropped_writes`); it is not disconnected.
+- Clients whose session has been idle-expired by the session manager are
+  evicted by the broker.
+- On shutdown, connected clients receive
+  `event: disconnect / data: {"reason":"server_shutdown"}`.
+- HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1.
+
+**Status endpoint** returns service and version identity, host, port, TLS flag,
+the compiled auth policy, active client count, buffer size, uptime, endpoint
+paths, and the `total_processed` / `dropped_writes` / `rejected_clients` /
+`auth_rejected` counters.
+
+> Without an `auth` block both endpoints are unauthenticated, and the stream
+> response carries `Access-Control-Allow-Origin: *`, so any web origin can read
+> it. Set `auth.type = "mtls"` (which requires `tls.client_auth`), bind to a
+> trusted interface, or put an authenticating reverse proxy in front.
+
+---
+
+## tcp
+
+Broadcasts formatted payloads to every connected TCP client.
 
 ```toml
-[pipelines.sinks.http.heartbeat]
+[[pipelines.plugin_sinks]]
-enabled = true
+id   = "tap"
-interval_ms = 30000
-include_timestamp = true
-include_stats = false
-format = "comment"  # comment|event|json
-```
-
-### TCP Sink
-
-TCP streaming server for debugging.
-
-```toml
-[[pipelines.sinks]]
 type = "tcp"
-
+[pipelines.plugin_sinks.config]
-[pipelines.sinks.tcp]
+host                 = "0.0.0.0"
-host = "0.0.0.0"
+port                 = 9090
-port = 9090
+buffer_size          = 1000
-buffer_size = 1000
+client_buffer_size   = 256
-max_connections = 100
+write_timeout_ms     = 5000
-keep_alive = true
+keep_alive           = true
 keep_alive_period_ms = 30000
-```
+max_connections      = 0
 
-**Configuration Options:**
+[pipelines.plugin_sinks.config.tls]
+enabled        = true
+cert_file      = "/etc/logwisp/tls/server.crt"
+key_file       = "/etc/logwisp/tls/server.key"
+client_auth    = true
+client_ca_file = "/etc/logwisp/tls/client-ca.crt"
+
+[pipelines.plugin_sinks.config.auth]
+type  = "mtls"
+allow = ["viewer-01"]
+```
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `host` | string | "0.0.0.0" | Bind address |
+| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
-| `port` | int | Required | Listen port |
+| `port` | int | **required** | Listen port |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `buffer_size` | int | `1000` | Sink input queue depth |
-| `max_connections` | int | 100 | Maximum concurrent clients |
+| `client_buffer_size` | int | `256` | Per-client send queue depth |
-| `keep_alive` | bool | true | Enable TCP keep-alive |
+| `write_timeout_ms` | int | `5000` | Per-write deadline |
-| `keep_alive_period_ms` | int | 30000 | Keep-alive interval |
+| `keep_alive` | bool | `true` | Enable TCP keep-alive on accepted connections |
+| `keep_alive_period_ms` | int | `30000` | Keep-alive idle period |
+| `max_connections` | int | `0` | Concurrent connection cap; `0` = unlimited |
+| `tls` | table | — | Listener TLS |
+| `auth` | table | — | Client authorization; see [Security](security.md#the-auth-block) |
 
-**Note:** TCP Sink has no authentication support (debugging only).
+**Behaviour**
 
-### HTTP Client Sink
+- The sink is write-only. Each connection also runs a reader that discards
+  inbound bytes; it exists to detect disconnects and to refresh session
+  activity when a client sends anything.
+- A write that misses its deadline means the kernel buffer stayed full for the
+  whole timeout, so the client is disconnected immediately rather than retried.
+- A client whose send queue is full has that event dropped (`dropped_writes`)
+  and stays connected.
+- With TLS enabled the handshake runs under a 10 s bound *after* the
+  `max_connections` check, so concurrent handshakes are bounded too.
+- With an `auth` block, authorization runs after that handshake and *before*
+  registration, so an unauthorized client never enters the client map and never
+  receives a broadcast. Its connection is closed, the rejection logged at WARN,
+  and `rejected_conns` incremented.
 
-Forward logs to remote HTTP endpoints.
+---
+
+## tcp_chain
+
+Forwards structured entries to a downstream LogWisp `tcp_chain` source over one
+persistent connection. See [Chaining](chaining.md).
 
 ```toml
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
-type = "http_client"
+id   = "to_relay"
+type = "tcp_chain"
+[pipelines.plugin_sinks.config]
+host                 = "relay.internal"
+port                 = 15801
+node                 = "edge-01"
+buffer_size          = 1000
+dial_timeout_ms      = 5000
+write_timeout_ms     = 5000
+backoff_min_ms       = 500
+backoff_max_ms       = 30000
+keep_alive           = true
+keep_alive_period_ms = 30000
 
-[pipelines.sinks.http_client]
+[pipelines.plugin_sinks.config.tls]
-url = "https://logs.example.com/ingest"
+enabled   = true
-buffer_size = 1000
+ca_file   = "/etc/logwisp/tls/ca.crt"
-batch_size = 100
+cert_file = "/etc/logwisp/tls/client.crt"
-batch_delay_ms = 1000
+key_file  = "/etc/logwisp/tls/client.key"
-timeout_seconds = 30
-max_retries = 3
-retry_delay_ms = 1000
-retry_backoff = 2.0
-insecure_skip_verify = false
 ```
 
-**Configuration Options:**
-
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `url` | string | Required | Target URL |
+| `host` | string | **required** | Downstream host |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `port` | int | **required** | Downstream port |
-| `batch_size` | int | 100 | Logs per request |
+| `node` | string | `os.Hostname()` | Origin label stamped on first-hop entries |
-| `batch_delay_ms` | int | 1000 | Max wait before sending |
+| `buffer_size` | int | `1000` | Sink input queue depth |
-| `timeout_seconds` | int | 30 | Request timeout |
+| `dial_timeout_ms` | int | `5000` | TCP connect timeout |
-| `max_retries` | int | 3 | Retry attempts |
+| `write_timeout_ms` | int | `5000` | Per-write deadline |
-| `retry_delay_ms` | int | 1000 | Initial retry delay |
+| `backoff_min_ms` | int | `500` | Reconnect backoff floor |
-| `retry_backoff` | float | 2.0 | Exponential backoff multiplier |
+| `backoff_max_ms` | int | `30000` | Reconnect backoff ceiling |
-| `insecure_skip_verify` | bool | false | Skip TLS verification |
+| `keep_alive` | bool | `true` | Enable TCP keep-alive |
+| `keep_alive_period_ms` | int | `30000` | Keep-alive idle period |
+| `tls` | table | — | Dialer TLS; `cert_file`/`key_file` present a client identity |
+| `auth` | table | — | Server identity pinning; see [Security](security.md#dialer-side-pinning) |
 
-### TCP Client Sink
+**Behaviour**
 
-Forward logs to remote TCP servers.
+- The connection is established lazily, so pipeline start does not depend on the
+  downstream being up.
+- Each entry is serialized as one canonical JSON line. Delivery holds the line
+  across reconnects until it is written or the process shuts down, with
+  exponential backoff plus ±20 % jitter between attempts.
+- Because delivery blocks the sink's run loop during an outage, back-pressure
+  surfaces as a full input queue and is counted by the pipeline as
+  `total_dropped_by_sink`.
+- With TLS, dial and handshake are bounded together by
+  `dial_timeout_ms` + 10 s.
+- Events arriving without a structured entry are wrapped from the formatted
+  payload and counted in `synthesized`.
+
+An `auth` block on a dialer pins the server's identity: the policy runs as part
+of the handshake, so a server it rejects is treated like any other connect
+failure and retried under the normal backoff.
+
+**Statistics**: `target`, `node`, `tls`, `auth`, `connected`, `reconnects`,
+`write_errors`, `synthesized`.
+
+---
+
+## http_chain
+
+Batches structured entries as NDJSON and POSTs them to a downstream LogWisp
+`http_chain` source.
 
 ```toml
-[[pipelines.sinks]]
+[[pipelines.plugin_sinks]]
-type = "tcp_client"
+id   = "to_collector"
+type = "http_chain"
+[pipelines.plugin_sinks.config]
+host               = "collector.internal"
+port               = 15802
+ingest_path        = "/ingest"
+node               = "edge-01"
+buffer_size        = 1000
+max_batch_count    = 100
+max_batch_bytes    = 1048576
+flush_interval_ms  = 1000
+request_timeout_ms = 10000
+backoff_min_ms     = 500
+backoff_max_ms     = 30000
 
-[pipelines.sinks.tcp_client]
+[pipelines.plugin_sinks.config.tls]
-host = "logs.example.com"
+enabled   = true
-port = 9090
+ca_file   = "/etc/logwisp/tls/ca.crt"
-buffer_size = 1000
+cert_file = "/etc/logwisp/tls/client.crt"
-dial_timeout = 10
+key_file  = "/etc/logwisp/tls/client.key"
-write_timeout = 30
-read_timeout = 10
-keep_alive = 30
-reconnect_delay_ms = 1000
-max_reconnect_delay_ms = 30000
-reconnect_backoff = 1.5
 ```
 
-**Configuration Options:**
-
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `host` | string | Required | Target host |
+| `host` | string | **required** | Downstream host |
-| `port` | int | Required | Target port |
+| `port` | int | **required** | Downstream port |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `ingest_path` | string | `/ingest` | Endpoint path; must start with `/` |
-| `dial_timeout` | int | 10 | Connection timeout (seconds) |
+| `node` | string | `os.Hostname()` | Origin label stamped on first-hop entries |
-| `write_timeout` | int | 30 | Write timeout (seconds) |
+| `buffer_size` | int | `1000` | Sink input queue depth |
-| `read_timeout` | int | 10 | Read timeout (seconds) |
+| `max_batch_count` | int | `100` | Flush after this many entries |
-| `keep_alive` | int | 30 | TCP keep-alive (seconds) |
+| `max_batch_bytes` | int | `1048576` | Flush after this many bytes (1 MiB) |
-| `reconnect_delay_ms` | int | 1000 | Initial reconnect delay |
+| `flush_interval_ms` | int | `1000` | Flush after this long |
-| `max_reconnect_delay_ms` | int | 30000 | Maximum reconnect delay |
+| `request_timeout_ms` | int | `10000` | Covers dial, write, and response |
-| `reconnect_backoff` | float | 1.5 | Backoff multiplier |
+| `backoff_min_ms` | int | `500` | Retry backoff floor |
+| `backoff_max_ms` | int | `30000` | Retry backoff ceiling |
+| `tls` | table | — | Dialer TLS; `cert_file`/`key_file` present a client identity |
+| `auth` | table | — | Server identity pinning; see [Security](security.md#dialer-side-pinning) |
 
-## Network Sink Features
+**Behaviour**
 
-### Network Rate Limiting
+- Delivery is at-least-once per batch: a retried batch can be delivered twice if
+  the first attempt succeeded but the response was lost.
+- Retries apply to transport errors, `408`, `429`, and `5xx`. Any other
+  non-2xx response is treated as permanent, and the batch is dropped and counted
+  in `dropped_batches`.
+- HTTP/2 is off by design; batched NDJSON POSTs gain nothing from it.
+- On shutdown a single best-effort flush of the pending batch is attempted.
 
-Available for HTTP and TCP sinks:
+**Statistics**: `target`, `node`, `tls`, `auth`, `batches_sent`,
+`request_errors`, `dropped_batches`, `synthesized`.
 
-```toml
+---
-[pipelines.sinks.http.net_limit]
-enabled = true
-max_connections_per_ip = 10
-max_connections_total = 100
-ip_whitelist = ["192.168.1.0/24"]
-ip_blacklist = ["10.0.0.0/8"]
-```
-
-### TLS Configuration (HTTP Only)
-
-```toml
-[pipelines.sinks.http.tls]
-enabled = true
-cert_file = "/path/to/cert.pem"
-key_file = "/path/to/key.pem"
-ca_file = "/path/to/ca.pem"
-min_version = "TLS1.2"
-client_auth = false
-```
-
-HTTP Client TLS:
-
-```toml
-[pipelines.sinks.http_client.tls]
-enabled = true
-server_name = "logs.example.com"
-skip_verify = false
-cert_file = "/path/to/client.pem"  # For mTLS
-key_file = "/path/to/client.key"   # For mTLS
-```
-
-### Authentication
-
-HTTP/HTTP Client authentication:
-
-```toml
-[pipelines.sinks.http_client.auth]
-type = "basic"  # none|basic|token|mtls
-username = "user"
-password = "pass"
-token = "bearer-token"
-```
-
-TCP Client authentication:
-
-```toml
-[pipelines.sinks.tcp_client.auth]
-type = "scram"  # none|scram
-username = "user"
-password = "pass"
-```
-
-## Sink Chaining
-
-Designed connection patterns:
-
-### Log Aggregation
-- **HTTP Client Sink → HTTP Source**: HTTPS with authentication
-- **TCP Client Sink → TCP Source**: Raw TCP with SCRAM
-
-### Live Monitoring
-- **HTTP Sink**: Browser-based SSE streaming
-- **TCP Sink**: Debug interface (telnet/netcat)
 
 ## Sink Statistics
 
-All sinks track:
+Every sink reports: `id`, `type`, `total_processed`, `active_connections`,
-- Total entries processed
+`start_time`, `last_processed`, and a type-specific `details` map.
-- Active connections
-- Failed sends
-- Retry attempts
-- Last processed timestamp

doc/sources.md

@@ -1,214 +1,289 @@
 # Input Sources
 
-LogWisp sources monitor various inputs and generate log entries for pipeline processing.
+Sources produce `core.LogEntry` values for a pipeline. Every source is declared
-
+as a `[[pipelines.plugin_sources]]` entry with an `id`, a `type`, and a
-## Source Types
+type-specific `config` table.
-
-### Directory Source
-
-Monitors a directory for log files matching a pattern.
 
 ```toml
-[[pipelines.sources]]
+[[pipelines.plugin_sources]]
-type = "directory"
+id   = "app_logs"
-
+type = "file"
-[pipelines.sources.directory]
+[pipelines.plugin_sources.config]
-path = "/var/log/myapp"
+directory = "/var/log/myapp"
-pattern = "*.log"          # Glob pattern
-check_interval_ms = 100    # Poll interval
-recursive = false           # Scan subdirectories
 ```
 
-**Configuration Options:**
+Registered types: `file`, `console`, `random`, `null`, `tcp_chain`,
+`http_chain`.
+
+Publication from any source is non-blocking. When a subscriber channel is full
+the entry is dropped and counted in `dropped_entries`.
+
+---
+
+## file
+
+Tails every file in a directory whose name matches a glob.
+
+```toml
+[[pipelines.plugin_sources]]
+id   = "app_logs"
+type = "file"
+[pipelines.plugin_sources.config]
+directory         = "/var/log/myapp"
+pattern           = "*.log"
+check_interval_ms = 100
+raw               = false
+from              = "end"
+```
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `path` | string | Required | Directory to monitor |
+| `directory` | string | **required** | Directory to scan; not recursive |
-| `pattern` | string | "*" | File pattern (glob) |
+| `pattern` | string | `*` | Glob over filenames; `*` and `?` only |
-| `check_interval_ms` | int | 100 | File check interval in milliseconds |
+| `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` |
-| `recursive` | bool | false | Include subdirectories |
+| `raw` | bool | `false` | Never parse a line: the whole line is the message |
+| `from` | string | `end` | Where a new watcher starts: `end` or `start` of the file |
 
-**Features:**
+**Behaviour**
-- Automatic file rotation detection
-- Position tracking (resume after restart)
-- Concurrent file monitoring
-- Pattern-based file selection
 
-### Stdin Source
+- `check_interval_ms` governs how often the directory is rescanned for new or
+  removed files. Tailing an already-open file polls on a **fixed 100 ms**
+  interval that this option does not change.
+- Each matched file gets its own watcher. Watchers for files that disappear are
+  stopped and removed on the next scan.
+- A new watcher seeks to end-of-file. Positions live in memory only, so a
+  restart resumes from the current end of each file and content written while
+  LogWisp was down is not read. `from = "start"` reads each file whole when its
+  watcher is created instead — what a process writing beside LogWisp needs, at
+  the cost of replaying a file already on disk at every restart.
+- Rotation is detected from size decrease, modification-time reset, a position
+  beyond end-of-file, or an inode change. An inode change where the new file is
+  already larger than the recorded position is treated as an atomic save, not a
+  rotation, and the position is preserved.
+- A line is parsed as JSON only when it is an object whose top-level keys are
+  all drawn from `time`, `level`, `msg` and `fields` — the four an entry can
+  carry. `time` is read as RFC3339Nano. Any other key, and any non-object line,
+  is kept whole as text with the level inferred from common markers
+  (`[ERROR]`, `WARN:`, and so on), because parsing it would drop the rest.
+- `raw = true` skips the JSON branch entirely. The line, plus its newline,
+  becomes the message; `fields` stays empty, the time is the read time, and the
+  level is inferred from the text as for any unparsed line. Paired with
+  `format.type = "raw"` this is byte-exact transport for records LogWisp's
+  envelope cannot hold — see [Formatters](formatters.md#raw).
+- `Source` is set to the file's base name.
 
-Reads log entries from standard input.
+**Statistics**: per-watcher size, position, entries read, rotation count, and
+last read time, plus `active_watchers`.
+
+---
+
+## console
+
+Reads newline-delimited entries from standard input.
 
 ```toml
-[[pipelines.sources]]
+[[pipelines.plugin_sources]]
-type = "stdin"
+id   = "stdin"
-
+type = "console"
-[pipelines.sources.stdin]
+[pipelines.plugin_sources.config]
 buffer_size = 1000
 ```
 
-**Configuration Options:**
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `buffer_size` | int | `1000` | Subscriber channel depth |
+
+At most **one** instance per pipeline: the type is registered with
+`MaxInstances: 1`, and a second instance is rejected at pipeline construction.
+The level is inferred from the line text, and `Source` is set to `console`.
+
+---
+
+## random
+
+Synthetic entry generator for development, smoke tests, and sanitizer testing.
+
+```toml
+[[pipelines.plugin_sources]]
+id   = "generator"
+type = "random"
+[pipelines.plugin_sources.config]
+interval_ms = 500
+jitter_ms   = 0
+format      = "txt"
+length      = 20
+special     = false
+```
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `interval_ms` | int | `500` | Emission period |
+| `jitter_ms` | int | `0` | Symmetric jitter; clamped to `interval_ms`, must be non-negative |
+| `format` | string | `txt` | `raw` (message only), `txt` (bracketed line), `json` (JSON object as the message) |
+| `length` | int | `20` | Message length in characters |
+| `special` | bool | `false` | Inject control and non-ASCII characters |
 
-**Features:**
+`special = true` is the intended way to exercise sanitizer policies: it inserts
-- Line-based processing
+control bytes and multi-byte Unicode into otherwise ordinary messages. Levels
-- Automatic level detection
+are chosen at random from DEBUG, INFO, WARN, ERROR.
-- Non-blocking reads
 
-### HTTP Source
+---
 
-REST endpoint for log ingestion.
+## null
+
+Produces nothing. Useful as a placeholder so a sink-only pipeline satisfies the
+"at least one source" requirement.
 
 ```toml
-[[pipelines.sources]]
+[[pipelines.plugin_sources]]
-type = "http"
+id   = "void"
-
+type = "null"
-[pipelines.sources.http]
-host = "0.0.0.0"
-port = 8081
-ingest_path = "/ingest"
-buffer_size = 1000
-max_body_size = 1048576  # 1MB
-read_timeout_ms = 10000
-write_timeout_ms = 10000
 ```
 
-**Configuration Options:**
+No options.
+
+---
+
+## tcp_chain
+
+Listens for persistent NDJSON streams from upstream LogWisp `tcp_chain` sinks.
+See [Chaining](chaining.md) for the protocol.
+
+```toml
+[[pipelines.plugin_sources]]
+id   = "ingest_tcp"
+type = "tcp_chain"
+[pipelines.plugin_sources.config]
+host             = "0.0.0.0"
+port             = 15801
+buffer_size      = 1000
+max_connections  = 0
+read_timeout_ms  = 0
+hello_timeout_ms = 10000
+trust_node       = true
+
+[pipelines.plugin_sources.config.tls]
+enabled        = true
+cert_file      = "/etc/logwisp/tls/server.crt"
+key_file       = "/etc/logwisp/tls/server.key"
+client_auth    = true
+client_ca_file = "/etc/logwisp/tls/client-ca.crt"
+min_version    = "1.3"
+
+[pipelines.plugin_sources.config.auth]
+type         = "mtls"
+allow        = ["edge-01", "edge-02"]
+node_binding = "force"
+```
 
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `host` | string | "0.0.0.0" | Bind address |
+| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
-| `port` | int | Required | Listen port |
+| `port` | int | **required** | Listen port, 1–65535 |
-| `ingest_path` | string | "/ingest" | Ingestion endpoint path |
+| `buffer_size` | int | `1000` | Subscriber channel depth |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `max_connections` | int | `0` | Concurrent connection cap; `0` = unlimited |
-| `max_body_size` | int | 1048576 | Maximum request body size |
+| `read_timeout_ms` | int | `0` | Per-connection idle read deadline; `0` = none |
-| `read_timeout_ms` | int | 10000 | Read timeout |
+| `hello_timeout_ms` | int | `10000` | Deadline for the hello preamble |
-| `write_timeout_ms` | int | 10000 | Write timeout |
+| `trust_node` | bool | `true` | `false` overrides the sender's node label with its remote address. Ignored when `auth.node_binding` is active |
+| `tls` | table | — | Listener TLS; see [Security](security.md) |
+| `auth` | table | — | Peer authorization and node binding; see [Security](security.md#the-auth-block) |
 
-**Input Formats:**
+**Behaviour**
-- Single JSON object
-- JSON array
-- Newline-delimited JSON (NDJSON)
-- Plain text (one entry per line)
 
-### TCP Source
+- TLS handshakes run explicitly with a 10 s bound before the preamble is read,
+  after the `max_connections` admission check.
+- Authorization runs between the handshake and the hello read, so an
+  unauthorized peer never gets a preamble parsed on its behalf. A rejection is
+  logged at WARN and counted in `rejected_conns`.
+- A connection is rejected if the first line is not a valid hello with a
+  matching protocol version.
+- The node label is then resolved: under `auth.node_binding` it comes from the
+  peer's certificate, otherwise `trust_node` governs. `force` also overrides the
+  `node` field on every individual entry; `assert` leaves per-entry labels to
+  `trust_node`, so a relay can forward other nodes' entries while proving its
+  own identity.
+- Each accepted connection gets a session recording the remote address, node
+  label, — under TLS — `tls` and `tls_peer_cn`, and — under auth —
+  `auth_method` and `auth_identity`.
+- A malformed entry line increments `parse_errors` and is skipped; the
+  connection survives. A line over 1 MiB is a protocol violation and terminates
+  the connection.
 
-Raw TCP socket listener for log ingestion.
+**Statistics**: `active_connections`, `rejected_conns`, `parse_errors`,
+`tls_handshake_errors`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
+`node_binding`.
+
+---
+
+## http_chain
+
+Accepts NDJSON batches POSTed by upstream LogWisp `http_chain` sinks.
 
 ```toml
-[[pipelines.sources]]
+[[pipelines.plugin_sources]]
-type = "tcp"
+id   = "ingest_http"
+type = "http_chain"
+[pipelines.plugin_sources.config]
+host            = "0.0.0.0"
+port            = 15802
+ingest_path     = "/ingest"
+buffer_size     = 1000
+max_body_bytes  = 8388608
+read_timeout_ms = 30000
+trust_node      = true
 
-[pipelines.sources.tcp]
+[pipelines.plugin_sources.config.tls]
-host = "0.0.0.0"
+enabled        = true
-port = 9091
+cert_file      = "/etc/logwisp/tls/server.crt"
-buffer_size = 1000
+key_file       = "/etc/logwisp/tls/server.key"
-read_timeout_ms = 10000
+client_auth    = true
-keep_alive = true
+client_ca_file = "/etc/logwisp/tls/client-ca.crt"
-keep_alive_period_ms = 30000
+
+[pipelines.plugin_sources.config.auth]
+type         = "mtls"
+allow        = ["edge-01", "edge-02"]
+node_binding = "force"
 ```
 
-**Configuration Options:**
-
 | Option | Type | Default | Description |
 |--------|------|---------|-------------|
-| `host` | string | "0.0.0.0" | Bind address |
+| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
-| `port` | int | Required | Listen port |
+| `port` | int | **required** | Listen port |
-| `buffer_size` | int | 1000 | Internal buffer size |
+| `ingest_path` | string | `/ingest` | Endpoint path; must start with `/` |
-| `read_timeout_ms` | int | 10000 | Read timeout |
+| `buffer_size` | int | `1000` | Subscriber channel depth |
-| `keep_alive` | bool | true | Enable TCP keep-alive |
+| `max_body_bytes` | int | `8388608` | Per-request body cap (8 MiB) |
-| `keep_alive_period_ms` | int | 30000 | Keep-alive interval |
+| `read_timeout_ms` | int | `30000` | Full request read deadline |
+| `trust_node` | bool | `true` | `false` overrides the sender's node label with its remote address. Ignored when `auth.node_binding` is active |
+| `tls` | table | — | Listener TLS |
+| `auth` | table | — | Peer authorization and node binding; see [Security](security.md#the-auth-block) |
 
-**Protocol:**
+**Behaviour**
-- Newline-delimited JSON
-- One log entry per line
-- UTF-8 encoding
 
-## Network Source Features
+- Only `POST` to `ingest_path` is routed; other methods get `405` with an
+  `Allow` header, and other paths get `404`.
+- Authorization runs before the body is read, so an unauthorized sender does not
+  get to stream `max_body_bytes` into the process. Both a policy rejection and a
+  node-binding failure answer `403`, distinct from the `400` used for protocol
+  errors, so a sender can tell "not allowed" from "malformed batch".
+- A missing or mismatched `X-Logwisp-Protocol` header is rejected with `400`.
+- Batch acceptance is atomic: entries are published only after the body reads
+  cleanly end to end. A transfer error rejects the whole batch (`400`, or `413`
+  when the body cap is hit) so the sender retries it. A malformed *line* inside
+  an otherwise clean transfer is skipped and counted in `parse_errors`.
+- Success is `204 No Content` with `X-Logwisp-Accepted` set to the number of
+  entries ingested.
+- Sessions are cached per remote host + node + authenticated identity, and
+  recreated after idle expiry. Including the identity in the key means two peers
+  sharing a remote address never share a session.
 
-### Network Rate Limiting
+**Statistics**: `total_requests`, `rejected_requests`, `parse_errors`,
+`cached_sessions`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
+`node_binding`.
 
-Available for HTTP and TCP sources:
+---
-
-```toml
-[pipelines.sources.http.net_limit]
-enabled = true
-max_connections_per_ip = 10
-max_connections_total = 100
-requests_per_second = 100.0
-burst_size = 200
-response_code = 429
-response_message = "Rate limit exceeded"
-ip_whitelist = ["192.168.1.0/24"]
-ip_blacklist = ["10.0.0.0/8"]
-```
-
-### TLS Configuration (HTTP Only)
-
-```toml
-[pipelines.sources.http.tls]
-enabled = true
-cert_file = "/path/to/cert.pem"
-key_file = "/path/to/key.pem"
-ca_file = "/path/to/ca.pem"
-min_version = "TLS1.2"
-client_auth = true
-client_ca_file = "/path/to/client-ca.pem"
-verify_client_cert = true
-```
-
-### Authentication
-
-HTTP Source authentication options:
-
-```toml
-[pipelines.sources.http.auth]
-type = "basic"  # none|basic|token|mtls
-realm = "LogWisp"
-
-# Basic auth
-[[pipelines.sources.http.auth.basic.users]]
-username = "admin"
-password_hash = "$argon2..."
-
-# Token auth
-[pipelines.sources.http.auth.token]
-tokens = ["token1", "token2"]
-```
-
-TCP Source authentication:
-
-```toml
-[pipelines.sources.tcp.auth]
-type = "scram"  # none|scram
-
-# SCRAM users
-[[pipelines.sources.tcp.auth.scram.users]]
-username = "user1"
-stored_key = "base64..."
-server_key = "base64..."
-salt = "base64..."
-argon_time = 3
-argon_memory = 65536
-argon_threads = 4
-```
 
 ## Source Statistics
 
-All sources track:
+Every source reports: `id`, `type`, `total_entries`, `dropped_entries`,
-- Total entries received
+`start_time`, `last_entry_time`, and a type-specific `details` map. These appear
-- Dropped entries (buffer full)
+in the status reporter output and in the `http` sink's status endpoint.
-- Invalid entries
-- Last entry timestamp
-- Active connections (network sources)
-- Source-specific metrics
-
-## Buffer Management
-
-Each source maintains internal buffers:
-- Default size: 1000 entries
-- Drop policy when full
-- Configurable per source
-- Non-blocking writes

go.mod

@@ -1,30 +1,14 @@
 module logwisp
 
-go 1.25.1
+go 1.27.1
 
 require (
-	github.com/lixenwraith/config v0.0.0-20251003140149-580459b815f6
+	github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98
-	github.com/lixenwraith/log v0.0.0-20251010094026-6a161eb2b686
+	github.com/lixenwraith/log v0.1.1-0.20260801090951-2c40643523b3
-	github.com/panjf2000/gnet/v2 v2.9.4
-	github.com/valyala/fasthttp v1.68.0
-	golang.org/x/crypto v0.43.0
-	golang.org/x/term v0.36.0
 )
 
 require (
-	github.com/BurntSushi/toml v1.5.0 // indirect
+	github.com/BurntSushi/toml v1.6.0 // indirect
-	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/klauspost/compress v1.18.1 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/panjf2000/ants/v2 v2.11.3 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
-
-replace github.com/mitchellh/mapstructure => github.com/go-viper/mapstructure v1.6.0

go.sum

@@ -1,48 +1,20 @@
-github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
+github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
-github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
-github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
-github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-viper/mapstructure v1.6.0 h1:0WdPOF2rmmQDN1xo8qIgxyugvLp71HrZSWyGLxofobw=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
-github.com/go-viper/mapstructure v1.6.0/go.mod h1:FcbLReH7/cjaC0RVQR+LHFIrBhHF3s1e/ud1KMDoBVw=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
+github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98 h1:VEFo2WsYgM5YYfzAfvXRKigCWr3zEz6h4M59N5TXMpk=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98/go.mod h1:J9ydxY7he4Dz+S59xKs/kDQ2YAv+zOLF05b3Rr2/ogE=
-github.com/lixenwraith/config v0.0.0-20251003140149-580459b815f6 h1:G9qP8biXBT6bwBOjEe1tZwjA0gPuB5DC+fLBRXDNXqo=
+github.com/lixenwraith/log v0.1.1-0.20260724174821-e688c5a07207 h1:EnDMcnFkgTwTGo1ojoJ85/b6aH3yxlBUAg6AefaBcrI=
-github.com/lixenwraith/config v0.0.0-20251003140149-580459b815f6/go.mod h1:I7ddNPT8MouXXz/ae4DQfBKMq5EisxdDLRX0C7Dv4O0=
+github.com/lixenwraith/log v0.1.1-0.20260724174821-e688c5a07207/go.mod h1:fvGT3IxlJQLJ1is4OP8Iet8gVzTJTGcuT90FFB2WBtc=
-github.com/lixenwraith/log v0.0.0-20251010094026-6a161eb2b686 h1:STgvFUpjvZquBF322PNLXaU67oEScewGDLy0aV+lIkY=
+github.com/lixenwraith/log v0.1.1-0.20260801090951-2c40643523b3 h1:5ojkxyuOKiUBRYhqPqKJS1lrqkjckYqYMNGBB49kdY0=
-github.com/lixenwraith/log v0.0.0-20251010094026-6a161eb2b686/go.mod h1:E7REMCVTr6DerzDtd2tpEEaZ9R9nduyAIKQFOqHqKr0=
+github.com/lixenwraith/log v0.1.1-0.20260801090951-2c40643523b3/go.mod h1:fvGT3IxlJQLJ1is4OP8Iet8gVzTJTGcuT90FFB2WBtc=
-github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
-github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
-github.com/panjf2000/gnet/v2 v2.9.4 h1:XvPCcaFwO4XWg4IgSfZnNV4dfDy5g++HIEx7sH0ldHc=
-github.com/panjf2000/gnet/v2 v2.9.4/go.mod h1:WQTxDWYuQ/hz3eccH0FN32IVuvZ19HewEWx0l62fx7E=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok=
-github.com/valyala/fasthttp v1.68.0/go.mod h1:5EXiRfYQAoiO/khu4oU9VISC/eVY6JqmSpPJoHCKsz4=
-github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
-github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
-go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
-go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
-golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
-golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
-golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
-golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
-gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/authz/authz.go

@@ -0,0 +1,318 @@
+// Package authz turns a verified certificate into an authorization decision.
+// It is the single seam between declarative auth config and the network
+// plugins: each one compiles a Policy at construction and calls Authorize per
+// connection (TCP) or per request (HTTP).
+//
+// New returns (nil, nil) when auth is disabled, mirroring tlsx.Server and
+// tlsx.Client, and every method tolerates a nil receiver — so call sites read
+// the same whether or not a policy is configured.
+package authz
+
+import (
+	"crypto/tls"
+	"fmt"
+	"regexp"
+	"strings"
+	"sync/atomic"
+
+	"logwisp/internal/config"
+	"logwisp/internal/tlsx"
+)
+
+// Authentication methods
+const (
+	MethodNone = "none"
+	MethodMTLS = "mtls"
+)
+
+// Node label binding modes. See ResolveNode and TrustsEntryNode for the
+// difference between assert and force.
+const (
+	BindingNone   = "none"
+	BindingAssert = "assert"
+	BindingForce  = "force"
+)
+
+// Role selects the validation and behavior appropriate to the call site
+type Role int
+
+const (
+	// RoleListener authorizes client certificates on a plugin with no node
+	// concept: the tcp and http sinks
+	RoleListener Role = iota
+	// RoleChainListener authorizes client certificates and binds the node
+	// label a peer declares: the tcp_chain and http_chain sources
+	RoleChainListener
+	// RoleDialer pins the server's identity beyond hostname verification:
+	// the tcp_chain and http_chain sinks
+	RoleDialer
+)
+
+// Policy is the compiled form of config.AuthOptions
+type Policy struct {
+	role     Role
+	identity string
+	allow    map[string]struct{}
+	patterns []*regexp.Regexp
+	binding  string
+
+	// Statistics
+	allowed  atomic.Uint64
+	rejected atomic.Uint64
+}
+
+// Identity is the outcome of a successful authorization. The zero value is
+// what a disabled policy yields.
+type Identity struct {
+	Name   string // the selected certificate field
+	Method string // MethodMTLS
+}
+
+// Apply stamps an authenticated identity onto session metadata. A zero
+// Identity (auth disabled) leaves the map untouched.
+func (id Identity) Apply(meta map[string]any) {
+	if id.Name == "" {
+		return
+	}
+	meta["auth_method"] = id.Method
+	meta["auth_identity"] = id.Name
+}
+
+// New compiles an auth policy, returning (nil, nil) when auth is disabled.
+// tlsOpts is the sibling `tls` block: an auth policy the transport cannot
+// enforce is rejected here rather than silently accepted, which is the
+// failure mode worth designing out.
+func New(o *config.AuthOptions, tlsOpts *config.TLSOptions, role Role) (*Policy, error) {
+	if o == nil {
+		return nil, nil
+	}
+	switch o.Type {
+	case "", MethodNone:
+		return nil, nil
+	case MethodMTLS:
+	default:
+		return nil, fmt.Errorf("auth: type %q (valid: %q, %q)", o.Type, MethodNone, MethodMTLS)
+	}
+
+	if tlsOpts == nil || !tlsOpts.Enabled {
+		return nil, fmt.Errorf("auth: type %q requires tls.enabled", MethodMTLS)
+	}
+	if role == RoleDialer {
+		// Identity from an unverified chain is a claim, not a fact
+		if tlsOpts.InsecureSkipVerify {
+			return nil, fmt.Errorf("auth: type %q cannot pin an identity with tls.insecure_skip_verify", MethodMTLS)
+		}
+	} else if !tlsOpts.ClientAuth {
+		return nil, fmt.Errorf("auth: type %q requires tls.client_auth", MethodMTLS)
+	}
+
+	identity := o.Identity
+	if identity == "" {
+		identity = tlsx.IdentityCN
+	}
+	switch identity {
+	case tlsx.IdentityCN, tlsx.IdentitySANDNS, tlsx.IdentitySANURI, tlsx.IdentitySANEmail:
+	default:
+		return nil, fmt.Errorf("auth: identity %q (valid: %q, %q, %q, %q)",
+			identity, tlsx.IdentityCN, tlsx.IdentitySANDNS, tlsx.IdentitySANURI, tlsx.IdentitySANEmail)
+	}
+
+	binding := o.NodeBinding
+	if role == RoleChainListener {
+		if binding == "" {
+			// The only setting under which a misconfigured or hostile edge
+			// cannot mislabel its entries
+			binding = BindingForce
+		}
+	} else if binding != "" && binding != BindingNone {
+		return nil, fmt.Errorf("auth: node_binding %q applies only to chain sources", binding)
+	} else {
+		binding = BindingNone
+	}
+	switch binding {
+	case BindingNone, BindingAssert, BindingForce:
+	default:
+		return nil, fmt.Errorf("auth: node_binding %q (valid: %q, %q, %q)",
+			binding, BindingNone, BindingAssert, BindingForce)
+	}
+
+	p := &Policy{
+		role:     role,
+		identity: identity,
+		binding:  binding,
+		allow:    make(map[string]struct{}, len(o.Allow)),
+	}
+	for _, a := range o.Allow {
+		if a = strings.TrimSpace(a); a != "" {
+			p.allow[a] = struct{}{}
+		}
+	}
+	for i, pat := range o.AllowPatterns {
+		re, err := regexp.Compile(pat)
+		if err != nil {
+			return nil, fmt.Errorf("auth: allow_patterns[%d] %q: %w", i, pat, err)
+		}
+		p.patterns = append(p.patterns, re)
+	}
+	return p, nil
+}
+
+// Authorize extracts and checks the peer identity from a completed handshake.
+// A nil policy authorizes everything and yields the zero Identity, so callers
+// need no branch on whether auth is configured.
+func (p *Policy) Authorize(cs *tls.ConnectionState) (Identity, error) {
+	if p == nil {
+		return Identity{}, nil
+	}
+	if cs == nil {
+		p.rejected.Add(1)
+		return Identity{}, fmt.Errorf("auth: peer is not on a TLS connection")
+	}
+	name := tlsx.PeerIdentity(*cs, p.identity)
+	if name == "" {
+		// An unusable identity field is a rejection, not an empty match
+		p.rejected.Add(1)
+		return Identity{}, fmt.Errorf("auth: peer certificate carries no %s identity", p.identity)
+	}
+	if !p.permits(name) {
+		p.rejected.Add(1)
+		return Identity{}, fmt.Errorf("auth: identity %q is not allowed", name)
+	}
+	p.allowed.Add(1)
+	return Identity{Name: name, Method: MethodMTLS}, nil
+}
+
+// VerifyConnection is assignable to tls.Config.VerifyConnection on a dialer,
+// so a server whose identity the policy rejects fails the handshake itself
+// rather than after the first write. It runs after the standard chain and
+// hostname checks, so the identity it reads is already verified.
+func (p *Policy) VerifyConnection(cs tls.ConnectionState) error {
+	_, err := p.Authorize(&cs)
+	return err
+}
+
+// permits reports whether an identity satisfies the allow list. An empty list
+// admits any identity the CA vouches for; that is the documented default, and
+// constructors log it at startup rather than leaving it silent.
+// Identities are not secrets, so ordinary comparison is fine.
+func (p *Policy) permits(name string) bool {
+	if len(p.allow) == 0 && len(p.patterns) == 0 {
+		return true
+	}
+	if _, ok := p.allow[name]; ok {
+		return true
+	}
+	for _, re := range p.patterns {
+		if re.MatchString(name) {
+			return true
+		}
+	}
+	return false
+}
+
+// ResolveNode returns the node label for a connection. With no policy, or
+// node_binding "none", trust_node governs as before: the declared label stands
+// only when trusted and non-empty, otherwise fallback (the remote address) is
+// used. Otherwise the label is bound to the authenticated identity.
+func (p *Policy) ResolveNode(declared, fallback string, trustNode bool, id Identity) (string, error) {
+	if p == nil || p.binding == BindingNone {
+		if declared == "" || !trustNode {
+			return fallback, nil
+		}
+		return declared, nil
+	}
+	if id.Name == "" {
+		return "", fmt.Errorf("auth: node_binding %q requires an authenticated identity", p.binding)
+	}
+	if p.binding == BindingForce {
+		return id.Name, nil
+	}
+	// BindingAssert: a mismatch is loud rather than silently corrected
+	if declared == "" {
+		return "", fmt.Errorf("auth: node_binding %q: peer %q declared no node label", BindingAssert, id.Name)
+	}
+	if declared != id.Name {
+		return "", fmt.Errorf("auth: node_binding %q: declared node %q does not match identity %q",
+			BindingAssert, declared, id.Name)
+	}
+	return declared, nil
+}
+
+// TrustsEntryNode reports whether node labels carried by individual entries
+// survive the policy. force relabels every entry, so an ingest boundary that
+// does not trust its peer gets exact attribution; assert pins only the
+// connection's own label, so a relay forwarding other nodes' entries proves
+// who it is while preserving their origin.
+func (p *Policy) TrustsEntryNode(trustNode bool) bool {
+	if p == nil {
+		return trustNode
+	}
+	if p.binding == BindingForce {
+		return false
+	}
+	return trustNode
+}
+
+// BindsNode reports whether the policy overrides trust_node
+func (p *Policy) BindsNode() bool {
+	return p != nil && p.binding != BindingNone
+}
+
+// NodeBinding returns the effective binding mode
+func (p *Policy) NodeBinding() string {
+	if p == nil {
+		return BindingNone
+	}
+	return p.binding
+}
+
+// Enabled reports whether a policy is in force
+func (p *Policy) Enabled() bool { return p != nil }
+
+// Unrestricted reports whether the policy admits any identity the CA vouches
+// for. Constructors log this at startup: it is a deliberate default, and a
+// silent one would be a footgun.
+func (p *Policy) Unrestricted() bool {
+	return p != nil && len(p.allow) == 0 && len(p.patterns) == 0
+}
+
+// Describe renders the policy for a startup log line
+func (p *Policy) Describe() string {
+	if p == nil {
+		return MethodNone
+	}
+	scope := fmt.Sprintf("%d exact, %d pattern(s)", len(p.allow), len(p.patterns))
+	if p.Unrestricted() {
+		scope = "any identity issued by the configured CA"
+	}
+	return fmt.Sprintf("%s identity=%s allow=[%s] node_binding=%s",
+		MethodMTLS, p.identity, scope, p.binding)
+}
+
+// Rejected returns the number of authorization failures
+func (p *Policy) Rejected() uint64 {
+	if p == nil {
+		return 0
+	}
+	return p.rejected.Load()
+}
+
+// Stats reports policy state for a plugin's stats details map. Merge it in
+// with maps.Copy so rejections surface in the status reporter and in the
+// http sink's status endpoint.
+func (p *Policy) Stats() map[string]any {
+	if p == nil {
+		return map[string]any{"auth": MethodNone}
+	}
+	d := map[string]any{
+		"auth":              MethodMTLS,
+		"auth_identity":     p.identity,
+		"auth_unrestricted": p.Unrestricted(),
+		"auth_allowed":      p.allowed.Load(),
+		"auth_rejected":     p.rejected.Load(),
+	}
+	if p.role == RoleChainListener {
+		d["node_binding"] = p.binding
+	}
+	return d
+}

internal/authz/authz_test.go

@@ -0,0 +1,298 @@
+package authz
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"net/url"
+	"testing"
+
+	"logwisp/internal/config"
+	"logwisp/internal/tlsx"
+)
+
+// peerState fakes a completed handshake. Only the leaf's identity fields are
+// read: the chain is verified by crypto/tls before a policy ever sees it.
+func peerState(leaf *x509.Certificate) *tls.ConnectionState {
+	return &tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}}
+}
+
+func leafCN(cn string) *x509.Certificate {
+	return &x509.Certificate{Subject: pkix.Name{CommonName: cn}}
+}
+
+func mtlsListenerTLS() *config.TLSOptions {
+	return &config.TLSOptions{Enabled: true, ClientAuth: true}
+}
+
+func TestPeerIdentityModes(t *testing.T) {
+	uri, err := url.Parse("spiffe://example.org/edge-01")
+	if err != nil {
+		t.Fatalf("parse uri: %v", err)
+	}
+	leaf := &x509.Certificate{
+		Subject:        pkix.Name{CommonName: "edge-01"},
+		DNSNames:       []string{"edge-01.internal", "alt.internal"},
+		URIs:           []*url.URL{uri},
+		EmailAddresses: []string{"ops@example.org"},
+	}
+	cs := peerState(leaf)
+
+	cases := map[string]string{
+		tlsx.IdentityCN:       "edge-01",
+		tlsx.IdentitySANDNS:   "edge-01.internal",
+		tlsx.IdentitySANURI:   "spiffe://example.org/edge-01",
+		tlsx.IdentitySANEmail: "ops@example.org",
+		"":                    "edge-01", // empty mode defaults to CN
+		"nonsense":            "",
+	}
+	for mode, want := range cases {
+		if got := tlsx.PeerIdentity(*cs, mode); got != want {
+			t.Errorf("PeerIdentity(%q) = %q, want %q", mode, got, want)
+		}
+	}
+
+	// A mode the certificate does not carry yields no identity
+	bare := peerState(leafCN("edge-01"))
+	if got := tlsx.PeerIdentity(*bare, tlsx.IdentitySANDNS); got != "" {
+		t.Errorf("PeerIdentity(san_dns) on bare cert = %q, want empty", got)
+	}
+	// No peer certificate at all
+	if got := tlsx.PeerIdentity(tls.ConnectionState{}, tlsx.IdentityCN); got != "" {
+		t.Errorf("PeerIdentity with no peer certs = %q, want empty", got)
+	}
+}
+
+func TestNewDisabled(t *testing.T) {
+	for _, o := range []*config.AuthOptions{nil, {}, {Type: MethodNone}} {
+		p, err := New(o, nil, RoleListener)
+		if err != nil {
+			t.Fatalf("New(%+v) error: %v", o, err)
+		}
+		if p != nil {
+			t.Fatalf("New(%+v) = %v, want nil policy", o, p)
+		}
+	}
+}
+
+// A nil policy must behave as if auth were never configured
+func TestNilPolicyIsTransparent(t *testing.T) {
+	var p *Policy
+	id, err := p.Authorize(nil)
+	if err != nil || id.Name != "" {
+		t.Fatalf("nil Authorize = (%+v, %v), want (zero, nil)", id, err)
+	}
+	if p.Enabled() || p.BindsNode() || p.Unrestricted() {
+		t.Fatal("nil policy reports itself active")
+	}
+	if p.NodeBinding() != BindingNone {
+		t.Fatalf("nil NodeBinding = %q", p.NodeBinding())
+	}
+	if !p.TrustsEntryNode(true) || p.TrustsEntryNode(false) {
+		t.Fatal("nil policy must defer to trust_node")
+	}
+	// trust_node semantics are unchanged without a policy
+	node, err := p.ResolveNode("edge-01", "10.0.0.5", true, Identity{})
+	if err != nil || node != "edge-01" {
+		t.Fatalf("nil ResolveNode(trust) = (%q, %v), want edge-01", node, err)
+	}
+	node, err = p.ResolveNode("edge-01", "10.0.0.5", false, Identity{})
+	if err != nil || node != "10.0.0.5" {
+		t.Fatalf("nil ResolveNode(no trust) = (%q, %v), want 10.0.0.5", node, err)
+	}
+	node, err = p.ResolveNode("", "10.0.0.5", true, Identity{})
+	if err != nil || node != "10.0.0.5" {
+		t.Fatalf("nil ResolveNode(no label) = (%q, %v), want 10.0.0.5", node, err)
+	}
+}
+
+func TestNewValidation(t *testing.T) {
+	tests := []struct {
+		name string
+		auth *config.AuthOptions
+		tls  *config.TLSOptions
+		role Role
+	}{
+		{"unknown type", &config.AuthOptions{Type: "kerberos"}, mtlsListenerTLS(), RoleListener},
+		{"no tls", &config.AuthOptions{Type: MethodMTLS}, nil, RoleListener},
+		{"tls disabled", &config.AuthOptions{Type: MethodMTLS}, &config.TLSOptions{}, RoleListener},
+		{"no client_auth", &config.AuthOptions{Type: MethodMTLS}, &config.TLSOptions{Enabled: true}, RoleListener},
+		{"unknown identity", &config.AuthOptions{Type: MethodMTLS, Identity: "serial"}, mtlsListenerTLS(), RoleListener},
+		{"bad pattern", &config.AuthOptions{Type: MethodMTLS, AllowPatterns: []string{"^edge-("}}, mtlsListenerTLS(), RoleListener},
+		{"unknown binding", &config.AuthOptions{Type: MethodMTLS, NodeBinding: "maybe"}, mtlsListenerTLS(), RoleChainListener},
+		{"binding on plain listener", &config.AuthOptions{Type: MethodMTLS, NodeBinding: BindingForce}, mtlsListenerTLS(), RoleListener},
+		{"binding on dialer", &config.AuthOptions{Type: MethodMTLS, NodeBinding: BindingForce}, &config.TLSOptions{Enabled: true}, RoleDialer},
+		{"dialer skips verify", &config.AuthOptions{Type: MethodMTLS}, &config.TLSOptions{Enabled: true, InsecureSkipVerify: true}, RoleDialer},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			if _, err := New(tc.auth, tc.tls, tc.role); err == nil {
+				t.Fatal("expected an error, got nil")
+			}
+		})
+	}
+
+	// A dialer needs TLS but not client_auth: it pins the server's identity
+	if _, err := New(&config.AuthOptions{Type: MethodMTLS}, &config.TLSOptions{Enabled: true}, RoleDialer); err != nil {
+		t.Fatalf("dialer policy rejected: %v", err)
+	}
+}
+
+func TestAuthorizeMatching(t *testing.T) {
+	p, err := New(&config.AuthOptions{
+		Type:          MethodMTLS,
+		Allow:         []string{"edge-01", " edge-02 "},
+		AllowPatterns: []string{`^relay-\d{2}$`},
+	}, mtlsListenerTLS(), RoleListener)
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if p.Unrestricted() {
+		t.Fatal("policy with an allow list reports unrestricted")
+	}
+
+	allowed := []string{"edge-01", "edge-02", "relay-07"}
+	for _, cn := range allowed {
+		id, err := p.Authorize(peerState(leafCN(cn)))
+		if err != nil {
+			t.Errorf("Authorize(%q): %v", cn, err)
+			continue
+		}
+		if id.Name != cn || id.Method != MethodMTLS {
+			t.Errorf("Authorize(%q) = %+v", cn, id)
+		}
+	}
+
+	denied := []string{"edge-99", "relay-007", "prefix-relay-07", "", "EDGE-01"}
+	for _, cn := range denied {
+		if _, err := p.Authorize(peerState(leafCN(cn))); err == nil {
+			t.Errorf("Authorize(%q) allowed, want rejection", cn)
+		}
+	}
+
+	if got, want := p.Rejected(), uint64(len(denied)); got != want {
+		t.Errorf("Rejected = %d, want %d", got, want)
+	}
+	stats := p.Stats()
+	if stats["auth_allowed"].(uint64) != uint64(len(allowed)) {
+		t.Errorf("auth_allowed = %v, want %d", stats["auth_allowed"], len(allowed))
+	}
+	if _, ok := stats["node_binding"]; ok {
+		t.Error("plain listener stats report node_binding")
+	}
+}
+
+// Empty allow and allow_patterns admits any CA-vouched identity, but still
+// records it and still refuses a certificate with no usable identity field
+func TestAuthorizeUnrestricted(t *testing.T) {
+	p, err := New(&config.AuthOptions{Type: MethodMTLS}, mtlsListenerTLS(), RoleListener)
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if !p.Unrestricted() {
+		t.Fatal("empty allow list should be unrestricted")
+	}
+	id, err := p.Authorize(peerState(leafCN("anyone")))
+	if err != nil || id.Name != "anyone" {
+		t.Fatalf("Authorize = (%+v, %v)", id, err)
+	}
+	if _, err := p.Authorize(peerState(leafCN(""))); err == nil {
+		t.Error("certificate with no CN was authorized")
+	}
+	if _, err := p.Authorize(&tls.ConnectionState{}); err == nil {
+		t.Error("connection with no peer certificate was authorized")
+	}
+	if _, err := p.Authorize(nil); err == nil {
+		t.Error("non-TLS connection was authorized")
+	}
+}
+
+func TestResolveNodeBindings(t *testing.T) {
+	newChain := func(binding string) *Policy {
+		p, err := New(&config.AuthOptions{Type: MethodMTLS, NodeBinding: binding}, mtlsListenerTLS(), RoleChainListener)
+		if err != nil {
+			t.Fatalf("New(%q): %v", binding, err)
+		}
+		return p
+	}
+	id := Identity{Name: "edge-01", Method: MethodMTLS}
+
+	// Default under mtls is force
+	if got := newChain("").NodeBinding(); got != BindingForce {
+		t.Errorf("default node_binding = %q, want %q", got, BindingForce)
+	}
+
+	// force ignores the declared label, however it was spoofed
+	force := newChain(BindingForce)
+	for _, declared := range []string{"edge-99", "", "edge-01"} {
+		node, err := force.ResolveNode(declared, "10.0.0.5", true, id)
+		if err != nil || node != "edge-01" {
+			t.Errorf("force ResolveNode(%q) = (%q, %v), want edge-01", declared, node, err)
+		}
+	}
+	if force.TrustsEntryNode(true) {
+		t.Error("force must not trust per-entry node labels")
+	}
+
+	// assert rejects a mismatch and an omission, and leaves per-entry labels
+	// alone so a relay can forward other nodes' entries
+	assert := newChain(BindingAssert)
+	node, err := assert.ResolveNode("edge-01", "10.0.0.5", true, id)
+	if err != nil || node != "edge-01" {
+		t.Errorf("assert ResolveNode(match) = (%q, %v)", node, err)
+	}
+	if _, err := assert.ResolveNode("edge-99", "10.0.0.5", true, id); err == nil {
+		t.Error("assert accepted a mismatched node label")
+	}
+	if _, err := assert.ResolveNode("", "10.0.0.5", true, id); err == nil {
+		t.Error("assert accepted a missing node label")
+	}
+	if !assert.TrustsEntryNode(true) || assert.TrustsEntryNode(false) {
+		t.Error("assert must leave per-entry node labels to trust_node")
+	}
+
+	// none leaves trust_node governing entirely
+	none := newChain(BindingNone)
+	if none.BindsNode() {
+		t.Error("node_binding none should not bind")
+	}
+	node, err = none.ResolveNode("edge-99", "10.0.0.5", true, id)
+	if err != nil || node != "edge-99" {
+		t.Errorf("none ResolveNode = (%q, %v), want edge-99", node, err)
+	}
+	node, err = none.ResolveNode("edge-99", "10.0.0.5", false, id)
+	if err != nil || node != "10.0.0.5" {
+		t.Errorf("none ResolveNode(no trust) = (%q, %v), want 10.0.0.5", node, err)
+	}
+
+	// Binding without an authenticated identity is a refusal, not a fallback
+	if _, err := force.ResolveNode("edge-01", "10.0.0.5", true, Identity{}); err == nil {
+		t.Error("force resolved a node without an identity")
+	}
+}
+
+func TestIdentityApply(t *testing.T) {
+	meta := map[string]any{"type": "tcp_chain"}
+	Identity{}.Apply(meta)
+	if len(meta) != 1 {
+		t.Fatalf("zero identity stamped metadata: %v", meta)
+	}
+	Identity{Name: "edge-01", Method: MethodMTLS}.Apply(meta)
+	if meta["auth_identity"] != "edge-01" || meta["auth_method"] != MethodMTLS {
+		t.Fatalf("metadata = %v", meta)
+	}
+}
+
+func TestVerifyConnectionPinsServer(t *testing.T) {
+	p, err := New(&config.AuthOptions{Type: MethodMTLS, Allow: []string{"relay.internal"}},
+		&config.TLSOptions{Enabled: true}, RoleDialer)
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	if err := p.VerifyConnection(*peerState(leafCN("relay.internal"))); err != nil {
+		t.Errorf("pinned server rejected: %v", err)
+	}
+	if err := p.VerifyConnection(*peerState(leafCN("impostor.internal"))); err == nil {
+		t.Error("unpinned server accepted")
+	}
+}

internal/chain/proto.go

@@ -0,0 +1,98 @@
+package chain
+
+import (
+	"encoding/json"
+	"fmt"
+	"logwisp/internal/core"
+	"math/rand/v2"
+	"time"
+)
+
+// ProtocolVersion is declared in the hello preamble
+const ProtocolVersion = 1
+
+// Hello is the first NDJSON line sent by the dialing side after connect.
+// Reserved for future revisions: auth credential, feature flags (ack, compression).
+type Hello struct {
+	LogWisp int    `json:"logwisp"`
+	Node    string `json:"node,omitempty"`
+	// Auth     string   `json:"auth,omitempty"`
+	// Features []string `json:"features,omitempty"`
+}
+
+// HTTP transport mapping of the chain protocol.
+// Hello preamble equivalent: protocol + node carried as request headers.
+// Reserved extension point: Authorization header for auth, TLS at transport.
+const (
+	HeaderProtocol    = "X-Logwisp-Protocol"
+	HeaderNode        = "X-Logwisp-Node"
+	HeaderAccepted    = "X-Logwisp-Accepted"
+	ContentTypeNDJSON = "application/x-ndjson"
+)
+
+// EncodeHello serializes a newline-terminated hello preamble
+func EncodeHello(node string) ([]byte, error) {
+	b, err := json.Marshal(Hello{LogWisp: ProtocolVersion, Node: node})
+	if err != nil {
+		return nil, err
+	}
+	return append(b, '\n'), nil
+}
+
+// DecodeHello parses and validates a hello preamble line
+func DecodeHello(line []byte) (Hello, error) {
+	var h Hello
+	if err := json.Unmarshal(line, &h); err != nil {
+		return h, fmt.Errorf("malformed hello: %w", err)
+	}
+	if h.LogWisp != ProtocolVersion {
+		return h, fmt.Errorf("unsupported protocol version: %d", h.LogWisp)
+	}
+	return h, nil
+}
+
+// DecodeEntry parses a canonical LogEntry line and applies the node trust policy
+func DecodeEntry(line []byte, connNode string, trustNode bool) (core.LogEntry, error) {
+	var entry core.LogEntry
+	if err := json.Unmarshal(line, &entry); err != nil {
+		return core.LogEntry{}, err
+	}
+	if entry.Time.IsZero() {
+		entry.Time = time.Now()
+	}
+	if entry.Node == "" || !trustNode {
+		entry.Node = connNode
+	}
+	entry.RawSize = int64(len(line))
+	return entry, nil
+}
+
+// EntryFromEvent extracts the structured entry, stamping node identity at
+// first hop. Second return is true when synthesized from a formatted payload.
+func EntryFromEvent(event core.TransportEvent, node, fallbackSource string) (core.LogEntry, bool) {
+	entry := event.Entry
+	synthesized := false
+	if entry.Time.IsZero() {
+		synthesized = true
+		entry = core.LogEntry{
+			Time:    event.Time,
+			Source:  fallbackSource,
+			Message: string(event.Payload),
+		}
+	}
+	if entry.Node == "" {
+		entry.Node = node
+	}
+	return entry, synthesized
+}
+
+// BackoffDelay computes exponential backoff with ±20% jitter
+func BackoffDelay(minD, maxD time.Duration, failures int) time.Duration {
+	d := maxD
+	if failures < 63 {
+		if v := minD << uint(failures-1); v > 0 && v < maxD {
+			d = v
+		}
+	}
+	return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
+}

internal/config/config.go

@@ -0,0 +1,405 @@
+package config
+
+// --- LogWisp Configuration Options ---
+
+// Config is the top-level configuration structure for the LogWisp application
+type Config struct {
+	// Top-level flags for application control
+	ShowVersion bool `toml:"version"`
+	Quiet       bool `toml:"quiet"`
+
+	// Runtime behavior flags
+	StatusReporter   bool `toml:"status_reporter"`
+	ConfigAutoReload bool `toml:"auto_reload"`
+
+	// Configuration file path
+	ConfigFile string `toml:"config_file"`
+
+	// Existing fields
+	Logging   *LogConfig       `toml:"logging"`
+	Pipelines []PipelineConfig `toml:"pipelines"`
+}
+
+// --- Logging Options ---
+
+// LogConfig represents the logging configuration for the LogWisp application itself
+type LogConfig struct {
+	// Output mode: "file", "stdout", "stderr", "split", "all", "none"
+	Output string `toml:"output"`
+
+	// Log level: "debug", "info", "warn", "error"
+	Level string `toml:"level"`
+
+	// Format: "raw", "txt", "json"
+	Format string `toml:"format"`
+
+	// Sanitization policy for console output
+	Sanitization string `toml:"sanitization"`
+
+	// File output settings (when Output includes "file" or "all")
+	File *LogFileConfig `toml:"file"`
+
+	// Console output settings
+	Console *LogConsoleConfig `toml:"console"`
+}
+
+// LogFileConfig defines settings for file-based application logging
+type LogFileConfig struct {
+	// Directory for log files
+	Directory string `toml:"directory"`
+
+	// Base name for log files
+	Name string `toml:"name"`
+
+	// Maximum size per log file in MB
+	MaxSizeMB int64 `toml:"max_size_mb"`
+
+	// Maximum total size of all logs in MB
+	MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
+
+	// Log retention in hours (0 = disabled)
+	RetentionHours float64 `toml:"retention_hours"`
+}
+
+// LogConsoleConfig defines settings for console-based application logging
+type LogConsoleConfig struct {
+	// Target for console output: "stdout", "stderr"
+	Target string `toml:"target"`
+}
+
+// --- Pipeline ---
+
+// PipelineConfig defines a complete data flow from sources to sinks
+type PipelineConfig struct {
+	Name string      `toml:"name"`
+	Flow *FlowConfig `toml:"flow"`
+
+	PluginSources []PluginSourceConfig `toml:"plugin_sources,omitempty"`
+	PluginSinks   []PluginSinkConfig   `toml:"plugin_sinks,omitempty"`
+}
+
+// --- Flow ---
+
+// FlowConfig consolidates all processing stages between sources and sinks
+type FlowConfig struct {
+	Heartbeat *HeartbeatConfig `toml:"heartbeat"`
+	RateLimit *RateLimitConfig `toml:"rate_limit"`
+	Filters   []FilterConfig   `toml:"filters"`
+	Format    *FormatConfig    `toml:"format"`
+}
+
+// --- Heartbeat Options ---
+
+// HeartbeatConfig defines settings for periodic keep-alive or status messages
+type HeartbeatConfig struct {
+	Enabled          bool   `toml:"enabled"`
+	IntervalMS       int64  `toml:"interval_ms"`
+	IncludeTimestamp bool   `toml:"include_timestamp"`
+	IncludeStats     bool   `toml:"include_stats"`
+	Format           string `toml:"format"`
+}
+
+// --- Formatter Options ---
+
+// FormatConfig is a polymorphic struct representing log entry formatting options
+type FormatConfig struct {
+	Type            string `toml:"type"` // "json", "txt", "raw"
+	Flags           int64  `toml:"flags"`
+	TimestampFormat string `toml:"timestamp_format"`
+	SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell"
+}
+
+// --- Rate Limit Options ---
+
+// RateLimitPolicy defines the action to take when a rate limit is exceeded
+type RateLimitPolicy int
+
+const (
+	// PolicyPass allows all logs through, effectively disabling the limiter
+	PolicyPass RateLimitPolicy = iota
+	// PolicyDrop drops logs that exceed the rate limit
+	PolicyDrop
+)
+
+// RateLimitConfig defines the configuration for pipeline-level rate limiting
+type RateLimitConfig struct {
+	// Rate is the number of log entries allowed per second. Default: 0 (disabled)
+	Rate float64 `toml:"rate"`
+	// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate
+	Burst float64 `toml:"burst"`
+	// Policy defines the action to take when the limit is exceeded. "pass" or "drop"
+	Policy string `toml:"policy"`
+	// MaxEntrySizeBytes is the maximum allowed size for a single log entry. 0 = no limit
+	MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes"`
+}
+
+// --- Filter Options ---
+
+// FilterType represents the filter's behavior (include or exclude)
+type FilterType string
+
+const (
+	// FilterTypeInclude specifies that only matching logs will pass
+	FilterTypeInclude FilterType = "include" // Whitelist - only matching logs pass
+	// FilterTypeExclude specifies that matching logs will be dropped
+	FilterTypeExclude FilterType = "exclude" // Blacklist - matching logs are dropped
+)
+
+// FilterLogic represents how multiple filter patterns are combined
+type FilterLogic string
+
+const (
+	// FilterLogicOr specifies that a match on any pattern is sufficient
+	FilterLogicOr FilterLogic = "or" // Match any pattern
+	// FilterLogicAnd specifies that all patterns must match
+	FilterLogicAnd FilterLogic = "and" // Match all patterns
+)
+
+// FilterConfig represents the configuration for a single filter
+type FilterConfig struct {
+	Type     FilterType  `toml:"type"`
+	Logic    FilterLogic `toml:"logic"`
+	Patterns []string    `toml:"patterns"`
+}
+
+// --- Source Options ---
+
+// PluginSourceConfig represents a source plugin instance configuration
+type PluginSourceConfig struct {
+	ID         string         `toml:"id"`
+	Type       string         `toml:"type"`
+	Config     map[string]any `toml:"config"`
+	ConfigFile string         `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
+}
+
+// // SourceConfig is a polymorphic struct representing a single data source
+// type SourceConfig struct {
+// 	Type string `toml:"type"`
+//
+// 	// Polymorphic - only one populated based on type
+// 	File    *FileSourceOptions    `toml:"file,omitempty"`
+// 	Console *ConsoleSourceOptions `toml:"console,omitempty"`
+// }
+
+// NullSourceOptions defines settings for a null source (no configuration needed)
+type NullSourceOptions struct{}
+
+// RandomSourceOptions defines settings for a random log generator source
+type RandomSourceOptions struct {
+	IntervalMS int64  `toml:"interval_ms"`
+	JitterMS   int64  `toml:"jitter_ms"`
+	Format     string `toml:"format"`
+	Length     int64  `toml:"length"`
+	Special    bool   `toml:"special"`
+}
+
+// FileSourceOptions defines settings for a file-based source
+type FileSourceOptions struct {
+	Directory       string `toml:"directory"`
+	Pattern         string `toml:"pattern"` // glob pattern
+	CheckIntervalMS int64  `toml:"check_interval_ms"`
+	Raw             bool   `toml:"raw"`  // keep the whole line as the message, never parse it
+	From            string `toml:"from"` // "end" (default) or "start" of a newly discovered file
+}
+
+// ConsoleSourceOptions defines settings for a stdin-based source
+type ConsoleSourceOptions struct {
+	BufferSize int64 `toml:"buffer_size"`
+}
+
+// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting
+// NDJSON entries from upstream logwisp tcp_chain sinks
+type TCPChainSourceOptions struct {
+	TLS            *TLSOptions  `toml:"tls"`
+	Host           string       `toml:"host"`
+	Port           int64        `toml:"port"`
+	BufferSize     int64        `toml:"buffer_size"`
+	MaxConnections int64        `toml:"max_connections"`  // 0 = unlimited
+	ReadTimeoutMS  int64        `toml:"read_timeout_ms"`  // per-connection idle deadline, 0 = none
+	HelloTimeoutMS int64        `toml:"hello_timeout_ms"` // preamble deadline
+	TrustNode      bool         `toml:"trust_node"`       // false: force node label from remote address
+	Auth           *AuthOptions `toml:"auth"`
+	// Future: password auth block
+}
+
+// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
+// NDJSON batches from upstream logwisp http_chain sinks
+type HTTPChainSourceOptions struct {
+	TLS           *TLSOptions  `toml:"tls"`
+	Host          string       `toml:"host"`
+	Port          int64        `toml:"port"`
+	IngestPath    string       `toml:"ingest_path"`
+	BufferSize    int64        `toml:"buffer_size"`
+	MaxBodyBytes  int64        `toml:"max_body_bytes"`  // per-request cap
+	ReadTimeoutMS int64        `toml:"read_timeout_ms"` // full request read deadline
+	TrustNode     bool         `toml:"trust_node"`      // false: force node label from remote address
+	Auth          *AuthOptions `toml:"auth"`
+	// Future: password auth block
+}
+
+// --- Sink Options ---
+
+// PluginSinkConfig represents a sink plugin instance configuration
+type PluginSinkConfig struct {
+	ID         string         `toml:"id"`
+	Type       string         `toml:"type"`
+	Config     map[string]any `toml:"config"`
+	ConfigFile string         `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
+}
+
+// // SinkConfig is a polymorphic struct representing a single data sink
+// type SinkConfig struct {
+// 	Type string `toml:"type"`
+//
+// 	// Polymorphic - only one populated based on type
+// 	Console *ConsoleSinkOptions `toml:"console,omitempty"`
+// 	File    *FileSinkOptions    `toml:"file,omitempty"`
+// }
+
+// NullSinkOptions defines settings for a null sink (no configuration needed)
+type NullSinkOptions struct{}
+
+// ConsoleSinkOptions defines settings for a console-based sink
+type ConsoleSinkOptions struct {
+	Target     string `toml:"target"` // "stdout", "stderr"
+	BufferSize int64  `toml:"buffer_size"`
+}
+
+// FileSinkOptions defines settings for a file-based sink
+type FileSinkOptions struct {
+	Directory       string  `toml:"directory"`
+	Name            string  `toml:"name"`
+	MaxSizeMB       int64   `toml:"max_size_mb"`
+	MaxTotalSizeMB  int64   `toml:"max_total_size_mb"`
+	MinDiskFreeMB   int64   `toml:"min_disk_free_mb"`
+	RetentionHours  float64 `toml:"retention_hours"`
+	BufferSize      int64   `toml:"buffer_size"`
+	FlushIntervalMs int64   `toml:"flush_interval_ms"`
+}
+
+// TCPSinkOptions defines settings for a TCP server sink
+type TCPSinkOptions struct {
+	TLS               *TLSOptions  `toml:"tls"`
+	Host              string       `toml:"host"`
+	Port              int64        `toml:"port"`
+	BufferSize        int64        `toml:"buffer_size"`        // sink input queue
+	ClientBufferSize  int64        `toml:"client_buffer_size"` // per-client send queue
+	WriteTimeoutMS    int64        `toml:"write_timeout_ms"`   // per-write deadline
+	KeepAlive         bool         `toml:"keep_alive"`
+	KeepAlivePeriodMS int64        `toml:"keep_alive_period_ms"`
+	MaxConnections    int64        `toml:"max_connections"` // 0 = unlimited
+	Auth              *AuthOptions `toml:"auth"`
+	// Future: password auth block
+}
+
+// HTTPSinkOptions defines settings for an HTTP SSE server sink
+type HTTPSinkOptions struct {
+	TLS              *TLSOptions  `toml:"tls"`
+	Host             string       `toml:"host"`
+	Port             int64        `toml:"port"`
+	StreamPath       string       `toml:"stream_path"`
+	StatusPath       string       `toml:"status_path"`
+	BufferSize       int64        `toml:"buffer_size"`        // sink input queue
+	ClientBufferSize int64        `toml:"client_buffer_size"` // per-client send queue
+	WriteTimeoutMS   int64        `toml:"write_timeout_ms"`   // per-SSE-write deadline, 0 = none
+	MaxConnections   int64        `toml:"max_connections"`    // 0 = unlimited
+	Auth             *AuthOptions `toml:"auth"`
+	// Future: password auth block
+}
+
+// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
+// entries to a downstream logwisp tcp_chain source
+type TCPChainSinkOptions struct {
+	TLS               *TLSOptions  `toml:"tls"`
+	Node              string       `toml:"node"` // origin label, default: os.Hostname()
+	Host              string       `toml:"host"`
+	Port              int64        `toml:"port"`
+	BufferSize        int64        `toml:"buffer_size"`
+	DialTimeoutMS     int64        `toml:"dial_timeout_ms"`
+	WriteTimeoutMS    int64        `toml:"write_timeout_ms"`
+	BackoffMinMS      int64        `toml:"backoff_min_ms"`
+	BackoffMaxMS      int64        `toml:"backoff_max_ms"`
+	KeepAlivePeriodMS int64        `toml:"keep_alive_period_ms"`
+	KeepAlive         bool         `toml:"keep_alive"`
+	Auth              *AuthOptions `toml:"auth"`
+	// Future: password auth block
+}
+
+// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting
+// NDJSON batches to a downstream logwisp http_chain source
+type HTTPChainSinkOptions struct {
+	TLS              *TLSOptions  `toml:"tls"`
+	Node             string       `toml:"node"` // origin label, default: os.Hostname()
+	Host             string       `toml:"host"`
+	Port             int64        `toml:"port"`
+	IngestPath       string       `toml:"ingest_path"`
+	BufferSize       int64        `toml:"buffer_size"`
+	MaxBatchCount    int64        `toml:"max_batch_count"`
+	MaxBatchBytes    int64        `toml:"max_batch_bytes"`
+	FlushIntervalMS  int64        `toml:"flush_interval_ms"`
+	RequestTimeoutMS int64        `toml:"request_timeout_ms"` // covers dial + write + response
+	BackoffMinMS     int64        `toml:"backoff_min_ms"`
+	BackoffMaxMS     int64        `toml:"backoff_max_ms"`
+	Auth             *AuthOptions `toml:"auth"`
+	// Future: password auth block
+}
+
+// --- Auth Options ---
+
+// AuthOptions defines certificate-based authorization for network plugins.
+// It sits beside `tls` rather than inside it: TLS answers "is this channel
+// private and does the peer chain to a CA", auth answers "may *this* peer do
+// *this*". One shape serves both roles:
+//   - Listeners (tcp/http sinks, tcp_chain/http_chain sources) authorize the
+//     peer's client certificate; type "mtls" requires tls.client_auth.
+//   - Dialers (tcp_chain/http_chain sinks) pin the server's identity beyond
+//     hostname verification.
+type AuthOptions struct {
+	// Method: "none" (default, preserves pre-auth behavior) | "mtls"
+	Type string `toml:"type"`
+
+	// Certificate field carrying the identity:
+	// "cn" (default) | "san_dns" | "san_uri" | "san_email"
+	Identity string `toml:"identity"`
+
+	// Exact identity matches. Empty Allow *and* AllowPatterns means "any
+	// identity the CA vouches for" - today's behavior, but with the identity
+	// recorded and node binding available.
+	Allow []string `toml:"allow"`
+
+	// RE2 patterns matched against the identity; anchor them yourself
+	AllowPatterns []string `toml:"allow_patterns"`
+
+	// Chain sources only: "none" | "assert" | "force" (default "force" when
+	// Type is "mtls"). Overrides trust_node.
+	NodeBinding string `toml:"node_binding"`
+}
+
+// --- TLS Options ---
+
+// TLSOptions defines transport security for network sources and sinks.
+// One shape serves both roles so the config block is uniform:
+//   - Listeners (tcp/http sinks, tcp_chain/http_chain sources) use
+//     cert_file/key_file as server identity; client_auth/client_ca_file
+//     require and verify peer certificates (mTLS).
+//   - Dialers (tcp_chain/http_chain sinks) use ca_file/server_name to verify
+//     the server; cert_file/key_file present a client identity (mTLS).
+type TLSOptions struct {
+	Enabled bool `toml:"enabled"`
+
+	// Local identity: required for listeners, optional for dialers (mTLS)
+	CertFile string `toml:"cert_file"`
+	KeyFile  string `toml:"key_file"`
+
+	// Listener-side peer verification (mTLS)
+	ClientAuth   bool   `toml:"client_auth"`
+	ClientCAFile string `toml:"client_ca_file"`
+
+	// Dialer-side peer verification
+	CAFile             string `toml:"ca_file"`     // empty = system trust store
+	ServerName         string `toml:"server_name"` // default: config host
+	InsecureSkipVerify bool   `toml:"insecure_skip_verify"`
+
+	// Minimum protocol version: "1.2" | "1.3" (default "1.3")
+	MinVersion string `toml:"min_version"`
+}

internal/config/loader.go

@@ -1,4 +1,3 @@
-// FILE: logwisp/src/internal/config/loader.go
 package config
 
 import (
@@ -8,75 +7,15 @@ import (
 	"path/filepath"
 	"strings"
 
+	"logwisp/internal/core"
+
 	lconfig "github.com/lixenwraith/config"
 )
 
+// configManager holds the global instance of the configuration manager
 var configManager *lconfig.Config
 
-// Hot reload access
+// Load is the single entry point for loading all application configuration
-func GetConfigManager() *lconfig.Config {
-	return configManager
-}
-
-func defaults() *Config {
-	return &Config{
-		// Top-level flag defaults
-		Background:  false,
-		ShowVersion: false,
-		Quiet:       false,
-
-		// Runtime behavior defaults
-		DisableStatusReporter: false,
-		ConfigAutoReload:      false,
-
-		// Child process indicator
-		BackgroundDaemon: false,
-
-		// Existing defaults
-		Logging: &LogConfig{
-			Output: "stdout",
-			Level:  "info",
-			File: &LogFileConfig{
-				Directory:      "./log",
-				Name:           "logwisp",
-				MaxSizeMB:      100,
-				MaxTotalSizeMB: 1000,
-				RetentionHours: 168, // 7 days
-			},
-			Console: &LogConsoleConfig{
-				Target: "stdout",
-				Format: "txt",
-			},
-		},
-		Pipelines: []PipelineConfig{
-			{
-				Name: "default",
-				Sources: []SourceConfig{
-					{
-						Type: "directory",
-						Directory: &DirectorySourceOptions{
-							Path:            "./",
-							Pattern:         "*.log",
-							CheckIntervalMS: int64(100),
-						},
-					},
-				},
-				Sinks: []SinkConfig{
-					{
-						Type: "console",
-						Console: &ConsoleSinkOptions{
-							Target:     "stdout",
-							Colorize:   false,
-							BufferSize: 100,
-						},
-					},
-				},
-			},
-		},
-	}
-}
-
-// Single entry point for loading all configuration
 func Load(args []string) (*Config, error) {
 	configPath, isExplicit := resolveConfigPath(args)
 	// Build configuration with all sources
@@ -110,7 +49,9 @@ func Load(args []string) (*Config, error) {
 		// Handle file not found errors - maintain existing behavior
 		if errors.Is(err, lconfig.ErrConfigNotFound) {
 			if isExplicit {
-				return nil, fmt.Errorf("config file not found: %s", configPath)
+				// Return empty config with file path
+				finalConfig.ConfigFile = configPath
+				return finalConfig, fmt.Errorf("config file not found: %s", configPath)
 			}
 			// If the default config file is not found, it's not an error, default/cli/env will be used
 		} else {
@@ -124,10 +65,102 @@ func Load(args []string) (*Config, error) {
 	// Store the manager for hot reload
 	configManager = cfg
 
+	// Surface typo'd flags (e.g. --status-reporter vs --status_reporter);
+	// pre-logger phase, stderr only, suppressed in quiet mode
+	if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet {
+		fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
+	}
+
+	// Start watcher if auto-reload is enabled
+	if finalConfig.ConfigAutoReload {
+		watchOpts := lconfig.WatchOptions{
+			PollInterval:      core.ReloadWatchPollInterval,
+			Debounce:          core.ReloadWatchDebounce,
+			ReloadTimeout:     core.ReloadWatchTimeout,
+			VerifyPermissions: true,
+		}
+		cfg.AutoUpdateWithOptions(watchOpts)
+	}
+
 	return finalConfig, nil
 }
 
-// Returns the configuration file path
+// GetConfigManager returns the global configuration manager instance for hot-reloading
+func GetConfigManager() *lconfig.Config {
+	return configManager
+}
+
+// defaults provides the default configuration values for the application
+func defaults() *Config {
+	return &Config{
+		// Top-level flag defaults
+		ShowVersion: false,
+		Quiet:       false,
+
+		// Runtime behavior defaults
+		StatusReporter:   true,
+		ConfigAutoReload: false,
+
+		// Existing defaults
+		Logging: &LogConfig{
+			Output: "stdout",
+			Level:  "info",
+			Format: "txt",
+			File: &LogFileConfig{
+				Directory:      "./log",
+				Name:           "logwisp",
+				MaxSizeMB:      100,
+				MaxTotalSizeMB: 1000,
+				RetentionHours: 168, // 7 days
+			},
+			Console: &LogConsoleConfig{
+				Target: "stdout",
+			},
+		},
+		Pipelines: []PipelineConfig{
+			{
+				Name: "default_pipeline",
+				Flow: &FlowConfig{
+					RateLimit: &RateLimitConfig{
+						Rate:              5,
+						Burst:             10,
+						Policy:            "drop",
+						MaxEntrySizeBytes: 65536,
+					},
+					Format: &FormatConfig{
+						Type:            "json",
+						SanitizerPolicy: "json",
+					},
+				},
+				PluginSources: []PluginSourceConfig{
+					{
+						ID:   "default_source",
+						Type: "random",
+						Config: map[string]any{
+							"special": true,
+						},
+						// Config: &FileSourceOptions{
+						// 	Directory:       "./",
+						// 	Pattern:         "*.log",
+						// 	CheckIntervalMS: int64(100),
+					},
+				},
+				PluginSinks: []PluginSinkConfig{
+					{
+						ID:   "default_sink",
+						Type: "console",
+						Config: map[string]any{
+							"target":      "stdout",
+							"buffer_size": 100,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// resolveConfigPath determines the configuration file path based on CLI args, env vars, and default locations
 func resolveConfigPath(args []string) (path string, isExplicit bool) {
 	// 1. Check for --config flag in command-line arguments (highest precedence)
 	for i, arg := range args {
@@ -163,6 +196,7 @@ func resolveConfigPath(args []string) (path string, isExplicit bool) {
 	return "logwisp.toml", false
 }
 
+// customEnvTransform converts TOML-style config paths (e.g., logging.level) to environment variable format (LOGGING_LEVEL)
 func customEnvTransform(path string) string {
 	env := strings.ReplaceAll(path, ".", "_")
 	env = strings.ToUpper(env)

internal/config/validate.go

@@ -0,0 +1,83 @@
+package config
+
+import (
+	"fmt"
+
+	lconfig "github.com/lixenwraith/config"
+)
+
+// ValidateConfig validates top-level structure only
+// Value range validation is delegated to component constructors
+func ValidateConfig(cfg *Config) error {
+	if cfg == nil {
+		return fmt.Errorf("config is nil")
+	}
+
+	if len(cfg.Pipelines) == 0 {
+		return fmt.Errorf("no pipelines configured")
+	}
+
+	// Reject duplicate pipeline names (service map is keyed by name)
+	names := make(map[string]struct{}, len(cfg.Pipelines))
+	for i, p := range cfg.Pipelines {
+		if _, dup := names[p.Name]; dup {
+			return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
+		}
+		names[p.Name] = struct{}{}
+	}
+
+	if err := validateLogConfig(cfg.Logging); err != nil {
+		return fmt.Errorf("logging: %w", err)
+	}
+
+	for i, p := range cfg.Pipelines {
+		if err := lconfig.NonEmpty(p.Name); err != nil {
+			return fmt.Errorf("pipeline[%d].name: %w", i, err)
+		}
+		if len(p.PluginSources) == 0 {
+			return fmt.Errorf("pipeline[%d]: no sources defined", i)
+		}
+		if len(p.PluginSinks) == 0 {
+			return fmt.Errorf("pipeline[%d]: no sinks defined", i)
+		}
+	}
+
+	return nil
+}
+
+// validateLogConfig validates application logging settings
+func validateLogConfig(cfg *LogConfig) error {
+	if cfg == nil {
+		return nil
+	}
+
+	validateOutput := lconfig.OneOf("file", "stdout", "stderr", "split", "all", "none")
+	if err := validateOutput(cfg.Output); err != nil {
+		return fmt.Errorf("output: %w", err)
+	}
+
+	validateLevel := lconfig.OneOf("debug", "info", "warn", "error")
+	if err := validateLevel(cfg.Level); err != nil {
+		return fmt.Errorf("level: %w", err)
+	}
+
+	if cfg.Format != "" {
+		if err := lconfig.OneOf("raw", "txt", "json")(cfg.Format); err != nil {
+			return fmt.Errorf("format: %w", err)
+		}
+	}
+	if cfg.Sanitization != "" {
+		if err := lconfig.OneOf("raw", "json", "txt", "shell")(cfg.Sanitization); err != nil {
+			return fmt.Errorf("sanitization: %w", err)
+		}
+	}
+
+	if cfg.Console != nil {
+		validateTarget := lconfig.OneOf("stdout", "stderr", "split")
+		if err := validateTarget(cfg.Console.Target); err != nil {
+			return fmt.Errorf("console.target: %w", err)
+		}
+	}
+
+	return nil
+}

internal/core/capability.go

@@ -0,0 +1,20 @@
+package core
+
+// Capability represents a plugin feature
+type Capability string
+
+const (
+	// Network capabilities
+	CapNetLimit Capability = "netlimit"
+	CapTLS      Capability = "tls"
+	CapAuth     Capability = "auth"
+
+	// Session capabilities
+	CapSessionAware   Capability = "session_aware"
+	CapMultiSession   Capability = "multi_session"
+	CapSingleInstance Capability = "single_instance"
+
+	// Stream capabilities
+	CapBidirectional Capability = "bidirectional"
+	CapCompression   Capability = "compression"
+)

internal/core/const.go

@@ -0,0 +1,29 @@
+package core
+
+import (
+	"time"
+)
+
+const (
+	MaxLogEntryBytes = 1024 * 1024
+
+	FileWatcherPollInterval = 100 * time.Millisecond
+
+	SessionDefaultMaxIdleTime = 30 * time.Minute
+
+	SessionCleanupInterval = 5 * time.Minute
+
+	ServiceStatsUpdateInterval = 1 * time.Second
+
+	ShutdownTimeout = 10 * time.Second
+
+	ConfigReloadTimeout = 30 * time.Second
+
+	LoggerShutdownTimeout = 2 * time.Second
+
+	ReloadWatchPollInterval = time.Second
+
+	ReloadWatchDebounce = 500 * time.Millisecond
+
+	ReloadWatchTimeout = 30 * time.Second
+)

internal/core/flow.go

@@ -0,0 +1,27 @@
+package core
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// LogEntry represents a single log record flowing through the pipeline
+type LogEntry struct {
+	Time    time.Time       `json:"time"`
+	Node    string          `json:"node,omitempty"` // origin node identity for chained topologies; first hop stamps, relays preserve
+	Source  string          `json:"source"`
+	Level   string          `json:"level,omitempty"`
+	Message string          `json:"message"`
+	Fields  json.RawMessage `json:"fields,omitempty"`
+	RawSize int64           `json:"-"`
+}
+
+// TransportEvent contains the final payload and minimal metadata needed by sinks
+type TransportEvent struct {
+	Time time.Time
+	// Formatted, serialized log payload
+	Payload []byte
+	// Structured entry for re-serializing sinks (chain links). Zero Time => absent
+	Entry LogEntry
+}
+

internal/filter/chain.go

@@ -1,17 +1,16 @@
-// FILE: logwisp/src/internal/filter/chain.go
 package filter
 
 import (
 	"fmt"
 	"sync/atomic"
 
-	"logwisp/src/internal/config"
+	"logwisp/internal/config"
-	"logwisp/src/internal/core"
+	"logwisp/internal/core"
 
 	"github.com/lixenwraith/log"
 )
 
-// Manages multiple filters in sequence
+// Chain manages a sequence of filters, applying them in order
 type Chain struct {
 	filters []*Filter
 	logger  *log.Logger
@@ -21,7 +20,7 @@ type Chain struct {
 	totalPassed    atomic.Uint64
 }
 
-// Creates a new filter chain from configurations
+// NewChain creates a new filter chain from a slice of filter configurations
 func NewChain(configs []config.FilterConfig, logger *log.Logger) (*Chain, error) {
 	chain := &Chain{
 		filters: make([]*Filter, 0, len(configs)),
@@ -42,7 +41,7 @@ func NewChain(configs []config.FilterConfig, logger *log.Logger) (*Chain, error)
 	return chain, nil
 }
 
-// Runs all filters in sequence, returns true if the entry passes all filters
+// Apply runs a log entry through all filters in the chain
 func (c *Chain) Apply(entry core.LogEntry) bool {
 	c.totalProcessed.Add(1)
 
@@ -67,7 +66,7 @@ func (c *Chain) Apply(entry core.LogEntry) bool {
 	return true
 }
 
-// Returns chain statistics
+// GetStats returns aggregated statistics for the entire chain
 func (c *Chain) GetStats() map[string]any {
 	filterStats := make([]map[string]any, len(c.filters))
 	for i, filter := range c.filters {

internal/filter/filter.go

@@ -1,4 +1,3 @@
-// FILE: logwisp/src/internal/filter/filter.go
 package filter
 
 import (
@@ -7,13 +6,14 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"logwisp/src/internal/config"
+	"logwisp/internal/config"
-	"logwisp/src/internal/core"
+	"logwisp/internal/core"
 
+	lconfig "github.com/lixenwraith/config"
 	"github.com/lixenwraith/log"
 )
 
-// Applies regex-based filtering to log entries
+// Filter applies regex-based filtering to log entries
 type Filter struct {
 	config   config.FilterConfig
 	patterns []*regexp.Regexp
@@ -26,8 +26,22 @@ type Filter struct {
 	totalDropped   atomic.Uint64
 }
 
-// Creates a new filter from configuration
+// NewFilter creates a new filter from a configuration
 func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) {
+	// Validate enums before setting defaults
+	if cfg.Type != "" {
+		validateType := lconfig.OneOf(config.FilterTypeInclude, config.FilterTypeExclude)
+		if err := validateType(cfg.Type); err != nil {
+			return nil, fmt.Errorf("type: %w", err)
+		}
+	}
+	if cfg.Logic != "" {
+		validateLogic := lconfig.OneOf(config.FilterLogicOr, config.FilterLogicAnd)
+		if err := validateLogic(cfg.Logic); err != nil {
+			return nil, fmt.Errorf("logic: %w", err)
+		}
+	}
+
 	// Set defaults
 	if cfg.Type == "" {
 		cfg.Type = config.FilterTypeInclude
@@ -46,7 +60,7 @@ func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) {
 	for i, pattern := range cfg.Patterns {
 		re, err := regexp.Compile(pattern)
 		if err != nil {
-			return nil, fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
+			return nil, fmt.Errorf("pattern[%d] '%s': %w", i, pattern, err)
 		}
 		f.patterns = append(f.patterns, re)
 	}
@@ -60,7 +74,7 @@ func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) {
 	return f, nil
 }
 
-// Checks if a log entry should be passed through
+// Apply determines if a log entry should be passed through based on the filter's rules
 func (f *Filter) Apply(entry core.LogEntry) bool {
 	f.totalProcessed.Add(1)
 
@@ -130,7 +144,44 @@ func (f *Filter) Apply(entry core.LogEntry) bool {
 	return shouldPass
 }
 
-// Checks if text matches the patterns according to the logic
+// GetStats returns the filter's current statistics
+func (f *Filter) GetStats() map[string]any {
+	return map[string]any{
+		"type":            f.config.Type,
+		"logic":           f.config.Logic,
+		"pattern_count":   len(f.patterns),
+		"total_processed": f.totalProcessed.Load(),
+		"total_matched":   f.totalMatched.Load(),
+		"total_dropped":   f.totalDropped.Load(),
+	}
+}
+
+// UpdatePatterns allows for dynamic, thread-safe updates to the filter's regex patterns
+func (f *Filter) UpdatePatterns(patterns []string) error {
+	compiled := make([]*regexp.Regexp, 0, len(patterns))
+
+	// Compile all patterns first
+	for i, pattern := range patterns {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
+		}
+		compiled = append(compiled, re)
+	}
+
+	// Update atomically
+	f.mu.Lock()
+	f.patterns = compiled
+	f.config.Patterns = patterns
+	f.mu.Unlock()
+
+	f.logger.Info("msg", "Filter patterns updated",
+		"component", "filter",
+		"pattern_count", len(patterns))
+	return nil
+}
+
+// matches checks if the given text matches the filter's patterns according to its logic
 func (f *Filter) matches(text string) bool {
 	switch f.config.Logic {
 	case config.FilterLogicOr:
@@ -159,40 +210,3 @@ func (f *Filter) matches(text string) bool {
 		return false
 	}
 }
-
-// Returns filter statistics
-func (f *Filter) GetStats() map[string]any {
-	return map[string]any{
-		"type":            f.config.Type,
-		"logic":           f.config.Logic,
-		"pattern_count":   len(f.patterns),
-		"total_processed": f.totalProcessed.Load(),
-		"total_matched":   f.totalMatched.Load(),
-		"total_dropped":   f.totalDropped.Load(),
-	}
-}
-
-// Allows dynamic pattern updates
-func (f *Filter) UpdatePatterns(patterns []string) error {
-	compiled := make([]*regexp.Regexp, 0, len(patterns))
-
-	// Compile all patterns first
-	for i, pattern := range patterns {
-		re, err := regexp.Compile(pattern)
-		if err != nil {
-			return fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
-		}
-		compiled = append(compiled, re)
-	}
-
-	// Update atomically
-	f.mu.Lock()
-	f.patterns = compiled
-	f.config.Patterns = patterns
-	f.mu.Unlock()
-
-	f.logger.Info("msg", "Filter patterns updated",
-		"component", "filter",
-		"pattern_count", len(patterns))
-	return nil
-}

internal/flow/flow.go

@@ -0,0 +1,164 @@
+package flow
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/filter"
+	"logwisp/internal/format"
+
+	"github.com/lixenwraith/log"
+)
+
+// Flow manages the complete processing pipeline for log entries:
+// LogEntry -> Rate Limiter -> Filters -> Formatter (with Sanitizer) -> TransportEvent
+type Flow struct {
+	rateLimiter *RateLimiter
+	filterChain *filter.Chain
+	formatter   format.Formatter
+	heartbeat   *HeartbeatGenerator
+	logger      *log.Logger
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	totalDropped   atomic.Uint64
+	totalFormatted atomic.Uint64
+}
+
+// NewFlow creates a flow processor from configuration
+func NewFlow(cfg *config.FlowConfig, logger *log.Logger) (*Flow, error) {
+	if cfg == nil {
+		cfg = &config.FlowConfig{}
+	}
+
+	f := &Flow{
+		logger: logger,
+	}
+
+	// Create rate limiter if configured
+	if cfg.RateLimit != nil {
+		limiter, err := NewRateLimiter(*cfg.RateLimit, logger)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create rate limiter: %w", err)
+		}
+		f.rateLimiter = limiter
+	}
+
+	// Create filter chain if configured
+	if len(cfg.Filters) > 0 {
+		chain, err := filter.NewChain(cfg.Filters, logger)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create filter chain: %w", err)
+		}
+		f.filterChain = chain
+	}
+
+	// Create formatter with sanitizer integration
+	formatter, err := format.NewFormatter(cfg.Format)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create formatter: %w", err)
+	}
+	f.formatter = formatter
+
+	// Create heartbeat generator with the same formatter if configured
+	if cfg.Heartbeat != nil {
+		hb, err := NewHeartbeatGenerator(cfg.Heartbeat, formatter, logger)
+		if err != nil {
+			return nil, fmt.Errorf("heartbeat: %w", err)
+		}
+		f.heartbeat = hb
+	}
+
+	logger.Info("msg", "Flow processor created",
+		"component", "flow",
+		"rate_limiter", f.rateLimiter != nil,
+		"filter_chain", f.filterChain != nil,
+		"formatter", formatter.Name(),
+		"heartbeat", f.heartbeat != nil)
+
+	return f, nil
+}
+
+// Process applies all flow stages to a log entry
+// Returns TransportEvent and whether entry passed all stages
+func (f *Flow) Process(entry core.LogEntry) (core.TransportEvent, bool) {
+	f.totalProcessed.Add(1)
+
+	// Stage 1: Rate limiting
+	if f.rateLimiter != nil {
+		if !f.rateLimiter.Allow(entry) {
+			f.totalDropped.Add(1)
+			return core.TransportEvent{}, false
+		}
+	}
+
+	// Stage 2: Filtering
+	if f.filterChain != nil {
+		if !f.filterChain.Apply(entry) {
+			f.totalDropped.Add(1)
+			return core.TransportEvent{}, false
+		}
+	}
+
+	// Stage 3: Formatting
+	formatted, err := f.formatter.Format(entry)
+	if err != nil {
+		f.logger.Error("msg", "Failed to format log entry",
+			"component", "flow",
+			"error", err)
+		f.totalDropped.Add(1)
+		return core.TransportEvent{}, false
+	}
+
+	f.totalFormatted.Add(1)
+
+	// Create transport event
+	event := core.TransportEvent{
+		Time:    entry.Time,
+		Payload: formatted,
+		Entry:   entry, // Carry structured entry so chain sinks are format-independent
+	}
+
+	return event, true
+}
+
+// StartHeartbeat starts the heartbeat generator if configured
+// Returns channel that emits heartbeat events
+func (f *Flow) StartHeartbeat(ctx context.Context) <-chan core.TransportEvent {
+	if f.heartbeat == nil {
+		return nil
+	}
+	return f.heartbeat.Start(ctx)
+}
+
+// GetStats returns flow statistics
+func (f *Flow) GetStats() map[string]any {
+	stats := map[string]any{
+		"total_processed": f.totalProcessed.Load(),
+		"total_dropped":   f.totalDropped.Load(),
+		"total_formatted": f.totalFormatted.Load(),
+	}
+
+	if f.rateLimiter != nil {
+		stats["rate_limiter"] = f.rateLimiter.GetStats()
+	}
+
+	if f.filterChain != nil {
+		stats["filters"] = f.filterChain.GetStats()
+	}
+
+	if f.formatter != nil {
+		stats["formatter"] = f.formatter.Name()
+	}
+
+	if f.heartbeat != nil {
+		stats["heartbeat_enabled"] = true
+		stats["heartbeat_interval_ms"] = f.heartbeat.IntervalMS()
+	}
+
+	return stats
+}
+

internal/flow/heartbeat.go

@@ -0,0 +1,171 @@
+package flow
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/format"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/log/formatter"
+)
+
+const (
+	MinHeartbeatIntervalMS     = 100
+	DefaultHeartbeatIntervalMS = 1000
+	DefaultHeartbeatFormat     = "txt"
+)
+
+// HeartbeatGenerator produces periodic heartbeat events
+type HeartbeatGenerator struct {
+	config    *config.HeartbeatConfig
+	formatter format.Formatter // Use flow's formatter
+	logger    *log.Logger
+	beatCount atomic.Uint64
+	lastBeat  atomic.Value // time.Time
+}
+
+// NewHeartbeatGenerator creates a new heartbeat generator
+func NewHeartbeatGenerator(cfg *config.HeartbeatConfig, formatter format.Formatter, logger *log.Logger) (*HeartbeatGenerator, error) {
+	if cfg == nil || !cfg.Enabled {
+		return nil, nil
+	}
+
+	// Validate
+	if cfg.IntervalMS == 0 {
+		cfg.IntervalMS = DefaultHeartbeatIntervalMS
+	} else if cfg.IntervalMS < MinHeartbeatIntervalMS {
+		return nil, fmt.Errorf("interval_ms: must be >= %d, got %d", MinHeartbeatIntervalMS, cfg.IntervalMS)
+	}
+
+	validateFormat := lconfig.OneOf("txt", "json", "raw", "")
+	if err := validateFormat(cfg.Format); err != nil {
+		return nil, fmt.Errorf("format: %w", err)
+	}
+
+	// Defaults
+	if cfg.Format == "" {
+		cfg.Format = DefaultHeartbeatFormat
+	}
+
+	hg := &HeartbeatGenerator{
+		config:    cfg,
+		formatter: formatter,
+		logger:    logger,
+	}
+	hg.lastBeat.Store(time.Time{})
+	return hg, nil
+}
+
+// Start begins generating heartbeat events
+func (hg *HeartbeatGenerator) Start(ctx context.Context) <-chan core.TransportEvent {
+	ch := make(chan core.TransportEvent)
+
+	go func() {
+		defer close(ch)
+
+		ticker := time.NewTicker(time.Duration(hg.config.IntervalMS) * time.Millisecond)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case t := <-ticker.C:
+				event := hg.generateHeartbeat(t)
+				select {
+				case ch <- event:
+					hg.beatCount.Add(1)
+					hg.lastBeat.Store(t)
+				case <-ctx.Done():
+					return
+				}
+			}
+		}
+	}()
+
+	return ch
+}
+
+// generateHeartbeat creates a heartbeat transport event
+func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent {
+	// Create heartbeat as LogEntry for consistent formatting
+	entry := core.LogEntry{
+		Time:    t,
+		Source:  "heartbeat",
+		Level:   "INFO",
+		Message: "heartbeat",
+	}
+
+	// Add stats if configured
+	if hg.config.IncludeStats {
+		fields := map[string]any{
+			"type":       "heartbeat",
+			"beat_count": hg.beatCount.Load(),
+		}
+
+		if last, ok := hg.lastBeat.Load().(time.Time); ok && !last.IsZero() {
+			fields["interval_ms"] = t.Sub(last).Milliseconds()
+		}
+
+		fieldsJSON, _ := json.Marshal(fields)
+		entry.Fields = fieldsJSON
+	}
+
+	// Use formatter to generate payload
+	var payload []byte
+	var err error
+
+	// Check if we need special formatting for heartbeat
+	if hg.config.Format == "comment" {
+		// SSE comment format - bypass formatter for this special case
+		if hg.config.IncludeStats {
+			beatNum := hg.beatCount.Load()
+			payload = []byte(": heartbeat " + t.Format(time.RFC3339) +
+				" [#" + strconv.FormatUint(beatNum, 10) + "]\n")
+		} else {
+			payload = []byte(": heartbeat " + t.Format(time.RFC3339) + "\n")
+		}
+	} else {
+		// Use flow's formatter for consistent formatting
+		if adapter, ok := hg.formatter.(*format.FormatterAdapter); ok {
+			// Customize flags for heartbeat if needed
+			customFlags := int64(0)
+			if !hg.config.IncludeTimestamp {
+				// Remove timestamp flag if not wanted
+				customFlags = formatter.FlagShowLevel
+			} else {
+				customFlags = formatter.FlagDefault
+			}
+			payload, err = adapter.FormatWithFlags(entry, customFlags)
+		} else {
+			// Fallback to standard format
+			payload, err = hg.formatter.Format(entry)
+		}
+
+		if err != nil {
+			hg.logger.Error("msg", "Failed to format heartbeat",
+				"error", err)
+			// Fallback to simple text
+			payload = []byte("heartbeat: " + t.Format(time.RFC3339) + "\n")
+		}
+	}
+
+	return core.TransportEvent{
+		Time:    t,
+		Payload: payload,
+		Entry:   entry, // heartbeats traverse chain links as structured entries
+	}
+}
+
+// IntervalMS returns the heartbeat interval in milliseconds
+func (hg *HeartbeatGenerator) IntervalMS() int64 {
+	return hg.config.IntervalMS
+}

internal/flow/ratelimiter.go

@@ -1,19 +1,21 @@
-// FILE: logwisp/src/internal/limit/rate.go
+package flow
-package limit
 
 import (
+	"fmt"
 	"strings"
 	"sync/atomic"
 
-	"logwisp/src/internal/config"
+	"logwisp/internal/config"
-	"logwisp/src/internal/core"
+	"logwisp/internal/core"
+	"logwisp/internal/tokenbucket"
 
+	lconfig "github.com/lixenwraith/config"
 	"github.com/lixenwraith/log"
 )
 
-// Enforces rate limits on log entries flowing through a pipeline.
+// RateLimiter enforces rate limits on log entries flowing through a pipeline
 type RateLimiter struct {
-	bucket *TokenBucket
+	bucket *tokenbucket.TokenBucket
 	policy config.RateLimitPolicy
 	logger *log.Logger
 
@@ -23,41 +25,51 @@ type RateLimiter struct {
 	droppedCount       atomic.Uint64
 }
 
-// Creates a new rate limiter. If cfg.Rate is 0, it returns nil.
+// NewRateLimiter creates a new pipeline-level rate limiter from configuration
 func NewRateLimiter(cfg config.RateLimitConfig, logger *log.Logger) (*RateLimiter, error) {
+	// Rate <= 0 means disabled
 	if cfg.Rate <= 0 {
 		return nil, nil // No rate limit
 	}
 
+	// Validate
+	if err := lconfig.NonNegative(cfg.Rate); err != nil {
+		return nil, fmt.Errorf("rate: %w", err)
+	}
+	if err := lconfig.NonNegative(cfg.Burst); err != nil {
+		return nil, fmt.Errorf("burst: %w", err)
+	}
+	if err := lconfig.NonNegative(cfg.MaxEntrySizeBytes); err != nil {
+		return nil, fmt.Errorf("max_entry_size_bytes: %w", err)
+	}
+
+	// Defaults
 	burst := cfg.Burst
 	if burst <= 0 {
-		burst = cfg.Rate // Default burst to rate
+		burst = cfg.Rate
 	}
 
 	var policy config.RateLimitPolicy
 	switch strings.ToLower(cfg.Policy) {
 	case "drop":
 		policy = config.PolicyDrop
-	default:
+	case "pass", "":
 		policy = config.PolicyPass
+	default:
+		return nil, fmt.Errorf("policy: must be one of [drop, pass], got %s", cfg.Policy)
 	}
 
 	l := &RateLimiter{
-		bucket:            NewTokenBucket(burst, cfg.Rate),
+		bucket:            tokenbucket.New(burst, cfg.Rate),
 		policy:            policy,
 		logger:            logger,
 		maxEntrySizeBytes: cfg.MaxEntrySizeBytes,
 	}
 
-	if cfg.Rate > 0 {
-		l.bucket = NewTokenBucket(burst, cfg.Rate)
-	}
-
 	return l, nil
 }
 
-// Checks if a log entry is allowed to pass based on the rate limit.
+// Allow checks if a log entry is permitted to pass based on the rate limit
-// It returns true if the entry should pass, false if it should be dropped.
 func (l *RateLimiter) Allow(entry core.LogEntry) bool {
 	if l == nil || l.policy == config.PolicyPass {
 		return true
@@ -83,7 +95,7 @@ func (l *RateLimiter) Allow(entry core.LogEntry) bool {
 	return true
 }
 
-// GetStats returns the statistics for the limiter.
+// GetStats returns statistics for the rate limiter
 func (l *RateLimiter) GetStats() map[string]any {
 	if l == nil {
 		return map[string]any{
@@ -93,6 +105,8 @@ func (l *RateLimiter) GetStats() map[string]any {
 
 	stats := map[string]any{
 		"enabled":               true,
+		"rate":                  l.bucket.Rate(),
+		"burst":                 l.bucket.Capacity(),
 		"dropped_total":         l.droppedCount.Load(),
 		"dropped_by_size_total": l.droppedBySizeCount.Load(),
 		"policy":                policyString(l.policy),
@@ -100,13 +114,13 @@ func (l *RateLimiter) GetStats() map[string]any {
 	}
 
 	if l.bucket != nil {
-		stats["tokens"] = l.bucket.Tokens()
+		stats["available_tokens"] = l.bucket.Tokens()
 	}
 
 	return stats
 }
 
-// policyString returns the string representation of the policy.
+// policyString returns the string representation of a rate limit policy
 func policyString(p config.RateLimitPolicy) string {
 	switch p {
 	case config.PolicyDrop:

internal/format/adapter.go

@@ -0,0 +1,160 @@
+package format
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log/formatter"
+	"github.com/lixenwraith/log/sanitizer"
+)
+
+const (
+	DefaultFormatType = "raw"
+)
+
+// FormatterAdapter wraps log/formatter for logwisp compatibility
+type FormatterAdapter struct {
+	formatter *formatter.Formatter
+	format    string
+	flags     int64
+	mu        sync.Mutex // formatter reuses internal buffer, not goroutine-safe
+}
+
+// NewFormatterAdapter creates adapter from config
+func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
+	// Validate
+	if cfg.Type != "" {
+		validateType := lconfig.OneOf("json", "txt", "text", "raw")
+		if err := validateType(cfg.Type); err != nil {
+			return nil, fmt.Errorf("type: %w", err)
+		}
+	}
+
+	if cfg.SanitizerPolicy != "" {
+		validatePolicy := lconfig.OneOf("raw", "json", "txt", "shell")
+		if err := validatePolicy(cfg.SanitizerPolicy); err != nil {
+			return nil, fmt.Errorf("sanitizer_policy: %w", err)
+		}
+	}
+
+	// Defaults
+	if cfg.Type == "" {
+		cfg.Type = DefaultFormatType
+	}
+
+	// Create sanitizer based on policy
+	var s *sanitizer.Sanitizer
+	if cfg.SanitizerPolicy != "" {
+		s = sanitizer.New().Policy(sanitizer.PolicyPreset(cfg.SanitizerPolicy))
+	} else {
+		// Default sanitizer policy based on format type
+		switch cfg.Type {
+		case "json":
+			s = sanitizer.New().Policy(sanitizer.PolicyJSON)
+		case "txt", "text":
+			s = sanitizer.New().Policy(sanitizer.PolicyTxt)
+		default:
+			s = sanitizer.New().Policy(sanitizer.PolicyRaw)
+		}
+	}
+
+	// Create formatter with sanitizer
+	f := formatter.New(s).Type(cfg.Type)
+
+	if cfg.TimestampFormat != "" {
+		f.TimestampFormat(cfg.TimestampFormat)
+	}
+
+	// Build flags from config
+	flags := cfg.Flags
+	if flags == 0 {
+		if cfg.Type == "raw" {
+			flags = formatter.FlagRaw
+		} else {
+			flags = formatter.FlagDefault
+		}
+	}
+
+	return &FormatterAdapter{
+		formatter: f,
+		format:    cfg.Type,
+		flags:     flags,
+	}, nil
+}
+
+// Format implements Formatter interface
+func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
+	return a.serialize(entry, a.flags), nil
+}
+
+// FormatWithFlags allows custom flags for specific formatting needs
+func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
+	return a.serialize(entry, customFlags), nil
+}
+
+// serialize renders an entry under the given flags. The returned slice is a
+// copy: the underlying formatter reuses one buffer and sinks retain payloads.
+func (a *FormatterAdapter) serialize(entry core.LogEntry, flags int64) []byte {
+	args, flags := formatArgs(entry, flags)
+
+	a.mu.Lock()
+	out := bytes.Clone(a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args))
+	a.mu.Unlock()
+	return out
+}
+
+// formatArgs pairs the entry with its flags. FlagRaw keeps the fields JSON
+// verbatim beside the message rather than silently overriding the caller's
+// choice of passthrough; every other mode renders it as a JSON object.
+func formatArgs(entry core.LogEntry, flags int64) ([]any, int64) {
+	if len(entry.Fields) == 0 {
+		return []any{entry.Message}, flags
+	}
+	if flags&formatter.FlagRaw != 0 {
+		if entry.Message == "" {
+			return []any{[]byte(entry.Fields)}, flags
+		}
+		return []any{entry.Message, []byte(entry.Fields)}, flags
+	}
+
+	var fields map[string]any
+	if err := json.Unmarshal(entry.Fields, &fields); err != nil || len(fields) == 0 {
+		return []any{entry.Message}, flags
+	}
+	return []any{entry.Message, fields}, flags | formatter.FlagStructuredJSON
+}
+
+// Name returns formatter type
+func (a *FormatterAdapter) Name() string {
+	return a.format
+}
+
+// mapLevel maps string level to int64
+func mapLevel(level string) int64 {
+	switch level {
+	case "DEBUG", "debug":
+		return -4
+	case "INFO", "info":
+		return 0
+	case "WARN", "warn", "WARNING", "warning":
+		return 4
+	case "ERROR", "error":
+		return 8
+	default:
+		return 0
+	}
+}
+
+// sourceLabel prefixes origin node onto source (syslog HOSTNAME + TAG convention)
+func sourceLabel(entry core.LogEntry) string {
+	if entry.Node == "" {
+		return entry.Source
+	}
+	return entry.Node + "/" + entry.Source
+}

internal/format/format.go

@@ -0,0 +1,28 @@
+package format
+
+import (
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+)
+
+// Formatter defines the interface for transforming a LogEntry into a byte slice
+type Formatter interface {
+	// Format takes a LogEntry and returns the formatted log as a byte slice
+	Format(entry core.LogEntry) ([]byte, error)
+
+	// Name returns the formatter's type name (e.g., "json", "raw")
+	Name() string
+}
+
+// NewFormatter creates a Formatter using formatter/sanitizer packages
+func NewFormatter(cfg *config.FormatConfig) (Formatter, error) {
+	if cfg == nil {
+		cfg = &config.FormatConfig{
+			Type:            DefaultFormatType,
+			Flags:           0,
+			SanitizerPolicy: "raw",
+		}
+	}
+
+	return NewFormatterAdapter(cfg)
+}

internal/pipeline/pipeline.go

@@ -0,0 +1,470 @@
+package pipeline
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/flow"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// Pipeline manages the flow of data from sources, through filters, to sinks
+type Pipeline struct {
+	Config *config.PipelineConfig
+
+	// Components
+	Registry *Registry
+	Sources  map[string]source.Source // Track instances by ID
+	Sinks    map[string]sink.Sink
+	Sessions *session.Manager
+
+	// Pipeline flow
+	Flow   *flow.Flow
+	Stats  *PipelineStats
+	logger *log.Logger
+
+	// Runtime
+	ctx     context.Context
+	cancel  context.CancelFunc
+	wg      sync.WaitGroup
+	running atomic.Bool
+}
+
+// PipelineStats contains runtime statistics for a pipeline
+type PipelineStats struct {
+	StartTime                 time.Time
+	TotalEntriesDroppedBySink atomic.Uint64
+	SourceStats               []source.SourceStats
+	SinkStats                 []sink.SinkStats
+	FlowStats                 map[string]any
+}
+
+// NewPipeline creates a new pipeline with registry support
+func NewPipeline(
+	cfg *config.PipelineConfig,
+	logger *log.Logger,
+) (*Pipeline, error) {
+	// Create pipeline context
+	pipelineCtx, pipelineCancel := context.WithCancel(context.Background())
+
+	// Create session manager with default timeout
+	sessionManager := session.NewManager(core.SessionDefaultMaxIdleTime)
+
+	// Create pipeline instance with registry
+	pipeline := &Pipeline{
+		Config:   cfg,
+		Registry: NewRegistry(cfg.Name, logger),
+		Sessions: sessionManager,
+		Sources:  make(map[string]source.Source),
+		Sinks:    make(map[string]sink.Sink),
+		Stats:    &PipelineStats{},
+		logger:   logger,
+		ctx:      pipelineCtx,
+		cancel:   pipelineCancel,
+	}
+
+	// Create flow processor
+	flowProcessor, err := flow.NewFlow(cfg.Flow, logger)
+	if err != nil {
+		// If flow fails, stop session manager
+		sessionManager.Stop()
+		return nil, fmt.Errorf("failed to create flow processor: %w", err)
+	}
+	pipeline.Flow = flowProcessor
+
+	// Initialize sources and sinks
+	if err := pipeline.initializeComponents(); err != nil {
+		pipelineCancel()
+		return nil, err
+	}
+
+	return pipeline, nil
+}
+
+func (p *Pipeline) initializeComponents() error {
+	// Create sources based on plugin config if available
+	if len(p.Config.PluginSources) > 0 {
+		for _, srcCfg := range p.Config.PluginSources {
+			// Create session proxy for this source instance
+			sessionProxy := session.NewProxy(p.Sessions, srcCfg.ID)
+
+			src, err := p.Registry.CreateSource(
+				srcCfg.ID,
+				srcCfg.Type,
+				srcCfg.Config,
+				p.logger,
+				sessionProxy,
+			)
+			if err != nil {
+				return fmt.Errorf("failed to create source %s: %w", srcCfg.ID, err)
+			}
+
+			// Check and inject capabilities using core interfaces
+			if err := p.initSourceCapabilities(src, srcCfg); err != nil {
+				return fmt.Errorf("failed to initiate capabilities for source %s: %w", srcCfg.ID, err)
+			}
+
+			p.Sources[srcCfg.ID] = src
+		}
+	} else {
+		return fmt.Errorf("no plugin sources defined")
+	}
+
+	// Create sinks based on plugin config if available
+	if len(p.Config.PluginSinks) > 0 {
+		for _, sinkCfg := range p.Config.PluginSinks {
+			// Create session proxy for this sink instance
+			sessionProxy := session.NewProxy(p.Sessions, sinkCfg.ID)
+
+			snk, err := p.Registry.CreateSink(
+				sinkCfg.ID,
+				sinkCfg.Type,
+				sinkCfg.Config,
+				p.logger,
+				sessionProxy,
+			)
+			if err != nil {
+				return fmt.Errorf("failed to create sink %s: %w", sinkCfg.ID, err)
+			}
+
+			// Check and inject capabilities using core interfaces
+			if err := p.initSinkCapabilities(snk, sinkCfg); err != nil {
+				return fmt.Errorf("failed to initiate capabilities for sink %s: %w", sinkCfg.ID, err)
+			}
+
+			p.Sinks[sinkCfg.ID] = snk
+		}
+	} else {
+		return fmt.Errorf("no plugin sinks defined")
+	}
+
+	return nil
+}
+
+// initSourceCapabilities checks and injects optional capabilities
+func (p *Pipeline) initSourceCapabilities(s source.Source, cfg config.PluginSourceConfig) error {
+	// Initiate and activate source capabilities
+	var hasTLS, hasAuth bool
+	for _, c := range s.Capabilities() {
+		switch c {
+		// Network capabilities
+		case core.CapNetLimit:
+			continue // No-op for now, placeholder
+		case core.CapTLS:
+			hasTLS = true
+		case core.CapAuth:
+			hasAuth = true
+
+		// Session capabilities
+		case core.CapSessionAware:
+		case core.CapMultiSession:
+			continue // TODO
+
+		default:
+			return fmt.Errorf("unknown capability type: %s", c)
+		}
+	}
+
+	if err := checkAuthCapability(hasTLS, hasAuth); err != nil {
+		return fmt.Errorf("source %s: %w", cfg.ID, err)
+	}
+
+	return nil
+}
+
+// checkAuthCapability rejects a plugin that decides on peer identity without a
+// transport that verifies one - the decision would rest on an unauthenticated
+// claim
+func checkAuthCapability(hasTLS, hasAuth bool) error {
+	if hasAuth && !hasTLS {
+		return fmt.Errorf("capability %q requires %q", core.CapAuth, core.CapTLS)
+	}
+	return nil
+}
+
+// initSinkCapabilities checks and injects optional capabilities
+func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig) error {
+	// Initiate and activate sink capabilities
+	var hasTLS, hasAuth bool
+	for _, c := range s.Capabilities() {
+		switch c {
+		// Network capabilities
+		case core.CapNetLimit:
+			continue // No-op for now, placeholder
+		case core.CapTLS:
+			hasTLS = true
+		case core.CapAuth:
+			hasAuth = true
+
+		// Session capabilities
+		case core.CapSessionAware:
+		case core.CapMultiSession:
+			continue // TODO
+
+		default:
+			return fmt.Errorf("unknown capability type: %s", c)
+		}
+	}
+
+	if err := checkAuthCapability(hasTLS, hasAuth); err != nil {
+		return fmt.Errorf("sink %s: %w", cfg.ID, err)
+	}
+
+	return nil
+}
+
+// run is the central processing loop that connects sources, flow, and sinks
+func (p *Pipeline) run() {
+	defer p.wg.Done()
+	defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name)
+
+	var componentWg sync.WaitGroup
+	// Start a goroutine for each source to fan-in data
+	for _, src := range p.Sources {
+		componentWg.Add(1)
+		go func(s source.Source) {
+			defer componentWg.Done()
+			ch := s.Subscribe()
+			// Range allows in-flight data to drain cleanly once Source.Stop() closes the channel
+			for entry := range ch {
+				if event, passed := p.Flow.Process(entry); passed {
+					// Use non-blocking dispatcher
+					p.dispatch(event)
+				}
+			}
+		}(src)
+	}
+
+	var hbWg sync.WaitGroup
+	// Start heartbeat generator if enabled
+	if heartbeatCh := p.Flow.StartHeartbeat(p.ctx); heartbeatCh != nil {
+		hbWg.Add(1)
+		go func() {
+			defer hbWg.Done()
+			for {
+				select {
+				case event, ok := <-heartbeatCh:
+					if !ok {
+						return
+					}
+					// Use non-blocking dispatcher
+					p.dispatch(event)
+				case <-p.ctx.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	componentWg.Wait()
+
+	// Terminate internal contexts (heartbeat) once flow is complete
+	p.cancel()
+	hbWg.Wait()
+}
+
+// dispatch performs a non-blocking send to all sinks.
+// A full/stalled sink must never block the run loop or starve sibling sinks.
+func (p *Pipeline) dispatch(event core.TransportEvent) {
+	for _, snk := range p.Sinks {
+		select {
+		case snk.Input() <- event:
+		default:
+			// Buffer full - drop to avoid deadlocking the pipeline
+			p.Stats.TotalEntriesDroppedBySink.Add(1)
+		}
+	}
+}
+
+// Start starts the pipeline operation and all its components including flow, sources, and sinks
+func (p *Pipeline) Start() error {
+	if !p.running.CompareAndSwap(false, true) {
+		return fmt.Errorf("pipeline %s is already running", p.Config.Name)
+	}
+
+	p.logger.Info("msg", "Starting pipeline", "pipeline", p.Config.Name)
+	p.ctx, p.cancel = context.WithCancel(context.Background())
+
+	// Start all sinks
+	for id, s := range p.Sinks {
+		if err := s.Start(p.ctx); err != nil {
+			return fmt.Errorf("failed to start sink %s: %w", id, err)
+		}
+	}
+
+	// Start all sources
+	for id, src := range p.Sources {
+		if err := src.Start(); err != nil {
+			return fmt.Errorf("failed to start source %s: %w", id, err)
+		}
+	}
+
+	// Start the central processing loop
+	p.Stats.StartTime = time.Now()
+	p.wg.Add(1)
+	go p.run()
+
+	return nil
+}
+
+// Stop stops the pipeline operation and all its components including flow, sources, and sinks
+func (p *Pipeline) Stop() error {
+	if !p.running.CompareAndSwap(true, false) {
+		return fmt.Errorf("pipeline %s is not running", p.Config.Name)
+	}
+
+	p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name)
+
+	// 1. Stop all sources concurrently to halt new data ingress and close their channels
+	var sourceWg sync.WaitGroup
+	for _, src := range p.Sources {
+		sourceWg.Add(1)
+		go func(s source.Source) {
+			defer sourceWg.Done()
+			s.Stop()
+		}(src)
+	}
+	sourceWg.Wait()
+
+	// 2. Wait for the run loop to finish processing and sending all in-flight data
+	// run() inherently calls p.cancel() when the source channels are empty
+	p.wg.Wait()
+
+	// 3. Stop all sinks concurrently now that no new data will be sent
+	var sinkWg sync.WaitGroup
+	for _, s := range p.Sinks {
+		sinkWg.Add(1)
+		go func(snk sink.Sink) {
+			defer sinkWg.Done()
+			snk.Stop()
+		}(s)
+	}
+	sinkWg.Wait()
+
+	p.logger.Info("msg", "Pipeline stopped", "pipeline", p.Config.Name)
+	return nil
+}
+
+// Shutdown gracefully stops the pipeline and all its components, deinitializing them for app shutdown or complete pipeline removal by service
+func (p *Pipeline) Shutdown() {
+	p.logger.Info("msg", "Shutting down pipeline",
+		"component", "pipeline",
+		"pipeline", p.Config.Name)
+
+	// Ensure the pipeline is stopped before shutting down
+	if p.running.Load() {
+		if err := p.Stop(); err != nil {
+			p.logger.Error("msg", "Error stopping pipeline during shutdown", "error", err)
+		}
+	}
+
+	// Stop long-running components
+	if p.Sessions != nil {
+		p.Sessions.Stop()
+	}
+
+	p.logger.Info("msg", "Pipeline shutdown complete",
+		"component", "pipeline",
+		"pipeline", p.Config.Name)
+}
+
+// GetStats returns a map of pipeline statistics
+func (p *Pipeline) GetStats() map[string]any {
+	// Recovery to handle concurrent access during shutdown
+	// When service is shutting down, sources/sinks might be nil or partially stopped
+	defer func() {
+		if r := recover(); r != nil {
+			p.logger.Error("msg", "Panic getting pipeline stats",
+				"pipeline", p.Config.Name,
+				"panic", r)
+		}
+	}()
+
+	// 1. Live collect source stats
+	sources := make([]map[string]any, 0, len(p.Sources))
+	for _, src := range p.Sources {
+		if src == nil {
+			continue
+		}
+		s := src.GetStats()
+		sources = append(sources, map[string]any{
+			"id":              s.ID,
+			"type":            s.Type,
+			"total_entries":   s.TotalEntries,
+			"dropped_entries": s.DroppedEntries,
+			"start_time":      s.StartTime,
+			"last_entry_time": s.LastEntryTime,
+			"details":         s.Details,
+		})
+	}
+
+	// 2. Live collect sink stats
+	sinks := make([]map[string]any, 0, len(p.Sinks))
+	for _, snk := range p.Sinks {
+		if snk == nil {
+			continue
+		}
+		s := snk.GetStats()
+		sinks = append(sinks, map[string]any{
+			"id":                 s.ID,
+			"type":               s.Type,
+			"total_processed":    s.TotalProcessed,
+			"active_connections": s.ActiveConnections,
+			"start_time":         s.StartTime,
+			"last_processed":     s.LastProcessed,
+			"details":            s.Details,
+		})
+	}
+
+	// 3. Collect flow stats and calculate filtered total
+	var flowStats map[string]any
+	var totalFiltered uint64
+	var totalProcessed uint64
+
+	if p.Flow != nil {
+		flowStats = p.Flow.GetStats()
+
+		// Map the top-level processed counter directly from Flow's source of truth
+		if tp, ok := flowStats["total_processed"].(uint64); ok {
+			totalProcessed = tp
+		}
+
+		// Calculate total dropped specifically by the filter chain
+		if filters, ok := flowStats["filters"].(map[string]any); ok {
+			if totalPassed, ok := filters["total_passed"].(uint64); ok {
+				if tProc, ok := filters["total_processed"].(uint64); ok {
+					totalFiltered = tProc - totalPassed
+				}
+			}
+		}
+	}
+
+	// 4. Calculate Uptime
+	var uptime int
+	if p.running.Load() && !p.Stats.StartTime.IsZero() {
+		uptime = int(time.Since(p.Stats.StartTime).Seconds())
+	}
+
+	return map[string]any{
+		"name":                  p.Config.Name,
+		"running":               p.running.Load(),
+		"uptime_seconds":        uptime,
+		"total_processed":       totalProcessed,
+		"total_filtered":        totalFiltered,
+		"total_dropped_by_sink": p.Stats.TotalEntriesDroppedBySink.Load(),
+		"source_count":          len(p.Sources),
+		"sources":               sources,
+		"sink_count":            len(p.Sinks),
+		"sinks":                 sinks,
+		"flow":                  flowStats,
+	}
+}

internal/pipeline/registry.go

@@ -0,0 +1,222 @@
+package pipeline
+
+import (
+	"fmt"
+	"sync"
+
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// SourceFactory creates source instances with required dependencies
+type SourceFactory func(
+	id string,
+	config map[string]any,
+	logger *log.Logger,
+	sessions *session.Proxy,
+) (source.Source, error)
+
+// SinkFactory creates sink instances with required dependencies
+type SinkFactory func(
+	id string,
+	config map[string]any,
+	logger *log.Logger,
+	sessions *session.Proxy,
+) (sink.Sink, error)
+
+// Registry manages plugin instances for a single pipeline
+type Registry struct {
+	pipelineName string
+
+	// Instance tracking
+	sourceInstances map[string]source.Source
+	sinkInstances   map[string]sink.Sink
+	// Type count tracking (for single instance enforcement)
+	sourceTypeCounts map[string]int
+	sinkTypeCounts   map[string]int
+
+	mu     sync.RWMutex
+	logger *log.Logger
+}
+
+// NewRegistry creates a new registry for a pipeline
+func NewRegistry(pipelineName string, logger *log.Logger) *Registry {
+	return &Registry{
+		pipelineName:     pipelineName,
+		sourceInstances:  make(map[string]source.Source),
+		sinkInstances:    make(map[string]sink.Sink),
+		sourceTypeCounts: make(map[string]int),
+		sinkTypeCounts:   make(map[string]int),
+		logger:           logger,
+	}
+}
+
+// CreateSource creates and tracks a source instance
+func (r *Registry) CreateSource(
+	id string,
+	pluginType string,
+	config map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Check for duplicate instance ID
+	if _, exists := r.sourceInstances[id]; exists {
+		return nil, fmt.Errorf("source instance with ID %s already exists", id)
+	}
+
+	// Check single instance constraint
+	if meta, ok := plugin.GetSourceMetadata(pluginType); ok {
+		if meta.MaxInstances == 1 && r.sourceTypeCounts[pluginType] >= 1 {
+			return nil, fmt.Errorf("source type %s only allows single instance", pluginType)
+		}
+	}
+
+	// Get source constructor
+	constructor, ok := plugin.GetSource(pluginType)
+	if !ok {
+		return nil, fmt.Errorf("unknown source type: %s", pluginType)
+	}
+
+	// Create instance
+	src, err := constructor(id, config, logger, proxy)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create source %s: %w", id, err)
+	}
+
+	// Track instance
+	r.sourceInstances[id] = src
+	r.sourceTypeCounts[pluginType]++
+
+	r.logger.Info("msg", "Created source instance",
+		"pipeline", r.pipelineName,
+		"id", id,
+		"type", pluginType)
+
+	return src, nil
+}
+
+// CreateSink creates and tracks a sink instance
+func (r *Registry) CreateSink(
+	id string,
+	pluginType string,
+	config map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Check for duplicate instance ID
+	if _, exists := r.sinkInstances[id]; exists {
+		return nil, fmt.Errorf("sink instance with ID %s already exists", id)
+	}
+
+	// Check single instance constraint
+	if meta, ok := plugin.GetSinkMetadata(pluginType); ok {
+		if meta.MaxInstances == 1 && r.sinkTypeCounts[pluginType] >= 1 {
+			return nil, fmt.Errorf("sink type %s only allows single instance", pluginType)
+		}
+	}
+
+	// Get sink constructor
+	constructor, ok := plugin.GetSink(pluginType)
+	if !ok {
+		return nil, fmt.Errorf("unknown sink type: %s", pluginType)
+	}
+
+	// Create instance
+	snk, err := constructor(id, config, logger, proxy)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create sink %s: %w", id, err)
+	}
+
+	// Track instance
+	r.sinkInstances[id] = snk
+	r.sinkTypeCounts[pluginType]++
+
+	r.logger.Info("msg", "Created sink instance",
+		"pipeline", r.pipelineName,
+		"id", id,
+		"type", pluginType)
+
+	return snk, nil
+}
+
+// GetSourceInstance retrieves a source instance by ID
+func (r *Registry) GetSourceInstance(id string) (source.Source, bool) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	src, exists := r.sourceInstances[id]
+	return src, exists
+}
+
+// GetSinkInstance retrieves a sink instance by ID
+func (r *Registry) GetSinkInstance(id string) (sink.Sink, bool) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	snk, exists := r.sinkInstances[id]
+	return snk, exists
+}
+
+// GetAllSources returns all source instances
+func (r *Registry) GetAllSources() map[string]source.Source {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	sources := make(map[string]source.Source, len(r.sourceInstances))
+	for k, v := range r.sourceInstances {
+		sources[k] = v
+	}
+	return sources
+}
+
+// GetAllSinks returns all sink instances
+func (r *Registry) GetAllSinks() map[string]sink.Sink {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	sinks := make(map[string]sink.Sink, len(r.sinkInstances))
+	for k, v := range r.sinkInstances {
+		sinks[k] = v
+	}
+	return sinks
+}
+
+// RemoveSource removes a source instance
+func (r *Registry) RemoveSource(id string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Decrement type count
+	if src, exists := r.sourceInstances[id]; exists {
+		stats := src.GetStats()
+		if pluginType, ok := stats.Details["type"].(string); ok {
+			r.sourceTypeCounts[pluginType]--
+		}
+	}
+
+	delete(r.sourceInstances, id)
+}
+
+// RemoveSink removes a sink instance
+func (r *Registry) RemoveSink(id string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// Decrement type count
+	if snk, exists := r.sinkInstances[id]; exists {
+		stats := snk.GetStats()
+		if pluginType, ok := stats.Details["type"].(string); ok {
+			r.sinkTypeCounts[pluginType]--
+		}
+	}
+
+	delete(r.sinkInstances, id)
+}

internal/plugin/factory.go

@@ -0,0 +1,189 @@
+package plugin
+
+import (
+	"fmt"
+	"sync"
+
+	"logwisp/internal/core"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// SourceFactory creates source instances
+type SourceFactory func(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	sessions *session.Proxy,
+) (source.Source, error)
+
+// SinkFactory creates sink instances
+type SinkFactory func(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	sessions *session.Proxy,
+) (sink.Sink, error)
+
+// PluginMetadata stores metadata about a plugin type
+type PluginMetadata struct {
+	Capabilities []core.Capability
+	MaxInstances int // 0 = unlimited, 1 = single instance only
+}
+
+// registry encapsulates all plugin factories with lazy initialization
+type registry struct {
+	sourceFactories map[string]SourceFactory
+	sinkFactories   map[string]SinkFactory
+	sourceMetadata  map[string]*PluginMetadata
+	sinkMetadata    map[string]*PluginMetadata
+	mu              sync.RWMutex
+}
+
+var (
+	globalRegistry *registry
+	once           sync.Once
+)
+
+// getRegistry returns the singleton registry, initializing on first access
+func getRegistry() *registry {
+	once.Do(func() {
+		globalRegistry = &registry{
+			sourceFactories: make(map[string]SourceFactory),
+			sinkFactories:   make(map[string]SinkFactory),
+			sourceMetadata:  make(map[string]*PluginMetadata),
+			sinkMetadata:    make(map[string]*PluginMetadata),
+		}
+	})
+	return globalRegistry
+}
+
+// RegisterSource registers a source factory function
+func RegisterSource(name string, constructor SourceFactory) error {
+	r := getRegistry()
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if _, exists := r.sourceFactories[name]; exists {
+		return fmt.Errorf("source type %s already registered", name)
+	}
+	r.sourceFactories[name] = constructor
+
+	// Set default metadata
+	r.sourceMetadata[name] = &PluginMetadata{
+		MaxInstances: 0, // Unlimited by default
+	}
+
+	return nil
+}
+
+// RegisterSink registers a sink factory function
+func RegisterSink(name string, constructor SinkFactory) error {
+	r := getRegistry()
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if _, exists := r.sinkFactories[name]; exists {
+		return fmt.Errorf("sink type %s already registered", name)
+	}
+	r.sinkFactories[name] = constructor
+
+	// Set default metadata
+	r.sinkMetadata[name] = &PluginMetadata{
+		MaxInstances: 0, // Unlimited by default
+	}
+
+	return nil
+}
+
+// SetSourceMetadata sets metadata for a source type (call after RegisterSource)
+func SetSourceMetadata(name string, metadata *PluginMetadata) error {
+	r := getRegistry()
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if _, exists := r.sourceFactories[name]; !exists {
+		return fmt.Errorf("source type %s not registered", name)
+	}
+	r.sourceMetadata[name] = metadata
+
+	return nil
+}
+
+// SetSinkMetadata sets metadata for a sink type (call after RegisterSink)
+func SetSinkMetadata(name string, metadata *PluginMetadata) error {
+	r := getRegistry()
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if _, exists := r.sinkFactories[name]; !exists {
+		return fmt.Errorf("sink type %s not registered", name)
+	}
+	r.sinkMetadata[name] = metadata
+	return nil
+}
+
+// GetSource retrieves a source factory function
+func GetSource(name string) (SourceFactory, bool) {
+	r := getRegistry()
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	constructor, exists := r.sourceFactories[name]
+	return constructor, exists
+}
+
+// GetSink retrieves a sink factory function
+func GetSink(name string) (SinkFactory, bool) {
+	r := getRegistry()
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	constructor, exists := r.sinkFactories[name]
+	return constructor, exists
+}
+
+// GetSourceMetadata retrieves metadata for a source type
+func GetSourceMetadata(name string) (*PluginMetadata, bool) {
+	r := getRegistry()
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	meta, exists := r.sourceMetadata[name]
+	return meta, exists
+}
+
+// GetSinkMetadata retrieves metadata for a sink type
+func GetSinkMetadata(name string) (*PluginMetadata, bool) {
+	r := getRegistry()
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	meta, exists := r.sinkMetadata[name]
+	return meta, exists
+}
+
+// ListSources returns all registered source types
+func ListSources() []string {
+	r := getRegistry()
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	types := make([]string, 0, len(r.sourceFactories))
+	for t := range r.sourceFactories {
+		types = append(types, t)
+	}
+	return types
+}
+
+// ListSinks returns all registered sink types
+func ListSinks() []string {
+	r := getRegistry()
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	types := make([]string, 0, len(r.sinkFactories))
+	for t := range r.sinkFactories {
+		types = append(types, t)
+	}
+	return types
+}

internal/sanitize/sanitize.go

@@ -0,0 +1,70 @@
+package sanitize
+
+import (
+	"encoding/hex"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+)
+
+// String sanitizes a string by replacing non-printable characters with hex encoding
+// Non-printable characters are encoded as <hex> (e.g., newline becomes <0a>)
+func String(data string) string {
+	// Fast path: check if sanitization is needed
+	needsSanitization := false
+	for _, r := range data {
+		if !strconv.IsPrint(r) {
+			needsSanitization = true
+			break
+		}
+	}
+
+	if !needsSanitization {
+		return data
+	}
+
+	// Pre-allocate builder for efficiency
+	var builder strings.Builder
+	builder.Grow(len(data))
+
+	for _, r := range data {
+		if strconv.IsPrint(r) {
+			builder.WriteRune(r)
+		} else {
+			// Encode non-printable rune as <hex>
+			var runeBytes [utf8.UTFMax]byte
+			n := utf8.EncodeRune(runeBytes[:], r)
+			builder.WriteByte('<')
+			builder.WriteString(hex.EncodeToString(runeBytes[:n]))
+			builder.WriteByte('>')
+		}
+	}
+
+	return builder.String()
+}
+
+// Bytes sanitizes a byte slice by converting to string and sanitizing
+func Bytes(data []byte) []byte {
+	return []byte(String(string(data)))
+}
+
+// Rune sanitizes a single rune, returning its string representation
+func Rune(r rune) string {
+	if strconv.IsPrint(r) {
+		return string(r)
+	}
+
+	var runeBytes [utf8.UTFMax]byte
+	n := utf8.EncodeRune(runeBytes[:], r)
+	return "<" + hex.EncodeToString(runeBytes[:n]) + ">"
+}
+
+// IsSafe checks if a string contains only printable characters
+func IsSafe(data string) bool {
+	for _, r := range data {
+		if !strconv.IsPrint(r) {
+			return false
+		}
+	}
+	return true
+}

internal/service/service.go

@@ -0,0 +1,212 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	"logwisp/internal/config"
+	"logwisp/internal/pipeline"
+
+	"github.com/lixenwraith/log"
+)
+
+// Service manages a collection of log processing pipelines
+type Service struct {
+	pipelines map[string]*pipeline.Pipeline
+	mu        sync.RWMutex
+	ctx       context.Context
+	cancel    context.CancelFunc
+	wg        sync.WaitGroup
+	logger    *log.Logger
+}
+
+// NewService creates a new, empty service
+func NewService(ctx context.Context, cfg *config.Config, logger *log.Logger) (*Service, error) {
+	serviceCtx, cancel := context.WithCancel(ctx)
+	svc := &Service{
+		pipelines: make(map[string]*pipeline.Pipeline),
+		ctx:       serviceCtx,
+		cancel:    cancel,
+		logger:    logger,
+	}
+
+	var errs error
+	// Initialize pipelines
+	for _, pipelineCfg := range cfg.Pipelines {
+		pipelineName := pipelineCfg.Name
+		logger.Info("msg", "Initializing pipeline", "pipeline", pipelineName)
+
+		// Create the pipeline
+		if pl, err := pipeline.NewPipeline(&pipelineCfg, logger); err != nil {
+			logger.Error("msg", "Failed to create pipeline",
+				"pipeline", pipelineCfg.Name,
+				"error", err)
+			errs = errors.Join(errs, fmt.Errorf("failed to initialize pipeline %s: %w", pipelineName, err))
+		} else {
+			svc.pipelines[pipelineName] = pl
+		}
+	}
+
+	logger.Info("msg", "Service initialization completed", "pipelines", len(svc.pipelines))
+
+	return svc, errs
+}
+
+// Start starts all or specific pipelines
+func (svc *Service) Start(names ...string) error {
+	svc.mu.RLock()
+	defer svc.mu.RUnlock()
+
+	var errs error
+	// If no names are provided, start all pipelines
+	if len(names) == 0 {
+		svc.logger.Info("msg", "Starting all pipelines")
+		for name, p := range svc.pipelines {
+			if err := p.Start(); err != nil {
+				errs = errors.Join(errs, fmt.Errorf("failed to start pipeline %s: %w", name, err))
+			}
+		}
+	} else {
+		// Start only the specified pipelines
+		svc.logger.Info("msg", "Starting specified pipelines", "pipelines", names)
+		for _, name := range names {
+			if p, exists := svc.pipelines[name]; exists {
+				if err := p.Start(); err != nil {
+					errs = errors.Join(errs, fmt.Errorf("failed to start pipeline %s: %w", name, err))
+				}
+			} else {
+				errs = errors.Join(errs, fmt.Errorf("pipeline %s not found", name))
+			}
+		}
+	}
+
+	svc.logger.Debug("msg", "Finished starting pipeline(s)", "pipelines", names)
+
+	return errs
+}
+
+// Stop stops all or specific pipeline
+func (svc *Service) Stop(names ...string) error {
+	svc.mu.RLock()
+	defer svc.mu.RUnlock()
+
+	var errs error
+
+	// If no names are provided, stop all pipelines
+	if len(names) == 0 {
+		svc.logger.Info("msg", "Stopping all pipelines")
+		for name, p := range svc.pipelines {
+			if err := p.Stop(); err != nil {
+				errs = errors.Join(errs, fmt.Errorf("failed to stop pipeline %s: %w", name, err))
+			}
+		}
+	} else {
+		// Stop only the specified pipelines
+		svc.logger.Info("msg", "Stopping specified pipelines", "pipelines", names)
+		for _, name := range names {
+			if p, exists := svc.pipelines[name]; exists {
+				if err := p.Stop(); err != nil {
+					errs = errors.Join(errs, fmt.Errorf("failed to stop pipeline %s: %w", name, err))
+				}
+			} else {
+				errs = errors.Join(errs, fmt.Errorf("pipeline %s not found", name))
+			}
+		}
+	}
+
+	svc.logger.Debug("msg", "Finished stopping pipeline(s)", "pipelines", names)
+
+	return errs
+}
+
+// GetPipeline returns a pipeline by its name
+func (svc *Service) GetPipeline(name string) (*pipeline.Pipeline, error) {
+	svc.mu.RLock()
+	defer svc.mu.RUnlock()
+
+	pipeline, exists := svc.pipelines[name]
+	if !exists {
+		return nil, fmt.Errorf("pipeline '%s' not found", name)
+	}
+	return pipeline, nil
+}
+
+// ListPipelines returns the names of all currently managed pipelines
+func (svc *Service) ListPipelines() []string {
+	svc.mu.RLock()
+	defer svc.mu.RUnlock()
+
+	names := make([]string, 0, len(svc.pipelines))
+	for name := range svc.pipelines {
+		names = append(names, name)
+	}
+	return names
+}
+
+// RemovePipeline stops and removes a pipeline from the service
+func (svc *Service) RemovePipeline(name string) error {
+	svc.mu.Lock()
+	defer svc.mu.Unlock()
+
+	pl, exists := svc.pipelines[name]
+	if !exists {
+		err := fmt.Errorf("pipeline '%s' not found", name)
+		svc.logger.Warn("msg", "Cannot remove non-existent pipeline",
+			"component", "service",
+			"pipeline", name,
+			"error", err)
+		return err
+	}
+
+	svc.logger.Info("msg", "Removing pipeline", "pipeline", name)
+	pl.Shutdown()
+	delete(svc.pipelines, name)
+	return nil
+}
+
+// Shutdown gracefully stops all pipelines managed by the service
+func (svc *Service) Shutdown() {
+	svc.logger.Info("msg", "Service shutdown initiated")
+
+	svc.mu.Lock()
+	pipelines := make([]*pipeline.Pipeline, 0, len(svc.pipelines))
+	for _, pl := range svc.pipelines {
+		pipelines = append(pipelines, pl)
+	}
+	svc.mu.Unlock()
+
+	// Stop all pipelines concurrently
+	var wg sync.WaitGroup
+	for _, pl := range pipelines {
+		wg.Add(1)
+		go func(p *pipeline.Pipeline) {
+			defer wg.Done()
+			p.Shutdown()
+		}(pl)
+	}
+	wg.Wait()
+
+	svc.cancel()
+	svc.wg.Wait()
+
+	svc.logger.Info("msg", "Service shutdown complete")
+}
+
+// GetGlobalStats returns statistics for all pipelines
+func (svc *Service) GetGlobalStats() map[string]any {
+	svc.mu.RLock()
+	defer svc.mu.RUnlock()
+
+	stats := map[string]any{
+		"pipelines":       make(map[string]any),
+		"total_pipelines": len(svc.pipelines),
+	}
+
+	for name, pl := range svc.pipelines {
+		stats["pipelines"].(map[string]any)[name] = pl.GetStats()
+	}
+
+	return stats
+}

internal/session/proxy.go

@@ -0,0 +1,82 @@
+package session
+
+import (
+	"sync"
+)
+
+// Proxy provides filtered access to session management for a specific plugin instance
+type Proxy struct {
+	manager    *Manager
+	instanceID string
+	mu         sync.RWMutex
+}
+
+// NewProxy creates a session proxy for a specific plugin instance
+func NewProxy(manager *Manager, instanceID string) *Proxy {
+	return &Proxy{
+		manager:    manager,
+		instanceID: instanceID,
+	}
+}
+
+// CreateSession creates a new session scoped to this instance
+func (p *Proxy) CreateSession(remoteAddr string, metadata map[string]any) *Session {
+	if metadata == nil {
+		metadata = make(map[string]any)
+	}
+
+	// Add instance ID to metadata
+	metadata["instance_id"] = p.instanceID
+
+	// Create session with instance-scoped source
+	session := p.manager.CreateSession(remoteAddr, p.instanceID, metadata)
+	session.InstanceID = p.instanceID
+
+	return session
+}
+
+// GetSession retrieves a session if it belongs to this instance
+func (p *Proxy) GetSession(sessionID string) (*Session, bool) {
+	session, exists := p.manager.GetSession(sessionID)
+	if !exists || session.InstanceID != p.instanceID {
+		return nil, false
+	}
+	return session, true
+}
+
+// RemoveSession removes a session if it belongs to this instance
+func (p *Proxy) RemoveSession(sessionID string) bool {
+	if session, exists := p.GetSession(sessionID); exists {
+		p.manager.RemoveSession(session.ID)
+		return true
+	}
+	return false
+}
+
+// GetActiveSessions returns all active sessions for this instance
+func (p *Proxy) GetActiveSessions() []*Session {
+	allSessions := p.manager.GetSessionsBySource(p.instanceID)
+
+	// Filter by instance ID
+	var filtered []*Session
+	for _, session := range allSessions {
+		if session.InstanceID == p.instanceID {
+			filtered = append(filtered, session)
+		}
+	}
+	return filtered
+}
+
+// UpdateActivity updates activity for a session if it belongs to this instance
+func (p *Proxy) UpdateActivity(sessionID string) bool {
+	if session, exists := p.GetSession(sessionID); exists {
+		p.manager.UpdateActivity(session.ID)
+		return true
+	}
+	return false
+}
+
+// GetInstanceID returns the instance ID this proxy is bound to
+func (p *Proxy) GetInstanceID() string {
+	return p.instanceID
+}

internal/session/session.go

@@ -0,0 +1,294 @@
+package session
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"sync"
+	"time"
+
+	"logwisp/internal/core"
+)
+
+// Session represents a connection session
+type Session struct {
+	InstanceID   string         // Plugin instance identifier
+	ID           string         // Unique session identifier
+	RemoteAddr   string         // Client address
+	CreatedAt    time.Time      // Session creation time
+	LastActivity time.Time      // Last activity timestamp
+	Metadata     map[string]any // Optional metadata (e.g., TLS info)
+
+	// Connection context
+	Source string // Source type: "tcp_source", "http_source", "tcp_sink", etc.
+}
+
+// Manager handles the lifecycle of sessions
+type Manager struct {
+	sessions map[string]*Session
+	mu       sync.RWMutex
+
+	// Cleanup configuration
+	maxIdleTime   time.Duration
+	cleanupTicker *time.Ticker
+	done          chan struct{}
+
+	// Expiry callbacks by source type
+	expiryCallbacks map[string]func(sessionID, remoteAddr string)
+	callbacksMu     sync.RWMutex
+}
+
+// NewManager creates a new session manager with a specified idle timeout
+func NewManager(maxIdleTime time.Duration) *Manager {
+	if maxIdleTime == 0 {
+		maxIdleTime = core.SessionDefaultMaxIdleTime
+	}
+
+	m := &Manager{
+		sessions:        make(map[string]*Session),
+		maxIdleTime:     maxIdleTime,
+		done:            make(chan struct{}),
+		expiryCallbacks: make(map[string]func(sessionID, remoteAddr string)),
+	}
+
+	// Start cleanup routine
+	m.startCleanup()
+
+	return m
+}
+
+// CreateSession creates and stores a new session for a connection
+func (m *Manager) CreateSession(remoteAddr string, source string, metadata map[string]any) *Session {
+	session := &Session{
+		ID:           generateSessionID(),
+		RemoteAddr:   remoteAddr,
+		CreatedAt:    time.Now(),
+		LastActivity: time.Now(),
+		Source:       source,
+		Metadata:     metadata,
+	}
+
+	if metadata == nil {
+		session.Metadata = make(map[string]any)
+	}
+
+	m.StoreSession(session)
+	return session
+}
+
+// StoreSession adds a session to the manager
+func (m *Manager) StoreSession(session *Session) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.sessions[session.ID] = session
+}
+
+// GetSession retrieves a session by its unique ID
+func (m *Manager) GetSession(sessionID string) (*Session, bool) {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	session, exists := m.sessions[sessionID]
+	return session, exists
+}
+
+// RemoveSession removes a session from the manager
+func (m *Manager) RemoveSession(sessionID string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	delete(m.sessions, sessionID)
+}
+
+// UpdateActivity updates the last activity timestamp for a session
+func (m *Manager) UpdateActivity(sessionID string) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if session, exists := m.sessions[sessionID]; exists {
+		session.LastActivity = time.Now()
+	}
+}
+
+// IsSessionActive checks if a session exists and has not been idle for too long
+func (m *Manager) IsSessionActive(sessionID string) bool {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	if session, exists := m.sessions[sessionID]; exists {
+		// Session exists and hasn't exceeded idle timeout
+		return time.Since(session.LastActivity) < m.maxIdleTime
+	}
+	return false
+}
+
+// GetActiveSessions returns a snapshot of all currently active sessions
+func (m *Manager) GetActiveSessions() []*Session {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	sessions := make([]*Session, 0, len(m.sessions))
+	for _, session := range m.sessions {
+		sessions = append(sessions, session)
+	}
+	return sessions
+}
+
+// GetSessionCount returns the number of active sessions
+func (m *Manager) GetSessionCount() int {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+	return len(m.sessions)
+}
+
+// GetSessionsBySource returns all sessions matching a specific source type
+func (m *Manager) GetSessionsBySource(source string) []*Session {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	var sessions []*Session
+	for _, session := range m.sessions {
+		if session.Source == source {
+			sessions = append(sessions, session)
+		}
+	}
+	return sessions
+}
+
+// GetActiveSessionsBySource returns all active sessions for a given source
+func (m *Manager) GetActiveSessionsBySource(source string) []*Session {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	var sessions []*Session
+	now := time.Now()
+
+	for _, session := range m.sessions {
+		if session.Source == source && now.Sub(session.LastActivity) < m.maxIdleTime {
+			sessions = append(sessions, session)
+		}
+	}
+	return sessions
+}
+
+// GetStats returns statistics about the session manager
+func (m *Manager) GetStats() map[string]any {
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
+	sourceCounts := make(map[string]int)
+	var totalSessions int
+	var oldestSession time.Time
+	var newestSession time.Time
+
+	for _, session := range m.sessions {
+		totalSessions++
+		sourceCounts[session.Source]++
+
+		if oldestSession.IsZero() || session.CreatedAt.Before(oldestSession) {
+			oldestSession = session.CreatedAt
+		}
+		if newestSession.IsZero() || session.CreatedAt.After(newestSession) {
+			newestSession = session.CreatedAt
+		}
+	}
+
+	stats := map[string]any{
+		"total_sessions":   totalSessions,
+		"sessions_by_type": sourceCounts,
+		"max_idle_time":    m.maxIdleTime.String(),
+	}
+
+	if !oldestSession.IsZero() {
+		stats["oldest_session_age"] = time.Since(oldestSession).String()
+	}
+	if !newestSession.IsZero() {
+		stats["newest_session_age"] = time.Since(newestSession).String()
+	}
+
+	return stats
+}
+
+// Stop gracefully stops the session manager and its cleanup goroutine
+func (m *Manager) Stop() {
+	close(m.done)
+	if m.cleanupTicker != nil {
+		m.cleanupTicker.Stop()
+	}
+}
+
+// RegisterExpiryCallback registers a callback function to be executed when a session expires
+func (m *Manager) RegisterExpiryCallback(source string, callback func(sessionID, remoteAddr string)) {
+	m.callbacksMu.Lock()
+	defer m.callbacksMu.Unlock()
+
+	if m.expiryCallbacks == nil {
+		m.expiryCallbacks = make(map[string]func(sessionID, remoteAddr string))
+	}
+	m.expiryCallbacks[source] = callback
+}
+
+// UnregisterExpiryCallback removes an expiry callback for a given source type
+func (m *Manager) UnregisterExpiryCallback(source string) {
+	m.callbacksMu.Lock()
+	defer m.callbacksMu.Unlock()
+
+	delete(m.expiryCallbacks, source)
+}
+
+// startCleanup initializes the periodic cleanup of idle sessions
+func (m *Manager) startCleanup() {
+	m.cleanupTicker = time.NewTicker(core.SessionCleanupInterval)
+
+	go func() {
+		for {
+			select {
+			case <-m.cleanupTicker.C:
+				m.cleanupIdleSessions()
+			case <-m.done:
+				return
+			}
+		}
+	}()
+}
+
+// cleanupIdleSessions removes sessions that have exceeded the maximum idle time.
+func (m *Manager) cleanupIdleSessions() {
+	now := time.Now()
+	expiredSessions := make([]*Session, 0)
+
+	m.mu.Lock()
+	for id, session := range m.sessions {
+		idleTime := now.Sub(session.LastActivity)
+
+		if idleTime > m.maxIdleTime {
+			expiredSessions = append(expiredSessions, session)
+			delete(m.sessions, id)
+		}
+	}
+	m.mu.Unlock()
+
+	if len(expiredSessions) > 0 {
+		m.callbacksMu.RLock()
+		callbacks := make(map[string]func(sessionID, remoteAddr string))
+		for k, v := range m.expiryCallbacks {
+			callbacks[k] = v
+		}
+		m.callbacksMu.RUnlock()
+
+		for _, session := range expiredSessions {
+			if callback, exists := callbacks[session.Source]; exists {
+				// Call callback to notify owner
+				go callback(session.ID, session.RemoteAddr)
+			}
+		}
+	}
+}
+
+// generateSessionID creates a unique, random session identifier.
+func generateSessionID() string {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		// Fallback to timestamp-based ID
+		return fmt.Sprintf("session_%d", time.Now().UnixNano())
+	}
+	return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(b)
+}

internal/sink/console/console.go

@@ -0,0 +1,209 @@
+package console
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSink("console", NewConsoleSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register console sink: %v", err))
+	}
+}
+
+// ConsoleSink writes log entries to the console (stdout/stderr) using an dedicated logger instance
+type ConsoleSink struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Configuration
+	config *config.ConsoleSinkOptions
+
+	// Application
+	input  chan core.TransportEvent
+	output io.Writer
+	logger *log.Logger // application logger
+
+	// Runtime
+	done      chan struct{}
+	startTime time.Time
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+const (
+	// Defaults
+	DefaultConsoleTarget     = "stdout"
+	DefaultConsoleBufferSize = 1000
+)
+
+// NewConsoleSinkPlugin creates a console sink through plugin factory
+func NewConsoleSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	opts := &config.ConsoleSinkOptions{}
+
+	// Scan config map into struct
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	// Validate and apply defaults
+	if opts.Target == "" {
+		opts.Target = DefaultConsoleTarget
+	} else {
+		validateTarget := lconfig.OneOf("stdout", "stderr")
+		if err := validateTarget(opts.Target); err != nil {
+			return nil, fmt.Errorf("target: %w", err)
+		}
+	}
+
+	var output io.Writer
+	switch opts.Target {
+	case "stdout":
+		output = os.Stdout
+	case "stderr":
+		output = os.Stderr
+	}
+
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultConsoleBufferSize
+	}
+
+	// Create and return plugin instance
+	cs := &ConsoleSink{
+		id:     id,
+		proxy:  proxy,
+		config: opts,
+		input:  make(chan core.TransportEvent, opts.BufferSize),
+		output: output,
+		done:   make(chan struct{}),
+		logger: logger,
+	}
+	cs.lastProcessed.Store(time.Time{})
+
+	// Create session for output
+	cs.session = proxy.CreateSession(
+		fmt.Sprintf("console:%s", opts.Target),
+		map[string]any{
+			"instance_id": id,
+			"type":        "console",
+			"target":      opts.Target,
+		},
+	)
+
+	cs.logger.Info("msg", "Console sink initialized",
+		"component", "console_sink",
+		"instance_id", id,
+		"target", opts.Target,
+	)
+
+	return cs, nil
+}
+
+// Capabilities returns supported capabilities
+func (cs *ConsoleSink) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware, // Single output session
+	}
+}
+
+// Input returns the channel for sending transport events
+func (cs *ConsoleSink) Input() chan<- core.TransportEvent {
+	return cs.input
+}
+
+// Start begins the processing loop
+func (cs *ConsoleSink) Start(ctx context.Context) error {
+	cs.startTime = time.Now()
+	go cs.processLoop(ctx)
+	cs.logger.Info("msg", "Console sink started",
+		"component", "console_sink",
+		"target", cs.config.Target)
+	return nil
+}
+
+// Stop gracefully shuts down the sink
+func (cs *ConsoleSink) Stop() {
+	cs.logger.Info("msg", "Stopping console sink", "target", cs.config.Target)
+
+	// Remove session
+	if cs.session != nil {
+		cs.proxy.RemoveSession(cs.session.ID)
+	}
+
+	close(cs.done)
+
+	cs.logger.Info("msg", "Console sink stopped",
+		"instance_id", cs.id,
+		"target", cs.config.Target,
+		"instance_id", cs.id,
+	)
+}
+
+// GetStats returns sink statistics
+func (cs *ConsoleSink) GetStats() sink.SinkStats {
+	lastProc, _ := cs.lastProcessed.Load().(time.Time)
+
+	return sink.SinkStats{
+		ID:             cs.id,
+		Type:           "console",
+		TotalProcessed: cs.totalProcessed.Load(),
+		StartTime:      cs.startTime,
+		LastProcessed:  lastProc,
+		Details: map[string]any{
+			"target":      cs.config.Target,
+			"buffer_size": cs.config.BufferSize,
+		},
+	}
+}
+
+// processLoop reads transport events and writes to console
+func (cs *ConsoleSink) processLoop(ctx context.Context) {
+	for {
+		select {
+		case event, ok := <-cs.input:
+			if !ok {
+				return
+			}
+
+			// Write pre-formatted payload directly to output
+			if _, err := cs.output.Write(event.Payload); err != nil {
+				cs.logger.Error("msg", "Failed to write to console",
+					"component", "console_sink",
+					"target", cs.config.Target,
+					"error", err)
+				continue
+			}
+
+			cs.totalProcessed.Add(1)
+			cs.lastProcessed.Store(time.Now())
+
+		case <-ctx.Done():
+			return
+		case <-cs.done:
+			return
+		}
+	}
+}

internal/sink/file/file.go

@@ -0,0 +1,262 @@
+package file
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSink("file", NewFileSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register file sink: %v", err))
+	}
+}
+
+// FileSink writes log entries to files with rotation
+type FileSink struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Configuration
+	config *config.FileSinkOptions
+
+	// Application
+	input  chan core.TransportEvent
+	writer *log.Logger // internal logger for file writing
+	logger *log.Logger // application logger
+
+	// Runtime
+	done      chan struct{}
+	startTime time.Time
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+const (
+	// Defaults
+	DefaultFileMaxSizeMB       = 100
+	DefaultFileMaxTotalSizeMB  = 1000
+	DefaultFileMinDiskFreeMB   = 100
+	DefaultFileRetentionHours  = 168 // 7 days
+	DefaultFileBufferSize      = 1000
+	DefaultFileFlushIntervalMs = 100
+)
+
+// NewFileSinkPlugin creates a file sink through plugin factory
+func NewFileSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	// Create empty config struct
+	opts := &config.FileSinkOptions{}
+
+	// Scan config map into struct
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	// Validate
+	if err := lconfig.NonEmpty(opts.Directory); err != nil {
+		return nil, fmt.Errorf("directory: %w", err)
+	}
+	if err := lconfig.NonEmpty(opts.Name); err != nil {
+		return nil, fmt.Errorf("name: %w", err)
+	}
+
+	// Defaults
+	if opts.MaxSizeMB <= 0 {
+		opts.MaxSizeMB = DefaultFileMaxSizeMB
+	}
+	if opts.MaxTotalSizeMB <= 0 {
+		opts.MaxTotalSizeMB = DefaultFileMaxTotalSizeMB
+	}
+	if opts.MinDiskFreeMB < 0 {
+		opts.MinDiskFreeMB = DefaultFileMinDiskFreeMB
+	}
+	if opts.RetentionHours <= 0 {
+		opts.RetentionHours = DefaultFileRetentionHours
+	}
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultFileBufferSize
+	}
+	if opts.FlushIntervalMs <= 0 {
+		opts.FlushIntervalMs = DefaultFileFlushIntervalMs
+	}
+
+	// Create configuration for the internal log writer
+	writerConfig := log.DefaultConfig()
+	writerConfig.Directory = opts.Directory
+	writerConfig.Name = opts.Name
+	writerConfig.MaxSizeKB = opts.MaxSizeMB * 1000
+	writerConfig.MaxTotalSizeKB = opts.MaxTotalSizeMB * 1000
+	writerConfig.MinDiskFreeKB = opts.MinDiskFreeMB * 1000
+	writerConfig.RetentionPeriodHrs = opts.RetentionHours
+	writerConfig.BufferSize = opts.BufferSize
+	writerConfig.FlushIntervalMs = opts.FlushIntervalMs
+	// Sink logic
+	writerConfig.EnableConsole = false
+	writerConfig.EnableFile = true
+	writerConfig.ShowTimestamp = false
+	writerConfig.ShowLevel = false
+	writerConfig.Format = "raw"
+
+	// Create internal logger for file writing
+	writer := log.NewLogger()
+	if err := writer.ApplyConfig(writerConfig); err != nil {
+		return nil, fmt.Errorf("failed to initialize file writer: %w", err)
+	}
+
+	fs := &FileSink{
+		id:     id,
+		proxy:  proxy,
+		config: opts,
+		input:  make(chan core.TransportEvent, opts.BufferSize),
+		writer: writer,
+		done:   make(chan struct{}),
+		logger: logger,
+	}
+	fs.lastProcessed.Store(time.Time{})
+
+	// Create session for file output
+	fs.session = proxy.CreateSession(
+		fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Name),
+		map[string]any{
+			"instance_id": id,
+			"type":        "file",
+			"directory":   opts.Directory,
+			"name":        opts.Name,
+		},
+	)
+
+	fs.logger.Info("msg", "File sink initialized",
+		"component", "file_sink",
+		"instance_id", id,
+		"directory", opts.Directory,
+		"name", opts.Name)
+
+	return fs, nil
+}
+
+// Capabilities returns supported capabilities
+func (fs *FileSink) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware, // Single output session
+	}
+}
+
+// Input returns the channel for sending transport events
+func (fs *FileSink) Input() chan<- core.TransportEvent {
+	return fs.input
+}
+
+// Start begins the processing loop for the sink
+func (fs *FileSink) Start(ctx context.Context) error {
+	// Start the internal file writer
+	if err := fs.writer.Start(); err != nil {
+		return fmt.Errorf("failed to start file writer: %w", err)
+	}
+
+	fs.startTime = time.Now()
+	go fs.processLoop(ctx)
+
+	fs.logger.Info("msg", "File sink started",
+		"component", "file_sink",
+	)
+	fs.logger.Debug("msg", "File sink config",
+		"component", "file_sink",
+		"directory", fs.config.Directory,
+		"name", fs.config.Name,
+		"max_size_mb", fs.config.MaxSizeMB,
+		"max_total_size_mb", fs.config.MaxTotalSizeMB,
+		"min_disk_free_mb", fs.config.MinDiskFreeMB,
+		"retention_hours", fs.config.RetentionHours,
+		"buffer_size", fs.config.BufferSize,
+		"flush_interval_ms", fs.config.FlushIntervalMs,
+	)
+
+	return nil
+}
+
+// Stop gracefully shuts down the sink
+func (fs *FileSink) Stop() {
+	fs.logger.Info("msg", "Stopping file sink",
+		"component", "file_sink",
+		"directory", fs.config.Directory,
+		"name", fs.config.Name)
+
+	close(fs.done)
+
+	// Remove session
+	if fs.session != nil {
+		fs.proxy.RemoveSession(fs.session.ID)
+	}
+
+	// Shutdown the writer with timeout
+	if err := fs.writer.Shutdown(core.LoggerShutdownTimeout); err != nil {
+		fs.logger.Error("msg", "Error shutting down file writer",
+			"component", "file_sink",
+			"error", err)
+	}
+
+	fs.logger.Info("msg", "File sink stopped",
+		"component", "file_sink",
+		"instance_id", fs.id,
+		"total_processed", fs.totalProcessed.Load())
+}
+
+// GetStats returns the sink's statistics
+func (fs *FileSink) GetStats() sink.SinkStats {
+	return sink.SinkStats{
+		ID:             fs.id,
+		Type:           "file",
+		TotalProcessed: fs.totalProcessed.Load(),
+		StartTime:      fs.startTime,
+		LastProcessed:  fs.lastProcessed.Load().(time.Time),
+		Details: map[string]any{
+			"directory": fs.config.Directory,
+			"name":      fs.config.Name,
+		},
+	}
+}
+
+// processLoop reads transport events and writes to file
+func (fs *FileSink) processLoop(ctx context.Context) {
+	for {
+		select {
+		case event, ok := <-fs.input:
+			if !ok {
+				return
+			}
+
+			// Write the pre-formatted payload directly
+			// The writer handles rotation automatically based on configuration
+			fs.writer.Write(string(event.Payload))
+
+			fs.totalProcessed.Add(1)
+			fs.lastProcessed.Store(time.Now())
+
+		case <-ctx.Done():
+			return
+
+		case <-fs.done:
+			return
+		}
+	}
+}

internal/sink/http/http.go

@@ -0,0 +1,567 @@
+package http
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/authz"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/tlsx"
+	"logwisp/internal/version"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+func init() {
+	if err := plugin.RegisterSink("http", NewHTTPSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register http sink: %v", err))
+	}
+}
+
+const (
+	DefaultHTTPHost             = "0.0.0.0"
+	DefaultHTTPBufferSize       = 1000
+	DefaultHTTPClientBufferSize = 256
+	DefaultHTTPStreamPath       = "/stream"
+	DefaultHTTPStatusPath       = "/status"
+	HTTPReadHeaderTimeout       = 10 * time.Second
+	HTTPShutdownTimeout         = 2 * time.Second
+)
+
+// HTTPSink streams log entries via Server-Sent Events
+// Server.WriteTimeout is deliberately unset (it would terminate long-lived SSE streams)
+// per-write deadlines are applied via http.ResponseController
+type HTTPSink struct {
+	// Plugin identity and session management
+	id    string
+	proxy *session.Proxy
+
+	// Configuration
+	config *config.HTTPSinkOptions
+	addr   string
+
+	// Network
+	server *http.Server
+
+	// Application
+	input  chan core.TransportEvent
+	logger *log.Logger
+
+	// Client registry
+	clients      map[uint64]*sseClient
+	clientsMu    sync.Mutex
+	nextClientID atomic.Uint64
+	writeTimeout time.Duration
+
+	// TLS
+	tlsConfig *tls.Config
+
+	// Authorization
+	auth *authz.Policy
+
+	// Runtime
+	done      chan struct{}
+	stopOnce  sync.Once
+	wg        sync.WaitGroup
+	startTime time.Time
+
+	// Statistics
+	activeClients   atomic.Int64
+	totalProcessed  atomic.Uint64
+	droppedWrites   atomic.Uint64
+	rejectedClients atomic.Uint64
+	lastProcessed   atomic.Value // time.Time
+}
+
+// sseClient is a registered stream consumer with a bounded send queue
+type sseClient struct {
+	send      chan []byte
+	sessionID string
+}
+
+// NewHTTPSinkPlugin creates a http sink through plugin factory
+func NewHTTPSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	opts := &config.HTTPSinkOptions{
+		Host:           DefaultHTTPHost,
+		WriteTimeoutMS: 0, // SSE indefinite streaming
+	}
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+	if err := lconfig.Port(opts.Port); err != nil {
+		return nil, fmt.Errorf("port: %w", err)
+	}
+	if opts.StreamPath == "" {
+		opts.StreamPath = DefaultHTTPStreamPath
+	} else if !strings.HasPrefix(opts.StreamPath, "/") {
+		return nil, fmt.Errorf("stream_path: must start with '/'")
+	}
+	if opts.StatusPath == "" {
+		opts.StatusPath = DefaultHTTPStatusPath
+	} else if !strings.HasPrefix(opts.StatusPath, "/") {
+		return nil, fmt.Errorf("status_path: must start with '/'")
+	}
+	if opts.StreamPath == opts.StatusPath {
+		return nil, fmt.Errorf("stream_path and status_path must differ")
+	}
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultHTTPBufferSize
+	}
+	if opts.ClientBufferSize <= 0 {
+		opts.ClientBufferSize = DefaultHTTPClientBufferSize
+	}
+	tlsCfg, err := tlsx.Server(opts.TLS)
+	if err != nil {
+		return nil, err
+	}
+	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleListener)
+	if err != nil {
+		return nil, err
+	}
+
+	h := &HTTPSink{
+		id:           id,
+		proxy:        proxy,
+		config:       opts,
+		addr:         net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
+		input:        make(chan core.TransportEvent, opts.BufferSize),
+		done:         make(chan struct{}),
+		logger:       logger,
+		clients:      make(map[uint64]*sseClient),
+		writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
+		tlsConfig:    tlsCfg,
+		auth:         authPolicy,
+	}
+	h.lastProcessed.Store(time.Time{})
+
+	logger.Info("msg", " HTTP sink initialized",
+		"component", "http_sink",
+		"instance_id", id,
+		"host", opts.Host,
+		"port", opts.Port,
+		"stream_path", opts.StreamPath,
+		"status_path", opts.StatusPath,
+		"tls", tlsCfg != nil,
+		"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
+		"auth", authPolicy.Describe())
+	if authPolicy.Unrestricted() {
+		logger.Warn("msg", "Auth policy admits any identity the configured CA vouches for",
+			"component", "http_sink",
+			"instance_id", id,
+			"hint", "set auth.allow or auth.allow_patterns to authorize named clients")
+	}
+	return h, nil
+}
+
+// Capabilities returns supported capabilities
+func (h *HTTPSink) Capabilities() []core.Capability {
+	caps := []core.Capability{core.CapSessionAware, core.CapMultiSession}
+	if h.tlsConfig != nil {
+		caps = append(caps, core.CapTLS)
+	}
+	if h.auth.Enabled() {
+		caps = append(caps, core.CapAuth) // authorizes clients, not just the CA
+	}
+	return caps
+}
+
+// Input returns the channel for sending transport events
+func (h *HTTPSink) Input() chan<- core.TransportEvent {
+	return h.input
+}
+
+// Start binds the listener and serves stream/status endpoints
+func (h *HTTPSink) Start(ctx context.Context) error {
+	// IPv4-only, parity with existing network sinks.
+	// TLS is applied via server.TLSConfig + ServeTLS below, not by wrapping
+	// ln; net/http then owns handshake, ALPN (h2), and per-conn errors.
+	ln, err := net.Listen("tcp4", h.addr)
+	if err != nil {
+		return fmt.Errorf("http sink bind %s: %w", h.addr, err)
+	}
+
+	mux := http.NewServeMux()
+	// Method-scoped patterns: mux answers 405 with Allow header on non-GET
+	mux.HandleFunc(http.MethodGet+" "+h.config.StreamPath, h.handleStream)
+	mux.HandleFunc(http.MethodGet+" "+h.config.StatusPath, h.handleStatus)
+
+	// One wrapper covers stream and status, and keeps the handlers themselves
+	// unaware of authorization
+	var handler http.Handler = mux
+	if h.auth.Enabled() {
+		handler = h.authMiddleware(handler)
+	}
+
+	h.server = &http.Server{
+		Handler:           handler,
+		ReadHeaderTimeout: HTTPReadHeaderTimeout,
+		// WriteTimeout unset by design: SSE responses are long-lived.
+		// net/http bounds the TLS handshake by min(ReadHeaderTimeout,
+		// ReadTimeout, WriteTimeout), so ReadHeaderTimeout covers it here.
+		ErrorLog: tlsx.HTTPErrorLog(h.logger, "http_sink"),
+	}
+	h.startTime = time.Now()
+
+	h.wg.Add(1)
+	go h.brokerLoop(ctx)
+
+	serve := h.server.Serve
+	if h.tlsConfig != nil {
+		h.server.TLSConfig = h.tlsConfig
+		serve = func(l net.Listener) error { return h.server.ServeTLS(l, "", "") }
+	}
+
+	go func() {
+		if err := serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			h.logger.Error("msg", "HTTP server terminated",
+				"component", "http_sink",
+				"instance_id", h.id,
+				"error", err)
+		}
+	}()
+
+	go func() {
+		select {
+		case <-ctx.Done():
+			h.shutdown()
+		case <-h.done:
+		}
+	}()
+
+	h.logger.Info("msg", " HTTP server started",
+		"component", "http_sink",
+		"instance_id", h.id,
+		"addr", h.addr)
+	return nil
+}
+
+// Stop gracefully shuts down the sink
+func (h *HTTPSink) Stop() {
+	h.logger.Info("msg", "Stopping HTTP sink",
+		"component", "http_sink",
+		"instance_id", h.id)
+
+	h.shutdown()
+	h.wg.Wait()
+
+	h.logger.Info("msg", " HTTP sink stopped",
+		"component", "http_sink",
+		"instance_id", h.id,
+		"total_processed", h.totalProcessed.Load())
+}
+
+// shutdown funnels ctx-cancel and Stop() teardown through a single path.
+// done is closed first so SSE handlers exit and Shutdown can complete;
+// Server.Close force-closes any handler stalled in a deadline-free write.
+func (h *HTTPSink) shutdown() {
+	h.stopOnce.Do(func() {
+		close(h.done)
+		if h.server != nil {
+			sctx, cancel := context.WithTimeout(context.Background(), HTTPShutdownTimeout)
+			defer cancel()
+			if err := h.server.Shutdown(sctx); err != nil {
+				h.server.Close()
+			}
+		}
+	})
+}
+
+// removeClient unregisters a client; the first caller closes the send
+// channel and removes the session. Broker (stale-session eviction) and
+// stream handler (disconnect) may race here safely.
+func (h *HTTPSink) removeClient(id uint64) {
+	h.clientsMu.Lock()
+	c, ok := h.clients[id]
+	if ok {
+		delete(h.clients, id)
+	}
+	h.clientsMu.Unlock()
+	if ok {
+		close(c.send)
+		h.proxy.RemoveSession(c.sessionID)
+	}
+}
+
+// brokerLoop fans out transport events to all client queues, non-blocking,
+// and evicts clients whose sessions were idle-expired by the session manager
+func (h *HTTPSink) brokerLoop(ctx context.Context) {
+	defer h.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-h.done:
+			return
+		case event, ok := <-h.input:
+			if !ok {
+				return
+			}
+			h.totalProcessed.Add(1)
+			h.lastProcessed.Store(time.Now())
+
+			var stale []uint64
+			h.clientsMu.Lock()
+			for id, c := range h.clients {
+				if _, exists := h.proxy.GetSession(c.sessionID); !exists {
+					stale = append(stale, id)
+					continue
+				}
+				select {
+				case c.send <- event.Payload:
+					h.proxy.UpdateActivity(c.sessionID)
+				default:
+					h.droppedWrites.Add(1)
+				}
+			}
+			h.clientsMu.Unlock()
+
+			for _, id := range stale {
+				h.removeClient(id)
+			}
+		}
+	}
+}
+
+// handleStream serves one client's SSE stream
+func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
+	if h.config.MaxConnections > 0 && h.activeClients.Load() >= h.config.MaxConnections {
+		h.rejectedClients.Add(1)
+		http.Error(w, "too many clients", http.StatusServiceUnavailable)
+		return
+	}
+
+	rc := http.NewResponseController(w)
+	remote := r.RemoteAddr
+
+	meta := map[string]any{
+		"type": "http_client",
+	}
+	if r.TLS != nil {
+		meta["tls"] = true
+		if cn := tlsx.PeerCN(*r.TLS); cn != "" {
+			meta["tls_peer_cn"] = cn
+		}
+	}
+	// Set by authMiddleware; absent when auth is disabled
+	ident, _ := r.Context().Value(identityKey{}).(authz.Identity)
+	ident.Apply(meta)
+	sess := h.proxy.CreateSession(remote, meta)
+
+	c := &sseClient{
+		send:      make(chan []byte, h.config.ClientBufferSize),
+		sessionID: sess.ID,
+	}
+	id := h.nextClientID.Add(1)
+
+	h.clientsMu.Lock()
+	h.clients[id] = c
+	h.clientsMu.Unlock()
+
+	count := h.activeClients.Add(1)
+	h.logger.Debug("msg", "HTTP client connected",
+		"component", "http_sink",
+		"remote_addr", remote,
+		"session_id", sess.ID,
+		"client_id", id,
+		"auth_identity", ident.Name,
+		"active_clients", count)
+
+	defer func() {
+		h.removeClient(id)
+		newCount := h.activeClients.Add(-1)
+		h.logger.Debug("msg", "HTTP client disconnected",
+			"component", "http_sink",
+			"remote_addr", remote,
+			"session_id", sess.ID,
+			"client_id", id,
+			"active_clients", newCount)
+	}()
+
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	w.Header().Set("X-Accel-Buffering", "no")
+	w.WriteHeader(http.StatusOK)
+
+	// Connected event with metadata, parity with fasthttp sink
+	info, _ := json.Marshal(map[string]any{
+		"client_id":   strconv.FormatUint(id, 10),
+		"session_id":  sess.ID,
+		"instance_id": h.id,
+		"stream_path": h.config.StreamPath,
+		"status_path": h.config.StatusPath,
+		"buffer_size": h.config.ClientBufferSize,
+	})
+	fmt.Fprintf(w, "event: connected\ndata: %s\n\n", info)
+	if err := rc.Flush(); err != nil {
+		return
+	}
+
+	clientGone := r.Context().Done()
+	for {
+		select {
+		case payload, ok := <-c.send:
+			if !ok {
+				return // broker evicted (stale session)
+			}
+			if h.writeTimeout > 0 {
+				_ = rc.SetWriteDeadline(time.Now().Add(h.writeTimeout))
+			}
+			if err := writeSSE(w, payload); err != nil {
+				return
+			}
+			if err := rc.Flush(); err != nil {
+				return
+			}
+			h.proxy.UpdateActivity(sess.ID)
+		case <-clientGone:
+			return
+		case <-h.done:
+			fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
+			rc.Flush()
+			return
+		}
+	}
+}
+
+// handleStatus provides a JSON status report
+func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
+	status := map[string]any{
+		"service":     "LogWisp",
+		"version":     version.Short(),
+		"instance_id": h.id,
+		"server": map[string]any{
+			"type":           "http",
+			"host":           h.config.Host,
+			"port":           h.config.Port,
+			"tls":            h.tlsConfig != nil,
+			"auth":           h.auth.Describe(),
+			"active_clients": h.activeClients.Load(),
+			"buffer_size":    h.config.BufferSize,
+			"uptime_seconds": int(time.Since(h.startTime).Seconds()),
+		},
+		"endpoints": map[string]string{
+			"stream": h.config.StreamPath,
+			"status": h.config.StatusPath,
+		},
+		"statistics": map[string]any{
+			"total_processed":  h.totalProcessed.Load(),
+			"dropped_writes":   h.droppedWrites.Load(),
+			"rejected_clients": h.rejectedClients.Load(),
+			"auth_rejected":    h.auth.Rejected(),
+		},
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(status)
+}
+
+// GetStats returns sink statistics
+func (h *HTTPSink) GetStats() sink.SinkStats {
+	lastProc, _ := h.lastProcessed.Load().(time.Time)
+	details := map[string]any{
+		"host":             h.config.Host,
+		"port":             h.config.Port,
+		"buffer_size":      h.config.BufferSize,
+		"tls":              h.tlsConfig != nil,
+		"dropped_writes":   h.droppedWrites.Load(),
+		"rejected_clients": h.rejectedClients.Load(),
+		"endpoints": map[string]string{
+			"stream": h.config.StreamPath,
+			"status": h.config.StatusPath,
+		},
+	}
+	maps.Copy(details, h.auth.Stats())
+
+	return sink.SinkStats{
+		ID:                h.id,
+		Type:              "http",
+		TotalProcessed:    h.totalProcessed.Load(),
+		ActiveConnections: h.activeClients.Load(),
+		StartTime:         h.startTime,
+		LastProcessed:     lastProc,
+		Details:           details,
+	}
+}
+
+// identityKey carries the authorized identity from the middleware to the
+// handlers; absent when auth is disabled
+type identityKey struct{}
+
+// authMiddleware gates every endpoint on the client certificate policy.
+// The rejection carries no detail: the status endpoint already exposes host,
+// port, and throughput counters, so a 403 should not add the shape of the
+// policy on top of that.
+func (h *HTTPSink) authMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ident, err := h.auth.Authorize(r.TLS)
+		if err != nil {
+			h.logger.Warn("msg", "Request rejected by auth policy",
+				"component", "http_sink",
+				"instance_id", h.id,
+				"remote_addr", r.RemoteAddr,
+				"path", r.URL.Path,
+				"error", err)
+			http.Error(w, "forbidden", http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), identityKey{}, ident)))
+	})
+}
+
+// writeSSE frames a payload per the W3C SSE spec (multi-line safe)
+func writeSSE(w http.ResponseWriter, payload []byte) error {
+	for _, line := range splitLines(payload) {
+		if _, err := fmt.Fprintf(w, "data: %s\n", line); err != nil {
+			return err
+		}
+	}
+	_, err := fmt.Fprint(w, "\n")
+	return err
+}
+
+// splitLines splits payload by newlines, trimming a single trailing newline
+func splitLines(data []byte) [][]byte {
+	if len(data) == 0 {
+		return nil
+	}
+	if data[len(data)-1] == '\n' {
+		data = data[:len(data)-1]
+	}
+	var lines [][]byte
+	start := 0
+	for i := 0; i < len(data); i++ {
+		if data[i] == '\n' {
+			lines = append(lines, data[start:i])
+			start = i + 1
+		}
+	}
+	if start < len(data) {
+		lines = append(lines, data[start:])
+	}
+	if len(lines) == 0 {
+		return [][]byte{data}
+	}
+	return lines
+}

internal/sink/httpchain/httpchain.go

@@ -0,0 +1,466 @@
+package httpchain
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"maps"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/authz"
+	"logwisp/internal/chain"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/tlsx"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+func init() {
+	if err := plugin.RegisterSink("http_chain", NewHTTPChainSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register http_chain sink: %v", err))
+	}
+}
+
+const (
+	DefaultHTTPChainSinkBufferSize       = 1000
+	DefaultHTTPChainSinkIngestPath       = "/ingest"
+	DefaultHTTPChainSinkMaxBatchCount    = 100
+	DefaultHTTPChainSinkMaxBatchBytes    = 1024 * 1024
+	DefaultHTTPChainSinkFlushIntervalMS  = 1000
+	DefaultHTTPChainSinkRequestTimeoutMS = 10000
+	DefaultHTTPChainSinkBackoffMinMS     = 500
+	DefaultHTTPChainSinkBackoffMaxMS     = 30000
+)
+
+// HTTPChainSink batches structured entries and posts NDJSON to a downstream
+// http_chain source. Delivery is at-least-once per batch.
+type HTTPChainSink struct {
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+	config  *config.HTTPChainSinkOptions
+
+	node string
+	url  string
+
+	tlsEnabled bool
+	mtls       bool
+
+	// Authorization: pins the downstream server's identity
+	auth *authz.Policy
+
+	client *http.Client
+	input  chan core.TransportEvent
+	logger *log.Logger
+
+	// Batch state owned exclusively by run loop goroutine
+	batch      bytes.Buffer
+	batchCount int64
+
+	reqTimeout time.Duration
+	done       chan struct{}
+	wg         sync.WaitGroup
+	startTime  time.Time
+
+	totalProcessed atomic.Uint64
+	batchesSent    atomic.Uint64
+	requestErrors  atomic.Uint64
+	droppedBatches atomic.Uint64
+	synthesized    atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+// NewHTTPChainSinkPlugin creates an http_chain sink through plugin factory
+func NewHTTPChainSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	opts := &config.HTTPChainSinkOptions{}
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+	if err := lconfig.NonEmpty(opts.Host); err != nil {
+		return nil, fmt.Errorf("host: %w", err)
+	}
+	if err := lconfig.Port(opts.Port); err != nil {
+		return nil, fmt.Errorf("port: %w", err)
+	}
+	if opts.IngestPath == "" {
+		opts.IngestPath = DefaultHTTPChainSinkIngestPath
+	} else if !strings.HasPrefix(opts.IngestPath, "/") {
+		return nil, fmt.Errorf("ingest_path: must start with '/'")
+	}
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultHTTPChainSinkBufferSize
+	}
+	if opts.MaxBatchCount <= 0 {
+		opts.MaxBatchCount = DefaultHTTPChainSinkMaxBatchCount
+	}
+	if opts.MaxBatchBytes <= 0 {
+		opts.MaxBatchBytes = DefaultHTTPChainSinkMaxBatchBytes
+	}
+	if opts.FlushIntervalMS <= 0 {
+		opts.FlushIntervalMS = DefaultHTTPChainSinkFlushIntervalMS
+	}
+	if opts.RequestTimeoutMS <= 0 {
+		opts.RequestTimeoutMS = DefaultHTTPChainSinkRequestTimeoutMS
+	}
+	if opts.BackoffMinMS <= 0 {
+		opts.BackoffMinMS = DefaultHTTPChainSinkBackoffMinMS
+	}
+	if opts.BackoffMaxMS < opts.BackoffMinMS {
+		opts.BackoffMaxMS = DefaultHTTPChainSinkBackoffMaxMS
+	}
+
+	node := opts.Node
+	if node == "" {
+		if hn, err := os.Hostname(); err == nil {
+			node = hn
+		} else {
+			node = "unknown"
+		}
+	}
+
+	tlsCfg, err := tlsx.Client(opts.TLS, opts.Host)
+	if err != nil {
+		return nil, err
+	}
+	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleDialer)
+	if err != nil {
+		return nil, err
+	}
+	if authPolicy.Enabled() {
+		// Runs after the standard chain and hostname checks, so a server the
+		// policy rejects fails the handshake instead of the first request
+		tlsCfg.VerifyConnection = authPolicy.VerifyConnection
+	}
+
+	addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
+
+	transport := &http.Transport{
+		DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
+			// IPv4-only, aligns with tcp/http sinks
+			d := net.Dialer{}
+			return d.DialContext(ctx, "tcp4", address)
+		},
+		MaxIdleConnsPerHost: 2,
+		IdleConnTimeout:     90 * time.Second,
+		DisableCompression:  true,
+		TLSClientConfig:     tlsCfg, // nil = plaintext
+		TLSHandshakeTimeout: tlsx.HandshakeTimeout,
+		// h2 stays off: custom DialContext disables auto-ALPN and batched
+		// NDJSON POSTs gain nothing from it
+	}
+
+	scheme := "http"
+	if tlsCfg != nil {
+		scheme = "https"
+	}
+
+	t := &HTTPChainSink{
+		id:         id,
+		proxy:      proxy,
+		config:     opts,
+		node:       node,
+		tlsEnabled: tlsCfg != nil,
+		mtls:       tlsCfg != nil && len(tlsCfg.Certificates) > 0,
+		auth:       authPolicy,
+		url:        scheme + "://" + addr + opts.IngestPath,
+		client:     &http.Client{Transport: transport},
+		input:      make(chan core.TransportEvent, opts.BufferSize),
+		done:       make(chan struct{}),
+		logger:     logger,
+		reqTimeout: time.Duration(opts.RequestTimeoutMS) * time.Millisecond,
+	}
+	t.lastProcessed.Store(time.Time{})
+
+	t.session = proxy.CreateSession(
+		"http_chain://"+addr,
+		map[string]any{
+			"instance_id": id,
+			"type":        "http_chain",
+			"target":      t.url,
+			"node":        node,
+		},
+	)
+
+	logger.Info("msg", "HTTP chain sink initialized",
+		"component", "http_chain_sink",
+		"instance_id", id,
+		"target", t.url,
+		"node", node,
+		"tls", t.tlsEnabled,
+		"mtls", t.mtls,
+		"auth", authPolicy.Describe())
+	return t, nil
+}
+
+// Capabilities returns supported capabilities
+func (t *HTTPChainSink) Capabilities() []core.Capability {
+	caps := []core.Capability{core.CapSessionAware}
+	if t.tlsEnabled {
+		caps = append(caps, core.CapTLS)
+	}
+	if t.auth.Enabled() {
+		caps = append(caps, core.CapAuth) // pins the server identity
+	}
+	return caps
+}
+
+// Input returns the channel for sending transport events
+func (t *HTTPChainSink) Input() chan<- core.TransportEvent {
+	return t.input
+}
+
+// Start launches the batching loop; downstream availability is not required
+func (t *HTTPChainSink) Start(ctx context.Context) error {
+	t.startTime = time.Now()
+	t.wg.Add(1)
+	go t.runLoop(ctx)
+
+	t.logger.Info("msg", "HTTP chain sink started",
+		"component", "http_chain_sink",
+		"instance_id", t.id,
+		"target", t.url)
+	return nil
+}
+
+// Stop terminates the loop. Worst case: one in-flight request timeout plus
+// one final-flush request timeout.
+func (t *HTTPChainSink) Stop() {
+	t.logger.Info("msg", "Stopping HTTP chain sink",
+		"component", "http_chain_sink",
+		"instance_id", t.id)
+
+	close(t.done)
+	t.wg.Wait()
+	t.client.CloseIdleConnections()
+
+	if t.session != nil {
+		t.proxy.RemoveSession(t.session.ID)
+	}
+
+	t.logger.Info("msg", "HTTP chain sink stopped",
+		"component", "http_chain_sink",
+		"instance_id", t.id,
+		"total_processed", t.totalProcessed.Load())
+}
+
+// GetStats returns sink statistics
+func (t *HTTPChainSink) GetStats() sink.SinkStats {
+	lastProc, _ := t.lastProcessed.Load().(time.Time)
+	details := map[string]any{
+		"target":          t.url,
+		"node":            t.node,
+		"tls":             t.tlsEnabled,
+		"batches_sent":    t.batchesSent.Load(),
+		"request_errors":  t.requestErrors.Load(),
+		"dropped_batches": t.droppedBatches.Load(),
+		"synthesized":     t.synthesized.Load(),
+	}
+	maps.Copy(details, t.auth.Stats())
+
+	return sink.SinkStats{
+		ID:             t.id,
+		Type:           "http_chain",
+		TotalProcessed: t.totalProcessed.Load(),
+		StartTime:      t.startTime,
+		LastProcessed:  lastProc,
+		Details:        details,
+	}
+}
+
+// runLoop batches events and flushes on size or interval
+func (t *HTTPChainSink) runLoop(ctx context.Context) {
+	defer t.wg.Done()
+
+	// Fold done channel into a context for request/backoff interruption
+	runCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go func() {
+		select {
+		case <-t.done:
+			cancel()
+		case <-runCtx.Done():
+		}
+	}()
+
+	ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-runCtx.Done():
+			t.finalFlush()
+			return
+		case <-ticker.C:
+			if t.batchCount > 0 && !t.flush(runCtx) {
+				t.finalFlush()
+				return
+			}
+		case event, ok := <-t.input:
+			if !ok {
+				t.finalFlush()
+				return
+			}
+			t.append(event)
+			if t.batchCount >= t.config.MaxBatchCount ||
+				int64(t.batch.Len()) >= t.config.MaxBatchBytes {
+				if !t.flush(runCtx) {
+					t.finalFlush()
+					return
+				}
+			}
+		}
+	}
+}
+
+// append serializes one event into the pending batch
+func (t *HTTPChainSink) append(event core.TransportEvent) {
+	entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
+	if synthesized {
+		t.synthesized.Add(1)
+	}
+	line, err := json.Marshal(entry)
+	if err != nil {
+		// Non-transient: drop entry
+		t.logger.Error("msg", "Failed to marshal chain entry",
+			"component", "http_chain_sink",
+			"error", err)
+		return
+	}
+	t.batch.Write(line)
+	t.batch.WriteByte('\n')
+	t.batchCount++
+}
+
+// flush delivers the pending batch, retrying transient failures with backoff.
+// Returns false when shutdown interrupts delivery; undelivered batch is dropped
+// by finalFlush semantics (batch already consumed here).
+func (t *HTTPChainSink) flush(ctx context.Context) bool {
+	body := bytes.Clone(t.batch.Bytes())
+	count := t.batchCount
+	t.batch.Reset()
+	t.batchCount = 0
+
+	failures := 0
+	for {
+		if failures > 0 && !t.waitBackoff(ctx, failures) {
+			t.droppedBatches.Add(1)
+			return false
+		}
+		transient, err := t.post(ctx, body)
+		if err == nil {
+			t.batchesSent.Add(1)
+			t.totalProcessed.Add(uint64(count))
+			t.lastProcessed.Store(time.Now())
+			t.proxy.UpdateActivity(t.session.ID)
+			return true
+		}
+		t.requestErrors.Add(1)
+		if !transient {
+			t.droppedBatches.Add(1)
+			t.logger.Error("msg", "Chain batch rejected, dropping",
+				"component", "http_chain_sink",
+				"target", t.url,
+				"entries", count,
+				"error", err)
+			return true
+		}
+		if ctx.Err() != nil {
+			t.droppedBatches.Add(1)
+			return false
+		}
+		failures++
+		t.logger.Warn("msg", "Chain batch delivery failed",
+			"component", "http_chain_sink",
+			"target", t.url,
+			"attempt", failures,
+			"error", err)
+	}
+}
+
+// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
+func (t *HTTPChainSink) finalFlush() {
+	if t.batchCount == 0 {
+		return
+	}
+	fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
+	defer cancel()
+
+	count := t.batchCount
+	if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
+		t.droppedBatches.Add(1)
+		t.logger.Warn("msg", "Final chain batch dropped on shutdown",
+			"component", "http_chain_sink",
+			"entries", count,
+			"error", err)
+		return
+	}
+	t.batchesSent.Add(1)
+	t.totalProcessed.Add(uint64(count))
+}
+
+// post sends one NDJSON batch; transient=true marks retryable failures
+func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool, err error) {
+	reqCtx, cancel := context.WithTimeout(ctx, t.reqTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, t.url, bytes.NewReader(body))
+	if err != nil {
+		return false, err
+	}
+	req.Header.Set("Content-Type", chain.ContentTypeNDJSON)
+	req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion))
+	req.Header.Set(chain.HeaderNode, t.node)
+	// Future: Authorization header for auth
+
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return true, err
+	}
+	defer resp.Body.Close()
+	// Drain for connection reuse
+	io.Copy(io.Discard, resp.Body)
+
+	switch {
+	case resp.StatusCode >= 200 && resp.StatusCode < 300:
+		return false, nil
+	case resp.StatusCode == http.StatusRequestTimeout,
+		resp.StatusCode == http.StatusTooManyRequests,
+		resp.StatusCode >= 500:
+		return true, fmt.Errorf("status %s", resp.Status)
+	default:
+		return false, fmt.Errorf("status %s", resp.Status)
+	}
+}
+
+// waitBackoff sleeps for the computed delay, interruptible by shutdown
+func (t *HTTPChainSink) waitBackoff(ctx context.Context, failures int) bool {
+	minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
+	maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
+	timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+		return true
+	case <-ctx.Done():
+		return false
+	}
+}

internal/sink/null/null.go

@@ -0,0 +1,146 @@
+package null
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSink("null", NewNullSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register null sink: %v", err))
+	}
+}
+
+// NullSink discards all received transport events, used for testing
+type NullSink struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Application
+	input  chan core.TransportEvent
+	logger *log.Logger
+
+	// Runtime
+	done      chan struct{}
+	startTime time.Time
+
+	// Statistics
+	totalReceived atomic.Uint64
+	totalBytes    atomic.Uint64
+	lastReceived  atomic.Value // time.Time
+}
+
+// NewNullSinkPlugin creates a null sink through plugin factory
+func NewNullSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	ns := &NullSink{
+		id:     id,
+		proxy:  proxy,
+		input:  make(chan core.TransportEvent, 1000),
+		done:   make(chan struct{}),
+		logger: logger,
+	}
+	ns.lastReceived.Store(time.Time{})
+
+	// Create session for null sink
+	ns.session = proxy.CreateSession(
+		"null://devnull",
+		map[string]any{
+			"instance_id": id,
+			"type":        "null",
+		},
+	)
+
+	logger.Debug("msg", "Null sink initialized",
+		"component", "null_sink",
+		"instance_id", id)
+
+	return ns, nil
+}
+
+// Capabilities returns supported capabilities
+func (ns *NullSink) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware,
+	}
+}
+
+// Input returns the channel for sending transport events
+func (ns *NullSink) Input() chan<- core.TransportEvent {
+	return ns.input
+}
+
+// Start begins the processing loop
+func (ns *NullSink) Start(ctx context.Context) error {
+
+	ns.startTime = time.Now()
+	go ns.processLoop(ctx)
+	ns.logger.Debug("msg", "Null sink started",
+		"component", "null_sink",
+		"instance_id", ns.id)
+	return nil
+}
+
+// Stop gracefully shuts down the sink
+func (ns *NullSink) Stop() {
+	if ns.session != nil {
+		ns.proxy.RemoveSession(ns.session.ID)
+	}
+	close(ns.done)
+	ns.logger.Debug("msg", "Null sink stopped",
+		"instance_id", ns.id,
+		"total_received", ns.totalReceived.Load())
+}
+
+// GetStats returns sink statistics
+func (ns *NullSink) GetStats() sink.SinkStats {
+	lastRcv, _ := ns.lastReceived.Load().(time.Time)
+
+	return sink.SinkStats{
+		ID:             ns.id,
+		Type:           "null",
+		TotalProcessed: ns.totalReceived.Load(),
+		StartTime:      ns.startTime,
+		LastProcessed:  lastRcv,
+		Details: map[string]any{
+			"total_bytes": ns.totalBytes.Load(),
+		},
+	}
+}
+
+// processLoop reads transport events and discards them
+func (ns *NullSink) processLoop(ctx context.Context) {
+	for {
+		select {
+		case event, ok := <-ns.input:
+			if !ok {
+				return
+			}
+			// Discard the event, only update stats
+			ns.totalReceived.Add(1)
+			ns.totalBytes.Add(uint64(len(event.Payload)))
+			ns.lastReceived.Store(time.Now())
+
+		case <-ctx.Done():
+			return
+		case <-ns.done:
+			return
+		}
+	}
+}

internal/sink/sink.go

@@ -0,0 +1,37 @@
+package sink
+
+import (
+	"context"
+	"time"
+
+	"logwisp/internal/core"
+)
+
+// Sink represents an output data stream.
+type Sink interface {
+	// Capabilities returns a slice of supported Source capabilities
+	Capabilities() []core.Capability
+
+	// Input returns the channel for sending transport events to this sink.
+	Input() chan<- core.TransportEvent
+
+	// Start begins processing transport events.
+	Start(ctx context.Context) error
+
+	// Stop gracefully shuts down the sink.
+	Stop()
+
+	// GetStats returns sink statistics.
+	GetStats() SinkStats
+}
+
+// SinkStats contains statistics about a sink.
+type SinkStats struct {
+	ID                string
+	Type              string
+	TotalProcessed    uint64
+	ActiveConnections int64
+	StartTime         time.Time
+	LastProcessed     time.Time
+	Details           map[string]any
+}

internal/sink/tcp/tcp.go

@@ -0,0 +1,482 @@
+package tcp
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"maps"
+	"net"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/authz"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/tlsx"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+func init() {
+	if err := plugin.RegisterSink("tcp", NewTCPSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register tcp sink: %v", err))
+	}
+}
+
+const (
+	DefaultTCPHost              = "0.0.0.0"
+	DefaultTCPBufferSize        = 1000
+	DefaultTCPClientBufferSize  = 256
+	DefaultTCPWriteTimeoutMS    = 5000
+	DefaultTCPKeepAlivePeriodMS = 30000
+)
+
+// TCPSink streams formatted log entries to connected TCP clients
+// Concurrency model: one broadcast loop fans out into bounded per-client queues
+// each connection owns a writer goroutine (drains queue) and a reader goroutine (disconnect detection)
+// A stalled client drops events, never the pipeline.
+type TCPSink struct {
+	// Plugin identity and session management
+	id    string
+	proxy *session.Proxy
+
+	// Configuration
+	config *config.TCPSinkOptions
+	addr   string
+
+	// Network
+	listener net.Listener
+
+	// Application
+	input  chan core.TransportEvent
+	logger *log.Logger
+
+	// Client registry
+	clients      map[uint64]*tcpClient
+	clientsMu    sync.Mutex
+	nextClientID atomic.Uint64
+	writeTimeout time.Duration
+
+	// TLS
+	tlsConfig          *tls.Config
+	tlsHandshakeErrors atomic.Uint64
+
+	// Authorization
+	auth *authz.Policy
+
+	// Runtime
+	done      chan struct{}
+	stopOnce  sync.Once
+	wg        sync.WaitGroup
+	startTime time.Time
+
+	// Statistics
+	activeConns    atomic.Int64
+	totalProcessed atomic.Uint64
+	writeErrors    atomic.Uint64
+	droppedWrites  atomic.Uint64
+	rejectedConns  atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+// tcpClient pairs a connection with its bounded send queue.
+// send is written by the broadcast loop (non-blocking) and drained by the
+// writer goroutine; closed signals reader-detected disconnect.
+type tcpClient struct {
+	conn      net.Conn
+	send      chan []byte
+	sessionID string
+	closed    chan struct{}
+}
+
+// NewTCPSinkPlugin creates a tcp sink through plugin factory
+func NewTCPSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	opts := &config.TCPSinkOptions{
+		Host:      DefaultTCPHost,
+		KeepAlive: true,
+	}
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+	if err := lconfig.Port(opts.Port); err != nil {
+		return nil, fmt.Errorf("port: %w", err)
+	}
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultTCPBufferSize
+	}
+	if opts.ClientBufferSize <= 0 {
+		opts.ClientBufferSize = DefaultTCPClientBufferSize
+	}
+	if opts.WriteTimeoutMS <= 0 {
+		opts.WriteTimeoutMS = DefaultTCPWriteTimeoutMS
+	}
+	if opts.KeepAlivePeriodMS <= 0 {
+		opts.KeepAlivePeriodMS = DefaultTCPKeepAlivePeriodMS
+	}
+	tlsCfg, err := tlsx.Server(opts.TLS)
+	if err != nil {
+		return nil, err
+	}
+	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleListener)
+	if err != nil {
+		return nil, err
+	}
+
+	t := &TCPSink{
+		id:           id,
+		proxy:        proxy,
+		config:       opts,
+		addr:         net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
+		input:        make(chan core.TransportEvent, opts.BufferSize),
+		done:         make(chan struct{}),
+		logger:       logger,
+		clients:      make(map[uint64]*tcpClient),
+		writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
+		tlsConfig:    tlsCfg,
+		auth:         authPolicy,
+	}
+	t.lastProcessed.Store(time.Time{})
+
+	logger.Info("msg", " TCP sink initialized",
+		"component", "tcp_sink",
+		"instance_id", id,
+		"host", opts.Host,
+		"port", opts.Port,
+		"tls", tlsCfg != nil,
+		"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
+		"auth", authPolicy.Describe())
+	if authPolicy.Unrestricted() {
+		logger.Warn("msg", "Auth policy admits any identity the configured CA vouches for",
+			"component", "tcp_sink",
+			"instance_id", id,
+			"hint", "set auth.allow or auth.allow_patterns to authorize named clients")
+	}
+	return t, nil
+}
+
+// Capabilities returns supported capabilities
+func (t *TCPSink) Capabilities() []core.Capability {
+	caps := []core.Capability{core.CapSessionAware, core.CapMultiSession}
+	if t.tlsConfig != nil {
+		caps = append(caps, core.CapTLS)
+	}
+	if t.auth.Enabled() {
+		caps = append(caps, core.CapAuth) // authorizes clients, not just the CA
+	}
+	return caps
+}
+
+// Input returns the channel for sending transport events
+func (t *TCPSink) Input() chan<- core.TransportEvent {
+	return t.input
+}
+
+// listen creates the server listener, TLS-wrapped when configured.
+// Handshake is deferred: tls.NewListener conns handshake explicitly in
+// handleConn under tlsx.HandshakeTimeout, post max_connections admission.
+func (t *TCPSink) listen() (net.Listener, error) {
+	lc := net.ListenConfig{}
+	if t.config.KeepAlive {
+		lc.KeepAliveConfig = net.KeepAliveConfig{
+			Enable: true,
+			Idle:   time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
+		}
+	}
+	// IPv4-only, parity with existing network sinks
+	ln, err := lc.Listen(context.Background(), "tcp4", t.addr)
+	if err != nil {
+		return nil, err
+	}
+	if t.tlsConfig != nil {
+		ln = tls.NewListener(ln, t.tlsConfig)
+	}
+	return ln, nil
+}
+
+// Start binds the listener and launches accept and broadcast loops
+func (t *TCPSink) Start(ctx context.Context) error {
+	ln, err := t.listen()
+	if err != nil {
+		return fmt.Errorf("tcp sink bind %s: %w", t.addr, err)
+	}
+	t.listener = ln
+	t.startTime = time.Now()
+
+	t.wg.Add(2)
+	go t.acceptLoop()
+	go t.broadcastLoop(ctx)
+
+	// Pipeline context cancellation mirrors gnet engine stop: cease accepting
+	// and tear down existing connections
+	go func() {
+		select {
+		case <-ctx.Done():
+			t.shutdown()
+		case <-t.done:
+		}
+	}()
+
+	t.logger.Info("msg", " TCP server started",
+		"component", "tcp_sink",
+		"instance_id", t.id,
+		"addr", t.addr)
+	return nil
+}
+
+// Stop gracefully shuts down the sink
+func (t *TCPSink) Stop() {
+	t.logger.Info("msg", "Stopping TCP sink",
+		"component", "tcp_sink",
+		"instance_id", t.id)
+
+	t.shutdown()
+	t.wg.Wait()
+
+	t.logger.Info("msg", " TCP sink stopped",
+		"component", "tcp_sink",
+		"instance_id", t.id,
+		"total_processed", t.totalProcessed.Load())
+}
+
+// shutdown funnels ctx-cancel and Stop() teardown through a single path
+func (t *TCPSink) shutdown() {
+	t.stopOnce.Do(func() {
+		close(t.done)
+		if t.listener != nil {
+			t.listener.Close() // unblocks acceptLoop
+		}
+		t.clientsMu.Lock()
+		for _, c := range t.clients {
+			c.conn.Close() // unblocks per-connection readers
+		}
+		t.clientsMu.Unlock()
+	})
+}
+
+// acceptLoop accepts client connections until listener close
+func (t *TCPSink) acceptLoop() {
+	defer t.wg.Done()
+	for {
+		conn, err := t.listener.Accept()
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) {
+				return
+			}
+			t.logger.Warn("msg", "Accept error",
+				"component", "tcp_sink",
+				"error", err)
+			continue
+		}
+
+		if t.config.MaxConnections > 0 && t.activeConns.Load() >= t.config.MaxConnections {
+			// Load/admit race can over-admit by a conn under burst; acceptable
+			t.rejectedConns.Add(1)
+			conn.Close()
+			continue
+		}
+
+		// Certificate authorization runs in handleConn post-handshake,
+		// pre-registration. Password-auth extension point: preamble
+		// verification belongs at the same place.
+
+		t.wg.Add(1)
+		go t.handleConn(conn)
+	}
+}
+
+// handleConn registers the client and runs its writer; a companion reader
+// goroutine drains inbound bytes for disconnect detection
+func (t *TCPSink) handleConn(conn net.Conn) {
+	defer t.wg.Done()
+	remote := conn.RemoteAddr().String()
+
+	// Counted from accept: max_connections bounds concurrent handshakes too
+	count := t.activeConns.Add(1)
+	defer func() {
+		newCount := t.activeConns.Add(-1)
+		t.logger.Debug("msg", "TCP connection closed",
+			"component", "tcp_sink",
+			"remote_addr", remote,
+			"active_connections", newCount)
+	}()
+
+	meta := map[string]any{
+		"type":        "tcp_client",
+		"remote_addr": remote,
+	}
+	var tlsState *tls.ConnectionState
+	if tc, ok := conn.(*tls.Conn); ok {
+		hctx, cancel := context.WithTimeout(context.Background(), tlsx.HandshakeTimeout)
+		err := tc.HandshakeContext(hctx)
+		cancel()
+		if err != nil {
+			t.tlsHandshakeErrors.Add(1)
+			t.logger.Debug("msg", "TLS handshake failed",
+				"component", "tcp_sink",
+				"remote_addr", remote,
+				"error", err)
+			conn.Close()
+			return
+		}
+		cs := tc.ConnectionState()
+		tlsState = &cs
+		meta["tls"] = true
+		if cn := tlsx.PeerCN(cs); cn != "" {
+			meta["tls_peer_cn"] = cn
+		}
+	}
+
+	// Authorize before registration, so an unauthorized peer never enters the
+	// client map and never receives a broadcast
+	ident, err := t.auth.Authorize(tlsState)
+	if err != nil {
+		t.rejectedConns.Add(1)
+		t.logger.Warn("msg", "Connection rejected by auth policy",
+			"component", "tcp_sink",
+			"instance_id", t.id,
+			"remote_addr", remote,
+			"error", err)
+		conn.Close()
+		return
+	}
+	ident.Apply(meta)
+
+	sess := t.proxy.CreateSession(remote, meta)
+	c := &tcpClient{
+		conn:      conn,
+		send:      make(chan []byte, t.config.ClientBufferSize),
+		sessionID: sess.ID,
+		closed:    make(chan struct{}),
+	}
+	id := t.nextClientID.Add(1)
+
+	t.clientsMu.Lock()
+	t.clients[id] = c
+	t.clientsMu.Unlock()
+
+	t.logger.Debug("msg", "TCP connection opened",
+		"component", "tcp_sink",
+		"remote_addr", remote,
+		"session_id", sess.ID,
+		"auth_identity", ident.Name,
+		"active_connections", count)
+
+	defer func() {
+		t.clientsMu.Lock()
+		delete(t.clients, id)
+		t.clientsMu.Unlock()
+		conn.Close()
+		<-c.closed // reader has exited
+		t.proxy.RemoveSession(sess.ID)
+	}()
+
+	// Reader: sink is write-only; drain and discard inbound bytes to detect
+	// disconnect and refresh session activity on client traffic
+	go func() {
+		defer close(c.closed)
+		buf := make([]byte, 4096)
+		for {
+			n, err := conn.Read(buf)
+			if n > 0 {
+				t.proxy.UpdateActivity(sess.ID)
+			}
+			if err != nil {
+				return
+			}
+		}
+	}()
+
+	// Writer: synchronous lib write with deadline. A failed write means
+	// the kernel buffer stayed full for the full deadline - connection is
+	// dead or hopelessly stalled, so disconnect immediately (no gnet-style
+	// consecutive-error counter needed for transient async callback errors).
+	for {
+		select {
+		case data := <-c.send:
+			if t.writeTimeout > 0 {
+				conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
+			}
+			if _, err := conn.Write(data); err != nil {
+				t.writeErrors.Add(1)
+				t.logger.Debug("msg", "Write failed, closing client",
+					"component", "tcp_sink",
+					"remote_addr", remote,
+					"error", err)
+				return
+			}
+			t.proxy.UpdateActivity(sess.ID)
+		case <-c.closed:
+			return
+		case <-t.done:
+			return
+		}
+	}
+}
+
+// broadcastLoop fans out transport events to all client queues, non-blocking
+func (t *TCPSink) broadcastLoop(ctx context.Context) {
+	defer t.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.done:
+			return
+		case event, ok := <-t.input:
+			if !ok {
+				return
+			}
+			t.totalProcessed.Add(1)
+			t.lastProcessed.Store(time.Now())
+
+			t.clientsMu.Lock()
+			for _, c := range t.clients {
+				select {
+				case c.send <- event.Payload:
+				default:
+					// Slow client: drop its event, never stall siblings
+					t.droppedWrites.Add(1)
+				}
+			}
+			t.clientsMu.Unlock()
+		}
+	}
+}
+
+// GetStats returns sink statistics
+func (t *TCPSink) GetStats() sink.SinkStats {
+	lastProc, _ := t.lastProcessed.Load().(time.Time)
+	details := map[string]any{
+		"host":                 t.config.Host,
+		"port":                 t.config.Port,
+		"buffer_size":          t.config.BufferSize,
+		"write_errors":         t.writeErrors.Load(),
+		"dropped_writes":       t.droppedWrites.Load(),
+		"rejected_conns":       t.rejectedConns.Load(),
+		"tls":                  t.tlsConfig != nil,
+		"tls_handshake_errors": t.tlsHandshakeErrors.Load(),
+	}
+	maps.Copy(details, t.auth.Stats())
+
+	return sink.SinkStats{
+		ID:                t.id,
+		Type:              "tcp",
+		TotalProcessed:    t.totalProcessed.Load(),
+		ActiveConnections: t.activeConns.Load(),
+		StartTime:         t.startTime,
+		LastProcessed:     lastProc,
+		Details:           details,
+	}
+}

internal/sink/tcpchain/tcpchain.go

@@ -0,0 +1,443 @@
+package tcpchain
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"maps"
+	"math/rand/v2"
+	"net"
+	"os"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/authz"
+	"logwisp/internal/chain"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/sink"
+	"logwisp/internal/tlsx"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+func init() {
+	if err := plugin.RegisterSink("tcp_chain", NewTCPChainSinkPlugin); err != nil {
+		panic(fmt.Sprintf("failed to register tcp_chain sink: %v", err))
+	}
+}
+
+const (
+	DefaultChainSinkBufferSize        = 1000
+	DefaultChainSinkDialTimeoutMS     = 5000
+	DefaultChainSinkWriteTimeoutMS    = 5000
+	DefaultChainSinkBackoffMinMS      = 500
+	DefaultChainSinkBackoffMaxMS      = 30000
+	DefaultChainSinkKeepAlivePeriodMS = 30000
+)
+
+// TCPChainSink forwards structured entries to a downstream tcp_chain source
+type TCPChainSink struct {
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+	config  *config.TCPChainSinkOptions
+
+	node      string
+	addr      string
+	helloLine []byte
+	tlsConfig *tls.Config
+
+	// Authorization: pins the downstream server's identity
+	auth *authz.Policy
+
+	input  chan core.TransportEvent
+	logger *log.Logger
+
+	// conn owned exclusively by run loop goroutine
+	conn          net.Conn
+	everConnected bool
+	dialTimeout   time.Duration
+	writeTimeout  time.Duration
+
+	done      chan struct{}
+	wg        sync.WaitGroup
+	startTime time.Time
+
+	totalProcessed atomic.Uint64
+	writeErrors    atomic.Uint64
+	reconnects     atomic.Uint64
+	synthesized    atomic.Uint64
+	connected      atomic.Bool
+	lastProcessed  atomic.Value // time.Time
+}
+
+// NewTCPChainSinkPlugin creates a tcp_chain sink through plugin factory
+func NewTCPChainSinkPlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (sink.Sink, error) {
+	opts := &config.TCPChainSinkOptions{
+		KeepAlive: true,
+	}
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+	if err := lconfig.NonEmpty(opts.Host); err != nil {
+		return nil, fmt.Errorf("host: %w", err)
+	}
+	if err := lconfig.Port(opts.Port); err != nil {
+		return nil, fmt.Errorf("port: %w", err)
+	}
+
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultChainSinkBufferSize
+	}
+	if opts.DialTimeoutMS <= 0 {
+		opts.DialTimeoutMS = DefaultChainSinkDialTimeoutMS
+	}
+	if opts.WriteTimeoutMS <= 0 {
+		opts.WriteTimeoutMS = DefaultChainSinkWriteTimeoutMS
+	}
+	if opts.BackoffMinMS <= 0 {
+		opts.BackoffMinMS = DefaultChainSinkBackoffMinMS
+	}
+	if opts.BackoffMaxMS < opts.BackoffMinMS {
+		opts.BackoffMaxMS = DefaultChainSinkBackoffMaxMS
+	}
+	if opts.KeepAlivePeriodMS <= 0 {
+		opts.KeepAlivePeriodMS = DefaultChainSinkKeepAlivePeriodMS
+	}
+
+	node := opts.Node
+	if node == "" {
+		if hn, err := os.Hostname(); err == nil {
+			node = hn
+		} else {
+			node = "unknown"
+		}
+	}
+
+	helloLine, err := chain.EncodeHello(node)
+	if err != nil {
+		return nil, fmt.Errorf("hello: %w", err)
+	}
+	tlsCfg, err := tlsx.Client(opts.TLS, opts.Host)
+	if err != nil {
+		return nil, err
+	}
+	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleDialer)
+	if err != nil {
+		return nil, err
+	}
+	if authPolicy.Enabled() {
+		// Runs after the standard chain and hostname checks, so a server the
+		// policy rejects fails the handshake instead of the first write
+		tlsCfg.VerifyConnection = authPolicy.VerifyConnection
+	}
+
+	t := &TCPChainSink{
+		id:           id,
+		proxy:        proxy,
+		config:       opts,
+		node:         node,
+		addr:         net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
+		helloLine:    helloLine,
+		tlsConfig:    tlsCfg,
+		auth:         authPolicy,
+		input:        make(chan core.TransportEvent, opts.BufferSize),
+		done:         make(chan struct{}),
+		logger:       logger,
+		dialTimeout:  time.Duration(opts.DialTimeoutMS) * time.Millisecond,
+		writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
+	}
+	t.lastProcessed.Store(time.Time{})
+
+	t.session = proxy.CreateSession(
+		"tcp_chain://"+t.addr,
+		map[string]any{
+			"instance_id": id,
+			"type":        "tcp_chain",
+			"target":      t.addr,
+			"node":        node,
+		},
+	)
+
+	logger.Info("msg", "TCP chain sink initialized",
+		"component", "tcp_chain_sink",
+		"instance_id", id,
+		"target", t.addr,
+		"node", node,
+		"tls", tlsCfg != nil,
+		"mtls", tlsCfg != nil && len(tlsCfg.Certificates) > 0,
+		"auth", authPolicy.Describe())
+	return t, nil
+}
+
+// Capabilities returns supported capabilities
+func (t *TCPChainSink) Capabilities() []core.Capability {
+	caps := []core.Capability{core.CapSessionAware}
+	if t.tlsConfig != nil {
+		caps = append(caps, core.CapTLS)
+	}
+	if t.auth.Enabled() {
+		caps = append(caps, core.CapAuth) // pins the server identity
+	}
+	return caps
+}
+
+// Input returns the channel for sending transport events
+func (t *TCPChainSink) Input() chan<- core.TransportEvent {
+	return t.input
+}
+
+// Start launches the forwarding loop; connection is established lazily so
+// pipeline start does not depend on downstream availability
+func (t *TCPChainSink) Start(ctx context.Context) error {
+	t.startTime = time.Now()
+	t.wg.Add(1)
+	go t.runLoop(ctx)
+
+	t.logger.Info("msg", "TCP chain sink started",
+		"component", "tcp_chain_sink",
+		"instance_id", t.id,
+		"target", t.addr)
+	return nil
+}
+
+// Stop terminates the forwarding loop. Worst-case latency: one write timeout
+// plus one backoff wait (both interruptible or bounded).
+func (t *TCPChainSink) Stop() {
+	t.logger.Info("msg", "Stopping TCP chain sink",
+		"component", "tcp_chain_sink",
+		"instance_id", t.id)
+
+	close(t.done)
+	t.wg.Wait()
+
+	if t.session != nil {
+		t.proxy.RemoveSession(t.session.ID)
+	}
+
+	t.logger.Info("msg", "TCP chain sink stopped",
+		"component", "tcp_chain_sink",
+		"instance_id", t.id,
+		"total_processed", t.totalProcessed.Load())
+}
+
+// GetStats returns sink statistics
+func (t *TCPChainSink) GetStats() sink.SinkStats {
+	lastProc, _ := t.lastProcessed.Load().(time.Time)
+	var active int64
+	if t.connected.Load() {
+		active = 1
+	}
+	details := map[string]any{
+		"target":       t.addr,
+		"node":         t.node,
+		"tls":          t.tlsConfig != nil,
+		"connected":    t.connected.Load(),
+		"reconnects":   t.reconnects.Load(),
+		"write_errors": t.writeErrors.Load(),
+		"synthesized":  t.synthesized.Load(),
+	}
+	maps.Copy(details, t.auth.Stats())
+
+	return sink.SinkStats{
+		ID:                t.id,
+		Type:              "tcp_chain",
+		TotalProcessed:    t.totalProcessed.Load(),
+		ActiveConnections: active,
+		StartTime:         t.startTime,
+		LastProcessed:     lastProc,
+		Details:           details,
+	}
+}
+
+// runLoop consumes transport events and forwards them downstream
+func (t *TCPChainSink) runLoop(ctx context.Context) {
+	defer t.wg.Done()
+	defer t.closeConn()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.done:
+			return
+		case event, ok := <-t.input:
+			if !ok {
+				return
+			}
+			entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
+			if synthesized {
+				t.synthesized.Add(1)
+			}
+			line, err := json.Marshal(entry)
+			if err != nil {
+				// Non-transient: drop
+				t.logger.Error("msg", "Failed to marshal chain entry",
+					"component", "tcp_chain_sink",
+					"error", err)
+				continue
+			}
+			if !t.deliver(ctx, append(line, '\n')) {
+				return // shutdown during retry
+			}
+			t.totalProcessed.Add(1)
+			t.lastProcessed.Store(time.Now())
+			t.proxy.UpdateActivity(t.session.ID)
+		}
+	}
+}
+
+// toEntry extracts the structured entry, stamping node identity at first hop
+func (t *TCPChainSink) toEntry(event core.TransportEvent) core.LogEntry {
+	entry := event.Entry
+	if entry.Time.IsZero() {
+		// Defensive: event without structured entry, wrap formatted payload
+		t.synthesized.Add(1)
+		entry = core.LogEntry{
+			Time:    event.Time,
+			Source:  t.id,
+			Message: string(event.Payload),
+		}
+	}
+	if entry.Node == "" {
+		entry.Node = t.node
+	}
+	return entry
+}
+
+// deliver writes one line, holding it across reconnects until sent or shutdown.
+// Backpressure during outage propagates to the pipeline dispatch drop counter.
+func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
+	failures := 0
+	for {
+		if t.conn == nil {
+			if failures > 0 && !t.waitBackoff(ctx, failures) {
+				return false
+			}
+			if err := t.connect(ctx); err != nil {
+				if ctx.Err() != nil {
+					return false
+				}
+				failures++
+				t.logger.Debug("msg", "Chain connect failed",
+					"component", "tcp_chain_sink",
+					"target", t.addr,
+					"attempt", failures,
+					"error", err)
+				continue
+			}
+		}
+
+		t.conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
+		if _, err := t.conn.Write(line); err != nil {
+			t.writeErrors.Add(1)
+			failures++
+			t.logger.Warn("msg", "Chain write failed",
+				"component", "tcp_chain_sink",
+				"target", t.addr,
+				"error", err)
+			t.closeConn()
+			continue
+		}
+		return true
+	}
+}
+
+// connect performs a single dial (+ TLS handshake) + hello attempt
+func (t *TCPChainSink) connect(ctx context.Context) error {
+	nd := net.Dialer{Timeout: t.dialTimeout}
+	if t.config.KeepAlive {
+		nd.KeepAliveConfig = net.KeepAliveConfig{
+			Enable: true,
+			Idle:   time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
+		}
+	}
+
+	var conn net.Conn
+	var err error
+	if t.tlsConfig != nil {
+		// nd.Timeout only bounds the TCP connect; tls.Dialer runs the
+		// handshake under ctx, so bound dial + handshake together here
+		dctx, cancel := context.WithTimeout(ctx, t.dialTimeout+tlsx.HandshakeTimeout)
+		td := tls.Dialer{NetDialer: &nd, Config: t.tlsConfig}
+		conn, err = td.DialContext(dctx, "tcp4", t.addr) // IPv4-only
+		cancel()
+	} else {
+		conn, err = nd.DialContext(ctx, "tcp4", t.addr) // IPv4-only
+	}
+	if err != nil {
+		return err
+	}
+
+	conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
+	if _, err := conn.Write(t.helloLine); err != nil {
+		conn.Close()
+		return fmt.Errorf("hello: %w", err)
+	}
+
+	t.conn = conn
+	t.connected.Store(true)
+	if t.everConnected {
+		t.reconnects.Add(1)
+	}
+	t.everConnected = true
+
+	t.logger.Info("msg", "Chain link established",
+		"component", "tcp_chain_sink",
+		"target", t.addr,
+		"node", t.node,
+		"tls", t.tlsConfig != nil)
+	return nil
+}
+
+// closeConn tears down the current connection (run loop goroutine only)
+func (t *TCPChainSink) closeConn() {
+	if t.conn != nil {
+		t.conn.Close()
+		t.conn = nil
+	}
+	t.connected.Store(false)
+}
+
+// waitBackoff sleeps for the computed delay, interruptible by shutdown
+func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
+	minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
+	maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
+	timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
+	defer timer.Stop()
+
+	select {
+	case <-timer.C:
+		return true
+	case <-ctx.Done():
+		return false
+	case <-t.done:
+		return false
+	}
+}
+
+// backoffDelay computes exponential backoff with ±20% jitter
+func (t *TCPChainSink) backoffDelay(failures int) time.Duration {
+	minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
+	maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
+
+	d := maxD
+	if failures < 63 {
+		if v := minD << uint(failures-1); v > 0 && v < maxD {
+			d = v
+		}
+	}
+	return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
+}

internal/source/console/console.go

@@ -0,0 +1,224 @@
+package console
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSource("console", NewConsoleSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register console source: %v", err))
+	}
+
+	// Console stdin can only have one reader
+	if err := plugin.SetSourceMetadata("console", &plugin.PluginMetadata{
+		Capabilities: []core.Capability{core.CapSessionAware, core.CapSingleInstance},
+		MaxInstances: 1,
+	}); err != nil {
+		panic(fmt.Sprintf("failed to set console source metadata: %v", err))
+	}
+}
+
+// ConsoleSource reads log entries from the standard input stream
+type ConsoleSource struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Configuration
+	config *config.ConsoleSourceOptions
+
+	// Application
+	subscribers []chan core.LogEntry
+	logger      *log.Logger
+
+	// Runtime
+	done chan struct{}
+
+	// Statistics
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	startTime      time.Time
+	lastEntryTime  atomic.Value // time.Time
+}
+
+const (
+	DefaultConsoleSourceBufferSize = 1000
+)
+
+// NewConsoleSourcePlugin creates a console source through plugin factory
+func NewConsoleSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	opts := &config.ConsoleSourceOptions{}
+
+	// Scan config map
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	// Validate and apply defaults
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultConsoleSourceBufferSize
+	}
+
+	// Create and return plugin instance
+	cs := &ConsoleSource{
+		id:          id,
+		proxy:       proxy,
+		config:      opts,
+		subscribers: make([]chan core.LogEntry, 0),
+		done:        make(chan struct{}),
+		logger:      logger,
+	}
+	cs.lastEntryTime.Store(time.Time{})
+
+	// Create session
+	cs.session = proxy.CreateSession(
+		"console_stdin",
+		map[string]any{
+			"instance_id": id,
+			"type":        "console",
+		},
+	)
+
+	cs.logger.Info("msg", "Console source initialized",
+		"component", "console_source",
+		"instance_id", id)
+
+	return cs, nil
+}
+
+// Capabilities returns supported capabilities
+func (s *ConsoleSource) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware, // Single console session
+	}
+}
+
+// Subscribe returns a channel for receiving log entries.
+func (s *ConsoleSource) Subscribe() <-chan core.LogEntry {
+	ch := make(chan core.LogEntry, s.config.BufferSize)
+	s.subscribers = append(s.subscribers, ch)
+	return ch
+}
+
+// Start begins reading from the standard input.
+func (s *ConsoleSource) Start() error {
+	s.startTime = time.Now()
+	go s.readLoop()
+
+	// Update session activity
+	s.proxy.UpdateActivity(s.session.ID)
+
+	s.logger.Info("msg", "Console source started",
+		"component", "console_source",
+		"instance_id", s.id)
+	return nil
+}
+
+// Stop signals the source to stop reading.
+func (s *ConsoleSource) Stop() {
+	close(s.done)
+
+	// Remove session
+	if s.session != nil {
+		s.proxy.RemoveSession(s.session.ID)
+	}
+
+	// Close subscriber channels
+	for _, ch := range s.subscribers {
+		close(ch)
+	}
+
+	s.logger.Info("msg", "Console source stopped",
+		"component", "console_source",
+		"instance_id", s.id)
+}
+
+// GetStats returns the source's statistics
+func (s *ConsoleSource) GetStats() source.SourceStats {
+	lastEntry, _ := s.lastEntryTime.Load().(time.Time)
+
+	return source.SourceStats{
+		Type:           "console",
+		TotalEntries:   s.totalEntries.Load(),
+		DroppedEntries: s.droppedEntries.Load(),
+		StartTime:      s.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        map[string]any{},
+	}
+}
+
+// readLoop continuously reads lines from stdin and publishes them
+func (s *ConsoleSource) readLoop() {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		select {
+		case <-s.done:
+			return
+		default:
+			// Update session activity on each read
+			s.proxy.UpdateActivity(s.session.ID)
+
+			// Get raw line
+			lineBytes := scanner.Bytes()
+			if len(lineBytes) == 0 {
+				continue
+			}
+
+			// Add newline back (scanner strips it)
+			lineWithNewline := append(lineBytes, '\n')
+
+			entry := core.LogEntry{
+				Time:    time.Now(),
+				Source:  "console",
+				Message: string(lineWithNewline), // Keep newline
+				Level:   source.ExtractLogLevel(string(lineBytes)),
+				RawSize: int64(len(lineWithNewline)),
+			}
+
+			s.publish(entry)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		s.logger.Error("msg", "Scanner error reading stdin",
+			"component", "console_source",
+			"instance_id", s.id,
+			"error", err)
+	}
+}
+
+// publish sends a log entry to all subscribers
+func (s *ConsoleSource) publish(entry core.LogEntry) {
+	s.totalEntries.Add(1)
+	s.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range s.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			s.droppedEntries.Add(1)
+			s.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+				"component", "console_source")
+		}
+	}
+}

internal/source/file/file.go

@@ -0,0 +1,372 @@
+package file
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSource("file", NewFileSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register file source: %v", err))
+	}
+}
+
+// FileSource monitors log files and tails them
+type FileSource struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Configuration
+	config *config.FileSourceOptions
+
+	// Application
+	subscribers []chan core.LogEntry
+	watchers    map[string]*fileWatcher
+	logger      *log.Logger
+
+	// Runtime
+	mu     sync.RWMutex
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+
+	// Statistics
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	startTime      time.Time
+	lastEntryTime  atomic.Value // time.Time
+}
+
+const (
+	DefaultFileSourcePattern         = "*"
+	DefaultFileSourceCheckIntervalMS = 100
+	MinFileSourceCheckIntervalMS     = 10
+	DefaultFileSourceFrom            = "end"
+)
+
+// NewFileSourcePlugin creates a file source through plugin factory
+func NewFileSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	opts := &config.FileSourceOptions{}
+
+	// Use lconfig to scan map into struct (overriding defaults)
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	// Validate and apply defaults
+	if err := lconfig.NonEmpty(opts.Directory); err != nil {
+		return nil, fmt.Errorf("directory: %w", err)
+	}
+
+	if opts.Pattern == "" {
+		opts.Pattern = DefaultFileSourcePattern
+	}
+	if opts.CheckIntervalMS <= 0 {
+		opts.CheckIntervalMS = DefaultFileSourceCheckIntervalMS
+	} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
+		return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
+	}
+	if opts.From == "" {
+		opts.From = DefaultFileSourceFrom
+	} else if err := lconfig.OneOf("start", "end")(opts.From); err != nil {
+		return nil, fmt.Errorf("from: %w", err)
+	}
+
+	// Create and return plugin instance
+	fs := &FileSource{
+		id:          id,
+		proxy:       proxy,
+		config:      opts,
+		subscribers: make([]chan core.LogEntry, 0),
+		watchers:    make(map[string]*fileWatcher),
+		logger:      logger,
+	}
+	fs.lastEntryTime.Store(time.Time{})
+
+	fs.session = proxy.CreateSession(
+		fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Pattern),
+		map[string]any{
+			"instance_id": id,
+			"type":        "file",
+			"directory":   opts.Directory,
+			"pattern":     opts.Pattern,
+		},
+	)
+
+	fs.logger.Info("msg", "File source initialized",
+		"component", "file_source",
+		"instance_id", id,
+		"directory", opts.Directory,
+		"pattern", opts.Pattern,
+		"raw", opts.Raw,
+		"from", opts.From)
+
+	return fs, nil
+}
+
+// Capabilities returns supported capabilities
+func (fs *FileSource) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware, // Tracks sessions per file
+		core.CapMultiSession, // Multiple file sessions
+	}
+}
+
+// Subscribe returns a channel for receiving log entries
+func (fs *FileSource) Subscribe() <-chan core.LogEntry {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	ch := make(chan core.LogEntry, 1000)
+	fs.subscribers = append(fs.subscribers, ch)
+	return ch
+}
+
+// Start begins the file monitoring loop
+func (fs *FileSource) Start() error {
+	fs.ctx, fs.cancel = context.WithCancel(context.Background())
+	fs.startTime = time.Now()
+	fs.wg.Add(1)
+	go fs.monitorLoop()
+
+	fs.logger.Info("msg", "File source started",
+		"component", "File_source",
+		"path", fs.config.Directory,
+		"pattern", fs.config.Pattern,
+		"check_interval_ms", fs.config.CheckIntervalMS)
+	return nil
+}
+
+// Stop gracefully shuts down the file source and all file watchers
+func (fs *FileSource) Stop() {
+	if fs.cancel != nil {
+		fs.cancel()
+	}
+	fs.wg.Wait()
+
+	fs.proxy.RemoveSession(fs.session.ID)
+
+	fs.mu.Lock()
+	for _, w := range fs.watchers {
+		w.stop()
+	}
+	for _, ch := range fs.subscribers {
+		close(ch)
+	}
+	fs.mu.Unlock()
+
+	fs.logger.Info("msg", "File source stopped",
+		"component", "file_source",
+		"instance_id", fs.id,
+		"path", fs.config.Directory)
+}
+
+// GetStats returns the source's statistics, including active watchers.
+func (fs *FileSource) GetStats() source.SourceStats {
+	lastEntry, _ := fs.lastEntryTime.Load().(time.Time)
+
+	fs.mu.RLock()
+	watcherCount := int64(len(fs.watchers))
+	details := make(map[string]any)
+
+	// Add watcher details
+	watchers := make([]map[string]any, 0, watcherCount)
+	for _, w := range fs.watchers {
+		info := w.getInfo()
+		watchers = append(watchers, map[string]any{
+			"directory":    info.Directory,
+			"size":         info.Size,
+			"position":     info.Position,
+			"entries_read": info.EntriesRead,
+			"rotations":    info.Rotations,
+			"last_read":    info.LastReadTime,
+		})
+	}
+	details["watchers"] = watchers
+	details["active_watchers"] = watcherCount
+	fs.mu.RUnlock()
+
+	return source.SourceStats{
+		ID:             fs.id,
+		Type:           "file",
+		TotalEntries:   fs.totalEntries.Load(),
+		DroppedEntries: fs.droppedEntries.Load(),
+		StartTime:      fs.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        details,
+	}
+}
+
+// monitorLoop periodically scans path for new or changed files.
+func (fs *FileSource) monitorLoop() {
+	defer fs.wg.Done()
+
+	fs.checkTargets()
+
+	ticker := time.NewTicker(time.Duration(fs.config.CheckIntervalMS) * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-fs.ctx.Done():
+			return
+		case <-ticker.C:
+			fs.checkTargets()
+		}
+	}
+}
+
+// checkTargets finds matching files and ensures watchers are running for them.
+func (fs *FileSource) checkTargets() {
+	files, err := fs.scanFile()
+	if err != nil {
+		fs.logger.Warn("msg", "Failed to scan file",
+			"component", "file_source",
+			"path", fs.config.Directory,
+			"pattern", fs.config.Pattern,
+			"error", err)
+		return
+	}
+
+	for _, file := range files {
+		fs.ensureWatcher(file)
+	}
+
+	fs.cleanupWatchers()
+}
+
+// ensureWatcher creates and starts a new file watcher if one doesn't exist for the given path.
+func (fs *FileSource) ensureWatcher(path string) {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	if _, exists := fs.watchers[path]; exists {
+		return
+	}
+
+	w := newFileWatcher(path, fs.config.Raw, fs.config.From == "start", fs.publish, fs.logger)
+	fs.watchers[path] = w
+
+	fs.logger.Debug("msg", "Created file watcher",
+		"component", "file_source",
+		"path", path)
+
+	fs.wg.Add(1)
+	go func() {
+		defer fs.wg.Done()
+		if err := w.watch(fs.ctx); err != nil {
+			if errors.Is(err, context.Canceled) {
+				fs.logger.Debug("msg", "Watcher cancelled",
+					"component", "file_source",
+					"path", path)
+			} else {
+				fs.logger.Error("msg", "Watcher failed",
+					"component", "file_source",
+					"path", path,
+					"error", err)
+			}
+		}
+
+		fs.mu.Lock()
+		delete(fs.watchers, path)
+		fs.mu.Unlock()
+	}()
+}
+
+// cleanupWatchers stops and removes watchers for files that no longer exist.
+func (fs *FileSource) cleanupWatchers() {
+	fs.mu.Lock()
+	defer fs.mu.Unlock()
+
+	for path, w := range fs.watchers {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			w.stop()
+			delete(fs.watchers, path)
+			fs.logger.Debug("msg", "Cleaned up watcher for non-existent file",
+				"component", "file_source",
+				"path", path)
+		}
+	}
+}
+
+// publish sends a log entry to all subscribers.
+func (fs *FileSource) publish(entry core.LogEntry) {
+	fs.mu.RLock()
+	defer fs.mu.RUnlock()
+
+	fs.totalEntries.Add(1)
+	fs.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range fs.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			fs.droppedEntries.Add(1)
+			fs.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+				"component", "file_source")
+		}
+	}
+}
+
+// scanFile finds all files in the configured path that match the pattern.
+func (fs *FileSource) scanFile() ([]string, error) {
+	entries, err := os.ReadDir(fs.config.Directory)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert glob pattern to regex
+	regexPattern := globToRegex(fs.config.Pattern)
+	re, err := regexp.Compile(regexPattern)
+	if err != nil {
+		return nil, fmt.Errorf("invalid pattern regex: %w", err)
+	}
+
+	var files []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		name := entry.Name()
+		if re.MatchString(name) {
+			files = append(files, filepath.Join(fs.config.Directory, name))
+		}
+	}
+
+	return files, nil
+}
+
+// globToRegex converts a simple glob pattern to a regular expression.
+func globToRegex(glob string) string {
+	regex := regexp.QuoteMeta(glob)
+	regex = strings.ReplaceAll(regex, `\*`, `.*`)
+	regex = strings.ReplaceAll(regex, `\?`, `.`)
+	return "^" + regex + "$"
+}
+

internal/source/file/file_watcher.go

@@ -1,5 +1,4 @@
-// FILE: logwisp/src/internal/source/file_watcher.go
+package file
-package source
 
 import (
 	"bufio"
@@ -9,20 +8,20 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"syscall"
 	"time"
 
-	"logwisp/src/internal/core"
+	"logwisp/internal/core"
+	"logwisp/internal/source"
 
 	"github.com/lixenwraith/log"
 )
 
-// Contains information about a file watcher
+// WatcherInfo contains snapshot information about a file watcher's state
 type WatcherInfo struct {
-	Path         string
+	Directory    string
 	Size         int64
 	Position     int64
 	ModTime      time.Time
@@ -31,9 +30,11 @@ type WatcherInfo struct {
 	Rotations    int64
 }
 
+// fileWatcher tails a single file, handles rotations, and sends new lines to a callback
 type fileWatcher struct {
-	path         string
+	directory    string
 	callback     func(core.LogEntry)
+	raw          bool
 	position     int64
 	size         int64
 	inode        uint64
@@ -46,23 +47,31 @@ type fileWatcher struct {
 	logger       *log.Logger
 }
 
-func newFileWatcher(path string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
+// newFileWatcher creates a new watcher for a specific file path.
+// A start position of 0 reads an existing file whole; -1 seeks to its end.
+func newFileWatcher(directory string, raw, fromStart bool, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
+	position := int64(-1)
+	if fromStart {
+		position = 0
+	}
 	w := &fileWatcher{
-		path:     path,
+		directory: directory,
-		callback: callback,
+		callback:  callback,
-		position: -1,
+		raw:       raw,
-		logger:   logger,
+		position:  position,
+		logger:    logger,
 	}
 	w.lastReadTime.Store(time.Time{})
 	return w
 }
 
+// watch starts the main monitoring loop for the file
 func (w *fileWatcher) watch(ctx context.Context) error {
-	if err := w.seekToEnd(); err != nil {
+	if err := w.initPosition(); err != nil {
-		return fmt.Errorf("seekToEnd failed: %w", err)
+		return fmt.Errorf("initPosition failed: %w", err)
 	}
 
-	ticker := time.NewTicker(100 * time.Millisecond)
+	ticker := time.NewTicker(core.FileWatcherPollInterval)
 	defer ticker.Stop()
 
 	for {
@@ -81,51 +90,36 @@ func (w *fileWatcher) watch(ctx context.Context) error {
 	}
 }
 
-func (w *fileWatcher) seekToEnd() error {
+// stop signals the watcher to terminate its loop
-	file, err := os.Open(w.path)
+func (w *fileWatcher) stop() {
-	if err != nil {
-		if os.IsNotExist(err) {
-			w.mu.Lock()
-			w.position = 0
-			w.size = 0
-			w.modTime = time.Now()
-			w.inode = 0
-			w.mu.Unlock()
-			return nil
-		}
-		return err
-	}
-	defer file.Close()
-
-	info, err := file.Stat()
-	if err != nil {
-		return err
-	}
-
 	w.mu.Lock()
-	defer w.mu.Unlock()
+	w.stopped = true
-
+	w.mu.Unlock()
-	// Keep existing position (including 0)
-	// First time initialization seeks to the end of the file
-	if w.position == -1 {
-		pos, err := file.Seek(0, io.SeekEnd)
-		if err != nil {
-			return err
-		}
-		w.position = pos
-	}
-
-	w.size = info.Size()
-	w.modTime = info.ModTime()
-	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		w.inode = stat.Ino
-	}
-
-	return nil
 }
 
+// getInfo returns a snapshot of the watcher's current statistics
+func (w *fileWatcher) getInfo() WatcherInfo {
+	w.mu.Lock()
+	info := WatcherInfo{
+		Directory:   w.directory,
+		Size:        w.size,
+		Position:    w.position,
+		ModTime:     w.modTime,
+		EntriesRead: w.entriesRead.Load(),
+		Rotations:   w.rotationSeq,
+	}
+	w.mu.Unlock()
+
+	if lastRead, ok := w.lastReadTime.Load().(time.Time); ok {
+		info.LastReadTime = lastRead
+	}
+
+	return info
+}
+
+// checkFile examines the file for changes, rotations, or new content
 func (w *fileWatcher) checkFile() error {
-	file, err := os.Open(w.path)
+	file, err := os.Open(w.directory)
 	if err != nil {
 		if os.IsNotExist(err) {
 			// File doesn't exist yet, keep watching
@@ -133,7 +127,7 @@ func (w *fileWatcher) checkFile() error {
 		}
 		w.logger.Error("msg", "Failed to open file for checking",
 			"component", "file_watcher",
-			"path", w.path,
+			"directory", w.directory,
 			"error", err)
 		return err
 	}
@@ -143,7 +137,7 @@ func (w *fileWatcher) checkFile() error {
 	if err != nil {
 		w.logger.Error("msg", "Failed to stat file",
 			"component", "file_watcher",
-			"path", w.path,
+			"directory", w.directory,
 			"error", err)
 		return err
 	}
@@ -213,7 +207,7 @@ func (w *fileWatcher) checkFile() error {
 
 			w.logger.Debug("msg", "Atomic file update detected",
 				"component", "file_watcher",
-				"path", w.path,
+				"directory", w.directory,
 				"old_inode", oldInode,
 				"new_inode", currentInode,
 				"position", oldPos,
@@ -232,26 +226,26 @@ func (w *fileWatcher) checkFile() error {
 
 		w.callback(core.LogEntry{
 			Time:    time.Now(),
-			Source:  filepath.Base(w.path),
+			Source:  filepath.Base(w.directory),
 			Level:   "INFO",
 			Message: fmt.Sprintf("Log rotation detected (#%d): %s", seq, rotationReason),
 		})
 
 		w.logger.Info("msg", "Log rotation detected",
 			"component", "file_watcher",
-			"path", w.path,
+			"directory", w.directory,
 			"sequence", seq,
 			"reason", rotationReason)
 	}
 
-	// Only read if there's new content
+	// Read if there's new content OR if we need to continue from position
 	if currentSize > startPos {
 		if _, err := file.Seek(startPos, io.SeekStart); err != nil {
 			return err
 		}
 
 		scanner := bufio.NewScanner(file)
-		scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+		scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
 
 		for scanner.Scan() {
 			line := scanner.Text()
@@ -271,7 +265,7 @@ func (w *fileWatcher) checkFile() error {
 		if err := scanner.Err(); err != nil {
 			w.logger.Error("msg", "Scanner error while reading file",
 				"component", "file_watcher",
-				"path", w.path,
+				"directory", w.directory,
 				"position", startPos,
 				"error", err)
 			return err
@@ -310,94 +304,117 @@ func (w *fileWatcher) checkFile() error {
 	return nil
 }
 
-func (w *fileWatcher) parseLine(line string) core.LogEntry {
+// initPosition records the file's metadata and, unless the watcher was created
-	var jsonLog struct {
+// to read from the start, sets the initial read position to the end
-		Time    string          `json:"time"`
+func (w *fileWatcher) initPosition() error {
-		Level   string          `json:"level"`
+	file, err := os.Open(w.directory)
-		Message string          `json:"msg"`
+	if err != nil {
-		Fields  json.RawMessage `json:"fields"`
+		if os.IsNotExist(err) {
+			w.mu.Lock()
+			w.position = 0
+			w.size = 0
+			w.modTime = time.Now()
+			w.inode = 0
+			w.mu.Unlock()
+			return nil
+		}
+		return err
+	}
+	defer file.Close()
+
+	info, err := file.Stat()
+	if err != nil {
+		return err
 	}
 
-	if err := json.Unmarshal([]byte(line), &jsonLog); err == nil {
+	w.mu.Lock()
-		timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time)
+	defer w.mu.Unlock()
+
+	if w.position == -1 {
+		pos, err := file.Seek(0, io.SeekEnd)
 		if err != nil {
-			timestamp = time.Now()
+			return err
-		}
-
-		return core.LogEntry{
-			Time:    timestamp,
-			Source:  filepath.Base(w.path),
-			Level:   jsonLog.Level,
-			Message: jsonLog.Message,
-			Fields:  jsonLog.Fields,
 		}
+		w.position = pos
 	}
 
-	level := extractLogLevel(line)
+	w.size = info.Size()
-
+	w.modTime = info.ModTime()
-	return core.LogEntry{
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		Time:    time.Now(),
+		w.inode = stat.Ino
-		Source:  filepath.Base(w.path),
-		Level:   level,
-		Message: line,
 	}
+
+	return nil
 }
 
-func extractLogLevel(line string) string {
+// isStopped checks if the watcher has been instructed to stop
-	patterns := []struct {
-		patterns []string
-		level    string
-	}{
-		{[]string{"[ERROR]", "ERROR:", " ERROR ", "ERR:", "[ERR]", "FATAL:", "[FATAL]"}, "ERROR"},
-		{[]string{"[WARN]", "WARN:", " WARN ", "WARNING:", "[WARNING]"}, "WARN"},
-		{[]string{"[INFO]", "INFO:", " INFO ", "[INF]", "INF:"}, "INFO"},
-		{[]string{"[DEBUG]", "DEBUG:", " DEBUG ", "[DBG]", "DBG:"}, "DEBUG"},
-		{[]string{"[TRACE]", "TRACE:", " TRACE "}, "TRACE"},
-	}
-
-	upperLine := strings.ToUpper(line)
-	for _, group := range patterns {
-		for _, pattern := range group.patterns {
-			if strings.Contains(upperLine, pattern) {
-				return group.level
-			}
-		}
-	}
-
-	return ""
-}
-
-func (w *fileWatcher) getInfo() WatcherInfo {
-	w.mu.Lock()
-	info := WatcherInfo{
-		Path:        w.path,
-		Size:        w.size,
-		Position:    w.position,
-		ModTime:     w.modTime,
-		EntriesRead: w.entriesRead.Load(),
-		Rotations:   w.rotationSeq,
-	}
-	w.mu.Unlock()
-
-	if lastRead, ok := w.lastReadTime.Load().(time.Time); ok {
-		info.LastReadTime = lastRead
-	}
-
-	return info
-}
-
-func (w *fileWatcher) close() {
-	w.stop()
-}
-
-func (w *fileWatcher) stop() {
-	w.mu.Lock()
-	w.stopped = true
-	w.mu.Unlock()
-}
-
 func (w *fileWatcher) isStopped() bool {
 	w.mu.Lock()
 	defer w.mu.Unlock()
 	return w.stopped
 }
+
+// parseLine converts a line into an entry, as JSON when nothing would be lost
+func (w *fileWatcher) parseLine(line string) core.LogEntry {
+	if w.raw {
+		// Newline restored: sinks write the payload as it stands
+		return core.LogEntry{
+			Time:    time.Now(),
+			Source:  filepath.Base(w.directory),
+			Level:   source.ExtractLogLevel(line),
+			Message: line + "\n",
+		}
+	}
+
+	if entry, ok := w.parseJSON(line); ok {
+		return entry
+	}
+
+	return core.LogEntry{
+		Time:    time.Now(),
+		Source:  filepath.Base(w.directory),
+		Level:   source.ExtractLogLevel(line),
+		Message: line,
+	}
+}
+
+// parseJSON decodes a line into the entry envelope. A top-level key LogEntry
+// cannot carry refuses the whole line, so a richer record reaches the pipeline
+// as text rather than silently reduced to the four keys kept here.
+func (w *fileWatcher) parseJSON(line string) (core.LogEntry, bool) {
+	if len(line) == 0 || line[0] != '{' {
+		return core.LogEntry{}, false
+	}
+
+	var obj map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(line), &obj); err != nil || len(obj) == 0 {
+		return core.LogEntry{}, false
+	}
+
+	entry := core.LogEntry{Time: time.Now(), Source: filepath.Base(w.directory)}
+	for key, val := range obj {
+		var err error
+		switch key {
+		case "time":
+			var ts string
+			if json.Unmarshal(val, &ts) == nil {
+				if t, terr := time.Parse(time.RFC3339Nano, ts); terr == nil {
+					entry.Time = t
+				}
+			}
+		case "level":
+			err = json.Unmarshal(val, &entry.Level)
+		case "msg":
+			err = json.Unmarshal(val, &entry.Message)
+		case "fields":
+			entry.Fields = val
+		default:
+			return core.LogEntry{}, false
+		}
+		if err != nil {
+			return core.LogEntry{}, false
+		}
+	}
+
+	return entry, true
+}

internal/source/httpchain/httpchain.go

@@ -0,0 +1,412 @@
+package httpchain
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"maps"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/authz"
+	"logwisp/internal/chain"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+	"logwisp/internal/tlsx"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+func init() {
+	if err := plugin.RegisterSource("http_chain", NewHTTPChainSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register http_chain source: %v", err))
+	}
+}
+
+const (
+	DefaultHTTPChainSourceBufferSize    = 1000
+	DefaultHTTPChainSourceIngestPath    = "/ingest"
+	DefaultHTTPChainSourceMaxBodyBytes  = 8 * 1024 * 1024
+	DefaultHTTPChainSourceReadTimeoutMS = 30000
+	HTTPChainReadHeaderTimeout          = 10 * time.Second
+	HTTPChainServerShutdownTimeout      = 2 * time.Second
+)
+
+// HTTPChainSource accepts NDJSON batches from upstream http_chain sinks
+type HTTPChainSource struct {
+	id     string
+	proxy  *session.Proxy
+	config *config.HTTPChainSourceOptions
+
+	subscribers []chan core.LogEntry
+	server      *http.Server
+	logger      *log.Logger
+
+	// TLS
+	tlsConfig *tls.Config
+
+	// Authorization
+	auth *authz.Policy
+
+	// Session cache: one session per remote host + node + authenticated identity
+	sessions   map[string]string // key -> sessionID
+	sessionsMu sync.Mutex
+
+	mu sync.RWMutex
+
+	startTime        time.Time
+	totalEntries     atomic.Uint64
+	droppedEntries   atomic.Uint64
+	parseErrors      atomic.Uint64
+	totalRequests    atomic.Uint64
+	rejectedRequests atomic.Uint64
+	lastEntryTime    atomic.Value // time.Time
+}
+
+// NewHTTPChainSourcePlugin creates an http_chain source through plugin factory
+func NewHTTPChainSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	opts := &config.HTTPChainSourceOptions{
+		Host:      "0.0.0.0",
+		TrustNode: true,
+	}
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+	if err := lconfig.Port(opts.Port); err != nil {
+		return nil, fmt.Errorf("port: %w", err)
+	}
+	if opts.IngestPath == "" {
+		opts.IngestPath = DefaultHTTPChainSourceIngestPath
+	} else if !strings.HasPrefix(opts.IngestPath, "/") {
+		return nil, fmt.Errorf("ingest_path: must start with '/'")
+	}
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultHTTPChainSourceBufferSize
+	}
+	if opts.MaxBodyBytes <= 0 {
+		opts.MaxBodyBytes = DefaultHTTPChainSourceMaxBodyBytes
+	}
+	if opts.ReadTimeoutMS <= 0 {
+		opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS
+	}
+	tlsCfg, err := tlsx.Server(opts.TLS)
+	if err != nil {
+		return nil, err
+	}
+	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleChainListener)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &HTTPChainSource{
+		id:          id,
+		proxy:       proxy,
+		config:      opts,
+		subscribers: make([]chan core.LogEntry, 0),
+		sessions:    make(map[string]string),
+		logger:      logger,
+		tlsConfig:   tlsCfg,
+		auth:        authPolicy,
+	}
+	s.lastEntryTime.Store(time.Time{})
+
+	logger.Info("msg", "HTTP chain source initialized",
+		"component", "http_chain_source",
+		"instance_id", id,
+		"host", opts.Host,
+		"port", opts.Port,
+		"ingest_path", opts.IngestPath,
+		"tls", tlsCfg != nil,
+		"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
+		"auth", authPolicy.Describe())
+	if authPolicy.Unrestricted() {
+		logger.Warn("msg", "Auth policy admits any identity the configured CA vouches for",
+			"component", "http_chain_source",
+			"instance_id", id,
+			"hint", "set auth.allow or auth.allow_patterns to authorize named peers")
+	}
+	if authPolicy.BindsNode() {
+		logger.Info("msg", "Node labels bound to peer identity; trust_node is ignored",
+			"component", "http_chain_source",
+			"instance_id", id,
+			"node_binding", authPolicy.NodeBinding(),
+			"trust_node", opts.TrustNode)
+	}
+	return s, nil
+}
+
+// Capabilities returns supported capabilities
+func (s *HTTPChainSource) Capabilities() []core.Capability {
+	caps := []core.Capability{core.CapSessionAware, core.CapMultiSession}
+	if s.tlsConfig != nil {
+		caps = append(caps, core.CapTLS)
+	}
+	if s.auth.Enabled() {
+		caps = append(caps, core.CapAuth) // authorizes peers, not just the CA
+	}
+	return caps
+}
+
+// Subscribe returns a channel for receiving log entries
+func (s *HTTPChainSource) Subscribe() <-chan core.LogEntry {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	ch := make(chan core.LogEntry, s.config.BufferSize)
+	s.subscribers = append(s.subscribers, ch)
+	return ch
+}
+
+// Start binds the listener and serves the ingest endpoint
+func (s *HTTPChainSource) Start() error {
+	addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
+	// IPv4-only, aligns with tcp/http sinks
+	ln, err := net.Listen("tcp4", addr)
+	if err != nil {
+		return fmt.Errorf("listen %s: %w", addr, err)
+	}
+
+	mux := http.NewServeMux()
+	// Method-scoped pattern: mux answers 405 with Allow header on non-POST
+	mux.HandleFunc(http.MethodPost+" "+s.config.IngestPath, s.handleIngest)
+
+	s.server = &http.Server{
+		Handler:           mux,
+		ReadTimeout:       time.Duration(s.config.ReadTimeoutMS) * time.Millisecond,
+		ReadHeaderTimeout: HTTPChainReadHeaderTimeout,
+		// TLS handshake bounded by min(ReadTimeout, ReadHeaderTimeout)
+		ErrorLog: tlsx.HTTPErrorLog(s.logger, "http_chain_source"),
+	}
+	s.startTime = time.Now()
+
+	serve := s.server.Serve
+	if s.tlsConfig != nil {
+		s.server.TLSConfig = s.tlsConfig
+		serve = func(l net.Listener) error { return s.server.ServeTLS(l, "", "") }
+	}
+
+	go func() {
+		if err := serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			s.logger.Error("msg", "HTTP chain server terminated",
+				"component", "http_chain_source",
+				"instance_id", s.id,
+				"error", err)
+		}
+	}()
+
+	s.logger.Info("msg", "HTTP chain source started",
+		"component", "http_chain_source",
+		"instance_id", s.id,
+		"addr", addr)
+	return nil
+}
+
+// Stop shuts down the server, sessions, and subscriber channels
+func (s *HTTPChainSource) Stop() {
+	if s.server != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), HTTPChainServerShutdownTimeout)
+		defer cancel()
+		s.server.Shutdown(ctx)
+	}
+
+	s.sessionsMu.Lock()
+	for _, id := range s.sessions {
+		s.proxy.RemoveSession(id)
+	}
+	s.sessions = make(map[string]string)
+	s.sessionsMu.Unlock()
+
+	s.mu.Lock()
+	for _, ch := range s.subscribers {
+		close(ch)
+	}
+	s.mu.Unlock()
+
+	s.logger.Info("msg", "HTTP chain source stopped",
+		"component", "http_chain_source",
+		"instance_id", s.id)
+}
+
+// GetStats returns the source's statistics
+func (s *HTTPChainSource) GetStats() source.SourceStats {
+	lastEntry, _ := s.lastEntryTime.Load().(time.Time)
+
+	s.sessionsMu.Lock()
+	cachedSessions := len(s.sessions)
+	s.sessionsMu.Unlock()
+
+	details := map[string]any{
+		"host":              s.config.Host,
+		"port":              s.config.Port,
+		"ingest_path":       s.config.IngestPath,
+		"tls":               s.tlsConfig != nil,
+		"total_requests":    s.totalRequests.Load(),
+		"rejected_requests": s.rejectedRequests.Load(),
+		"parse_errors":      s.parseErrors.Load(),
+		"cached_sessions":   cachedSessions,
+		"trust_node":        s.config.TrustNode,
+	}
+	maps.Copy(details, s.auth.Stats())
+
+	return source.SourceStats{
+		ID:             s.id,
+		Type:           "http_chain",
+		TotalEntries:   s.totalEntries.Load(),
+		DroppedEntries: s.droppedEntries.Load(),
+		StartTime:      s.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        details,
+	}
+}
+
+// handleIngest validates protocol headers and ingests one NDJSON batch.
+// Batch acceptance is atomic: entries publish only after a clean full read.
+func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) {
+	s.totalRequests.Add(1)
+
+	// Authorize before the body is read: an unauthorized sender should not get
+	// to stream max_body_bytes into the process. 403 is distinct from the 400
+	// used for protocol errors, so a sender can tell "not allowed" from
+	// "malformed batch".
+	ident, err := s.auth.Authorize(r.TLS)
+	if err != nil {
+		s.rejectedRequests.Add(1)
+		s.logger.Warn("msg", "Request rejected by auth policy",
+			"component", "http_chain_source",
+			"instance_id", s.id,
+			"remote_addr", r.RemoteAddr,
+			"error", err)
+		http.Error(w, "forbidden", http.StatusForbidden)
+		return
+	}
+
+	if r.Header.Get(chain.HeaderProtocol) != strconv.Itoa(chain.ProtocolVersion) {
+		s.rejectedRequests.Add(1)
+		http.Error(w, "unsupported protocol version", http.StatusBadRequest)
+		return
+	}
+
+	remoteHost := r.RemoteAddr
+	if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
+		remoteHost = host
+	}
+	declaredNode := r.Header.Get(chain.HeaderNode)
+	connNode, err := s.auth.ResolveNode(declaredNode, remoteHost, s.config.TrustNode, ident)
+	if err != nil {
+		s.rejectedRequests.Add(1)
+		s.logger.Warn("msg", "Request rejected by node binding",
+			"component", "http_chain_source",
+			"instance_id", s.id,
+			"remote_addr", r.RemoteAddr,
+			"declared_node", declaredNode,
+			"error", err)
+		http.Error(w, "forbidden", http.StatusForbidden)
+		return
+	}
+	// force relabels every entry, so a sender cannot smuggle a foreign origin
+	// through the per-entry node field either
+	trustEntryNode := s.auth.TrustsEntryNode(s.config.TrustNode)
+
+	body := http.MaxBytesReader(w, r.Body, s.config.MaxBodyBytes)
+	scanner := bufio.NewScanner(body)
+	scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
+
+	entries := make([]core.LogEntry, 0, 128)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+		entry, err := chain.DecodeEntry(line, connNode, trustEntryNode)
+		if err != nil {
+			// Content error within a clean transfer: skip line, keep batch
+			s.parseErrors.Add(1)
+			continue
+		}
+		entries = append(entries, entry)
+	}
+	if err := scanner.Err(); err != nil {
+		// Transfer error: reject batch without partial ingestion, sender retries
+		s.rejectedRequests.Add(1)
+		var maxErr *http.MaxBytesError
+		if errors.As(err, &maxErr) {
+			http.Error(w, "body too large", http.StatusRequestEntityTooLarge)
+			return
+		}
+		s.logger.Debug("msg", "Chain batch read failed",
+			"component", "http_chain_source",
+			"remote_addr", r.RemoteAddr,
+			"error", err)
+		http.Error(w, "malformed body", http.StatusBadRequest)
+		return
+	}
+
+	for _, entry := range entries {
+		s.publish(entry)
+	}
+	s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode, r.TLS, ident))
+
+	w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries)))
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// sessionFor returns the cached session for a remote+node+identity,
+// recreating after idle expiry. Identity is part of the key so two peers
+// sharing a remote address never share a session.
+func (s *HTTPChainSource) sessionFor(remoteHost, node string, cs *tls.ConnectionState, ident authz.Identity) string {
+	key := remoteHost + "|" + node + "|" + ident.Name
+	s.sessionsMu.Lock()
+	defer s.sessionsMu.Unlock()
+
+	if id, ok := s.sessions[key]; ok {
+		if _, exists := s.proxy.GetSession(id); exists {
+			return id
+		}
+	}
+	meta := map[string]any{
+		"type": "http_chain",
+		"node": node,
+	}
+	if cs != nil {
+		meta["tls"] = true
+		if cn := tlsx.PeerCN(*cs); cn != "" {
+			meta["tls_peer_cn"] = cn
+		}
+	}
+	ident.Apply(meta)
+	sess := s.proxy.CreateSession(remoteHost, meta)
+	s.sessions[key] = sess.ID
+	return sess.ID
+}
+
+// publish sends a log entry to all subscribers
+func (s *HTTPChainSource) publish(entry core.LogEntry) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	s.totalEntries.Add(1)
+	s.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range s.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			s.droppedEntries.Add(1)
+		}
+	}
+}

internal/source/null/null.go

@@ -0,0 +1,125 @@
+package null
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSource("null", NewNullSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register null source: %v", err))
+	}
+}
+
+// NullSource generates no log entries, used for testing
+type NullSource struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Application
+	subscribers []chan core.LogEntry
+	logger      *log.Logger
+
+	// Runtime
+	done chan struct{}
+
+	// Statistics
+	totalEntries  atomic.Uint64
+	startTime     time.Time
+	lastEntryTime atomic.Value // time.Time
+}
+
+// NewNullSourcePlugin creates a null source through plugin factory
+func NewNullSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	ns := &NullSource{
+		id:          id,
+		proxy:       proxy,
+		subscribers: make([]chan core.LogEntry, 0),
+		done:        make(chan struct{}),
+		logger:      logger,
+	}
+	ns.lastEntryTime.Store(time.Time{})
+
+	// Create session for null source
+	ns.session = proxy.CreateSession(
+		"null://void",
+		map[string]any{
+			"instance_id": id,
+			"type":        "null",
+		},
+	)
+
+	logger.Debug("msg", "Null source initialized",
+		"component", "null_source",
+		"instance_id", id)
+
+	return ns, nil
+}
+
+// Capabilities returns supported capabilities
+func (ns *NullSource) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware,
+	}
+}
+
+// Subscribe returns a channel for receiving log entries
+func (ns *NullSource) Subscribe() <-chan core.LogEntry {
+	ch := make(chan core.LogEntry, 1000)
+	ns.subscribers = append(ns.subscribers, ch)
+	return ch
+}
+
+// Start begins the source operation (no-op for null source)
+func (ns *NullSource) Start() error {
+	ns.startTime = time.Now()
+	ns.proxy.UpdateActivity(ns.session.ID)
+	ns.logger.Debug("msg", "Null source started",
+		"component", "null_source",
+		"instance_id", ns.id)
+	return nil
+}
+
+// Stop signals the source to stop
+func (ns *NullSource) Stop() {
+	close(ns.done)
+	if ns.session != nil {
+		ns.proxy.RemoveSession(ns.session.ID)
+	}
+	for _, ch := range ns.subscribers {
+		close(ch)
+	}
+	ns.logger.Debug("msg", "Null source stopped",
+		"component", "null_source",
+		"instance_id", ns.id)
+}
+
+// GetStats returns the source's statistics
+func (ns *NullSource) GetStats() source.SourceStats {
+	lastEntry, _ := ns.lastEntryTime.Load().(time.Time)
+
+	return source.SourceStats{
+		ID:            ns.id,
+		Type:          "null",
+		TotalEntries:  ns.totalEntries.Load(),
+		StartTime:     ns.startTime,
+		LastEntryTime: lastEntry,
+		Details:       map[string]any{},
+	}
+}

internal/source/random/random.go

@@ -0,0 +1,358 @@
+package random
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+// init registers the component in plugin factory
+func init() {
+	if err := plugin.RegisterSource("random", NewRandomSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register random source: %v", err))
+	}
+}
+
+// RandomSource generates random log entries for testing
+type RandomSource struct {
+	// Plugin identity and session management
+	id      string
+	proxy   *session.Proxy
+	session *session.Session
+
+	// Configuration
+	config *config.RandomSourceOptions
+
+	// Application
+	subscribers []chan core.LogEntry
+	logger      *log.Logger
+	rng         *rand.Rand
+	mu          sync.RWMutex
+
+	// Runtime
+	done   chan struct{}
+	wg     sync.WaitGroup
+	cancel chan struct{}
+
+	// Statistics
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	startTime      time.Time
+	lastEntryTime  atomic.Value // time.Time
+}
+
+const (
+	DefaultRandomSourceIntervalMS = 500
+	DefaultRandomSourceFormat     = "txt"
+	DefaultRandomSourceLength     = 20
+)
+
+// NewRandomSourcePlugin creates a random source through plugin factory
+func NewRandomSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	// Step 1: Create empty config struct with defaults
+	opts := &config.RandomSourceOptions{
+		IntervalMS: 500,
+		JitterMS:   0,
+		Format:     "txt",
+		Length:     20,
+		Special:    false,
+	}
+
+	// Scan config map
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	// Defaults
+	if opts.IntervalMS <= 0 {
+		opts.IntervalMS = DefaultRandomSourceIntervalMS
+	}
+	if opts.Format == "" {
+		opts.Format = DefaultRandomSourceFormat
+	}
+	if opts.Length <= 0 {
+		opts.Length = DefaultRandomSourceLength
+	}
+
+	// Validate
+	if opts.JitterMS < 0 {
+		return nil, fmt.Errorf("jitter_ms cannot be negative")
+	}
+	if opts.JitterMS > opts.IntervalMS {
+		opts.JitterMS = opts.IntervalMS
+	}
+
+	validateFormat := lconfig.OneOf("raw", "txt", "json")
+	if err := validateFormat(opts.Format); err != nil {
+		return nil, fmt.Errorf("format: %w", err)
+	}
+
+	rs := &RandomSource{
+		id:          id,
+		proxy:       proxy,
+		config:      opts,
+		subscribers: make([]chan core.LogEntry, 0),
+		done:        make(chan struct{}),
+		cancel:      make(chan struct{}),
+		logger:      logger,
+		rng:         rand.New(rand.NewSource(time.Now().UnixNano())),
+	}
+	rs.lastEntryTime.Store(time.Time{})
+
+	// Create session for random source
+	rs.session = proxy.CreateSession(
+		fmt.Sprintf("random://%s", id),
+		map[string]any{
+			"instance_id": id,
+			"type":        "random",
+			"format":      opts.Format,
+			"interval_ms": opts.IntervalMS,
+		},
+	)
+
+	logger.Debug("msg", "Random source initialized",
+		"component", "random_source",
+		"instance_id", id,
+		"format", opts.Format,
+		"interval_ms", opts.IntervalMS,
+		"jitter_ms", opts.JitterMS)
+
+	return rs, nil
+}
+
+// Capabilities returns supported capabilities
+func (rs *RandomSource) Capabilities() []core.Capability {
+	return []core.Capability{
+		core.CapSessionAware,
+	}
+}
+
+// Subscribe returns a channel for receiving log entries
+func (rs *RandomSource) Subscribe() <-chan core.LogEntry {
+	rs.mu.Lock()
+	defer rs.mu.Unlock()
+	ch := make(chan core.LogEntry, 1000)
+	rs.subscribers = append(rs.subscribers, ch)
+	return ch
+}
+
+// Start begins generating random log entries
+func (rs *RandomSource) Start() error {
+	rs.startTime = time.Now()
+	rs.wg.Add(1)
+	go rs.generateLoop()
+
+	rs.proxy.UpdateActivity(rs.session.ID)
+	rs.logger.Debug("msg", "Random source started",
+		"component", "random_source",
+		"instance_id", rs.id)
+	return nil
+}
+
+// Stop signals the source to stop generating
+func (rs *RandomSource) Stop() {
+	close(rs.cancel)
+	rs.wg.Wait()
+
+	if rs.session != nil {
+		rs.proxy.RemoveSession(rs.session.ID)
+	}
+
+	rs.mu.Lock()
+	for _, ch := range rs.subscribers {
+		close(ch)
+	}
+	rs.mu.Unlock()
+
+	rs.logger.Debug("msg", "Random source stopped",
+		"component", "random_source",
+		"instance_id", rs.id,
+		"total_entries", rs.totalEntries.Load())
+}
+
+// GetStats returns the source's statistics
+func (rs *RandomSource) GetStats() source.SourceStats {
+	lastEntry, _ := rs.lastEntryTime.Load().(time.Time)
+
+	return source.SourceStats{
+		ID:             rs.id,
+		Type:           "random",
+		TotalEntries:   rs.totalEntries.Load(),
+		DroppedEntries: rs.droppedEntries.Load(),
+		StartTime:      rs.startTime,
+		LastEntryTime:  lastEntry,
+		Details: map[string]any{
+			"format":      rs.config.Format,
+			"interval_ms": rs.config.IntervalMS,
+			"jitter_ms":   rs.config.JitterMS,
+			"length":      rs.config.Length,
+			"special":     rs.config.Special,
+		},
+	}
+}
+
+// generateLoop continuously generates random log entries at configured intervals
+func (rs *RandomSource) generateLoop() {
+	defer rs.wg.Done()
+
+	for {
+		// Calculate next interval with jitter
+		interval := time.Duration(rs.config.IntervalMS) * time.Millisecond
+		if rs.config.JitterMS > 0 {
+			jitter := time.Duration(rs.rng.Intn(int(rs.config.JitterMS))) * time.Millisecond
+			interval = interval - time.Duration(rs.config.JitterMS/2)*time.Millisecond + jitter
+		}
+
+		select {
+		case <-time.After(interval):
+			entry := rs.generateEntry()
+			rs.publish(entry)
+			rs.proxy.UpdateActivity(rs.session.ID)
+		case <-rs.cancel:
+			return
+		case <-rs.done:
+			return
+		}
+	}
+}
+
+// generateEntry creates a random log entry based on configured format
+func (rs *RandomSource) generateEntry() core.LogEntry {
+	now := time.Now()
+
+	switch rs.config.Format {
+	case "raw":
+		message := rs.generateRandomString(int(rs.config.Length))
+		return core.LogEntry{
+			Time:    now,
+			Source:  fmt.Sprintf("random_%s", rs.id),
+			Message: message,
+			RawSize: int64(len(message) + 1), // +1 for newline
+		}
+
+	case "txt":
+		level := rs.randomLogLevel()
+		message := rs.generateRandomString(int(rs.config.Length))
+		formatted := fmt.Sprintf("[%s] [%s] random_%s - %s",
+			now.Format(time.RFC3339),
+			level,
+			rs.id,
+			message)
+		return core.LogEntry{
+			Time:    now,
+			Source:  fmt.Sprintf("random_%s", rs.id),
+			Level:   level,
+			Message: formatted,
+			RawSize: int64(len(formatted) + 1),
+		}
+
+	case "json":
+		level := rs.randomLogLevel()
+		message := rs.generateRandomString(int(rs.config.Length))
+		data := map[string]any{
+			"time":    now.Format(time.RFC3339Nano),
+			"level":   level,
+			"source":  fmt.Sprintf("random_%s", rs.id),
+			"message": message,
+		}
+		jsonBytes, _ := json.Marshal(data)
+		return core.LogEntry{
+			Time:    now,
+			Source:  fmt.Sprintf("random_%s", rs.id),
+			Level:   level,
+			Message: string(jsonBytes),
+			RawSize: int64(len(jsonBytes) + 1),
+		}
+
+	default:
+		return core.LogEntry{}
+	}
+}
+
+// generateRandomString creates a random string of specified length
+func (rs *RandomSource) generateRandomString(length int) string {
+	const normalChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "
+	const specialChars = "\t\n\r\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F"
+	const unicodeChars = "™€¢£¥§©®°±µ¶·ÀÉÑÖÜßäëïöü←↑→↓∀∃∅∇∈∉∪∩≈≠≤≥"
+
+	result := make([]byte, 0, length)
+
+	if rs.config.Special && length >= 3 {
+		// Reserve space for at least one special and one unicode char
+		normalLength := length - 2
+
+		// Generate normal characters
+		for i := 0; i < normalLength; i++ {
+			result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
+		}
+
+		// Insert special character at random position
+		specialPos := rs.rng.Intn(len(result) + 1)
+		specialChar := specialChars[rs.rng.Intn(len(specialChars))]
+		result = append(result[:specialPos], append([]byte{specialChar}, result[specialPos:]...)...)
+
+		// Insert unicode character at random position
+		unicodePos := rs.rng.Intn(len(result) + 1)
+		unicodeChar := unicodeChars[rs.rng.Intn(len(unicodeChars)/3)*3:]
+		if len(unicodeChar) >= 3 {
+			unicodeBytes := []byte(unicodeChar[:3])
+			if unicodePos == len(result) {
+				result = append(result, unicodeBytes...)
+			} else {
+				result = append(result[:unicodePos], append(unicodeBytes, result[unicodePos:]...)...)
+			}
+		}
+
+		// Trim to exact length if needed
+		if len(result) > length {
+			result = result[:length]
+		}
+	} else {
+		// Normal generation without special characters
+		for i := 0; i < length; i++ {
+			result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
+		}
+	}
+
+	return string(result)
+}
+
+// randomLogLevel returns a random log level
+func (rs *RandomSource) randomLogLevel() string {
+	levels := []string{"DEBUG", "INFO", "WARN", "ERROR"}
+	return levels[rs.rng.Intn(len(levels))]
+}
+
+// publish sends a log entry to all subscribers
+func (rs *RandomSource) publish(entry core.LogEntry) {
+	rs.mu.RLock()
+	defer rs.mu.RUnlock()
+
+	rs.totalEntries.Add(1)
+	rs.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range rs.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			rs.droppedEntries.Add(1)
+		}
+	}
+}

internal/source/source.go

@@ -0,0 +1,62 @@
+package source
+
+import (
+	"strings"
+	"time"
+
+	"logwisp/internal/core"
+)
+
+// Source represents an input data stream for log entries
+type Source interface {
+	// Capabilities returns a slice of supported Source capabilities
+	Capabilities() []core.Capability
+
+	// Subscribe returns a channel that receives log entries from the source
+	Subscribe() <-chan core.LogEntry
+
+	// Start begins reading from the source
+	Start() error
+
+	// Stop gracefully shuts down the source
+	Stop()
+
+	// SourceStats contains statistics about a source
+	GetStats() SourceStats
+}
+
+// SourceStats contains statistics about a source
+type SourceStats struct {
+	ID             string
+	Type           string
+	TotalEntries   uint64
+	DroppedEntries uint64
+	StartTime      time.Time
+	LastEntryTime  time.Time
+	Details        map[string]any
+}
+
+// ExtractLogLevel heuristically determines the log level from a line of text
+func ExtractLogLevel(line string) string {
+	patterns := []struct {
+		patterns []string
+		level    string
+	}{
+		{[]string{"[ERROR]", "ERROR:", " ERROR ", "ERR:", "[ERR]", "FATAL:", "[FATAL]"}, "ERROR"},
+		{[]string{"[WARN]", "WARN:", " WARN ", "WARNING:", "[WARNING]"}, "WARN"},
+		{[]string{"[INFO]", "INFO:", " INFO ", "[INF]", "INF:"}, "INFO"},
+		{[]string{"[DEBUG]", "DEBUG:", " DEBUG ", "[DBG]", "DBG:"}, "DEBUG"},
+		{[]string{"[TRACE]", "TRACE:", " TRACE "}, "TRACE"},
+	}
+
+	upperLine := strings.ToUpper(line)
+	for _, group := range patterns {
+		for _, pattern := range group.patterns {
+			if strings.Contains(upperLine, pattern) {
+				return group.level
+			}
+		}
+	}
+
+	return ""
+}

internal/source/tcpchain/tcpchain.go

@@ -0,0 +1,429 @@
+package tcpchain
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"maps"
+	"net"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/internal/authz"
+	"logwisp/internal/chain"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
+	"logwisp/internal/plugin"
+	"logwisp/internal/session"
+	"logwisp/internal/source"
+	"logwisp/internal/tlsx"
+
+	lconfig "github.com/lixenwraith/config"
+	"github.com/lixenwraith/log"
+)
+
+func init() {
+	if err := plugin.RegisterSource("tcp_chain", NewTCPChainSourcePlugin); err != nil {
+		panic(fmt.Sprintf("failed to register tcp_chain source: %v", err))
+	}
+}
+
+const (
+	DefaultChainSourceBufferSize     = 1000
+	DefaultChainSourceHelloTimeoutMS = 10000
+)
+
+// TCPChainSource accepts connections from upstream tcp_chain sinks and ingests NDJSON entries
+type TCPChainSource struct {
+	id     string
+	proxy  *session.Proxy
+	config *config.TCPChainSourceOptions
+
+	subscribers []chan core.LogEntry
+	listener    net.Listener
+	conns       map[net.Conn]struct{}
+	logger      *log.Logger
+
+	// TLS
+	tlsConfig          *tls.Config
+	tlsHandshakeErrors atomic.Uint64
+
+	// Authorization
+	auth *authz.Policy
+
+	mu     sync.RWMutex
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+
+	startTime      time.Time
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	parseErrors    atomic.Uint64
+	rejectedConns  atomic.Uint64
+	activeConns    atomic.Int64
+	lastEntryTime  atomic.Value // time.Time
+}
+
+// NewTCPChainSourcePlugin creates a tcp_chain source through plugin factory
+func NewTCPChainSourcePlugin(
+	id string,
+	configMap map[string]any,
+	logger *log.Logger,
+	proxy *session.Proxy,
+) (source.Source, error) {
+	opts := &config.TCPChainSourceOptions{
+		Host:      "0.0.0.0",
+		TrustNode: true,
+	}
+	if err := lconfig.ScanMap(configMap, opts); err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+	if err := lconfig.Port(opts.Port); err != nil {
+		return nil, fmt.Errorf("port: %w", err)
+	}
+	if opts.BufferSize <= 0 {
+		opts.BufferSize = DefaultChainSourceBufferSize
+	}
+	if opts.HelloTimeoutMS <= 0 {
+		opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS
+	}
+	tlsCfg, err := tlsx.Server(opts.TLS)
+	if err != nil {
+		return nil, err
+	}
+	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleChainListener)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &TCPChainSource{
+		id:          id,
+		proxy:       proxy,
+		config:      opts,
+		subscribers: make([]chan core.LogEntry, 0),
+		conns:       make(map[net.Conn]struct{}),
+		logger:      logger,
+		tlsConfig:   tlsCfg,
+		auth:        authPolicy,
+	}
+	s.lastEntryTime.Store(time.Time{})
+
+	logger.Info("msg", "TCP chain source initialized",
+		"component", "tcp_chain_source",
+		"instance_id", id,
+		"host", opts.Host,
+		"port", opts.Port,
+		"tls", tlsCfg != nil,
+		"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
+		"auth", authPolicy.Describe())
+	if authPolicy.Unrestricted() {
+		logger.Warn("msg", "Auth policy admits any identity the configured CA vouches for",
+			"component", "tcp_chain_source",
+			"instance_id", id,
+			"hint", "set auth.allow or auth.allow_patterns to authorize named peers")
+	}
+	if authPolicy.BindsNode() {
+		logger.Info("msg", "Node labels bound to peer identity; trust_node is ignored",
+			"component", "tcp_chain_source",
+			"instance_id", id,
+			"node_binding", authPolicy.NodeBinding(),
+			"trust_node", opts.TrustNode)
+	}
+	return s, nil
+}
+
+// Capabilities returns supported capabilities
+func (s *TCPChainSource) Capabilities() []core.Capability {
+	caps := []core.Capability{core.CapSessionAware, core.CapMultiSession}
+	if s.tlsConfig != nil {
+		caps = append(caps, core.CapTLS)
+	}
+	if s.auth.Enabled() {
+		caps = append(caps, core.CapAuth) // authorizes peers, not just the CA
+	}
+	return caps
+}
+
+// Subscribe returns a channel for receiving log entries
+func (s *TCPChainSource) Subscribe() <-chan core.LogEntry {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	ch := make(chan core.LogEntry, s.config.BufferSize)
+	s.subscribers = append(s.subscribers, ch)
+	return ch
+}
+
+// Start binds the listener and begins accepting connections
+func (s *TCPChainSource) Start() error {
+	addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
+	// IPv4-only. TLS-wrapped when configured; handshake runs explicitly in
+	// handleConn under tlsx.HandshakeTimeout, pre-hello.
+	ln, err := net.Listen("tcp4", addr)
+	if err != nil {
+		return fmt.Errorf("listen %s: %w", addr, err)
+	}
+	if s.tlsConfig != nil {
+		ln = tls.NewListener(ln, s.tlsConfig)
+	}
+	s.listener = ln
+	s.ctx, s.cancel = context.WithCancel(context.Background())
+	s.startTime = time.Now()
+
+	s.wg.Add(1)
+	go s.acceptLoop()
+
+	s.logger.Info("msg", "TCP chain source started",
+		"component", "tcp_chain_source",
+		"instance_id", s.id,
+		"addr", addr)
+	return nil
+}
+
+// Stop closes the listener, all connections, and subscriber channels
+func (s *TCPChainSource) Stop() {
+	if s.cancel != nil {
+		s.cancel()
+	}
+	if s.listener != nil {
+		s.listener.Close()
+	}
+
+	s.mu.Lock()
+	for conn := range s.conns {
+		conn.Close() // unblocks per-connection reads
+	}
+	s.mu.Unlock()
+
+	s.wg.Wait()
+
+	s.mu.Lock()
+	for _, ch := range s.subscribers {
+		close(ch)
+	}
+	s.mu.Unlock()
+
+	s.logger.Info("msg", "TCP chain source stopped",
+		"component", "tcp_chain_source",
+		"instance_id", s.id)
+}
+
+// GetStats returns the source's statistics
+func (s *TCPChainSource) GetStats() source.SourceStats {
+	lastEntry, _ := s.lastEntryTime.Load().(time.Time)
+	details := map[string]any{
+		"host":                 s.config.Host,
+		"port":                 s.config.Port,
+		"tls":                  s.tlsConfig != nil,
+		"tls_handshake_errors": s.tlsHandshakeErrors.Load(),
+		"active_connections":   s.activeConns.Load(),
+		"rejected_conns":       s.rejectedConns.Load(),
+		"parse_errors":         s.parseErrors.Load(),
+		"trust_node":           s.config.TrustNode,
+	}
+	maps.Copy(details, s.auth.Stats())
+
+	return source.SourceStats{
+		ID:             s.id,
+		Type:           "tcp_chain",
+		TotalEntries:   s.totalEntries.Load(),
+		DroppedEntries: s.droppedEntries.Load(),
+		StartTime:      s.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        details,
+	}
+}
+
+// acceptLoop accepts upstream connections until listener close
+func (s *TCPChainSource) acceptLoop() {
+	defer s.wg.Done()
+	for {
+		conn, err := s.listener.Accept()
+		if err != nil {
+			if errors.Is(err, net.ErrClosed) || s.ctx.Err() != nil {
+				return
+			}
+			s.logger.Warn("msg", "Accept error",
+				"component", "tcp_chain_source",
+				"error", err)
+			continue
+		}
+
+		if s.config.MaxConnections > 0 && s.activeConns.Load() >= s.config.MaxConnections {
+			s.rejectedConns.Add(1)
+			conn.Close()
+			continue
+		}
+
+		s.mu.Lock()
+		s.conns[conn] = struct{}{}
+		s.mu.Unlock()
+
+		s.wg.Add(1)
+		go s.handleConn(conn)
+	}
+}
+
+// handleConn validates the hello preamble, then streams entries until EOF/error
+func (s *TCPChainSource) handleConn(conn net.Conn) {
+	defer s.wg.Done()
+	remote := conn.RemoteAddr().String()
+	s.activeConns.Add(1)
+
+	var sessID string
+	defer func() {
+		conn.Close()
+		s.mu.Lock()
+		delete(s.conns, conn)
+		s.mu.Unlock()
+		if sessID != "" {
+			s.proxy.RemoveSession(sessID)
+		}
+		s.activeConns.Add(-1)
+	}()
+
+	var tlsState *tls.ConnectionState
+	if tc, ok := conn.(*tls.Conn); ok {
+		hctx, cancel := context.WithTimeout(s.ctx, tlsx.HandshakeTimeout)
+		err := tc.HandshakeContext(hctx)
+		cancel()
+		if err != nil {
+			s.tlsHandshakeErrors.Add(1)
+			s.logger.Warn("msg", "TLS handshake failed",
+				"component", "tcp_chain_source",
+				"remote_addr", remote,
+				"error", err)
+			return // deferred cleanup closes conn
+		}
+		cs := tc.ConnectionState()
+		tlsState = &cs
+	}
+
+	// Authorize before a preamble is parsed on an unauthorized peer's behalf
+	ident, err := s.auth.Authorize(tlsState)
+	if err != nil {
+		s.rejectedConns.Add(1)
+		s.logger.Warn("msg", "Connection rejected by auth policy",
+			"component", "tcp_chain_source",
+			"instance_id", s.id,
+			"remote_addr", remote,
+			"error", err)
+		return // deferred cleanup closes conn
+	}
+
+	scanner := bufio.NewScanner(conn)
+	// Oversized line (> MaxLogEntryBytes) is a protocol violation; scanner is
+	// unrecoverable after ErrTooLong, connection terminates
+	scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
+
+	// Hello preamble
+	conn.SetReadDeadline(time.Now().Add(time.Duration(s.config.HelloTimeoutMS) * time.Millisecond))
+	if !scanner.Scan() {
+		s.logger.Warn("msg", "Connection closed before hello",
+			"component", "tcp_chain_source",
+			"remote_addr", remote,
+			"error", scanner.Err())
+		return
+	}
+	hello, err := chain.DecodeHello(scanner.Bytes())
+	if err != nil {
+		s.logger.Warn("msg", "Rejected chain connection",
+			"component", "tcp_chain_source",
+			"remote_addr", remote,
+			"error", err)
+		return
+	}
+
+	fallbackNode := remote
+	if host, _, splitErr := net.SplitHostPort(remote); splitErr == nil {
+		fallbackNode = host
+	}
+	connNode, err := s.auth.ResolveNode(hello.Node, fallbackNode, s.config.TrustNode, ident)
+	if err != nil {
+		s.rejectedConns.Add(1)
+		s.logger.Warn("msg", "Connection rejected by node binding",
+			"component", "tcp_chain_source",
+			"instance_id", s.id,
+			"remote_addr", remote,
+			"declared_node", hello.Node,
+			"error", err)
+		return
+	}
+	// force relabels every entry, so an edge cannot smuggle a foreign origin
+	// through the per-entry node field either
+	trustEntryNode := s.auth.TrustsEntryNode(s.config.TrustNode)
+
+	meta := map[string]any{
+		"type": "tcp_chain",
+		"node": connNode,
+	}
+	if tlsState != nil {
+		meta["tls"] = true
+		if cn := tlsx.PeerCN(*tlsState); cn != "" {
+			meta["tls_peer_cn"] = cn
+		}
+	}
+	ident.Apply(meta)
+	sess := s.proxy.CreateSession(remote, meta)
+	sessID = sess.ID
+
+	s.logger.Info("msg", "Chain connection established",
+		"component", "tcp_chain_source",
+		"remote_addr", remote,
+		"node", connNode,
+		"auth_identity", ident.Name)
+
+	idle := time.Duration(s.config.ReadTimeoutMS) * time.Millisecond
+	for {
+		if idle > 0 {
+			conn.SetReadDeadline(time.Now().Add(idle))
+		} else {
+			conn.SetReadDeadline(time.Time{})
+		}
+		if !scanner.Scan() {
+			if err := scanner.Err(); err != nil && !errors.Is(err, net.ErrClosed) {
+				s.logger.Debug("msg", "Chain read terminated",
+					"component", "tcp_chain_source",
+					"remote_addr", remote,
+					"error", err)
+			}
+			return
+		}
+		line := scanner.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+		s.proxy.UpdateActivity(sessID)
+
+		entry, err := chain.DecodeEntry(line, connNode, trustEntryNode)
+		if err != nil {
+			s.parseErrors.Add(1)
+			s.logger.Debug("msg", "Dropped malformed chain entry",
+				"component", "tcp_chain_source",
+				"error", err)
+			continue
+		}
+		s.publish(entry)
+	}
+}
+
+// publish sends a log entry to all subscribers
+func (s *TCPChainSource) publish(entry core.LogEntry) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	s.totalEntries.Add(1)
+	s.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range s.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			s.droppedEntries.Add(1)
+		}
+	}
+}

internal/tlsx/tlsx.go

@@ -0,0 +1,177 @@
+// Package tlsx builds crypto/tls configurations from config.TLSOptions.
+// It is the single seam between declarative TLS config and the stdlib;
+// each network plugin calls exactly one constructor.
+package tlsx
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	stdlog "log"
+	"os"
+	"strings"
+	"time"
+
+	"logwisp/internal/config"
+
+	"github.com/lixenwraith/log"
+)
+
+// HandshakeTimeout bounds TLS handshakes on both accept and dial paths
+const HandshakeTimeout = 10 * time.Second
+
+// Server builds the *tls.Config for listener plugins
+// (tcp/http sinks, tcp_chain/http_chain sources). Returns (nil, nil) when disabled.
+func Server(o *config.TLSOptions) (*tls.Config, error) {
+	if o == nil || !o.Enabled {
+		return nil, nil
+	}
+	if o.CertFile == "" || o.KeyFile == "" {
+		return nil, fmt.Errorf("tls: cert_file and key_file are required for listeners")
+	}
+	cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
+	if err != nil {
+		return nil, fmt.Errorf("tls: load keypair: %w", err)
+	}
+	mv, err := minVersion(o.MinVersion)
+	if err != nil {
+		return nil, err
+	}
+	cfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		MinVersion:   mv,
+	}
+	if o.ClientAuth {
+		if o.ClientCAFile == "" {
+			return nil, fmt.Errorf("tls: client_auth requires client_ca_file")
+		}
+		pool, err := loadPool(o.ClientCAFile)
+		if err != nil {
+			return nil, err
+		}
+		cfg.ClientCAs = pool
+		cfg.ClientAuth = tls.RequireAndVerifyClientCert
+	}
+	return cfg, nil
+}
+
+// Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain
+// sinks). host seeds ServerName when no override is set; Go verifies IP SANs
+// when host is an address. Returns (nil, nil) when disabled.
+func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
+	if o == nil || !o.Enabled {
+		return nil, nil
+	}
+	mv, err := minVersion(o.MinVersion)
+	if err != nil {
+		return nil, err
+	}
+	cfg := &tls.Config{
+		MinVersion:         mv,
+		ServerName:         o.ServerName,
+		InsecureSkipVerify: o.InsecureSkipVerify,
+	}
+	if cfg.ServerName == "" {
+		cfg.ServerName = host
+	}
+	if o.CAFile != "" {
+		pool, err := loadPool(o.CAFile)
+		if err != nil {
+			return nil, err
+		}
+		cfg.RootCAs = pool
+	}
+	if (o.CertFile == "") != (o.KeyFile == "") {
+		return nil, fmt.Errorf("tls: cert_file and key_file must be set together")
+	}
+	if o.CertFile != "" {
+		cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("tls: load keypair: %w", err)
+		}
+		cfg.Certificates = []tls.Certificate{cert}
+	}
+	return cfg, nil
+}
+
+// Identity modes for PeerIdentity, mirroring the auth.identity config values.
+// Validity is enforced at policy construction in internal/authz.
+const (
+	IdentityCN       = "cn"
+	IdentitySANDNS   = "san_dns"
+	IdentitySANURI   = "san_uri"
+	IdentitySANEmail = "san_email"
+)
+
+// PeerCN returns the subject CN of the verified peer leaf, "" if none
+func PeerCN(cs tls.ConnectionState) string {
+	return PeerIdentity(cs, IdentityCN)
+}
+
+// PeerIdentity returns the field named by mode from the verified peer leaf,
+// "" when the certificate does not carry it or mode is unknown. The chain,
+// signature, and validity window are already checked by the handshake, so
+// this is pure field selection.
+func PeerIdentity(cs tls.ConnectionState, mode string) string {
+	if len(cs.PeerCertificates) == 0 {
+		return ""
+	}
+	leaf := cs.PeerCertificates[0]
+	switch mode {
+	case "", IdentityCN:
+		return leaf.Subject.CommonName
+	case IdentitySANDNS:
+		if len(leaf.DNSNames) > 0 {
+			return leaf.DNSNames[0]
+		}
+	case IdentitySANURI:
+		if len(leaf.URIs) > 0 {
+			return leaf.URIs[0].String()
+		}
+	case IdentitySANEmail:
+		if len(leaf.EmailAddresses) > 0 {
+			return leaf.EmailAddresses[0]
+		}
+	}
+	return ""
+}
+
+// HTTPErrorLog adapts the structured logger for http.Server.ErrorLog so TLS
+// handshake failures don't bypass log routing straight to stderr (which would
+// violate the console sanitization policy).
+func HTTPErrorLog(l *log.Logger, component string) *stdlog.Logger {
+	return stdlog.New(errLogWriter{l: l, component: component}, "", 0)
+}
+
+type errLogWriter struct {
+	l         *log.Logger
+	component string
+}
+
+func (w errLogWriter) Write(p []byte) (int, error) {
+	w.l.Warn("msg", strings.TrimSpace(string(p)), "component", w.component)
+	return len(p), nil
+}
+
+func loadPool(file string) (*x509.CertPool, error) {
+	pemBytes, err := os.ReadFile(file)
+	if err != nil {
+		return nil, fmt.Errorf("tls: read CA file: %w", err)
+	}
+	pool := x509.NewCertPool()
+	if !pool.AppendCertsFromPEM(pemBytes) {
+		return nil, fmt.Errorf("tls: no certificates found in %s", file)
+	}
+	return pool, nil
+}
+
+func minVersion(s string) (uint16, error) {
+	switch s {
+	case "", "1.3":
+		return tls.VersionTLS13, nil
+	case "1.2":
+		return tls.VersionTLS12, nil
+	default:
+		return 0, fmt.Errorf("tls: min_version %q (valid: \"1.2\", \"1.3\")", s)
+	}
+}

internal/tokenbucket/bucket.go

@@ -1,13 +1,11 @@
-// FILE: logwisp/src/internal/limit/token_bucket.go
+package tokenbucket
-package limit
 
 import (
 	"sync"
 	"time"
 )
 
-// TokenBucket implements a token bucket rate limiter
+// TokenBucket implements a thread-safe token bucket rate limiter
-// Safe for concurrent use.
 type TokenBucket struct {
 	capacity   float64
 	tokens     float64
@@ -16,8 +14,8 @@ type TokenBucket struct {
 	mu         sync.Mutex
 }
 
-// Creates a new token bucket with given capacity and refill rate
+// New creates a new token bucket with given capacity and refill rate
-func NewTokenBucket(capacity float64, refillRate float64) *TokenBucket {
+func New(capacity float64, refillRate float64) *TokenBucket {
 	return &TokenBucket{
 		capacity:   capacity,
 		tokens:     capacity, // Start full
@@ -26,12 +24,12 @@ func NewTokenBucket(capacity float64, refillRate float64) *TokenBucket {
 	}
 }
 
-// Attempts to consume one token, returns true if allowed
+// Allow attempts to consume one token, returns true if allowed
 func (tb *TokenBucket) Allow() bool {
 	return tb.AllowN(1)
 }
 
-// Attempts to consume n tokens, returns true if allowed
+// AllowN attempts to consume n tokens, returns true if allowed
 func (tb *TokenBucket) AllowN(n float64) bool {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
@@ -45,7 +43,7 @@ func (tb *TokenBucket) AllowN(n float64) bool {
 	return false
 }
 
-// Returns the current number of available tokens
+// Tokens returns the current number of available tokens
 func (tb *TokenBucket) Tokens() float64 {
 	tb.mu.Lock()
 	defer tb.mu.Unlock()
@@ -54,7 +52,7 @@ func (tb *TokenBucket) Tokens() float64 {
 	return tb.tokens
 }
 
-// Adds tokens based on time elapsed since last refill
+// refill adds tokens based on time elapsed since last refill
 // MUST be called with mutex held
 func (tb *TokenBucket) refill() {
 	now := time.Now()
@@ -73,3 +71,17 @@ func (tb *TokenBucket) refill() {
 	}
 	tb.lastRefill = now
 }
+
+// Rate returns the refill rate in tokens per second
+func (tb *TokenBucket) Rate() float64 {
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+	return tb.refillRate
+}
+
+// Capacity returns the bucket capacity
+func (tb *TokenBucket) Capacity() float64 {
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+	return tb.capacity
+}

internal/version/version.go

@@ -0,0 +1,25 @@
+package version
+
+import "fmt"
+
+var (
+	// Version is the application version, set at compile time via -ldflags
+	Version = "dev"
+	// GitCommit is the git commit hash, set at compile time
+	GitCommit = "unknown"
+	// BuildTime is the application build time, set at compile time
+	BuildTime = "unknown"
+)
+
+// String returns a detailed, formatted version string including commit and build time
+func String() string {
+	if Version == "dev" {
+		return fmt.Sprintf("dev (commit: %s, built: %s)", GitCommit, BuildTime)
+	}
+	return fmt.Sprintf("%s (commit: %s, built: %s)", Version, GitCommit, BuildTime)
+}
+
+// Short returns just the version tag
+func Short() string {
+	return Version
+}

src/cmd/logwisp/bootstrap.go

@@ -1,132 +0,0 @@
-// FILE: logwisp/src/cmd/logwisp/bootstrap.go
-package main
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/service"
-	"logwisp/src/internal/version"
-
-	"github.com/lixenwraith/log"
-)
-
-// Creates and initializes the log transport service
-func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service, error) {
-	// Create service with logger dependency injection
-	svc := service.NewService(ctx, logger)
-
-	// Initialize pipelines
-	successCount := 0
-	for _, pipelineCfg := range cfg.Pipelines {
-		logger.Info("msg", "Initializing pipeline", "pipeline", pipelineCfg.Name)
-
-		// Create the pipeline
-		if err := svc.NewPipeline(&pipelineCfg); err != nil {
-			logger.Error("msg", "Failed to create pipeline",
-				"pipeline", pipelineCfg.Name,
-				"error", err)
-			continue
-		}
-		successCount++
-		displayPipelineEndpoints(pipelineCfg)
-	}
-
-	if successCount == 0 {
-		return nil, fmt.Errorf("no pipelines successfully started (attempted %d)", len(cfg.Pipelines))
-	}
-
-	logger.Info("msg", "LogWisp started",
-		"version", version.Short(),
-		"pipelines", successCount)
-
-	return svc, nil
-}
-
-// Sets up the logger based on configuration
-func initializeLogger(cfg *config.Config) error {
-	logger = log.NewLogger()
-	logCfg := log.DefaultConfig()
-
-	if cfg.Quiet {
-		// In quiet mode, disable ALL logging output
-		logCfg.Level = 255 // A level that disables all output
-		logCfg.EnableFile = false
-		logCfg.EnableConsole = false
-		return logger.ApplyConfig(logCfg)
-	}
-
-	// Determine log level
-	levelValue, err := parseLogLevel(cfg.Logging.Level)
-	if err != nil {
-		return fmt.Errorf("invalid log level: %w", err)
-	}
-	logCfg.Level = levelValue
-
-	// Configure based on output mode
-	switch cfg.Logging.Output {
-	case "none":
-		logCfg.EnableFile = false
-		logCfg.EnableConsole = false
-	case "stdout":
-		logCfg.EnableFile = false
-		logCfg.EnableConsole = true
-		logCfg.ConsoleTarget = "stdout"
-	case "stderr":
-		logCfg.EnableFile = false
-		logCfg.EnableConsole = true
-		logCfg.ConsoleTarget = "stderr"
-	case "split":
-		logCfg.EnableFile = false
-		logCfg.EnableConsole = true
-		logCfg.ConsoleTarget = "split"
-	case "file":
-		logCfg.EnableFile = true
-		logCfg.EnableConsole = false
-		configureFileLogging(logCfg, cfg)
-	case "all":
-		logCfg.EnableFile = true
-		logCfg.EnableConsole = true
-		logCfg.ConsoleTarget = "split"
-		configureFileLogging(logCfg, cfg)
-	default:
-		return fmt.Errorf("invalid log output mode: %s", cfg.Logging.Output)
-	}
-
-	// Apply format if specified
-	if cfg.Logging.Console != nil && cfg.Logging.Console.Format != "" {
-		logCfg.Format = cfg.Logging.Console.Format
-	}
-
-	return logger.ApplyConfig(logCfg)
-}
-
-// Sets up file-based logging parameters
-func configureFileLogging(logCfg *log.Config, cfg *config.Config) {
-	if cfg.Logging.File != nil {
-		logCfg.Directory = cfg.Logging.File.Directory
-		logCfg.Name = cfg.Logging.File.Name
-		logCfg.MaxSizeKB = cfg.Logging.File.MaxSizeMB * 1000
-		logCfg.MaxTotalSizeKB = cfg.Logging.File.MaxTotalSizeMB * 1000
-		if cfg.Logging.File.RetentionHours > 0 {
-			logCfg.RetentionPeriodHrs = cfg.Logging.File.RetentionHours
-		}
-	}
-}
-
-func parseLogLevel(level string) (int64, error) {
-	switch strings.ToLower(level) {
-	case "debug":
-		return log.LevelDebug, nil
-	case "info":
-		return log.LevelInfo, nil
-	case "warn", "warning":
-		return log.LevelWarn, nil
-	case "error":
-		return log.LevelError, nil
-	default:
-		return 0, fmt.Errorf("unknown log level: %s", level)
-	}
-}

src/cmd/logwisp/commands/auth.go

@@ -1,355 +0,0 @@
-// FILE: src/cmd/logwisp/commands/auth.go
-package commands
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-	"strings"
-	"syscall"
-
-	"logwisp/src/internal/auth"
-	"logwisp/src/internal/core"
-
-	"golang.org/x/term"
-)
-
-type AuthCommand struct {
-	output io.Writer
-	errOut io.Writer
-}
-
-func NewAuthCommand() *AuthCommand {
-	return &AuthCommand{
-		output: os.Stdout,
-		errOut: os.Stderr,
-	}
-}
-
-func (ac *AuthCommand) Execute(args []string) error {
-	cmd := flag.NewFlagSet("auth", flag.ContinueOnError)
-	cmd.SetOutput(ac.errOut)
-
-	var (
-		// User credentials
-		username     = cmd.String("u", "", "Username")
-		usernameLong = cmd.String("user", "", "Username")
-		password     = cmd.String("p", "", "Password (will prompt if not provided)")
-		passwordLong = cmd.String("password", "", "Password (will prompt if not provided)")
-
-		// Auth type selection (multiple ways to specify)
-		authType     = cmd.String("t", "", "Auth type: basic, scram, or token")
-		authTypeLong = cmd.String("type", "", "Auth type: basic, scram, or token")
-		useScram     = cmd.Bool("s", false, "Generate SCRAM credentials (TCP)")
-		useScramLong = cmd.Bool("scram", false, "Generate SCRAM credentials (TCP)")
-		useBasic     = cmd.Bool("b", false, "Generate basic auth credentials (HTTP)")
-		useBasicLong = cmd.Bool("basic", false, "Generate basic auth credentials (HTTP)")
-
-		// Token generation
-		genToken     = cmd.Bool("k", false, "Generate random bearer token")
-		genTokenLong = cmd.Bool("token", false, "Generate random bearer token")
-		tokenLen     = cmd.Int("l", 32, "Token length in bytes")
-		tokenLenLong = cmd.Int("length", 32, "Token length in bytes")
-
-		// Migration option
-		migrate     = cmd.Bool("m", false, "Convert basic auth PHC to SCRAM")
-		migrateLong = cmd.Bool("migrate", false, "Convert basic auth PHC to SCRAM")
-		phcHash     = cmd.String("phc", "", "PHC hash to migrate (required with --migrate)")
-	)
-
-	cmd.Usage = func() {
-		fmt.Fprintln(ac.errOut, "Generate authentication credentials for LogWisp")
-		fmt.Fprintln(ac.errOut, "\nUsage: logwisp auth [options]")
-		fmt.Fprintln(ac.errOut, "\nExamples:")
-		fmt.Fprintln(ac.errOut, "  # Generate basic auth hash for HTTP sources/sinks")
-		fmt.Fprintln(ac.errOut, "  logwisp auth -u admin -b")
-		fmt.Fprintln(ac.errOut, "  logwisp auth --user=admin --basic")
-		fmt.Fprintln(ac.errOut, "  ")
-		fmt.Fprintln(ac.errOut, "  # Generate SCRAM credentials for TCP")
-		fmt.Fprintln(ac.errOut, "  logwisp auth -u tcpuser -s")
-		fmt.Fprintln(ac.errOut, "  logwisp auth --user=tcpuser --scram")
-		fmt.Fprintln(ac.errOut, "  ")
-		fmt.Fprintln(ac.errOut, "  # Generate bearer token")
-		fmt.Fprintln(ac.errOut, "  logwisp auth -k -l 64")
-		fmt.Fprintln(ac.errOut, "  logwisp auth --token --length=64")
-		fmt.Fprintln(ac.errOut, "\nOptions:")
-		cmd.PrintDefaults()
-	}
-
-	if err := cmd.Parse(args); err != nil {
-		return err
-	}
-
-	// Check for unparsed arguments
-	if cmd.NArg() > 0 {
-		return fmt.Errorf("unexpected argument(s): %s", strings.Join(cmd.Args(), " "))
-	}
-
-	// Merge short and long form values
-	finalUsername := coalesceString(*username, *usernameLong)
-	finalPassword := coalesceString(*password, *passwordLong)
-	finalAuthType := coalesceString(*authType, *authTypeLong)
-	finalGenToken := coalesceBool(*genToken, *genTokenLong)
-	finalTokenLen := coalesceInt(*tokenLen, *tokenLenLong, core.DefaultTokenLength)
-	finalUseScram := coalesceBool(*useScram, *useScramLong)
-	finalUseBasic := coalesceBool(*useBasic, *useBasicLong)
-	finalMigrate := coalesceBool(*migrate, *migrateLong)
-
-	// Handle migration mode
-	if finalMigrate {
-		if *phcHash == "" || finalUsername == "" || finalPassword == "" {
-			return fmt.Errorf("--migrate requires --user, --password, and --phc flags")
-		}
-		return ac.migrateToScram(finalUsername, finalPassword, *phcHash)
-	}
-
-	// Determine auth type from flags
-	if finalGenToken || finalAuthType == "token" {
-		return ac.generateToken(finalTokenLen)
-	}
-
-	// Determine credential type
-	credType := "basic" // default
-
-	// Check explicit type flags
-	if finalUseScram || finalAuthType == "scram" {
-		credType = "scram"
-	} else if finalUseBasic || finalAuthType == "basic" {
-		credType = "basic"
-	} else if finalAuthType != "" {
-		return fmt.Errorf("invalid auth type: %s (valid: basic, scram, token)", finalAuthType)
-	}
-
-	// Username required for password-based auth
-	if finalUsername == "" {
-		cmd.Usage()
-		return fmt.Errorf("username required for %s auth generation", credType)
-	}
-
-	return ac.generatePasswordHash(finalUsername, finalPassword, credType)
-}
-
-func (ac *AuthCommand) Description() string {
-	return "Generate authentication credentials (passwords, tokens, SCRAM)"
-}
-
-func (ac *AuthCommand) Help() string {
-	return `Auth Command - Generate authentication credentials for LogWisp
-
-Usage: 
-  logwisp auth [options]
-
-Authentication Types:
-  HTTP/HTTPS Sources & Sinks (TLS required):
-    - Basic Auth: Username/password with Argon2id hashing
-    - Bearer Token: Random cryptographic tokens
-  
-  TCP Sources & Sinks (No TLS):
-    - SCRAM: Argon2-SCRAM-SHA256 for plaintext connections
-
-Options:
-  -u, --user <name>        Username for credential generation
-  -p, --password <pass>    Password (will prompt if not provided)
-  -t, --type <type>        Auth type: "basic", "scram", or "token"
-  -b, --basic              Generate basic auth credentials (HTTP/HTTPS)
-  -s, --scram              Generate SCRAM credentials (TCP)
-  -k, --token              Generate random bearer token
-  -l, --length <bytes>     Token length in bytes (default: 32)
-
-Examples:
-Examples:
-  # Generate basic auth hash for HTTP/HTTPS (with TLS)
-  logwisp auth -u admin -b
-  logwisp auth --user=admin --basic
-  
-  # Generate SCRAM credentials for TCP (without TLS)  
-  logwisp auth -u tcpuser -s
-  logwisp auth --user=tcpuser --type=scram
-  
-  # Generate 64-byte bearer token
-  logwisp auth -k -l 64
-  logwisp auth --token --length=64
-  
-  # Convert existing basic auth to SCRAM (HTTPS to TCP conversion)
-  logwisp auth -u admin -m --phc='$argon2id$v=19$m=65536...' --password='secret'
-
-Output:
-  The command outputs configuration snippets ready to paste into logwisp.toml
-  and the raw credential values for external auth files.
-
-Security Notes:
-  - Basic auth and tokens require TLS encryption for HTTP connections
-  - SCRAM provides authentication but NOT encryption for TCP connections
-  - Use strong passwords (12+ characters with mixed case, numbers, symbols)
-  - Store credentials securely and never commit them to version control
-`
-}
-
-func (ac *AuthCommand) generatePasswordHash(username, password, credType string) error {
-	// Get password if not provided
-	if password == "" {
-		var err error
-		password, err = ac.promptForPassword()
-		if err != nil {
-			return err
-		}
-	}
-
-	switch credType {
-	case "basic":
-		return ac.generateBasicAuth(username, password)
-	case "scram":
-		return ac.generateScramAuth(username, password)
-	default:
-		return fmt.Errorf("invalid credential type: %s", credType)
-	}
-}
-
-// promptForPassword handles password prompting with confirmation
-func (ac *AuthCommand) promptForPassword() (string, error) {
-	pass1 := ac.promptPassword("Enter password: ")
-	pass2 := ac.promptPassword("Confirm password: ")
-	if pass1 != pass2 {
-		return "", fmt.Errorf("passwords don't match")
-	}
-	return pass1, nil
-}
-
-func (ac *AuthCommand) promptPassword(prompt string) string {
-	fmt.Fprint(ac.errOut, prompt)
-	password, err := term.ReadPassword(syscall.Stdin)
-	fmt.Fprintln(ac.errOut)
-	if err != nil {
-		fmt.Fprintf(ac.errOut, "Failed to read password: %v\n", err)
-		os.Exit(1)
-	}
-	return string(password)
-}
-
-// generateBasicAuth creates Argon2id hash for HTTP basic auth
-func (ac *AuthCommand) generateBasicAuth(username, password string) error {
-	// Generate salt
-	salt := make([]byte, core.Argon2SaltLen)
-	if _, err := rand.Read(salt); err != nil {
-		return fmt.Errorf("failed to generate salt: %w", err)
-	}
-
-	// Generate Argon2id hash
-	cred, err := auth.DeriveCredential(username, password, salt,
-		core.Argon2Time, core.Argon2Memory, core.Argon2Threads)
-	if err != nil {
-		return fmt.Errorf("failed to derive credential: %w", err)
-	}
-
-	// Output configuration snippets
-	fmt.Fprintln(ac.output, "\n# Basic Auth Configuration (HTTP sources/sinks)")
-	fmt.Fprintln(ac.output, "# REQUIRES HTTPS/TLS for security")
-	fmt.Fprintln(ac.output, "# Add to logwisp.toml under [[pipelines]]:")
-	fmt.Fprintln(ac.output, "")
-	fmt.Fprintln(ac.output, "[pipelines.auth]")
-	fmt.Fprintln(ac.output, `type = "basic"`)
-	fmt.Fprintln(ac.output, "")
-	fmt.Fprintln(ac.output, "[[pipelines.auth.basic_auth.users]]")
-	fmt.Fprintf(ac.output, "username = %q\n", username)
-	fmt.Fprintf(ac.output, "password_hash = %q\n\n", cred.PHCHash)
-
-	fmt.Fprintln(ac.output, "# For external users file:")
-	fmt.Fprintf(ac.output, "%s:%s\n", username, cred.PHCHash)
-
-	return nil
-}
-
-// generateScramAuth creates Argon2id-SCRAM-SHA256 credentials for TCP
-func (ac *AuthCommand) generateScramAuth(username, password string) error {
-	// Generate salt
-	salt := make([]byte, core.Argon2SaltLen)
-	if _, err := rand.Read(salt); err != nil {
-		return fmt.Errorf("failed to generate salt: %w", err)
-	}
-
-	// Use internal auth package to derive SCRAM credentials
-	cred, err := auth.DeriveCredential(username, password, salt,
-		core.Argon2Time, core.Argon2Memory, core.Argon2Threads)
-	if err != nil {
-		return fmt.Errorf("failed to derive SCRAM credential: %w", err)
-	}
-
-	// Output SCRAM configuration
-	fmt.Fprintln(ac.output, "\n# SCRAM Auth Configuration (TCP sources/sinks)")
-	fmt.Fprintln(ac.output, "# Provides authentication but NOT encryption")
-	fmt.Fprintln(ac.output, "# Add to logwisp.toml under [[pipelines]]:")
-	fmt.Fprintln(ac.output, "")
-	fmt.Fprintln(ac.output, "[pipelines.auth]")
-	fmt.Fprintln(ac.output, `type = "scram"`)
-	fmt.Fprintln(ac.output, "")
-	fmt.Fprintln(ac.output, "[[pipelines.auth.scram_auth.users]]")
-	fmt.Fprintf(ac.output, "username = %q\n", username)
-	fmt.Fprintf(ac.output, "stored_key = %q\n", base64.StdEncoding.EncodeToString(cred.StoredKey))
-	fmt.Fprintf(ac.output, "server_key = %q\n", base64.StdEncoding.EncodeToString(cred.ServerKey))
-	fmt.Fprintf(ac.output, "salt = %q\n", base64.StdEncoding.EncodeToString(cred.Salt))
-	fmt.Fprintf(ac.output, "argon_time = %d\n", cred.ArgonTime)
-	fmt.Fprintf(ac.output, "argon_memory = %d\n", cred.ArgonMemory)
-	fmt.Fprintf(ac.output, "argon_threads = %d\n\n", cred.ArgonThreads)
-
-	fmt.Fprintln(ac.output, "# Note: SCRAM provides authentication only.")
-	fmt.Fprintln(ac.output, "# Use TLS/mTLS for encryption if needed.")
-
-	return nil
-}
-
-func (ac *AuthCommand) generateToken(length int) error {
-	if length < 16 {
-		fmt.Fprintln(ac.errOut, "Warning: tokens < 16 bytes are cryptographically weak")
-	}
-	if length > 512 {
-		return fmt.Errorf("token length exceeds maximum (512 bytes)")
-	}
-
-	token := make([]byte, length)
-	if _, err := rand.Read(token); err != nil {
-		return fmt.Errorf("failed to generate random bytes: %w", err)
-	}
-
-	b64 := base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(token)
-	hex := fmt.Sprintf("%x", token)
-
-	fmt.Fprintln(ac.output, "\n# Token Configuration")
-	fmt.Fprintln(ac.output, "# Add to logwisp.toml:")
-	fmt.Fprintf(ac.output, "tokens = [%q]\n\n", b64)
-
-	fmt.Fprintln(ac.output, "# Generated Token:")
-	fmt.Fprintf(ac.output, "Base64: %s\n", b64)
-	fmt.Fprintf(ac.output, "Hex:    %s\n", hex)
-
-	return nil
-}
-
-// migrateToScram converts basic auth PHC hash to SCRAM credentials
-func (ac *AuthCommand) migrateToScram(username, password, phcHash string) error {
-	// CHANGED: Moved from internal/auth to CLI command layer
-	cred, err := auth.MigrateFromPHC(username, password, phcHash)
-	if err != nil {
-		return fmt.Errorf("migration failed: %w", err)
-	}
-
-	// Output SCRAM configuration (reuse format from generateScramAuth)
-	fmt.Fprintln(ac.output, "\n# Migrated SCRAM Credentials")
-	fmt.Fprintln(ac.output, "# Add to logwisp.toml under [[pipelines]]:")
-	fmt.Fprintln(ac.output, "")
-	fmt.Fprintln(ac.output, "[pipelines.auth]")
-	fmt.Fprintln(ac.output, `type = "scram"`)
-	fmt.Fprintln(ac.output, "")
-	fmt.Fprintln(ac.output, "[[pipelines.auth.scram_auth.users]]")
-	fmt.Fprintf(ac.output, "username = %q\n", username)
-	fmt.Fprintf(ac.output, "stored_key = %q\n", base64.StdEncoding.EncodeToString(cred.StoredKey))
-	fmt.Fprintf(ac.output, "server_key = %q\n", base64.StdEncoding.EncodeToString(cred.ServerKey))
-	fmt.Fprintf(ac.output, "salt = %q\n", base64.StdEncoding.EncodeToString(cred.Salt))
-	fmt.Fprintf(ac.output, "argon_time = %d\n", cred.ArgonTime)
-	fmt.Fprintf(ac.output, "argon_memory = %d\n", cred.ArgonMemory)
-	fmt.Fprintf(ac.output, "argon_threads = %d\n", cred.ArgonThreads)
-
-	return nil
-}

src/cmd/logwisp/commands/help.go

@@ -1,123 +0,0 @@
-// FILE: src/cmd/logwisp/commands/help.go
-package commands
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-const generalHelpTemplate = `LogWisp: A flexible log transport and processing tool.
-
-Usage: 
-  logwisp [command] [options]
-  logwisp [options]
-
-Commands:
-%s
-
-Application Options:
-  -c, --config <path>      Path to configuration file (default: logwisp.toml)
-  -h, --help               Display this help message and exit
-  -v, --version            Display version information and exit  
-  -b, --background         Run LogWisp in the background as a daemon
-  -q, --quiet              Suppress all console output, including errors
-
-Runtime Options:
-  --disable-status-reporter  Disable the periodic status reporter
-  --config-auto-reload       Enable config reload on file change
-
-For command-specific help:
-  logwisp help <command>
-  logwisp <command> --help
-
-Configuration Sources (Precedence: CLI > Env > File > Defaults):
-  - CLI flags override all other settings
-  - Environment variables override file settings  
-  - TOML configuration file is the primary method
-
-Examples:
-  # Generate password for admin user
-  logwisp auth -u admin
-  
-  # Start service with custom config
-  logwisp -c /etc/logwisp/prod.toml
-  
-  # Run in background with config reload
-  logwisp -b --config-auto-reload
-
-For detailed configuration options, please refer to the documentation.
-`
-
-// HelpCommand handles help display
-type HelpCommand struct {
-	router *CommandRouter
-}
-
-// NewHelpCommand creates a new help command
-func NewHelpCommand(router *CommandRouter) *HelpCommand {
-	return &HelpCommand{router: router}
-}
-
-// Execute displays help information
-func (c *HelpCommand) Execute(args []string) error {
-	// Check if help is requested for a specific command
-	if len(args) > 0 && args[0] != "" {
-		cmdName := args[0]
-
-		if handler, exists := c.router.GetCommand(cmdName); exists {
-			fmt.Print(handler.Help())
-			return nil
-		}
-
-		return fmt.Errorf("unknown command: %s", cmdName)
-	}
-
-	// Display general help with command list
-	fmt.Printf(generalHelpTemplate, c.formatCommandList())
-	return nil
-}
-
-// formatCommandList creates a formatted list of available commands
-func (c *HelpCommand) formatCommandList() string {
-	commands := c.router.GetCommands()
-
-	// Sort command names for consistent output
-	names := make([]string, 0, len(commands))
-	maxLen := 0
-	for name := range commands {
-		names = append(names, name)
-		if len(name) > maxLen {
-			maxLen = len(name)
-		}
-	}
-	sort.Strings(names)
-
-	// Format each command with aligned descriptions
-	var lines []string
-	for _, name := range names {
-		handler := commands[name]
-		padding := strings.Repeat(" ", maxLen-len(name)+2)
-		lines = append(lines, fmt.Sprintf("  %s%s%s", name, padding, handler.Description()))
-	}
-
-	return strings.Join(lines, "\n")
-}
-
-func (c *HelpCommand) Description() string {
-	return "Display help information"
-}
-
-func (c *HelpCommand) Help() string {
-	return `Help Command - Display help information
-
-Usage:
-  logwisp help              Show general help
-  logwisp help <command>    Show help for a specific command
-  
-Examples:
-  logwisp help              # Show general help
-  logwisp help auth         # Show auth command help
-  logwisp auth --help       # Alternative way to get command help
-`
-}

src/cmd/logwisp/commands/router.go

@@ -1,118 +0,0 @@
-// FILE: src/cmd/logwisp/commands/router.go
-package commands
-
-import (
-	"fmt"
-	"os"
-)
-
-// Handler defines the interface for subcommands
-type Handler interface {
-	Execute(args []string) error
-	Description() string
-	Help() string
-}
-
-// CommandRouter handles subcommand routing before main app initialization
-type CommandRouter struct {
-	commands map[string]Handler
-}
-
-// NewCommandRouter creates and initializes the command router
-func NewCommandRouter() *CommandRouter {
-	router := &CommandRouter{
-		commands: make(map[string]Handler),
-	}
-
-	// Register available commands
-	router.commands["auth"] = NewAuthCommand()
-	router.commands["tls"] = NewTLSCommand()
-	router.commands["version"] = NewVersionCommand()
-	router.commands["help"] = NewHelpCommand(router)
-
-	return router
-}
-
-// Route checks for and executes subcommands
-func (r *CommandRouter) Route(args []string) (bool, error) {
-	if len(args) < 2 {
-		return false, nil // No command specified, let main app continue
-	}
-
-	cmdName := args[1]
-
-	// Special case: help flag at any position shows general help
-	for _, arg := range args[1:] {
-		if arg == "-h" || arg == "--help" {
-			// If it's after a valid command, show command-specific help
-			if handler, exists := r.commands[cmdName]; exists && cmdName != "help" {
-				fmt.Print(handler.Help())
-				return true, nil
-			}
-			// Otherwise show general help
-			return true, r.commands["help"].Execute(nil)
-		}
-	}
-
-	// Check if this is a known command
-	handler, exists := r.commands[cmdName]
-	if !exists {
-		// Check if it looks like a mistyped command (not a flag)
-		if cmdName[0] != '-' {
-			return false, fmt.Errorf("unknown command: %s\n\nRun 'logwisp help' for usage", cmdName)
-		}
-		// It's a flag, let main app handle it
-		return false, nil
-	}
-
-	// Execute the command
-	return true, handler.Execute(args[2:])
-}
-
-// GetCommand returns a command handler by name
-func (r *CommandRouter) GetCommand(name string) (Handler, bool) {
-	cmd, exists := r.commands[name]
-	return cmd, exists
-}
-
-// GetCommands returns all registered commands
-func (r *CommandRouter) GetCommands() map[string]Handler {
-	return r.commands
-}
-
-// ShowCommands displays available subcommands
-func (r *CommandRouter) ShowCommands() {
-	for name, handler := range r.commands {
-		fmt.Fprintf(os.Stderr, "  %-10s %s\n", name, handler.Description())
-	}
-	fmt.Fprintln(os.Stderr, "\nUse 'logwisp <command> --help' for command-specific help")
-}
-
-// Helper functions to merge short and long options
-func coalesceString(values ...string) string {
-	for _, v := range values {
-		if v != "" {
-			return v
-		}
-	}
-	return ""
-}
-
-func coalesceInt(primary, secondary, defaultVal int) int {
-	if primary != defaultVal {
-		return primary
-	}
-	if secondary != defaultVal {
-		return secondary
-	}
-	return defaultVal
-}
-
-func coalesceBool(values ...bool) bool {
-	for _, v := range values {
-		if v {
-			return true
-		}
-	}
-	return false
-}

src/cmd/logwisp/commands/tls.go

@@ -1,563 +0,0 @@
-// FILE: src/cmd/logwisp/commands/tls.go
-package commands
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"flag"
-	"fmt"
-	"io"
-	"math/big"
-	"net"
-	"os"
-	"strings"
-	"time"
-)
-
-type TLSCommand struct {
-	output io.Writer
-	errOut io.Writer
-}
-
-func NewTLSCommand() *TLSCommand {
-	return &TLSCommand{
-		output: os.Stdout,
-		errOut: os.Stderr,
-	}
-}
-
-func (tc *TLSCommand) Execute(args []string) error {
-	cmd := flag.NewFlagSet("tls", flag.ContinueOnError)
-	cmd.SetOutput(tc.errOut)
-
-	// Certificate type flags
-	var (
-		genCA     = cmd.Bool("ca", false, "Generate CA certificate")
-		genServer = cmd.Bool("server", false, "Generate server certificate")
-		genClient = cmd.Bool("client", false, "Generate client certificate")
-		selfSign  = cmd.Bool("self-signed", false, "Generate self-signed certificate")
-
-		// Common options - short forms
-		commonName = cmd.String("cn", "", "Common name (required)")
-		org        = cmd.String("o", "LogWisp", "Organization")
-		country    = cmd.String("c", "US", "Country code")
-		validDays  = cmd.Int("d", 365, "Validity period in days")
-		keySize    = cmd.Int("b", 2048, "RSA key size")
-
-		// Common options - long forms
-		commonNameLong = cmd.String("common-name", "", "Common name (required)")
-		orgLong        = cmd.String("org", "LogWisp", "Organization")
-		countryLong    = cmd.String("country", "US", "Country code")
-		validDaysLong  = cmd.Int("days", 365, "Validity period in days")
-		keySizeLong    = cmd.Int("bits", 2048, "RSA key size")
-
-		// Server/Client specific - short forms
-		hosts  = cmd.String("h", "", "Comma-separated hostnames/IPs")
-		caFile = cmd.String("ca-cert", "", "CA certificate file")
-		caKey  = cmd.String("ca-key", "", "CA key file")
-
-		// Server/Client specific - long forms
-		hostsLong = cmd.String("hosts", "", "Comma-separated hostnames/IPs")
-
-		// Output files
-		certOut = cmd.String("cert-out", "", "Output certificate file")
-		keyOut  = cmd.String("key-out", "", "Output key file")
-	)
-
-	cmd.Usage = func() {
-		fmt.Fprintln(tc.errOut, "Generate TLS certificates for LogWisp")
-		fmt.Fprintln(tc.errOut, "\nUsage: logwisp tls [options]")
-		fmt.Fprintln(tc.errOut, "\nExamples:")
-		fmt.Fprintln(tc.errOut, "  # Generate self-signed certificate")
-		fmt.Fprintln(tc.errOut, "  logwisp tls --self-signed --cn localhost --hosts localhost,127.0.0.1")
-		fmt.Fprintln(tc.errOut, "  ")
-		fmt.Fprintln(tc.errOut, "  # Generate CA certificate")
-		fmt.Fprintln(tc.errOut, "  logwisp tls --ca --cn \"LogWisp CA\" --cert-out ca.crt --key-out ca.key")
-		fmt.Fprintln(tc.errOut, "  ")
-		fmt.Fprintln(tc.errOut, "  # Generate server certificate signed by CA")
-		fmt.Fprintln(tc.errOut, "  logwisp tls --server --cn server.example.com --hosts server.example.com \\")
-		fmt.Fprintln(tc.errOut, "              --ca-cert ca.crt --ca-key ca.key")
-		fmt.Fprintln(tc.errOut, "\nOptions:")
-		cmd.PrintDefaults()
-		fmt.Fprintln(tc.errOut)
-	}
-
-	if err := cmd.Parse(args); err != nil {
-		return err
-	}
-
-	// Check for unparsed arguments
-	if cmd.NArg() > 0 {
-		return fmt.Errorf("unexpected argument(s): %s", strings.Join(cmd.Args(), " "))
-	}
-
-	// Merge short and long options
-	finalCN := coalesceString(*commonName, *commonNameLong)
-	finalOrg := coalesceString(*org, *orgLong, "LogWisp")
-	finalCountry := coalesceString(*country, *countryLong, "US")
-	finalDays := coalesceInt(*validDays, *validDaysLong, 365)
-	finalKeySize := coalesceInt(*keySize, *keySizeLong, 2048)
-	finalHosts := coalesceString(*hosts, *hostsLong)
-	finalCAFile := *caFile   // no short form
-	finalCAKey := *caKey     // no short form
-	finalCertOut := *certOut // no short form
-	finalKeyOut := *keyOut   // no short form
-
-	// Validate common name
-	if finalCN == "" {
-		cmd.Usage()
-		return fmt.Errorf("common name (--cn) is required")
-	}
-
-	// Validate RSA key size
-	if finalKeySize != 2048 && finalKeySize != 3072 && finalKeySize != 4096 {
-		return fmt.Errorf("invalid key size: %d (valid: 2048, 3072, 4096)", finalKeySize)
-	}
-
-	// Route to appropriate generator
-	switch {
-	case *genCA:
-		return tc.generateCA(finalCN, finalOrg, finalCountry, finalDays, finalKeySize, finalCertOut, finalKeyOut)
-	case *selfSign:
-		return tc.generateSelfSigned(finalCN, finalOrg, finalCountry, finalHosts, finalDays, finalKeySize, finalCertOut, finalKeyOut)
-	case *genServer:
-		return tc.generateServerCert(finalCN, finalOrg, finalCountry, finalHosts, finalCAFile, finalCAKey, finalDays, finalKeySize, finalCertOut, finalKeyOut)
-	case *genClient:
-		return tc.generateClientCert(finalCN, finalOrg, finalCountry, finalCAFile, finalCAKey, finalDays, finalKeySize, finalCertOut, finalKeyOut)
-	default:
-		cmd.Usage()
-		return fmt.Errorf("specify certificate type: --ca, --self-signed, --server, or --client")
-	}
-}
-
-func (tc *TLSCommand) Description() string {
-	return "Generate TLS certificates (CA, server, client, self-signed)"
-}
-
-func (tc *TLSCommand) Help() string {
-	return `TLS Command - Generate TLS certificates for LogWisp
-
-Usage: 
-  logwisp tls [options]
-
-Certificate Types:
-  --ca           Generate Certificate Authority (CA) certificate
-  --server       Generate server certificate (requires CA or self-signed)
-  --client       Generate client certificate (for mTLS)
-  --self-signed  Generate self-signed certificate (single cert for testing)
-
-Common Options:
-  --cn, --common-name <name>     Common Name (required)
-  -o, --org <organization>       Organization name (default: "LogWisp")
-  -c, --country <code>           Country code (default: "US")
-  -d, --days <number>            Validity period in days (default: 365)
-  -b, --bits <size>              RSA key size (default: 2048)
-
-Server Certificate Options:
-  -h, --hosts <list>             Comma-separated hostnames/IPs
-                                 Example: "localhost,10.0.0.1,example.com"
-  --ca-cert <file>       CA certificate file (for signing)
-  --ca-key <file>        CA key file (for signing)
-
-Output Options:
-  --cert-out <file>      Output certificate file (default: stdout)
-  --key-out <file>       Output private key file (default: stdout)
-
-Examples:
-  # Generate self-signed certificate for testing
-  logwisp tls --self-signed --cn localhost --hosts "localhost,127.0.0.1" \
-    --cert-out server.crt --key-out server.key
-  
-  # Generate CA certificate
-  logwisp tls --ca --cn "LogWisp CA" --days 3650 \
-    --cert-out ca.crt --key-out ca.key
-  
-  # Generate server certificate signed by CA
-  logwisp tls --server --cn "logwisp.example.com" \
-    --hosts "logwisp.example.com,10.0.0.100" \
-    --ca-cert ca.crt --ca-key ca.key \
-    --cert-out server.crt --key-out server.key
-  
-  # Generate client certificate for mTLS
-  logwisp tls --client --cn "client1" \
-    --ca-cert ca.crt --ca-key ca.key \
-    --cert-out client.crt --key-out client.key
-
-Security Notes:
-  - Keep private keys secure and never share them
-  - Use 2048-bit RSA minimum, 3072 or 4096 for higher security
-  - For production, use certificates from a trusted CA
-  - Self-signed certificates are only for development/testing
-  - Rotate certificates before expiration
-`
-}
-
-// Create and manage private CA
-func (tc *TLSCommand) generateCA(cn, org, country string, days, bits int, certFile, keyFile string) error {
-	// Generate RSA key
-	priv, err := rsa.GenerateKey(rand.Reader, bits)
-	if err != nil {
-		return fmt.Errorf("failed to generate key: %w", err)
-	}
-
-	// Create certificate template
-	serialNumber, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	template := x509.Certificate{
-		SerialNumber: serialNumber,
-		Subject: pkix.Name{
-			Organization: []string{org},
-			Country:      []string{country},
-			CommonName:   cn,
-		},
-		NotBefore:             time.Now(),
-		NotAfter:              time.Now().AddDate(0, 0, days),
-		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
-		BasicConstraintsValid: true,
-		IsCA:                  true,
-	}
-
-	// Generate certificate
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	if err != nil {
-		return fmt.Errorf("failed to create certificate: %w", err)
-	}
-
-	// Default output files
-	if certFile == "" {
-		certFile = "ca.crt"
-	}
-	if keyFile == "" {
-		keyFile = "ca.key"
-	}
-
-	// Save certificate
-	if err := saveCert(certFile, certDER); err != nil {
-		return err
-	}
-	if err := saveKey(keyFile, priv); err != nil {
-		return err
-	}
-
-	fmt.Printf("✓ CA certificate generated:\n")
-	fmt.Printf("  Certificate: %s\n", certFile)
-	fmt.Printf("  Private key: %s (mode 0600)\n", keyFile)
-	fmt.Printf("  Valid for:   %d days\n", days)
-	fmt.Printf("  Common name: %s\n", cn)
-
-	return nil
-}
-
-func parseHosts(hostList string) ([]string, []net.IP) {
-	var dnsNames []string
-	var ipAddrs []net.IP
-
-	if hostList == "" {
-		return dnsNames, ipAddrs
-	}
-
-	hosts := strings.Split(hostList, ",")
-	for _, h := range hosts {
-		h = strings.TrimSpace(h)
-		if ip := net.ParseIP(h); ip != nil {
-			ipAddrs = append(ipAddrs, ip)
-		} else {
-			dnsNames = append(dnsNames, h)
-		}
-	}
-
-	return dnsNames, ipAddrs
-}
-
-// Generate self-signed certificate
-func (tc *TLSCommand) generateSelfSigned(cn, org, country, hosts string, days, bits int, certFile, keyFile string) error {
-	// 1. Generate an RSA private key with the specified bit size
-	priv, err := rsa.GenerateKey(rand.Reader, bits)
-	if err != nil {
-		return fmt.Errorf("failed to generate private key: %w", err)
-	}
-
-	// 2. Parse the hosts string into DNS names and IP addresses
-	dnsNames, ipAddrs := parseHosts(hosts)
-
-	// 3. Create the certificate template
-	serialNumber, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-
-	template := x509.Certificate{
-		SerialNumber: serialNumber,
-		Subject: pkix.Name{
-			CommonName:   cn,
-			Organization: []string{org},
-			Country:      []string{country},
-		},
-		NotBefore: time.Now(),
-		NotAfter:  time.Now().AddDate(0, 0, days),
-
-		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
-		IsCA:        false,
-
-		DNSNames:    dnsNames,
-		IPAddresses: ipAddrs,
-	}
-
-	// 4. Create the self-signed certificate
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	if err != nil {
-		return fmt.Errorf("failed to create certificate: %w", err)
-	}
-
-	// 5. Default output filenames
-	if certFile == "" {
-		certFile = "server.crt"
-	}
-	if keyFile == "" {
-		keyFile = "server.key"
-	}
-
-	// 6. Save the certificate with 0644 permissions
-	if err := saveCert(certFile, certDER); err != nil {
-		return err
-	}
-	if err := saveKey(keyFile, priv); err != nil {
-		return err
-	}
-
-	// 7. Print summary
-	fmt.Printf("\n✓ Self-signed certificate generated:\n")
-	fmt.Printf("  Certificate: %s\n", certFile)
-	fmt.Printf("  Private Key: %s (mode 0600)\n", keyFile)
-	fmt.Printf("  Valid for:   %d days\n", days)
-	fmt.Printf("  Common Name: %s\n", cn)
-	if len(hosts) > 0 {
-		fmt.Printf("  Hosts (SANs): %s\n", hosts)
-	}
-
-	return nil
-}
-
-// Generate server cert with CA
-func (tc *TLSCommand) generateServerCert(cn, org, country, hosts, caFile, caKeyFile string, days, bits int, certFile, keyFile string) error {
-	caCert, caKey, err := loadCA(caFile, caKeyFile)
-	if err != nil {
-		return err
-	}
-
-	priv, err := rsa.GenerateKey(rand.Reader, bits)
-	if err != nil {
-		return fmt.Errorf("failed to generate server private key: %w", err)
-	}
-
-	dnsNames, ipAddrs := parseHosts(hosts)
-	serialNumber, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	certExpiry := time.Now().AddDate(0, 0, days)
-	if certExpiry.After(caCert.NotAfter) {
-		return fmt.Errorf("certificate validity period (%d days) exceeds CA expiry (%s)", days, caCert.NotAfter.Format(time.RFC3339))
-	}
-
-	template := x509.Certificate{
-		SerialNumber: serialNumber,
-		Subject: pkix.Name{
-			CommonName:   cn,
-			Organization: []string{org},
-			Country:      []string{country},
-		},
-		NotBefore:   time.Now(),
-		NotAfter:    certExpiry,
-		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		DNSNames:    dnsNames,
-		IPAddresses: ipAddrs,
-	}
-
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, caCert, &priv.PublicKey, caKey)
-	if err != nil {
-		return fmt.Errorf("failed to sign server certificate: %w", err)
-	}
-
-	if certFile == "" {
-		certFile = "server.crt"
-	}
-	if keyFile == "" {
-		keyFile = "server.key"
-	}
-
-	if err := saveCert(certFile, certDER); err != nil {
-		return err
-	}
-	if err := saveKey(keyFile, priv); err != nil {
-		return err
-	}
-
-	fmt.Printf("\n✓ Server certificate generated:\n")
-	fmt.Printf("  Certificate: %s\n", certFile)
-	fmt.Printf("  Private Key: %s (mode 0600)\n", keyFile)
-	fmt.Printf("  Signed by:   CN=%s\n", caCert.Subject.CommonName)
-	if len(hosts) > 0 {
-		fmt.Printf("  Hosts (SANs): %s\n", hosts)
-	}
-	return nil
-}
-
-// Generate client cert with CA
-func (tc *TLSCommand) generateClientCert(cn, org, country, caFile, caKeyFile string, days, bits int, certFile, keyFile string) error {
-	caCert, caKey, err := loadCA(caFile, caKeyFile)
-	if err != nil {
-		return err
-	}
-
-	priv, err := rsa.GenerateKey(rand.Reader, bits)
-	if err != nil {
-		return fmt.Errorf("failed to generate client private key: %w", err)
-	}
-
-	serialNumber, _ := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
-	certExpiry := time.Now().AddDate(0, 0, days)
-	if certExpiry.After(caCert.NotAfter) {
-		return fmt.Errorf("certificate validity period (%d days) exceeds CA expiry (%s)", days, caCert.NotAfter.Format(time.RFC3339))
-	}
-
-	template := x509.Certificate{
-		SerialNumber: serialNumber,
-		Subject: pkix.Name{
-			CommonName:   cn,
-			Organization: []string{org},
-			Country:      []string{country},
-		},
-		NotBefore:   time.Now(),
-		NotAfter:    certExpiry,
-		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
-	}
-
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, caCert, &priv.PublicKey, caKey)
-	if err != nil {
-		return fmt.Errorf("failed to sign client certificate: %w", err)
-	}
-
-	if certFile == "" {
-		certFile = "client.crt"
-	}
-	if keyFile == "" {
-		keyFile = "client.key"
-	}
-
-	if err := saveCert(certFile, certDER); err != nil {
-		return err
-	}
-	if err := saveKey(keyFile, priv); err != nil {
-		return err
-	}
-
-	fmt.Printf("\n✓ Client certificate generated:\n")
-	fmt.Printf("  Certificate: %s\n", certFile)
-	fmt.Printf("  Private Key: %s (mode 0600)\n", keyFile)
-	fmt.Printf("  Signed by:   CN=%s\n", caCert.Subject.CommonName)
-	return nil
-}
-
-// Load cert with CA
-func loadCA(certFile, keyFile string) (*x509.Certificate, *rsa.PrivateKey, error) {
-	// Load CA certificate
-	certPEM, err := os.ReadFile(certFile)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read CA certificate: %w", err)
-	}
-
-	certBlock, _ := pem.Decode(certPEM)
-	if certBlock == nil || certBlock.Type != "CERTIFICATE" {
-		return nil, nil, fmt.Errorf("invalid CA certificate format")
-	}
-
-	caCert, err := x509.ParseCertificate(certBlock.Bytes)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to parse CA certificate: %w", err)
-	}
-
-	// Load CA private key
-	keyPEM, err := os.ReadFile(keyFile)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read CA key: %w", err)
-	}
-
-	keyBlock, _ := pem.Decode(keyPEM)
-	if keyBlock == nil {
-		return nil, nil, fmt.Errorf("invalid CA key format")
-	}
-
-	var caKey *rsa.PrivateKey
-	switch keyBlock.Type {
-	case "RSA PRIVATE KEY":
-		caKey, err = x509.ParsePKCS1PrivateKey(keyBlock.Bytes)
-	case "PRIVATE KEY":
-		parsedKey, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes)
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to parse CA key: %w", err)
-		}
-		var ok bool
-		caKey, ok = parsedKey.(*rsa.PrivateKey)
-		if !ok {
-			return nil, nil, fmt.Errorf("CA key is not RSA")
-		}
-	default:
-		return nil, nil, fmt.Errorf("unsupported CA key type: %s", keyBlock.Type)
-	}
-
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to parse CA private key: %w", err)
-	}
-
-	// Verify CA certificate is actually a CA
-	if !caCert.IsCA {
-		return nil, nil, fmt.Errorf("certificate is not a CA certificate")
-	}
-
-	return caCert, caKey, nil
-}
-
-func saveCert(filename string, certDER []byte) error {
-	certFile, err := os.Create(filename)
-	if err != nil {
-		return fmt.Errorf("failed to create certificate file: %w", err)
-	}
-	defer certFile.Close()
-
-	if err := pem.Encode(certFile, &pem.Block{
-		Type:  "CERTIFICATE",
-		Bytes: certDER,
-	}); err != nil {
-		return fmt.Errorf("failed to write certificate: %w", err)
-	}
-
-	// Set readable permissions
-	if err := os.Chmod(filename, 0644); err != nil {
-		return fmt.Errorf("failed to set certificate permissions: %w", err)
-	}
-
-	return nil
-}
-
-func saveKey(filename string, key *rsa.PrivateKey) error {
-	keyFile, err := os.Create(filename)
-	if err != nil {
-		return fmt.Errorf("failed to create key file: %w", err)
-	}
-	defer keyFile.Close()
-
-	privKeyDER := x509.MarshalPKCS1PrivateKey(key)
-	if err := pem.Encode(keyFile, &pem.Block{
-		Type:  "RSA PRIVATE KEY",
-		Bytes: privKeyDER,
-	}); err != nil {
-		return fmt.Errorf("failed to write private key: %w", err)
-	}
-
-	// Set restricted permissions for private key
-	if err := os.Chmod(filename, 0600); err != nil {
-		return fmt.Errorf("failed to set key permissions: %w", err)
-	}
-
-	return nil
-}

src/cmd/logwisp/commands/version.go

@@ -1,41 +0,0 @@
-// FILE: src/cmd/logwisp/commands/version.go
-package commands
-
-import (
-	"fmt"
-
-	"logwisp/src/internal/version"
-)
-
-// VersionCommand handles version display
-type VersionCommand struct{}
-
-// NewVersionCommand creates a new version command
-func NewVersionCommand() *VersionCommand {
-	return &VersionCommand{}
-}
-
-func (c *VersionCommand) Execute(args []string) error {
-	fmt.Println(version.String())
-	return nil
-}
-
-func (c *VersionCommand) Description() string {
-	return "Show version information"
-}
-
-func (c *VersionCommand) Help() string {
-	return `Version Command - Show LogWisp version information
-
-Usage:
-  logwisp version
-  logwisp -v
-  logwisp --version
-
-Output includes:
-  - Version number
-  - Build date
-  - Git commit hash (if available)
-  - Go version used for compilation
-`
-}

src/cmd/logwisp/main.go

@@ -1,195 +0,0 @@
-// FILE: logwisp/src/cmd/logwisp/main.go
-package main
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/signal"
-	"strings"
-	"syscall"
-	"time"
-
-	"logwisp/src/cmd/logwisp/commands"
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/version"
-
-	"github.com/lixenwraith/log"
-)
-
-var logger *log.Logger
-
-func main() {
-	// Handle subcommands before any config loading
-	// This prevents flag conflicts with lixenwraith/config
-	router := commands.NewCommandRouter()
-	handled, err := router.Route(os.Args)
-
-	if err != nil {
-		// Command execution error
-		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
-		os.Exit(1)
-	}
-
-	if handled {
-		// Command was successfully handled
-		os.Exit(0)
-	}
-
-	// No subcommand, continue with main application
-
-	// Emulates nohup
-	signal.Ignore(syscall.SIGHUP)
-
-	// Load configuration with automatic CLI parsing
-	cfg, err := config.Load(os.Args[1:])
-	if err != nil {
-		if strings.Contains(err.Error(), "not found") && cfg != nil && cfg.ConfigFile != "" {
-			fmt.Fprintf(os.Stderr, "Error: Config file not found: %s\n", cfg.ConfigFile)
-			os.Exit(2)
-		}
-		fmt.Fprintf(os.Stderr, "Error: Failed to load config: %v\n", err)
-		os.Exit(1)
-	}
-
-	// Initialize output handler
-	InitOutputHandler(cfg.Quiet)
-
-	// Handle version
-	if cfg.ShowVersion {
-		fmt.Println(version.String())
-		os.Exit(0)
-	}
-
-	// Background mode spawns a child with internal --background-daemon flag.
-	if cfg.Background && !cfg.BackgroundDaemon {
-		// Prepare arguments for the child process, including originals and daemon flag.
-		args := append(os.Args[1:], "--background-daemon")
-
-		cmd := exec.Command(os.Args[0], args...)
-
-		if err := cmd.Start(); err != nil {
-			FatalError(1, "Failed to start background process: %v\n", err)
-		}
-
-		Print("Started LogWisp in background (PID: %d)\n", cmd.Process.Pid)
-		os.Exit(0) // The parent process exits successfully.
-	}
-
-	// Initialize logger instance and apply configuration
-	if err := initializeLogger(cfg); err != nil {
-		FatalError(1, "Failed to initialize logger: %v\n", err)
-	}
-	defer shutdownLogger()
-
-	// Start the logger
-	if err := logger.Start(); err != nil {
-		FatalError(1, "Failed to start logger: %v\n", err)
-	}
-
-	// Log startup information
-	logger.Info("msg", "LogWisp starting",
-		"version", version.String(),
-		"config_file", cfg.ConfigFile,
-		"log_output", cfg.Logging.Output,
-		"background_mode", cfg.Background)
-
-	// Create context for shutdown
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// Service and hot reload management
-	var reloadManager *ReloadManager
-
-	if cfg.ConfigAutoReload && cfg.ConfigFile != "" {
-		// Use reload manager for dynamic configuration
-		logger.Info("msg", "Config auto-reload enabled",
-			"config_file", cfg.ConfigFile)
-
-		reloadManager = NewReloadManager(cfg.ConfigFile, cfg, logger)
-
-		if err := reloadManager.Start(ctx); err != nil {
-			logger.Error("msg", "Failed to start reload manager", "error", err)
-			os.Exit(1)
-		}
-		defer reloadManager.Shutdown()
-
-		// Setup signal handler with reload support
-		signalHandler := NewSignalHandler(reloadManager, logger)
-		defer signalHandler.Stop()
-
-		// Handle signals in background
-		go func() {
-			sig := signalHandler.Handle(ctx)
-			if sig != nil {
-				logger.Info("msg", "Shutdown signal received",
-					"signal", sig)
-				cancel() // Trigger shutdown
-			}
-		}()
-	} else {
-		// Traditional static bootstrap
-		logger.Info("msg", "Config auto-reload disabled")
-
-		svc, err := bootstrapService(ctx, cfg)
-		if err != nil {
-			logger.Error("msg", "Failed to bootstrap service", "error", err)
-			os.Exit(1)
-		}
-
-		// Start status reporter if enabled (static mode)
-		if !cfg.DisableStatusReporter {
-			go statusReporter(svc, ctx)
-		}
-
-		// Setup traditional signal handling
-		sigChan := make(chan os.Signal, 1)
-		signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
-
-		// Wait for shutdown signal
-		sig := <-sigChan
-
-		// Handle SIGKILL for immediate shutdown
-		if sig == syscall.SIGKILL {
-			os.Exit(137) // Standard exit code for SIGKILL (128 + 9)
-		}
-
-		logger.Info("msg", "Shutdown signal received, starting graceful shutdown...")
-
-		// Shutdown service with timeout
-		shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer shutdownCancel()
-
-		done := make(chan struct{})
-		go func() {
-			svc.Shutdown()
-			close(done)
-		}()
-
-		select {
-		case <-done:
-			logger.Info("msg", "Shutdown complete")
-		case <-shutdownCtx.Done():
-			logger.Error("msg", "Shutdown timeout exceeded - forcing exit")
-			os.Exit(1)
-		}
-
-		return // Exit from static mode
-	}
-
-	// Wait for context cancellation
-	<-ctx.Done()
-
-	// Shutdown is handled by ReloadManager.Shutdown() in defer
-	logger.Info("msg", "Shutdown complete")
-}
-
-func shutdownLogger() {
-	if logger != nil {
-		if err := logger.Shutdown(2 * time.Second); err != nil {
-			// Best effort - can't log the shutdown error
-			Error("Logger shutdown error: %v\n", err)
-		}
-	}
-}

src/cmd/logwisp/reload.go

@@ -1,371 +0,0 @@
-// FILE: src/cmd/logwisp/reload.go
-package main
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/service"
-
-	lconfig "github.com/lixenwraith/config"
-	"github.com/lixenwraith/log"
-)
-
-// Handles configuration hot reload
-type ReloadManager struct {
-	configPath  string
-	service     *service.Service
-	cfg         *config.Config
-	lcfg        *lconfig.Config
-	logger      *log.Logger
-	mu          sync.RWMutex
-	reloadingMu sync.Mutex
-	isReloading bool
-	shutdownCh  chan struct{}
-	wg          sync.WaitGroup
-
-	// Status reporter management
-	statusReporterCancel context.CancelFunc
-	statusReporterMu     sync.Mutex
-}
-
-// Creates a new reload manager
-func NewReloadManager(configPath string, initialCfg *config.Config, logger *log.Logger) *ReloadManager {
-	return &ReloadManager{
-		configPath: configPath,
-		cfg:        initialCfg,
-		logger:     logger,
-		shutdownCh: make(chan struct{}),
-	}
-}
-
-// Begins watching for configuration changes
-func (rm *ReloadManager) Start(ctx context.Context) error {
-	// Bootstrap initial service
-	svc, err := bootstrapService(ctx, rm.cfg)
-	if err != nil {
-		return fmt.Errorf("failed to bootstrap initial service: %w", err)
-	}
-
-	rm.mu.Lock()
-	rm.service = svc
-	rm.mu.Unlock()
-
-	// Start status reporter for initial service
-	if !rm.cfg.DisableStatusReporter {
-		rm.startStatusReporter(ctx, svc)
-	}
-
-	// Use the same lconfig instance from initial load
-	lcfg := config.GetConfigManager()
-	if lcfg == nil {
-		// Config manager not initialized - potential for config bypass
-		return fmt.Errorf("config manager not initialized - cannot enable hot reload")
-	}
-
-	rm.lcfg = lcfg
-
-	// Enable auto-update with custom options
-	watchOpts := lconfig.WatchOptions{
-		PollInterval:      time.Second,
-		Debounce:          500 * time.Millisecond,
-		ReloadTimeout:     30 * time.Second,
-		VerifyPermissions: true,
-	}
-	lcfg.AutoUpdateWithOptions(watchOpts)
-
-	// Start watching for changes
-	rm.wg.Add(1)
-	go rm.watchLoop(ctx)
-
-	rm.logger.Info("msg", "Configuration hot reload enabled",
-		"config_file", rm.configPath)
-
-	return nil
-}
-
-// Monitors configuration changes
-func (rm *ReloadManager) watchLoop(ctx context.Context) {
-	defer rm.wg.Done()
-
-	changeCh := rm.lcfg.Watch()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-rm.shutdownCh:
-			return
-		case changedPath := <-changeCh:
-			// Handle special notifications
-			switch changedPath {
-			case "file_deleted":
-				rm.logger.Error("msg", "Configuration file deleted",
-					"action", "keeping current configuration")
-				continue
-			case "permissions_changed":
-				// Config file permissions changed suspiciously, overlap with file permission check
-				rm.logger.Error("msg", "Configuration file permissions changed",
-					"action", "reload blocked for security")
-				continue
-			case "reload_timeout":
-				rm.logger.Error("msg", "Configuration reload timed out",
-					"action", "keeping current configuration")
-				continue
-			default:
-				if strings.HasPrefix(changedPath, "reload_error:") {
-					rm.logger.Error("msg", "Configuration reload error",
-						"error", strings.TrimPrefix(changedPath, "reload_error:"),
-						"action", "keeping current configuration")
-					continue
-				}
-			}
-
-			// Verify file permissions before reload
-			if err := verifyFilePermissions(rm.configPath); err != nil {
-				rm.logger.Error("msg", "Configuration file permission check failed",
-					"path", rm.configPath,
-					"error", err,
-					"action", "reload blocked for security")
-				continue
-			}
-
-			// Trigger reload for any pipeline-related change
-			if rm.shouldReload(changedPath) {
-				rm.triggerReload(ctx)
-			}
-		}
-	}
-}
-
-// Verify file permissions for security
-func verifyFilePermissions(path string) error {
-	info, err := os.Stat(path)
-	if err != nil {
-		return fmt.Errorf("failed to stat config file: %w", err)
-	}
-
-	// Extract file mode and system stats
-	mode := info.Mode()
-	stat, ok := info.Sys().(*syscall.Stat_t)
-	if !ok {
-		return fmt.Errorf("unable to get file ownership info")
-	}
-
-	// Check ownership - must be current user or root
-	currentUID := uint32(os.Getuid())
-	if stat.Uid != currentUID && stat.Uid != 0 {
-		return fmt.Errorf("config file owned by uid %d, expected %d or 0", stat.Uid, currentUID)
-	}
-
-	// Check permissions - must not be writable by group or other
-	perm := mode.Perm()
-	if perm&0022 != 0 {
-		// Group or other has write permission
-		return fmt.Errorf("insecure permissions %04o - file must not be writable by group/other", perm)
-	}
-
-	return nil
-}
-
-// Determines if a config change requires service reload
-func (rm *ReloadManager) shouldReload(path string) bool {
-	// Pipeline changes always require reload
-	if strings.HasPrefix(path, "pipelines.") || path == "pipelines" {
-		return true
-	}
-
-	// Logging changes don't require service reload
-	if strings.HasPrefix(path, "logging.") {
-		return false
-	}
-
-	// Status reporter changes
-	if path == "disable_status_reporter" {
-		return true
-	}
-
-	return false
-}
-
-// Performs the actual reload
-func (rm *ReloadManager) triggerReload(ctx context.Context) {
-	// Prevent concurrent reloads
-	rm.reloadingMu.Lock()
-	if rm.isReloading {
-		rm.reloadingMu.Unlock()
-		rm.logger.Debug("msg", "Reload already in progress, skipping")
-		return
-	}
-	rm.isReloading = true
-	rm.reloadingMu.Unlock()
-
-	defer func() {
-		rm.reloadingMu.Lock()
-		rm.isReloading = false
-		rm.reloadingMu.Unlock()
-	}()
-
-	rm.logger.Info("msg", "Starting configuration hot reload")
-
-	// Create reload context with timeout
-	reloadCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
-	defer cancel()
-
-	if err := rm.performReload(reloadCtx); err != nil {
-		rm.logger.Error("msg", "Hot reload failed",
-			"error", err,
-			"action", "keeping current configuration and services")
-		return
-	}
-
-	rm.logger.Info("msg", "Configuration hot reload completed successfully")
-}
-
-// Executes the reload process
-func (rm *ReloadManager) performReload(ctx context.Context) error {
-	// Get updated config from lconfig
-	updatedCfg, err := rm.lcfg.AsStruct()
-	if err != nil {
-		return fmt.Errorf("failed to get updated config: %w", err)
-	}
-
-	// AsStruct returns the target pointer, not a new instance
-	newCfg := updatedCfg.(*config.Config)
-
-	// Validate the new config
-	if err := config.ValidateConfig(newCfg); err != nil {
-		return fmt.Errorf("updated config validation failed: %w", err)
-	}
-
-	// Get current service snapshot
-	rm.mu.RLock()
-	oldService := rm.service
-	rm.mu.RUnlock()
-
-	// Try to bootstrap with new configuration
-	rm.logger.Debug("msg", "Bootstrapping new service with updated config")
-	newService, err := bootstrapService(ctx, newCfg)
-	if err != nil {
-		// Bootstrap failed - keep old services running
-		return fmt.Errorf("failed to bootstrap new service (old service still active): %w", err)
-	}
-
-	// Bootstrap succeeded - swap services atomically
-	rm.mu.Lock()
-	rm.service = newService
-	rm.cfg = newCfg
-	rm.mu.Unlock()
-
-	// Stop old status reporter and start new one
-	rm.restartStatusReporter(ctx, newService)
-
-	// Gracefully shutdown old services after swap to minimize downtime
-	go rm.shutdownOldServices(oldService)
-
-	return nil
-}
-
-// Gracefully shuts down old services
-func (rm *ReloadManager) shutdownOldServices(svc *service.Service) {
-	// Give connections time to drain
-	rm.logger.Debug("msg", "Draining connections from old services")
-	time.Sleep(2 * time.Second)
-
-	if svc != nil {
-		rm.logger.Info("msg", "Shutting down old service")
-		svc.Shutdown()
-	}
-
-	rm.logger.Debug("msg", "Old services shutdown complete")
-}
-
-// Starts a new status reporter
-func (rm *ReloadManager) startStatusReporter(ctx context.Context, svc *service.Service) {
-	rm.statusReporterMu.Lock()
-	defer rm.statusReporterMu.Unlock()
-
-	// Create cancellable context for status reporter
-	reporterCtx, cancel := context.WithCancel(ctx)
-	rm.statusReporterCancel = cancel
-
-	go statusReporter(svc, reporterCtx)
-	rm.logger.Debug("msg", "Started status reporter")
-}
-
-// Stops old and starts new status reporter
-func (rm *ReloadManager) restartStatusReporter(ctx context.Context, newService *service.Service) {
-	if rm.cfg.DisableStatusReporter {
-		// Just stop the old one if disabled
-		rm.stopStatusReporter()
-		return
-	}
-
-	rm.statusReporterMu.Lock()
-	defer rm.statusReporterMu.Unlock()
-
-	// Stop old reporter
-	if rm.statusReporterCancel != nil {
-		rm.statusReporterCancel()
-		rm.logger.Debug("msg", "Stopped old status reporter")
-	}
-
-	// Start new reporter
-	reporterCtx, cancel := context.WithCancel(ctx)
-	rm.statusReporterCancel = cancel
-
-	go statusReporter(newService, reporterCtx)
-	rm.logger.Debug("msg", "Started new status reporter")
-}
-
-// Stops the status reporter
-func (rm *ReloadManager) stopStatusReporter() {
-	rm.statusReporterMu.Lock()
-	defer rm.statusReporterMu.Unlock()
-
-	if rm.statusReporterCancel != nil {
-		rm.statusReporterCancel()
-		rm.statusReporterCancel = nil
-		rm.logger.Debug("msg", "Stopped status reporter")
-	}
-}
-
-// Stops the reload manager
-func (rm *ReloadManager) Shutdown() {
-	rm.logger.Info("msg", "Shutting down reload manager")
-
-	// Stop status reporter
-	rm.stopStatusReporter()
-
-	// Stop watching
-	close(rm.shutdownCh)
-	rm.wg.Wait()
-
-	// Stop config watching
-	if rm.lcfg != nil {
-		rm.lcfg.StopAutoUpdate()
-	}
-
-	// Shutdown current services
-	rm.mu.RLock()
-	currentService := rm.service
-	rm.mu.RUnlock()
-
-	if currentService != nil {
-		rm.logger.Info("msg", "Shutting down service")
-		currentService.Shutdown()
-	}
-}
-
-// Returns the current service (thread-safe)
-func (rm *ReloadManager) GetService() *service.Service {
-	rm.mu.RLock()
-	defer rm.mu.RUnlock()
-	return rm.service
-}

src/cmd/logwisp/signal.go

@@ -1,65 +0,0 @@
-// FILE: src/cmd/logwisp/signals.go
-package main
-
-import (
-	"context"
-	"os"
-	"os/signal"
-	"syscall"
-
-	"github.com/lixenwraith/log"
-)
-
-// Manages OS signals
-type SignalHandler struct {
-	reloadManager *ReloadManager
-	logger        *log.Logger
-	sigChan       chan os.Signal
-}
-
-// Creates a signal handler
-func NewSignalHandler(rm *ReloadManager, logger *log.Logger) *SignalHandler {
-	sh := &SignalHandler{
-		reloadManager: rm,
-		logger:        logger,
-		sigChan:       make(chan os.Signal, 1),
-	}
-
-	// Register for signals
-	signal.Notify(sh.sigChan,
-		syscall.SIGINT,
-		syscall.SIGTERM,
-		syscall.SIGHUP,  // Traditional reload signal
-		syscall.SIGUSR1, // Alternative reload signal
-	)
-
-	return sh
-}
-
-// Processes signals
-func (sh *SignalHandler) Handle(ctx context.Context) os.Signal {
-	for {
-		select {
-		case sig := <-sh.sigChan:
-			switch sig {
-			case syscall.SIGHUP, syscall.SIGUSR1:
-				sh.logger.Info("msg", "Reload signal received",
-					"signal", sig)
-				// Trigger manual reload
-				go sh.reloadManager.triggerReload(ctx)
-				// Continue handling signals
-			default:
-				// Return termination signals
-				return sig
-			}
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-// Cleans up signal handling
-func (sh *SignalHandler) Stop() {
-	signal.Stop(sh.sigChan)
-	close(sh.sigChan)
-}

src/cmd/logwisp/status.go

@@ -1,259 +0,0 @@
-// FILE: logwisp/src/cmd/logwisp/status.go
-package main
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/service"
-)
-
-// Periodically logs service status
-func statusReporter(service *service.Service, ctx context.Context) {
-	ticker := time.NewTicker(30 * time.Second)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			// Clean shutdown
-			return
-		case <-ticker.C:
-			if service == nil {
-				logger.Warn("msg", "Status reporter: service is nil",
-					"component", "status_reporter")
-				return
-			}
-
-			// Safely get stats with recovery
-			func() {
-				defer func() {
-					if r := recover(); r != nil {
-						logger.Error("msg", "Panic in status reporter",
-							"component", "status_reporter",
-							"panic", r)
-					}
-				}()
-
-				stats := service.GetGlobalStats()
-				totalPipelines, ok := stats["total_pipelines"].(int)
-				if !ok || totalPipelines == 0 {
-					logger.Warn("msg", "No active pipelines in status report",
-						"component", "status_reporter")
-					return
-				}
-
-				logger.Debug("msg", "Status report",
-					"component", "status_reporter",
-					"active_pipelines", totalPipelines,
-					"time", time.Now().Format("15:04:05"))
-
-				// Log individual pipeline status
-				pipelines := stats["pipelines"].(map[string]any)
-				for name, pipelineStats := range pipelines {
-					logPipelineStatus(name, pipelineStats.(map[string]any))
-				}
-			}()
-		}
-	}
-}
-
-// Logs the status of an individual pipeline
-func logPipelineStatus(name string, stats map[string]any) {
-	statusFields := []any{
-		"msg", "Pipeline status",
-		"pipeline", name,
-	}
-
-	// Add processing statistics
-	if totalProcessed, ok := stats["total_processed"].(uint64); ok {
-		statusFields = append(statusFields, "entries_processed", totalProcessed)
-	}
-	if totalFiltered, ok := stats["total_filtered"].(uint64); ok {
-		statusFields = append(statusFields, "entries_filtered", totalFiltered)
-	}
-
-	// Add source count
-	if sourceCount, ok := stats["source_count"].(int); ok {
-		statusFields = append(statusFields, "sources", sourceCount)
-	}
-
-	// Add sink statistics
-	if sinks, ok := stats["sinks"].([]map[string]any); ok {
-		tcpConns := int64(0)
-		httpConns := int64(0)
-
-		for _, sink := range sinks {
-			sinkType := sink["type"].(string)
-			if activeConns, ok := sink["active_connections"].(int64); ok {
-				switch sinkType {
-				case "tcp":
-					tcpConns += activeConns
-				case "http":
-					httpConns += activeConns
-				}
-			}
-		}
-
-		if tcpConns > 0 {
-			statusFields = append(statusFields, "tcp_connections", tcpConns)
-		}
-		if httpConns > 0 {
-			statusFields = append(statusFields, "http_connections", httpConns)
-		}
-	}
-
-	logger.Debug(statusFields...)
-}
-
-// Logs the configured endpoints for a pipeline
-func displayPipelineEndpoints(cfg config.PipelineConfig) {
-	// Display sink endpoints
-	for i, sinkCfg := range cfg.Sinks {
-		switch sinkCfg.Type {
-		case "tcp":
-			if sinkCfg.TCP != nil {
-				host := "0.0.0.0"
-				if sinkCfg.TCP.Host != "" {
-					host = sinkCfg.TCP.Host
-				}
-
-				logger.Info("msg", "TCP endpoint configured",
-					"component", "main",
-					"pipeline", cfg.Name,
-					"sink_index", i,
-					"listen", fmt.Sprintf("%s:%d", host, sinkCfg.TCP.Port))
-
-				// Display net limit info if configured
-				if sinkCfg.TCP.NetLimit != nil && sinkCfg.TCP.NetLimit.Enabled {
-					logger.Info("msg", "TCP net limiting enabled",
-						"pipeline", cfg.Name,
-						"sink_index", i,
-						"requests_per_second", sinkCfg.TCP.NetLimit.RequestsPerSecond,
-						"burst_size", sinkCfg.TCP.NetLimit.BurstSize)
-				}
-			}
-
-		case "http":
-			if sinkCfg.HTTP != nil {
-				host := "0.0.0.0"
-				if sinkCfg.HTTP.Host != "" {
-					host = sinkCfg.HTTP.Host
-				}
-
-				streamPath := "/stream"
-				statusPath := "/status"
-				if sinkCfg.HTTP.StreamPath != "" {
-					streamPath = sinkCfg.HTTP.StreamPath
-				}
-				if sinkCfg.HTTP.StatusPath != "" {
-					statusPath = sinkCfg.HTTP.StatusPath
-				}
-
-				logger.Info("msg", "HTTP endpoints configured",
-					"pipeline", cfg.Name,
-					"sink_index", i,
-					"listen", fmt.Sprintf("%s:%d", host, sinkCfg.HTTP.Port),
-					"stream_url", fmt.Sprintf("http://%s:%d%s", host, sinkCfg.HTTP.Port, streamPath),
-					"status_url", fmt.Sprintf("http://%s:%d%s", host, sinkCfg.HTTP.Port, statusPath))
-
-				// Display net limit info if configured
-				if sinkCfg.HTTP.NetLimit != nil && sinkCfg.HTTP.NetLimit.Enabled {
-					logger.Info("msg", "HTTP net limiting enabled",
-						"pipeline", cfg.Name,
-						"sink_index", i,
-						"requests_per_second", sinkCfg.HTTP.NetLimit.RequestsPerSecond,
-						"burst_size", sinkCfg.HTTP.NetLimit.BurstSize)
-				}
-			}
-
-		case "file":
-			if sinkCfg.File != nil {
-				logger.Info("msg", "File sink configured",
-					"pipeline", cfg.Name,
-					"sink_index", i,
-					"directory", sinkCfg.File.Directory,
-					"name", sinkCfg.File.Name)
-			}
-
-		case "console":
-			if sinkCfg.Console != nil {
-				logger.Info("msg", "Console sink configured",
-					"pipeline", cfg.Name,
-					"sink_index", i,
-					"target", sinkCfg.Console.Target)
-			}
-		}
-	}
-
-	// Display source endpoints with host support
-	for i, sourceCfg := range cfg.Sources {
-		switch sourceCfg.Type {
-		case "http":
-			if sourceCfg.HTTP != nil {
-				host := "0.0.0.0"
-				if sourceCfg.HTTP.Host != "" {
-					host = sourceCfg.HTTP.Host
-				}
-
-				displayHost := host
-				if host == "0.0.0.0" {
-					displayHost = "localhost"
-				}
-
-				ingestPath := "/ingest"
-				if sourceCfg.HTTP.IngestPath != "" {
-					ingestPath = sourceCfg.HTTP.IngestPath
-				}
-
-				logger.Info("msg", "HTTP source configured",
-					"pipeline", cfg.Name,
-					"source_index", i,
-					"listen", fmt.Sprintf("%s:%d", host, sourceCfg.HTTP.Port),
-					"ingest_url", fmt.Sprintf("http://%s:%d%s", displayHost, sourceCfg.HTTP.Port, ingestPath))
-			}
-
-		case "tcp":
-			if sourceCfg.TCP != nil {
-				host := "0.0.0.0"
-				if sourceCfg.TCP.Host != "" {
-					host = sourceCfg.TCP.Host
-				}
-
-				displayHost := host
-				if host == "0.0.0.0" {
-					displayHost = "localhost"
-				}
-
-				logger.Info("msg", "TCP source configured",
-					"pipeline", cfg.Name,
-					"source_index", i,
-					"listen", fmt.Sprintf("%s:%d", host, sourceCfg.TCP.Port),
-					"endpoint", fmt.Sprintf("%s:%d", displayHost, sourceCfg.TCP.Port))
-			}
-
-		case "directory":
-			if sourceCfg.Directory != nil {
-				logger.Info("msg", "Directory source configured",
-					"pipeline", cfg.Name,
-					"source_index", i,
-					"path", sourceCfg.Directory.Path,
-					"pattern", sourceCfg.Directory.Pattern)
-			}
-
-		case "stdin":
-			logger.Info("msg", "Stdin source configured",
-				"pipeline", cfg.Name,
-				"source_index", i)
-		}
-	}
-
-	// Display filter information
-	if len(cfg.Filters) > 0 {
-		logger.Info("msg", "Filters configured",
-			"pipeline", cfg.Name,
-			"filter_count", len(cfg.Filters))
-	}
-}

src/internal/auth/authenticator.go

@@ -1,213 +0,0 @@
-// FILE: logwisp/src/internal/auth/authenticator.go
-package auth
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"logwisp/src/internal/config"
-
-	"github.com/lixenwraith/log"
-)
-
-// Prevent unbounded map growth
-const maxAuthTrackedIPs = 10000
-
-// Handles all authentication methods for a pipeline
-type Authenticator struct {
-	config *config.ServerAuthConfig
-	logger *log.Logger
-	tokens map[string]bool // token -> valid
-	mu     sync.RWMutex
-
-	// Session tracking
-	sessions  map[string]*Session
-	sessionMu sync.RWMutex
-}
-
-// TODO: only one connection per user, token, mtls
-// TODO: implement tracker logic
-// Represents an authenticated connection
-type Session struct {
-	ID           string
-	Username     string
-	Method       string // basic, token, mtls
-	RemoteAddr   string
-	CreatedAt    time.Time
-	LastActivity time.Time
-}
-
-// Creates a new authenticator from config
-func NewAuthenticator(cfg *config.ServerAuthConfig, logger *log.Logger) (*Authenticator, error) {
-	// SCRAM is handled by ScramManager in sources
-	if cfg == nil || cfg.Type == "none" || cfg.Type == "scram" {
-		return nil, nil
-	}
-
-	a := &Authenticator{
-		config:   cfg,
-		logger:   logger,
-		tokens:   make(map[string]bool),
-		sessions: make(map[string]*Session),
-	}
-
-	// Initialize tokens
-	if cfg.Type == "token" && cfg.Token != nil {
-		for _, token := range cfg.Token.Tokens {
-			a.tokens[token] = true
-		}
-	}
-
-	// Start session cleanup
-	go a.sessionCleanup()
-
-	logger.Info("msg", "Authenticator initialized",
-		"component", "auth",
-		"type", cfg.Type)
-
-	return a, nil
-}
-
-// Handles HTTP authentication headers
-func (a *Authenticator) AuthenticateHTTP(authHeader, remoteAddr string) (*Session, error) {
-	if a == nil || a.config.Type == "none" {
-		return &Session{
-			ID:         generateSessionID(),
-			Method:     "none",
-			RemoteAddr: remoteAddr,
-			CreatedAt:  time.Now(),
-		}, nil
-	}
-
-	var session *Session
-	var err error
-
-	switch a.config.Type {
-	case "token":
-		session, err = a.authenticateToken(authHeader, remoteAddr)
-	default:
-		err = fmt.Errorf("unsupported auth type: %s", a.config.Type)
-	}
-
-	if err != nil {
-		time.Sleep(500 * time.Millisecond)
-		return nil, err
-	}
-
-	return session, nil
-}
-
-func (a *Authenticator) authenticateToken(authHeader, remoteAddr string) (*Session, error) {
-	if !strings.HasPrefix(authHeader, "Token") {
-		return nil, fmt.Errorf("invalid token auth header")
-	}
-
-	token := authHeader[7:]
-	return a.validateToken(token, remoteAddr)
-}
-
-func (a *Authenticator) validateToken(token, remoteAddr string) (*Session, error) {
-	// Check static tokens first
-	a.mu.RLock()
-	isValid := a.tokens[token]
-	a.mu.RUnlock()
-
-	if !isValid {
-		return nil, fmt.Errorf("invalid token")
-	}
-
-	session := &Session{
-		ID:           generateSessionID(),
-		Method:       "token",
-		RemoteAddr:   remoteAddr,
-		CreatedAt:    time.Now(),
-		LastActivity: time.Now(),
-	}
-	a.storeSession(session)
-	return session, nil
-}
-
-func (a *Authenticator) storeSession(session *Session) {
-	a.sessionMu.Lock()
-	a.sessions[session.ID] = session
-	a.sessionMu.Unlock()
-
-	a.logger.Info("msg", "Session created",
-		"component", "auth",
-		"session_id", session.ID,
-		"username", session.Username,
-		"method", session.Method,
-		"remote_addr", session.RemoteAddr)
-}
-
-func (a *Authenticator) sessionCleanup() {
-	ticker := time.NewTicker(5 * time.Minute)
-	defer ticker.Stop()
-
-	for range ticker.C {
-		a.sessionMu.Lock()
-		now := time.Now()
-		for id, session := range a.sessions {
-			if now.Sub(session.LastActivity) > 30*time.Minute {
-				delete(a.sessions, id)
-				a.logger.Debug("msg", "Session expired",
-					"component", "auth",
-					"session_id", id)
-			}
-		}
-		a.sessionMu.Unlock()
-	}
-}
-
-func generateSessionID() string {
-	b := make([]byte, 32)
-	if _, err := rand.Read(b); err != nil {
-		// Fallback to a less secure method if crypto/rand fails
-		return fmt.Sprintf("fallback-%d", time.Now().UnixNano())
-	}
-	return base64.URLEncoding.EncodeToString(b)
-}
-
-// Checks if a session is still valid
-func (a *Authenticator) ValidateSession(sessionID string) bool {
-	if a == nil {
-		return true
-	}
-
-	a.sessionMu.RLock()
-	session, exists := a.sessions[sessionID]
-	a.sessionMu.RUnlock()
-
-	if !exists {
-		return false
-	}
-
-	// Update activity
-	a.sessionMu.Lock()
-	session.LastActivity = time.Now()
-	a.sessionMu.Unlock()
-
-	return true
-}
-
-// Returns authentication statistics
-func (a *Authenticator) GetStats() map[string]any {
-	if a == nil {
-		return map[string]any{"enabled": false}
-	}
-
-	a.sessionMu.RLock()
-	sessionCount := len(a.sessions)
-	a.sessionMu.RUnlock()
-
-	return map[string]any{
-		"enabled":         true,
-		"type":            a.config.Type,
-		"active_sessions": sessionCount,
-		"static_tokens":   len(a.tokens),
-	}
-}

src/internal/auth/scram_client.go

@@ -1,106 +0,0 @@
-// FILE: src/internal/auth/scram_client.go
-package auth
-
-import (
-	"crypto/rand"
-	"crypto/sha256"
-	"crypto/subtle"
-	"encoding/base64"
-	"fmt"
-
-	"golang.org/x/crypto/argon2"
-)
-
-// Client handles SCRAM client-side authentication
-type ScramClient struct {
-	Username string
-	Password string
-
-	// Handshake state
-	clientNonce string
-	serverFirst *ServerFirst
-	authMessage string
-	serverKey   []byte
-}
-
-// NewScramClient creates SCRAM client
-func NewScramClient(username, password string) *ScramClient {
-	return &ScramClient{
-		Username: username,
-		Password: password,
-	}
-}
-
-// StartAuthentication generates ClientFirst message
-func (c *ScramClient) StartAuthentication() (*ClientFirst, error) {
-	// Generate client nonce
-	nonce := make([]byte, 32)
-	if _, err := rand.Read(nonce); err != nil {
-		return nil, fmt.Errorf("failed to generate nonce: %w", err)
-	}
-	c.clientNonce = base64.StdEncoding.EncodeToString(nonce)
-
-	return &ClientFirst{
-		Username:    c.Username,
-		ClientNonce: c.clientNonce,
-	}, nil
-}
-
-// ProcessServerFirst handles server challenge
-func (c *ScramClient) ProcessServerFirst(msg *ServerFirst) (*ClientFinal, error) {
-	c.serverFirst = msg
-
-	// Decode salt
-	salt, err := base64.StdEncoding.DecodeString(msg.Salt)
-	if err != nil {
-		return nil, fmt.Errorf("invalid salt encoding: %w", err)
-	}
-
-	// Derive keys using Argon2id
-	saltedPassword := argon2.IDKey([]byte(c.Password), salt,
-		msg.ArgonTime, msg.ArgonMemory, msg.ArgonThreads, 32)
-
-	clientKey := computeHMAC(saltedPassword, []byte("Client Key"))
-	serverKey := computeHMAC(saltedPassword, []byte("Server Key"))
-	storedKey := sha256.Sum256(clientKey)
-
-	// Build auth message
-	clientFirstBare := fmt.Sprintf("u=%s,n=%s", c.Username, c.clientNonce)
-	clientFinalBare := fmt.Sprintf("r=%s", msg.FullNonce)
-	c.authMessage = clientFirstBare + "," + msg.Marshal() + "," + clientFinalBare
-
-	// Compute client proof
-	clientSignature := computeHMAC(storedKey[:], []byte(c.authMessage))
-	clientProof := xorBytes(clientKey, clientSignature)
-
-	// Store server key for verification
-	c.serverKey = serverKey
-
-	return &ClientFinal{
-		FullNonce:   msg.FullNonce,
-		ClientProof: base64.StdEncoding.EncodeToString(clientProof),
-	}, nil
-}
-
-// VerifyServerFinal validates server signature
-func (c *ScramClient) VerifyServerFinal(msg *ServerFinal) error {
-	if c.authMessage == "" || c.serverKey == nil {
-		return fmt.Errorf("invalid handshake state")
-	}
-
-	// Compute expected server signature
-	expectedSig := computeHMAC(c.serverKey, []byte(c.authMessage))
-
-	// Decode received signature
-	receivedSig, err := base64.StdEncoding.DecodeString(msg.ServerSignature)
-	if err != nil {
-		return fmt.Errorf("invalid signature encoding: %w", err)
-	}
-
-	// ☢ SECURITY: Constant-time comparison
-	if subtle.ConstantTimeCompare(expectedSig, receivedSig) != 1 {
-		return fmt.Errorf("server authentication failed")
-	}
-
-	return nil
-}

src/internal/auth/scram_credential.go

@@ -1,108 +0,0 @@
-// FILE: src/internal/auth/scram_credential.go
-package auth
-
-import (
-	"crypto/hmac"
-	"crypto/sha256"
-	"crypto/subtle"
-	"encoding/base64"
-	"fmt"
-	"strings"
-
-	"logwisp/src/internal/core"
-
-	"golang.org/x/crypto/argon2"
-)
-
-// Credential stores SCRAM authentication data
-type Credential struct {
-	Username     string
-	Salt         []byte // 16+ bytes
-	ArgonTime    uint32 // e.g., 3
-	ArgonMemory  uint32 // e.g., 64*1024 KiB
-	ArgonThreads uint8  // e.g., 4
-	StoredKey    []byte // SHA256(ClientKey)
-	ServerKey    []byte // For server auth
-	PHCHash      string
-}
-
-// DeriveCredential creates SCRAM credential from password
-func DeriveCredential(username, password string, salt []byte, time, memory uint32, threads uint8) (*Credential, error) {
-	if len(salt) < 16 {
-		return nil, fmt.Errorf("salt must be at least 16 bytes")
-	}
-
-	// Derive salted password using Argon2id
-	saltedPassword := argon2.IDKey([]byte(password), salt, time, memory, threads, core.Argon2KeyLen)
-
-	// Construct PHC format for basic auth compatibility
-	saltB64 := base64.RawStdEncoding.EncodeToString(salt)
-	hashB64 := base64.RawStdEncoding.EncodeToString(saltedPassword)
-	phcHash := fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s",
-		argon2.Version, memory, time, threads, saltB64, hashB64)
-
-	// Derive keys
-	clientKey := computeHMAC(saltedPassword, []byte("Client Key"))
-	serverKey := computeHMAC(saltedPassword, []byte("Server Key"))
-	storedKey := sha256.Sum256(clientKey)
-
-	return &Credential{
-		Username:     username,
-		Salt:         salt,
-		ArgonTime:    time,
-		ArgonMemory:  memory,
-		ArgonThreads: threads,
-		StoredKey:    storedKey[:],
-		ServerKey:    serverKey,
-		PHCHash:      phcHash,
-	}, nil
-}
-
-// MigrateFromPHC converts existing Argon2 PHC hash to SCRAM credential
-func MigrateFromPHC(username, password, phcHash string) (*Credential, error) {
-	// Parse PHC: $argon2id$v=19$m=65536,t=3,p=4$salt$hash
-	parts := strings.Split(phcHash, "$")
-	if len(parts) != 6 || parts[1] != "argon2id" {
-		return nil, fmt.Errorf("invalid PHC format")
-	}
-
-	var memory, time uint32
-	var threads uint8
-	fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &time, &threads)
-
-	salt, err := base64.RawStdEncoding.DecodeString(parts[4])
-	if err != nil {
-		return nil, fmt.Errorf("invalid salt encoding: %w", err)
-	}
-
-	expectedHash, err := base64.RawStdEncoding.DecodeString(parts[5])
-	if err != nil {
-		return nil, fmt.Errorf("invalid hash encoding: %w", err)
-	}
-
-	// Verify password matches
-	computedHash := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(expectedHash)))
-	if subtle.ConstantTimeCompare(computedHash, expectedHash) != 1 {
-		return nil, fmt.Errorf("password verification failed")
-	}
-
-	// Now derive SCRAM credential
-	return DeriveCredential(username, password, salt, time, memory, threads)
-}
-
-func computeHMAC(key, message []byte) []byte {
-	mac := hmac.New(sha256.New, key)
-	mac.Write(message)
-	return mac.Sum(nil)
-}
-
-func xorBytes(a, b []byte) []byte {
-	if len(a) != len(b) {
-		panic("xor length mismatch")
-	}
-	result := make([]byte, len(a))
-	for i := range a {
-		result[i] = a[i] ^ b[i]
-	}
-	return result
-}

src/internal/auth/scram_manager.go

@@ -1,83 +0,0 @@
-// FILE: src/internal/auth/scram_manager.go
-package auth
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"fmt"
-
-	"logwisp/src/internal/config"
-)
-
-// ScramManager provides high-level SCRAM operations with rate limiting
-type ScramManager struct {
-	server *ScramServer
-}
-
-// NewScramManager creates SCRAM manager
-func NewScramManager(scramAuthCfg *config.ScramAuthConfig) *ScramManager {
-	manager := &ScramManager{
-		server: NewScramServer(),
-	}
-
-	// Load users from SCRAM config
-	for _, user := range scramAuthCfg.Users {
-		storedKey, err := base64.StdEncoding.DecodeString(user.StoredKey)
-		if err != nil {
-			// Skip user with invalid stored key
-			continue
-		}
-
-		serverKey, err := base64.StdEncoding.DecodeString(user.ServerKey)
-		if err != nil {
-			// Skip user with invalid server key
-			continue
-		}
-
-		salt, err := base64.StdEncoding.DecodeString(user.Salt)
-		if err != nil {
-			// Skip user with invalid salt
-			continue
-		}
-
-		cred := &Credential{
-			Username:     user.Username,
-			StoredKey:    storedKey,
-			ServerKey:    serverKey,
-			Salt:         salt,
-			ArgonTime:    user.ArgonTime,
-			ArgonMemory:  user.ArgonMemory,
-			ArgonThreads: user.ArgonThreads,
-		}
-		manager.server.AddCredential(cred)
-	}
-
-	return manager
-}
-
-// RegisterUser creates new user credential
-func (sm *ScramManager) RegisterUser(username, password string) error {
-	salt := make([]byte, 16)
-	if _, err := rand.Read(salt); err != nil {
-		return fmt.Errorf("salt generation failed: %w", err)
-	}
-
-	cred, err := DeriveCredential(username, password, salt,
-		sm.server.DefaultTime, sm.server.DefaultMemory, sm.server.DefaultThreads)
-	if err != nil {
-		return err
-	}
-
-	sm.server.AddCredential(cred)
-	return nil
-}
-
-// HandleClientFirst wraps server's HandleClientFirst
-func (sm *ScramManager) HandleClientFirst(msg *ClientFirst) (*ServerFirst, error) {
-	return sm.server.HandleClientFirst(msg)
-}
-
-// HandleClientFinal wraps server's HandleClientFinal
-func (sm *ScramManager) HandleClientFinal(msg *ClientFinal) (*ServerFinal, error) {
-	return sm.server.HandleClientFinal(msg)
-}

src/internal/auth/scram_message.go

@@ -1,38 +0,0 @@
-// FILE: src/internal/auth/scram_message.go
-package auth
-
-import (
-	"fmt"
-)
-
-// ClientFirst initiates authentication
-type ClientFirst struct {
-	Username    string `json:"u"`
-	ClientNonce string `json:"n"`
-}
-
-// ServerFirst contains server challenge
-type ServerFirst struct {
-	FullNonce    string `json:"r"` // client_nonce + server_nonce
-	Salt         string `json:"s"` // base64
-	ArgonTime    uint32 `json:"t"`
-	ArgonMemory  uint32 `json:"m"`
-	ArgonThreads uint8  `json:"p"`
-}
-
-// ClientFinal contains client proof
-type ClientFinal struct {
-	FullNonce   string `json:"r"`
-	ClientProof string `json:"p"` // base64
-}
-
-// ServerFinal contains server signature for mutual auth
-type ServerFinal struct {
-	ServerSignature string `json:"v"` // base64
-	SessionID       string `json:"sid,omitempty"`
-}
-
-func (sf *ServerFirst) Marshal() string {
-	return fmt.Sprintf("r=%s,s=%s,t=%d,m=%d,p=%d",
-		sf.FullNonce, sf.Salt, sf.ArgonTime, sf.ArgonMemory, sf.ArgonThreads)
-}

src/internal/auth/scram_protocol.go

@@ -1,117 +0,0 @@
-// FILE: src/internal/auth/scram_protocol.go
-package auth
-
-import (
-	"encoding/json"
-	"fmt"
-	"strings"
-	"time"
-
-	"github.com/lixenwraith/log"
-	"github.com/panjf2000/gnet/v2"
-)
-
-// ScramProtocolHandler handles SCRAM message exchange for TCP
-type ScramProtocolHandler struct {
-	manager *ScramManager
-	logger  *log.Logger
-}
-
-// NewScramProtocolHandler creates protocol handler
-func NewScramProtocolHandler(manager *ScramManager, logger *log.Logger) *ScramProtocolHandler {
-	return &ScramProtocolHandler{
-		manager: manager,
-		logger:  logger,
-	}
-}
-
-// HandleAuthMessage processes a complete auth line from buffer
-func (sph *ScramProtocolHandler) HandleAuthMessage(line []byte, conn gnet.Conn) (authenticated bool, session *Session, err error) {
-	// Parse SCRAM messages
-	parts := strings.Fields(string(line))
-	if len(parts) < 2 {
-		conn.AsyncWrite([]byte("SCRAM-FAIL Invalid message format\n"), nil)
-		return false, nil, fmt.Errorf("invalid message format")
-	}
-
-	switch parts[0] {
-	case "SCRAM-FIRST":
-		// Parse ClientFirst JSON
-		var clientFirst ClientFirst
-		if err := json.Unmarshal([]byte(parts[1]), &clientFirst); err != nil {
-			conn.AsyncWrite([]byte("SCRAM-FAIL Invalid JSON\n"), nil)
-			return false, nil, fmt.Errorf("invalid JSON")
-		}
-
-		// Process with SCRAM server
-		serverFirst, err := sph.manager.HandleClientFirst(&clientFirst)
-		if err != nil {
-			// Still send challenge to prevent user enumeration
-			response, _ := json.Marshal(serverFirst)
-			conn.AsyncWrite([]byte(fmt.Sprintf("SCRAM-CHALLENGE %s\n", response)), nil)
-			return false, nil, err
-		}
-
-		// Send ServerFirst challenge
-		response, _ := json.Marshal(serverFirst)
-		conn.AsyncWrite([]byte(fmt.Sprintf("SCRAM-CHALLENGE %s\n", response)), nil)
-		return false, nil, nil // Not authenticated yet
-
-	case "SCRAM-PROOF":
-		// Parse ClientFinal JSON
-		var clientFinal ClientFinal
-		if err := json.Unmarshal([]byte(parts[1]), &clientFinal); err != nil {
-			conn.AsyncWrite([]byte("SCRAM-FAIL Invalid JSON\n"), nil)
-			return false, nil, fmt.Errorf("invalid JSON")
-		}
-
-		// Verify proof
-		serverFinal, err := sph.manager.HandleClientFinal(&clientFinal)
-		if err != nil {
-			conn.AsyncWrite([]byte("SCRAM-FAIL Authentication failed\n"), nil)
-			return false, nil, err
-		}
-
-		// Authentication successful
-		session = &Session{
-			ID:         serverFinal.SessionID,
-			Method:     "scram-sha-256",
-			RemoteAddr: conn.RemoteAddr().String(),
-			CreatedAt:  time.Now(),
-		}
-
-		// Send ServerFinal with signature
-		response, _ := json.Marshal(serverFinal)
-		conn.AsyncWrite([]byte(fmt.Sprintf("SCRAM-OK %s\n", response)), nil)
-
-		return true, session, nil
-
-	default:
-		conn.AsyncWrite([]byte("SCRAM-FAIL Unknown command\n"), nil)
-		return false, nil, fmt.Errorf("unknown command: %s", parts[0])
-	}
-}
-
-// FormatSCRAMRequest formats a SCRAM protocol message for TCP
-func FormatSCRAMRequest(command string, data interface{}) (string, error) {
-	jsonData, err := json.Marshal(data)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshal %s: %w", command, err)
-	}
-	return fmt.Sprintf("%s %s\n", command, jsonData), nil
-}
-
-// ParseSCRAMResponse parses a SCRAM protocol response from TCP
-func ParseSCRAMResponse(response string) (command string, data string, err error) {
-	response = strings.TrimSpace(response)
-	parts := strings.SplitN(response, " ", 2)
-	if len(parts) < 1 {
-		return "", "", fmt.Errorf("empty response")
-	}
-
-	command = parts[0]
-	if len(parts) > 1 {
-		data = parts[1]
-	}
-	return command, data, nil
-}

src/internal/auth/scram_server.go

@@ -1,174 +0,0 @@
-// FILE: src/internal/auth/scram_server.go
-package auth
-
-import (
-	"crypto/rand"
-	"crypto/sha256"
-	"crypto/subtle"
-	"encoding/base64"
-	"fmt"
-	"sync"
-	"time"
-
-	"logwisp/src/internal/core"
-)
-
-// Server handles SCRAM authentication
-type ScramServer struct {
-	credentials map[string]*Credential
-	handshakes  map[string]*HandshakeState
-	mu          sync.RWMutex
-
-	// TODO: configurability useful? to be included in config or refactor to use core.const directly for simplicity
-	// Default Argon2 params for new registrations
-	DefaultTime    uint32
-	DefaultMemory  uint32
-	DefaultThreads uint8
-}
-
-// HandshakeState tracks ongoing authentication
-type HandshakeState struct {
-	Username    string
-	ClientNonce string
-	ServerNonce string
-	FullNonce   string
-	Credential  *Credential
-	CreatedAt   time.Time
-}
-
-// NewScramServer creates SCRAM server
-func NewScramServer() *ScramServer {
-	return &ScramServer{
-		credentials:    make(map[string]*Credential),
-		handshakes:     make(map[string]*HandshakeState),
-		DefaultTime:    core.Argon2Time,
-		DefaultMemory:  core.Argon2Memory,
-		DefaultThreads: core.Argon2Threads,
-	}
-}
-
-// AddCredential registers user credential
-func (s *ScramServer) AddCredential(cred *Credential) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.credentials[cred.Username] = cred
-}
-
-// HandleClientFirst processes initial auth request
-func (s *ScramServer) HandleClientFirst(msg *ClientFirst) (*ServerFirst, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	// Check if user exists
-	cred, exists := s.credentials[msg.Username]
-	if !exists {
-		// Prevent user enumeration - still generate response
-		salt := make([]byte, 16)
-		rand.Read(salt)
-		serverNonce := generateNonce()
-
-		return &ServerFirst{
-			FullNonce:    msg.ClientNonce + serverNonce,
-			Salt:         base64.StdEncoding.EncodeToString(salt),
-			ArgonTime:    s.DefaultTime,
-			ArgonMemory:  s.DefaultMemory,
-			ArgonThreads: s.DefaultThreads,
-		}, fmt.Errorf("invalid credentials")
-	}
-
-	// Generate server nonce
-	serverNonce := generateNonce()
-	fullNonce := msg.ClientNonce + serverNonce
-
-	// Store handshake state
-	state := &HandshakeState{
-		Username:    msg.Username,
-		ClientNonce: msg.ClientNonce,
-		ServerNonce: serverNonce,
-		FullNonce:   fullNonce,
-		Credential:  cred,
-		CreatedAt:   time.Now(),
-	}
-	s.handshakes[fullNonce] = state
-
-	// Cleanup old handshakes
-	s.cleanupHandshakes()
-
-	return &ServerFirst{
-		FullNonce:    fullNonce,
-		Salt:         base64.StdEncoding.EncodeToString(cred.Salt),
-		ArgonTime:    cred.ArgonTime,
-		ArgonMemory:  cred.ArgonMemory,
-		ArgonThreads: cred.ArgonThreads,
-	}, nil
-}
-
-// HandleClientFinal verifies client proof
-func (s *ScramServer) HandleClientFinal(msg *ClientFinal) (*ServerFinal, error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	state, exists := s.handshakes[msg.FullNonce]
-	if !exists {
-		return nil, fmt.Errorf("invalid nonce or expired handshake")
-	}
-	defer delete(s.handshakes, msg.FullNonce)
-
-	// Check timeout
-	if time.Since(state.CreatedAt) > 60*time.Second {
-		return nil, fmt.Errorf("handshake timeout")
-	}
-
-	// Decode client proof
-	clientProof, err := base64.StdEncoding.DecodeString(msg.ClientProof)
-	if err != nil {
-		return nil, fmt.Errorf("invalid proof encoding")
-	}
-
-	// Build auth message
-	clientFirstBare := fmt.Sprintf("u=%s,n=%s", state.Username, state.ClientNonce)
-	serverFirst := &ServerFirst{
-		FullNonce:    state.FullNonce,
-		Salt:         base64.StdEncoding.EncodeToString(state.Credential.Salt),
-		ArgonTime:    state.Credential.ArgonTime,
-		ArgonMemory:  state.Credential.ArgonMemory,
-		ArgonThreads: state.Credential.ArgonThreads,
-	}
-	clientFinalBare := fmt.Sprintf("r=%s", msg.FullNonce)
-	authMessage := clientFirstBare + "," + serverFirst.Marshal() + "," + clientFinalBare
-
-	// Compute client signature
-	clientSignature := computeHMAC(state.Credential.StoredKey, []byte(authMessage))
-
-	// XOR to get ClientKey
-	clientKey := xorBytes(clientProof, clientSignature)
-
-	// Verify by computing StoredKey
-	computedStoredKey := sha256.Sum256(clientKey)
-	if subtle.ConstantTimeCompare(computedStoredKey[:], state.Credential.StoredKey) != 1 {
-		return nil, fmt.Errorf("authentication failed")
-	}
-
-	// Generate server signature for mutual auth
-	serverSignature := computeHMAC(state.Credential.ServerKey, []byte(authMessage))
-
-	return &ServerFinal{
-		ServerSignature: base64.StdEncoding.EncodeToString(serverSignature),
-		SessionID:       generateSessionID(),
-	}, nil
-}
-
-func (s *ScramServer) cleanupHandshakes() {
-	cutoff := time.Now().Add(-60 * time.Second)
-	for nonce, state := range s.handshakes {
-		if state.CreatedAt.Before(cutoff) {
-			delete(s.handshakes, nonce)
-		}
-	}
-}
-
-func generateNonce() string {
-	b := make([]byte, 32)
-	rand.Read(b)
-	return base64.StdEncoding.EncodeToString(b)
-}

src/internal/config/config.go

@@ -1,379 +0,0 @@
-// FILE: logwisp/src/internal/config/config.go
-package config
-
-// --- LogWisp Configuration Options ---
-
-type Config struct {
-	// Top-level flags for application control
-	Background  bool `toml:"background"`
-	ShowVersion bool `toml:"version"`
-	Quiet       bool `toml:"quiet"`
-
-	// Runtime behavior flags
-	DisableStatusReporter bool `toml:"disable_status_reporter"`
-	ConfigAutoReload      bool `toml:"config_auto_reload"`
-
-	// Internal flag indicating demonized child process (DO NOT SET IN CONFIG FILE)
-	BackgroundDaemon bool
-
-	// Configuration file path
-	ConfigFile string `toml:"config_file"`
-
-	// Existing fields
-	Logging   *LogConfig       `toml:"logging"`
-	Pipelines []PipelineConfig `toml:"pipelines"`
-}
-
-// --- Logging Options ---
-
-// Represents logging configuration for LogWisp
-type LogConfig struct {
-	// Output mode: "file", "stdout", "stderr", "split", "all", "none"
-	Output string `toml:"output"`
-
-	// Log level: "debug", "info", "warn", "error"
-	Level string `toml:"level"`
-
-	// File output settings (when Output includes "file" or "all")
-	File *LogFileConfig `toml:"file"`
-
-	// Console output settings
-	Console *LogConsoleConfig `toml:"console"`
-}
-
-type LogFileConfig struct {
-	// Directory for log files
-	Directory string `toml:"directory"`
-
-	// Base name for log files
-	Name string `toml:"name"`
-
-	// Maximum size per log file in MB
-	MaxSizeMB int64 `toml:"max_size_mb"`
-
-	// Maximum total size of all logs in MB
-	MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
-
-	// Log retention in hours (0 = disabled)
-	RetentionHours float64 `toml:"retention_hours"`
-}
-
-type LogConsoleConfig struct {
-	// Target for console output: "stdout", "stderr", "split"
-	// "split": info/debug to stdout, warn/error to stderr
-	Target string `toml:"target"`
-
-	// Format: "txt" or "json"
-	Format string `toml:"format"`
-}
-
-// --- Pipeline Options ---
-
-type PipelineConfig struct {
-	Name      string           `toml:"name"`
-	Sources   []SourceConfig   `toml:"sources"`
-	RateLimit *RateLimitConfig `toml:"rate_limit"`
-	Filters   []FilterConfig   `toml:"filters"`
-	Format    *FormatConfig    `toml:"format"`
-
-	Sinks []SinkConfig `toml:"sinks"`
-	// Auth  *ServerAuthConfig `toml:"auth"` // Global auth for pipeline
-}
-
-// Common configuration structs used across components
-
-type NetLimitConfig struct {
-	Enabled             bool     `toml:"enabled"`
-	MaxConnections      int64    `toml:"max_connections"`
-	RequestsPerSecond   float64  `toml:"requests_per_second"`
-	BurstSize           int64    `toml:"burst_size"`
-	ResponseMessage     string   `toml:"response_message"`
-	ResponseCode        int64    `toml:"response_code"` // Default: 429
-	MaxConnectionsPerIP int64    `toml:"max_connections_per_ip"`
-	MaxConnectionsTotal int64    `toml:"max_connections_total"`
-	IPWhitelist         []string `toml:"ip_whitelist"`
-	IPBlacklist         []string `toml:"ip_blacklist"`
-}
-
-type TLSConfig struct {
-	Enabled    bool   `toml:"enabled"`
-	CertFile   string `toml:"cert_file"`
-	KeyFile    string `toml:"key_file"`
-	CAFile     string `toml:"ca_file"`
-	ServerName string `toml:"server_name"` // for client verification
-	SkipVerify bool   `toml:"skip_verify"`
-
-	// Client certificate authentication
-	ClientAuth       bool   `toml:"client_auth"`
-	ClientCAFile     string `toml:"client_ca_file"`
-	VerifyClientCert bool   `toml:"verify_client_cert"`
-
-	// TLS version constraints
-	MinVersion string `toml:"min_version"` // "TLS1.2", "TLS1.3"
-	MaxVersion string `toml:"max_version"`
-
-	// Cipher suites (comma-separated list)
-	CipherSuites string `toml:"cipher_suites"`
-}
-
-type HeartbeatConfig struct {
-	Enabled          bool   `toml:"enabled"`
-	IntervalMS       int64  `toml:"interval_ms"`
-	IncludeTimestamp bool   `toml:"include_timestamp"`
-	IncludeStats     bool   `toml:"include_stats"`
-	Format           string `toml:"format"`
-}
-
-type ClientAuthConfig struct {
-	Type     string `toml:"type"` // "none", "basic", "token", "scram"
-	Username string `toml:"username"`
-	Password string `toml:"password"`
-	Token    string `toml:"token"`
-}
-
-// --- Source Options ---
-
-type SourceConfig struct {
-	Type string `toml:"type"`
-
-	// Polymorphic - only one populated based on type
-	Directory *DirectorySourceOptions `toml:"directory,omitempty"`
-	Stdin     *StdinSourceOptions     `toml:"stdin,omitempty"`
-	HTTP      *HTTPSourceOptions      `toml:"http,omitempty"`
-	TCP       *TCPSourceOptions       `toml:"tcp,omitempty"`
-}
-
-type DirectorySourceOptions struct {
-	Path            string `toml:"path"`
-	Pattern         string `toml:"pattern"` // glob pattern
-	CheckIntervalMS int64  `toml:"check_interval_ms"`
-	Recursive       bool   `toml:"recursive"` // TODO: implement logic
-}
-
-type StdinSourceOptions struct {
-	BufferSize int64 `toml:"buffer_size"`
-}
-
-type HTTPSourceOptions struct {
-	Host               string            `toml:"host"`
-	Port               int64             `toml:"port"`
-	IngestPath         string            `toml:"ingest_path"`
-	BufferSize         int64             `toml:"buffer_size"`
-	MaxRequestBodySize int64             `toml:"max_body_size"`
-	ReadTimeout        int64             `toml:"read_timeout_ms"`
-	WriteTimeout       int64             `toml:"write_timeout_ms"`
-	NetLimit           *NetLimitConfig   `toml:"net_limit"`
-	TLS                *TLSConfig        `toml:"tls"`
-	Auth               *ServerAuthConfig `toml:"auth"`
-}
-
-type TCPSourceOptions struct {
-	Host            string            `toml:"host"`
-	Port            int64             `toml:"port"`
-	BufferSize      int64             `toml:"buffer_size"`
-	ReadTimeout     int64             `toml:"read_timeout_ms"`
-	KeepAlive       bool              `toml:"keep_alive"`
-	KeepAlivePeriod int64             `toml:"keep_alive_period_ms"`
-	NetLimit        *NetLimitConfig   `toml:"net_limit"`
-	Auth            *ServerAuthConfig `toml:"auth"`
-}
-
-// --- Sink Options ---
-
-type SinkConfig struct {
-	Type string `toml:"type"`
-
-	// Polymorphic - only one populated based on type
-	Console    *ConsoleSinkOptions    `toml:"console,omitempty"`
-	File       *FileSinkOptions       `toml:"file,omitempty"`
-	HTTP       *HTTPSinkOptions       `toml:"http,omitempty"`
-	TCP        *TCPSinkOptions        `toml:"tcp,omitempty"`
-	HTTPClient *HTTPClientSinkOptions `toml:"http_client,omitempty"`
-	TCPClient  *TCPClientSinkOptions  `toml:"tcp_client,omitempty"`
-}
-
-type ConsoleSinkOptions struct {
-	Target     string `toml:"target"` // "stdout", "stderr", "split"
-	Colorize   bool   `toml:"colorize"`
-	BufferSize int64  `toml:"buffer_size"`
-}
-
-type FileSinkOptions struct {
-	Directory      string  `toml:"directory"`
-	Name           string  `toml:"name"`
-	MaxSizeMB      int64   `toml:"max_size_mb"`
-	MaxTotalSizeMB int64   `toml:"max_total_size_mb"`
-	MinDiskFreeMB  int64   `toml:"min_disk_free_mb"`
-	RetentionHours float64 `toml:"retention_hours"`
-	BufferSize     int64   `toml:"buffer_size"`
-	FlushInterval  int64   `toml:"flush_interval_ms"`
-}
-
-type HTTPSinkOptions struct {
-	Host         string            `toml:"host"`
-	Port         int64             `toml:"port"`
-	StreamPath   string            `toml:"stream_path"`
-	StatusPath   string            `toml:"status_path"`
-	BufferSize   int64             `toml:"buffer_size"`
-	WriteTimeout int64             `toml:"write_timeout_ms"`
-	Heartbeat    *HeartbeatConfig  `toml:"heartbeat"`
-	NetLimit     *NetLimitConfig   `toml:"net_limit"`
-	TLS          *TLSConfig        `toml:"tls"`
-	Auth         *ServerAuthConfig `toml:"auth"`
-}
-
-type TCPSinkOptions struct {
-	Host            string            `toml:"host"`
-	Port            int64             `toml:"port"`
-	BufferSize      int64             `toml:"buffer_size"`
-	WriteTimeout    int64             `toml:"write_timeout_ms"`
-	KeepAlive       bool              `toml:"keep_alive"`
-	KeepAlivePeriod int64             `toml:"keep_alive_period_ms"`
-	Heartbeat       *HeartbeatConfig  `toml:"heartbeat"`
-	NetLimit        *NetLimitConfig   `toml:"net_limit"`
-	Auth            *ServerAuthConfig `toml:"auth"`
-}
-
-type HTTPClientSinkOptions struct {
-	URL                string            `toml:"url"`
-	BufferSize         int64             `toml:"buffer_size"`
-	BatchSize          int64             `toml:"batch_size"`
-	BatchDelayMS       int64             `toml:"batch_delay_ms"`
-	Timeout            int64             `toml:"timeout_seconds"`
-	MaxRetries         int64             `toml:"max_retries"`
-	RetryDelayMS       int64             `toml:"retry_delay_ms"`
-	RetryBackoff       float64           `toml:"retry_backoff"`
-	InsecureSkipVerify bool              `toml:"insecure_skip_verify"`
-	TLS                *TLSConfig        `toml:"tls"`
-	Auth               *ClientAuthConfig `toml:"auth"`
-}
-
-type TCPClientSinkOptions struct {
-	Host                string            `toml:"host"`
-	Port                int64             `toml:"port"`
-	BufferSize          int64             `toml:"buffer_size"`
-	DialTimeout         int64             `toml:"dial_timeout_seconds"`
-	WriteTimeout        int64             `toml:"write_timeout_seconds"`
-	ReadTimeout         int64             `toml:"read_timeout_seconds"`
-	KeepAlive           int64             `toml:"keep_alive_seconds"`
-	ReconnectDelayMS    int64             `toml:"reconnect_delay_ms"`
-	MaxReconnectDelayMS int64             `toml:"max_reconnect_delay_ms"`
-	ReconnectBackoff    float64           `toml:"reconnect_backoff"`
-	Auth                *ClientAuthConfig `toml:"auth"`
-}
-
-// --- Rate Limit Options ---
-
-// Defines the action to take when a rate limit is exceeded.
-type RateLimitPolicy int
-
-const (
-	// PolicyPass allows all logs through, effectively disabling the limiter.
-	PolicyPass RateLimitPolicy = iota
-	// PolicyDrop drops logs that exceed the rate limit.
-	PolicyDrop
-)
-
-// Defines the configuration for pipeline-level rate limiting.
-type RateLimitConfig struct {
-	// Rate is the number of log entries allowed per second. Default: 0 (disabled).
-	Rate float64 `toml:"rate"`
-	// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate.
-	Burst float64 `toml:"burst"`
-	// Policy defines the action to take when the limit is exceeded. "pass" or "drop".
-	Policy string `toml:"policy"`
-	// MaxEntrySizeBytes is the maximum allowed size for a single log entry. 0 = no limit.
-	MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes"`
-}
-
-// --- Filter Options ---
-
-// Represents the filter type
-type FilterType string
-
-const (
-	FilterTypeInclude FilterType = "include" // Whitelist - only matching logs pass
-	FilterTypeExclude FilterType = "exclude" // Blacklist - matching logs are dropped
-)
-
-// Represents how multiple patterns are combined
-type FilterLogic string
-
-const (
-	FilterLogicOr  FilterLogic = "or"  // Match any pattern
-	FilterLogicAnd FilterLogic = "and" // Match all patterns
-)
-
-// Represents filter configuration
-type FilterConfig struct {
-	Type     FilterType  `toml:"type"`
-	Logic    FilterLogic `toml:"logic"`
-	Patterns []string    `toml:"patterns"`
-}
-
-// --- Formatter Options ---
-
-type FormatConfig struct {
-	// Format configuration - polymorphic like sources/sinks
-	Type string `toml:"type"` // "json", "txt", "raw"
-
-	// Only one will be populated based on format type
-	JSONFormatOptions *JSONFormatterOptions `toml:"json,omitempty"`
-	TxtFormatOptions  *TxtFormatterOptions  `toml:"txt,omitempty"`
-	RawFormatOptions  *RawFormatterOptions  `toml:"raw,omitempty"`
-}
-
-type JSONFormatterOptions struct {
-	Pretty         bool   `toml:"pretty"`
-	TimestampField string `toml:"timestamp_field"`
-	LevelField     string `toml:"level_field"`
-	MessageField   string `toml:"message_field"`
-	SourceField    string `toml:"source_field"`
-}
-
-type TxtFormatterOptions struct {
-	Template        string `toml:"template"`
-	TimestampFormat string `toml:"timestamp_format"`
-}
-
-type RawFormatterOptions struct {
-	AddNewLine bool `toml:"add_new_line"`
-}
-
-// --- Server-side Auth (for sources) ---
-
-type BasicAuthConfig struct {
-	Users []BasicAuthUser `toml:"users"`
-	Realm string          `toml:"realm"`
-}
-
-type BasicAuthUser struct {
-	Username     string `toml:"username"`
-	PasswordHash string `toml:"password_hash"` // Argon2
-}
-
-type ScramAuthConfig struct {
-	Users []ScramUser `toml:"users"`
-}
-
-type ScramUser struct {
-	Username     string `toml:"username"`
-	StoredKey    string `toml:"stored_key"` // base64
-	ServerKey    string `toml:"server_key"` // base64
-	Salt         string `toml:"salt"`       // base64
-	ArgonTime    uint32 `toml:"argon_time"`
-	ArgonMemory  uint32 `toml:"argon_memory"`
-	ArgonThreads uint8  `toml:"argon_threads"`
-}
-
-type TokenAuthConfig struct {
-	Tokens []string `toml:"tokens"`
-}
-
-// Server auth wrapper (for sources accepting connections)
-type ServerAuthConfig struct {
-	Type  string           `toml:"type"` // "none", "basic", "token", "scram"
-	Basic *BasicAuthConfig `toml:"basic,omitempty"`
-	Token *TokenAuthConfig `toml:"token,omitempty"`
-	Scram *ScramAuthConfig `toml:"scram,omitempty"`
-}

src/internal/config/validation.go

@@ -1,945 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"net/url"
-	"path/filepath"
-	"regexp"
-	"strings"
-	"time"
-
-	lconfig "github.com/lixenwraith/config"
-)
-
-// validateConfig is the centralized validator for the entire configuration
-// This replaces the old (c *Config) validate() method
-func ValidateConfig(cfg *Config) error {
-	if cfg == nil {
-		return fmt.Errorf("config is nil")
-	}
-
-	if len(cfg.Pipelines) == 0 {
-		return fmt.Errorf("no pipelines configured")
-	}
-
-	if err := validateLogConfig(cfg.Logging); err != nil {
-		return fmt.Errorf("logging config: %w", err)
-	}
-
-	// Track used ports across all pipelines
-	allPorts := make(map[int64]string)
-	pipelineNames := make(map[string]bool)
-
-	for i, pipeline := range cfg.Pipelines {
-		if err := validatePipeline(i, &pipeline, pipelineNames, allPorts); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func validateLogConfig(cfg *LogConfig) error {
-	validOutputs := map[string]bool{
-		"file": true, "stdout": true, "stderr": true,
-		"split": true, "all": true, "none": true,
-	}
-	if !validOutputs[cfg.Output] {
-		return fmt.Errorf("invalid log output mode: %s", cfg.Output)
-	}
-
-	validLevels := map[string]bool{
-		"debug": true, "info": true, "warn": true, "error": true,
-	}
-	if !validLevels[cfg.Level] {
-		return fmt.Errorf("invalid log level: %s", cfg.Level)
-	}
-
-	if cfg.Console != nil {
-		validTargets := map[string]bool{
-			"stdout": true, "stderr": true, "split": true,
-		}
-		if !validTargets[cfg.Console.Target] {
-			return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
-		}
-
-		validFormats := map[string]bool{
-			"txt": true, "json": true, "": true,
-		}
-		if !validFormats[cfg.Console.Format] {
-			return fmt.Errorf("invalid console format: %s", cfg.Console.Format)
-		}
-	}
-
-	return nil
-}
-
-func validatePipeline(index int, p *PipelineConfig, pipelineNames map[string]bool, allPorts map[int64]string) error {
-	// Validate pipeline name
-	if err := lconfig.NonEmpty(p.Name); err != nil {
-		return fmt.Errorf("pipeline %d: missing name", index)
-	}
-
-	if pipelineNames[p.Name] {
-		return fmt.Errorf("pipeline %d: duplicate name '%s'", index, p.Name)
-	}
-	pipelineNames[p.Name] = true
-
-	// Must have at least one source
-	if len(p.Sources) == 0 {
-		return fmt.Errorf("pipeline '%s': no sources specified", p.Name)
-	}
-
-	// Validate each source
-	for j, source := range p.Sources {
-		if err := validateSourceConfig(p.Name, j, &source); err != nil {
-			return err
-		}
-	}
-
-	// Validate rate limit if present
-	if p.RateLimit != nil {
-		if err := validateRateLimit(p.Name, p.RateLimit); err != nil {
-			return err
-		}
-	}
-
-	// Validate filters
-	for j, filter := range p.Filters {
-		if err := validateFilter(p.Name, j, &filter); err != nil {
-			return err
-		}
-	}
-
-	// Validate formatter configuration
-	if err := validateFormatterConfig(p); err != nil {
-		return fmt.Errorf("pipeline '%s': %w", p.Name, err)
-	}
-
-	// Must have at least one sink
-	if len(p.Sinks) == 0 {
-		return fmt.Errorf("pipeline '%s': no sinks specified", p.Name)
-	}
-
-	// Validate each sink
-	for j, sink := range p.Sinks {
-		if err := validateSinkConfig(p.Name, j, &sink, allPorts); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// validateSourceConfig validates typed source configuration
-func validateSourceConfig(pipelineName string, index int, s *SourceConfig) error {
-	if err := lconfig.NonEmpty(s.Type); err != nil {
-		return fmt.Errorf("pipeline '%s' source[%d]: missing type", pipelineName, index)
-	}
-
-	// Count how many source configs are populated
-	populated := 0
-	var populatedType string
-
-	if s.Directory != nil {
-		populated++
-		populatedType = "directory"
-	}
-	if s.Stdin != nil {
-		populated++
-		populatedType = "stdin"
-	}
-	if s.HTTP != nil {
-		populated++
-		populatedType = "http"
-	}
-	if s.TCP != nil {
-		populated++
-		populatedType = "tcp"
-	}
-
-	if populated == 0 {
-		return fmt.Errorf("pipeline '%s' source[%d]: no configuration provided for type '%s'",
-			pipelineName, index, s.Type)
-	}
-	if populated > 1 {
-		return fmt.Errorf("pipeline '%s' source[%d]: multiple configurations provided, only one allowed",
-			pipelineName, index)
-	}
-	if populatedType != s.Type {
-		return fmt.Errorf("pipeline '%s' source[%d]: type mismatch - type is '%s' but config is for '%s'",
-			pipelineName, index, s.Type, populatedType)
-	}
-
-	// Validate specific source type
-	switch s.Type {
-	case "directory":
-		return validateDirectorySource(pipelineName, index, s.Directory)
-	case "stdin":
-		return validateStdinSource(pipelineName, index, s.Stdin)
-	case "http":
-		return validateHTTPSource(pipelineName, index, s.HTTP)
-	case "tcp":
-		return validateTCPSource(pipelineName, index, s.TCP)
-	default:
-		return fmt.Errorf("pipeline '%s' source[%d]: unknown type '%s'", pipelineName, index, s.Type)
-	}
-}
-
-func validateDirectorySource(pipelineName string, index int, opts *DirectorySourceOptions) error {
-	if err := lconfig.NonEmpty(opts.Path); err != nil {
-		return fmt.Errorf("pipeline '%s' source[%d]: directory requires 'path'", pipelineName, index)
-	} else {
-		absPath, err := filepath.Abs(opts.Path)
-		if err != nil {
-			return fmt.Errorf("invalid path %s: %w", opts.Path, err)
-		}
-		opts.Path = absPath
-	}
-
-	// Check for directory traversal
-	// TODO: traversal check only if optional security settings from cli/env set
-	if strings.Contains(opts.Path, "..") {
-		return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal", pipelineName, index)
-	}
-
-	// Validate pattern if provided
-	if opts.Pattern != "" {
-		if strings.Count(opts.Pattern, "*") == 0 && strings.Count(opts.Pattern, "?") == 0 {
-			// If no wildcards, ensure valid filename
-			if filepath.Base(opts.Pattern) != opts.Pattern {
-				return fmt.Errorf("pipeline '%s' source[%d]: pattern contains path separators", pipelineName, index)
-			}
-		}
-	} else {
-		opts.Pattern = "*"
-	}
-
-	// Validate check interval
-	if opts.CheckIntervalMS < 10 {
-		return fmt.Errorf("pipeline '%s' source[%d]: check_interval_ms must be at least 10ms", pipelineName, index)
-	}
-
-	return nil
-}
-
-func validateStdinSource(pipelineName string, index int, opts *StdinSourceOptions) error {
-	if opts.BufferSize < 0 {
-		return fmt.Errorf("pipeline '%s' source[%d]: buffer_size must be positive", pipelineName, index)
-	} else if opts.BufferSize == 0 {
-		opts.BufferSize = 1000
-	}
-	return nil
-}
-
-func validateHTTPSource(pipelineName string, index int, opts *HTTPSourceOptions) error {
-	// Validate port
-	if err := lconfig.Port(opts.Port); err != nil {
-		return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
-	}
-
-	// Set defaults
-	if opts.Host == "" {
-		opts.Host = "0.0.0.0"
-	}
-	if opts.IngestPath == "" {
-		opts.IngestPath = "/ingest"
-	}
-	if opts.MaxRequestBodySize <= 0 {
-		opts.MaxRequestBodySize = 10 * 1024 * 1024 // 10MB default
-	}
-	if opts.ReadTimeout <= 0 {
-		opts.ReadTimeout = 5000 // 5 seconds
-	}
-	if opts.WriteTimeout <= 0 {
-		opts.WriteTimeout = 5000 // 5 seconds
-	}
-
-	// Validate host if specified
-	if opts.Host != "" && opts.Host != "0.0.0.0" {
-		if err := lconfig.IPAddress(opts.Host); err != nil {
-			return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
-		}
-	}
-
-	// Validate paths
-	if !strings.HasPrefix(opts.IngestPath, "/") {
-		return fmt.Errorf("pipeline '%s' source[%d]: ingest_path must start with /", pipelineName, index)
-	}
-
-	// Validate auth configuration
-	validHTTPSourceAuthTypes := map[string]bool{"basic": true, "token": true, "mtls": true}
-	if opts.Auth != nil && opts.Auth.Type != "none" && opts.Auth.Type != "" {
-		if !validHTTPSourceAuthTypes[opts.Auth.Type] {
-			return fmt.Errorf("pipeline '%s' source[%d]: %s is not a valid auth type",
-				pipelineName, index, opts.Auth.Type)
-		}
-		// All non-none auth types require TLS for HTTP
-		if opts.TLS == nil || !opts.TLS.Enabled {
-			return fmt.Errorf("pipeline '%s' source[%d]: %s auth requires TLS to be enabled",
-				pipelineName, index, opts.Auth.Type)
-		}
-
-		// Validate specific auth types
-		if err := validateServerAuth(pipelineName, opts.Auth); err != nil {
-			return fmt.Errorf("source[%d]: %w", index, err)
-		}
-	}
-
-	// Validate nested configs
-	if opts.NetLimit != nil {
-		if err := validateNetLimit(pipelineName, fmt.Sprintf("source[%d]", index), opts.NetLimit); err != nil {
-			return err
-		}
-	}
-
-	if opts.TLS != nil {
-		if err := validateTLS(pipelineName, fmt.Sprintf("source[%d]", index), opts.TLS); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func validateTCPSource(pipelineName string, index int, opts *TCPSourceOptions) error {
-	// Validate port
-	if err := lconfig.Port(opts.Port); err != nil {
-		return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
-	}
-
-	// Set defaults
-	if opts.Host == "" {
-		opts.Host = "0.0.0.0"
-	}
-	if opts.ReadTimeout <= 0 {
-		opts.ReadTimeout = 5000 // 5 seconds
-	}
-	if !opts.KeepAlive {
-		opts.KeepAlive = true // Default enabled
-	}
-	if opts.KeepAlivePeriod <= 0 {
-		opts.KeepAlivePeriod = 30000 // 30 seconds
-	}
-
-	// Validate host if specified
-	if opts.Host != "" && opts.Host != "0.0.0.0" {
-		if err := lconfig.IPAddress(opts.Host); err != nil {
-			return fmt.Errorf("pipeline '%s' source[%d]: %w", pipelineName, index, err)
-		}
-	}
-
-	// TCP source does NOT support TLS
-	// Validate auth configuration - only none and scram are allowed
-	if opts.Auth != nil {
-		switch opts.Auth.Type {
-		case "", "none":
-			// OK
-		case "scram":
-			// SCRAM doesn't require TLS
-			if err := validateServerAuth(pipelineName, opts.Auth); err != nil {
-				return fmt.Errorf("source[%d]: %w", index, err)
-			}
-		default:
-			return fmt.Errorf("pipeline '%s' source[%d]: TCP source only supports 'none' or 'scram' auth (got '%s')",
-				pipelineName, index, opts.Auth.Type)
-		}
-	}
-
-	// Validate NetLimit if present
-	if opts.NetLimit != nil {
-		if err := validateNetLimit(pipelineName, fmt.Sprintf("source[%d]", index), opts.NetLimit); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// validateSinkConfig validates typed sink configuration
-func validateSinkConfig(pipelineName string, index int, s *SinkConfig, allPorts map[int64]string) error {
-	if err := lconfig.NonEmpty(s.Type); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: missing type", pipelineName, index)
-	}
-
-	// Count populated sink configs
-	populated := 0
-	var populatedType string
-
-	if s.Console != nil {
-		populated++
-		populatedType = "console"
-	}
-	if s.File != nil {
-		populated++
-		populatedType = "file"
-	}
-	if s.HTTP != nil {
-		populated++
-		populatedType = "http"
-	}
-	if s.TCP != nil {
-		populated++
-		populatedType = "tcp"
-	}
-	if s.HTTPClient != nil {
-		populated++
-		populatedType = "http_client"
-	}
-	if s.TCPClient != nil {
-		populated++
-		populatedType = "tcp_client"
-	}
-
-	if populated == 0 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: no configuration provided for type '%s'",
-			pipelineName, index, s.Type)
-	}
-	if populated > 1 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: multiple configurations provided, only one allowed",
-			pipelineName, index)
-	}
-	if populatedType != s.Type {
-		return fmt.Errorf("pipeline '%s' sink[%d]: type mismatch - type is '%s' but config is for '%s'",
-			pipelineName, index, s.Type, populatedType)
-	}
-
-	// Validate specific sink type
-	switch s.Type {
-	case "console":
-		return validateConsoleSink(pipelineName, index, s.Console)
-	case "file":
-		return validateFileSink(pipelineName, index, s.File)
-	case "http":
-		return validateHTTPSink(pipelineName, index, s.HTTP, allPorts)
-	case "tcp":
-		return validateTCPSink(pipelineName, index, s.TCP, allPorts)
-	case "http_client":
-		return validateHTTPClientSink(pipelineName, index, s.HTTPClient)
-	case "tcp_client":
-		return validateTCPClientSink(pipelineName, index, s.TCPClient)
-	default:
-		return fmt.Errorf("pipeline '%s' sink[%d]: unknown type '%s'", pipelineName, index, s.Type)
-	}
-}
-
-func validateConsoleSink(pipelineName string, index int, opts *ConsoleSinkOptions) error {
-	if opts.BufferSize < 1 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: buffer_size must be positive", pipelineName, index)
-	}
-	return nil
-}
-
-func validateFileSink(pipelineName string, index int, opts *FileSinkOptions) error {
-	if err := lconfig.NonEmpty(opts.Directory); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: file requires 'directory'", pipelineName, index)
-	}
-
-	if err := lconfig.NonEmpty(opts.Name); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: file requires 'name'", pipelineName, index)
-	}
-
-	if opts.BufferSize <= 0 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: max_size_mb must be positive", pipelineName, index)
-	}
-
-	// Validate sizes
-	if opts.MaxSizeMB < 0 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: max_size_mb must be positive", pipelineName, index)
-	}
-
-	if opts.MaxTotalSizeMB <= 0 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: max_total_size_mb cannot be negative", pipelineName, index)
-	}
-
-	if opts.MinDiskFreeMB < 0 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: min_disk_free_mb must be positive", pipelineName, index)
-	}
-
-	if opts.RetentionHours <= 0 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: retention_hours cannot be negative", pipelineName, index)
-	}
-
-	return nil
-}
-
-func validateHTTPSink(pipelineName string, index int, opts *HTTPSinkOptions, allPorts map[int64]string) error {
-	// Validate port
-	if err := lconfig.Port(opts.Port); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
-	}
-
-	// Check port conflicts
-	if existing, exists := allPorts[opts.Port]; exists {
-		return fmt.Errorf("pipeline '%s' sink[%d]: port %d already used by %s",
-			pipelineName, index, opts.Port, existing)
-	}
-	allPorts[opts.Port] = fmt.Sprintf("%s-http[%d]", pipelineName, index)
-
-	// Validate host if specified
-	if opts.Host != "" {
-		if err := lconfig.IPAddress(opts.Host); err != nil {
-			return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
-		}
-	}
-
-	// Validate paths
-	if !strings.HasPrefix(opts.StreamPath, "/") {
-		return fmt.Errorf("pipeline '%s' sink[%d]: stream_path must start with /", pipelineName, index)
-	}
-
-	if !strings.HasPrefix(opts.StatusPath, "/") {
-		return fmt.Errorf("pipeline '%s' sink[%d]: status_path must start with /", pipelineName, index)
-	}
-
-	// Validate buffer
-	if opts.BufferSize < 1 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: buffer_size must be positive", pipelineName, index)
-	}
-
-	// Validate nested configs
-	if opts.Heartbeat != nil {
-		if err := validateHeartbeat(pipelineName, fmt.Sprintf("sink[%d]", index), opts.Heartbeat); err != nil {
-			return err
-		}
-	}
-
-	if opts.NetLimit != nil {
-		if err := validateNetLimit(pipelineName, fmt.Sprintf("sink[%d]", index), opts.NetLimit); err != nil {
-			return err
-		}
-	}
-
-	if opts.TLS != nil {
-		if err := validateTLS(pipelineName, fmt.Sprintf("sink[%d]", index), opts.TLS); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func validateTCPSink(pipelineName string, index int, opts *TCPSinkOptions, allPorts map[int64]string) error {
-	// Validate port
-	if err := lconfig.Port(opts.Port); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
-	}
-
-	// Check port conflicts
-	if existing, exists := allPorts[opts.Port]; exists {
-		return fmt.Errorf("pipeline '%s' sink[%d]: port %d already used by %s",
-			pipelineName, index, opts.Port, existing)
-	}
-	allPorts[opts.Port] = fmt.Sprintf("%s-tcp[%d]", pipelineName, index)
-
-	// Validate host if specified
-	if opts.Host != "" {
-		if err := lconfig.IPAddress(opts.Host); err != nil {
-			return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
-		}
-	}
-
-	// Validate buffer
-	if opts.BufferSize < 1 {
-		return fmt.Errorf("pipeline '%s' sink[%d]: buffer_size must be positive", pipelineName, index)
-	}
-
-	// Validate nested configs
-	if opts.Heartbeat != nil {
-		if err := validateHeartbeat(pipelineName, fmt.Sprintf("sink[%d]", index), opts.Heartbeat); err != nil {
-			return err
-		}
-	}
-
-	if opts.NetLimit != nil {
-		if err := validateNetLimit(pipelineName, fmt.Sprintf("sink[%d]", index), opts.NetLimit); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func validateHTTPClientSink(pipelineName string, index int, opts *HTTPClientSinkOptions) error {
-	// Validate URL
-	if err := lconfig.NonEmpty(opts.URL); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: http_client requires 'url'", pipelineName, index)
-	}
-
-	parsedURL, err := url.Parse(opts.URL)
-	if err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: invalid URL: %w", pipelineName, index, err)
-	}
-
-	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
-		return fmt.Errorf("pipeline '%s' sink[%d]: URL must use http or https scheme", pipelineName, index)
-	}
-
-	isHTTPS := parsedURL.Scheme == "https"
-
-	// Set defaults for unspecified fields
-	if opts.BufferSize <= 0 {
-		opts.BufferSize = 1000
-	}
-	if opts.BatchSize <= 0 {
-		opts.BatchSize = 100
-	}
-	if opts.BatchDelayMS <= 0 {
-		opts.BatchDelayMS = 1000 // 1 second in ms
-	}
-	if opts.Timeout <= 0 {
-		opts.Timeout = 30 // 30 seconds
-	}
-	if opts.MaxRetries < 0 {
-		opts.MaxRetries = 3
-	}
-	if opts.RetryDelayMS <= 0 {
-		opts.RetryDelayMS = 1000 // 1 second in ms
-	}
-	if opts.RetryBackoff < 1.0 {
-		opts.RetryBackoff = 2.0
-	}
-
-	// Validate auth configuration
-	if opts.Auth != nil {
-		switch opts.Auth.Type {
-		case "basic":
-			if opts.Auth.Username == "" || opts.Auth.Password == "" {
-				return fmt.Errorf("pipeline '%s' sink[%d]: username and password required for basic auth",
-					pipelineName, index)
-			}
-			if !isHTTPS && !opts.InsecureSkipVerify {
-				return fmt.Errorf("pipeline '%s' sink[%d]: basic auth requires HTTPS (security: credentials would be sent in plaintext)",
-					pipelineName, index)
-			}
-
-		case "token":
-			if opts.Auth.Token == "" {
-				return fmt.Errorf("pipeline '%s' sink[%d]: token required for %s auth",
-					pipelineName, index, opts.Auth.Type)
-			}
-			if !isHTTPS && !opts.InsecureSkipVerify {
-				return fmt.Errorf("pipeline '%s' sink[%d]: %s auth requires HTTPS (security: token would be sent in plaintext)",
-					pipelineName, index, opts.Auth.Type)
-			}
-
-		case "mtls":
-			if !isHTTPS {
-				return fmt.Errorf("pipeline '%s' sink[%d]: mTLS requires HTTPS",
-					pipelineName, index)
-			}
-			// mTLS certs should be in TLS config, not auth config
-			if opts.TLS == nil || opts.TLS.CertFile == "" || opts.TLS.KeyFile == "" {
-				return fmt.Errorf("pipeline '%s' sink[%d]: cert_file and key_file required in TLS config for mTLS auth",
-					pipelineName, index)
-			}
-
-		case "none", "":
-			// Clear any credentials if auth is "none" or empty
-			if opts.Auth != nil {
-				opts.Auth.Username = ""
-				opts.Auth.Password = ""
-				opts.Auth.Token = ""
-			}
-
-		default:
-			return fmt.Errorf("pipeline '%s' sink[%d]: invalid auth type '%s' (valid: none, basic, token, mtls)",
-				pipelineName, index, opts.Auth.Type)
-		}
-	}
-
-	// Validate TLS config if present
-	if opts.TLS != nil {
-		if err := validateTLS(pipelineName, fmt.Sprintf("sink[%d]", index), opts.TLS); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func validateTCPClientSink(pipelineName string, index int, opts *TCPClientSinkOptions) error {
-	// Validate host and port
-	if err := lconfig.NonEmpty(opts.Host); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: tcp_client requires 'host'", pipelineName, index)
-	}
-
-	if err := lconfig.Port(opts.Port); err != nil {
-		return fmt.Errorf("pipeline '%s' sink[%d]: %w", pipelineName, index, err)
-	}
-
-	// Set defaults
-	if opts.BufferSize <= 0 {
-		opts.BufferSize = 1000
-	}
-	if opts.DialTimeout <= 0 {
-		opts.DialTimeout = 10 // 10 seconds
-	}
-	if opts.WriteTimeout <= 0 {
-		opts.WriteTimeout = 30 // 30 seconds
-	}
-	if opts.ReadTimeout <= 0 {
-		opts.ReadTimeout = 10 // 10 seconds
-	}
-	if opts.KeepAlive <= 0 {
-		opts.KeepAlive = 30 // 30 seconds
-	}
-	if opts.ReconnectDelayMS <= 0 {
-		opts.ReconnectDelayMS = 1000 // 1 second in ms
-	}
-	if opts.MaxReconnectDelayMS <= 0 {
-		opts.MaxReconnectDelayMS = 30000 // 30 seconds in ms
-	}
-	if opts.ReconnectBackoff < 1.0 {
-		opts.ReconnectBackoff = 1.5
-	}
-
-	// Validate auth configuration
-	if opts.Auth != nil {
-		switch opts.Auth.Type {
-
-		case "scram":
-			if opts.Auth.Username == "" || opts.Auth.Password == "" {
-				return fmt.Errorf("pipeline '%s' sink[%d]: username and password required for SCRAM auth",
-					pipelineName, index)
-			}
-			// SCRAM doesn't require TLS as it uses challenge-response
-
-		case "none", "":
-			// Clear credentials
-			if opts.Auth != nil {
-				opts.Auth.Username = ""
-				opts.Auth.Password = ""
-				opts.Auth.Token = ""
-			}
-
-		default:
-			return fmt.Errorf("pipeline '%s' sink[%d]: invalid auth type '%s' (valid: none, basic, token, scram, mtls)",
-				pipelineName, index, opts.Auth.Type)
-		}
-	}
-
-	return nil
-}
-
-// validateFormatterConfig validates formatter configuration
-func validateFormatterConfig(p *PipelineConfig) error {
-	if p.Format == nil {
-		p.Format = &FormatConfig{
-			Type: "raw",
-		}
-	} else if p.Format.Type == "" {
-		p.Format.Type = "raw" // Default
-	}
-
-	switch p.Format.Type {
-
-	case "raw":
-		if p.Format.RawFormatOptions == nil {
-			p.Format.RawFormatOptions = &RawFormatterOptions{}
-		}
-
-	case "txt":
-		if p.Format.TxtFormatOptions == nil {
-			p.Format.TxtFormatOptions = &TxtFormatterOptions{}
-		}
-
-		// Default template format
-		templateStr := "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}{{ if .Fields }} {{.Fields}}{{ end }}"
-		if p.Format.TxtFormatOptions.Template != "" {
-			p.Format.TxtFormatOptions.Template = templateStr
-		}
-
-		// Default timestamp format
-		timestampFormat := time.RFC3339
-		if p.Format.TxtFormatOptions.TimestampFormat != "" {
-			p.Format.TxtFormatOptions.TimestampFormat = timestampFormat
-		}
-
-	case "json":
-		if p.Format.JSONFormatOptions == nil {
-			p.Format.JSONFormatOptions = &JSONFormatterOptions{}
-		}
-	}
-
-	return nil
-}
-
-// Helper validation functions for nested configs
-func validateNetLimit(pipelineName, location string, nl *NetLimitConfig) error {
-	if !nl.Enabled {
-		return nil // Skip validation if disabled
-	}
-
-	if nl.MaxConnections < 0 {
-		return fmt.Errorf("pipeline '%s' %s: max_connections cannot be negative", pipelineName, location)
-	}
-
-	if nl.BurstSize < 0 {
-		return fmt.Errorf("pipeline '%s' %s: burst_size cannot be negative", pipelineName, location)
-	}
-
-	return nil
-}
-
-func validateTLS(pipelineName, location string, tls *TLSConfig) error {
-	if !tls.Enabled {
-		return nil // Skip validation if disabled
-	}
-
-	// If TLS enabled, cert and key files required (unless skip verify)
-	if !tls.SkipVerify {
-		if tls.CertFile == "" || tls.KeyFile == "" {
-			return fmt.Errorf("pipeline '%s' %s: TLS enabled requires cert_file and key_file", pipelineName, location)
-		}
-	}
-
-	return nil
-}
-
-func validateHeartbeat(pipelineName, location string, hb *HeartbeatConfig) error {
-	if !hb.Enabled {
-		return nil // Skip validation if disabled
-	}
-
-	if hb.IntervalMS < 1000 { // At least 1 second
-		return fmt.Errorf("pipeline '%s' %s: heartbeat interval must be at least 1000ms", pipelineName, location)
-	}
-
-	return nil
-}
-
-func validateServerAuth(pipelineName string, auth *ServerAuthConfig) error {
-	if auth.Type == "" || auth.Type == "none" {
-		return nil
-	}
-
-	// Count populated auth configs
-	populated := 0
-	var populatedType string
-
-	if auth.Basic != nil {
-		populated++
-		populatedType = "basic"
-	}
-	if auth.Token != nil {
-		populated++
-		populatedType = "token"
-	}
-	if auth.Scram != nil {
-		populated++
-		populatedType = "scram"
-	}
-
-	if populated == 0 {
-		return fmt.Errorf("pipeline '%s': auth type '%s' specified but config missing", pipelineName, auth.Type)
-	}
-	if populated > 1 {
-		return fmt.Errorf("pipeline '%s': multiple auth configurations provided", pipelineName)
-	}
-	if populatedType != auth.Type {
-		return fmt.Errorf("pipeline '%s': auth type mismatch - type is '%s' but config is for '%s'",
-			pipelineName, auth.Type, populatedType)
-	}
-
-	// Validate specific auth type
-	switch auth.Type {
-	case "basic":
-		if len(auth.Basic.Users) == 0 {
-			return fmt.Errorf("pipeline '%s': basic auth requires at least one user", pipelineName)
-		}
-		for i, user := range auth.Basic.Users {
-			if err := lconfig.NonEmpty(user.Username); err != nil {
-				return fmt.Errorf("pipeline '%s': basic auth user[%d] missing username", pipelineName, i)
-			}
-			if err := lconfig.NonEmpty(user.PasswordHash); err != nil {
-				return fmt.Errorf("pipeline '%s': basic auth user[%d] missing password_hash", pipelineName, i)
-			}
-		}
-	case "token":
-		if len(auth.Token.Tokens) == 0 {
-			return fmt.Errorf("pipeline '%s': token auth requires at least one token", pipelineName)
-		}
-	case "scram":
-		if len(auth.Scram.Users) == 0 {
-			return fmt.Errorf("pipeline '%s': scram auth requires at least one user", pipelineName)
-		}
-		for i, user := range auth.Scram.Users {
-			if err := lconfig.NonEmpty(user.Username); err != nil {
-				return fmt.Errorf("pipeline '%s': scram auth user[%d] missing username", pipelineName, i)
-			}
-			// Validate required SCRAM fields
-			if user.StoredKey == "" || user.ServerKey == "" || user.Salt == "" {
-				return fmt.Errorf("pipeline '%s': scram auth user[%d] missing required fields", pipelineName, i)
-			}
-		}
-	default:
-		return fmt.Errorf("pipeline '%s': unknown auth type '%s'", pipelineName, auth.Type)
-	}
-
-	return nil
-}
-
-func validateRateLimit(pipelineName string, cfg *RateLimitConfig) error {
-	if cfg == nil {
-		return nil
-	}
-
-	if cfg.Rate < 0 {
-		return fmt.Errorf("pipeline '%s': rate limit rate cannot be negative", pipelineName)
-	}
-
-	if cfg.Burst < 0 {
-		return fmt.Errorf("pipeline '%s': rate limit burst cannot be negative", pipelineName)
-	}
-
-	if cfg.MaxEntrySizeBytes < 0 {
-		return fmt.Errorf("pipeline '%s': max entry size bytes cannot be negative", pipelineName)
-	}
-
-	// Validate policy
-	switch strings.ToLower(cfg.Policy) {
-	case "", "pass", "drop":
-		// Valid policies
-	default:
-		return fmt.Errorf("pipeline '%s': invalid rate limit policy '%s' (must be 'pass' or 'drop')",
-			pipelineName, cfg.Policy)
-	}
-
-	return nil
-}
-
-func validateFilter(pipelineName string, filterIndex int, cfg *FilterConfig) error {
-	// Validate filter type
-	switch cfg.Type {
-	case FilterTypeInclude, FilterTypeExclude, "":
-		// Valid types
-	default:
-		return fmt.Errorf("pipeline '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
-			pipelineName, filterIndex, cfg.Type)
-	}
-
-	// Validate filter logic
-	switch cfg.Logic {
-	case FilterLogicOr, FilterLogicAnd, "":
-		// Valid logic
-	default:
-		return fmt.Errorf("pipeline '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
-			pipelineName, filterIndex, cfg.Logic)
-	}
-
-	// Empty patterns is valid - passes everything
-	if len(cfg.Patterns) == 0 {
-		return nil
-	}
-
-	// Validate regex patterns
-	for i, pattern := range cfg.Patterns {
-		if _, err := regexp.Compile(pattern); err != nil {
-			return fmt.Errorf("pipeline '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
-				pipelineName, filterIndex, i, pattern, err)
-		}
-	}
-
-	return nil
-}

src/internal/core/const.go

@@ -1,13 +0,0 @@
-// FILE: logwisp/src/internal/core/const.go
-package core
-
-// Argon2id parameters
-const (
-	Argon2Time    = 3
-	Argon2Memory  = 64 * 1024 // 64 MB
-	Argon2Threads = 4
-	Argon2SaltLen = 16
-	Argon2KeyLen  = 32
-)
-
-const DefaultTokenLength = 32

src/internal/core/types.go

@@ -1,17 +0,0 @@
-// FILE: logwisp/src/internal/core/types.go
-package core
-
-import (
-	"encoding/json"
-	"time"
-)
-
-// Represents a single log record flowing through the pipeline
-type LogEntry struct {
-	Time    time.Time       `json:"time"`
-	Source  string          `json:"source"`
-	Level   string          `json:"level,omitempty"`
-	Message string          `json:"message"`
-	Fields  json.RawMessage `json:"fields,omitempty"`
-	RawSize int64           `json:"-"`
-}

src/internal/format/format.go

@@ -1,34 +0,0 @@
-// FILE: logwisp/src/internal/format/format.go
-package format
-
-import (
-	"fmt"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-
-	"github.com/lixenwraith/log"
-)
-
-// Defines the interface for transforming a LogEntry into a byte slice.
-type Formatter interface {
-	// Format takes a LogEntry and returns the formatted log as a byte slice.
-	Format(entry core.LogEntry) ([]byte, error)
-
-	// Name returns the formatter type name
-	Name() string
-}
-
-// Creates a new Formatter based on the provided configuration.
-func NewFormatter(cfg *config.FormatConfig, logger *log.Logger) (Formatter, error) {
-	switch cfg.Type {
-	case "json":
-		return NewJSONFormatter(cfg.JSONFormatOptions, logger)
-	case "txt":
-		return NewTxtFormatter(cfg.TxtFormatOptions, logger)
-	case "raw", "":
-		return NewRawFormatter(cfg.RawFormatOptions, logger)
-	default:
-		return nil, fmt.Errorf("unknown formatter type: %s", cfg.Type)
-	}
-}

src/internal/format/json.go

@@ -1,134 +0,0 @@
-// FILE: logwisp/src/internal/format/json.go
-package format
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-
-	"github.com/lixenwraith/log"
-)
-
-// Produces structured JSON logs
-type JSONFormatter struct {
-	config *config.JSONFormatterOptions
-	logger *log.Logger
-}
-
-// Creates a new JSON formatter
-func NewJSONFormatter(opts *config.JSONFormatterOptions, logger *log.Logger) (*JSONFormatter, error) {
-	f := &JSONFormatter{
-		config: opts,
-		logger: logger,
-	}
-
-	return f, nil
-}
-
-// Formats the log entry as JSON
-func (f *JSONFormatter) Format(entry core.LogEntry) ([]byte, error) {
-	// Start with a clean map
-	output := make(map[string]any)
-
-	// First, populate with LogWisp metadata
-	output[f.config.TimestampField] = entry.Time.Format(time.RFC3339Nano)
-	output[f.config.LevelField] = entry.Level
-	output[f.config.SourceField] = entry.Source
-
-	// Try to parse the message as JSON
-	var msgData map[string]any
-	if err := json.Unmarshal([]byte(entry.Message), &msgData); err == nil {
-		// Message is valid JSON - merge fields
-		// LogWisp metadata takes precedence
-		for k, v := range msgData {
-			// Don't overwrite our standard fields
-			if k != f.config.TimestampField && k != f.config.LevelField && k != f.config.SourceField {
-				output[k] = v
-			}
-		}
-
-		// If the original JSON had these fields, log that we're overriding
-		if _, hasTime := msgData[f.config.TimestampField]; hasTime {
-			f.logger.Debug("msg", "Overriding timestamp from JSON message",
-				"component", "json_formatter",
-				"original", msgData[f.config.TimestampField],
-				"logwisp", output[f.config.TimestampField])
-		}
-	} else {
-		// Message is not valid JSON - add as message field
-		output[f.config.MessageField] = entry.Message
-	}
-
-	// Add any additional fields from LogEntry.Fields
-	if len(entry.Fields) > 0 {
-		var fields map[string]any
-		if err := json.Unmarshal(entry.Fields, &fields); err == nil {
-			// Merge additional fields, but don't override existing
-			for k, v := range fields {
-				if _, exists := output[k]; !exists {
-					output[k] = v
-				}
-			}
-		}
-	}
-
-	// Marshal to JSON
-	var result []byte
-	var err error
-	if f.config.Pretty {
-		result, err = json.MarshalIndent(output, "", "  ")
-	} else {
-		result, err = json.Marshal(output)
-	}
-
-	if err != nil {
-		return nil, fmt.Errorf("failed to marshal JSON: %w", err)
-	}
-
-	// Add newline
-	return append(result, '\n'), nil
-}
-
-// Returns the formatter name
-func (f *JSONFormatter) Name() string {
-	return "json"
-}
-
-// Formats multiple entries as a JSON array
-// This is a special method for sinks that need to batch entries
-func (f *JSONFormatter) FormatBatch(entries []core.LogEntry) ([]byte, error) {
-	// For batching, we need to create an array of formatted objects
-	batch := make([]json.RawMessage, 0, len(entries))
-
-	for _, entry := range entries {
-		// Format each entry without the trailing newline
-		formatted, err := f.Format(entry)
-		if err != nil {
-			f.logger.Warn("msg", "Failed to format entry in batch",
-				"component", "json_formatter",
-				"error", err)
-			continue
-		}
-
-		// Remove the trailing newline for array elements
-		if len(formatted) > 0 && formatted[len(formatted)-1] == '\n' {
-			formatted = formatted[:len(formatted)-1]
-		}
-
-		batch = append(batch, formatted)
-	}
-
-	// Marshal the entire batch as an array
-	var result []byte
-	var err error
-	if f.config.Pretty {
-		result, err = json.MarshalIndent(batch, "", "  ")
-	} else {
-		result, err = json.Marshal(batch)
-	}
-
-	return result, err
-}

src/internal/format/raw.go

@@ -1,38 +0,0 @@
-// FILE: logwisp/src/internal/format/raw.go
-package format
-
-import (
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-
-	"github.com/lixenwraith/log"
-)
-
-// Outputs the log message as-is with a newline
-type RawFormatter struct {
-	config *config.RawFormatterOptions
-	logger *log.Logger
-}
-
-// Creates a new raw formatter
-func NewRawFormatter(cfg *config.RawFormatterOptions, logger *log.Logger) (*RawFormatter, error) {
-	return &RawFormatter{
-		config: cfg,
-		logger: logger,
-	}, nil
-}
-
-// Returns the message with a newline appended
-func (f *RawFormatter) Format(entry core.LogEntry) ([]byte, error) {
-	// TODO: Standardize not to add "\n" when processing raw, check lixenwraith/log for consistency
-	if f.config.AddNewLine {
-		return append([]byte(entry.Message), '\n'), nil
-	} else {
-		return []byte(entry.Message), nil
-	}
-}
-
-// Returns the formatter name
-func (f *RawFormatter) Name() string {
-	return "raw"
-}

src/internal/format/txt.go

@@ -1,97 +0,0 @@
-// FILE: logwisp/src/internal/format/txt.go
-package format
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-	"text/template"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-
-	"github.com/lixenwraith/log"
-)
-
-// Produces human-readable text logs using templates
-type TxtFormatter struct {
-	config   *config.TxtFormatterOptions
-	template *template.Template
-	logger   *log.Logger
-}
-
-// Creates a new text formatter
-func NewTxtFormatter(opts *config.TxtFormatterOptions, logger *log.Logger) (*TxtFormatter, error) {
-	f := &TxtFormatter{
-		config: opts,
-		logger: logger,
-	}
-
-	// Create template with helper functions
-	funcMap := template.FuncMap{
-		"FmtTime": func(t time.Time) string {
-			return t.Format(f.config.TimestampFormat)
-		},
-		"ToUpper":   strings.ToUpper,
-		"ToLower":   strings.ToLower,
-		"TrimSpace": strings.TrimSpace,
-	}
-
-	tmpl, err := template.New("log").Funcs(funcMap).Parse(f.config.Template)
-	if err != nil {
-		return nil, fmt.Errorf("invalid template: %w", err)
-	}
-
-	f.template = tmpl
-	return f, nil
-}
-
-// Formats the log entry using the template
-func (f *TxtFormatter) Format(entry core.LogEntry) ([]byte, error) {
-	// Prepare data for template
-	data := map[string]any{
-		"Timestamp": entry.Time,
-		"Level":     entry.Level,
-		"Source":    entry.Source,
-		"Message":   entry.Message,
-	}
-
-	// Set default level if empty
-	if data["Level"] == "" {
-		data["Level"] = "INFO"
-	}
-
-	// Add fields if present
-	if len(entry.Fields) > 0 {
-		data["Fields"] = string(entry.Fields)
-	}
-
-	var buf bytes.Buffer
-	if err := f.template.Execute(&buf, data); err != nil {
-		// Fallback: return a basic formatted message
-		f.logger.Debug("msg", "Template execution failed, using fallback",
-			"component", "txt_formatter",
-			"error", err)
-
-		fallback := fmt.Sprintf("[%s] [%s] %s - %s\n",
-			entry.Time.Format(f.config.TimestampFormat),
-			strings.ToUpper(entry.Level),
-			entry.Source,
-			entry.Message)
-		return []byte(fallback), nil
-	}
-
-	// Ensure newline at end
-	result := buf.Bytes()
-	if len(result) == 0 || result[len(result)-1] != '\n' {
-		result = append(result, '\n')
-	}
-
-	return result, nil
-}
-
-// Returns the formatter name
-func (f *TxtFormatter) Name() string {
-	return "txt"
-}

src/internal/limit/net.go

@@ -1,893 +0,0 @@
-// FILE: logwisp/src/internal/limit/net.go
-package limit
-
-import (
-	"context"
-	"net"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"logwisp/src/internal/config"
-
-	"github.com/lixenwraith/log"
-)
-
-// DenialReason indicates why a request was denied
-type DenialReason string
-
-// ** THIS PROGRAM IS IPV4 ONLY !!**
-const (
-	// IPv4Only is the enforcement message for IPv6 rejection
-	IPv4Only = "IPv4-only (IPv6 not supported)"
-)
-
-const (
-	ReasonAllowed           DenialReason = ""
-	ReasonBlacklisted       DenialReason = "IP denied by blacklist"
-	ReasonNotWhitelisted    DenialReason = "IP not in whitelist"
-	ReasonRateLimited       DenialReason = "Rate limit exceeded"
-	ReasonConnectionLimited DenialReason = "Connection limit exceeded"
-	ReasonInvalidIP         DenialReason = "Invalid IP address"
-)
-
-// NetLimiter manages net limiting for a transport
-type NetLimiter struct {
-	config *config.NetLimitConfig
-	logger *log.Logger
-
-	// IP Access Control Lists
-	ipWhitelist []*net.IPNet
-	ipBlacklist []*net.IPNet
-
-	// Per-IP limiters
-	ipLimiters map[string]*ipLimiter
-	ipMu       sync.RWMutex
-
-	// Global limiter for the transport
-	globalLimiter *TokenBucket
-
-	// Connection tracking
-	ipConnections    map[string]*connTracker
-	userConnections  map[string]*connTracker
-	tokenConnections map[string]*connTracker
-	totalConnections atomic.Int64
-	connMu           sync.RWMutex
-
-	// Statistics
-	totalRequests      atomic.Uint64
-	blockedByBlacklist atomic.Uint64
-	blockedByWhitelist atomic.Uint64
-	blockedByRateLimit atomic.Uint64
-	blockedByConnLimit atomic.Uint64
-	blockedByInvalidIP atomic.Uint64
-	uniqueIPs          atomic.Uint64
-
-	// Cleanup
-	lastCleanup   time.Time
-	cleanupMu     sync.Mutex
-	cleanupActive atomic.Bool
-
-	// Lifecycle management
-	ctx         context.Context
-	cancel      context.CancelFunc
-	cleanupDone chan struct{}
-}
-
-type ipLimiter struct {
-	bucket      *TokenBucket
-	lastSeen    time.Time
-	connections atomic.Int64
-}
-
-// Connection tracking with activity timestamp
-type connTracker struct {
-	connections atomic.Int64
-	lastSeen    time.Time
-	mu          sync.Mutex
-}
-
-// Creates a new net limiter
-func NewNetLimiter(cfg *config.NetLimitConfig, logger *log.Logger) *NetLimiter {
-	if cfg == nil {
-		return nil
-	}
-
-	// Return nil only if nothing is configured
-	hasACL := len(cfg.IPWhitelist) > 0 || len(cfg.IPBlacklist) > 0
-	hasRateLimit := cfg.Enabled
-
-	if !hasACL && !hasRateLimit {
-		return nil
-	}
-
-	if logger == nil {
-		panic("netlimit.New: logger cannot be nil")
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	l := &NetLimiter{
-		config:           cfg,
-		logger:           logger,
-		ipWhitelist:      make([]*net.IPNet, 0),
-		ipBlacklist:      make([]*net.IPNet, 0),
-		ipLimiters:       make(map[string]*ipLimiter),
-		ipConnections:    make(map[string]*connTracker),
-		userConnections:  make(map[string]*connTracker),
-		tokenConnections: make(map[string]*connTracker),
-		lastCleanup:      time.Now(),
-		ctx:              ctx,
-		cancel:           cancel,
-		cleanupDone:      make(chan struct{}),
-	}
-
-	// Parse IP lists
-	l.parseIPLists()
-
-	// Start cleanup goroutine only if rate limiting is enabled
-	if cfg.Enabled {
-		go l.cleanupLoop()
-	}
-
-	logger.Info("msg", "Net limiter initialized",
-		"component", "netlimit",
-		"acl_enabled", hasACL,
-		"rate_limiting", cfg.Enabled,
-		"whitelist_rules", len(l.ipWhitelist),
-		"blacklist_rules", len(l.ipBlacklist),
-		"requests_per_second", cfg.RequestsPerSecond,
-		"burst_size", cfg.BurstSize,
-		"max_connections_per_ip", cfg.MaxConnectionsPerIP,
-		"max_connections_total", cfg.MaxConnectionsTotal)
-
-	return l
-}
-
-// parseIPLists parses and validates IP whitelist/blacklist
-func (l *NetLimiter) parseIPLists() {
-	// Parse whitelist
-	for _, entry := range l.config.IPWhitelist {
-		if ipNet := l.parseIPEntry(entry, "whitelist"); ipNet != nil {
-			l.ipWhitelist = append(l.ipWhitelist, ipNet)
-		}
-	}
-
-	// Parse blacklist
-	for _, entry := range l.config.IPBlacklist {
-		if ipNet := l.parseIPEntry(entry, "blacklist"); ipNet != nil {
-			l.ipBlacklist = append(l.ipBlacklist, ipNet)
-		}
-	}
-}
-
-// parseIPEntry parses a single IP or CIDR entry
-func (l *NetLimiter) parseIPEntry(entry, listType string) *net.IPNet {
-	// Handle single IP
-	if !strings.Contains(entry, "/") {
-		ip := net.ParseIP(entry)
-		if ip == nil {
-			l.logger.Warn("msg", "Invalid IP entry",
-				"component", "netlimit",
-				"list", listType,
-				"entry", entry)
-			return nil
-		}
-
-		// Reject IPv6
-		if ip.To4() == nil {
-			l.logger.Warn("msg", "IPv6 address rejected",
-				"component", "netlimit",
-				"list", listType,
-				"entry", entry,
-				"reason", IPv4Only)
-			return nil
-		}
-
-		return &net.IPNet{IP: ip.To4(), Mask: net.CIDRMask(32, 32)}
-	}
-
-	// Parse CIDR
-	ipAddr, ipNet, err := net.ParseCIDR(entry)
-	if err != nil {
-		l.logger.Warn("msg", "Invalid CIDR entry",
-			"component", "netlimit",
-			"list", listType,
-			"entry", entry,
-			"error", err)
-		return nil
-	}
-
-	// Reject IPv6 CIDR
-	if ipAddr.To4() == nil {
-		l.logger.Warn("msg", "IPv6 CIDR rejected",
-			"component", "netlimit",
-			"list", listType,
-			"entry", entry,
-			"reason", IPv4Only)
-		return nil
-	}
-
-	// Ensure mask is IPv4
-	_, bits := ipNet.Mask.Size()
-	if bits != 32 {
-		l.logger.Warn("msg", "Non-IPv4 CIDR mask rejected",
-			"component", "netlimit",
-			"list", listType,
-			"entry", entry,
-			"mask_bits", bits,
-			"reason", IPv4Only)
-		return nil
-	}
-
-	return &net.IPNet{IP: ipAddr.To4(), Mask: ipNet.Mask}
-}
-
-// checkIPAccess checks if an IP is allowed by ACLs
-func (l *NetLimiter) checkIPAccess(ip net.IP) DenialReason {
-	// 1. Check blacklist first (deny takes precedence)
-	for _, ipNet := range l.ipBlacklist {
-		if ipNet.Contains(ip) {
-			l.blockedByBlacklist.Add(1)
-			l.logger.Debug("msg", "IP denied by blacklist",
-				"component", "netlimit",
-				"ip", ip.String(),
-				"rule", ipNet.String())
-			return ReasonBlacklisted
-		}
-	}
-
-	// 2. If whitelist is configured, IP must be in it
-	if len(l.ipWhitelist) > 0 {
-		for _, ipNet := range l.ipWhitelist {
-			if ipNet.Contains(ip) {
-				l.logger.Debug("msg", "IP allowed by whitelist",
-					"component", "netlimit",
-					"ip", ip.String(),
-					"rule", ipNet.String())
-				return ReasonAllowed
-			}
-		}
-		l.blockedByWhitelist.Add(1)
-		l.logger.Debug("msg", "IP not in whitelist",
-			"component", "netlimit",
-			"ip", ip.String())
-		return ReasonNotWhitelisted
-	}
-
-	return ReasonAllowed
-}
-
-func (l *NetLimiter) Shutdown() {
-	if l == nil {
-		return
-	}
-
-	l.logger.Info("msg", "Shutting down net limiter", "component", "netlimit")
-
-	// Cancel context to stop cleanup goroutine
-	l.cancel()
-
-	// Wait for cleanup goroutine to finish
-	select {
-	case <-l.cleanupDone:
-		l.logger.Debug("msg", "Cleanup goroutine stopped", "component", "netlimit")
-	case <-time.After(2 * time.Second):
-		l.logger.Warn("msg", "Cleanup goroutine shutdown timeout", "component", "netlimit")
-	}
-}
-
-// Checks if an HTTP request should be allowed: IP access control + connection limits (IP only) + calls
-func (l *NetLimiter) CheckHTTP(remoteAddr string) (allowed bool, statusCode int64, message string) {
-	if l == nil {
-		return true, 0, ""
-	}
-
-	l.totalRequests.Add(1)
-
-	// Parse IP address
-	ipStr, _, err := net.SplitHostPort(remoteAddr)
-	if err != nil {
-		l.logger.Warn("msg", "Failed to parse remote addr",
-			"component", "netlimit",
-			"remote_addr", remoteAddr,
-			"error", err)
-		return true, 0, ""
-	}
-
-	ip := net.ParseIP(ipStr)
-	if ip == nil {
-		l.blockedByInvalidIP.Add(1)
-		l.logger.Warn("msg", "Failed to parse IP",
-			"component", "netlimit",
-			"ip", ipStr)
-		return false, 403, string(ReasonInvalidIP)
-	}
-
-	// Reject IPv6 connections
-	if !isIPv4(ip) {
-		l.blockedByInvalidIP.Add(1)
-		l.logger.Warn("msg", "IPv6 connection rejected",
-			"component", "netlimit",
-			"ip", ipStr,
-			"reason", IPv4Only)
-		return false, 403, IPv4Only
-	}
-
-	// Normalize to IPv4 representation
-	ip = ip.To4()
-
-	// Check IP access control
-	if reason := l.checkIPAccess(ip); reason != ReasonAllowed {
-		return false, 403, string(reason)
-	}
-
-	// If rate limiting is not enabled, allow
-	if !l.config.Enabled {
-		return true, 0, ""
-	}
-
-	// Check connection limits
-	if l.config.MaxConnectionsPerIP > 0 {
-		l.connMu.RLock()
-		tracker, exists := l.ipConnections[ipStr]
-		l.connMu.RUnlock()
-
-		if exists && tracker.connections.Load() >= l.config.MaxConnectionsPerIP {
-			l.blockedByConnLimit.Add(1)
-			statusCode = l.config.ResponseCode
-			if statusCode == 0 {
-				statusCode = 429
-			}
-			return false, statusCode, string(ReasonConnectionLimited)
-		}
-	}
-
-	// Check rate limit
-	if !l.checkIPLimit(ipStr) {
-		l.blockedByRateLimit.Add(1)
-		statusCode = l.config.ResponseCode
-		if statusCode == 0 {
-			statusCode = 429
-		}
-		message = l.config.ResponseMessage
-		if message == "" {
-			message = string(ReasonRateLimited)
-		}
-		return false, statusCode, message
-	}
-
-	return true, 0, ""
-}
-
-// Update connection activity
-func (l *NetLimiter) updateConnectionActivity(ip string) {
-	l.connMu.RLock()
-	tracker, exists := l.ipConnections[ip]
-	l.connMu.RUnlock()
-
-	if exists {
-		tracker.mu.Lock()
-		tracker.lastSeen = time.Now()
-		tracker.mu.Unlock()
-	}
-}
-
-// Checks if a TCP connection should be allowed: IP access control + calls checkIPLimit()
-func (l *NetLimiter) CheckTCP(remoteAddr net.Addr) bool {
-	if l == nil {
-		return true
-	}
-
-	l.totalRequests.Add(1)
-
-	// Extract IP from TCP addr
-	tcpAddr, ok := remoteAddr.(*net.TCPAddr)
-	if !ok {
-		l.blockedByInvalidIP.Add(1)
-		return false
-	}
-
-	// Reject IPv6 connections
-	if !isIPv4(tcpAddr.IP) {
-		l.blockedByInvalidIP.Add(1)
-		l.logger.Warn("msg", "IPv6 TCP connection rejected",
-			"component", "netlimit",
-			"ip", tcpAddr.IP.String(),
-			"reason", IPv4Only)
-		return false
-	}
-
-	// Normalize to IPv4 representation
-	ip := tcpAddr.IP.To4()
-
-	// Check IP access control
-	if reason := l.checkIPAccess(ip); reason != ReasonAllowed {
-		return false
-	}
-
-	// If rate limiting is not enabled, allow
-	if !l.config.Enabled {
-		return true
-	}
-
-	// Check rate limit
-	ipStr := tcpAddr.IP.String()
-	if !l.checkIPLimit(ipStr) {
-		l.blockedByRateLimit.Add(1)
-		return false
-	}
-
-	return true
-}
-
-func isIPv4(ip net.IP) bool {
-	return ip.To4() != nil
-}
-
-// Tracks a new connection for an IP
-func (l *NetLimiter) AddConnection(remoteAddr string) {
-	if l == nil {
-		return
-	}
-
-	ip, _, err := net.SplitHostPort(remoteAddr)
-	if err != nil {
-		l.logger.Warn("msg", "Failed to parse remote address in AddConnection",
-			"component", "netlimit",
-			"remote_addr", remoteAddr,
-			"error", err)
-		return
-	}
-
-	// IP validation
-	parsedIP := net.ParseIP(ip)
-	if parsedIP == nil {
-		l.logger.Warn("msg", "Failed to parse IP in AddConnection",
-			"component", "netlimit",
-			"ip", ip)
-		return
-	}
-
-	// Only supporting ipv4
-	if !isIPv4(parsedIP) {
-		return
-	}
-
-	l.connMu.Lock()
-	tracker, exists := l.ipConnections[ip]
-	if !exists {
-		// Create new tracker with timestamp
-		tracker = &connTracker{
-			lastSeen: time.Now(),
-		}
-		l.ipConnections[ip] = tracker
-	}
-	l.connMu.Unlock()
-
-	newCount := tracker.connections.Add(1)
-	// Update activity timestamp
-	tracker.mu.Lock()
-	tracker.lastSeen = time.Now()
-	tracker.mu.Unlock()
-
-	l.logger.Debug("msg", "Connection added",
-		"ip", ip,
-		"connections", newCount)
-}
-
-// Removes a connection for an IP
-func (l *NetLimiter) RemoveConnection(remoteAddr string) {
-	if l == nil {
-		return
-	}
-
-	ip, _, err := net.SplitHostPort(remoteAddr)
-	if err != nil {
-		l.logger.Warn("msg", "Failed to parse remote address in RemoveConnection",
-			"component", "netlimit",
-			"remote_addr", remoteAddr,
-			"error", err)
-		return
-	}
-
-	// IP validation
-	parsedIP := net.ParseIP(ip)
-	if parsedIP == nil {
-		l.logger.Warn("msg", "Failed to parse IP in RemoveConnection",
-			"component", "netlimit",
-			"ip", ip)
-		return
-	}
-
-	// Only supporting ipv4
-	if !isIPv4(parsedIP) {
-		return
-	}
-
-	l.connMu.RLock()
-	tracker, exists := l.ipConnections[ip]
-	l.connMu.RUnlock()
-
-	if exists {
-		newCount := tracker.connections.Add(-1)
-		l.logger.Debug("msg", "Connection removed",
-			"ip", ip,
-			"connections", newCount)
-
-		if newCount <= 0 {
-			// Clean up if no more connections
-			l.connMu.Lock()
-			if tracker.connections.Load() <= 0 {
-				delete(l.ipConnections, ip)
-			}
-			l.connMu.Unlock()
-		}
-	}
-}
-
-// Returns net limiter statistics
-func (l *NetLimiter) GetStats() map[string]any {
-	if l == nil {
-		return map[string]any{"enabled": false}
-	}
-
-	// Get active rate limiters count
-	l.ipMu.RLock()
-	activeIPs := len(l.ipLimiters)
-	l.ipMu.RUnlock()
-
-	// Get connection tracker counts and calculate total active connections
-	l.connMu.RLock()
-	ipConnTrackers := len(l.ipConnections)
-	userConnTrackers := len(l.userConnections)
-	tokenConnTrackers := len(l.tokenConnections)
-
-	// Calculate actual connection count by summing all IP connections
-	// Potentially more accurate than totalConnections counter which might drift
-	// TODO: test and refactor if they match
-	actualIPConnections := 0
-	for _, tracker := range l.ipConnections {
-		actualIPConnections += int(tracker.connections.Load())
-	}
-
-	actualUserConnections := 0
-	for _, tracker := range l.userConnections {
-		actualUserConnections += int(tracker.connections.Load())
-	}
-
-	actualTokenConnections := 0
-	for _, tracker := range l.tokenConnections {
-		actualTokenConnections += int(tracker.connections.Load())
-	}
-
-	// Use the counter for total (should match actualIPConnections in most cases)
-	totalConns := l.totalConnections.Load()
-	l.connMu.RUnlock()
-
-	// Calculate total blocked
-	totalBlocked := l.blockedByBlacklist.Load() +
-		l.blockedByWhitelist.Load() +
-		l.blockedByRateLimit.Load() +
-		l.blockedByConnLimit.Load() +
-		l.blockedByInvalidIP.Load()
-
-	return map[string]any{
-		"enabled":        true,
-		"total_requests": l.totalRequests.Load(),
-		"total_blocked":  totalBlocked,
-		"blocked_breakdown": map[string]uint64{
-			"blacklist":  l.blockedByBlacklist.Load(),
-			"whitelist":  l.blockedByWhitelist.Load(),
-			"rate_limit": l.blockedByRateLimit.Load(),
-			"conn_limit": l.blockedByConnLimit.Load(),
-			"invalid_ip": l.blockedByInvalidIP.Load(),
-		},
-		"rate_limiting": map[string]any{
-			"enabled":             l.config.Enabled,
-			"requests_per_second": l.config.RequestsPerSecond,
-			"burst_size":          l.config.BurstSize,
-			"active_ip_limiters":  activeIPs, // IPs being rate-limited
-		},
-		"access_control": map[string]any{
-			"whitelist_rules": len(l.ipWhitelist),
-			"blacklist_rules": len(l.ipBlacklist),
-		},
-		"connections": map[string]any{
-			// Actual counts
-			"total_active":             totalConns,             // Counter-based total
-			"active_ip_connections":    actualIPConnections,    // Sum of all IP connections
-			"active_user_connections":  actualUserConnections,  // Sum of all user connections
-			"active_token_connections": actualTokenConnections, // Sum of all token connections
-
-			// Tracker counts (number of unique IPs/users/tokens being tracked)
-			"tracked_ips":    ipConnTrackers,
-			"tracked_users":  userConnTrackers,
-			"tracked_tokens": tokenConnTrackers,
-
-			// Configuration limits (0 = disabled)
-			"limit_per_ip": l.config.MaxConnectionsPerIP,
-			"limit_total":  l.config.MaxConnectionsTotal,
-		},
-	}
-}
-
-// Performs IP net limit check (req/sec)
-func (l *NetLimiter) checkIPLimit(ip string) bool {
-	// Validate IP format
-	parsedIP := net.ParseIP(ip)
-	if parsedIP == nil || !isIPv4(parsedIP) {
-		l.logger.Warn("msg", "Invalid or non-IPv4 address in rate limiter",
-			"component", "netlimit",
-			"ip", ip)
-		return false
-	}
-
-	// Maybe run cleanup
-	l.maybeCleanup()
-
-	// IP limit
-	l.ipMu.Lock()
-	lim, exists := l.ipLimiters[ip]
-	if !exists {
-		// Create new limiter for this IP
-		lim = &ipLimiter{
-			bucket: NewTokenBucket(
-				float64(l.config.BurstSize),
-				l.config.RequestsPerSecond,
-			),
-			lastSeen: time.Now(),
-		}
-		l.ipLimiters[ip] = lim
-		l.uniqueIPs.Add(1)
-
-		l.logger.Debug("msg", "Created new IP limiter",
-			"ip", ip,
-			"total_ips", l.uniqueIPs.Load())
-	} else {
-		lim.lastSeen = time.Now()
-	}
-	l.ipMu.Unlock()
-
-	// Rate limit check
-	allowed := lim.bucket.Allow()
-	if !allowed {
-		l.blockedByRateLimit.Add(1)
-	}
-
-	return allowed
-}
-
-// Runs cleanup if enough time has passed
-func (l *NetLimiter) maybeCleanup() {
-	l.cleanupMu.Lock()
-
-	// Check if enough time has passed
-	if time.Since(l.lastCleanup) < 30*time.Second {
-		l.cleanupMu.Unlock()
-		return
-	}
-
-	// Check if cleanup already running
-	if !l.cleanupActive.CompareAndSwap(false, true) {
-		l.cleanupMu.Unlock()
-		return
-	}
-
-	l.lastCleanup = time.Now()
-	l.cleanupMu.Unlock()
-
-	// Run cleanup async
-	go func() {
-		defer l.cleanupActive.Store(false)
-		l.cleanup()
-	}()
-}
-
-// Removes stale IP limiters
-func (l *NetLimiter) cleanup() {
-	staleTimeout := 5 * time.Minute
-	now := time.Now()
-
-	l.ipMu.Lock()
-	defer l.ipMu.Unlock()
-
-	// Clean up rate limiters
-	l.ipMu.Lock()
-	cleaned := 0
-	for ip, lim := range l.ipLimiters {
-		if now.Sub(lim.lastSeen) > staleTimeout {
-			delete(l.ipLimiters, ip)
-			cleaned++
-		}
-	}
-	l.ipMu.Unlock()
-
-	if cleaned > 0 {
-		l.logger.Debug("msg", "Cleaned up stale IP limiters",
-			"component", "netlimit",
-			"cleaned", cleaned,
-			"remaining", len(l.ipLimiters))
-	}
-
-	// Clean up stale connection trackers
-	l.connMu.Lock()
-
-	// Clean IP connections
-	ipCleaned := 0
-	for ip, tracker := range l.ipConnections {
-		tracker.mu.Lock()
-		lastSeen := tracker.lastSeen
-		tracker.mu.Unlock()
-
-		if now.Sub(lastSeen) > staleTimeout && tracker.connections.Load() <= 0 {
-			delete(l.ipConnections, ip)
-			ipCleaned++
-		}
-	}
-
-	// Clean user connections
-	userCleaned := 0
-	for user, tracker := range l.userConnections {
-		tracker.mu.Lock()
-		lastSeen := tracker.lastSeen
-		tracker.mu.Unlock()
-
-		if now.Sub(lastSeen) > staleTimeout && tracker.connections.Load() <= 0 {
-			delete(l.userConnections, user)
-			userCleaned++
-		}
-	}
-
-	// Clean token connections
-	tokenCleaned := 0
-	for token, tracker := range l.tokenConnections {
-		tracker.mu.Lock()
-		lastSeen := tracker.lastSeen
-		tracker.mu.Unlock()
-
-		if now.Sub(lastSeen) > staleTimeout && tracker.connections.Load() <= 0 {
-			delete(l.tokenConnections, token)
-			tokenCleaned++
-		}
-	}
-
-	l.connMu.Unlock()
-
-	if ipCleaned > 0 || userCleaned > 0 || tokenCleaned > 0 {
-		l.logger.Debug("msg", "Cleaned up stale connection trackers",
-			"component", "netlimit",
-			"ip_cleaned", ipCleaned,
-			"user_cleaned", userCleaned,
-			"token_cleaned", tokenCleaned,
-			"ip_remaining", len(l.ipConnections),
-			"user_remaining", len(l.userConnections),
-			"token_remaining", len(l.tokenConnections))
-	}
-}
-
-// Runs periodic cleanup
-func (l *NetLimiter) cleanupLoop() {
-	defer close(l.cleanupDone)
-
-	ticker := time.NewTicker(1 * time.Minute)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-l.ctx.Done():
-			// Exit when context is cancelled
-			l.logger.Debug("msg", "Cleanup loop stopping", "component", "netlimit")
-			return
-		case <-ticker.C:
-			l.cleanup()
-		}
-	}
-}
-
-// Tracks a new connection with optional user/token info: Connection limits (IP/user/token/total) for TCP only
-func (l *NetLimiter) TrackConnection(ip string, user string, token string) bool {
-	if l == nil {
-		return true
-	}
-
-	l.connMu.Lock()
-	defer l.connMu.Unlock()
-
-	// Check total connections limit (0 = disabled)
-	if l.config.MaxConnectionsTotal > 0 {
-		currentTotal := l.totalConnections.Load()
-		if currentTotal >= l.config.MaxConnectionsTotal {
-			l.blockedByConnLimit.Add(1)
-			l.logger.Debug("msg", "TCP connection blocked by total limit",
-				"component", "netlimit",
-				"current_total", currentTotal,
-				"max_connections_total", l.config.MaxConnectionsTotal)
-			return false
-		}
-	}
-
-	// Check per-IP connection limit (0 = disabled)
-	if l.config.MaxConnectionsPerIP > 0 && ip != "" {
-		tracker, exists := l.ipConnections[ip]
-		if !exists {
-			tracker = &connTracker{lastSeen: time.Now()}
-			l.ipConnections[ip] = tracker
-		}
-		if tracker.connections.Load() >= l.config.MaxConnectionsPerIP {
-			l.blockedByConnLimit.Add(1)
-			l.logger.Debug("msg", "TCP connection blocked by IP limit",
-				"component", "netlimit",
-				"ip", ip,
-				"current", tracker.connections.Load(),
-				"max", l.config.MaxConnectionsPerIP)
-			return false
-		}
-	}
-
-	// All checks passed, increment counters
-	l.totalConnections.Add(1)
-
-	if ip != "" && l.config.MaxConnectionsPerIP > 0 {
-		if tracker, exists := l.ipConnections[ip]; exists {
-			tracker.connections.Add(1)
-			tracker.mu.Lock()
-			tracker.lastSeen = time.Now()
-			tracker.mu.Unlock()
-		}
-	}
-
-	return true
-}
-
-// Releases a tracked connection
-func (l *NetLimiter) ReleaseConnection(ip string, user string, token string) {
-	if l == nil {
-		return
-	}
-
-	l.connMu.Lock()
-	defer l.connMu.Unlock()
-
-	// Decrement total
-	if l.totalConnections.Load() > 0 {
-		l.totalConnections.Add(-1)
-	}
-
-	// Decrement IP counter
-	if ip != "" {
-		if tracker, exists := l.ipConnections[ip]; exists {
-			if tracker.connections.Load() > 0 {
-				tracker.connections.Add(-1)
-			}
-			tracker.mu.Lock()
-			tracker.lastSeen = time.Now()
-			tracker.mu.Unlock()
-		}
-	}
-
-	// Decrement user counter
-	if user != "" {
-		if tracker, exists := l.userConnections[user]; exists {
-			if tracker.connections.Load() > 0 {
-				tracker.connections.Add(-1)
-			}
-			tracker.mu.Lock()
-			tracker.lastSeen = time.Now()
-			tracker.mu.Unlock()
-		}
-	}
-
-	// Decrement token counter
-	if token != "" {
-		if tracker, exists := l.tokenConnections[token]; exists {
-			if tracker.connections.Load() > 0 {
-				tracker.connections.Add(-1)
-			}
-			tracker.mu.Lock()
-			tracker.lastSeen = time.Now()
-			tracker.mu.Unlock()
-		}
-	}
-}

src/internal/service/pipeline.go

@@ -1,281 +0,0 @@
-// FILE: logwisp/src/internal/service/pipeline.go
-package service
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/filter"
-	"logwisp/src/internal/format"
-	"logwisp/src/internal/limit"
-	"logwisp/src/internal/sink"
-	"logwisp/src/internal/source"
-
-	"github.com/lixenwraith/log"
-)
-
-// Manages the flow of data from sources through filters to sinks
-type Pipeline struct {
-	Config      *config.PipelineConfig
-	Sources     []source.Source
-	RateLimiter *limit.RateLimiter
-	FilterChain *filter.Chain
-	Sinks       []sink.Sink
-	Stats       *PipelineStats
-	logger      *log.Logger
-
-	ctx    context.Context
-	cancel context.CancelFunc
-	wg     sync.WaitGroup
-}
-
-// Contains statistics for a pipeline
-type PipelineStats struct {
-	StartTime                      time.Time
-	TotalEntriesProcessed          atomic.Uint64
-	TotalEntriesDroppedByRateLimit atomic.Uint64
-	TotalEntriesFiltered           atomic.Uint64
-	SourceStats                    []source.SourceStats
-	SinkStats                      []sink.SinkStats
-	FilterStats                    map[string]any
-}
-
-// Creates and starts a new pipeline
-func (s *Service) NewPipeline(cfg *config.PipelineConfig) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	if _, exists := s.pipelines[cfg.Name]; exists {
-		err := fmt.Errorf("pipeline '%s' already exists", cfg.Name)
-		s.logger.Error("msg", "Failed to create pipeline - duplicate name",
-			"component", "service",
-			"pipeline", cfg.Name,
-			"error", err)
-		return err
-	}
-
-	s.logger.Debug("msg", "Creating pipeline", "pipeline", cfg.Name)
-
-	// Create pipeline context
-	pipelineCtx, pipelineCancel := context.WithCancel(s.ctx)
-
-	// Create pipeline instance
-	pipeline := &Pipeline{
-		Config: cfg,
-		Stats: &PipelineStats{
-			StartTime: time.Now(),
-		},
-		ctx:    pipelineCtx,
-		cancel: pipelineCancel,
-		logger: s.logger,
-	}
-
-	// Create sources
-	for i, srcCfg := range cfg.Sources {
-		src, err := s.createSource(&srcCfg)
-		if err != nil {
-			pipelineCancel()
-			return fmt.Errorf("failed to create source[%d]: %w", i, err)
-		}
-		pipeline.Sources = append(pipeline.Sources, src)
-	}
-
-	// Create pipeline rate limiter
-	if cfg.RateLimit != nil {
-		limiter, err := limit.NewRateLimiter(*cfg.RateLimit, s.logger)
-		if err != nil {
-			pipelineCancel()
-			return fmt.Errorf("failed to create pipeline rate limiter: %w", err)
-		}
-		pipeline.RateLimiter = limiter
-	}
-
-	// Create filter chain
-	if len(cfg.Filters) > 0 {
-		chain, err := filter.NewChain(cfg.Filters, s.logger)
-		if err != nil {
-			pipelineCancel()
-			return fmt.Errorf("failed to create filter chain: %w", err)
-		}
-		pipeline.FilterChain = chain
-	}
-
-	// Create formatter for the pipeline
-	formatter, err := format.NewFormatter(cfg.Format, s.logger)
-	if err != nil {
-		pipelineCancel()
-		return fmt.Errorf("failed to create formatter: %w", err)
-	}
-
-	// Create sinks
-	for i, sinkCfg := range cfg.Sinks {
-		sinkInst, err := s.createSink(sinkCfg, formatter)
-		if err != nil {
-			pipelineCancel()
-			return fmt.Errorf("failed to create sink[%d]: %w", i, err)
-		}
-		pipeline.Sinks = append(pipeline.Sinks, sinkInst)
-	}
-
-	// Start all sources
-	for i, src := range pipeline.Sources {
-		if err := src.Start(); err != nil {
-			pipeline.Shutdown()
-			return fmt.Errorf("failed to start source[%d]: %w", i, err)
-		}
-	}
-
-	// Start all sinks
-	for i, sinkInst := range pipeline.Sinks {
-		if err := sinkInst.Start(pipelineCtx); err != nil {
-			pipeline.Shutdown()
-			return fmt.Errorf("failed to start sink[%d]: %w", i, err)
-		}
-	}
-
-	// Wire sources to sinks through filters
-	s.wirePipeline(pipeline)
-
-	// Start stats updater
-	pipeline.startStatsUpdater(pipelineCtx)
-
-	s.pipelines[cfg.Name] = pipeline
-	s.logger.Info("msg", "Pipeline created successfully",
-		"pipeline", cfg.Name)
-	return nil
-}
-
-// Gracefully stops the pipeline
-func (p *Pipeline) Shutdown() {
-	p.logger.Info("msg", "Shutting down pipeline",
-		"component", "pipeline",
-		"pipeline", p.Config.Name)
-
-	// Cancel context to stop processing
-	p.cancel()
-
-	// Stop all sinks first
-	var wg sync.WaitGroup
-	for _, s := range p.Sinks {
-		wg.Add(1)
-		go func(sink sink.Sink) {
-			defer wg.Done()
-			sink.Stop()
-		}(s)
-	}
-	wg.Wait()
-
-	// Stop all sources
-	for _, src := range p.Sources {
-		wg.Add(1)
-		go func(source source.Source) {
-			defer wg.Done()
-			source.Stop()
-		}(src)
-	}
-	wg.Wait()
-
-	// Wait for processing goroutines
-	p.wg.Wait()
-
-	p.logger.Info("msg", "Pipeline shutdown complete",
-		"component", "pipeline",
-		"pipeline", p.Config.Name)
-}
-
-// Returns pipeline statistics
-func (p *Pipeline) GetStats() map[string]any {
-	// Recovery to handle concurrent access during shutdown
-	// When service is shutting down, sources/sinks might be nil or partially stopped
-	defer func() {
-		if r := recover(); r != nil {
-			p.logger.Error("msg", "Panic getting pipeline stats",
-				"pipeline", p.Config.Name,
-				"panic", r)
-		}
-	}()
-
-	// Collect source stats
-	sourceStats := make([]map[string]any, 0, len(p.Sources))
-	for _, src := range p.Sources {
-		if src == nil {
-			continue // Skip nil sources
-		}
-
-		stats := src.GetStats()
-		sourceStats = append(sourceStats, map[string]any{
-			"type":            stats.Type,
-			"total_entries":   stats.TotalEntries,
-			"dropped_entries": stats.DroppedEntries,
-			"start_time":      stats.StartTime,
-			"last_entry_time": stats.LastEntryTime,
-			"details":         stats.Details,
-		})
-	}
-
-	// Collect rate limit stats
-	var rateLimitStats map[string]any
-	if p.RateLimiter != nil {
-		rateLimitStats = p.RateLimiter.GetStats()
-	}
-
-	// Collect filter stats
-	var filterStats map[string]any
-	if p.FilterChain != nil {
-		filterStats = p.FilterChain.GetStats()
-	}
-
-	// Collect sink stats
-	sinkStats := make([]map[string]any, 0, len(p.Sinks))
-	for _, s := range p.Sinks {
-		if s == nil {
-			continue // Skip nil sinks
-		}
-
-		stats := s.GetStats()
-		sinkStats = append(sinkStats, map[string]any{
-			"type":               stats.Type,
-			"total_processed":    stats.TotalProcessed,
-			"active_connections": stats.ActiveConnections,
-			"start_time":         stats.StartTime,
-			"last_processed":     stats.LastProcessed,
-			"details":            stats.Details,
-		})
-	}
-
-	return map[string]any{
-		"name":                     p.Config.Name,
-		"uptime_seconds":           int(time.Since(p.Stats.StartTime).Seconds()),
-		"total_processed":          p.Stats.TotalEntriesProcessed.Load(),
-		"total_dropped_rate_limit": p.Stats.TotalEntriesDroppedByRateLimit.Load(),
-		"total_filtered":           p.Stats.TotalEntriesFiltered.Load(),
-		"sources":                  sourceStats,
-		"rate_limiter":             rateLimitStats,
-		"sinks":                    sinkStats,
-		"filters":                  filterStats,
-		"source_count":             len(p.Sources),
-		"sink_count":               len(p.Sinks),
-		"filter_count":             len(p.Config.Filters),
-	}
-}
-
-// Runs periodic stats updates
-func (p *Pipeline) startStatsUpdater(ctx context.Context) {
-	go func() {
-		ticker := time.NewTicker(1 * time.Second)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticker.C:
-				// Periodic stats updates if needed
-			}
-		}
-	}()
-}

src/internal/service/service.go

@@ -1,249 +0,0 @@
-// FILE: logwisp/src/internal/service/service.go
-package service
-
-import (
-	"context"
-	"fmt"
-	"sync"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-	"logwisp/src/internal/format"
-	"logwisp/src/internal/sink"
-	"logwisp/src/internal/source"
-
-	"github.com/lixenwraith/log"
-)
-
-// Service manages multiple pipelines
-type Service struct {
-	pipelines map[string]*Pipeline
-	mu        sync.RWMutex
-	ctx       context.Context
-	cancel    context.CancelFunc
-	wg        sync.WaitGroup
-	logger    *log.Logger
-}
-
-// Creates a new service
-func NewService(ctx context.Context, logger *log.Logger) *Service {
-	serviceCtx, cancel := context.WithCancel(ctx)
-	return &Service{
-		pipelines: make(map[string]*Pipeline),
-		ctx:       serviceCtx,
-		cancel:    cancel,
-		logger:    logger,
-	}
-}
-
-// Connects sources to sinks through filters
-func (s *Service) wirePipeline(p *Pipeline) {
-	// For each source, subscribe and process entries
-	for _, src := range p.Sources {
-		srcChan := src.Subscribe()
-
-		// Create a processing goroutine for this source
-		p.wg.Add(1)
-		go func(source source.Source, entries <-chan core.LogEntry) {
-			defer p.wg.Done()
-
-			// Panic recovery to prevent single source from crashing pipeline
-			defer func() {
-				if r := recover(); r != nil {
-					s.logger.Error("msg", "Panic in pipeline processing",
-						"pipeline", p.Config.Name,
-						"source", source.GetStats().Type,
-						"panic", r)
-
-					// Ensure failed pipelines don't leave resources hanging
-					go func() {
-						s.logger.Warn("msg", "Shutting down pipeline due to panic",
-							"pipeline", p.Config.Name)
-						if err := s.RemovePipeline(p.Config.Name); err != nil {
-							s.logger.Error("msg", "Failed to remove panicked pipeline",
-								"pipeline", p.Config.Name,
-								"error", err)
-						}
-					}()
-				}
-			}()
-
-			for {
-				select {
-				case <-p.ctx.Done():
-					return
-				case entry, ok := <-entries:
-					if !ok {
-						return
-					}
-
-					p.Stats.TotalEntriesProcessed.Add(1)
-
-					// Apply pipeline rate limiter
-					if p.RateLimiter != nil {
-						if !p.RateLimiter.Allow(entry) {
-							p.Stats.TotalEntriesDroppedByRateLimit.Add(1)
-							continue // Drop the entry
-						}
-					}
-
-					// Apply filters if configured
-					if p.FilterChain != nil {
-						if !p.FilterChain.Apply(entry) {
-							p.Stats.TotalEntriesFiltered.Add(1)
-							continue
-						}
-					}
-
-					// Send to all sinks
-					for _, sinkInst := range p.Sinks {
-						select {
-						case sinkInst.Input() <- entry:
-						case <-p.ctx.Done():
-							return
-						default:
-							// Drop if sink buffer is full, may flood logging for slow client
-							s.logger.Debug("msg", "Dropped log entry - sink buffer full",
-								"pipeline", p.Config.Name)
-						}
-					}
-				}
-			}
-		}(src, srcChan)
-	}
-}
-
-// Creates a source instance based on configuration
-func (s *Service) createSource(cfg *config.SourceConfig) (source.Source, error) {
-	switch cfg.Type {
-	case "directory":
-		return source.NewDirectorySource(cfg.Directory, s.logger)
-	case "stdin":
-		return source.NewStdinSource(cfg.Stdin, s.logger)
-	case "http":
-		return source.NewHTTPSource(cfg.HTTP, s.logger)
-	case "tcp":
-		return source.NewTCPSource(cfg.TCP, s.logger)
-	default:
-		return nil, fmt.Errorf("unknown source type: %s", cfg.Type)
-	}
-}
-
-// Creates a sink instance based on configuration
-func (s *Service) createSink(cfg config.SinkConfig, formatter format.Formatter) (sink.Sink, error) {
-
-	switch cfg.Type {
-	case "http":
-		if cfg.HTTP == nil {
-			return nil, fmt.Errorf("HTTP sink configuration missing")
-		}
-		return sink.NewHTTPSink(cfg.HTTP, s.logger, formatter)
-
-	case "tcp":
-		if cfg.TCP == nil {
-			return nil, fmt.Errorf("TCP sink configuration missing")
-		}
-		return sink.NewTCPSink(cfg.TCP, s.logger, formatter)
-
-	case "http_client":
-		return sink.NewHTTPClientSink(cfg.HTTPClient, s.logger, formatter)
-	case "tcp_client":
-		return sink.NewTCPClientSink(cfg.TCPClient, s.logger, formatter)
-	case "file":
-		return sink.NewFileSink(cfg.File, s.logger, formatter)
-	case "console":
-		return sink.NewConsoleSink(cfg.Console, s.logger, formatter)
-	default:
-		return nil, fmt.Errorf("unknown sink type: %s", cfg.Type)
-	}
-}
-
-// Returns a pipeline by name
-func (s *Service) GetPipeline(name string) (*Pipeline, error) {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	pipeline, exists := s.pipelines[name]
-	if !exists {
-		return nil, fmt.Errorf("pipeline '%s' not found", name)
-	}
-	return pipeline, nil
-}
-
-// Returns all pipeline names
-func (s *Service) ListPipelines() []string {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	names := make([]string, 0, len(s.pipelines))
-	for name := range s.pipelines {
-		names = append(names, name)
-	}
-	return names
-}
-
-// Stops and removes a pipeline
-func (s *Service) RemovePipeline(name string) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-
-	pipeline, exists := s.pipelines[name]
-	if !exists {
-		err := fmt.Errorf("pipeline '%s' not found", name)
-		s.logger.Warn("msg", "Cannot remove non-existent pipeline",
-			"component", "service",
-			"pipeline", name,
-			"error", err)
-		return err
-	}
-
-	s.logger.Info("msg", "Removing pipeline", "pipeline", name)
-	pipeline.Shutdown()
-	delete(s.pipelines, name)
-	return nil
-}
-
-// Stops all pipelines
-func (s *Service) Shutdown() {
-	s.logger.Info("msg", "Service shutdown initiated")
-
-	s.mu.Lock()
-	pipelines := make([]*Pipeline, 0, len(s.pipelines))
-	for _, pipeline := range s.pipelines {
-		pipelines = append(pipelines, pipeline)
-	}
-	s.mu.Unlock()
-
-	// Stop all pipelines concurrently
-	var wg sync.WaitGroup
-	for _, pipeline := range pipelines {
-		wg.Add(1)
-		go func(p *Pipeline) {
-			defer wg.Done()
-			p.Shutdown()
-		}(pipeline)
-	}
-	wg.Wait()
-
-	s.cancel()
-	s.wg.Wait()
-
-	s.logger.Info("msg", "Service shutdown complete")
-}
-
-// Returns statistics for all pipelines
-func (s *Service) GetGlobalStats() map[string]any {
-	s.mu.RLock()
-	defer s.mu.RUnlock()
-
-	stats := map[string]any{
-		"pipelines":       make(map[string]any),
-		"total_pipelines": len(s.pipelines),
-	}
-
-	for name, pipeline := range s.pipelines {
-		stats["pipelines"].(map[string]any)[name] = pipeline.GetStats()
-	}
-
-	return stats
-}

src/internal/sink/console.go

@@ -1,163 +0,0 @@
-// FILE: logwisp/src/internal/sink/console.go
-package sink
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"strings"
-	"sync/atomic"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-	"logwisp/src/internal/format"
-
-	"github.com/lixenwraith/log"
-)
-
-// ConsoleSink writes log entries to the console (stdout/stderr) using an dedicated logger instance
-type ConsoleSink struct {
-	config    *config.ConsoleSinkOptions
-	input     chan core.LogEntry
-	writer    *log.Logger // Dedicated internal logger instance for console writing
-	done      chan struct{}
-	startTime time.Time
-	logger    *log.Logger // Application logger for app logs
-	formatter format.Formatter
-
-	// Statistics
-	totalProcessed atomic.Uint64
-	lastProcessed  atomic.Value // time.Time
-}
-
-// Creates a new console sink
-func NewConsoleSink(opts *config.ConsoleSinkOptions, appLogger *log.Logger, formatter format.Formatter) (*ConsoleSink, error) {
-	if opts == nil {
-		return nil, fmt.Errorf("console sink options cannot be nil")
-	}
-
-	// Set defaults if not configured
-	if opts.Target == "" {
-		opts.Target = "stdout"
-	}
-	if opts.BufferSize <= 0 {
-		opts.BufferSize = 1000
-	}
-
-	// Dedicated logger instance as console writer
-	writer, err := log.NewBuilder().
-		EnableFile(false).
-		EnableConsole(true).
-		ConsoleTarget(opts.Target).
-		Format("raw").        // Passthrough pre-formatted messages
-		ShowTimestamp(false). // Disable writer's own timestamp
-		ShowLevel(false).     // Disable writer's own level prefix
-		Build()
-
-	if err != nil {
-		return nil, fmt.Errorf("failed to create console writer: %w", err)
-	}
-
-	s := &ConsoleSink{
-		config:    opts,
-		input:     make(chan core.LogEntry, opts.BufferSize),
-		writer:    writer,
-		done:      make(chan struct{}),
-		startTime: time.Now(),
-		logger:    appLogger,
-		formatter: formatter,
-	}
-	s.lastProcessed.Store(time.Time{})
-
-	return s, nil
-}
-
-func (s *ConsoleSink) Input() chan<- core.LogEntry {
-	return s.input
-}
-
-func (s *ConsoleSink) Start(ctx context.Context) error {
-	// Start the internal writer's processing goroutine.
-	if err := s.writer.Start(); err != nil {
-		return fmt.Errorf("failed to start console writer: %w", err)
-	}
-	go s.processLoop(ctx)
-	s.logger.Info("msg", "Console sink started",
-		"component", "console_sink",
-		"target", s.writer.GetConfig().ConsoleTarget)
-	return nil
-}
-
-func (s *ConsoleSink) Stop() {
-	target := s.writer.GetConfig().ConsoleTarget
-	s.logger.Info("msg", "Stopping console sink", "target", target)
-	close(s.done)
-
-	// Shutdown the internal writer with a timeout.
-	if err := s.writer.Shutdown(2 * time.Second); err != nil {
-		s.logger.Error("msg", "Error shutting down console writer",
-			"component", "console_sink",
-			"error", err)
-	}
-	s.logger.Info("msg", "Console sink stopped", "target", target)
-}
-
-func (s *ConsoleSink) GetStats() SinkStats {
-	lastProc, _ := s.lastProcessed.Load().(time.Time)
-
-	return SinkStats{
-		Type:           "console",
-		TotalProcessed: s.totalProcessed.Load(),
-		StartTime:      s.startTime,
-		LastProcessed:  lastProc,
-		Details: map[string]any{
-			"target": s.writer.GetConfig().ConsoleTarget,
-		},
-	}
-}
-
-// processLoop reads entries, formats them, and passes them to the internal writer.
-func (s *ConsoleSink) processLoop(ctx context.Context) {
-	for {
-		select {
-		case entry, ok := <-s.input:
-			if !ok {
-				return
-			}
-
-			s.totalProcessed.Add(1)
-			s.lastProcessed.Store(time.Now())
-
-			// Format the entry using the pipeline's configured formatter.
-			formatted, err := s.formatter.Format(entry)
-			if err != nil {
-				s.logger.Error("msg", "Failed to format log entry for console",
-					"component", "console_sink",
-					"error", err)
-				continue
-			}
-
-			// Convert to string to prevent hex encoding of []byte by log package
-			// Strip new line, writer adds it
-			message := string(bytes.TrimSuffix(formatted, []byte{'\n'}))
-			switch strings.ToUpper(entry.Level) {
-			case "DEBUG":
-				s.writer.Debug(message)
-			case "INFO":
-				s.writer.Info(message)
-			case "WARN", "WARNING":
-				s.writer.Warn(message)
-			case "ERROR", "FATAL":
-				s.writer.Error(message)
-			default:
-				s.writer.Message(message)
-			}
-
-		case <-ctx.Done():
-			return
-		case <-s.done:
-			return
-		}
-	}
-}

src/internal/sink/file.go

@@ -1,138 +0,0 @@
-// FILE: logwisp/src/internal/sink/file.go
-package sink
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"sync/atomic"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/core"
-	"logwisp/src/internal/format"
-
-	"github.com/lixenwraith/log"
-)
-
-// Writes log entries to files with rotation
-type FileSink struct {
-	config    *config.FileSinkOptions
-	input     chan core.LogEntry
-	writer    *log.Logger // Internal logger instance for file writing
-	done      chan struct{}
-	startTime time.Time
-	logger    *log.Logger // Application logger
-	formatter format.Formatter
-
-	// Statistics
-	totalProcessed atomic.Uint64
-	lastProcessed  atomic.Value // time.Time
-}
-
-// Creates a new file sink
-func NewFileSink(opts *config.FileSinkOptions, logger *log.Logger, formatter format.Formatter) (*FileSink, error) {
-	if opts == nil {
-		return nil, fmt.Errorf("file sink options cannot be nil")
-	}
-
-	// Create configuration for the internal log writer
-	writerConfig := log.DefaultConfig()
-	writerConfig.Directory = opts.Directory
-	writerConfig.Name = opts.Name
-	writerConfig.EnableConsole = false // File only
-	writerConfig.ShowTimestamp = false // We already have timestamps in entries
-	writerConfig.ShowLevel = false     // We already have levels in entries
-
-	// Create internal logger for file writing
-	writer := log.NewLogger()
-	if err := writer.ApplyConfig(writerConfig); err != nil {
-		return nil, fmt.Errorf("failed to initialize file writer: %w", err)
-	}
-
-	fs := &FileSink{
-		input:     make(chan core.LogEntry, opts.BufferSize),
-		writer:    writer,
-		done:      make(chan struct{}),
-		startTime: time.Now(),
-		logger:    logger,
-		formatter: formatter,
-	}
-	fs.lastProcessed.Store(time.Time{})
-
-	return fs, nil
-}
-
-func (fs *FileSink) Input() chan<- core.LogEntry {
-	return fs.input
-}
-
-func (fs *FileSink) Start(ctx context.Context) error {
-	// Start the internal file writer
-	if err := fs.writer.Start(); err != nil {
-		return fmt.Errorf("failed to start sink file writer: %w", err)
-	}
-
-	go fs.processLoop(ctx)
-	fs.logger.Info("msg", "File sink started", "component", "file_sink")
-	return nil
-}
-
-func (fs *FileSink) Stop() {
-	fs.logger.Info("msg", "Stopping file sink")
-	close(fs.done)
-
-	// Shutdown the writer with timeout
-	if err := fs.writer.Shutdown(2 * time.Second); err != nil {
-		fs.logger.Error("msg", "Error shutting down file writer",
-			"component", "file_sink",
-			"error", err)
-	}
-
-	fs.logger.Info("msg", "File sink stopped")
-}
-
-func (fs *FileSink) GetStats() SinkStats {
-	lastProc, _ := fs.lastProcessed.Load().(time.Time)
-
-	return SinkStats{
-		Type:           "file",
-		TotalProcessed: fs.totalProcessed.Load(),
-		StartTime:      fs.startTime,
-		LastProcessed:  lastProc,
-		Details:        map[string]any{},
-	}
-}
-
-func (fs *FileSink) processLoop(ctx context.Context) {
-	for {
-		select {
-		case entry, ok := <-fs.input:
-			if !ok {
-				return
-			}
-
-			fs.totalProcessed.Add(1)
-			fs.lastProcessed.Store(time.Now())
-
-			// Format using the formatter instead of fmt.Sprintf
-			formatted, err := fs.formatter.Format(entry)
-			if err != nil {
-				fs.logger.Error("msg", "Failed to format log entry",
-					"component", "file_sink",
-					"error", err)
-				continue
-			}
-
-			// Convert to string to prevent hex encoding of []byte by log package
-			// Strip new line, writer adds it
-			message := string(bytes.TrimSuffix(formatted, []byte{'\n'}))
-			fs.writer.Message(message)
-
-		case <-ctx.Done():
-			return
-		case <-fs.done:
-			return
-		}
-	}
-}