v0.2.0 restructured to pipeline architecture, dirty

Makefile

@@ -1,31 +1,52 @@
-# FILE: Makefile
+# LogWisp Makefile
-BINARY_NAME := logwisp
+# Compatible with GNU Make (Linux) and BSD Make (FreeBSD)
-VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
-GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
-BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S')
 
-LDFLAGS := -ldflags "-X 'logwisp/src/internal/version.Version=$(VERSION)' \
+BINARY_NAME = logwisp
+BUILD_DIR = bin
+BINARY_PATH = $(BUILD_DIR)/$(BINARY_NAME)
+VERSION != git describe --tags --always --dirty 2>/dev/null || echo "dev"
+GIT_COMMIT != git rev-parse --short HEAD 2>/dev/null || echo "unknown"
+BUILD_TIME != date -u '+%Y-%m-%d_%H:%M:%S'
+
+# Go build variables
+GO = go
+GOFLAGS =
+LDFLAGS = -X 'logwisp/src/internal/version.Version=$(VERSION)' \
           -X 'logwisp/src/internal/version.GitCommit=$(GIT_COMMIT)' \
-                     -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'"
+          -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'
 
-.PHONY: build
+# Installation directories
+PREFIX ?= /usr/local
+BINDIR = $(PREFIX)/bin
+
+# Default target
+all: build
+
+# Build the binary
 build:
-	go build $(LDFLAGS) -o $(BINARY_NAME) ./src/cmd/logwisp
+	mkdir -p $(BUILD_DIR)
+	$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./src/cmd/logwisp
 
-.PHONY: install
+# Install the binary
 install: build
-	install -m 755 $(BINARY_NAME) /usr/local/bin/
+	install -m 755 $(BINARY_PATH) $(BINDIR)/
 
-.PHONY: clean
+# Uninstall the binary
+uninstall:
+	rm -f $(BINDIR)/$(BINARY_PATH)
+
+# Clean build artifacts
 clean:
-	rm -f $(BINARY_NAME)
+	rm -f $(BINARY_PATH)
 
-.PHONY: test
+# Development build with race detector
-test:
+dev:
-	go test -v ./...
+	$(GO) build $(GOFLAGS) -race -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./src/cmd/logwisp
 
-.PHONY: release
+# Show current version
-release:
+version:
-	@if [ -z "$(TAG)" ]; then echo "TAG is required: make release TAG=v1.0.0"; exit 1; fi
+	@echo "Version: $(VERSION)"
-	git tag -a $(TAG) -m "Release $(TAG)"
+	@echo "Commit: $(GIT_COMMIT)"
-	git push origin $(TAG)
+	@echo "Build Time: $(BUILD_TIME)"
+
+.PHONY: all build install uninstall clean dev version

README.md

@@ -3,7 +3,7 @@
     <td width="200" valign="middle">
       <img src="asset/logwisp-logo.svg" alt="LogWisp Logo" width="200"/>
     </td>
-    <td valign="middle">
+    <td>
       <h1>LogWisp</h1>
       <p>
         <a href="https://golang.org"><img src="https://img.shields.io/badge/Go-1.24-00ADD8?style=flat&logo=go" alt="Go"></a>
@@ -14,30 +14,29 @@
   </tr>
 </table>
 
+**Flexible log monitoring with real-time streaming over HTTP/SSE and TCP**
 
-**Multi-stream log monitoring with real-time streaming over HTTP/SSE and TCP**
+LogWisp watches log files and streams updates to connected clients in real-time using a pipeline architecture: **sources → filters → sinks**. Perfect for monitoring multiple applications, filtering noise, and routing logs to multiple destinations.
-
-LogWisp watches log files and streams updates to connected clients in real-time. Perfect for monitoring multiple applications, filtering noise, and centralizing log access.
 
 ## 🚀 Quick Start
 
 ```bash
 # Install
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+git clone https://github.com/lixenwraith/logwisp.git
+cd logwisp
+make install
 
 # Run with defaults (monitors *.log in current directory)
 logwisp
-
-# Stream logs (from another terminal)
-curl -N http://localhost:8080/stream
 ```
 
 ## ✨ Key Features
 
+- **🔧 Pipeline Architecture** - Flexible source → filter → sink processing
 - **📡 Real-time Streaming** - SSE (HTTP) and TCP protocols
 - **🔍 Pattern Filtering** - Include/exclude logs with regex patterns
 - **🛡️ Rate Limiting** - Protect against abuse with configurable limits
-- **📊 Multi-stream** - Monitor different log sources simultaneously
+- **📊 Multi-pipeline** - Process different log sources simultaneously
 - **🔄 Rotation Aware** - Handles log rotation seamlessly
 - **⚡ High Performance** - Minimal CPU/memory footprint
 
@@ -50,33 +49,6 @@ Complete documentation is available in the [`doc/`](doc/) directory:
 - [**CLI Reference**](doc/cli.md) - Command-line interface
 - [**Examples**](doc/examples/) - Ready-to-use configurations
 
-## 💻 Basic Usage
-
-### Monitor application logs with filtering:
-
-```toml
-# ~/.config/logwisp.toml
-[[streams]]
-name = "myapp"
-
-[streams.monitor]
-targets = [{ path = "/var/log/myapp", pattern = "*.log" }]
-
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "WARN", "CRITICAL"]
-
-[streams.httpserver]
-enabled = true
-port = 8080
-```
-
-### Run multiple streams:
-
-```bash
-logwisp --router --config /etc/logwisp/multi-stream.toml
-```
-
 ## 📄 License
 
 BSD-3-Clause

config/logwisp.toml.defaults

@@ -1,713 +1,221 @@
-# LogWisp Configuration File - Complete Reference
+# LogWisp Default Configuration and Guide
 # Default path: ~/.config/logwisp.toml
-# Override with: ./logwisp --config /path/to/config.toml
+# Override: logwisp --config /path/to/config.toml
-
-# This is a complete configuration reference showing all available options.
-# Default values are uncommented, alternatives and examples are commented.
-
-# ==============================================================================
-# LOGGING CONFIGURATION (LogWisp's own operational logs)
-# ==============================================================================
-# Controls where and how LogWisp logs its own operational messages.
-# This is separate from the logs being monitored and streamed.
 
+# ============================================================================
+# LOGGING (LogWisp's operational logs)
+# ============================================================================
 [logging]
-# Output mode: where to write LogWisp's operational logs
+# Output mode: file, stdout, stderr, both, none
-# Options: "file", "stdout", "stderr", "both", "none"
-# - file: Write only to log files
-# - stdout: Write only to standard output
-# - stderr: Write only to standard error (default for containers)
-# - both: Write to both file and console
-# - none: Disable logging (⚠️ SECURITY: Not recommended)
 output = "stderr"
 
-# Minimum log level for operational logs
+# Log level: debug, info, warn, error
-# Options: "debug", "info", "warn", "error"
-# - debug: Maximum verbosity, includes internal state changes
-# - info: Normal operational messages (default)
-# - warn: Warnings and errors only
-# - error: Errors only
 level = "info"
 
-# File output configuration (used when output includes "file" or "both")
+# File output settings (when output includes "file")
 [logging.file]
-# Directory for log files
 directory = "./logs"
-
-# Base name for log files (will append timestamp and .log)
 name = "logwisp"
-
-# Maximum size per log file before rotation (megabytes)
 max_size_mb = 100
-
-# Maximum total size of all log files (megabytes)
-# Oldest files are deleted when limit is reached
 max_total_size_mb = 1000
-
-# How long to keep log files (hours)
-# 0 = no time-based deletion
 retention_hours = 168.0  # 7 days
 
-# Console output configuration
+# Console output settings
 [logging.console]
-# Target for console output
+target = "stderr"  # stdout, stderr, split
-# Options: "stdout", "stderr", "split"
+format = "txt"     # txt, json
-# - stdout: All logs to standard output
-# - stderr: All logs to standard error (default)
-# - split: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
-target = "stderr"
 
-# Output format
+# ============================================================================
-# Options: "txt", "json"
+# PIPELINE CONFIGURATION
-# - txt: Human-readable text format
+# ============================================================================
-# - json: Structured JSON for log aggregation
+# Each [[pipelines]] defines an independent log processing pipeline
-format = "txt"
+# Structure: sources → filters → sinks
 
-# ==============================================================================
+[[pipelines]]
-# STREAM CONFIGURATION
+# Unique pipeline identifier (used in router paths)
-# ==============================================================================
-# Each [[streams]] section defines an independent log monitoring stream.
-# You can have multiple streams, each with its own settings.
-
-# ------------------------------------------------------------------------------
-# Default Stream - Monitors current directory
-# ------------------------------------------------------------------------------
-[[streams]]
-# Stream identifier used in logs, metrics, and router paths
-# Must be unique across all streams
 name = "default"
 
-# File monitoring configuration
+# ----------------------------------------------------------------------------
-[streams.monitor]
+# SOURCES - Input data sources
-# How often to check for new log entries (milliseconds)
+# ----------------------------------------------------------------------------
-# Lower = faster detection but more CPU usage
+[[pipelines.sources]]
-# Range: 10-60000 (0.01 to 60 seconds)
+# Source type: directory, file, stdin
-check_interval_ms = 100
+type = "directory"
 
-# Targets to monitor - can be files or directories
+# Type-specific options
-# At least one target is required
+options = {
-targets = [
+    path = "./",
-    # Monitor all .log files in current directory
+    pattern = "*.log",
-    { path = "./", pattern = "*.log", is_file = false },
+    check_interval_ms = 100  # How often to check for new entries (10-60000)
+}
 
-    # Example: Monitor specific file
+# Additional source examples:
-    # { path = "/var/log/app.log", is_file = true },
+# [[pipelines.sources]]
+# type = "file"
+# options = { path = "/var/log/app.log" }
+#
+# [[pipelines.sources]]
+# type = "stdin"
+# options = {}
 
-    # Example: Multiple patterns in a directory
+# ----------------------------------------------------------------------------
-    # { path = "/logs", pattern = "*.log", is_file = false },
+# FILTERS - Log entry filtering (optional)
-    # { path = "/logs", pattern = "*.txt", is_file = false },
+# ----------------------------------------------------------------------------
-]
-
-# Filter configuration (optional) - controls which logs are streamed
 # Multiple filters are applied sequentially - all must pass
-# Empty patterns array means "match everything"
 
-# Example: Include only errors and warnings
+# [[pipelines.filters]]
-# [[streams.filters]]
+# type = "include"     # include (whitelist) or exclude (blacklist)
-# type = "include"              # "include" (whitelist) or "exclude" (blacklist)
+# logic = "or"         # or (match any) or and (match all)
-# logic = "or"                  # "or" (match any) or "and" (match all)
 # patterns = [
-#     "(?i)error",              # Case-insensitive error matching
+#     "ERROR",
-#     "(?i)warn"                # Case-insensitive warning matching
+#     "(?i)warn",      # Case-insensitive
+#     "\\bfatal\\b"    # Word boundary
 # ]
 
-# Example: Exclude debug and trace logs
+# ----------------------------------------------------------------------------
-# [[streams.filters]]
+# SINKS - Output destinations
-# type = "exclude"
+# ----------------------------------------------------------------------------
-# patterns = ["DEBUG", "TRACE", "VERBOSE"]
+[[pipelines.sinks]]
+# Sink type: http, tcp, file, stdout, stderr
+type = "http"
 
-# HTTP Server configuration (SSE/Server-Sent Events)
+# Type-specific options
-[streams.httpserver]
+options = {
-# Enable/disable HTTP server for this stream
+    port = 8080,
-enabled = true
+    buffer_size = 1000,
+    stream_path = "/stream",
+    status_path = "/status",
 
-# Port to listen on (1-65535)
+    # Heartbeat configuration
-# Each stream needs a unique port unless using router mode
+    heartbeat = {
-port = 8080
+        enabled = true,
-
+        interval_seconds = 30,
-# Per-client buffer size (number of messages)
+        format = "comment",      # comment or json
-# Larger = handles bursts better, more memory per client
+        include_timestamp = true,
-buffer_size = 1000
-
-# Endpoint paths (must start with /)
-stream_path = "/stream"     # SSE stream endpoint
-status_path = "/status"     # Statistics endpoint
-
-# Keep-alive heartbeat configuration
-# Prevents connection timeout on quiet logs
-[streams.httpserver.heartbeat]
-# Enable/disable heartbeat messages
-enabled = true
-
-# Interval between heartbeats (seconds)
-# Range: 1-3600 (1 second to 1 hour)
-interval_seconds = 30
-
-# Heartbeat format
-# Options: "comment", "json"
-# - comment: SSE comment format (: heartbeat)
-# - json: JSON event format (data: {"type":"heartbeat"})
-format = "comment"
-
-# Include timestamp in heartbeat
-include_timestamp = true
-
-# Include connection statistics
         include_stats = false
+    },
 
-# Rate limiting configuration (disabled by default)
+    # Rate limiting (optional)
-# Protects against abuse and resource exhaustion
+    rate_limit = {
-[streams.httpserver.rate_limit]
+        enabled = false,
-# Enable/disable rate limiting
+        requests_per_second = 10.0,
-enabled = false
+        burst_size = 20,
+        limit_by = "ip",         # ip or global
+        max_connections_per_ip = 5,
+        max_total_connections = 100,
+        response_code = 429,
+        response_message = "Rate limit exceeded"
+    }
 
-# Token refill rate (requests per second)
+    # SSL/TLS (planned)
-# Float value, e.g., 0.5 = 1 request every 2 seconds
+    # ssl = {
-# requests_per_second = 10.0
+    #     enabled = false,
-
+    #     cert_file = "/path/to/cert.pem",
-# Maximum burst capacity (token bucket size)
-# Should be 2-3x requests_per_second for normal usage
-# burst_size = 20
-
-# Rate limit strategy
-# Options: "ip", "global"
-# - ip: Each client IP gets its own limit
-# - global: All clients share one limit
-# limit_by = "ip"
-
-# HTTP response code when rate limited
-# Common: 429 (Too Many Requests), 503 (Service Unavailable)
-# response_code = 429
-
-# Response message when rate limited
-# response_message = "Rate limit exceeded"
-
-# Maximum concurrent connections per IP address
-# 0 = unlimited
-# max_connections_per_ip = 5
-
-# Maximum total concurrent connections
-# 0 = unlimited
-# max_total_connections = 100
-
-# SSL/TLS configuration (planned feature)
-# [streams.httpserver.ssl]
-# enabled = false
-# cert_file = "/path/to/cert.pem"
     #     key_file = "/path/to/key.pem"
-# min_version = "TLS1.2"        # Minimum TLS version
+    # }
-# client_auth = false           # Require client certificates
+}
 
-# TCP Server configuration (optional)
+# Additional sink examples:
-# Raw TCP streaming for high-performance scenarios
-# [streams.tcpserver]
-# enabled = false
-# port = 9090
-# buffer_size = 5000            # Larger buffer for TCP
-#
-# [streams.tcpserver.heartbeat]
-# enabled = true
-# interval_seconds = 60
-# include_timestamp = true
-# include_stats = false
-#
-# [streams.tcpserver.rate_limit]
-# enabled = false
-# requests_per_second = 5.0
-# burst_size = 10
-# limit_by = "ip"
 
-# Authentication configuration (planned feature)
+# [[pipelines.sinks]]
-# [streams.auth]
+# type = "tcp"
-# type = "none"                 # Options: "none", "basic", "bearer"
+# options = {
+#     port = 9090,
+#     buffer_size = 5000,
+#     heartbeat = { enabled = true, interval_seconds = 60 }
+# }
+
+# [[pipelines.sinks]]
+# type = "file"
+# options = {
+#     directory = "/var/log/logwisp",
+#     name = "app",
+#     max_size_mb = 100,
+#     retention_hours = 168.0
+# }
+
+# [[pipelines.sinks]]
+# type = "stdout"
+# options = { buffer_size = 500 }
+
+# ----------------------------------------------------------------------------
+# AUTHENTICATION (optional, applies to network sinks)
+# ----------------------------------------------------------------------------
+# [pipelines.auth]
+# type = "none"  # none, basic, bearer
 #
-# # Basic authentication
+# [pipelines.auth.basic_auth]
-# [streams.auth.basic_auth]
-# users_file = "/etc/logwisp/users.htpasswd"
 # realm = "LogWisp"
-#
+# users = [
-# # IP-based access control
+#     { username = "admin", password_hash = "$2a$10$..." }
-# ip_whitelist = ["192.168.1.0/24", "10.0.0.0/8"]
+# ]
-# ip_blacklist = []
+# ip_whitelist = ["192.168.1.0/24"]
 
-# ------------------------------------------------------------------------------
+# ============================================================================
-# Example: Application Logs Stream with Error Filtering
+# COMPLETE EXAMPLES
-# ------------------------------------------------------------------------------
+# ============================================================================
-# [[streams]]
+
-# name = "app"
+# Example: Production logs with filtering and multiple outputs
+# [[pipelines]]
+# name = "production"
 #
-# [streams.monitor]
+# [[pipelines.sources]]
-# check_interval_ms = 50          # Fast detection for active logs
+# type = "directory"
-# targets = [
+# options = { path = "/var/log/app", pattern = "*.log", check_interval_ms = 50 }
-#     # Monitor specific application log directory
-#     { path = "/var/log/myapp", pattern = "*.log", is_file = false },
-#     # Also monitor specific file
-#     { path = "/var/log/myapp/app.log", is_file = true },
-# ]
 #
-# # Filter 1: Include only errors and warnings
+# [[pipelines.filters]]
-# [[streams.filters]]
 # type = "include"
-# logic = "or"                    # Match ANY of these patterns
+# patterns = ["ERROR", "WARN", "CRITICAL"]
-# patterns = [
-#     "(?i)\\berror\\b",          # Word boundary error (case-insensitive)
-#     "(?i)\\bwarn(ing)?\\b",     # warn or warning
-#     "(?i)\\bfatal\\b",          # fatal
-#     "(?i)\\bcritical\\b",       # critical
-#     "(?i)exception",            # exception anywhere
-#     "(?i)fail(ed|ure)?",        # fail, failed, failure
-#     "panic",                    # Go panics
-#     "traceback",                # Python tracebacks
-# ]
 #
-# # Filter 2: Exclude health check noise
+# [[pipelines.filters]]
-# [[streams.filters]]
 # type = "exclude"
-# patterns = [
+# patterns = ["/health", "/metrics"]
-#     "/health",
-#     "/metrics",
-#     "/ping",
-#     "GET /favicon.ico",
-#     "ELB-HealthChecker",
-#     "kube-probe"
-# ]
 #
-# [streams.httpserver]
+# [[pipelines.sinks]]
-# enabled = true
+# type = "http"
-# port = 8081                     # Different port for each stream
+# options = {
-# buffer_size = 2000              # Larger buffer for busy logs
+#     port = 8080,
-# stream_path = "/logs"           # Custom path
+#     rate_limit = { enabled = true, requests_per_second = 25.0 }
-# status_path = "/health"         # Custom health endpoint
+# }
 #
-# # JSON heartbeat format for programmatic clients
+# [[pipelines.sinks]]
-# [streams.httpserver.heartbeat]
+# type = "file"
-# enabled = true
+# options = { directory = "/var/log/archive", name = "errors" }
-# interval_seconds = 20
-# format = "json"                 # JSON event format
-# include_timestamp = true
-# include_stats = true            # Include active client count
-#
-# # Moderate rate limiting for public access
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 25.0
-# burst_size = 50
-# limit_by = "ip"
-# max_connections_per_ip = 10
-# max_total_connections = 200
 
-# ------------------------------------------------------------------------------
+# Example: Multi-source aggregation
-# Example: System Logs Stream (TCP + HTTP) with Security Filtering
+# [[pipelines]]
-# ------------------------------------------------------------------------------
+# name = "aggregated"
-# [[streams]]
-# name = "system"
 #
-# [streams.monitor]
+# [[pipelines.sources]]
-# check_interval_ms = 1000        # Check every second (system logs update slowly)
+# type = "directory"
-# targets = [
+# options = { path = "/var/log/nginx", pattern = "*.log" }
-#     { path = "/var/log/syslog", is_file = true },
-#     { path = "/var/log/auth.log", is_file = true },
-#     { path = "/var/log/kern.log", is_file = true },
-#     { path = "/var/log/messages", is_file = true },
-# ]
 #
-# # Include only security-relevant logs
+# [[pipelines.sources]]
-# [[streams.filters]]
+# type = "directory"
-# type = "include"
+# options = { path = "/var/log/app", pattern = "*.log" }
-# logic = "or"
-# patterns = [
-#     "(?i)auth",
-#     "(?i)sudo",
-#     "(?i)ssh",
-#     "(?i)login",
-#     "(?i)permission",
-#     "(?i)denied",
-#     "(?i)unauthorized",
-#     "(?i)security",
-#     "(?i)selinux",
-#     "kernel:.*audit",
-#     "COMMAND=",                 # sudo commands
-#     "session opened",
-#     "session closed"
-# ]
 #
-# # TCP Server for high-performance streaming
+# [[pipelines.sinks]]
-# [streams.tcpserver]
+# type = "tcp"
-# enabled = true
+# options = { port = 9090 }
-# port = 9090
-# buffer_size = 5000
-#
-# # TCP heartbeat (always JSON format)
-# [streams.tcpserver.heartbeat]
-# enabled = true
-# interval_seconds = 60           # Less frequent for TCP
-# include_timestamp = true
-# include_stats = false
-#
-# # TCP rate limiting
-# [streams.tcpserver.rate_limit]
-# enabled = true
-# requests_per_second = 5.0       # Limit TCP connections
-# burst_size = 10
-# limit_by = "ip"
-#
-# # Also expose via HTTP
-# [streams.httpserver]
-# enabled = true
-# port = 8082
-# buffer_size = 1000
-# stream_path = "/stream"
-# status_path = "/status"
-#
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 5.0
-# burst_size = 10
-# max_connections_per_ip = 2      # Strict for security logs
 
-# ------------------------------------------------------------------------------
+# ============================================================================
-# Example: High-Volume Debug Logs with Performance Filtering
+# ROUTER MODE
-# ------------------------------------------------------------------------------
+# ============================================================================
-# [[streams]]
-# name = "debug"
-#
-# [streams.monitor]
-# check_interval_ms = 5000        # Check every 5 seconds (high volume)
-# targets = [
-#     { path = "/tmp/debug", pattern = "*.debug", is_file = false },
-#     { path = "/var/log/debug", pattern = "debug-*.log", is_file = false },
-# ]
-#
-# # Exclude verbose debug output
-# [[streams.filters]]
-# type = "exclude"
-# patterns = [
-#     "TRACE",
-#     "VERBOSE",
-#     "entering function",
-#     "exiting function",
-#     "memory dump",
-#     "hex dump",
-#     "stack trace",
-#     "goroutine [0-9]+"
-# ]
-#
-# # Include only specific modules
-# [[streams.filters]]
-# type = "include"
-# patterns = [
-#     "module=(api|database|auth)",
-#     "component=(router|handler)",
-#     "service=(payment|order|user)"
-# ]
-#
-# [streams.httpserver]
-# enabled = true
-# port = 8083
-# buffer_size = 10000             # Very large buffer
-# stream_path = "/debug"
-# status_path = "/stats"
-#
-# # Disable heartbeat for high-volume streams
-# [streams.httpserver.heartbeat]
-# enabled = false
-#
-# # Aggressive rate limiting
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 1.0       # Very restrictive
-# burst_size = 5
-# limit_by = "ip"
-# max_connections_per_ip = 1      # One connection per IP
-# response_code = 503             # Service Unavailable
-# response_message = "Debug stream overloaded"
-
-# ------------------------------------------------------------------------------
-# Example: Multi-Application with Router Mode
-# ------------------------------------------------------------------------------
 # Run with: logwisp --router
-#
+# Allows multiple pipelines to share HTTP ports via path-based routing
-# [[streams]]
+# Access: http://localhost:8080/{pipeline_name}/stream
-# name = "frontend"
+# Global status: http://localhost:8080/status
-# [streams.monitor]
-# targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
-# [[streams.filters]]
-# type = "exclude"
-# patterns = ["GET /static/", "GET /assets/"]
-# [streams.httpserver]
-# enabled = true
-# port = 8080                     # Same port OK in router mode
-#
-# [[streams]]
-# name = "backend"
-# [streams.monitor]
-# targets = [{ path = "/var/log/api", pattern = "*.log" }]
-# [[streams.filters]]
-# type = "include"
-# patterns = ["ERROR", "WARN", "timeout", "failed"]
-# [streams.httpserver]
-# enabled = true
-# port = 8080                     # Shared port in router mode
-#
-# [[streams]]
-# name = "database"
-# [streams.monitor]
-# targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
-# [streams.httpserver]
-# enabled = true
-# port = 8080
-#
-# # Access via:
-# # http://localhost:8080/frontend/stream
-# # http://localhost:8080/backend/stream
-# # http://localhost:8080/database/stream
-# # http://localhost:8080/status (global)
 
-# ==============================================================================
+# ============================================================================
-# FILTER PATTERN REFERENCE
+# QUICK REFERENCE
-# ==============================================================================
+# ============================================================================
+# Source types: directory, file, stdin
+# Sink types: http, tcp, file, stdout, stderr
+# Filter types: include, exclude
+# Filter logic: or, and
 #
-# Basic Patterns:
+# Common patterns:
-# - "ERROR"                    # Exact match (case sensitive)
+# "(?i)error"          - Case-insensitive
-# - "(?i)error"               # Case-insensitive
+# "\\berror\\b"        - Word boundary
-# - "\\berror\\b"             # Word boundary (won't match "errorCode")
+# "^ERROR"             - Start of line
-# - "error|warn|fatal"        # Multiple options (OR)
+# "status=[4-5]\\d{2}" - HTTP errors
-# - "(error|warn) level"      # Group with context
-#
-# Position Patterns:
-# - "^\\[ERROR\\]"            # Line starts with [ERROR]
-# - "ERROR:$"                 # Line ends with ERROR:
-# - "^\\d{4}-\\d{2}-\\d{2}"  # Line starts with date
-#
-# Complex Patterns:
-# - "status=[4-5][0-9]{2}"    # HTTP 4xx or 5xx status codes
-# - "duration>[0-9]{4}ms"     # Duration over 999ms
-# - "user_id=\"[^\"]+\""      # Extract user_id values
-# - "\\[ERROR\\].*database"   # ERROR followed by database
-# - "(?i)\\b(error|fail|critical)\\b"  # Multiple error words
-#
-# Log Level Patterns:
-# - "\\[(ERROR|WARN|FATAL)\\]"     # Common formats
-# - "level=(error|warning|critical)" # Key-value format
-# - "ERROR\\s*:"                    # ERROR with optional space
-# - "<(Error|Warning)>"             # XML-style
-#
-# Application Patterns:
-# - "com\\.mycompany\\..*Exception"  # Java exceptions
-# - "at .+\\(.+\\.java:[0-9]+\\)"   # Java stack traces
-# - "File \".+\", line [0-9]+"      # Python tracebacks
-# - "panic: .+"                     # Go panics
-# - "/api/v[0-9]+/"                 # API versioned paths
-#
-# Performance Patterns:
-# - "took [0-9]{4,}ms"              # Operations over 999ms
-# - "memory usage: [8-9][0-9]%"    # High memory usage
-# - "queue size: [0-9]{4,}"        # Large queues
-# - "timeout|timed out"             # Timeouts
-#
-# Security Patterns:
-# - "unauthorized|forbidden"         # Access denied
-# - "invalid token|expired token"   # Auth failures
-# - "SQL injection|XSS"             # Security threats
-# - "failed login.*IP: ([0-9.]+)"  # Failed logins with IP
-#
-# Performance Tips:
-# - Avoid nested quantifiers: "((a+)+)+" causes catastrophic backtracking
-# - Use anchors when possible: "^ERROR" is faster than "ERROR"
-# - Prefer character classes: "[0-9]" over "\\d" for clarity
-# - Use non-capturing groups: "(?:error|warn)" when not extracting
-# - Test complex patterns with sample data before deployment
-# - Consider using multiple simple patterns instead of one complex pattern
-#
-# Security Considerations:
-# - Be aware of ReDoS (Regular Expression Denial of Service)
-# - Limit pattern complexity for public-facing streams
-# - Monitor filter processing time in statistics
-# - Consider pre-filtering very high volume streams
-# - Use explicit allow-lists for sensitive logs
-
-# ==============================================================================
-# RATE LIMITING GUIDE
-# ==============================================================================
-#
-# Token Bucket Algorithm:
-# - Each client (IP) or global limit gets a bucket with 'burst_size' tokens
-# - Tokens refill at 'requests_per_second' rate
-# - Each request consumes one token
-# - Provides smooth rate limiting without hard cutoffs
-#
-# Configuration Examples:
-#
-# Light Protection (default for most streams):
-# requests_per_second = 10.0
-# burst_size = 20              # Handle short bursts
-#
-# Moderate Protection (public endpoints):
-# requests_per_second = 5.0
-# burst_size = 15
-# max_connections_per_ip = 5
-#
-# Strict Protection (sensitive logs):
-# requests_per_second = 1.0
-# burst_size = 3
-# max_connections_per_ip = 1
-# limit_by = "ip"
-#
-# Global Limiting (shared resource):
-# requests_per_second = 50.0   # Total for all clients
-# burst_size = 100
-# limit_by = "global"
-# max_total_connections = 50
-#
-# Behavior:
-# - HTTP: Returns response_code (default 429) with JSON error
-# - TCP: Silently drops connection (no error message)
-# - Cleanup: Inactive IPs removed after 5 minutes
-# - Statistics: Available in /status endpoint
-#
-# Best Practices:
-# - Set burst_size to 2-3x requests_per_second
-# - Use per-IP limiting for fairness
-# - Use global limiting for resource protection
-# - Monitor rate limit statistics for tuning
-# - Consider different limits for different streams
-# - Enable for any public-facing endpoints
-
-# ==============================================================================
-# PERFORMANCE TUNING
-# ==============================================================================
-#
-# Monitor Check Interval:
-# - 10-50ms: Real-time monitoring, higher CPU usage
-# - 100-500ms: Good balance for active logs
-# - 1000-5000ms: Low-activity logs, minimal CPU
-# - 10000ms+: Very slow changing logs
-#
-# Buffer Sizes:
-# - HTTP: 100-1000 for normal use, 5000+ for high volume
-# - TCP: 1000-5000 typical, 10000+ for bulk streaming
-# - Larger = more memory per client, handles bursts better
-#
-# Connection Limits:
-# - Development: No limits needed
-# - Production: 5-10 connections per IP typical
-# - Public: 1-3 connections per IP
-# - Total: Based on available memory (each uses ~1-5MB)
-#
-# Filter Performance:
-# - Simple patterns: ~1μs per check
-# - Complex patterns: ~10-100μs per check
-# - Many patterns: Consider multiple streams instead
-# - Use exclude filters to drop noise early
-#
-# Memory Usage (approximate):
-# - Base process: ~10-20MB
-# - Per stream: ~5-10MB
-# - Per HTTP client: ~1-2MB
-# - Per TCP client: ~0.5-1MB
-# - Filter chain: ~1-5MB depending on patterns
-
-# ==============================================================================
-# DEPLOYMENT SCENARIOS
-# ==============================================================================
-#
-# Single Application:
-# - One stream with basic filtering
-# - Moderate rate limiting
-# - Standard check interval (100ms)
-#
-# Microservices:
-# - Multiple streams, one per service
-# - Router mode for unified access
-# - Different filter rules per service
-# - Service-specific rate limits
-#
-# High Security:
-# - Strict include filters
-# - Low rate limits (1-2 req/sec)
-# - Single connection per IP
-# - TCP for internal, HTTP for external
-#
-# High Performance:
-# - TCP streaming preferred
-# - Large buffers (10000+)
-# - Minimal filtering
-# - Higher check intervals
-# - No heartbeats
-#
-# Development/Testing:
-# - Multiple streams for different log levels
-# - No rate limiting
-# - Debug level logging
-# - Fast check intervals
-# - All filters disabled
-
-# ==============================================================================
-# TROUBLESHOOTING
-# ==============================================================================
-#
-# Common Issues:
-#
-# "No logs appearing":
-# - Check file paths and permissions
-# - Verify pattern matches filenames
-# - Check filters aren't too restrictive
-# - Enable debug logging: --log-level debug
-#
-# "High CPU usage":
-# - Increase check_interval_ms
-# - Reduce number of filter patterns
-# - Use simpler regex patterns
-# - Check for runaway log growth
-#
-# "Clients disconnecting":
-# - Enable heartbeats
-# - Check rate limiting settings
-# - Verify network connectivity
-# - Increase buffer sizes
-#
-# "Memory growth":
-# - Check client connection count
-# - Verify buffer sizes are reasonable
-# - Look for memory leaks in filters
-# - Enable connection limits
-#
-# Debug Commands:
-# - Check status: curl http://localhost:8080/status
-# - Test stream: curl -N http://localhost:8080/stream
-# - View logs: logwisp --log-level debug --log-output stderr
-# - Test filters: Use simple patterns first
-
-# ==============================================================================
-# FUTURE FEATURES (Roadmap)
-# ==============================================================================
-#
-# Authentication:
-# - Basic auth with htpasswd files
-# - Bearer token authentication
-# - JWT validation
-# - mTLS client certificates
-#
-# SSL/TLS:
-# - HTTPS endpoints
-# - TLS for TCP streams
-# - Certificate management
-# - Let's Encrypt integration
-#
-# Advanced Filtering:
-# - Lua scripting for complex logic
-# - Rate-based filtering (N per minute)
-# - Statistical anomaly detection
-# - Multi-line pattern matching
-#
-# Output Formats:
-# - JSON transformation
-# - Field extraction
-# - Custom formatting templates
-# - Compression (gzip)
-#
-# Integrations:
-# - Prometheus metrics
-# - OpenTelemetry traces
-# - Webhook notifications
-# - Cloud storage backends

config/logwisp.toml.minimal

@@ -1,40 +1,44 @@
 # LogWisp Minimal Configuration
 # Save as: ~/.config/logwisp.toml
 
-# Basic stream monitoring application logs
+# Basic pipeline monitoring application logs
-[[streams]]
+[[pipelines]]
 name = "app"
 
-[streams.monitor]
+# Source: Monitor log directory
-check_interval_ms = 100
+[[pipelines.sources]]
-targets = [
+type = "directory"
-    { path = "/var/log/myapp", pattern = "*.log", is_file = false }
+options = { path = "/var/log/myapp", pattern = "*.log", check_interval_ms = 100 }
-]
 
-[streams.httpserver]
+# Sink: HTTP streaming
-enabled = true
+[[pipelines.sinks]]
-port = 8080
+type = "http"
-stream_path = "/stream"
+options = {
+    port = 8080,
+    buffer_size = 1000,
+    stream_path = "/stream",
     status_path = "/status"
+}
 
 # Optional additions:
 
 # 1. Filter for errors only:
-# [[streams.filters]]
+# [[pipelines.filters]]
 # type = "include"
 # patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
 
 # 2. Enable rate limiting:
-# [streams.httpserver.rate_limit]
+# Modify the sink options above:
-# enabled = true
+# options = {
-# requests_per_second = 10.0
+#     port = 8080,
-# burst_size = 20
+#     buffer_size = 1000,
-# limit_by = "ip"
+#     rate_limit = { enabled = true, requests_per_second = 10.0, burst_size = 20 }
+# }
 
-# 3. Add heartbeat:
+# 3. Add file output:
-# [streams.httpserver.heartbeat]
+# [[pipelines.sinks]]
-# enabled = true
+# type = "file"
-# interval_seconds = 30
+# options = { directory = "/var/log/logwisp", name = "app" }
 
 # 4. Change LogWisp's own logging:
 # [logging]

doc/README.md

@@ -1,75 +1,26 @@
 # LogWisp Documentation
 
-Welcome to the LogWisp documentation. This guide covers all aspects of installing, configuring, and using LogWisp for multi-stream log monitoring.
+Documentation covers installation, configuration, and usage of LogWisp's pipeline-based log monitoring system.
 
 ## 📚 Documentation Index
 
 ### Getting Started
-- **[Installation Guide](installation.md)** - How to install LogWisp on various platforms
+- **[Installation Guide](installation.md)** - Platform-specific installation
-- **[Quick Start](quickstart.md)** - Get up and running in 5 minutes
+- **[Quick Start](quickstart.md)** - Get running in 5 minutes
-- **[Architecture Overview](architecture.md)** - System design and components
+- **[Architecture Overview](architecture.md)** - Pipeline design
 
 ### Configuration
-- **[Configuration Guide](configuration.md)** - Complete configuration reference
+- **[Configuration Guide](configuration.md)** - Complete reference
-- **[Environment Variables](environment.md)** - Environment variable reference
+- **[Environment Variables](environment.md)** - Container configuration
-- **[Command Line Options](cli.md)** - CLI flags and parameters
+- **[Command Line Options](cli.md)** - CLI reference
+- **[Sample Configurations](../config/)** - Default & Minimal Config
 
 ### Features
-- **[Filters Guide](filters.md)** - Pattern-based log filtering
+- **[Status Monitoring](status.md)** - Health checks
-- **[Rate Limiting](ratelimiting.md)** - Request and connection limiting
+- **[Filters Guide](filters.md)** - Pattern-based filtering
-- **[Router Mode](router.md)** - Path-based multi-stream routing
+- **[Rate Limiting](ratelimiting.md)** - Connection protection
-- **[Authentication](authentication.md)** - Securing your log streams *(planned)*
+- **[Router Mode](router.md)** - Multi-pipeline routing
-
+- **[Authentication](authentication.md)** - Access control *(planned)*
-### Operations
-- **[Monitoring & Status](monitoring.md)** - Health checks and statistics
-- **[Performance Tuning](performance.md)** - Optimization guidelines
-- **[Troubleshooting](troubleshooting.md)** - Common issues and solutions
-
-### Advanced Topics
-- **[Security Best Practices](security.md)** - Hardening your deployment
-- **[Integration Examples](integrations.md)** - Working with other tools
-- **[Development Guide](development.md)** - Contributing to LogWisp
-
-## 🚀 Quick Links
-
-- **[Example Configurations](examples/)** - Ready-to-use config templates
-- **[API Reference](api.md)** - SSE/TCP protocol documentation
-- **[Changelog](../CHANGELOG.md)** - Version history and updates
-
-## 💡 Common Use Cases
-
-### Single Application Monitoring
-Monitor logs from one application with basic filtering:
-```toml
-[[streams]]
-name = "myapp"
-[streams.monitor]
-targets = [{ path = "/var/log/myapp", pattern = "*.log" }]
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "WARN"]
-```
-
-### Multi-Service Architecture
-Monitor multiple services with different configurations:
-```bash
-logwisp --router --config /etc/logwisp/services.toml
-```
-
-### High-Security Environments
-Enable authentication and rate limiting:
-```toml
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-max_connections_per_ip = 3
-```
-
-## 🔍 Finding Help
-
-- **GitHub Issues**: [Report bugs or request features](https://github.com/logwisp/logwisp/issues)
-- **Discussions**: [Ask questions and share ideas](https://github.com/logwisp/logwisp/discussions)
-- **Examples**: Check the [examples directory](examples/) for common scenarios
 
 ## 📝 License
 

doc/architecture.md

@@ -0,0 +1,304 @@
+# Architecture Overview
+
+LogWisp implements a flexible pipeline architecture for real-time log processing and streaming.
+
+## Core Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────────┐
+│                              LogWisp Service                            │
+├─────────────────────────────────────────────────────────────────────────┤
+│                                                                         │
+│  ┌─────────────────────────── Pipeline 1 ───────────────────────────┐   │
+│  │                                                                  │   │
+│  │  Sources           Filters              Sinks                    │   │
+│  │  ┌──────┐        ┌────────┐          ┌──────┐                    │   │
+│  │  │ Dir  │──┐     │Include │     ┌────│ HTTP │←── Client 1        │   │
+│  │  └──────┘  │     │ ERROR  │     │    └──────┘                    │   │
+│  │            ├────▶│  WARN  │────▶├────┌──────┐                    │   │
+│  │  ┌──────┐  │     └────────┘     │    │ File │                    │   │
+│  │  │ File │──┘          ▼         │    └──────┘                    │   │
+│  │  └──────┘        ┌────────┐     │    ┌──────┐                    │   │
+│  │                  │Exclude │     └────│ TCP  │←── Client 2        │   │
+│  │                  │ DEBUG  │          └──────┘                    │   │
+│  │                  └────────┘                                      │   │
+│  └──────────────────────────────────────────────────────────────────┘   │
+│                                                                         │
+│  ┌─────────────────────────── Pipeline 2 ───────────────────────────┐   │
+│  │                                                                  │   │
+│  │  ┌──────┐                            ┌──────┐                    │   │
+│  │  │Stdin │────────────────────────────│Stdout│                    │   │
+│  │  └──────┘         (No Filters)       └──────┘                    │   │
+│  └──────────────────────────────────────────────────────────────────┘   │
+│                                                                         │
+│  ┌─────────────────────────── Pipeline N ───────────────────────────┐   │
+│  │  Multiple Sources → Filter Chain → Multiple Sinks                │   │
+│  └──────────────────────────────────────────────────────────────────┘   │
+└─────────────────────────────────────────────────────────────────────────┘
+```
+
+## Data Flow
+
+```
+Log Entry Flow:
+
+┌─────────┐     ┌─────────┐     ┌─────────┐     ┌─────────┐
+│  File   │     │  Parse  │     │ Filter  │     │ Format  │
+│ Watcher │────▶│  Entry  │────▶│  Chain  │────▶│  Send   │
+└─────────┘     └─────────┘     └─────────┘     └─────────┘
+     │               │               │               │
+     ▼               ▼               ▼               ▼
+  Detect         Extract         Include/        Deliver to
+  Changes        Timestamp       Exclude         Clients
+                 & Level         Patterns
+
+
+Entry Processing:
+
+1. Source Detection     2. Entry Creation      3. Filter Application
+   ┌──────────┐           ┌────────────┐         ┌─────────────┐
+   │New Entry │           │ Timestamp  │         │  Filter 1   │
+   │Detected  │──────────▶│   Level    │────────▶│ Include?    │
+   └──────────┘           │  Message   │         └──────┬──────┘
+                          └────────────┘                │
+                                                        ▼
+4. Sink Distribution                             ┌─────────────┐
+   ┌──────────┐                                  │  Filter 2   │
+   │   HTTP   │◀───┐                             │ Exclude?    │
+   └──────────┘    │                             └──────┬──────┘
+   ┌──────────┐    │                                    │
+   │   TCP    │◀───┼────────── Entry ◀──────────────────┘
+   └──────────┘    │           (if passed)
+   ┌──────────┐    │
+   │   File   │◀───┘
+   └──────────┘
+```
+
+## Component Details
+
+### Sources
+
+Sources monitor inputs and generate log entries:
+
+```
+Directory Source:
+┌─────────────────────────────────┐
+│        Directory Monitor        │
+├─────────────────────────────────┤
+│ • Pattern Matching (*.log)      │
+│ • File Rotation Detection       │
+│ • Position Tracking             │
+│ • Concurrent File Watching      │
+└─────────────────────────────────┘
+           │
+           ▼
+    ┌──────────────┐
+    │ File Watcher │ (per file)
+    ├──────────────┤
+    │ • Read New   │
+    │ • Track Pos  │
+    │ • Detect Rot │
+    └──────────────┘
+```
+
+### Filters
+
+Filters process entries through pattern matching:
+
+```
+Filter Chain:
+                 ┌─────────────┐
+Entry ──────────▶│  Filter 1   │
+                 │  (Include)  │
+                 └──────┬──────┘
+                        │ Pass?
+                        ▼
+                 ┌─────────────┐
+                 │  Filter 2   │
+                 │  (Exclude)  │
+                 └──────┬──────┘
+                        │ Pass?
+                        ▼
+                 ┌─────────────┐
+                 │  Filter N   │
+                 └──────┬──────┘
+                        │
+                        ▼
+                    To Sinks
+```
+
+### Sinks
+
+Sinks deliver processed entries to destinations:
+
+```
+HTTP Sink (SSE):
+┌───────────────────────────────────┐
+│            HTTP Server            │
+├───────────────────────────────────┤
+│    ┌─────────┐    ┌─────────┐     │
+│    │ Stream  │    │ Status  │     │
+│    │Endpoint │    │Endpoint │     │
+│    └────┬────┘    └────┬────┘     │
+│         │              │          │
+│    ┌────▼──────────────▼────┐     │
+│    │   Connection Manager   │     │
+│    ├────────────────────────┤     │
+│    │ • Rate Limiting        │     │
+│    │ • Heartbeat            │     │
+│    │ • Buffer Management    │     │
+│    └────────────────────────┘     │
+└───────────────────────────────────┘
+
+TCP Sink:
+┌───────────────────────────────────┐
+│            TCP Server             │
+├───────────────────────────────────┤
+│    ┌────────────────────────┐     │
+│    │    gnet Event Loop     │     │
+│    ├────────────────────────┤     │
+│    │ • Async I/O            │     │
+│    │ • Connection Pool      │     │
+│    │ • Rate Limiting        │     │
+│    └────────────────────────┘     │
+└───────────────────────────────────┘
+```
+
+## Router Mode
+
+In router mode, multiple pipelines share HTTP ports:
+
+```
+Router Architecture:
+                    ┌─────────────────┐
+                    │   HTTP Router   │
+                    │    Port 8080    │
+                    └────────┬────────┘
+                             │
+        ┌────────────────────┼────────────────────┐
+        │                    │                    │
+   /app/stream          /db/stream         /sys/stream
+        │                    │                    │
+   ┌────▼────┐         ┌────▼────┐         ┌────▼────┐
+   │Pipeline │         │Pipeline │         │Pipeline │
+   │  "app"  │         │  "db"   │         │  "sys"  │
+   └─────────┘         └─────────┘         └─────────┘
+
+Path Routing:
+Client Request ──▶ Router ──▶ Parse Path ──▶ Find Pipeline ──▶ Route
+                                  │
+                                  ▼
+                           Extract Pipeline Name
+                           from /pipeline/endpoint
+```
+
+## Memory Management
+
+```
+Buffer Flow:
+┌──────────┐     ┌──────────┐     ┌──────────┐
+│  Source  │     │ Pipeline │     │   Sink   │
+│  Buffer  │────▶│  Buffer  │────▶│  Buffer  │
+│ (1000)   │     │  (chan)  │     │ (1000)   │
+└──────────┘     └──────────┘     └──────────┘
+     │                │                 │
+     ▼                ▼                 ▼
+ Drop if full    Backpressure      Drop if full
+ (counted)        (blocking)        (counted)
+```
+
+## Rate Limiting
+
+```
+Token Bucket Algorithm:
+┌─────────────────────────────┐
+│        Token Bucket         │
+├─────────────────────────────┤
+│ Capacity: burst_size        │
+│ Refill: requests_per_second │
+│                             │
+│   ┌─────────────────────┐   │
+│   │ ● ● ● ● ● ● ○ ○ ○ ○ │   │
+│   └─────────────────────┘   │
+│    6/10 tokens available    │
+└─────────────────────────────┘
+         │
+         ▼
+   Request arrives
+         │
+         ▼
+   Token available? ──No──▶ Reject (429)
+         │
+        Yes
+         ▼
+   Consume token ──▶ Allow request
+```
+
+## Concurrency Model
+
+```
+Goroutine Structure:
+
+Main ────┬──── Pipeline 1 ────┬──── Source Reader 1
+         │                    ├──── Source Reader 2
+         │                    ├──── Filter Processor
+         │                    ├──── HTTP Server
+         │                    └──── TCP Server
+         │
+         ├──── Pipeline 2 ────┬──── Source Reader
+         │                    └──── Sink Writers
+         │
+         └──── HTTP Router (if enabled)
+
+Channel Communication:
+Source ──chan──▶ Filter ──chan──▶ Sink
+  │                                 │
+  └── Non-blocking send ────────────┘
+      (drop & count if full)
+```
+
+## Configuration Loading
+
+```
+Priority Order:
+1. CLI Flags ─────────┐
+2. Environment Vars ──┼──▶ Merge ──▶ Final Config
+3. Config File ───────┤
+4. Defaults ──────────┘
+
+Example:
+CLI:     --log-level debug
+Env:     LOGWISP_PIPELINES_0_NAME=app
+File:    pipelines.toml
+Default: buffer_size = 1000
+```
+
+## Security Architecture
+
+```
+Security Layers:
+
+┌─────────────────────────────────────┐
+│         Network Layer               │
+├─────────────────────────────────────┤
+│ • Rate Limiting (per IP/global)     │
+│ • Connection Limits                 │
+│ • TLS/SSL (planned)                 │
+└──────────────┬──────────────────────┘
+               │
+┌──────────────▼──────────────────────┐
+│      Authentication Layer           │
+├─────────────────────────────────────┤
+│ • Basic Auth (planned)              │
+│ • Bearer Tokens (planned)           │
+│ • IP Whitelisting (planned)         │
+└──────────────┬──────────────────────┘
+               │
+┌──────────────▼──────────────────────┐
+│      Application Layer              │
+├─────────────────────────────────────┤
+│ • Input Validation                  │
+│ • Path Traversal Prevention         │
+│ • Resource Limits                   │
+└─────────────────────────────────────┘
+```

doc/cli.md

@@ -1,6 +1,6 @@
 # Command Line Interface
 
-LogWisp provides a comprehensive set of command-line options for controlling its behavior without modifying configuration files.
+LogWisp CLI options for controlling behavior without modifying configuration files.
 
 ## Synopsis
 
@@ -11,145 +11,92 @@ logwisp [options]
 ## General Options
 
 ### `--config <path>`
-Specify the configuration file location.
+Configuration file location.
 - **Default**: `~/.config/logwisp.toml`
 - **Example**: `logwisp --config /etc/logwisp/production.toml`
 
 ### `--router`
-Enable HTTP router mode for path-based routing of multiple streams.
+Enable HTTP router mode for path-based routing.
-- **Default**: `false` (standalone mode)
+- **Default**: `false`
-- **Use case**: Consolidate multiple HTTP streams on shared ports
 - **Example**: `logwisp --router`
 
 ### `--version`
-Display version information and exit.
+Display version information.
-- **Example**: `logwisp --version`
 
 ### `--background`
-Run LogWisp as a background process.
+Run as background process.
-- **Default**: `false` (foreground mode)
 - **Example**: `logwisp --background`
 
 ## Logging Options
 
-These options override the corresponding configuration file settings.
+Override configuration file settings:
 
 ### `--log-output <mode>`
-Control where LogWisp writes its own operational logs.
+LogWisp's operational log output.
 - **Values**: `file`, `stdout`, `stderr`, `both`, `none`
-- **Default**: Configured value or `stderr`
 - **Example**: `logwisp --log-output both`
 
-#### Output Modes:
-- `file`: Write logs only to files
-- `stdout`: Write logs only to standard output
-- `stderr`: Write logs only to standard error
-- `both`: Write logs to both files and console
-- `none`: Disable logging (⚠️ SECURITY: Not recommended)
-
 ### `--log-level <level>`
-Set the minimum log level for LogWisp's operational logs.
+Minimum log level.
 - **Values**: `debug`, `info`, `warn`, `error`
-- **Default**: Configured value or `info`
 - **Example**: `logwisp --log-level debug`
 
 ### `--log-file <path>`
-Specify the log file path when using file output.
+Log file path (with file output).
-- **Default**: Configured value or `./logs/logwisp.log`
+- **Example**: `logwisp --log-file /var/log/logwisp/app.log`
-- **Example**: `logwisp --log-output file --log-file /var/log/logwisp/app.log`
 
 ### `--log-dir <directory>`
-Specify the log directory when using file output.
+Log directory (with file output).
-- **Default**: Configured value or `./logs`
+- **Example**: `logwisp --log-dir /var/log/logwisp`
-- **Example**: `logwisp --log-output file --log-dir /var/log/logwisp`
 
 ### `--log-console <target>`
-Control console output destination when using `stdout`, `stderr`, or `both` modes.
+Console output destination.
 - **Values**: `stdout`, `stderr`, `split`
-- **Default**: `stderr`
+- **Example**: `logwisp --log-console split`
-- **Example**: `logwisp --log-output both --log-console split`
-
-#### Console Targets:
-- `stdout`: All logs to standard output
-- `stderr`: All logs to standard error
-- `split`: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
 
 ## Examples
 
 ### Basic Usage
 ```bash
-# Start with default configuration
+# Default configuration
 logwisp
 
-# Use a specific configuration file
+# Specific configuration
 logwisp --config /etc/logwisp/production.toml
 ```
 
-### Development Mode
+### Development
 ```bash
-# Enable debug logging to console
+# Debug mode
 logwisp --log-output stderr --log-level debug
 
-# Debug with file output
+# With file output
 logwisp --log-output both --log-level debug --log-dir ./debug-logs
 ```
 
-### Production Deployment
+### Production
 ```bash
-# File logging with info level
+# File logging
-logwisp --log-output file --log-dir /var/log/logwisp --log-level info
+logwisp --log-output file --log-dir /var/log/logwisp
 
-# Background mode with custom config
+# Background with router
-logwisp --background --config /etc/logwisp/prod.toml
+logwisp --background --router --config /etc/logwisp/prod.toml
-
-# Router mode for multiple services
-logwisp --router --config /etc/logwisp/services.toml
-```
-
-### Troubleshooting
-```bash
-# Maximum verbosity to stderr
-logwisp --log-output stderr --log-level debug
-
-# Check version
-logwisp --version
-
-# Test configuration without backgrounding
-logwisp --config test.toml --log-level debug
 ```
 
 ## Priority Order
 
-Configuration values are applied in the following priority order (highest to lowest):
+1. **Command-line flags** (highest)
-
+2. **Environment variables**
-1. **Command-line flags** - Explicitly specified options
+3. **Configuration file**
-2. **Environment variables** - `LOGWISP_*` prefixed variables
+4. **Built-in defaults** (lowest)
-3. **Configuration file** - TOML configuration
-4. **Built-in defaults** - Hardcoded fallback values
 
 ## Exit Codes
 
-- `0`: Successful execution
+- `0`: Success
-- `1`: General error (configuration, startup failure)
+- `1`: General error
-- `2`: Invalid command-line arguments
+- `2`: Invalid arguments
 
 ## Signals
 
-LogWisp responds to the following signals:
-
 - `SIGINT` (Ctrl+C): Graceful shutdown
 - `SIGTERM`: Graceful shutdown
-- `SIGKILL`: Immediate termination (not recommended)
-
-During graceful shutdown, LogWisp will:
-1. Stop accepting new connections
-2. Finish streaming to existing clients
-3. Flush all buffers
-4. Close all file handles
-5. Exit cleanly
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [Environment Variables](environment.md) - Environment variable options
-- [Router Mode](router.md) - Path-based routing details

doc/configuration.md

@@ -1,10 +1,9 @@
 # Configuration Guide
 
-LogWisp uses TOML format for configuration with sensible defaults for all settings.
+LogWisp uses TOML format with a flexible **source → filter → sink** pipeline architecture.
 
 ## Configuration File Location
 
-Default search order:
 1. Command line: `--config /path/to/config.toml`
 2. Environment: `$LOGWISP_CONFIG_FILE`
 3. User config: `~/.config/logwisp.toml`
@@ -13,342 +12,281 @@ Default search order:
 ## Configuration Structure
 
 ```toml
-# Optional: LogWisp's own logging configuration
+# Optional: LogWisp's own logging
 [logging]
-output = "stderr"  # file, stdout, stderr, both, none
+output = "stderr"
-level = "info"     # debug, info, warn, error
+level = "info"
 
-# Required: At least one stream
+# Required: At least one pipeline
-[[streams]]
+[[pipelines]]
-name = "default"   # Unique identifier
+name = "default"
 
-[streams.monitor]  # Required: What to monitor
+# Sources (required)
-# ... monitor settings ...
+[[pipelines.sources]]
+type = "directory"
+options = { ... }
 
-[streams.httpserver]  # Optional: HTTP/SSE server
+# Filters (optional)
-# ... HTTP settings ...
+[[pipelines.filters]]
+type = "include"
+patterns = [...]
 
-[streams.tcpserver]   # Optional: TCP server
+# Sinks (required)
-# ... TCP settings ...
+[[pipelines.sinks]]
-
+type = "http"
-[[streams.filters]]   # Optional: Log filtering
+options = { ... }
-# ... filter settings ...
 ```
 
 ## Logging Configuration
 
-Controls LogWisp's operational logging (not the logs being monitored).
+Controls LogWisp's operational logging:
 
 ```toml
 [logging]
-output = "stderr"  # Where to write LogWisp's logs
+output = "stderr"  # file, stdout, stderr, both, none
-level = "info"     # Minimum log level
+level = "info"     # debug, info, warn, error
 
-# File output settings (when output includes "file")
 [logging.file]
-directory = "./logs"          # Log directory
+directory = "./logs"
-name = "logwisp"             # Base filename
+name = "logwisp"
-max_size_mb = 100            # Rotate at this size
+max_size_mb = 100
-max_total_size_mb = 1000     # Total size limit
+max_total_size_mb = 1000
-retention_hours = 168        # Keep for 7 days
+retention_hours = 168
 
-# Console output settings
 [logging.console]
 target = "stderr"  # stdout, stderr, split
 format = "txt"     # txt or json
 ```
 
-## Stream Configuration
+## Pipeline Configuration
 
-Each `[[streams]]` section defines an independent log monitoring pipeline.
+Each `[[pipelines]]` section defines an independent processing pipeline.
 
-### Monitor Settings
+### Sources
 
-What files or directories to watch:
+Input data sources:
 
+#### Directory Source
 ```toml
-[streams.monitor]
+[[pipelines.sources]]
-check_interval_ms = 100  # How often to check for new entries
+type = "directory"
-
+options = {
-# Monitor targets (at least one required)
+    path = "/var/log/myapp",      # Directory to monitor
-targets = [
+    pattern = "*.log",            # File pattern (glob)
-    # Watch all .log files in a directory
+    check_interval_ms = 100       # Check interval (10-60000)
-    { path = "/var/log/myapp", pattern = "*.log", is_file = false },
+}
-    
-    # Watch a specific file
-    { path = "/var/log/app.log", is_file = true },
-    
-    # Multiple patterns
-    { path = "/logs", pattern = "app-*.log", is_file = false },
-    { path = "/logs", pattern = "error-*.txt", is_file = false }
-]
 ```
 
-### HTTP Server (SSE)
+#### File Source
-
-Server-Sent Events streaming over HTTP:
-
 ```toml
-[streams.httpserver]
+[[pipelines.sources]]
-enabled = true
+type = "file"
-port = 8080
+options = {
-buffer_size = 1000      # Per-client event buffer
+    path = "/var/log/app.log"     # Specific file
-stream_path = "/stream" # SSE endpoint
+}
-status_path = "/status" # Statistics endpoint
-
-# Keep-alive heartbeat
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-format = "comment"        # "comment" or "json"
-include_timestamp = true
-include_stats = false
-
-# Rate limiting (optional)
-[streams.httpserver.rate_limit]
-enabled = false
-requests_per_second = 10.0
-burst_size = 20
-limit_by = "ip"           # "ip" or "global"
-response_code = 429
-response_message = "Rate limit exceeded"
-max_connections_per_ip = 5
-max_total_connections = 100
 ```
 
-### TCP Server
+#### Stdin Source
-
-Raw TCP streaming for high performance:
-
 ```toml
-[streams.tcpserver]
+[[pipelines.sources]]
-enabled = true
+type = "stdin"
-port = 9090
+options = {}
-buffer_size = 5000  # Larger buffer for TCP
-
-# Heartbeat (always JSON format for TCP)
-[streams.tcpserver.heartbeat]
-enabled = true
-interval_seconds = 60
-include_timestamp = true
-include_stats = false
-
-# Rate limiting
-[streams.tcpserver.rate_limit]
-enabled = false
-requests_per_second = 5.0
-burst_size = 10
-limit_by = "ip"
 ```
 
 ### Filters
 
-Control which log entries are streamed:
+Control which log entries pass through:
 
 ```toml
 # Include filter - only matching logs pass
-[[streams.filters]]
+[[pipelines.filters]]
 type = "include"
-logic = "or"      # "or" = match any, "and" = match all
+logic = "or"      # or: match any, and: match all
 patterns = [
     "ERROR",
-    "WARN",
+    "(?i)warn",   # Case-insensitive
-    "CRITICAL"
+    "\\bfatal\\b" # Word boundary
 ]
 
 # Exclude filter - matching logs are dropped
-[[streams.filters]]
+[[pipelines.filters]]
 type = "exclude"
-patterns = [
+patterns = ["DEBUG", "health-check"]
-    "DEBUG",
+```
-    "health check"
+
-]
+### Sinks
+
+Output destinations:
+
+#### HTTP Sink (SSE)
+```toml
+[[pipelines.sinks]]
+type = "http"
+options = {
+    port = 8080,
+    buffer_size = 1000,
+    stream_path = "/stream",
+    status_path = "/status",
+    
+    # Heartbeat
+    heartbeat = {
+        enabled = true,
+        interval_seconds = 30,
+        format = "comment",  # comment or json
+        include_timestamp = true
+    },
+    
+    # Rate limiting
+    rate_limit = {
+        enabled = true,
+        requests_per_second = 10.0,
+        burst_size = 20,
+        limit_by = "ip",  # ip or global
+        max_connections_per_ip = 5
+    }
+}
+```
+
+#### TCP Sink
+```toml
+[[pipelines.sinks]]
+type = "tcp"
+options = {
+    port = 9090,
+    buffer_size = 5000,
+    heartbeat = { enabled = true, interval_seconds = 60 },
+    rate_limit = { enabled = true, requests_per_second = 5.0 }
+}
+```
+
+#### File Sink
+```toml
+[[pipelines.sinks]]
+type = "file"
+options = {
+    directory = "/var/log/logwisp",
+    name = "app",
+    max_size_mb = 100,
+    retention_hours = 168.0,
+    buffer_size = 2000
+}
+```
+
+#### Console Sinks
+```toml
+[[pipelines.sinks]]
+type = "stdout"  # or "stderr"
+options = { buffer_size = 500 }
 ```
 
 ## Complete Examples
 
-### Minimal Configuration
+### Basic Application Monitoring
 
 ```toml
-[[streams]]
+[[pipelines]]
-name = "simple"
+name = "app"
-[streams.monitor]
+
-targets = [{ path = "./logs", pattern = "*.log" }]
+[[pipelines.sources]]
-[streams.httpserver]
+type = "directory"
-enabled = true
+options = { path = "/var/log/app", pattern = "*.log" }
-port = 8080
+
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
 ```
 
-### Production Web Application
+### Production with Filtering
 
 ```toml
 [logging]
 output = "file"
 level = "info"
-[logging.file]
-directory = "/var/log/logwisp"
-max_size_mb = 500
-retention_hours = 336  # 14 days
 
-[[streams]]
+[[pipelines]]
-name = "webapp"
+name = "production"
 
-[streams.monitor]
+[[pipelines.sources]]
-check_interval_ms = 50
+type = "directory"
-targets = [
+options = { path = "/var/log/app", pattern = "*.log", check_interval_ms = 50 }
-    { path = "/var/log/nginx", pattern = "access.log*" },
-    { path = "/var/log/nginx", pattern = "error.log*" },
-    { path = "/var/log/myapp", pattern = "*.log" }
-]
 
-# Only errors and warnings
+[[pipelines.filters]]
-[[streams.filters]]
 type = "include"
-logic = "or"
+patterns = ["ERROR", "WARN", "CRITICAL"]
-patterns = [
-    "\\b(ERROR|error|Error)\\b",
-    "\\b(WARN|WARNING|warn|warning)\\b",
-    "\\b(CRITICAL|FATAL|critical|fatal)\\b",
-    "status=[4-5][0-9][0-9]"  # HTTP errors
-]
 
-# Exclude noise
+[[pipelines.filters]]
-[[streams.filters]]
 type = "exclude"
-patterns = [
+patterns = ["/health", "/metrics"]
-    "/health",
-    "/metrics",
-    "ELB-HealthChecker"
-]
 
-[streams.httpserver]
+[[pipelines.sinks]]
-enabled = true
+type = "http"
-port = 8080
+options = {
-buffer_size = 2000
+    port = 8080,
+    rate_limit = { enabled = true, requests_per_second = 25.0 }
+}
 
-[streams.httpserver.rate_limit]
+[[pipelines.sinks]]
-enabled = true
+type = "file"
-requests_per_second = 25.0
+options = { directory = "/var/log/archive", name = "errors" }
-burst_size = 50
-max_connections_per_ip = 10
 ```
 
-### Multi-Service with Router
+### Multi-Source Aggregation
+
+```toml
+[[pipelines]]
+name = "aggregated"
+
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/nginx", pattern = "*.log" }
+
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
+
+[[pipelines.sources]]
+type = "stdin"
+options = {}
+
+[[pipelines.sinks]]
+type = "tcp"
+options = { port = 9090 }
+```
+
+### Router Mode
 
 ```toml
 # Run with: logwisp --router
 
-# Service 1: API
+[[pipelines]]
-[[streams]]
 name = "api"
-[streams.monitor]
+[[pipelines.sources]]
-targets = [{ path = "/var/log/api", pattern = "*.log" }]
+type = "directory"
-[streams.httpserver]
+options = { path = "/var/log/api", pattern = "*.log" }
-enabled = true
+[[pipelines.sinks]]
-port = 8080  # All streams can use same port in router mode
+type = "http"
+options = { port = 8080 }  # Same port OK in router mode
 
-# Service 2: Database
+[[pipelines]]
-[[streams]]
+name = "web"
-name = "database"
+[[pipelines.sources]]
-[streams.monitor]
+type = "directory"
-targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
+options = { path = "/var/log/nginx", pattern = "*.log" }
-[[streams.filters]]
+[[pipelines.sinks]]
-type = "include"
+type = "http"
-patterns = ["ERROR", "FATAL", "deadlock", "timeout"]
+options = { port = 8080 }  # Shared port
-[streams.httpserver]
-enabled = true
-port = 8080
 
-# Service 3: System
+# Access:
-[[streams]]
+# http://localhost:8080/api/stream
-name = "system"
+# http://localhost:8080/web/stream
-[streams.monitor]
+# http://localhost:8080/status
-targets = [
-    { path = "/var/log/syslog", is_file = true },
-    { path = "/var/log/auth.log", is_file = true }
-]
-[streams.tcpserver]
-enabled = true
-port = 9090
 ```
 
-### High-Security Configuration
-
-```toml
-[logging]
-output = "file"
-level = "warn"  # Less verbose
-
-[[streams]]
-name = "secure"
-
-[streams.monitor]
-targets = [{ path = "/var/log/secure", pattern = "*.log" }]
-
-# Only security events
-[[streams.filters]]
-type = "include"
-patterns = [
-    "auth",
-    "sudo",
-    "ssh",
-    "login",
-    "failed",
-    "denied"
-]
-
-[streams.httpserver]
-enabled = true
-port = 8443
-
-# Strict rate limiting
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 2.0
-burst_size = 3
-limit_by = "ip"
-max_connections_per_ip = 1
-response_code = 403  # Forbidden instead of 429
-
-# Future: Authentication
-# [streams.auth]
-# type = "basic"
-# [streams.auth.basic_auth]
-# users_file = "/etc/logwisp/users.htpasswd"
-```
-
-## Configuration Tips
-
-### Performance Tuning
-
-- **check_interval_ms**: Higher values reduce CPU usage
-- **buffer_size**: Larger buffers handle bursts better
-- **rate_limit**: Essential for public-facing streams
-
-### Filter Patterns
-
-- Use word boundaries: `\\berror\\b` (won't match "errorCode")
-- Case-insensitive: `(?i)error`
-- Anchors for speed: `^ERROR` faster than `ERROR`
-- Test complex patterns before deployment
-
-### Resource Limits
-
-- Each stream uses ~10-50MB RAM (depending on buffers)
-- CPU usage scales with check_interval and file activity
-- Network bandwidth depends on log volume and client count
-
 ## Validation
 
-LogWisp validates configuration on startup:
+LogWisp validates on startup:
-- Required fields (name, monitor targets)
+- Required fields (name, sources, sinks)
-- Port conflicts between streams
+- Port conflicts between pipelines
-- Pattern syntax for filters
+- Pattern syntax
 - Path accessibility
-
+- Rate limit values
-## See Also
-
-- [Environment Variables](environment.md) - Override via environment
-- [CLI Options](cli.md) - Override via command line
-- [Filter Guide](filters.md) - Advanced filtering patterns
-- [Examples](examples/) - Ready-to-use configurations

doc/environment.md

@@ -1,275 +1,148 @@
 # Environment Variables
 
-LogWisp supports comprehensive configuration through environment variables, allowing deployment without configuration files or dynamic overrides in containerized environments.
+Configure LogWisp through environment variables for containerized deployments.
 
 ## Naming Convention
 
-Environment variables follow a structured pattern:
 - **Prefix**: `LOGWISP_`
 - **Path separator**: `_` (underscore)
 - **Array indices**: Numeric suffix (0-based)
 - **Case**: UPPERCASE
 
-### Examples:
+Examples:
-- Config file setting: `logging.level = "debug"`
+- `logging.level` → `LOGWISP_LOGGING_LEVEL`
-- Environment variable: `LOGWISP_LOGGING_LEVEL=debug`
+- `pipelines[0].name` → `LOGWISP_PIPELINES_0_NAME`
-
-- Array element: `streams[0].name = "app"`
-- Environment variable: `LOGWISP_STREAMS_0_NAME=app`
 
 ## General Variables
 
 ### `LOGWISP_CONFIG_FILE`
-Path to the configuration file.
+Configuration file path.
-- **Default**: `~/.config/logwisp.toml`
+```bash
-- **Example**: `LOGWISP_CONFIG_FILE=/etc/logwisp/config.toml`
+export LOGWISP_CONFIG_FILE=/etc/logwisp/config.toml
+```
 
 ### `LOGWISP_CONFIG_DIR`
-Directory containing configuration files.
+Configuration directory.
-- **Usage**: Combined with `LOGWISP_CONFIG_FILE` for relative paths
-- **Example**:
 ```bash
 export LOGWISP_CONFIG_DIR=/etc/logwisp
 export LOGWISP_CONFIG_FILE=production.toml
-  # Loads: /etc/logwisp/production.toml
 ```
 
 ### `LOGWISP_DISABLE_STATUS_REPORTER`
-Disable the periodic status reporter.
+Disable periodic status reporting.
-- **Values**: `1` (disable), `0` or unset (enable)
+```bash
-- **Default**: `0` (enabled)
+export LOGWISP_DISABLE_STATUS_REPORTER=1
-- **Example**: `LOGWISP_DISABLE_STATUS_REPORTER=1`
+```
-
-### `LOGWISP_BACKGROUND`
-Internal marker for background process detection.
-- **Note**: Set automatically by `--background` flag
-- **Values**: `1` (background), unset (foreground)
 
 ## Logging Variables
 
-### `LOGWISP_LOGGING_OUTPUT`
-LogWisp's operational log output mode.
-- **Values**: `file`, `stdout`, `stderr`, `both`, `none`
-- **Example**: `LOGWISP_LOGGING_OUTPUT=both`
-
-### `LOGWISP_LOGGING_LEVEL`
-Minimum log level for operational logs.
-- **Values**: `debug`, `info`, `warn`, `error`
-- **Example**: `LOGWISP_LOGGING_LEVEL=debug`
-
-### File Logging
 ```bash
+# Output mode
+LOGWISP_LOGGING_OUTPUT=both
+
+# Log level
+LOGWISP_LOGGING_LEVEL=debug
+
+# File logging
 LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
 LOGWISP_LOGGING_FILE_NAME=logwisp
 LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
-LOGWISP_LOGGING_FILE_MAX_TOTAL_SIZE_MB=1000
+LOGWISP_LOGGING_FILE_RETENTION_HOURS=168
-LOGWISP_LOGGING_FILE_RETENTION_HOURS=168  # 7 days
+
+# Console logging
+LOGWISP_LOGGING_CONSOLE_TARGET=stderr
+LOGWISP_LOGGING_CONSOLE_FORMAT=json
 ```
 
-### Console Logging
+## Pipeline Configuration
+
+### Basic Pipeline
 ```bash
-LOGWISP_LOGGING_CONSOLE_TARGET=stderr     # stdout, stderr, split
+# Pipeline name
-LOGWISP_LOGGING_CONSOLE_FORMAT=txt        # txt, json
+LOGWISP_PIPELINES_0_NAME=app
-```
 
-## Stream Configuration
+# Source configuration
+LOGWISP_PIPELINES_0_SOURCES_0_TYPE=directory
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATH=/var/log/app
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATTERN="*.log"
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_CHECK_INTERVAL_MS=100
 
-Streams are configured using array indices (0-based).
+# Sink configuration
-
+LOGWISP_PIPELINES_0_SINKS_0_TYPE=http
-### Basic Stream Settings
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_PORT=8080
-```bash
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_BUFFER_SIZE=1000
-# First stream (index 0)
-LOGWISP_STREAMS_0_NAME=app
-LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=100
-
-# Second stream (index 1)
-LOGWISP_STREAMS_1_NAME=system
-LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
-```
-
-### Monitor Targets
-```bash
-# Single file target
-LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/app.log
-LOGWISP_STREAMS_0_MONITOR_TARGETS_0_IS_FILE=true
-
-# Directory with pattern
-LOGWISP_STREAMS_0_MONITOR_TARGETS_1_PATH=/var/log/myapp
-LOGWISP_STREAMS_0_MONITOR_TARGETS_1_PATTERN="*.log"
-LOGWISP_STREAMS_0_MONITOR_TARGETS_1_IS_FILE=false
 ```
 
 ### Filters
 ```bash
 # Include filter
-LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
+LOGWISP_PIPELINES_0_FILTERS_0_TYPE=include
-LOGWISP_STREAMS_0_FILTERS_0_LOGIC=or
+LOGWISP_PIPELINES_0_FILTERS_0_LOGIC=or
-LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN","CRITICAL"]'
+LOGWISP_PIPELINES_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
 
 # Exclude filter
-LOGWISP_STREAMS_0_FILTERS_1_TYPE=exclude
+LOGWISP_PIPELINES_0_FILTERS_1_TYPE=exclude
-LOGWISP_STREAMS_0_FILTERS_1_PATTERNS='["DEBUG","TRACE"]'
+LOGWISP_PIPELINES_0_FILTERS_1_PATTERNS='["DEBUG"]'
 ```
 
-### HTTP Server
+### HTTP Sink Options
 ```bash
-LOGWISP_STREAMS_0_HTTPSERVER_ENABLED=true
+# Basic
-LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_STREAM_PATH=/stream
-LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=1000
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_STATUS_PATH=/status
-LOGWISP_STREAMS_0_HTTPSERVER_STREAM_PATH=/stream
-LOGWISP_STREAMS_0_HTTPSERVER_STATUS_PATH=/status
 
 # Heartbeat
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INTERVAL_SECONDS=30
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_INTERVAL_SECONDS=30
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=comment
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_FORMAT=comment
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INCLUDE_TIMESTAMP=true
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INCLUDE_STATS=false
 
 # Rate Limiting
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_BURST_SIZE=20
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_BURST_SIZE=20
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_LIMIT_BY=ip
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_LIMIT_BY=ip
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_MAX_CONNECTIONS_PER_IP=5
 ```
 
-### TCP Server
+## Example
-```bash
-LOGWISP_STREAMS_0_TCPSERVER_ENABLED=true
-LOGWISP_STREAMS_0_TCPSERVER_PORT=9090
-LOGWISP_STREAMS_0_TCPSERVER_BUFFER_SIZE=5000
-
-# Rate Limiting
-LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_ENABLED=true
-LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=5.0
-LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_BURST_SIZE=10
-```
-
-## Complete Example
-
-Here's a complete example configuring two streams via environment variables:
 
 ```bash
 #!/bin/bash
 
-# Logging configuration
+# Logging
 export LOGWISP_LOGGING_OUTPUT=both
 export LOGWISP_LOGGING_LEVEL=info
-export LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
-export LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
 
-# Stream 0: Application logs
+# Pipeline 0: Application logs
-export LOGWISP_STREAMS_0_NAME=app
+export LOGWISP_PIPELINES_0_NAME=app
-export LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
+export LOGWISP_PIPELINES_0_SOURCES_0_TYPE=directory
-export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/myapp
+export LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATH=/var/log/myapp
-export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATTERN="*.log"
+export LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATTERN="*.log"
-export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_IS_FILE=false
 
-# Stream 0: Filters
+# Filters
-export LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
+export LOGWISP_PIPELINES_0_FILTERS_0_TYPE=include
-export LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
+export LOGWISP_PIPELINES_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
 
-# Stream 0: HTTP server
+# HTTP sink
-export LOGWISP_STREAMS_0_HTTPSERVER_ENABLED=true
+export LOGWISP_PIPELINES_0_SINKS_0_TYPE=http
-export LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
+export LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_PORT=8080
-export LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
+export LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_ENABLED=true
-export LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=25.0
+export LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=25.0
 
-# Stream 1: System logs
+# Pipeline 1: System logs
-export LOGWISP_STREAMS_1_NAME=system
+export LOGWISP_PIPELINES_1_NAME=system
-export LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
+export LOGWISP_PIPELINES_1_SOURCES_0_TYPE=file
-export LOGWISP_STREAMS_1_MONITOR_TARGETS_0_PATH=/var/log/syslog
+export LOGWISP_PIPELINES_1_SOURCES_0_OPTIONS_PATH=/var/log/syslog
-export LOGWISP_STREAMS_1_MONITOR_TARGETS_0_IS_FILE=true
 
-# Stream 1: TCP server
+# TCP sink
-export LOGWISP_STREAMS_1_TCPSERVER_ENABLED=true
+export LOGWISP_PIPELINES_1_SINKS_0_TYPE=tcp
-export LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
+export LOGWISP_PIPELINES_1_SINKS_0_OPTIONS_PORT=9090
 
-# Start LogWisp
 logwisp
 ```
 
-## Docker/Kubernetes Usage
+## Precedence
 
-Environment variables are ideal for containerized deployments:
+1. Command-line flags (highest)
-
+2. Environment variables
-### Docker
+3. Configuration file
-```dockerfile
+4. Defaults (lowest)
-FROM logwisp:latest
-ENV LOGWISP_LOGGING_OUTPUT=stdout
-ENV LOGWISP_STREAMS_0_NAME=container
-ENV LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/app
-ENV LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
-```
-
-### Docker Compose
-```yaml
-version: '3'
-services:
-  logwisp:
-    image: logwisp:latest
-    environment:
-      - LOGWISP_LOGGING_OUTPUT=stdout
-      - LOGWISP_STREAMS_0_NAME=webapp
-      - LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/logs
-      - LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
-    volumes:
-      - ./logs:/logs:ro
-    ports:
-      - "8080:8080"
-```
-
-### Kubernetes ConfigMap
-```yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: logwisp-config
-data:
-  LOGWISP_LOGGING_LEVEL: "info"
-  LOGWISP_STREAMS_0_NAME: "k8s-app"
-  LOGWISP_STREAMS_0_HTTPSERVER_PORT: "8080"
-```
-
-## Precedence Rules
-
-When the same setting is configured multiple ways, this precedence applies:
-
-1. **Command-line flags** (highest priority)
-2. **Environment variables**
-3. **Configuration file**
-4. **Default values** (lowest priority)
-
-Example:
-```bash
-# Config file has: logging.level = "info"
-export LOGWISP_LOGGING_LEVEL=warn
-logwisp --log-level debug
-
-# Result: log level will be "debug" (CLI flag wins)
-```
-
-## Debugging
-
-To see which environment variables LogWisp recognizes:
-```bash
-# List all LOGWISP variables
-env | grep ^LOGWISP_
-
-# Test configuration parsing
-LOGWISP_LOGGING_LEVEL=debug logwisp --version
-```
-
-## Security Considerations
-
-- **Sensitive Values**: Avoid putting passwords or tokens in environment variables
-- **Process Visibility**: Environment variables may be visible to other processes
-- **Container Security**: Use secrets management for sensitive configuration
-- **Logging**: Be careful not to log environment variable values
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [CLI Options](cli.md) - Command-line interface
-- [Docker Deployment](integrations.md#docker) - Container-specific guidance

doc/filters.md

@@ -1,21 +1,17 @@
 # Filter Guide
 
-LogWisp's filtering system allows you to control which log entries are streamed to clients, reducing noise and focusing on what matters.
+LogWisp filters control which log entries pass through pipelines using regular expressions.
 
 ## How Filters Work
 
-Filters use regular expressions to match log entries. Each filter can either:
+- **Include**: Only matching logs pass (whitelist)
-- **Include**: Only matching logs pass through (whitelist)
 - **Exclude**: Matching logs are dropped (blacklist)
+- Multiple filters apply sequentially - all must pass
 
-Multiple filters are applied sequentially - a log entry must pass ALL filters to be streamed.
+## Configuration
-
-## Filter Configuration
-
-### Basic Structure
 
 ```toml
-[[streams.filters]]
+[[pipelines.filters]]
 type = "include"    # or "exclude"
 logic = "or"        # or "and"
 patterns = [
@@ -26,115 +22,79 @@ patterns = [
 
 ### Filter Types
 
-#### Include Filter (Whitelist)
+#### Include Filter
-Only logs matching the patterns are streamed:
-
 ```toml
-[[streams.filters]]
+[[pipelines.filters]]
 type = "include"
 logic = "or"
-patterns = [
+patterns = ["ERROR", "WARN", "CRITICAL"]
-    "ERROR",
+# Only ERROR, WARN, or CRITICAL logs pass
-    "WARN",
-    "CRITICAL"
-]
-# Result: Only ERROR, WARN, or CRITICAL logs are streamed
 ```
 
-#### Exclude Filter (Blacklist)
+#### Exclude Filter
-Logs matching the patterns are dropped:
-
 ```toml
-[[streams.filters]]
+[[pipelines.filters]]
 type = "exclude"
-patterns = [
+patterns = ["DEBUG", "TRACE", "/health"]
-    "DEBUG",
+# DEBUG, TRACE, and health checks are dropped
-    "TRACE",
-    "/health"
-]
-# Result: DEBUG, TRACE, and health check logs are filtered out
 ```
 
 ### Logic Operators
 
-#### OR Logic (Default)
+- **OR**: Match ANY pattern (default)
-Log matches if ANY pattern matches:
+- **AND**: Match ALL patterns
 
 ```toml
-[[streams.filters]]
+# OR Logic
-type = "include"
 logic = "or"
-patterns = ["ERROR", "FAIL", "EXCEPTION"]
+patterns = ["ERROR", "FAIL"]
-# Matches: "ERROR: disk full" OR "FAIL: connection timeout" OR "NullPointerException"
+# Matches: "ERROR: disk full" OR "FAIL: timeout"
-```
 
-#### AND Logic
+# AND Logic
-Log matches only if ALL patterns match:
-
-```toml
-[[streams.filters]]
-type = "include"
 logic = "and"
 patterns = ["database", "timeout", "ERROR"]
 # Matches: "ERROR: database connection timeout"
-# Doesn't match: "ERROR: file not found" (missing "database" and "timeout")
+# Not: "ERROR: file not found"
 ```
 
 ## Pattern Syntax
 
-LogWisp uses Go's regular expression syntax (RE2):
+Go regular expressions (RE2):
-
-### Basic Patterns
 
+```toml
+"ERROR"              # Substring match
+"(?i)error"          # Case-insensitive
+"\\berror\\b"        # Word boundaries
+"^ERROR"             # Start of line
+"ERROR$"             # End of line
+"error|fail|warn"    # Alternatives
+```
+
+## Common Patterns
+
+### Log Levels
 ```toml
 patterns = [
-    "ERROR",                    # Exact substring match
+    "\\[(ERROR|WARN|INFO)\\]",      # [ERROR] format
-    "(?i)error",               # Case-insensitive
+    "(?i)\\b(error|warning)\\b",    # Word boundaries
-    "\\berror\\b",             # Word boundaries
+    "level=(error|warn)",           # key=value format
-    "^ERROR",                  # Start of line
-    "ERROR$",                  # End of line
-    "ERR(OR)?",                # Optional group
-    "error|fail|exception"     # Alternatives
 ]
 ```
 
-### Common Pattern Examples
+### Application Errors
-
-#### Log Levels
 ```toml
-# Standard log levels
+# Java
-patterns = [
-    "\\[(ERROR|WARN|INFO|DEBUG)\\]",     # [ERROR] format
-    "(?i)\\b(error|warning|info|debug)\\b", # Word boundaries
-    "level=(error|warn|info|debug)",      # key=value format
-    "<(Error|Warning|Info|Debug)>"        # XML-style
-]
-
-# Severity patterns
-patterns = [
-    "(?i)(fatal|critical|severe)",
-    "(?i)(error|fail|exception)",
-    "(?i)(warn|warning|caution)",
-    "panic:",                             # Go panics
-    "Traceback",                          # Python errors
-]
-```
-
-#### Application Errors
-```toml
-# Java/JVM
 patterns = [
     "Exception",
-    "\\.java:[0-9]+",                    # Stack trace lines
+    "at .+\\.java:[0-9]+",
-    "at com\\.mycompany\\.",             # Company packages
+    "NullPointerException"
-    "NullPointerException|ClassNotFoundException"
 ]
 
 # Python
 patterns = [
-    "Traceback \\(most recent call last\\)",
+    "Traceback",
     "File \".+\\.py\", line [0-9]+",
-    "(ValueError|TypeError|KeyError)"
+    "ValueError|TypeError"
 ]
 
 # Go
@@ -143,297 +103,73 @@ patterns = [
     "goroutine [0-9]+",
     "runtime error:"
 ]
+```
 
-# Node.js
+### Performance Issues
+```toml
 patterns = [
-    "Error:",
+    "took [0-9]{4,}ms",           # >999ms operations
-    "at .+ \\(.+\\.js:[0-9]+:[0-9]+\\)",
+    "timeout|timed out",
-    "UnhandledPromiseRejection"
+    "slow query",
+    "high cpu|cpu usage: [8-9][0-9]%"
 ]
 ```
 
-#### Performance Issues
+### HTTP Patterns
 ```toml
 patterns = [
-    "took [0-9]{4,}ms",                  # Operations over 999ms
+    "status[=:][4-5][0-9]{2}",    # 4xx/5xx codes
-    "duration>[0-9]{3,}s",               # Long durations
+    "HTTP/[0-9.]+ [4-5][0-9]{2}",
-    "timeout|timed out",                 # Timeouts
+    "\"/api/v[0-9]+/",            # API paths
-    "slow query",                        # Database
-    "memory pressure",                   # Memory issues
-    "high cpu|cpu usage: [8-9][0-9]%"   # CPU issues
-]
-```
-
-#### Security Patterns
-```toml
-patterns = [
-    "(?i)(unauthorized|forbidden|denied)",
-    "(?i)(auth|authentication) fail",
-    "invalid (token|session|credentials)",
-    "SQL injection|XSS|CSRF",
-    "brute force|rate limit",
-    "suspicious activity"
-]
-```
-
-#### HTTP Patterns
-```toml
-# Error status codes
-patterns = [
-    "status[=:][4-5][0-9]{2}",          # status=404, status:500
-    "HTTP/[0-9.]+ [4-5][0-9]{2}",       # HTTP/1.1 404
-    "\"status\":\\s*[4-5][0-9]{2}"      # JSON "status": 500
-]
-
-# Specific endpoints
-patterns = [
-    "\"(GET|POST|PUT|DELETE) /api/",
-    "/api/v[0-9]+/users",
-    "path=\"/admin"
 ]
 ```
 
 ## Filter Chains
 
-Multiple filters create a processing chain. Each filter must pass for the log to be streamed.
+### Error Monitoring
-
-### Example: Error Monitoring
 ```toml
-# Step 1: Include only errors and warnings
+# Include errors
-[[streams.filters]]
+[[pipelines.filters]]
 type = "include"
-logic = "or"
+patterns = ["(?i)\\b(error|fail|critical)\\b"]
-patterns = [
-    "(?i)\\b(error|fail|exception)\\b",
-    "(?i)\\b(warn|warning)\\b",
-    "(?i)\\b(critical|fatal|severe)\\b"
-]
 
-# Step 2: Exclude known non-issues
+# Exclude known non-issues
-[[streams.filters]]
+[[pipelines.filters]]
 type = "exclude"
-patterns = [
+patterns = ["Error: Expected", "/health"]
-    "Error: Expected behavior",
-    "Warning: Deprecated API",
-    "INFO.*error in message"          # INFO logs talking about errors
-]
-
-# Step 3: Exclude noisy sources
-[[streams.filters]]
-type = "exclude"
-patterns = [
-    "/health",
-    "/metrics",
-    "ELB-HealthChecker",
-    "Googlebot"
-]
 ```
 
-### Example: API Monitoring
+### API Monitoring
 ```toml
-# Include only API calls
+# Include API calls
-[[streams.filters]]
+[[pipelines.filters]]
 type = "include"
-patterns = [
+patterns = ["/api/", "/v[0-9]+/"]
-    "/api/",
-    "/v[0-9]+/"
-]
 
-# Exclude successful requests
+# Exclude successful
-[[streams.filters]]
+[[pipelines.filters]]
 type = "exclude"
-patterns = [
+patterns = ["\" 2[0-9]{2} "]
-    "\" 200 ",                        # HTTP 200 OK
-    "\" 201 ",                        # HTTP 201 Created
-    "\" 204 ",                        # HTTP 204 No Content
-    "\" 304 "                         # HTTP 304 Not Modified
-]
-
-# Exclude OPTIONS requests (CORS)
-[[streams.filters]]
-type = "exclude"
-patterns = [
-    "OPTIONS "
-]
 ```
 
-### Example: Security Audit
+## Performance Tips
-```toml
-# Include security-relevant events
-[[streams.filters]]
-type = "include"
-logic = "or"
-patterns = [
-    "(?i)auth",
-    "(?i)login|logout",
-    "(?i)sudo|root",
-    "(?i)ssh|sftp|ftp",
-    "(?i)firewall|iptables",
-    "COMMAND=",                       # sudo commands
-    "USER=",                          # user actions
-    "SELINUX"
-]
 
-# Must also contain failure/success indicators
+1. **Use anchors**: `^ERROR` faster than `ERROR`
-[[streams.filters]]
+2. **Avoid nested quantifiers**: `((a+)+)+`
-type = "include"
+3. **Non-capturing groups**: `(?:error|warn)`
-logic = "or"
+4. **Order by frequency**: Most common first
-patterns = [
+5. **Simple patterns**: Faster than complex regex
-    "(?i)(fail|denied|error)",
-    "(?i)(success|accepted|granted)",
-    "(?i)(invalid|unauthorized)"
-]
-```
-
-## Performance Considerations
-
-### Pattern Complexity
-
-Simple patterns are fast (~1μs per check):
-```toml
-patterns = ["ERROR", "WARN", "FATAL"]
-```
-
-Complex patterns are slower (~10-100μs per check):
-```toml
-patterns = [
-    "^\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]\\s+\\[(ERROR|WARN)\\]\\s+\\[([^\\]]+)\\]\\s+(.+)$"
-]
-```
-
-### Optimization Tips
-
-1. **Use anchors when possible**:
-   ```toml
-   "^ERROR" # Faster than "ERROR"
-   ```
-
-2. **Avoid nested quantifiers**:
-   ```toml
-   # BAD: Can cause exponential backtracking
-   "((a+)+)+"
-   
-   # GOOD: Linear time
-   "a+"
-   ```
-
-3. **Use non-capturing groups**:
-   ```toml
-   "(?:error|warn)" # Instead of "(error|warn)"
-   ```
-
-4. **Order patterns by frequency**:
-   ```toml
-   # Most common first
-   patterns = ["ERROR", "WARN", "INFO", "DEBUG"]
-   ```
-
-5. **Prefer character classes**:
-   ```toml
-   "[0-9]" # Instead of "\\d"
-   "[a-zA-Z]" # Instead of "\\w"
-   ```
 
 ## Testing Filters
 
-### Test Configuration
-Create a test configuration with sample logs:
-
-```toml
-[[streams]]
-name = "test"
-[streams.monitor]
-targets = [{ path = "./test-logs", pattern = "*.log" }]
-
-[[streams.filters]]
-type = "include"
-patterns = ["YOUR_PATTERN_HERE"]
-
-[streams.httpserver]
-enabled = true
-port = 8888
-```
-
-### Generate Test Logs
 ```bash
-# Create test log entries
+# Test configuration
-echo "[ERROR] Database connection failed" >> test-logs/app.log
+echo "[ERROR] Test" >> test.log
-echo "[INFO] User logged in" >> test-logs/app.log
+echo "[INFO] Test" >> test.log
-echo "[WARN] High memory usage: 85%" >> test-logs/app.log
 
-# Run LogWisp with debug logging
+# Run with debug
-logwisp --config test.toml --log-level debug
+logwisp --log-level debug
 
-# Check what passes through
+# Check output
-curl -N http://localhost:8888/stream
+curl -N http://localhost:8080/stream
 ```
-
-### Debug Filter Behavior
-Enable debug logging to see filter decisions:
-
-```bash
-logwisp --log-level debug --log-output stderr
-```
-
-Look for messages like:
-```
-Entry filtered out component=filter_chain filter_index=0 filter_type=include
-Entry passed all filters component=filter_chain
-```
-
-## Common Pitfalls
-
-### Case Sensitivity
-By default, patterns are case-sensitive:
-```toml
-# Won't match "error" or "Error"
-patterns = ["ERROR"]
-
-# Use case-insensitive flag
-patterns = ["(?i)error"]
-```
-
-### Partial Matches
-Patterns match substrings by default:
-```toml
-# Matches "ERROR", "ERRORS", "TERROR"
-patterns = ["ERROR"]
-
-# Use word boundaries for exact words
-patterns = ["\\bERROR\\b"]
-```
-
-### Special Characters
-Remember to escape regex special characters:
-```toml
-# Won't work as expected
-patterns = ["[ERROR]"]
-
-# Correct: escape brackets
-patterns = ["\\[ERROR\\]"]
-```
-
-### Performance Impact
-Too many complex patterns can impact performance:
-```toml
-# Consider splitting into multiple streams instead
-[[streams.filters]]
-patterns = [
-    # 50+ complex patterns...
-]
-```
-
-## Best Practices
-
-1. **Start Simple**: Begin with basic patterns and refine as needed
-2. **Test Thoroughly**: Use test logs to verify filter behavior
-3. **Monitor Performance**: Check filter statistics in `/status`
-4. **Document Patterns**: Comment complex patterns for maintenance
-5. **Use Multiple Streams**: Instead of complex filters, consider separate streams
-6. **Regular Review**: Periodically review and optimize filter rules
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [Performance Tuning](performance.md) - Optimization guidelines
-- [Examples](examples/) - Real-world filter configurations

doc/installation.md

@@ -1,137 +1,54 @@
 # Installation Guide
 
-This guide covers installing LogWisp on various platforms and deployment scenarios.
+Installation process on tested platforms.
 
 ## Requirements
 
-### System Requirements
+- **OS**: Linux, FreeBSD
+- **Architecture**: amd64
+- **Go**: 1.23+ (for building)
 
-- **OS**: Linux, macOS, FreeBSD, Windows (with WSL)
+## Installation
-- **Architecture**: amd64, arm64
-- **Memory**: 64MB minimum, 256MB recommended
-- **Disk**: 10MB for binary, plus log storage
-- **Go**: 1.23+ (for building from source)
-
-### Runtime Dependencies
-
-LogWisp is a single static binary with no runtime dependencies. It only requires:
-- Read access to monitored log files
-- Network access for serving streams
-- Write access for operational logs (optional)
-
-## Installation Methods
 
 ### Pre-built Binaries
 
-Download the latest release:
-
 ```bash
-# Linux (amd64)
+# Linux amd64
 wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
 chmod +x logwisp-linux-amd64
 sudo mv logwisp-linux-amd64 /usr/local/bin/logwisp
 
-# macOS (Intel)
+# macOS Intel
 wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-amd64
 chmod +x logwisp-darwin-amd64
 sudo mv logwisp-darwin-amd64 /usr/local/bin/logwisp
 
-# macOS (Apple Silicon)
+# Verify
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-arm64
-chmod +x logwisp-darwin-arm64
-sudo mv logwisp-darwin-arm64 /usr/local/bin/logwisp
-```
-
-Verify installation:
-```bash
 logwisp --version
 ```
 
 ### From Source
 
-Build from source code:
-
 ```bash
-# Clone repository
 git clone https://github.com/yourusername/logwisp.git
 cd logwisp
-
-# Build
 make build
-
-# Install
 sudo make install
-
-# Or install to custom location
-make install PREFIX=/opt/logwisp
 ```
 
-### Using Go Install
+### Go Install
-
-Install directly with Go:
 
 ```bash
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+go install github.com/lixenwraith/logwisp/src/cmd/logwisp@latest
 ```
+Note: Binary created with this method will not contain version information.
 
-Note: This installs to `$GOPATH/bin` (usually `~/go/bin`)
+## Platform-Specific
 
-### Docker
+### Linux (systemd)
-
-Official Docker image:
 
 ```bash
-# Pull image
+# Create service
-docker pull yourusername/logwisp:latest
-
-# Run with volume mount
-docker run -d \
-  --name logwisp \
-  -p 8080:8080 \
-  -v /var/log:/logs:ro \
-  -v $PWD/config.toml:/config/logwisp.toml:ro \
-  yourusername/logwisp:latest \
-  --config /config/logwisp.toml
-```
-
-Build your own image:
-
-```dockerfile
-FROM golang:1.23-alpine AS builder
-WORKDIR /build
-COPY . .
-RUN go build -o logwisp ./src/cmd/logwisp
-
-FROM alpine:latest
-RUN apk --no-cache add ca-certificates
-COPY --from=builder /build/logwisp /usr/local/bin/
-ENTRYPOINT ["logwisp"]
-```
-
-## Platform-Specific Instructions
-
-### Linux
-
-#### Debian/Ubuntu
-
-Create package (planned):
-```bash
-# Future feature
-sudo apt install logwisp
-```
-
-Manual installation:
-```bash
-# Download binary
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64 -O logwisp
-chmod +x logwisp
-sudo mv logwisp /usr/local/bin/
-
-# Create config directory
-sudo mkdir -p /etc/logwisp
-sudo cp config/logwisp.toml.example /etc/logwisp/logwisp.toml
-
-# Create systemd service
 sudo tee /etc/systemd/system/logwisp.service << EOF
 [Unit]
 Description=LogWisp Log Monitoring Service
@@ -140,21 +57,10 @@ After=network.target
 [Service]
 Type=simple
 User=logwisp
-Group=logwisp
 ExecStart=/usr/local/bin/logwisp --config /etc/logwisp/logwisp.toml
 Restart=always
-RestartSec=5
 StandardOutput=journal
 StandardError=journal
-SyslogIdentifier=logwisp
-
-# Security hardening
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadOnlyPaths=/var/log
-ReadWritePaths=/var/log/logwisp
 
 [Install]
 WantedBy=multi-user.target
@@ -163,9 +69,12 @@ EOF
 # Create user
 sudo useradd -r -s /bin/false logwisp
 
-# Create log directory
+# Create service user
-sudo mkdir -p /var/log/logwisp
+sudo useradd -r -s /bin/false logwisp
-sudo chown logwisp:logwisp /var/log/logwisp
+
+# Create configuration directory
+sudo mkdir -p /etc/logwisp
+sudo chown logwisp:logwisp /etc/logwisp
 
 # Enable and start
 sudo systemctl daemon-reload
@@ -173,93 +82,11 @@ sudo systemctl enable logwisp
 sudo systemctl start logwisp
 ```
 
-#### Red Hat/CentOS/Fedora
+### FreeBSD (rc.d)
 
 ```bash
-# Similar to Debian, but use:
+# Create service script
-sudo yum install wget  # or dnf on newer versions
+sudo tee /usr/local/etc/rc.d/logwisp << 'EOF'
-
-# SELinux context (if enabled)
-sudo semanage fcontext -a -t bin_t /usr/local/bin/logwisp
-sudo restorecon -v /usr/local/bin/logwisp
-```
-
-#### Arch Linux
-
-AUR package (community maintained):
-```bash
-# Future feature
-yay -S logwisp
-```
-
-### macOS
-
-#### Homebrew
-
-Formula (planned):
-```bash
-# Future feature
-brew install logwisp
-```
-
-#### Manual Installation
-
-```bash
-# Download and install
-curl -L https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-$(uname -m) -o logwisp
-chmod +x logwisp
-sudo mv logwisp /usr/local/bin/
-
-# Create LaunchDaemon
-sudo tee /Library/LaunchDaemons/com.logwisp.plist << EOF
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-    <key>Label</key>
-    <string>com.logwisp</string>
-    <key>ProgramArguments</key>
-    <array>
-        <string>/usr/local/bin/logwisp</string>
-        <string>--config</string>
-        <string>/usr/local/etc/logwisp/logwisp.toml</string>
-    </array>
-    <key>RunAtLoad</key>
-    <true/>
-    <key>KeepAlive</key>
-    <true/>
-    <key>StandardOutPath</key>
-    <string>/usr/local/var/log/logwisp.log</string>
-    <key>StandardErrorPath</key>
-    <string>/usr/local/var/log/logwisp.error.log</string>
-</dict>
-</plist>
-EOF
-
-# Load service
-sudo launchctl load /Library/LaunchDaemons/com.logwisp.plist
-```
-
-### FreeBSD
-
-#### Ports
-
-```bash
-# Future feature
-cd /usr/ports/sysutils/logwisp
-make install clean
-```
-
-#### Manual Installation
-
-```bash
-# Download
-fetch https://github.com/yourusername/logwisp/releases/latest/download/logwisp-freebsd-amd64
-chmod +x logwisp-freebsd-amd64
-mv logwisp-freebsd-amd64 /usr/local/bin/logwisp
-
-# RC script
-cat > /usr/local/etc/rc.d/logwisp << 'EOF'
 #!/bin/sh
 
 # PROVIDE: logwisp
@@ -273,319 +100,125 @@ rcvar="${name}_enable"
 command="/usr/local/bin/logwisp"
 command_args="--config /usr/local/etc/logwisp/logwisp.toml"
 pidfile="/var/run/${name}.pid"
+start_cmd="logwisp_start"
+stop_cmd="logwisp_stop"
+
+logwisp_start()
+{
+    echo "Starting logwisp service..."
+    /usr/sbin/daemon -c -f -p ${pidfile} ${command} ${command_args}
+}
+
+logwisp_stop()
+{
+    if [ -f ${pidfile} ]; then
+        echo "Stopping logwisp service..."
+        kill $(cat ${pidfile})
+        rm -f ${pidfile}
+    fi
+}
 
 load_rc_config $name
 : ${logwisp_enable:="NO"}
+: ${logwisp_config:="/usr/local/etc/logwisp/logwisp.toml"}
 
 run_rc_command "$1"
 EOF
 
-chmod +x /usr/local/etc/rc.d/logwisp
+# Make executable
+sudo chmod +x /usr/local/etc/rc.d/logwisp
 
-# Enable
+# Create service user
-sysrc logwisp_enable="YES"
+sudo pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin
-service logwisp start
-```
 
-### Windows
+# Create configuration directory
+sudo mkdir -p /usr/local/etc/logwisp
+sudo chown logwisp:logwisp /usr/local/etc/logwisp
 
-#### Windows Subsystem for Linux (WSL)
+# Enable service
+sudo sysrc logwisp_enable="YES"
 
-```bash
+# Start service
-# Inside WSL, follow Linux instructions
+sudo service logwisp start
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
-chmod +x logwisp-linux-amd64
-./logwisp-linux-amd64
-```
-
-#### Native Windows (planned)
-
-Future support for native Windows service.
-
-## Container Deployment
-
-### Docker Compose
-
-```yaml
-version: '3.8'
-
-services:
-  logwisp:
-    image: yourusername/logwisp:latest
-    container_name: logwisp
-    restart: unless-stopped
-    ports:
-      - "8080:8080"
-      - "9090:9090"  # If using TCP
-    volumes:
-      - /var/log:/logs:ro
-      - ./logwisp.toml:/config/logwisp.toml:ro
-    command: ["--config", "/config/logwisp.toml"]
-    environment:
-      - LOGWISP_LOGGING_LEVEL=info
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080/status"]
-      interval: 30s
-      timeout: 3s
-      retries: 3
-```
-
-### Kubernetes
-
-Deployment manifest:
-
-```yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: logwisp
-  labels:
-    app: logwisp
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: logwisp
-  template:
-    metadata:
-      labels:
-        app: logwisp
-    spec:
-      containers:
-      - name: logwisp
-        image: yourusername/logwisp:latest
-        args:
-          - --config
-          - /config/logwisp.toml
-        ports:
-        - containerPort: 8080
-          name: http
-        - containerPort: 9090
-          name: tcp
-        volumeMounts:
-        - name: logs
-          mountPath: /logs
-          readOnly: true
-        - name: config
-          mountPath: /config
-        livenessProbe:
-          httpGet:
-            path: /status
-            port: 8080
-          initialDelaySeconds: 10
-          periodSeconds: 30
-        readinessProbe:
-          httpGet:
-            path: /status
-            port: 8080
-          initialDelaySeconds: 5
-          periodSeconds: 10
-        resources:
-          requests:
-            memory: "128Mi"
-            cpu: "100m"
-          limits:
-            memory: "512Mi"
-            cpu: "500m"
-      volumes:
-      - name: logs
-        hostPath:
-          path: /var/log
-      - name: config
-        configMap:
-          name: logwisp-config
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: logwisp
-spec:
-  selector:
-    app: logwisp
-  ports:
-  - name: http
-    port: 8080
-    targetPort: 8080
-  - name: tcp
-    port: 9090
-    targetPort: 9090
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: logwisp-config
-data:
-  logwisp.toml: |
-    [[streams]]
-    name = "k8s"
-    [streams.monitor]
-    targets = [{ path = "/logs", pattern = "*.log" }]
-    [streams.httpserver]
-    enabled = true
-    port = 8080
 ```
 
 ## Post-Installation
 
 ### Verify Installation
-
-1. Check version:
 ```bash
+# Check version
 logwisp --version
-   ```
 
-2. Test configuration:
+# Test configuration
-   ```bash
 logwisp --config /etc/logwisp/logwisp.toml --log-level debug
-   ```
 
-3. Check service status:
+# Check service
-   ```bash
-   # systemd
 sudo systemctl status logwisp
-   
-   # macOS
-   sudo launchctl list | grep logwisp
-   
-   # FreeBSD
-   service logwisp status
 ```
 
-4. Test streaming:
+### Linux Service Status
 ```bash
-   curl -N http://localhost:8080/stream
+sudo systemctl status logwisp
 ```
 
-### Security Hardening
+### FreeBSD Service Status
-
-1. **Create dedicated user**:
 ```bash
-   sudo useradd -r -s /bin/false -d /var/lib/logwisp logwisp
+sudo service logwisp status
 ```
 
-2. **Set file permissions**:
-   ```bash
-   sudo chown root:root /usr/local/bin/logwisp
-   sudo chmod 755 /usr/local/bin/logwisp
-   sudo chown -R logwisp:logwisp /etc/logwisp
-   sudo chmod 640 /etc/logwisp/logwisp.toml
-   ```
-
-3. **Configure firewall**:
-   ```bash
-   # UFW
-   sudo ufw allow 8080/tcp comment "LogWisp HTTP"
-   
-   # firewalld
-   sudo firewall-cmd --permanent --add-port=8080/tcp
-   sudo firewall-cmd --reload
-   ```
-
-4. **Enable SELinux/AppArmor** (if applicable)
-
 ### Initial Configuration
 
-1. Copy example configuration:
+Create a basic configuration file:
-   ```bash
-   sudo cp /usr/local/share/logwisp/examples/logwisp.toml.example /etc/logwisp/logwisp.toml
-   ```
 
-2. Edit configuration:
-   ```bash
-   sudo nano /etc/logwisp/logwisp.toml
-   ```
-
-3. Set up log monitoring:
 ```toml
-   [[streams]]
+# /etc/logwisp/logwisp.toml (Linux)
+# /usr/local/etc/logwisp/logwisp.toml (FreeBSD)
+
+[[pipelines]]
 name = "myapp"
-   [streams.monitor]
+
-   targets = [
+[[pipelines.sources]]
-       { path = "/var/log/myapp", pattern = "*.log" }
+type = "directory"
-   ]
+options = { 
+    path = "/path/to/application/logs",
+    pattern = "*.log"
+}
+
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
 ```
 
-4. Restart service:
+Restart service after configuration changes:
+
+**Linux:**
 ```bash
 sudo systemctl restart logwisp
 ```
 
+**FreeBSD:**
+```bash
+sudo service logwisp restart
+```
+
 ## Uninstallation
 
 ### Linux
 ```bash
-# Stop service
 sudo systemctl stop logwisp
 sudo systemctl disable logwisp
-
-# Remove files
 sudo rm /usr/local/bin/logwisp
 sudo rm /etc/systemd/system/logwisp.service
 sudo rm -rf /etc/logwisp
-sudo rm -rf /var/log/logwisp
-
-# Remove user
 sudo userdel logwisp
 ```
 
-### macOS
+### FreeBSD
 ```bash
-# Stop service
+sudo service logwisp stop
-sudo launchctl unload /Library/LaunchDaemons/com.logwisp.plist
+sudo sysrc logwisp_enable="NO"
-
-# Remove files
 sudo rm /usr/local/bin/logwisp
-sudo rm /Library/LaunchDaemons/com.logwisp.plist
+sudo rm /usr/local/etc/rc.d/logwisp
 sudo rm -rf /usr/local/etc/logwisp
+sudo pw userdel logwisp
 ```
-
-### Docker
-```bash
-docker stop logwisp
-docker rm logwisp
-docker rmi yourusername/logwisp:latest
-```
-
-## Troubleshooting Installation
-
-### Permission Denied
-
-If you get permission errors:
-```bash
-# Check file ownership
-ls -la /usr/local/bin/logwisp
-
-# Fix permissions
-sudo chmod +x /usr/local/bin/logwisp
-
-# Check log directory
-sudo mkdir -p /var/log/logwisp
-sudo chown logwisp:logwisp /var/log/logwisp
-```
-
-### Service Won't Start
-
-Check logs:
-```bash
-# systemd
-sudo journalctl -u logwisp -f
-
-# Manual run
-sudo -u logwisp /usr/local/bin/logwisp --config /etc/logwisp/logwisp.toml
-```
-
-### Port Already in Use
-
-Find conflicting process:
-```bash
-sudo lsof -i :8080
-# or
-sudo netstat -tlnp | grep 8080
-```
-
-## See Also
-
-- [Quick Start](quickstart.md) - Get running quickly
-- [Configuration Guide](configuration.md) - Configure LogWisp
-- [Troubleshooting](troubleshooting.md) - Common issues
-- [Security Best Practices](security.md) - Hardening guide

doc/monitoring.md

@@ -1,511 +0,0 @@
-# Monitoring & Status Guide
-
-LogWisp provides comprehensive monitoring capabilities through status endpoints, operational logs, and metrics.
-
-## Status Endpoints
-
-### Stream Status
-
-Each stream exposes its own status endpoint:
-
-```bash
-# Standalone mode
-curl http://localhost:8080/status
-
-# Router mode
-curl http://localhost:8080/streamname/status
-```
-
-Example response:
-```json
-{
-  "service": "LogWisp",
-  "version": "1.0.0",
-  "server": {
-    "type": "http",
-    "port": 8080,
-    "active_clients": 5,
-    "buffer_size": 1000,
-    "uptime_seconds": 3600,
-    "mode": {
-      "standalone": true,
-      "router": false
-    }
-  },
-  "monitor": {
-    "active_watchers": 3,
-    "total_entries": 152341,
-    "dropped_entries": 12,
-    "start_time": "2024-01-20T10:00:00Z",
-    "last_entry_time": "2024-01-20T11:00:00Z"
-  },
-  "filters": {
-    "filter_count": 2,
-    "total_processed": 152341,
-    "total_passed": 48234,
-    "filters": [
-      {
-        "type": "include",
-        "logic": "or",
-        "pattern_count": 3,
-        "total_processed": 152341,
-        "total_matched": 48234,
-        "total_dropped": 0
-      }
-    ]
-  },
-  "features": {
-    "heartbeat": {
-      "enabled": true,
-      "interval": 30,
-      "format": "comment"
-    },
-    "rate_limit": {
-      "enabled": true,
-      "total_requests": 8234,
-      "blocked_requests": 89,
-      "active_ips": 12,
-      "total_connections": 5
-    }
-  }
-}
-```
-
-### Global Status (Router Mode)
-
-In router mode, a global status endpoint provides aggregated information:
-
-```bash
-curl http://localhost:8080/status
-```
-
-## Key Metrics
-
-### Monitor Metrics
-
-Track file watching performance:
-
-| Metric | Description | Healthy Range |
-|--------|-------------|---------------|
-| `active_watchers` | Number of files being watched | 1-1000 |
-| `total_entries` | Total log entries processed | Increasing |
-| `dropped_entries` | Entries dropped due to buffer full | < 1% of total |
-| `entries_per_second` | Current processing rate | Varies |
-
-### Connection Metrics
-
-Monitor client connections:
-
-| Metric | Description | Warning Signs |
-|--------|-------------|---------------|
-| `active_clients` | Current SSE connections | Near limit |
-| `tcp_connections` | Current TCP connections | Near limit |
-| `total_connections` | All active connections | > 80% of max |
-
-### Filter Metrics
-
-Understand filtering effectiveness:
-
-| Metric | Description | Optimization |
-|--------|-------------|--------------|
-| `total_processed` | Entries checked | - |
-| `total_passed` | Entries that passed | Very low = too restrictive |
-| `total_dropped` | Entries filtered out | Very high = review patterns |
-
-### Rate Limit Metrics
-
-Track rate limiting impact:
-
-| Metric | Description | Action Needed |
-|--------|-------------|---------------|
-| `blocked_requests` | Rejected requests | High = increase limits |
-| `active_ips` | Unique clients | High = scale out |
-| `blocked_percentage` | Rejection rate | > 10% = review |
-
-## Operational Logging
-
-### Log Levels
-
-Configure LogWisp's operational logging:
-
-```toml
-[logging]
-output = "both"     # file and stderr
-level = "info"      # info for production
-```
-
-Log levels and their use:
-- **DEBUG**: Detailed internal operations
-- **INFO**: Normal operations, connections
-- **WARN**: Recoverable issues
-- **ERROR**: Errors requiring attention
-
-### Important Log Messages
-
-#### Startup Messages
-```
-LogWisp starting version=1.0.0 config_file=/etc/logwisp.toml
-Stream registered with router stream=app
-TCP endpoint configured transport=system port=9090
-HTTP endpoints configured transport=app stream_url=http://localhost:8080/stream
-```
-
-#### Connection Events
-```
-HTTP client connected remote_addr=192.168.1.100:54231 active_clients=6
-HTTP client disconnected remote_addr=192.168.1.100:54231 active_clients=5
-TCP connection opened remote_addr=192.168.1.100:54232 active_connections=3
-```
-
-#### Error Conditions
-```
-Failed to open file for checking path=/var/log/app.log error=permission denied
-Scanner error while reading file path=/var/log/huge.log error=token too long
-Request rate limited ip=192.168.1.100
-Connection limit exceeded ip=192.168.1.100 connections=5 limit=5
-```
-
-#### Performance Warnings
-```
-Dropped log entry - subscriber buffer full
-Dropped entry for slow client remote_addr=192.168.1.100
-Check interval too small: 5ms (min: 10ms)
-```
-
-## Health Checks
-
-### Basic Health Check
-
-Simple up/down check:
-
-```bash
-#!/bin/bash
-# health_check.sh
-
-STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/status)
-
-if [ "$STATUS" -eq 200 ]; then
-    echo "LogWisp is healthy"
-    exit 0
-else
-    echo "LogWisp is unhealthy (status: $STATUS)"
-    exit 1
-fi
-```
-
-### Advanced Health Check
-
-Check specific conditions:
-
-```bash
-#!/bin/bash
-# advanced_health_check.sh
-
-RESPONSE=$(curl -s http://localhost:8080/status)
-
-# Check if processing logs
-ENTRIES=$(echo "$RESPONSE" | jq -r '.monitor.total_entries')
-if [ "$ENTRIES" -eq 0 ]; then
-    echo "WARNING: No log entries processed"
-    exit 1
-fi
-
-# Check dropped entries
-DROPPED=$(echo "$RESPONSE" | jq -r '.monitor.dropped_entries')
-TOTAL=$(echo "$RESPONSE" | jq -r '.monitor.total_entries')
-DROP_PERCENT=$(( DROPPED * 100 / TOTAL ))
-
-if [ "$DROP_PERCENT" -gt 5 ]; then
-    echo "WARNING: High drop rate: ${DROP_PERCENT}%"
-    exit 1
-fi
-
-# Check connections
-CONNECTIONS=$(echo "$RESPONSE" | jq -r '.server.active_clients')
-echo "OK: Processing logs, $CONNECTIONS active clients"
-exit 0
-```
-
-### Container Health Check
-
-Docker/Kubernetes configuration:
-
-```dockerfile
-# Dockerfile
-HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
-    CMD curl -f http://localhost:8080/status || exit 1
-```
-
-```yaml
-# Kubernetes
-livenessProbe:
-  httpGet:
-    path: /status
-    port: 8080
-  initialDelaySeconds: 10
-  periodSeconds: 30
-
-readinessProbe:
-  httpGet:
-    path: /status
-    port: 8080
-  initialDelaySeconds: 5
-  periodSeconds: 10
-```
-
-## Monitoring Integration
-
-### Prometheus Metrics
-
-Export metrics in Prometheus format:
-
-```bash
-#!/bin/bash
-# prometheus_exporter.sh
-
-while true; do
-    STATUS=$(curl -s http://localhost:8080/status)
-    
-    # Extract metrics
-    CLIENTS=$(echo "$STATUS" | jq -r '.server.active_clients')
-    ENTRIES=$(echo "$STATUS" | jq -r '.monitor.total_entries')
-    DROPPED=$(echo "$STATUS" | jq -r '.monitor.dropped_entries')
-    
-    # Output Prometheus format
-    cat << EOF
-# HELP logwisp_active_clients Number of active streaming clients
-# TYPE logwisp_active_clients gauge
-logwisp_active_clients $CLIENTS
-
-# HELP logwisp_total_entries Total log entries processed
-# TYPE logwisp_total_entries counter
-logwisp_total_entries $ENTRIES
-
-# HELP logwisp_dropped_entries Total log entries dropped
-# TYPE logwisp_dropped_entries counter
-logwisp_dropped_entries $DROPPED
-EOF
-
-    sleep 60
-done
-```
-
-### Grafana Dashboard
-
-Key panels for Grafana:
-
-1. **Active Connections**
-    - Query: `logwisp_active_clients`
-    - Visualization: Graph
-    - Alert: > 80% of max
-
-2. **Log Processing Rate**
-    - Query: `rate(logwisp_total_entries[5m])`
-    - Visualization: Graph
-    - Alert: < 1 entry/min
-
-3. **Drop Rate**
-    - Query: `rate(logwisp_dropped_entries[5m]) / rate(logwisp_total_entries[5m])`
-    - Visualization: Gauge
-    - Alert: > 5%
-
-4. **Rate Limit Rejections**
-    - Query: `rate(logwisp_blocked_requests[5m])`
-    - Visualization: Graph
-    - Alert: > 10/min
-
-### Datadog Integration
-
-Send custom metrics:
-
-```bash
-#!/bin/bash
-# datadog_metrics.sh
-
-while true; do
-    STATUS=$(curl -s http://localhost:8080/status)
-    
-    # Send metrics to Datadog
-    echo "$STATUS" | jq -r '
-        "logwisp.connections:\(.server.active_clients)|g",
-        "logwisp.entries:\(.monitor.total_entries)|c",
-        "logwisp.dropped:\(.monitor.dropped_entries)|c"
-    ' | while read metric; do
-        echo "$metric" | nc -u -w1 localhost 8125
-    done
-    
-    sleep 60
-done
-```
-
-## Performance Monitoring
-
-### CPU Usage
-
-Monitor CPU usage by component:
-
-```bash
-# Check process CPU
-top -p $(pgrep logwisp) -b -n 1
-
-# Profile CPU usage
-go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
-```
-
-Common CPU consumers:
-- File watching (reduce check_interval_ms)
-- Regex filtering (simplify patterns)
-- JSON encoding (reduce clients)
-
-### Memory Usage
-
-Track memory consumption:
-
-```bash
-# Check process memory
-ps aux | grep logwisp
-
-# Detailed memory stats
-cat /proc/$(pgrep logwisp)/status | grep -E "Vm(RSS|Size)"
-```
-
-Memory optimization:
-- Reduce buffer sizes
-- Limit connections
-- Simplify filters
-
-### Network Bandwidth
-
-Monitor streaming bandwidth:
-
-```bash
-# Network statistics
-netstat -i
-iftop -i eth0 -f "port 8080"
-
-# Connection count
-ss -tan | grep :8080 | wc -l
-```
-
-## Alerting
-
-### Basic Alerts
-
-Essential alerts to configure:
-
-| Alert | Condition | Severity |
-|-------|-----------|----------|
-| Service Down | Status endpoint fails | Critical |
-| High Drop Rate | > 10% entries dropped | Warning |
-| No Log Activity | 0 entries/min for 5 min | Warning |
-| Connection Limit | > 90% of max connections | Warning |
-| Rate Limit High | > 20% requests blocked | Warning |
-
-### Alert Script
-
-Example monitoring script:
-
-```bash
-#!/bin/bash
-# monitor_alerts.sh
-
-check_alert() {
-    local name=$1
-    local condition=$2
-    local message=$3
-    
-    if eval "$condition"; then
-        echo "ALERT: $name - $message"
-        # Send to alerting system
-        # curl -X POST https://alerts.example.com/...
-    fi
-}
-
-while true; do
-    STATUS=$(curl -s http://localhost:8080/status)
-    
-    if [ -z "$STATUS" ]; then
-        check_alert "SERVICE_DOWN" "true" "LogWisp not responding"
-        sleep 60
-        continue
-    fi
-    
-    # Extract metrics
-    DROPPED=$(echo "$STATUS" | jq -r '.monitor.dropped_entries')
-    TOTAL=$(echo "$STATUS" | jq -r '.monitor.total_entries')
-    CLIENTS=$(echo "$STATUS" | jq -r '.server.active_clients')
-    
-    # Check conditions
-    check_alert "HIGH_DROP_RATE" \
-        "[ $((DROPPED * 100 / TOTAL)) -gt 10 ]" \
-        "Drop rate above 10%"
-        
-    check_alert "HIGH_CONNECTIONS" \
-        "[ $CLIENTS -gt 90 ]" \
-        "Near connection limit: $CLIENTS/100"
-    
-    sleep 60
-done
-```
-
-## Troubleshooting with Monitoring
-
-### No Logs Appearing
-
-Check monitor stats:
-```bash
-curl -s http://localhost:8080/status | jq '.monitor'
-```
-
-Look for:
-- `active_watchers` = 0 (no files found)
-- `total_entries` not increasing (files not updating)
-
-### High CPU Usage
-
-Enable debug logging:
-```bash
-logwisp --log-level debug --log-output stderr
-```
-
-Watch for:
-- Frequent "checkFile" messages (reduce check_interval)
-- Many filter operations (optimize patterns)
-
-### Memory Growth
-
-Monitor over time:
-```bash
-while true; do
-    ps aux | grep logwisp | grep -v grep
-    curl -s http://localhost:8080/status | jq '.server.active_clients'
-    sleep 10
-done
-```
-
-### Connection Issues
-
-Check connection stats:
-```bash
-# Current connections
-curl -s http://localhost:8080/status | jq '.server'
-
-# Rate limit stats
-curl -s http://localhost:8080/status | jq '.features.rate_limit'
-```
-
-## Best Practices
-
-1. **Regular Monitoring**: Check status endpoints every 30-60 seconds
-2. **Set Alerts**: Configure alerts for critical conditions
-3. **Log Rotation**: Rotate LogWisp's own logs to prevent disk fill
-4. **Baseline Metrics**: Establish normal ranges for your environment
-5. **Capacity Planning**: Monitor trends for scaling decisions
-6. **Test Monitoring**: Verify alerts work before issues occur
-
-## See Also
-
-- [Performance Tuning](performance.md) - Optimization guide
-- [Troubleshooting](troubleshooting.md) - Common issues
-- [Configuration Guide](configuration.md) - Monitoring configuration
-- [Integration Examples](integrations.md) - Monitoring system integration

doc/quickstart.md

@@ -1,26 +1,20 @@
 # Quick Start Guide
 
-Get LogWisp up and running in 5 minutes!
+Get LogWisp up and running in 5 minutes with the new pipeline architecture!
 
 ## Installation
 
 ### From Source
 ```bash
-# Clone the repository
+git clone https://github.com/lixenwraith/logwisp.git
-git clone https://github.com/yourusername/logwisp.git
 cd logwisp
-
-# Build and install
 make install
-
-# Or just build
-make build
-./logwisp --version
 ```
 
 ### Using Go Install
+
 ```bash
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+go install github.com/lixenwraith/logwisp/src/cmd/logwisp@latest
 ```
 
 ## Basic Usage
@@ -35,22 +29,19 @@ logwisp
 
 ### 2. Stream Logs
 
-In another terminal, connect to the log stream:
+Connect to the log stream:
 
 ```bash
-# Using curl (SSE stream)
+# SSE stream
 curl -N http://localhost:8080/stream
 
 # Check status
 curl http://localhost:8080/status | jq .
 ```
 
-### 3. Create Some Logs
+### 3. Generate Test Logs
-
-Generate test logs to see streaming in action:
 
 ```bash
-# In a third terminal
 echo "[ERROR] Something went wrong!" >> test.log
 echo "[INFO] Application started" >> test.log
 echo "[WARN] Low memory warning" >> test.log
@@ -60,88 +51,78 @@ echo "[WARN] Low memory warning" >> test.log
 
 ### Monitor Specific Directory
 
-Create a configuration file `~/.config/logwisp.toml`:
+Create `~/.config/logwisp.toml`:
 
 ```toml
-[[streams]]
+[[pipelines]]
 name = "myapp"
 
-[streams.monitor]
+[[pipelines.sources]]
-targets = [
+type = "directory"
-    { path = "/var/log/myapp", pattern = "*.log", is_file = false }
+options = { path = "/var/log/myapp", pattern = "*.log" }
-]
 
-[streams.httpserver]
+[[pipelines.sinks]]
-enabled = true
+type = "http"
-port = 8080
+options = { port = 8080 }
 ```
 
-Run LogWisp:
+### Filter Only Errors
-```bash
-logwisp
-```
-
-### Filter Only Errors and Warnings
-
-Add filters to your configuration:
 
 ```toml
-[[streams]]
+[[pipelines]]
 name = "errors"
 
-[streams.monitor]
+[[pipelines.sources]]
-targets = [
+type = "directory"
-    { path = "./", pattern = "*.log" }
+options = { path = "./", pattern = "*.log" }
-]
 
-[[streams.filters]]
+[[pipelines.filters]]
 type = "include"
-patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
+patterns = ["ERROR", "WARN", "CRITICAL"]
 
-[streams.httpserver]
+[[pipelines.sinks]]
-enabled = true
+type = "http"
-port = 8080
+options = { port = 8080 }
 ```
 
-### Multiple Log Sources
+### Multiple Outputs
 
-Monitor different applications on different ports:
+Send logs to both HTTP stream and file:
 
 ```toml
-# Stream 1: Web application
+[[pipelines]]
-[[streams]]
+name = "multi-output"
-name = "webapp"
-[streams.monitor]
-targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
 
-# Stream 2: Database
+[[pipelines.sources]]
-[[streams]]
+type = "directory"
-name = "database"  
+options = { path = "/var/log/app", pattern = "*.log" }
-[streams.monitor]
+
-targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
+# HTTP streaming
-[streams.httpserver]
+[[pipelines.sinks]]
-enabled = true
+type = "http"
-port = 8081
+options = { port = 8080 }
+
+# File archival
+[[pipelines.sinks]]
+type = "file"
+options = { directory = "/var/log/archive", name = "app" }
 ```
 
 ### TCP Streaming
 
-For high-performance streaming, use TCP:
+For high-performance streaming:
 
 ```toml
-[[streams]]
+[[pipelines]]
 name = "highperf"
 
-[streams.monitor]
+[[pipelines.sources]]
-targets = [{ path = "/var/log/app", pattern = "*.log" }]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
 
-[streams.tcpserver]
+[[pipelines.sinks]]
-enabled = true
+type = "tcp"
-port = 9090
+options = { port = 9090, buffer_size = 5000 }
-buffer_size = 5000
 ```
 
 Connect with netcat:
@@ -151,16 +132,15 @@ nc localhost 9090
 
 ### Router Mode
 
-Consolidate multiple streams on one port using router mode:
+Run multiple pipelines on shared ports:
 
 ```bash
-# With the multi-stream config above
 logwisp --router
 
-# Access streams at:
+# Access pipelines at:
-# http://localhost:8080/webapp/stream
+# http://localhost:8080/myapp/stream
-# http://localhost:8080/database/stream
+# http://localhost:8080/errors/stream
-# http://localhost:8080/status (global status)
+# http://localhost:8080/status (global)
 ```
 
 ## Quick Tips
@@ -170,40 +150,23 @@ logwisp --router
 logwisp --log-level debug --log-output stderr
 ```
 
-### Run in Background
-```bash
-logwisp --background --config /etc/logwisp/prod.toml
-```
-
 ### Rate Limiting
-Protect your streams from abuse:
-
 ```toml
-[streams.httpserver.rate_limit]
+[[pipelines.sinks]]
-enabled = true
+type = "http"
-requests_per_second = 10.0
+options = {
+    port = 8080,
+    rate_limit = {
+        enabled = true,
+        requests_per_second = 10.0,
         burst_size = 20
-max_connections_per_ip = 5
+    }
+}
 ```
 
-### JSON Output Format
+### Console Output
-For structured logging:
-
 ```toml
-[logging.console]
+[[pipelines.sinks]]
-format = "json"
+type = "stdout"  # or "stderr"
+options = {}
 ```
-
-## What's Next?
-
-- Read the [Configuration Guide](configuration.md) for all options
-- Learn about [Filters](filters.md) for advanced pattern matching
-- Explore [Rate Limiting](ratelimiting.md) for production deployments
-- Check out [Example Configurations](examples/) for more scenarios
-
-## Getting Help
-
-- Run `logwisp --help` for CLI options
-- Check `http://localhost:8080/status` for runtime statistics
-- Enable debug logging for troubleshooting
-- Visit our [GitHub repository](https://github.com/yourusername/logwisp) for issues and discussions

doc/ratelimiting.md

@@ -1,136 +1,65 @@
 # Rate Limiting Guide
 
-LogWisp provides configurable rate limiting to protect against abuse, prevent resource exhaustion, and ensure fair access to log streams.
+LogWisp provides configurable rate limiting to protect against abuse and ensure fair access.
 
-## How Rate Limiting Works
+## How It Works
 
-LogWisp uses a **token bucket algorithm** for smooth, burst-tolerant rate limiting:
+Token bucket algorithm:
-
+1. Each client gets a bucket with fixed capacity
-1. Each client (or globally) gets a bucket with a fixed capacity
+2. Tokens refill at configured rate
-2. Tokens are added to the bucket at a configured rate
 3. Each request consumes one token
-4. If no tokens are available, the request is rejected
+4. No tokens = request rejected
-5. The bucket can accumulate tokens up to its capacity for bursts
 
 ## Configuration
 
-### Basic Configuration
-
 ```toml
-[streams.httpserver.rate_limit]
+[[pipelines.sinks]]
-enabled = true                    # Enable rate limiting
+type = "http"  # or "tcp"
-requests_per_second = 10.0        # Token refill rate
+options = {
-burst_size = 20                   # Maximum tokens (bucket capacity)
+    port = 8080,
-limit_by = "ip"                   # "ip" or "global"
+    rate_limit = {
+        enabled = true,
+        requests_per_second = 10.0,
+        burst_size = 20,
+        limit_by = "ip",  # or "global"
+        max_connections_per_ip = 5,
+        max_total_connections = 100,
+        response_code = 429,
+        response_message = "Rate limit exceeded"
+    }
+}
 ```
 
-### Complete Options
+## Strategies
-
-```toml
-[streams.httpserver.rate_limit]
-# Core settings
-enabled = true                    # Enable/disable rate limiting
-requests_per_second = 10.0        # Token generation rate (float)
-burst_size = 20                   # Token bucket capacity
-
-# Limiting strategy
-limit_by = "ip"                   # "ip" or "global"
-
-# Connection limits
-max_connections_per_ip = 5        # Max concurrent connections per IP
-max_total_connections = 100       # Max total concurrent connections
-
-# Response configuration
-response_code = 429               # HTTP status code when limited
-response_message = "Rate limit exceeded"  # Error message
-
-# Same options available for TCP
-[streams.tcpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0
-burst_size = 10
-limit_by = "ip"
-```
-
-## Limiting Strategies
 
 ### Per-IP Limiting (Default)
-
+Each IP gets its own bucket:
-Each client IP address gets its own token bucket:
-
 ```toml
-[streams.httpserver.rate_limit]
-enabled = true
 limit_by = "ip"
 requests_per_second = 10.0
-burst_size = 20
+# Client A: 10 req/sec
+# Client B: 10 req/sec
 ```
 
-**Use cases:**
-- Fair access for multiple users
-- Prevent single client from monopolizing resources
-- Public-facing endpoints
-
-**Example behavior:**
-- Client A: Can make 10 req/sec
-- Client B: Also can make 10 req/sec
-- Total: Up to 10 × number of clients
-
 ### Global Limiting
-
+All clients share one bucket:
-All clients share a single token bucket:
-
 ```toml
-[streams.httpserver.rate_limit]
-enabled = true
 limit_by = "global"
 requests_per_second = 50.0
-burst_size = 100
+# All clients combined: 50 req/sec
 ```
 
-**Use cases:**
-- Protect backend resources
-- Control total system load
-- Internal services with known clients
-
-**Example behavior:**
-- All clients combined: 50 req/sec max
-- One aggressive client can consume all tokens
-
 ## Connection Limits
 
-In addition to request rate limiting, you can limit concurrent connections:
-
-### Per-IP Connection Limit
-
 ```toml
-[streams.httpserver.rate_limit]
+max_connections_per_ip = 5    # Per IP
-max_connections_per_ip = 5    # Each IP can have max 5 connections
+max_total_connections = 100   # Total
 ```
 
-**Behavior:**
-- Prevents connection exhaustion attacks
-- Limits resource usage per client
-- Checked before rate limits
-
-### Total Connection Limit
-
-```toml
-[streams.httpserver.rate_limit]
-max_total_connections = 100   # Max 100 connections total
-```
-
-**Behavior:**
-- Protects server resources
-- Prevents memory exhaustion
-- Global limit across all IPs
-
 ## Response Behavior
 
-### HTTP Responses
+### HTTP
-
+Returns JSON with configured status:
-When rate limited, HTTP clients receive:
-
 ```json
 {
     "error": "Rate limit exceeded",
@@ -138,389 +67,59 @@ When rate limited, HTTP clients receive:
 }
 ```
 
-With these headers:
+### TCP
-- Status code: 429 (default) or configured value
+Connections silently dropped.
-- Content-Type: application/json
 
-Configure custom responses:
+## Examples
-
-```toml
-[streams.httpserver.rate_limit]
-response_code = 503                    # Service Unavailable
-response_message = "Server overloaded, please retry later"
-```
-
-### TCP Behavior
-
-TCP connections are **silently dropped** when rate limited:
-- No error message sent
-- Connection immediately closed
-- Prevents information leakage
-
-## Configuration Examples
 
 ### Light Protection
-
-For internal or trusted environments:
-
 ```toml
-[streams.httpserver.rate_limit]
+rate_limit = {
-enabled = true
+    enabled = true,
-requests_per_second = 50.0
+    requests_per_second = 50.0,
     burst_size = 100
-limit_by = "ip"
+}
 ```
 
 ### Moderate Protection
-
-For semi-public endpoints:
-
 ```toml
-[streams.httpserver.rate_limit]
+rate_limit = {
-enabled = true
+    enabled = true,
-requests_per_second = 10.0
+    requests_per_second = 10.0,
-burst_size = 30
+    burst_size = 30,
-limit_by = "ip"
     max_connections_per_ip = 5
-max_total_connections = 200
+}
 ```
 
 ### Strict Protection
-
-For public or sensitive endpoints:
-
 ```toml
-[streams.httpserver.rate_limit]
+rate_limit = {
-enabled = true
+    enabled = true,
-requests_per_second = 2.0
+    requests_per_second = 2.0,
-burst_size = 5
+    burst_size = 5,
-limit_by = "ip"
+    max_connections_per_ip = 2,
-max_connections_per_ip = 2
-max_total_connections = 50
     response_code = 503
-response_message = "Service temporarily unavailable"
-```
-
-### Debug/Development
-
-Disable for testing:
-
-```toml
-[streams.httpserver.rate_limit]
-enabled = false
-```
-
-## Use Case Scenarios
-
-### Public Log Viewer
-
-Prevent abuse while allowing legitimate use:
-
-```toml
-[[streams]]
-name = "public-logs"
-
-[streams.httpserver]
-enabled = true
-port = 8080
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0      # 5 new connections per second
-burst_size = 10                # Allow short bursts
-limit_by = "ip"
-max_connections_per_ip = 3     # Max 3 streams per user
-max_total_connections = 100
-```
-
-### Internal Monitoring
-
-Protect against accidental overload:
-
-```toml
-[[streams]]
-name = "internal-metrics"
-
-[streams.httpserver]
-enabled = true
-port = 8081
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 100.0    # High limit for internal use
-burst_size = 200
-limit_by = "global"            # Total system limit
-max_total_connections = 500
-```
-
-### High-Security Audit Logs
-
-Very restrictive access:
-
-```toml
-[[streams]]
-name = "audit"
-
-[streams.httpserver]
-enabled = true
-port = 8443
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 0.5      # 1 request every 2 seconds
-burst_size = 2
-limit_by = "ip"
-max_connections_per_ip = 1     # Single connection only
-max_total_connections = 10
-response_code = 403            # Forbidden (hide rate limit)
-response_message = "Access denied"
-```
-
-### Multi-Tenant Service
-
-Different limits per stream:
-
-```toml
-# Free tier
-[[streams]]
-name = "logs-free"
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 1.0
-burst_size = 5
-max_connections_per_ip = 1
-
-# Premium tier
-[[streams]]
-name = "logs-premium"
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 50.0
-burst_size = 100
-max_connections_per_ip = 10
-```
-
-## Monitoring Rate Limits
-
-### Status Endpoint
-
-Check rate limit statistics:
-
-```bash
-curl http://localhost:8080/status | jq '.server.features.rate_limit'
-```
-
-Response includes:
-```json
-{
-  "enabled": true,
-  "total_requests": 15234,
-  "blocked_requests": 89,
-  "active_ips": 12,
-  "total_connections": 8,
-  "config": {
-    "requests_per_second": 10,
-    "burst_size": 20,
-    "limit_by": "ip"
-  }
 }
 ```
 
-### Debug Logging
+## Monitoring
 
-Enable debug logs to see rate limit decisions:
+Check statistics:
+```bash
+curl http://localhost:8080/status | jq '.sinks[0].details.rate_limit'
+```
+
+## Testing
 
 ```bash
-logwisp --log-level debug
+# Test rate limits
-```
+for i in {1..20}; do
-
+    curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8080/status
-Look for messages:
-```
-Request rate limited ip=192.168.1.100
-Connection limit exceeded ip=192.168.1.100 connections=5 limit=5
-Created new IP limiter ip=192.168.1.100 total_ips=3
-```
-
-## Testing Rate Limits
-
-### Test Script
-
-```bash
-#!/bin/bash
-# Test rate limiting behavior
-
-URL="http://localhost:8080/stream"
-PARALLEL=10
-DURATION=10
-
-echo "Testing rate limits..."
-echo "URL: $URL"
-echo "Parallel connections: $PARALLEL"
-echo "Duration: ${DURATION}s"
-echo
-
-# Function to connect and count lines
-test_connection() {
-    local id=$1
-    local count=0
-    local start=$(date +%s)
-    
-    while (( $(date +%s) - start < DURATION )); do
-        if curl -s -N --max-time 1 "$URL" >/dev/null 2>&1; then
-            ((count++))
-            echo "[$id] Connected successfully (total: $count)"
-        else
-            echo "[$id] Rate limited!"
-        fi
-        sleep 0.1
-    done
-}
-
-# Run parallel connections
-for i in $(seq 1 $PARALLEL); do
-    test_connection $i &
-done
-
-wait
-echo "Test complete"
-```
-
-### Load Testing
-
-Using Apache Bench (ab):
-
-```bash
-# Test burst handling
-ab -n 100 -c 20 http://localhost:8080/status
-
-# Test sustained load
-ab -n 1000 -c 5 -r http://localhost:8080/status
-```
-
-Using curl:
-
-```bash
-# Test connection limit
-for i in {1..10}; do
-    curl -N http://localhost:8080/stream &
 done
 ```
 
-## Tuning Guidelines
+## Tuning
 
-### Setting requests_per_second
+- **requests_per_second**: Expected load
-
+- **burst_size**: 2-3× requests_per_second
-Consider:
+- **Connection limits**: Based on memory
-- Expected legitimate traffic
-- Server capacity
-- Client retry behavior
-
-**Formula**: `requests_per_second = expected_clients × requests_per_client`
-
-### Setting burst_size
-
-General rule: `burst_size = 2-3 × requests_per_second`
-
-Examples:
-- `10 req/s → burst_size = 20-30`
-- `1 req/s → burst_size = 3-5`
-- `100 req/s → burst_size = 200-300`
-
-### Connection Limits
-
-Based on available memory:
-- Each HTTP connection: ~1-2MB
-- Each TCP connection: ~0.5-1MB
-
-**Formula**: `max_connections = available_memory / memory_per_connection`
-
-## Common Issues
-
-### "All requests blocked"
-
-Check if:
-- Rate limits too strict
-- Burst size too small
-- Using global limiting with many clients
-
-### "Memory growth"
-
-Possible causes:
-- No connection limits set
-- Slow clients holding connections
-- Too high burst_size
-
-Solutions:
-```toml
-max_connections_per_ip = 5
-max_total_connections = 100
-```
-
-### "Legitimate users blocked"
-
-Consider:
-- Increasing burst_size for short spikes
-- Using per-IP instead of global limiting
-- Different streams for different user tiers
-
-## Security Considerations
-
-### Information Disclosure
-
-Rate limit responses can reveal information:
-
-```toml
-# Default - informative
-response_code = 429
-response_message = "Rate limit exceeded"
-
-# Security-focused - generic
-response_code = 503
-response_message = "Service unavailable"
-
-# High security - misleading
-response_code = 403
-response_message = "Forbidden"
-```
-
-### DDoS Protection
-
-Rate limiting helps but isn't complete DDoS protection:
-- Use with firewall rules
-- Consider CDN/proxy rate limiting
-- Monitor for distributed attacks
-
-### Resource Exhaustion
-
-Protect against:
-- Connection exhaustion
-- Memory exhaustion
-- CPU exhaustion
-
-```toml
-# Comprehensive protection
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-burst_size = 20
-max_connections_per_ip = 5
-max_total_connections = 100
-limit_by = "ip"
-```
-
-## Best Practices
-
-1. **Start Conservative**: Begin with strict limits and relax as needed
-2. **Monitor Statistics**: Use `/status` endpoint to track behavior
-3. **Test Thoroughly**: Verify limits work as expected under load
-4. **Document Limits**: Make rate limits clear to users
-5. **Provide Retry Info**: Help clients implement proper retry logic
-6. **Different Tiers**: Consider different limits for different user types
-7. **Regular Review**: Adjust limits based on usage patterns
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [Security Best Practices](security.md) - Security hardening
-- [Performance Tuning](performance.md) - Optimization guidelines
-- [Troubleshooting](troubleshooting.md) - Common issues

doc/router.md

@@ -1,520 +1,158 @@
 # Router Mode Guide
 
-Router mode allows multiple LogWisp streams to share HTTP ports through path-based routing, simplifying deployment and access control.
+Router mode enables multiple pipelines to share HTTP ports through path-based routing.
 
 ## Overview
 
-In standard mode, each stream requires its own port:
+**Standard mode**: Each pipeline needs its own port
-- Stream 1: `http://localhost:8080/stream`
+- Pipeline 1: `http://localhost:8080/stream`
-- Stream 2: `http://localhost:8081/stream`
+- Pipeline 2: `http://localhost:8081/stream`
-- Stream 3: `http://localhost:8082/stream`
 
-In router mode, streams share ports via paths:
+**Router mode**: Pipelines share ports via paths
-- Stream 1: `http://localhost:8080/app/stream`
+- Pipeline 1: `http://localhost:8080/app/stream`
-- Stream 2: `http://localhost:8080/database/stream`
+- Pipeline 2: `http://localhost:8080/database/stream`
-- Stream 3: `http://localhost:8080/system/stream`
 - Global status: `http://localhost:8080/status`
 
 ## Enabling Router Mode
 
-Start LogWisp with the `--router` flag:
-
 ```bash
-logwisp --router --config /etc/logwisp/multi-stream.toml
+logwisp --router --config /etc/logwisp/multi-pipeline.toml
 ```
 
 ## Configuration
 
-### Basic Router Configuration
-
 ```toml
-# All streams can use the same port in router mode
+# All pipelines can use the same port
-[[streams]]
+[[pipelines]]
 name = "app"
-[streams.monitor]
+[[pipelines.sources]]
-targets = [{ path = "/var/log/app", pattern = "*.log" }]
+type = "directory"
-[streams.httpserver]
+options = { path = "/var/log/app", pattern = "*.log" }
-enabled = true
+[[pipelines.sinks]]
-port = 8080  # Same port OK
+type = "http"
+options = { port = 8080 }  # Same port OK
 
-[[streams]]
+[[pipelines]]
 name = "database"
-[streams.monitor]
+[[pipelines.sources]]
-targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
+type = "directory"
-[streams.httpserver]
+options = { path = "/var/log/postgresql", pattern = "*.log" }
-enabled = true
+[[pipelines.sinks]]
-port = 8080  # Shared port
+type = "http"
-
+options = { port = 8080 }  # Shared port
-[[streams]]
-name = "nginx"
-[streams.monitor]
-targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080  # Shared port
 ```
 
-### Path Structure
+## Path Structure
 
-In router mode, paths are automatically prefixed with the stream name:
+Paths are prefixed with pipeline name:
 
-| Stream Name | Configuration Path | Router Mode Path |
+| Pipeline | Config Path | Router Path |
-|------------|-------------------|------------------|
+|----------|-------------|-------------|
 | `app` | `/stream` | `/app/stream` |
 | `app` | `/status` | `/app/status` |
 | `database` | `/stream` | `/database/stream` |
-| `database` | `/status` | `/database/status` |
 
 ### Custom Paths
 
-You can customize the paths in each stream:
-
 ```toml
-[[streams]]
+[[pipelines.sinks]]
-name = "api"
+type = "http"
-[streams.httpserver]
+options = {
-stream_path = "/logs"      # Becomes /api/logs
+    stream_path = "/logs",    # Becomes /app/logs
-status_path = "/health"    # Becomes /api/health
+    status_path = "/health"   # Becomes /app/health
+}
 ```
 
-## URL Endpoints
+## Endpoints
-
-### Stream Endpoints
-
-Access individual streams:
 
+### Pipeline Endpoints
 ```bash
-# SSE stream for 'app' logs
+# SSE stream
 curl -N http://localhost:8080/app/stream
 
-# Status for 'database' stream
+# Pipeline status
 curl http://localhost:8080/database/status
-
-# Custom path example
-curl -N http://localhost:8080/api/logs
 ```
 
 ### Global Status
-
-Router mode provides a global status endpoint:
-
 ```bash
-curl http://localhost:8080/status | jq .
+curl http://localhost:8080/status
 ```
 
-Returns aggregated information:
+Returns:
 ```json
 {
   "service": "LogWisp Router",
-  "version": "1.0.0",
+  "pipelines": {
-  "port": 8080,
+    "app": { /* stats */ },
-  "total_streams": 3,
+    "database": { /* stats */ }
-  "streams": {
-    "app": { /* stream stats */ },
-    "database": { /* stream stats */ },
-    "nginx": { /* stream stats */ }
   },
-  "router": {
+  "total_pipelines": 2
-    "uptime_seconds": 3600,
-    "total_requests": 15234,
-    "routed_requests": 15220,
-    "failed_requests": 14
 }
-}
-```
-
-## Port Sharing
-
-### How It Works
-
-1. Router server listens on configured ports
-2. Examines request path to determine target stream
-3. Routes request to appropriate stream handler
-4. Stream handles request as if standalone
-
-### Port Assignment Rules
-
-In router mode:
-- Multiple streams can use the same port
-- Router detects and consolidates shared ports
-- Each unique port gets one router server
-- TCP servers remain independent (no routing)
-
-Example with multiple ports:
-
-```toml
-# Streams 1-3 share port 8080
-[[streams]]
-name = "app"
-[streams.httpserver]
-port = 8080
-
-[[streams]]
-name = "db"
-[streams.httpserver]
-port = 8080
-
-[[streams]]
-name = "web"
-[streams.httpserver]
-port = 8080
-
-# Stream 4 uses different port
-[[streams]]
-name = "admin"
-[streams.httpserver]
-port = 9090
-
-# Result: 2 router servers (8080 and 9090)
 ```
 
 ## Use Cases
 
-### Microservices Architecture
+### Microservices
-
-Route logs from different services:
-
 ```toml
-[[streams]]
+[[pipelines]]
 name = "frontend"
-[streams.monitor]
+[[pipelines.sources]]
-targets = [{ path = "/var/log/frontend", pattern = "*.log" }]
+type = "directory"
-[streams.httpserver]
+options = { path = "/var/log/frontend", pattern = "*.log" }
-enabled = true
+[[pipelines.sinks]]
-port = 8080
+type = "http"
+options = { port = 8080 }
 
-[[streams]]
+[[pipelines]]
 name = "backend"
-[streams.monitor]
+[[pipelines.sources]]
-targets = [{ path = "/var/log/backend", pattern = "*.log" }]
+type = "directory"
-[streams.httpserver]
+options = { path = "/var/log/backend", pattern = "*.log" }
-enabled = true
+[[pipelines.sinks]]
-port = 8080
+type = "http"
+options = { port = 8080 }
 
-[[streams]]
+# Access:
-name = "worker"
+# http://localhost:8080/frontend/stream
-[streams.monitor]
+# http://localhost:8080/backend/stream
-targets = [{ path = "/var/log/worker", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
 ```
 
-Access via:
+### Environment-Based
-- Frontend logs: `http://localhost:8080/frontend/stream`
-- Backend logs: `http://localhost:8080/backend/stream`
-- Worker logs: `http://localhost:8080/worker/stream`
-
-### Environment-Based Routing
-
-Different log levels per environment:
-
 ```toml
-[[streams]]
+[[pipelines]]
 name = "prod"
-[streams.monitor]
+[[pipelines.filters]]
-targets = [{ path = "/logs/prod", pattern = "*.log" }]
-[[streams.filters]]
 type = "include"
 patterns = ["ERROR", "WARN"]
-[streams.httpserver]
+[[pipelines.sinks]]
-port = 8080
+type = "http"
+options = { port = 8080 }
 
-[[streams]]
+[[pipelines]]
-name = "staging"
-[streams.monitor]
-targets = [{ path = "/logs/staging", pattern = "*.log" }]
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "WARN", "INFO"]
-[streams.httpserver]
-port = 8080
-
-[[streams]]
 name = "dev"
-[streams.monitor]
-targets = [{ path = "/logs/dev", pattern = "*.log" }]
 # No filters - all logs
-[streams.httpserver]
+[[pipelines.sinks]]
-port = 8080
+type = "http"
-```
+options = { port = 8080 }
-
-### Department Access
-
-Separate streams for different teams:
-
-```toml
-[[streams]]
-name = "engineering"
-[streams.monitor]
-targets = [{ path = "/logs/apps", pattern = "*.log" }]
-[streams.httpserver]
-port = 8080
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 50.0
-
-[[streams]]
-name = "security"
-[streams.monitor]
-targets = [{ path = "/logs/audit", pattern = "*.log" }]
-[streams.httpserver]
-port = 8080
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0
-max_connections_per_ip = 1
-
-[[streams]]
-name = "support"
-[streams.monitor]
-targets = [{ path = "/logs/customer", pattern = "*.log" }]
-[[streams.filters]]
-type = "exclude"
-patterns = ["password", "token", "secret"]
-[streams.httpserver]
-port = 8080
-```
-
-## Advanced Features
-
-### Mixed Mode Deployment
-
-Combine router and standalone modes:
-
-```toml
-# Public streams via router
-[[streams]]
-name = "public-api"
-[streams.httpserver]
-enabled = true
-port = 8080  # Router mode
-
-[[streams]]
-name = "public-web"
-[streams.httpserver]
-enabled = true
-port = 8080  # Router mode
-
-# Internal stream standalone
-[[streams]]
-name = "internal"
-[streams.httpserver]
-enabled = true
-port = 9999  # Different port, standalone
-
-# High-performance TCP
-[[streams]]
-name = "metrics"
-[streams.tcpserver]
-enabled = true
-port = 9090  # TCP not affected by router
-```
-
-### Load Balancer Integration
-
-Router mode works well with load balancers:
-
-```nginx
-# Nginx configuration
-upstream logwisp {
-    server logwisp1:8080;
-    server logwisp2:8080;
-    server logwisp3:8080;
-}
-
-location /logs/ {
-    proxy_pass http://logwisp/;
-    proxy_http_version 1.1;
-    proxy_set_header Connection "";
-    proxy_buffering off;
-}
-```
-
-Access becomes:
-- `https://example.com/logs/app/stream`
-- `https://example.com/logs/database/stream`
-- `https://example.com/logs/status`
-
-### Path-Based Access Control
-
-Use reverse proxy for authentication:
-
-```nginx
-# Require auth for security logs
-location /logs/security/ {
-    auth_basic "Security Logs";
-    auth_basic_user_file /etc/nginx/security.htpasswd;
-    proxy_pass http://localhost:8080/security/;
-}
-
-# Public access for status
-location /logs/app/ {
-    proxy_pass http://localhost:8080/app/;
-}
 ```
 
 ## Limitations
 
-### Router Mode Limitations
+1. **HTTP Only**: Router mode only works for HTTP/SSE
+2. **No TCP Routing**: TCP remains on separate ports
+3. **Path Conflicts**: Pipeline names must be unique
 
-1. **HTTP Only**: Router mode only works for HTTP/SSE streams
+## Load Balancer Integration
-2. **No TCP Routing**: TCP streams remain on separate ports
-3. **Path Conflicts**: Stream names must be unique
-4. **Same Config**: All streams on a port share SSL/auth settings
 
-### When Not to Use Router Mode
+```nginx
+upstream logwisp {
+    server logwisp1:8080;
+    server logwisp2:8080;
+}
 
-- High-performance scenarios (use TCP)
+location /logs/ {
-- Streams need different SSL certificates
+    proxy_pass http://logwisp/;
-- Complex authentication per stream
+    proxy_buffering off;
-- Network isolation requirements
-
-## Troubleshooting
-
-### "Path not found"
-
-Check available routes:
-```bash
-curl http://localhost:8080/invalid-path
-```
-
-Response shows available routes:
-```json
-{
-  "error": "Not Found",
-  "requested_path": "/invalid-path",
-  "available_routes": [
-    "/status (global status)",
-    "/app/stream (stream: app)",
-    "/app/status (status: app)",
-    "/database/stream (stream: database)",
-    "/database/status (status: database)"
-  ]
 }
 ```
-
-### "Port conflict"
-
-If you see port conflicts:
-1. Ensure `--router` flag is used
-2. Check all streams have `httpserver.enabled = true`
-3. Verify no other services use the port
-
-### Debug Routing
-
-Enable debug logging:
-```bash
-logwisp --router --log-level debug
-```
-
-Look for routing decisions:
-```
-Router request method=GET path=/app/stream remote_addr=127.0.0.1:54321
-Routing request to stream stream=app original_path=/app/stream remaining_path=/stream
-```
-
-### Performance Impact
-
-Router mode adds minimal overhead:
-- ~100-200ns per request for path matching
-- Negligible memory overhead
-- No impact on streaming performance
-
-## Best Practices
-
-### Naming Conventions
-
-Use clear, consistent stream names:
-```toml
-# Good: Clear purpose
-name = "frontend-prod"
-name = "backend-staging"
-name = "worker-payments"
-
-# Bad: Ambiguous
-name = "logs1"
-name = "stream2"
-name = "test"
-```
-
-### Path Organization
-
-Group related streams:
-```
-/prod/frontend/stream
-/prod/backend/stream
-/staging/frontend/stream
-/staging/backend/stream
-```
-
-### Documentation
-
-Document your routing structure:
-```toml
-# Stream for production API logs
-# Access: https://logs.example.com/api-prod/stream
-[[streams]]
-name = "api-prod"
-```
-
-### Monitoring
-
-Use global status for overview:
-```bash
-# Monitor all streams
-watch -n 5 'curl -s localhost:8080/status | jq .streams'
-
-# Check specific stream
-curl -s localhost:8080/status | jq '.streams.app'
-```
-
-## Migration Guide
-
-### From Standalone to Router
-
-1. **Update configuration** - ensure consistent ports:
-   ```toml
-   # Change from different ports
-   [streams.httpserver]
-   port = 8080  # Was 8081, 8082, etc.
-   ```
-
-2. **Start with router flag**:
-   ```bash
-   logwisp --router --config existing.toml
-   ```
-
-3. **Update client URLs**:
-   ```bash
-   # Old: http://localhost:8081/stream
-   # New: http://localhost:8080/streamname/stream
-   ```
-
-4. **Update monitoring**:
-   ```bash
-   # Global status now available
-   curl http://localhost:8080/status
-   ```
-
-### Gradual Migration
-
-Run both modes during transition:
-```bash
-# Week 1: Run standalone (current)
-logwisp --config prod.toml
-
-# Week 2: Run both
-logwisp --config prod.toml &  # Standalone
-logwisp --router --config prod-router.toml &  # Router
-
-# Week 3: Router only
-logwisp --router --config prod.toml
-```
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Stream configuration
-- [HTTP Streaming](api.md#http-sse) - SSE protocol details
-- [Load Balancing](integrations.md#load-balancers) - Integration patterns
-- [Security Best Practices](security.md) - Securing router deployments

doc/status.md

@@ -0,0 +1,186 @@
+# Status Monitoring
+
+LogWisp provides comprehensive monitoring through status endpoints and operational logs.
+
+## Status Endpoints
+
+### Pipeline Status
+
+```bash
+# Standalone mode
+curl http://localhost:8080/status
+
+# Router mode
+curl http://localhost:8080/pipelinename/status
+```
+
+Example response:
+```json
+{
+  "service": "LogWisp",
+  "version": "1.0.0",
+  "server": {
+    "type": "http",
+    "port": 8080,
+    "active_clients": 5,
+    "buffer_size": 1000,
+    "uptime_seconds": 3600
+  },
+  "sources": [{
+    "type": "directory",
+    "total_entries": 152341,
+    "dropped_entries": 12,
+    "active_watchers": 3
+  }],
+  "filters": {
+    "filter_count": 2,
+    "total_processed": 152341,
+    "total_passed": 48234
+  },
+  "sinks": [{
+    "type": "http",
+    "total_processed": 48234,
+    "active_connections": 5
+  }]
+}
+```
+
+## Key Metrics
+
+### Source Metrics
+| Metric | Description | Healthy Range |
+|--------|-------------|---------------|
+| `active_watchers` | Files being watched | 1-1000 |
+| `total_entries` | Entries processed | Increasing |
+| `dropped_entries` | Buffer overflows | < 1% of total |
+
+### Sink Metrics
+| Metric | Description | Warning Signs |
+|--------|-------------|---------------|
+| `active_connections` | Current clients | Near limit |
+| `total_processed` | Entries sent | Should match filter output |
+
+### Filter Metrics
+| Metric | Description | Notes |
+|--------|-------------|-------|
+| `total_processed` | Entries checked | All entries |
+| `total_passed` | Passed filters | Check if too low/high |
+
+## Operational Logging
+
+### Log Levels
+```toml
+[logging]
+level = "info"  # debug, info, warn, error
+```
+
+### Important Messages
+
+**Startup**:
+```
+LogWisp starting version=1.0.0
+Pipeline created successfully pipeline=app
+HTTP server started port=8080
+```
+
+**Connections**:
+```
+HTTP client connected remote_addr=192.168.1.100 active_clients=6
+TCP connection opened active_connections=3
+```
+
+**Errors**:
+```
+Failed to open file path=/var/log/app.log error=permission denied
+Request rate limited ip=192.168.1.100
+```
+
+## Health Checks
+
+### Basic Check
+```bash
+#!/bin/bash
+if curl -s -f http://localhost:8080/status > /dev/null; then
+    echo "Healthy"
+else
+    echo "Unhealthy"
+    exit 1
+fi
+```
+
+### Advanced Check
+```bash
+#!/bin/bash
+STATUS=$(curl -s http://localhost:8080/status)
+DROPPED=$(echo "$STATUS" | jq '.sources[0].dropped_entries')
+TOTAL=$(echo "$STATUS" | jq '.sources[0].total_entries')
+
+if [ $((DROPPED * 100 / TOTAL)) -gt 5 ]; then
+    echo "High drop rate"
+    exit 1
+fi
+```
+
+### Docker
+```dockerfile
+HEALTHCHECK --interval=30s --timeout=3s \
+    CMD curl -f http://localhost:8080/status || exit 1
+```
+
+## Integration
+
+### Prometheus Export
+```bash
+#!/bin/bash
+STATUS=$(curl -s http://localhost:8080/status)
+
+cat << EOF
+# HELP logwisp_active_clients Active streaming clients
+# TYPE logwisp_active_clients gauge
+logwisp_active_clients $(echo "$STATUS" | jq '.server.active_clients')
+
+# HELP logwisp_total_entries Total log entries
+# TYPE logwisp_total_entries counter
+logwisp_total_entries $(echo "$STATUS" | jq '.sources[0].total_entries')
+EOF
+```
+
+### Alerts
+
+| Alert | Condition | Severity |
+|-------|-----------|----------|
+| Service Down | Status fails | Critical |
+| High Drops | >10% dropped | Warning |
+| No Activity | 0 entries/min | Warning |
+| Rate Limited | >20% blocked | Warning |
+
+## Performance Monitoring
+
+### CPU Usage
+```bash
+top -p $(pgrep logwisp)
+```
+
+### Memory Usage
+```bash
+ps aux | grep logwisp
+```
+
+### Connections
+```bash
+ss -tan | grep :8080 | wc -l
+```
+
+## Troubleshooting
+
+Enable debug logging:
+```bash
+logwisp --log-level debug --log-output stderr
+```
+
+Check specific components:
+```bash
+curl -s http://localhost:8080/status | jq '.sources'
+curl -s http://localhost:8080/status | jq '.filters'
+curl -s http://localhost:8080/status | jq '.sinks'
+```

doc/troubleshooting.md

@@ -1,537 +0,0 @@
-# Troubleshooting Guide
-
-This guide helps diagnose and resolve common issues with LogWisp.
-
-## Diagnostic Tools
-
-### Enable Debug Logging
-
-The first step in troubleshooting is enabling debug logs:
-
-```bash
-# Via command line
-logwisp --log-level debug --log-output stderr
-
-# Via environment
-export LOGWISP_LOGGING_LEVEL=debug
-logwisp
-
-# Via config
-[logging]
-level = "debug"
-output = "stderr"
-```
-
-### Check Status Endpoint
-
-Verify LogWisp is running and processing:
-
-```bash
-# Basic check
-curl http://localhost:8080/status
-
-# Pretty print
-curl -s http://localhost:8080/status | jq .
-
-# Check specific metrics
-curl -s http://localhost:8080/status | jq '.monitor'
-```
-
-### Test Log Streaming
-
-Verify streams are working:
-
-```bash
-# Test SSE stream (should show heartbeats if enabled)
-curl -N http://localhost:8080/stream
-
-# Test with timeout
-timeout 5 curl -N http://localhost:8080/stream
-
-# Test TCP stream
-nc localhost 9090
-```
-
-## Common Issues
-
-### No Logs Appearing
-
-**Symptoms:**
-- Stream connects but no log entries appear
-- Status shows `total_entries: 0`
-
-**Diagnosis:**
-
-1. Check monitor configuration:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.monitor'
-   ```
-
-2. Verify file paths exist:
-   ```bash
-   # Check your configured paths
-   ls -la /var/log/myapp/
-   ```
-
-3. Check file permissions:
-   ```bash
-   # LogWisp user must have read access
-   sudo -u logwisp ls /var/log/myapp/
-   ```
-
-4. Verify files match pattern:
-   ```bash
-   # If pattern is "*.log"
-   ls /var/log/myapp/*.log
-   ```
-
-5. Check if files are being updated:
-   ```bash
-   # Should show recent timestamps
-   ls -la /var/log/myapp/*.log
-   tail -f /var/log/myapp/app.log
-   ```
-
-**Solutions:**
-
-- Fix file permissions:
-  ```bash
-  sudo chmod 644 /var/log/myapp/*.log
-  sudo usermod -a -G adm logwisp  # Add to log group
-  ```
-
-- Correct path configuration:
-  ```toml
-  targets = [
-      { path = "/correct/path/to/logs", pattern = "*.log" }
-  ]
-  ```
-
-- Use absolute paths:
-  ```toml
-  # Bad: Relative path
-  targets = [{ path = "./logs", pattern = "*.log" }]
-  
-  # Good: Absolute path
-  targets = [{ path = "/var/log/app", pattern = "*.log" }]
-  ```
-
-### High CPU Usage
-
-**Symptoms:**
-- LogWisp process using excessive CPU
-- System slowdown
-
-**Diagnosis:**
-
-1. Check process CPU:
-   ```bash
-   top -p $(pgrep logwisp)
-   ```
-
-2. Review check intervals:
-   ```bash
-   grep check_interval /etc/logwisp/logwisp.toml
-   ```
-
-3. Count active watchers:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.monitor.active_watchers'
-   ```
-
-4. Check filter complexity:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.filters'
-   ```
-
-**Solutions:**
-
-- Increase check interval:
-  ```toml
-  [streams.monitor]
-  check_interval_ms = 1000  # Was 50ms
-  ```
-
-- Reduce watched files:
-  ```toml
-  # Instead of watching entire directory
-  targets = [
-      { path = "/var/log/specific-app.log", is_file = true }
-  ]
-  ```
-
-- Simplify filter patterns:
-  ```toml
-  # Complex regex (slow)
-  patterns = ["^\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]\\s+\\[(ERROR|WARN)\\]"]
-  
-  # Simple patterns (fast)
-  patterns = ["ERROR", "WARN"]
-  ```
-
-### Memory Growth
-
-**Symptoms:**
-- Increasing memory usage over time
-- Eventually runs out of memory
-
-**Diagnosis:**
-
-1. Monitor memory usage:
-   ```bash
-   watch -n 10 'ps aux | grep logwisp'
-   ```
-
-2. Check connection count:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.server.active_clients'
-   ```
-
-3. Check for dropped entries:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.monitor.dropped_entries'
-   ```
-
-**Solutions:**
-
-- Limit connections:
-  ```toml
-  [streams.httpserver.rate_limit]
-  enabled = true
-  max_connections_per_ip = 5
-  max_total_connections = 100
-  ```
-
-- Reduce buffer sizes:
-  ```toml
-  [streams.httpserver]
-  buffer_size = 500  # Was 5000
-  ```
-
-- Enable rate limiting:
-  ```toml
-  [streams.httpserver.rate_limit]
-  enabled = true
-  requests_per_second = 10.0
-  ```
-
-### Connection Refused
-
-**Symptoms:**
-- Cannot connect to LogWisp
-- `curl: (7) Failed to connect`
-
-**Diagnosis:**
-
-1. Check if LogWisp is running:
-   ```bash
-   ps aux | grep logwisp
-   systemctl status logwisp
-   ```
-
-2. Verify listening ports:
-   ```bash
-   sudo netstat -tlnp | grep logwisp
-   # or
-   sudo ss -tlnp | grep logwisp
-   ```
-
-3. Check firewall:
-   ```bash
-   sudo iptables -L -n | grep 8080
-   sudo ufw status
-   ```
-
-**Solutions:**
-
-- Start the service:
-  ```bash
-  sudo systemctl start logwisp
-  ```
-
-- Fix port configuration:
-  ```toml
-  [streams.httpserver]
-  enabled = true  # Must be true
-  port = 8080     # Correct port
-  ```
-
-- Open firewall:
-  ```bash
-  sudo ufw allow 8080/tcp
-  ```
-
-### Rate Limit Errors
-
-**Symptoms:**
-- HTTP 429 responses
-- "Rate limit exceeded" errors
-
-**Diagnosis:**
-
-1. Check rate limit stats:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.features.rate_limit'
-   ```
-
-2. Test rate limits:
-   ```bash
-   # Rapid requests
-   for i in {1..20}; do curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8080/status; done
-   ```
-
-**Solutions:**
-
-- Increase rate limits:
-  ```toml
-  [streams.httpserver.rate_limit]
-  requests_per_second = 50.0  # Was 10.0
-  burst_size = 100            # Was 20
-  ```
-
-- Use per-IP limiting:
-  ```toml
-  limit_by = "ip"  # Instead of "global"
-  ```
-
-- Disable for internal use:
-  ```toml
-  enabled = false
-  ```
-
-### Filter Not Working
-
-**Symptoms:**
-- Unwanted logs still appearing
-- Wanted logs being filtered out
-
-**Diagnosis:**
-
-1. Check filter configuration:
-   ```bash
-   curl -s http://localhost:8080/status | jq '.filters'
-   ```
-
-2. Test patterns:
-   ```bash
-   # Test regex pattern
-   echo "ERROR: test message" | grep -E "your-pattern"
-   ```
-
-3. Enable debug logging to see filter decisions:
-   ```bash
-   logwisp --log-level debug 2>&1 | grep filter
-   ```
-
-**Solutions:**
-
-- Fix pattern syntax:
-  ```toml
-  # Word boundaries
-  patterns = ["\\bERROR\\b"]  # Not "ERROR" which matches "TERROR"
-  
-  # Case insensitive
-  patterns = ["(?i)error"]
-  ```
-
-- Check filter order:
-  ```toml
-  # Include filters run first
-  [[streams.filters]]
-  type = "include"
-  patterns = ["ERROR", "WARN"]
-  
-  # Then exclude filters
-  [[streams.filters]]
-  type = "exclude"
-  patterns = ["IGNORE_THIS"]
-  ```
-
-- Use correct logic:
-  ```toml
-  logic = "or"   # Match ANY pattern
-  # not
-  logic = "and"  # Match ALL patterns
-  ```
-
-### Logs Dropping
-
-**Symptoms:**
-- `dropped_entries` counter increasing
-- Missing log entries in stream
-
-**Diagnosis:**
-
-1. Check drop statistics:
-   ```bash
-   curl -s http://localhost:8080/status | jq '{
-     dropped: .monitor.dropped_entries,
-     total: .monitor.total_entries,
-     percent: (.monitor.dropped_entries / .monitor.total_entries * 100)
-   }'
-   ```
-
-2. Monitor drop rate:
-   ```bash
-   watch -n 5 'curl -s http://localhost:8080/status | jq .monitor.dropped_entries'
-   ```
-
-**Solutions:**
-
-- Increase buffer sizes:
-  ```toml
-  [streams.httpserver]
-  buffer_size = 5000  # Was 1000
-  ```
-
-- Add flow control:
-  ```toml
-  [streams.monitor]
-  check_interval_ms = 500  # Slow down reading
-  ```
-
-- Reduce clients:
-  ```toml
-  [streams.httpserver.rate_limit]
-  max_total_connections = 50
-  ```
-
-## Performance Issues
-
-### Slow Response Times
-
-**Diagnosis:**
-```bash
-# Measure response time
-time curl -s http://localhost:8080/status > /dev/null
-
-# Check system load
-uptime
-top
-```
-
-**Solutions:**
-- Reduce concurrent operations
-- Increase system resources
-- Use TCP instead of HTTP for high volume
-
-### Network Bandwidth
-
-**Diagnosis:**
-```bash
-# Monitor network usage
-iftop -i eth0 -f "port 8080"
-
-# Check connection count
-ss -tan | grep :8080 | wc -l
-```
-
-**Solutions:**
-- Enable compression (future feature)
-- Filter more aggressively
-- Use TCP for local connections
-
-## Debug Commands
-
-### System Information
-
-```bash
-# LogWisp version
-logwisp --version
-
-# System resources
-free -h
-df -h
-ulimit -a
-
-# Network state
-ss -tlnp
-netstat -anp | grep logwisp
-```
-
-### Process Inspection
-
-```bash
-# Process details
-ps aux | grep logwisp
-
-# Open files
-lsof -p $(pgrep logwisp)
-
-# System calls (Linux)
-strace -p $(pgrep logwisp) -e trace=open,read,write
-
-# File system activity
-inotifywait -m /var/log/myapp/
-```
-
-### Configuration Validation
-
-```bash
-# Test configuration
-logwisp --config test.toml --log-level debug --log-output stderr
-
-# Check file syntax
-cat /etc/logwisp/logwisp.toml | grep -E "^\s*\["
-
-# Validate TOML
-python3 -m pip install toml
-python3 -c "import toml; toml.load('/etc/logwisp/logwisp.toml'); print('Valid')"
-```
-
-## Getting Help
-
-### Collect Diagnostic Information
-
-Create a diagnostic bundle:
-
-```bash
-#!/bin/bash
-# diagnostic.sh
-
-DIAG_DIR="logwisp-diag-$(date +%Y%m%d-%H%M%S)"
-mkdir -p "$DIAG_DIR"
-
-# Version
-logwisp --version > "$DIAG_DIR/version.txt" 2>&1
-
-# Configuration (sanitized)
-grep -v "password\|secret\|token" /etc/logwisp/logwisp.toml > "$DIAG_DIR/config.toml"
-
-# Status
-curl -s http://localhost:8080/status > "$DIAG_DIR/status.json"
-
-# System info
-uname -a > "$DIAG_DIR/system.txt"
-free -h >> "$DIAG_DIR/system.txt"
-df -h >> "$DIAG_DIR/system.txt"
-
-# Process info
-ps aux | grep logwisp > "$DIAG_DIR/process.txt"
-lsof -p $(pgrep logwisp) > "$DIAG_DIR/files.txt" 2>&1
-
-# Recent logs
-journalctl -u logwisp -n 1000 > "$DIAG_DIR/logs.txt" 2>&1
-
-# Create archive
-tar -czf "$DIAG_DIR.tar.gz" "$DIAG_DIR"
-rm -rf "$DIAG_DIR"
-
-echo "Diagnostic bundle created: $DIAG_DIR.tar.gz"
-```
-
-### Report Issues
-
-When reporting issues, include:
-1. LogWisp version
-2. Configuration (sanitized)
-3. Error messages
-4. Steps to reproduce
-5. Diagnostic bundle
-
-## See Also
-
-- [Monitoring Guide](monitoring.md) - Status and metrics
-- [Performance Tuning](performance.md) - Optimization
-- [Configuration Guide](configuration.md) - Settings reference
-- [FAQ](faq.md) - Frequently asked questions

src/cmd/logwisp/bootstrap.go

@@ -18,7 +18,7 @@ import (
 
 // bootstrapService creates and initializes the log transport service
 func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service, *service.HTTPRouter, error) {
-	// Create log transport service
+	// Create service
 	svc := service.New(ctx, logger)
 
 	// Create HTTP router if requested
@@ -28,75 +28,46 @@ func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service
 		logger.Info("msg", "HTTP router mode enabled")
 	}
 
-	// Initialize streams
+	// Initialize pipelines
 	successCount := 0
-	for _, streamCfg := range cfg.Streams {
+	for _, pipelineCfg := range cfg.Pipelines {
-		logger.Info("msg", "Initializing transport", "transport", streamCfg.Name)
+		logger.Info("msg", "Initializing pipeline", "pipeline", pipelineCfg.Name)
 
-		// Handle router mode configuration
+		// Create the pipeline
-		if *useRouter && streamCfg.HTTPServer != nil && streamCfg.HTTPServer.Enabled {
+		if err := svc.NewPipeline(pipelineCfg); err != nil {
-			if err := initializeRouterStream(svc, router, streamCfg); err != nil {
+			logger.Error("msg", "Failed to create pipeline",
-				logger.Error("msg", "Failed to initialize router stream",
+				"pipeline", pipelineCfg.Name,
-					"transport", streamCfg.Name,
 				"error", err)
 			continue
 		}
-		} else {
+
-			// Standard standalone mode
+		// If using router mode, register HTTP sinks
-			if err := svc.CreateStream(streamCfg); err != nil {
+		if *useRouter {
-				logger.Error("msg", "Failed to create transport",
+			pipeline, err := svc.GetPipeline(pipelineCfg.Name)
-					"transport", streamCfg.Name,
+			if err == nil && len(pipeline.HTTPSinks) > 0 {
+				if err := router.RegisterPipeline(pipeline); err != nil {
+					logger.Error("msg", "Failed to register pipeline with router",
+						"pipeline", pipelineCfg.Name,
 						"error", err)
-				continue
+				}
 			}
 		}
 
 		successCount++
-		displayStreamEndpoints(streamCfg, *useRouter)
+		displayPipelineEndpoints(pipelineCfg, *useRouter)
 	}
 
 	if successCount == 0 {
-		return nil, nil, fmt.Errorf("no streams successfully started (attempted %d)", len(cfg.Streams))
+		return nil, nil, fmt.Errorf("no pipelines successfully started (attempted %d)", len(cfg.Pipelines))
 	}
 
 	logger.Info("msg", "LogWisp started",
 		"version", version.Short(),
-		"transports", successCount)
+		"pipelines", successCount)
 
 	return svc, router, nil
 }
 
-// initializeRouterStream sets up a stream for router mode
-func initializeRouterStream(svc *service.Service, router *service.HTTPRouter, streamCfg config.StreamConfig) error {
-	// Temporarily disable standalone server startup
-	originalEnabled := streamCfg.HTTPServer.Enabled
-	streamCfg.HTTPServer.Enabled = false
-
-	if err := svc.CreateStream(streamCfg); err != nil {
-		return err
-	}
-
-	// Get the created transport and configure for router mode
-	stream, err := svc.GetStream(streamCfg.Name)
-	if err != nil {
-		return err
-	}
-
-	if stream.HTTPServer != nil {
-		stream.HTTPServer.SetRouterMode()
-		// Restore enabled state
-		stream.Config.HTTPServer.Enabled = originalEnabled
-
-		if err := router.RegisterStream(stream); err != nil {
-			return err
-		}
-
-		logger.Info("msg", "Stream registered with router", "stream", streamCfg.Name)
-	}
-
-	return nil
-}
-
 // initializeLogger sets up the logger based on configuration and CLI flags
 func initializeLogger(cfg *config.Config) error {
 	logger = log.NewLogger()

src/cmd/logwisp/flags.go

@@ -31,7 +31,7 @@ func init() {
 }
 
 func customUsage() {
-	fmt.Fprintf(os.Stderr, "LogWisp - Multi-Stream Log Monitoring Service\n\n")
+	fmt.Fprintf(os.Stderr, "LogWisp - Multi-Pipeline Log Processing Service\n\n")
 	fmt.Fprintf(os.Stderr, "Usage: %s [options]\n\n", os.Args[0])
 	fmt.Fprintf(os.Stderr, "Options:\n")
 
@@ -63,8 +63,8 @@ func customUsage() {
 	fmt.Fprintf(os.Stderr, "  # Run with custom config and override log level\n")
 	fmt.Fprintf(os.Stderr, "  %s --config /etc/logwisp.toml --log-level warn\n\n", os.Args[0])
 
-	fmt.Fprintf(os.Stderr, "  # Run in router mode with multiple streams\n")
+	fmt.Fprintf(os.Stderr, "  # Run in router mode with multiple pipelines\n")
-	fmt.Fprintf(os.Stderr, "  %s --router --config /etc/logwisp/multi-stream.toml\n\n", os.Args[0])
+	fmt.Fprintf(os.Stderr, "  %s --router --config /etc/logwisp/multi-pipeline.toml\n\n", os.Args[0])
 
 	fmt.Fprintf(os.Stderr, "Environment Variables:\n")
 	fmt.Fprintf(os.Stderr, "  LOGWISP_CONFIG_FILE              Config file path\n")

src/cmd/logwisp/main.go

@@ -81,7 +81,7 @@ func main() {
 	}
 
 	// Start status reporter if enabled
-	if shouldEnableStatusReporter() {
+	if enableStatusReporter() {
 		go statusReporter(svc)
 	}
 
@@ -123,7 +123,7 @@ func shutdownLogger() {
 	}
 }
 
-func shouldEnableStatusReporter() bool {
+func enableStatusReporter() bool {
 	// Status reporter can be disabled via environment variable
 	if os.Getenv("LOGWISP_DISABLE_STATUS_REPORTER") == "1" {
 		return false

src/cmd/logwisp/status.go

@@ -16,9 +16,9 @@ func statusReporter(service *service.Service) {
 
 	for range ticker.C {
 		stats := service.GetGlobalStats()
-		totalStreams := stats["total_streams"].(int)
+		totalPipelines := stats["total_pipelines"].(int)
-		if totalStreams == 0 {
+		if totalPipelines == 0 {
-			logger.Warn("msg", "No active streams in status report",
+			logger.Warn("msg", "No active pipelines in status report",
 				"component", "status_reporter")
 			return
 		}
@@ -26,94 +26,171 @@ func statusReporter(service *service.Service) {
 		// Log status at DEBUG level to avoid cluttering INFO logs
 		logger.Debug("msg", "Status report",
 			"component", "status_reporter",
-			"active_streams", totalStreams,
+			"active_pipelines", totalPipelines,
 			"time", time.Now().Format("15:04:05"))
 
-		// Log individual stream status
+		// Log individual pipeline status
-		for name, streamStats := range stats["streams"].(map[string]interface{}) {
+		pipelines := stats["pipelines"].(map[string]any)
-			logStreamStatus(name, streamStats.(map[string]interface{}))
+		for name, pipelineStats := range pipelines {
+			logPipelineStatus(name, pipelineStats.(map[string]any))
 		}
 	}
 }
 
-// logStreamStatus logs the status of an individual stream
+// logPipelineStatus logs the status of an individual pipeline
-func logStreamStatus(name string, stats map[string]interface{}) {
+func logPipelineStatus(name string, stats map[string]any) {
-	statusFields := []interface{}{
+	statusFields := []any{
-		"msg", "Stream status",
+		"msg", "Pipeline status",
-		"stream", name,
+		"pipeline", name,
 	}
 
-	// Add monitor statistics
+	// Add processing statistics
-	if monitor, ok := stats["monitor"].(map[string]interface{}); ok {
+	if totalProcessed, ok := stats["total_processed"].(uint64); ok {
-		statusFields = append(statusFields,
+		statusFields = append(statusFields, "entries_processed", totalProcessed)
-			"watchers", monitor["active_watchers"],
+	}
-			"entries", monitor["total_entries"])
+	if totalFiltered, ok := stats["total_filtered"].(uint64); ok {
+		statusFields = append(statusFields, "entries_filtered", totalFiltered)
 	}
 
-	// Add TCP server statistics
+	// Add source count
-	if tcp, ok := stats["tcp"].(map[string]interface{}); ok && tcp["enabled"].(bool) {
+	if sourceCount, ok := stats["source_count"].(int); ok {
-		statusFields = append(statusFields, "tcp_conns", tcp["connections"])
+		statusFields = append(statusFields, "sources", sourceCount)
 	}
 
-	// Add HTTP server statistics
+	// Add sink statistics
-	if http, ok := stats["http"].(map[string]interface{}); ok && http["enabled"].(bool) {
+	if sinks, ok := stats["sinks"].([]map[string]any); ok {
-		statusFields = append(statusFields, "http_conns", http["connections"])
+		tcpConns := 0
+		httpConns := 0
+
+		for _, sink := range sinks {
+			sinkType := sink["type"].(string)
+			if activeConns, ok := sink["active_connections"].(int32); ok {
+				switch sinkType {
+				case "tcp":
+					tcpConns += int(activeConns)
+				case "http":
+					httpConns += int(activeConns)
+				}
+			}
+		}
+
+		if tcpConns > 0 {
+			statusFields = append(statusFields, "tcp_connections", tcpConns)
+		}
+		if httpConns > 0 {
+			statusFields = append(statusFields, "http_connections", httpConns)
+		}
 	}
 
 	logger.Debug(statusFields...)
 }
 
-// displayStreamEndpoints logs the configured endpoints for a stream
+// displayPipelineEndpoints logs the configured endpoints for a pipeline
-func displayStreamEndpoints(cfg config.StreamConfig, routerMode bool) {
+func displayPipelineEndpoints(cfg config.PipelineConfig, routerMode bool) {
-	// Display TCP endpoints
+	// Display sink endpoints
-	if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
+	for i, sinkCfg := range cfg.Sinks {
+		switch sinkCfg.Type {
+		case "tcp":
+			if port, ok := toInt(sinkCfg.Options["port"]); ok {
 				logger.Info("msg", "TCP endpoint configured",
 					"component", "main",
-			"transport", cfg.Name,
+					"pipeline", cfg.Name,
-			"port", cfg.TCPServer.Port)
+					"sink_index", i,
+					"port", port)
 
-		if cfg.TCPServer.RateLimit != nil && cfg.TCPServer.RateLimit.Enabled {
+				// Display rate limit info if configured
+				if rl, ok := sinkCfg.Options["rate_limit"].(map[string]any); ok {
+					if enabled, ok := rl["enabled"].(bool); ok && enabled {
 						logger.Info("msg", "TCP rate limiting enabled",
-				"transport", cfg.Name,
+							"pipeline", cfg.Name,
-				"requests_per_second", cfg.TCPServer.RateLimit.RequestsPerSecond,
+							"sink_index", i,
-				"burst_size", cfg.TCPServer.RateLimit.BurstSize)
+							"requests_per_second", rl["requests_per_second"],
+							"burst_size", rl["burst_size"])
+					}
 				}
 			}
 
-	// Display HTTP endpoints
+		case "http":
-	if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
+			if port, ok := toInt(sinkCfg.Options["port"]); ok {
+				streamPath := "/transport"
+				statusPath := "/status"
+				if path, ok := sinkCfg.Options["stream_path"].(string); ok {
+					streamPath = path
+				}
+				if path, ok := sinkCfg.Options["status_path"].(string); ok {
+					statusPath = path
+				}
+
 				if routerMode {
 					logger.Info("msg", "HTTP endpoints configured",
-				"transport", cfg.Name,
+						"pipeline", cfg.Name,
-				"stream_path", fmt.Sprintf("/%s%s", cfg.Name, cfg.HTTPServer.StreamPath),
+						"sink_index", i,
-				"status_path", fmt.Sprintf("/%s%s", cfg.Name, cfg.HTTPServer.StatusPath))
+						"stream_path", fmt.Sprintf("/%s%s", cfg.Name, streamPath),
+						"status_path", fmt.Sprintf("/%s%s", cfg.Name, statusPath))
 				} else {
 					logger.Info("msg", "HTTP endpoints configured",
-				"transport", cfg.Name,
+						"pipeline", cfg.Name,
-				"stream_url", fmt.Sprintf("http://localhost:%d%s", cfg.HTTPServer.Port, cfg.HTTPServer.StreamPath),
+						"sink_index", i,
-				"status_url", fmt.Sprintf("http://localhost:%d%s", cfg.HTTPServer.Port, cfg.HTTPServer.StatusPath))
+						"stream_url", fmt.Sprintf("http://localhost:%d%s", port, streamPath),
+						"status_url", fmt.Sprintf("http://localhost:%d%s", port, statusPath))
 				}
 
-		if cfg.HTTPServer.RateLimit != nil && cfg.HTTPServer.RateLimit.Enabled {
+				// Display rate limit info if configured
+				if rl, ok := sinkCfg.Options["rate_limit"].(map[string]any); ok {
+					if enabled, ok := rl["enabled"].(bool); ok && enabled {
 						logger.Info("msg", "HTTP rate limiting enabled",
-				"transport", cfg.Name,
+							"pipeline", cfg.Name,
-				"requests_per_second", cfg.HTTPServer.RateLimit.RequestsPerSecond,
+							"sink_index", i,
-				"burst_size", cfg.HTTPServer.RateLimit.BurstSize,
+							"requests_per_second", rl["requests_per_second"],
-				"limit_by", cfg.HTTPServer.RateLimit.LimitBy)
+							"burst_size", rl["burst_size"],
+							"limit_by", rl["limit_by"])
+					}
+				}
+			}
+
+		case "file":
+			if dir, ok := sinkCfg.Options["directory"].(string); ok {
+				name, _ := sinkCfg.Options["name"].(string)
+				logger.Info("msg", "File sink configured",
+					"pipeline", cfg.Name,
+					"sink_index", i,
+					"directory", dir,
+					"name", name)
+			}
+
+		case "stdout", "stderr":
+			logger.Info("msg", "Console sink configured",
+				"pipeline", cfg.Name,
+				"sink_index", i,
+				"type", sinkCfg.Type)
+		}
 	}
 
 	// Display authentication information
 	if cfg.Auth != nil && cfg.Auth.Type != "none" {
 		logger.Info("msg", "Authentication enabled",
-				"transport", cfg.Name,
+			"pipeline", cfg.Name,
 			"auth_type", cfg.Auth.Type)
 	}
-	}
 
 	// Display filter information
 	if len(cfg.Filters) > 0 {
 		logger.Info("msg", "Filters configured",
-			"transport", cfg.Name,
+			"pipeline", cfg.Name,
 			"filter_count", len(cfg.Filters))
 	}
 }
+
+// Helper function for type conversion
+func toInt(v any) (int, bool) {
+	switch val := v.(type) {
+	case int:
+		return val, true
+	case int64:
+		return int(val), true
+	case float64:
+		return int(val), true
+	default:
+		return 0, false
+	}
+}

src/internal/config/auth.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/auth.go
 package config
 
+import "fmt"
+
 type AuthConfig struct {
 	// Authentication type: "none", "basic", "bearer", "mtls"
 	Type string `toml:"type"`
@@ -54,3 +56,24 @@ type JWTConfig struct {
 	// Expected audience
 	Audience string `toml:"audience"`
 }
+
+func validateAuth(pipelineName string, auth *AuthConfig) error {
+	if auth == nil {
+		return nil
+	}
+
+	validTypes := map[string]bool{"none": true, "basic": true, "bearer": true, "mtls": true}
+	if !validTypes[auth.Type] {
+		return fmt.Errorf("pipeline '%s': invalid auth type: %s", pipelineName, auth.Type)
+	}
+
+	if auth.Type == "basic" && auth.BasicAuth == nil {
+		return fmt.Errorf("pipeline '%s': basic auth type specified but config missing", pipelineName)
+	}
+
+	if auth.Type == "bearer" && auth.BearerAuth == nil {
+		return fmt.Errorf("pipeline '%s': bearer auth type specified but config missing", pipelineName)
+	}
+
+	return nil
+}

src/internal/config/config.go

@@ -5,10 +5,33 @@ type Config struct {
 	// Logging configuration
 	Logging *LogConfig `toml:"logging"`
 
-	// Stream configurations
+	// Pipeline configurations
-	Streams []StreamConfig `toml:"streams"`
+	Pipelines []PipelineConfig `toml:"pipelines"`
 }
 
-type MonitorConfig struct {
+// Helper functions to handle type conversions from any
-	CheckIntervalMs int `toml:"check_interval_ms"`
+func toInt(v any) (int, bool) {
+	switch val := v.(type) {
+	case int:
+		return val, true
+	case int64:
+		return int(val), true
+	case float64:
+		return int(val), true
+	default:
+		return 0, false
+	}
+}
+
+func toFloat(v any) (float64, bool) {
+	switch val := v.(type) {
+	case float64:
+		return val, true
+	case int:
+		return float64(val), true
+	case int64:
+		return float64(val), true
+	default:
+		return 0, false
+	}
 }

src/internal/config/filter.go

@@ -0,0 +1,44 @@
+// FILE: src/internal/config/filter.go
+package config
+
+import (
+	"fmt"
+	"regexp"
+
+	"logwisp/src/internal/filter"
+)
+
+func validateFilter(pipelineName string, filterIndex int, cfg *filter.Config) error {
+	// Validate filter type
+	switch cfg.Type {
+	case filter.TypeInclude, filter.TypeExclude, "":
+		// Valid types
+	default:
+		return fmt.Errorf("pipeline '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
+			pipelineName, filterIndex, cfg.Type)
+	}
+
+	// Validate filter logic
+	switch cfg.Logic {
+	case filter.LogicOr, filter.LogicAnd, "":
+		// Valid logic
+	default:
+		return fmt.Errorf("pipeline '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
+			pipelineName, filterIndex, cfg.Logic)
+	}
+
+	// Empty patterns is valid - passes everything
+	if len(cfg.Patterns) == 0 {
+		return nil
+	}
+
+	// Validate regex patterns
+	for i, pattern := range cfg.Patterns {
+		if _, err := regexp.Compile(pattern); err != nil {
+			return fmt.Errorf("pipeline '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
+				pipelineName, filterIndex, i, pattern, err)
+		}
+	}
+
+	return nil
+}

src/internal/config/loader.go

@@ -13,27 +13,35 @@ import (
 func defaults() *Config {
 	return &Config{
 		Logging: DefaultLogConfig(),
-		Streams: []StreamConfig{
+		Pipelines: []PipelineConfig{
 			{
 				Name: "default",
-				Monitor: &StreamMonitorConfig{
+				Sources: []SourceConfig{
-					CheckIntervalMs: 100,
+					{
-					Targets: []MonitorTarget{
+						Type: "directory",
-						{Path: "./", Pattern: "*.log", IsFile: false},
+						Options: map[string]any{
+							"path":              "./",
+							"pattern":           "*.log",
+							"check_interval_ms": 100,
+						},
+					},
+				},
+				Sinks: []SinkConfig{
+					{
+						Type: "http",
+						Options: map[string]any{
+							"port":        8080,
+							"buffer_size": 1000,
+							"stream_path": "/transport",
+							"status_path": "/status",
+							"heartbeat": map[string]any{
+								"enabled":           true,
+								"interval_seconds":  30,
+								"include_timestamp": true,
+								"include_stats":     false,
+								"format":            "comment",
 							},
 						},
-				HTTPServer: &HTTPConfig{
-					Enabled:    true,
-					Port:       8080,
-					BufferSize: 1000,
-					StreamPath: "/transport",
-					StatusPath: "/status",
-					Heartbeat: HeartbeatConfig{
-						Enabled:          true,
-						IntervalSeconds:  30,
-						IncludeTimestamp: true,
-						IncludeStats:     false,
-						Format:           "comment",
 					},
 				},
 			},

src/internal/config/logging.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/logging.go
 package config
 
+import "fmt"
+
 // LogConfig represents logging configuration for LogWisp
 type LogConfig struct {
 	// Output mode: "file", "stdout", "stderr", "both", "none"
@@ -60,3 +62,31 @@ func DefaultLogConfig() *LogConfig {
 		},
 	}
 }
+
+func validateLogConfig(cfg *LogConfig) error {
+	validOutputs := map[string]bool{
+		"file": true, "stdout": true, "stderr": true,
+		"both": true, "none": true,
+	}
+	if !validOutputs[cfg.Output] {
+		return fmt.Errorf("invalid log output mode: %s", cfg.Output)
+	}
+
+	validLevels := map[string]bool{
+		"debug": true, "info": true, "warn": true, "error": true,
+	}
+	if !validLevels[cfg.Level] {
+		return fmt.Errorf("invalid log level: %s", cfg.Level)
+	}
+
+	if cfg.Console != nil {
+		validTargets := map[string]bool{
+			"stdout": true, "stderr": true, "split": true,
+		}
+		if !validTargets[cfg.Console.Target] {
+			return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
+		}
+	}
+
+	return nil
+}

src/internal/config/pipeline.go

@@ -0,0 +1,276 @@
+// FILE: src/internal/config/pipeline.go
+package config
+
+import (
+	"fmt"
+	"logwisp/src/internal/filter"
+	"path/filepath"
+	"strings"
+)
+
+// PipelineConfig represents a data processing pipeline
+type PipelineConfig struct {
+	// Pipeline identifier (used in logs and metrics)
+	Name string `toml:"name"`
+
+	// Data sources for this pipeline
+	Sources []SourceConfig `toml:"sources"`
+
+	// Filter configuration
+	Filters []filter.Config `toml:"filters"`
+
+	// Output sinks for this pipeline
+	Sinks []SinkConfig `toml:"sinks"`
+
+	// Authentication/Authorization (applies to network sinks)
+	Auth *AuthConfig `toml:"auth"`
+}
+
+// SourceConfig represents an input data source
+type SourceConfig struct {
+	// Source type: "directory", "file", "stdin", etc.
+	Type string `toml:"type"`
+
+	// Type-specific configuration options
+	Options map[string]any `toml:"options"`
+
+	// Placeholder for future source-side rate limiting
+	// This will be used for features like aggregation and summarization
+	RateLimit *RateLimitConfig `toml:"rate_limit"`
+}
+
+// SinkConfig represents an output destination
+type SinkConfig struct {
+	// Sink type: "http", "tcp", "file", "stdout", "stderr"
+	Type string `toml:"type"`
+
+	// Type-specific configuration options
+	Options map[string]any `toml:"options"`
+}
+
+func validateSource(pipelineName string, sourceIndex int, cfg *SourceConfig) error {
+	if cfg.Type == "" {
+		return fmt.Errorf("pipeline '%s' source[%d]: missing type", pipelineName, sourceIndex)
+	}
+
+	switch cfg.Type {
+	case "directory":
+		// Validate directory source options
+		path, ok := cfg.Options["path"].(string)
+		if !ok || path == "" {
+			return fmt.Errorf("pipeline '%s' source[%d]: directory source requires 'path' option",
+				pipelineName, sourceIndex)
+		}
+
+		// Check for directory traversal
+		if strings.Contains(path, "..") {
+			return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal",
+				pipelineName, sourceIndex)
+		}
+
+		// Validate pattern if provided
+		if pattern, ok := cfg.Options["pattern"].(string); ok && pattern != "" {
+			// Try to compile as glob pattern (will be converted to regex internally)
+			if strings.Count(pattern, "*") == 0 && strings.Count(pattern, "?") == 0 {
+				// If no wildcards, ensure it's a valid filename
+				if filepath.Base(pattern) != pattern {
+					return fmt.Errorf("pipeline '%s' source[%d]: pattern contains path separators",
+						pipelineName, sourceIndex)
+				}
+			}
+		}
+
+		// Validate check interval if provided
+		if interval, ok := cfg.Options["check_interval_ms"]; ok {
+			if intVal, ok := toInt(interval); ok {
+				if intVal < 10 {
+					return fmt.Errorf("pipeline '%s' source[%d]: check interval too small: %d ms (min: 10ms)",
+						pipelineName, sourceIndex, intVal)
+				}
+			} else {
+				return fmt.Errorf("pipeline '%s' source[%d]: invalid check_interval_ms type",
+					pipelineName, sourceIndex)
+			}
+		}
+
+	case "file":
+		// Validate file source options
+		path, ok := cfg.Options["path"].(string)
+		if !ok || path == "" {
+			return fmt.Errorf("pipeline '%s' source[%d]: file source requires 'path' option",
+				pipelineName, sourceIndex)
+		}
+
+		// Check for directory traversal
+		if strings.Contains(path, "..") {
+			return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal",
+				pipelineName, sourceIndex)
+		}
+
+	case "stdin":
+		// No specific validation needed for stdin
+
+	default:
+		return fmt.Errorf("pipeline '%s' source[%d]: unknown source type '%s'",
+			pipelineName, sourceIndex, cfg.Type)
+	}
+
+	// Note: RateLimit field is ignored for now as it's a placeholder
+	return nil
+}
+
+func validateSink(pipelineName string, sinkIndex int, cfg *SinkConfig, allPorts map[int]string) error {
+	if cfg.Type == "" {
+		return fmt.Errorf("pipeline '%s' sink[%d]: missing type", pipelineName, sinkIndex)
+	}
+
+	switch cfg.Type {
+	case "http":
+		// Extract and validate HTTP configuration
+		port, ok := toInt(cfg.Options["port"])
+		if !ok || port < 1 || port > 65535 {
+			return fmt.Errorf("pipeline '%s' sink[%d]: invalid or missing HTTP port",
+				pipelineName, sinkIndex)
+		}
+
+		// Check port conflicts
+		if existing, exists := allPorts[port]; exists {
+			return fmt.Errorf("pipeline '%s' sink[%d]: HTTP port %d already used by %s",
+				pipelineName, sinkIndex, port, existing)
+		}
+		allPorts[port] = fmt.Sprintf("%s-http[%d]", pipelineName, sinkIndex)
+
+		// Validate buffer size
+		if bufSize, ok := toInt(cfg.Options["buffer_size"]); ok {
+			if bufSize < 1 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: HTTP buffer size must be positive: %d",
+					pipelineName, sinkIndex, bufSize)
+			}
+		}
+
+		// Validate paths if provided
+		if streamPath, ok := cfg.Options["stream_path"].(string); ok {
+			if !strings.HasPrefix(streamPath, "/") {
+				return fmt.Errorf("pipeline '%s' sink[%d]: stream path must start with /: %s",
+					pipelineName, sinkIndex, streamPath)
+			}
+		}
+
+		if statusPath, ok := cfg.Options["status_path"].(string); ok {
+			if !strings.HasPrefix(statusPath, "/") {
+				return fmt.Errorf("pipeline '%s' sink[%d]: status path must start with /: %s",
+					pipelineName, sinkIndex, statusPath)
+			}
+		}
+
+		// Validate heartbeat if present
+		if hb, ok := cfg.Options["heartbeat"].(map[string]any); ok {
+			if err := validateHeartbeatOptions("HTTP", pipelineName, sinkIndex, hb); err != nil {
+				return err
+			}
+		}
+
+		// Validate SSL if present
+		if ssl, ok := cfg.Options["ssl"].(map[string]any); ok {
+			if err := validateSSLOptions("HTTP", pipelineName, sinkIndex, ssl); err != nil {
+				return err
+			}
+		}
+
+		// Validate rate limit if present
+		if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
+			if err := validateRateLimitOptions("HTTP", pipelineName, sinkIndex, rl); err != nil {
+				return err
+			}
+		}
+
+	case "tcp":
+		// Extract and validate TCP configuration
+		port, ok := toInt(cfg.Options["port"])
+		if !ok || port < 1 || port > 65535 {
+			return fmt.Errorf("pipeline '%s' sink[%d]: invalid or missing TCP port",
+				pipelineName, sinkIndex)
+		}
+
+		// Check port conflicts
+		if existing, exists := allPorts[port]; exists {
+			return fmt.Errorf("pipeline '%s' sink[%d]: TCP port %d already used by %s",
+				pipelineName, sinkIndex, port, existing)
+		}
+		allPorts[port] = fmt.Sprintf("%s-tcp[%d]", pipelineName, sinkIndex)
+
+		// Validate buffer size
+		if bufSize, ok := toInt(cfg.Options["buffer_size"]); ok {
+			if bufSize < 1 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: TCP buffer size must be positive: %d",
+					pipelineName, sinkIndex, bufSize)
+			}
+		}
+
+		// Validate heartbeat if present
+		if hb, ok := cfg.Options["heartbeat"].(map[string]any); ok {
+			if err := validateHeartbeatOptions("TCP", pipelineName, sinkIndex, hb); err != nil {
+				return err
+			}
+		}
+
+		// Validate SSL if present
+		if ssl, ok := cfg.Options["ssl"].(map[string]any); ok {
+			if err := validateSSLOptions("TCP", pipelineName, sinkIndex, ssl); err != nil {
+				return err
+			}
+		}
+
+		// Validate rate limit if present
+		if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
+			if err := validateRateLimitOptions("TCP", pipelineName, sinkIndex, rl); err != nil {
+				return err
+			}
+		}
+
+	case "file":
+		// Validate file sink options
+		directory, ok := cfg.Options["directory"].(string)
+		if !ok || directory == "" {
+			return fmt.Errorf("pipeline '%s' sink[%d]: file sink requires 'directory' option",
+				pipelineName, sinkIndex)
+		}
+
+		name, ok := cfg.Options["name"].(string)
+		if !ok || name == "" {
+			return fmt.Errorf("pipeline '%s' sink[%d]: file sink requires 'name' option",
+				pipelineName, sinkIndex)
+		}
+
+		// Validate numeric options
+		if maxSize, ok := toInt(cfg.Options["max_size_mb"]); ok {
+			if maxSize < 1 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: max_size_mb must be positive: %d",
+					pipelineName, sinkIndex, maxSize)
+			}
+		}
+
+		if maxTotalSize, ok := toInt(cfg.Options["max_total_size_mb"]); ok {
+			if maxTotalSize < 0 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: max_total_size_mb cannot be negative: %d",
+					pipelineName, sinkIndex, maxTotalSize)
+			}
+		}
+
+		if retention, ok := toFloat(cfg.Options["retention_hours"]); ok {
+			if retention < 0 {
+				return fmt.Errorf("pipeline '%s' sink[%d]: retention_hours cannot be negative: %f",
+					pipelineName, sinkIndex, retention)
+			}
+		}
+
+	case "stdout", "stderr":
+		// No specific validation needed for console sinks
+
+	default:
+		return fmt.Errorf("pipeline '%s' sink[%d]: unknown sink type '%s'",
+			pipelineName, sinkIndex, cfg.Type)
+	}
+
+	return nil
+}

src/internal/config/server.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/server.go
 package config
 
+import "fmt"
+
 type TCPConfig struct {
 	Enabled    bool `toml:"enabled"`
 	Port       int  `toml:"port"`
@@ -64,3 +66,71 @@ type RateLimitConfig struct {
 	MaxConnectionsPerIP int `toml:"max_connections_per_ip"`
 	MaxTotalConnections int `toml:"max_total_connections"`
 }
+
+func validateHeartbeatOptions(serverType, pipelineName string, sinkIndex int, hb map[string]any) error {
+	if enabled, ok := hb["enabled"].(bool); ok && enabled {
+		interval, ok := toInt(hb["interval_seconds"])
+		if !ok || interval < 1 {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: heartbeat interval must be positive",
+				pipelineName, sinkIndex, serverType)
+		}
+
+		if format, ok := hb["format"].(string); ok {
+			if format != "json" && format != "comment" {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: heartbeat format must be 'json' or 'comment': %s",
+					pipelineName, sinkIndex, serverType, format)
+			}
+		}
+	}
+	return nil
+}
+
+func validateRateLimitOptions(serverType, pipelineName string, sinkIndex int, rl map[string]any) error {
+	if enabled, ok := rl["enabled"].(bool); !ok || !enabled {
+		return nil
+	}
+
+	// Validate requests per second
+	rps, ok := toFloat(rl["requests_per_second"])
+	if !ok || rps <= 0 {
+		return fmt.Errorf("pipeline '%s' sink[%d] %s: requests_per_second must be positive",
+			pipelineName, sinkIndex, serverType)
+	}
+
+	// Validate burst size
+	burst, ok := toInt(rl["burst_size"])
+	if !ok || burst < 1 {
+		return fmt.Errorf("pipeline '%s' sink[%d] %s: burst_size must be at least 1",
+			pipelineName, sinkIndex, serverType)
+	}
+
+	// Validate limit_by
+	if limitBy, ok := rl["limit_by"].(string); ok && limitBy != "" {
+		validLimitBy := map[string]bool{"ip": true, "global": true}
+		if !validLimitBy[limitBy] {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid limit_by value: %s (must be 'ip' or 'global')",
+				pipelineName, sinkIndex, serverType, limitBy)
+		}
+	}
+
+	// Validate response code
+	if respCode, ok := toInt(rl["response_code"]); ok {
+		if respCode > 0 && (respCode < 400 || respCode >= 600) {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: response_code must be 4xx or 5xx: %d",
+				pipelineName, sinkIndex, serverType, respCode)
+		}
+	}
+
+	// Validate connection limits
+	maxPerIP, perIPOk := toInt(rl["max_connections_per_ip"])
+	maxTotal, totalOk := toInt(rl["max_total_connections"])
+
+	if perIPOk && totalOk && maxPerIP > 0 && maxTotal > 0 {
+		if maxPerIP > maxTotal {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: max_connections_per_ip (%d) cannot exceed max_total_connections (%d)",
+				pipelineName, sinkIndex, serverType, maxPerIP, maxTotal)
+		}
+	}
+
+	return nil
+}

src/internal/config/ssl.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/ssl.go
 package config
 
+import "fmt"
+
 type SSLConfig struct {
 	Enabled  bool   `toml:"enabled"`
 	CertFile string `toml:"cert_file"`
@@ -18,3 +20,38 @@ type SSLConfig struct {
 	// Cipher suites (comma-separated list)
 	CipherSuites string `toml:"cipher_suites"`
 }
+
+func validateSSLOptions(serverType, pipelineName string, sinkIndex int, ssl map[string]any) error {
+	if enabled, ok := ssl["enabled"].(bool); ok && enabled {
+		certFile, certOk := ssl["cert_file"].(string)
+		keyFile, keyOk := ssl["key_file"].(string)
+
+		if !certOk || certFile == "" || !keyOk || keyFile == "" {
+			return fmt.Errorf("pipeline '%s' sink[%d] %s: SSL enabled but cert/key files not specified",
+				pipelineName, sinkIndex, serverType)
+		}
+
+		if clientAuth, ok := ssl["client_auth"].(bool); ok && clientAuth {
+			if caFile, ok := ssl["client_ca_file"].(string); !ok || caFile == "" {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: client auth enabled but CA file not specified",
+					pipelineName, sinkIndex, serverType)
+			}
+		}
+
+		// Validate TLS versions
+		validVersions := map[string]bool{"TLS1.0": true, "TLS1.1": true, "TLS1.2": true, "TLS1.3": true}
+		if minVer, ok := ssl["min_version"].(string); ok && minVer != "" {
+			if !validVersions[minVer] {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid min TLS version: %s",
+					pipelineName, sinkIndex, serverType, minVer)
+			}
+		}
+		if maxVer, ok := ssl["max_version"].(string); ok && maxVer != "" {
+			if !validVersions[maxVer] {
+				return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid max TLS version: %s",
+					pipelineName, sinkIndex, serverType, maxVer)
+			}
+		}
+	}
+	return nil
+}

src/internal/config/stream.go

@@ -1,49 +0,0 @@
-// FILE: src/internal/config/transport.go
-package config
-
-import (
-	"logwisp/src/internal/filter"
-)
-
-type StreamConfig struct {
-	// Stream identifier (used in logs and metrics)
-	Name string `toml:"name"`
-
-	// Monitor configuration for this transport
-	Monitor *StreamMonitorConfig `toml:"monitor"`
-
-	// Filter configuration
-	Filters []filter.Config `toml:"filters"`
-
-	// Server configurations
-	TCPServer  *TCPConfig  `toml:"tcpserver"`
-	HTTPServer *HTTPConfig `toml:"httpserver"`
-
-	// Authentication/Authorization
-	Auth *AuthConfig `toml:"auth"`
-}
-
-type StreamMonitorConfig struct {
-	CheckIntervalMs int             `toml:"check_interval_ms"`
-	Targets         []MonitorTarget `toml:"targets"`
-}
-
-type MonitorTarget struct {
-	Path    string `toml:"path"`
-	Pattern string `toml:"pattern"`
-	IsFile  bool   `toml:"is_file"`
-}
-
-func (s *StreamConfig) GetTargets(defaultTargets []MonitorTarget) []MonitorTarget {
-	if s.Monitor != nil && len(s.Monitor.Targets) > 0 {
-		return s.Monitor.Targets
-	}
-	return nil
-}
-
-func (s *StreamConfig) GetCheckInterval(defaultInterval int) int {
-	if s.Monitor != nil && s.Monitor.CheckIntervalMs > 0 {
-		return s.Monitor.CheckIntervalMs
-	}
-	return defaultInterval
-}

src/internal/config/validation.go

@@ -3,309 +3,67 @@ package config
 
 import (
 	"fmt"
-	"regexp"
-	"strings"
-
-	"logwisp/src/internal/filter"
 )
 
 func (c *Config) validate() error {
-	if len(c.Streams) == 0 {
+	if len(c.Pipelines) == 0 {
-		return fmt.Errorf("no streams configured")
+		return fmt.Errorf("no pipelines configured")
 	}
 
 	if err := validateLogConfig(c.Logging); err != nil {
 		return fmt.Errorf("logging config: %w", err)
 	}
 
-	// Validate each transport
+	// Track used ports across all pipelines
-	streamNames := make(map[string]bool)
+	allPorts := make(map[int]string)
-	streamPorts := make(map[int]string)
+	pipelineNames := make(map[string]bool)
 
-	for i, stream := range c.Streams {
+	for i, pipeline := range c.Pipelines {
-		if stream.Name == "" {
+		if pipeline.Name == "" {
-			return fmt.Errorf("transport %d: missing name", i)
+			return fmt.Errorf("pipeline %d: missing name", i)
 		}
 
-		if streamNames[stream.Name] {
+		if pipelineNames[pipeline.Name] {
-			return fmt.Errorf("transport %d: duplicate name '%s'", i, stream.Name)
+			return fmt.Errorf("pipeline %d: duplicate name '%s'", i, pipeline.Name)
 		}
-		streamNames[stream.Name] = true
+		pipelineNames[pipeline.Name] = true
 
-		// Stream must have monitor config with targets
+		// Pipeline must have at least one source
-		if stream.Monitor == nil || len(stream.Monitor.Targets) == 0 {
+		if len(pipeline.Sources) == 0 {
-			return fmt.Errorf("transport '%s': no monitor targets specified", stream.Name)
+			return fmt.Errorf("pipeline '%s': no sources specified", pipeline.Name)
 		}
 
-		// Validate check interval
+		// Validate sources
-		if stream.Monitor.CheckIntervalMs < 10 {
+		for j, source := range pipeline.Sources {
-			return fmt.Errorf("transport '%s': check interval too small: %d ms (min: 10ms)",
+			if err := validateSource(pipeline.Name, j, &source); err != nil {
-				stream.Name, stream.Monitor.CheckIntervalMs)
+				return err
-		}
-
-		// Validate targets
-		for j, target := range stream.Monitor.Targets {
-			if target.Path == "" {
-				return fmt.Errorf("transport '%s' target %d: empty path", stream.Name, j)
-			}
-			if strings.Contains(target.Path, "..") {
-				return fmt.Errorf("transport '%s' target %d: path contains directory traversal", stream.Name, j)
 			}
 		}
 
 		// Validate filters
-		for j, filterCfg := range stream.Filters {
+		for j, filterCfg := range pipeline.Filters {
-			if err := validateFilter(stream.Name, j, &filterCfg); err != nil {
+			if err := validateFilter(pipeline.Name, j, &filterCfg); err != nil {
 				return err
 			}
 		}
 
-		// Validate TCP server
+		// Pipeline must have at least one sink
-		if stream.TCPServer != nil && stream.TCPServer.Enabled {
+		if len(pipeline.Sinks) == 0 {
-			if stream.TCPServer.Port < 1 || stream.TCPServer.Port > 65535 {
+			return fmt.Errorf("pipeline '%s': no sinks specified", pipeline.Name)
-				return fmt.Errorf("transport '%s': invalid TCP port: %d", stream.Name, stream.TCPServer.Port)
-			}
-			if existing, exists := streamPorts[stream.TCPServer.Port]; exists {
-				return fmt.Errorf("transport '%s': TCP port %d already used by transport '%s'",
-					stream.Name, stream.TCPServer.Port, existing)
-			}
-			streamPorts[stream.TCPServer.Port] = stream.Name + "-tcp"
-
-			if stream.TCPServer.BufferSize < 1 {
-				return fmt.Errorf("transport '%s': TCP buffer size must be positive: %d",
-					stream.Name, stream.TCPServer.BufferSize)
 		}
 
-			if err := validateHeartbeat("TCP", stream.Name, &stream.TCPServer.Heartbeat); err != nil {
+		// Validate sinks and check for port conflicts
+		for j, sink := range pipeline.Sinks {
+			if err := validateSink(pipeline.Name, j, &sink, allPorts); err != nil {
 				return err
 			}
-
-			if err := validateSSL("TCP", stream.Name, stream.TCPServer.SSL); err != nil {
-				return err
-			}
-
-			if err := validateRateLimit("TCP", stream.Name, stream.TCPServer.RateLimit); err != nil {
-				return err
-			}
-		}
-
-		// Validate HTTP server
-		if stream.HTTPServer != nil && stream.HTTPServer.Enabled {
-			if stream.HTTPServer.Port < 1 || stream.HTTPServer.Port > 65535 {
-				return fmt.Errorf("transport '%s': invalid HTTP port: %d", stream.Name, stream.HTTPServer.Port)
-			}
-			if existing, exists := streamPorts[stream.HTTPServer.Port]; exists {
-				return fmt.Errorf("transport '%s': HTTP port %d already used by transport '%s'",
-					stream.Name, stream.HTTPServer.Port, existing)
-			}
-			streamPorts[stream.HTTPServer.Port] = stream.Name + "-http"
-
-			if stream.HTTPServer.BufferSize < 1 {
-				return fmt.Errorf("transport '%s': HTTP buffer size must be positive: %d",
-					stream.Name, stream.HTTPServer.BufferSize)
-			}
-
-			// Validate paths
-			if stream.HTTPServer.StreamPath == "" {
-				stream.HTTPServer.StreamPath = "/transport"
-			}
-			if stream.HTTPServer.StatusPath == "" {
-				stream.HTTPServer.StatusPath = "/status"
-			}
-			if !strings.HasPrefix(stream.HTTPServer.StreamPath, "/") {
-				return fmt.Errorf("transport '%s': transport path must start with /: %s",
-					stream.Name, stream.HTTPServer.StreamPath)
-			}
-			if !strings.HasPrefix(stream.HTTPServer.StatusPath, "/") {
-				return fmt.Errorf("transport '%s': status path must start with /: %s",
-					stream.Name, stream.HTTPServer.StatusPath)
-			}
-
-			if err := validateHeartbeat("HTTP", stream.Name, &stream.HTTPServer.Heartbeat); err != nil {
-				return err
-			}
-
-			if err := validateSSL("HTTP", stream.Name, stream.HTTPServer.SSL); err != nil {
-				return err
-			}
-
-			if err := validateRateLimit("HTTP", stream.Name, stream.HTTPServer.RateLimit); err != nil {
-				return err
-			}
-		}
-
-		// At least one server must be enabled
-		tcpEnabled := stream.TCPServer != nil && stream.TCPServer.Enabled
-		httpEnabled := stream.HTTPServer != nil && stream.HTTPServer.Enabled
-		if !tcpEnabled && !httpEnabled {
-			return fmt.Errorf("transport '%s': no servers enabled", stream.Name)
 		}
 
 		// Validate auth if present
-		if err := validateAuth(stream.Name, stream.Auth); err != nil {
+		if err := validateAuth(pipeline.Name, pipeline.Auth); err != nil {
 			return err
 		}
 	}
 
 	return nil
 }
-
-func validateHeartbeat(serverType, streamName string, hb *HeartbeatConfig) error {
-	if hb.Enabled {
-		if hb.IntervalSeconds < 1 {
-			return fmt.Errorf("transport '%s' %s: heartbeat interval must be positive: %d",
-				streamName, serverType, hb.IntervalSeconds)
-		}
-		if hb.Format != "json" && hb.Format != "comment" {
-			return fmt.Errorf("transport '%s' %s: heartbeat format must be 'json' or 'comment': %s",
-				streamName, serverType, hb.Format)
-		}
-	}
-	return nil
-}
-
-func validateSSL(serverType, streamName string, ssl *SSLConfig) error {
-	if ssl != nil && ssl.Enabled {
-		if ssl.CertFile == "" || ssl.KeyFile == "" {
-			return fmt.Errorf("transport '%s' %s: SSL enabled but cert/key files not specified",
-				streamName, serverType)
-		}
-
-		if ssl.ClientAuth && ssl.ClientCAFile == "" {
-			return fmt.Errorf("transport '%s' %s: client auth enabled but CA file not specified",
-				streamName, serverType)
-		}
-
-		// Validate TLS versions
-		validVersions := map[string]bool{"TLS1.0": true, "TLS1.1": true, "TLS1.2": true, "TLS1.3": true}
-		if ssl.MinVersion != "" && !validVersions[ssl.MinVersion] {
-			return fmt.Errorf("transport '%s' %s: invalid min TLS version: %s",
-				streamName, serverType, ssl.MinVersion)
-		}
-		if ssl.MaxVersion != "" && !validVersions[ssl.MaxVersion] {
-			return fmt.Errorf("transport '%s' %s: invalid max TLS version: %s",
-				streamName, serverType, ssl.MaxVersion)
-		}
-	}
-	return nil
-}
-
-func validateAuth(streamName string, auth *AuthConfig) error {
-	if auth == nil {
-		return nil
-	}
-
-	validTypes := map[string]bool{"none": true, "basic": true, "bearer": true, "mtls": true}
-	if !validTypes[auth.Type] {
-		return fmt.Errorf("transport '%s': invalid auth type: %s", streamName, auth.Type)
-	}
-
-	if auth.Type == "basic" && auth.BasicAuth == nil {
-		return fmt.Errorf("transport '%s': basic auth type specified but config missing", streamName)
-	}
-
-	if auth.Type == "bearer" && auth.BearerAuth == nil {
-		return fmt.Errorf("transport '%s': bearer auth type specified but config missing", streamName)
-	}
-
-	return nil
-}
-
-func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error {
-	if rl == nil || !rl.Enabled {
-		return nil
-	}
-
-	if rl.RequestsPerSecond <= 0 {
-		return fmt.Errorf("transport '%s' %s: requests_per_second must be positive: %f",
-			streamName, serverType, rl.RequestsPerSecond)
-	}
-
-	if rl.BurstSize < 1 {
-		return fmt.Errorf("transport '%s' %s: burst_size must be at least 1: %d",
-			streamName, serverType, rl.BurstSize)
-	}
-
-	validLimitBy := map[string]bool{"ip": true, "global": true, "": true}
-	if !validLimitBy[rl.LimitBy] {
-		return fmt.Errorf("transport '%s' %s: invalid limit_by value: %s (must be 'ip' or 'global')",
-			streamName, serverType, rl.LimitBy)
-	}
-
-	if rl.ResponseCode > 0 && (rl.ResponseCode < 400 || rl.ResponseCode >= 600) {
-		return fmt.Errorf("transport '%s' %s: response_code must be 4xx or 5xx: %d",
-			streamName, serverType, rl.ResponseCode)
-	}
-
-	if rl.MaxConnectionsPerIP > 0 && rl.MaxTotalConnections > 0 {
-		if rl.MaxConnectionsPerIP > rl.MaxTotalConnections {
-			return fmt.Errorf("stream '%s' %s: max_connections_per_ip (%d) cannot exceed max_total_connections (%d)",
-				streamName, serverType, rl.MaxConnectionsPerIP, rl.MaxTotalConnections)
-		}
-	}
-
-	return nil
-}
-
-func validateFilter(streamName string, filterIndex int, cfg *filter.Config) error {
-	// Validate filter type
-	switch cfg.Type {
-	case filter.TypeInclude, filter.TypeExclude, "":
-		// Valid types
-	default:
-		return fmt.Errorf("transport '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
-			streamName, filterIndex, cfg.Type)
-	}
-
-	// Validate filter logic
-	switch cfg.Logic {
-	case filter.LogicOr, filter.LogicAnd, "":
-		// Valid logic
-	default:
-		return fmt.Errorf("transport '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
-			streamName, filterIndex, cfg.Logic)
-	}
-
-	// Empty patterns is valid - passes everything
-	if len(cfg.Patterns) == 0 {
-		return nil
-	}
-
-	// Validate regex patterns
-	for i, pattern := range cfg.Patterns {
-		if _, err := regexp.Compile(pattern); err != nil {
-			return fmt.Errorf("transport '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
-				streamName, filterIndex, i, pattern, err)
-		}
-	}
-
-	return nil
-}
-
-func validateLogConfig(cfg *LogConfig) error {
-	validOutputs := map[string]bool{
-		"file": true, "stdout": true, "stderr": true,
-		"both": true, "none": true,
-	}
-	if !validOutputs[cfg.Output] {
-		return fmt.Errorf("invalid log output mode: %s", cfg.Output)
-	}
-
-	validLevels := map[string]bool{
-		"debug": true, "info": true, "warn": true, "error": true,
-	}
-	if !validLevels[cfg.Level] {
-		return fmt.Errorf("invalid log level: %s", cfg.Level)
-	}
-
-	if cfg.Console != nil {
-		validTargets := map[string]bool{
-			"stdout": true, "stderr": true, "split": true,
-		}
-		if !validTargets[cfg.Console.Target] {
-			return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
-		}
-	}
-
-	return nil
-}

src/internal/filter/chain.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"sync/atomic"
 
-	"logwisp/src/internal/monitor"
+	"logwisp/src/internal/source"
 
 	"github.com/lixenwraith/log"
 )
@@ -43,7 +43,7 @@ func NewChain(configs []Config, logger *log.Logger) (*Chain, error) {
 
 // Apply runs all filters in sequence
 // Returns true if the entry passes all filters
-func (c *Chain) Apply(entry monitor.LogEntry) bool {
+func (c *Chain) Apply(entry source.LogEntry) bool {
 	c.totalProcessed.Add(1)
 
 	// No filters means pass everything
@@ -68,13 +68,13 @@ func (c *Chain) Apply(entry monitor.LogEntry) bool {
 }
 
 // GetStats returns chain statistics
-func (c *Chain) GetStats() map[string]interface{} {
+func (c *Chain) GetStats() map[string]any {
-	filterStats := make([]map[string]interface{}, len(c.filters))
+	filterStats := make([]map[string]any, len(c.filters))
 	for i, filter := range c.filters {
 		filterStats[i] = filter.GetStats()
 	}
 
-	return map[string]interface{}{
+	return map[string]any{
 		"filter_count":    len(c.filters),
 		"total_processed": c.totalProcessed.Load(),
 		"total_passed":    c.totalPassed.Load(),

src/internal/filter/filter.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"logwisp/src/internal/monitor"
+	"logwisp/src/internal/source"
 
 	"github.com/lixenwraith/log"
 )
@@ -83,7 +83,7 @@ func New(cfg Config, logger *log.Logger) (*Filter, error) {
 }
 
 // Apply checks if a log entry should be passed through
-func (f *Filter) Apply(entry monitor.LogEntry) bool {
+func (f *Filter) Apply(entry source.LogEntry) bool {
 	f.totalProcessed.Add(1)
 
 	// No patterns means pass everything
@@ -152,8 +152,8 @@ func (f *Filter) matches(text string) bool {
 }
 
 // GetStats returns filter statistics
-func (f *Filter) GetStats() map[string]interface{} {
+func (f *Filter) GetStats() map[string]any {
-	return map[string]interface{}{
+	return map[string]any{
 		"type":            f.config.Type,
 		"logic":           f.config.Logic,
 		"pattern_count":   len(f.patterns),

src/internal/monitor/monitor.go

@@ -1,380 +0,0 @@
-// FILE: src/internal/monitor/monitor.go
-package monitor
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"regexp"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/lixenwraith/log"
-)
-
-type LogEntry struct {
-	Time    time.Time       `json:"time"`
-	Source  string          `json:"source"`
-	Level   string          `json:"level,omitempty"`
-	Message string          `json:"message"`
-	Fields  json.RawMessage `json:"fields,omitempty"`
-}
-
-type Monitor interface {
-	Start(ctx context.Context) error
-	Stop()
-	Subscribe() chan LogEntry
-	AddTarget(path, pattern string, isFile bool) error
-	RemoveTarget(path string) error
-	SetCheckInterval(interval time.Duration)
-	GetStats() Stats
-	GetActiveWatchers() []WatcherInfo
-}
-
-type Stats struct {
-	ActiveWatchers int
-	TotalEntries   uint64
-	DroppedEntries uint64
-	StartTime      time.Time
-	LastEntryTime  time.Time
-}
-
-type WatcherInfo struct {
-	Path         string
-	Size         int64
-	Position     int64
-	ModTime      time.Time
-	EntriesRead  uint64
-	LastReadTime time.Time
-	Rotations    int
-}
-
-type monitor struct {
-	subscribers    []chan LogEntry
-	targets        []target
-	watchers       map[string]*fileWatcher
-	mu             sync.RWMutex
-	ctx            context.Context
-	cancel         context.CancelFunc
-	wg             sync.WaitGroup
-	checkInterval  time.Duration
-	totalEntries   atomic.Uint64
-	droppedEntries atomic.Uint64
-	startTime      time.Time
-	lastEntryTime  atomic.Value // time.Time
-	logger         *log.Logger
-}
-
-type target struct {
-	path    string
-	pattern string
-	isFile  bool
-	regex   *regexp.Regexp
-}
-
-func New(logger *log.Logger) Monitor {
-	m := &monitor{
-		watchers:      make(map[string]*fileWatcher),
-		checkInterval: 100 * time.Millisecond,
-		startTime:     time.Now(),
-		logger:        logger,
-	}
-	m.lastEntryTime.Store(time.Time{})
-	return m
-}
-
-func (m *monitor) Subscribe() chan LogEntry {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	ch := make(chan LogEntry, 1000)
-	m.subscribers = append(m.subscribers, ch)
-	return ch
-}
-
-func (m *monitor) publish(entry LogEntry) {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	m.totalEntries.Add(1)
-	m.lastEntryTime.Store(entry.Time)
-
-	for _, ch := range m.subscribers {
-		select {
-		case ch <- entry:
-		default:
-			m.droppedEntries.Add(1)
-			m.logger.Debug("msg", "Dropped log entry - subscriber buffer full")
-		}
-	}
-}
-
-func (m *monitor) SetCheckInterval(interval time.Duration) {
-	m.mu.Lock()
-	m.checkInterval = interval
-	m.mu.Unlock()
-
-	m.logger.Debug("msg", "Check interval updated", "interval_ms", interval.Milliseconds())
-}
-
-func (m *monitor) AddTarget(path, pattern string, isFile bool) error {
-	absPath, err := filepath.Abs(path)
-	if err != nil {
-		m.logger.Error("msg", "Failed to resolve absolute path",
-			"component", "monitor",
-			"path", path,
-			"error", err)
-		return fmt.Errorf("invalid path %s: %w", path, err)
-	}
-
-	var compiledRegex *regexp.Regexp
-	if !isFile && pattern != "" {
-		regexPattern := globToRegex(pattern)
-		compiledRegex, err = regexp.Compile(regexPattern)
-		if err != nil {
-			m.logger.Error("msg", "Failed to compile pattern regex",
-				"component", "monitor",
-				"pattern", pattern,
-				"regex", regexPattern,
-				"error", err)
-			return fmt.Errorf("invalid pattern %s: %w", pattern, err)
-		}
-	}
-
-	m.mu.Lock()
-	m.targets = append(m.targets, target{
-		path:    absPath,
-		pattern: pattern,
-		isFile:  isFile,
-		regex:   compiledRegex,
-	})
-	m.mu.Unlock()
-
-	m.logger.Info("msg", "Added monitor target",
-		"component", "monitor",
-		"path", absPath,
-		"pattern", pattern,
-		"is_file", isFile)
-
-	return nil
-}
-
-func (m *monitor) RemoveTarget(path string) error {
-	absPath, err := filepath.Abs(path)
-	if err != nil {
-		return fmt.Errorf("invalid path %s: %w", path, err)
-	}
-
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	// Remove from targets
-	newTargets := make([]target, 0, len(m.targets))
-	for _, t := range m.targets {
-		if t.path != absPath {
-			newTargets = append(newTargets, t)
-		}
-	}
-	m.targets = newTargets
-
-	// Stop any watchers for this path
-	if w, exists := m.watchers[absPath]; exists {
-		w.stop()
-		delete(m.watchers, absPath)
-		m.logger.Info("msg", "Monitor started",
-			"component", "monitor",
-			"check_interval_ms", m.checkInterval.Milliseconds())
-	}
-
-	return nil
-}
-
-func (m *monitor) Start(ctx context.Context) error {
-	m.ctx, m.cancel = context.WithCancel(ctx)
-	m.wg.Add(1)
-	go m.monitorLoop()
-
-	m.logger.Info("msg", "Monitor started", "check_interval_ms", m.checkInterval.Milliseconds())
-	return nil
-}
-
-func (m *monitor) Stop() {
-	if m.cancel != nil {
-		m.cancel()
-	}
-	m.wg.Wait()
-
-	m.mu.Lock()
-	for _, w := range m.watchers {
-		w.close()
-	}
-	for _, ch := range m.subscribers {
-		close(ch)
-	}
-	m.mu.Unlock()
-
-	m.logger.Info("msg", "Monitor stopped")
-}
-
-func (m *monitor) GetStats() Stats {
-	lastEntry, _ := m.lastEntryTime.Load().(time.Time)
-
-	m.mu.RLock()
-	watcherCount := len(m.watchers)
-	m.mu.RUnlock()
-
-	return Stats{
-		ActiveWatchers: watcherCount,
-		TotalEntries:   m.totalEntries.Load(),
-		DroppedEntries: m.droppedEntries.Load(),
-		StartTime:      m.startTime,
-		LastEntryTime:  lastEntry,
-	}
-}
-
-func (m *monitor) GetActiveWatchers() []WatcherInfo {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	info := make([]WatcherInfo, 0, len(m.watchers))
-	for _, w := range m.watchers {
-		info = append(info, w.getInfo())
-	}
-	return info
-}
-
-func (m *monitor) monitorLoop() {
-	defer m.wg.Done()
-
-	m.checkTargets()
-
-	m.mu.RLock()
-	interval := m.checkInterval
-	m.mu.RUnlock()
-
-	ticker := time.NewTicker(interval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-m.ctx.Done():
-			return
-		case <-ticker.C:
-			m.checkTargets()
-
-			m.mu.RLock()
-			newInterval := m.checkInterval
-			m.mu.RUnlock()
-
-			if newInterval != interval {
-				ticker.Stop()
-				ticker = time.NewTicker(newInterval)
-				interval = newInterval
-			}
-		}
-	}
-}
-
-func (m *monitor) checkTargets() {
-	m.mu.RLock()
-	targets := make([]target, len(m.targets))
-	copy(targets, m.targets)
-	m.mu.RUnlock()
-
-	for _, t := range targets {
-		if t.isFile {
-			m.ensureWatcher(t.path)
-		} else {
-			// Directory scanning for pattern matching
-			files, err := m.scanDirectory(t.path, t.regex)
-			if err != nil {
-				m.logger.Warn("msg", "Failed to scan directory",
-					"component", "monitor",
-					"path", t.path,
-					"pattern", t.pattern,
-					"error", err)
-				continue
-			}
-
-			for _, file := range files {
-				m.ensureWatcher(file)
-			}
-		}
-	}
-
-	m.cleanupWatchers()
-}
-
-func (m *monitor) scanDirectory(dir string, pattern *regexp.Regexp) ([]string, error) {
-	entries, err := os.ReadDir(dir)
-	if err != nil {
-		return nil, err
-	}
-
-	var files []string
-	for _, entry := range entries {
-		if entry.IsDir() {
-			continue
-		}
-
-		name := entry.Name()
-		if pattern == nil || pattern.MatchString(name) {
-			files = append(files, filepath.Join(dir, name))
-		}
-	}
-
-	return files, nil
-}
-
-func (m *monitor) ensureWatcher(path string) {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	if _, exists := m.watchers[path]; exists {
-		return
-	}
-
-	w := newFileWatcher(path, m.publish, m.logger)
-	m.watchers[path] = w
-
-	m.logger.Debug("msg", "Created watcher", "path", path)
-
-	m.wg.Add(1)
-	go func() {
-		defer m.wg.Done()
-		if err := w.watch(m.ctx); err != nil {
-			// Log based on error type
-			if errors.Is(err, context.Canceled) {
-				m.logger.Debug("msg", "Watcher cancelled",
-					"component", "monitor",
-					"path", path)
-			} else {
-				m.logger.Error("msg", "Watcher failed",
-					"component", "monitor",
-					"path", path,
-					"error", err)
-			}
-		}
-
-		m.mu.Lock()
-		delete(m.watchers, path)
-		m.mu.Unlock()
-	}()
-}
-
-func (m *monitor) cleanupWatchers() {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	for path, w := range m.watchers {
-		if _, err := os.Stat(path); os.IsNotExist(err) {
-			w.stop()
-			delete(m.watchers, path)
-			m.logger.Debug("msg", "Cleaned up watcher for non-existent file", "path", path)
-		}
-	}
-}

src/internal/service/httprouter.go

@@ -8,10 +8,13 @@ import (
 	"sync/atomic"
 	"time"
 
+	"logwisp/src/internal/sink"
+
 	"github.com/lixenwraith/log"
 	"github.com/valyala/fasthttp"
 )
 
+// HTTPRouter manages HTTP routing for multiple pipelines
 type HTTPRouter struct {
 	service *Service
 	servers map[int]*routerServer // port -> server
@@ -25,6 +28,7 @@ type HTTPRouter struct {
 	failedRequests atomic.Uint64
 }
 
+// NewHTTPRouter creates a new HTTP router
 func NewHTTPRouter(service *Service, logger *log.Logger) *HTTPRouter {
 	return &HTTPRouter{
 		service:   service,
@@ -34,12 +38,23 @@ func NewHTTPRouter(service *Service, logger *log.Logger) *HTTPRouter {
 	}
 }
 
-func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
+// RegisterPipeline registers a pipeline's HTTP sinks with the router
-	if stream.HTTPServer == nil || stream.Config.HTTPServer == nil {
+func (r *HTTPRouter) RegisterPipeline(pipeline *Pipeline) error {
-		return nil // No HTTP server configured
+	// Register all HTTP sinks in the pipeline
+	for _, httpSink := range pipeline.HTTPSinks {
+		if err := r.registerHTTPSink(pipeline.Name, httpSink); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
-	port := stream.Config.HTTPServer.Port
+// registerHTTPSink registers a single HTTP sink
+func (r *HTTPRouter) registerHTTPSink(pipelineName string, httpSink *sink.HTTPSink) error {
+	// Get port from sink configuration
+	stats := httpSink.GetStats()
+	details := stats.Details
+	port := details["port"].(int)
 
 	r.mu.Lock()
 	rs, exists := r.servers[port]
@@ -47,7 +62,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
 		// Create new server for this port
 		rs = &routerServer{
 			port:      port,
-			routes:    make(map[string]*LogStream),
+			routes:    make(map[string]*routedSink),
 			router:    r,
 			startTime: time.Now(),
 			logger:    r.logger,
@@ -56,7 +71,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
 			Handler:           rs.requestHandler,
 			DisableKeepalive:  false,
 			StreamRequestBody: true,
-			CloseOnShutdown:   true, // Ensure connections close on shutdown
+			CloseOnShutdown:   true,
 		}
 		r.servers[port] = rs
 
@@ -79,54 +94,74 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
 	}
 	r.mu.Unlock()
 
-	// Register routes for this transport
+	// Register routes for this sink
 	rs.routeMu.Lock()
 	defer rs.routeMu.Unlock()
 
-	// Use transport name as path prefix
+	// Use pipeline name as path prefix
-	pathPrefix := "/" + stream.Name
+	pathPrefix := "/" + pipelineName
 
 	// Check for conflicts
-	for existingPath, existingStream := range rs.routes {
+	for existingPath, existing := range rs.routes {
 		if strings.HasPrefix(pathPrefix, existingPath) || strings.HasPrefix(existingPath, pathPrefix) {
-			return fmt.Errorf("path conflict: '%s' conflicts with existing transport '%s' at '%s'",
+			return fmt.Errorf("path conflict: '%s' conflicts with existing pipeline '%s' at '%s'",
-				pathPrefix, existingStream.Name, existingPath)
+				pathPrefix, existing.pipelineName, existingPath)
 		}
 	}
 
-	rs.routes[pathPrefix] = stream
+	// Set the sink to router mode
-	r.logger.Info("msg", "Registered transport route",
+	httpSink.SetRouterMode()
+
+	rs.routes[pathPrefix] = &routedSink{
+		pipelineName: pipelineName,
+		httpSink:     httpSink,
+	}
+
+	r.logger.Info("msg", "Registered pipeline route",
 		"component", "http_router",
-		"transport", stream.Name,
+		"pipeline", pipelineName,
 		"path", pathPrefix,
 		"port", port)
 	return nil
 }
 
+// UnregisterStream is deprecated
 func (r *HTTPRouter) UnregisterStream(streamName string) {
+	r.logger.Warn("msg", "UnregisterStream is deprecated",
+		"component", "http_router")
+}
+
+// UnregisterPipeline removes a pipeline's routes
+func (r *HTTPRouter) UnregisterPipeline(pipelineName string) {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 
 	for port, rs := range r.servers {
 		rs.routeMu.Lock()
-		for path, stream := range rs.routes {
+		for path, route := range rs.routes {
-			if stream.Name == streamName {
+			if route.pipelineName == pipelineName {
 				delete(rs.routes, path)
-				fmt.Printf("[ROUTER] Unregistered transport '%s' from path '%s' on port %d\n",
+				r.logger.Info("msg", "Unregistered pipeline route",
-					streamName, path, port)
+					"component", "http_router",
+					"pipeline", pipelineName,
+					"path", path,
+					"port", port)
 			}
 		}
 
 		// Check if server has no more routes
 		if len(rs.routes) == 0 {
-			fmt.Printf("[ROUTER] No routes left on port %d, considering shutdown\n", port)
+			r.logger.Info("msg", "No routes left on port, considering shutdown",
+				"component", "http_router",
+				"port", port)
 		}
 		rs.routeMu.Unlock()
 	}
 }
 
+// Shutdown stops all router servers
 func (r *HTTPRouter) Shutdown() {
-	fmt.Println("[ROUTER] Starting router shutdown...")
+	r.logger.Info("msg", "Starting router shutdown...")
 
 	r.mu.Lock()
 	defer r.mu.Unlock()
@@ -136,17 +171,23 @@ func (r *HTTPRouter) Shutdown() {
 		wg.Add(1)
 		go func(p int, s *routerServer) {
 			defer wg.Done()
-			fmt.Printf("[ROUTER] Shutting down server on port %d\n", p)
+			r.logger.Info("msg", "Shutting down server",
+				"component", "http_router",
+				"port", p)
 			if err := s.server.Shutdown(); err != nil {
-				fmt.Printf("[ROUTER] Error shutting down server on port %d: %v\n", p, err)
+				r.logger.Error("msg", "Error shutting down server",
+					"component", "http_router",
+					"port", p,
+					"error", err)
 			}
 		}(port, rs)
 	}
 	wg.Wait()
 
-	fmt.Println("[ROUTER] Router shutdown complete")
+	r.logger.Info("msg", "Router shutdown complete")
 }
 
+// GetStats returns router statistics
 func (r *HTTPRouter) GetStats() map[string]any {
 	r.mu.RLock()
 	defer r.mu.RUnlock()

src/internal/service/logstream.go

@@ -1,210 +0,0 @@
-// FILE: src/internal/service/logstream.go
-package service
-
-import (
-	"context"
-	"fmt"
-	"path/filepath"
-	"sync"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/filter"
-	"logwisp/src/internal/monitor"
-	"logwisp/src/internal/transport"
-
-	"github.com/lixenwraith/log"
-)
-
-type LogStream struct {
-	Name        string
-	Config      config.StreamConfig
-	Monitor     monitor.Monitor
-	FilterChain *filter.Chain
-	TCPServer   *transport.TCPStreamer
-	HTTPServer  *transport.HTTPStreamer
-	Stats       *StreamStats
-	logger      *log.Logger
-
-	ctx    context.Context
-	cancel context.CancelFunc
-}
-
-type StreamStats struct {
-	StartTime          time.Time
-	MonitorStats       monitor.Stats
-	TCPConnections     int32
-	HTTPConnections    int32
-	TotalBytesServed   uint64
-	TotalEntriesServed uint64
-	FilterStats        map[string]any
-}
-
-func (ls *LogStream) Shutdown() {
-	ls.logger.Info("msg", "Shutting down stream",
-		"component", "logstream",
-		"stream", ls.Name)
-
-	// Stop servers first
-	var wg sync.WaitGroup
-
-	if ls.TCPServer != nil {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			ls.TCPServer.Stop()
-		}()
-	}
-
-	if ls.HTTPServer != nil {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			ls.HTTPServer.Stop()
-		}()
-	}
-
-	// Cancel context
-	ls.cancel()
-
-	// Wait for servers
-	wg.Wait()
-
-	// Stop monitor
-	ls.Monitor.Stop()
-
-	ls.logger.Info("msg", "Stream shutdown complete",
-		"component", "logstream",
-		"stream", ls.Name)
-}
-
-func (ls *LogStream) GetStats() map[string]any {
-	monStats := ls.Monitor.GetStats()
-
-	stats := map[string]any{
-		"name":           ls.Name,
-		"uptime_seconds": int(time.Since(ls.Stats.StartTime).Seconds()),
-		"monitor":        monStats,
-	}
-
-	if ls.FilterChain != nil {
-		stats["filters"] = ls.FilterChain.GetStats()
-	}
-
-	if ls.TCPServer != nil {
-		currentConnections := ls.TCPServer.GetActiveConnections()
-
-		stats["tcp"] = map[string]interface{}{
-			"enabled":     true,
-			"port":        ls.Config.TCPServer.Port,
-			"connections": currentConnections,
-		}
-	}
-
-	if ls.HTTPServer != nil {
-		currentConnections := ls.HTTPServer.GetActiveConnections()
-
-		stats["http"] = map[string]interface{}{
-			"enabled":     true,
-			"port":        ls.Config.HTTPServer.Port,
-			"connections": currentConnections,
-			"stream_path": ls.Config.HTTPServer.StreamPath,
-			"status_path": ls.Config.HTTPServer.StatusPath,
-		}
-	}
-
-	return stats
-}
-
-func (ls *LogStream) UpdateTargets(targets []config.MonitorTarget) error {
-	// Validate new targets first
-	validatedTargets := make([]config.MonitorTarget, 0, len(targets))
-	for _, target := range targets {
-		// Basic validation
-		absPath, err := filepath.Abs(target.Path)
-		if err != nil {
-			ls.logger.Error("msg", "Invalid target path",
-				"component", "logstream",
-				"stream", ls.Name,
-				"path", target.Path,
-				"error", err)
-			return fmt.Errorf("invalid target path %s: %w", target.Path, err)
-		}
-		target.Path = absPath
-		validatedTargets = append(validatedTargets, target)
-	}
-
-	// Get current watchers
-	oldWatchers := ls.Monitor.GetActiveWatchers()
-
-	// Add new targets
-	for _, target := range validatedTargets {
-		if err := ls.Monitor.AddTarget(target.Path, target.Pattern, target.IsFile); err != nil {
-			ls.logger.Error("msg", "Failed to add monitor target - rolling back",
-				"component", "logstream",
-				"stream", ls.Name,
-				"target", target.Path,
-				"pattern", target.Pattern,
-				"error", err)
-			// Rollback: restore old watchers
-			for _, watcher := range oldWatchers {
-				// Best effort restoration
-				ls.Monitor.AddTarget(watcher.Path, "", false)
-			}
-			return fmt.Errorf("failed to add target %s: %w", target.Path, err)
-		}
-	}
-
-	// Only remove old targets after new ones are successfully added
-	for _, watcher := range oldWatchers {
-		ls.Monitor.RemoveTarget(watcher.Path)
-	}
-
-	ls.logger.Info("msg", "Updated monitor targets",
-		"component", "logstream",
-		"stream", ls.Name,
-		"old_count", len(oldWatchers),
-		"new_count", len(validatedTargets))
-
-	return nil
-}
-
-func (ls *LogStream) startStatsUpdater(ctx context.Context) {
-	go func() {
-		ticker := time.NewTicker(1 * time.Second)
-		defer ticker.Stop()
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-ticker.C:
-				// Update cached values
-				if ls.TCPServer != nil {
-					oldTCP := ls.Stats.TCPConnections
-					ls.Stats.TCPConnections = ls.TCPServer.GetActiveConnections()
-					if oldTCP != ls.Stats.TCPConnections {
-						// This debug should now show changes
-						ls.logger.Debug("msg", "TCP connection count changed",
-							"component", "logstream",
-							"stream", ls.Name,
-							"old", oldTCP,
-							"new", ls.Stats.TCPConnections)
-					}
-				}
-				if ls.HTTPServer != nil {
-					oldHTTP := ls.Stats.HTTPConnections
-					ls.Stats.HTTPConnections = ls.HTTPServer.GetActiveConnections()
-					if oldHTTP != ls.Stats.HTTPConnections {
-						// This debug should now show changes
-						ls.logger.Debug("msg", "HTTP connection count changed",
-							"component", "logstream",
-							"stream", ls.Name,
-							"old", oldHTTP,
-							"new", ls.Stats.HTTPConnections)
-					}
-				}
-			}
-		}
-	}()
-}

src/internal/service/pipeline.go

@@ -0,0 +1,150 @@
+// FILE: src/internal/service/pipeline.go
+package service
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/src/internal/config"
+	"logwisp/src/internal/filter"
+	"logwisp/src/internal/sink"
+	"logwisp/src/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// Pipeline manages the flow of data from sources through filters to sinks
+type Pipeline struct {
+	Name        string
+	Config      config.PipelineConfig
+	Sources     []source.Source
+	FilterChain *filter.Chain
+	Sinks       []sink.Sink
+	Stats       *PipelineStats
+	logger      *log.Logger
+
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+
+	// For HTTP sinks in router mode
+	HTTPSinks []*sink.HTTPSink
+	TCPSinks  []*sink.TCPSink
+}
+
+// PipelineStats contains statistics for a pipeline
+type PipelineStats struct {
+	StartTime             time.Time
+	TotalEntriesProcessed atomic.Uint64
+	TotalEntriesFiltered  atomic.Uint64
+	SourceStats           []source.SourceStats
+	SinkStats             []sink.SinkStats
+	FilterStats           map[string]any
+}
+
+// Shutdown gracefully stops the pipeline
+func (p *Pipeline) Shutdown() {
+	p.logger.Info("msg", "Shutting down pipeline",
+		"component", "pipeline",
+		"pipeline", p.Name)
+
+	// Cancel context to stop processing
+	p.cancel()
+
+	// Stop all sinks first
+	var wg sync.WaitGroup
+	for _, s := range p.Sinks {
+		wg.Add(1)
+		go func(sink sink.Sink) {
+			defer wg.Done()
+			sink.Stop()
+		}(s)
+	}
+	wg.Wait()
+
+	// Stop all sources
+	for _, src := range p.Sources {
+		wg.Add(1)
+		go func(source source.Source) {
+			defer wg.Done()
+			source.Stop()
+		}(src)
+	}
+	wg.Wait()
+
+	// Wait for processing goroutines
+	p.wg.Wait()
+
+	p.logger.Info("msg", "Pipeline shutdown complete",
+		"component", "pipeline",
+		"pipeline", p.Name)
+}
+
+// GetStats returns pipeline statistics
+func (p *Pipeline) GetStats() map[string]any {
+	// Collect source stats
+	sourceStats := make([]map[string]any, len(p.Sources))
+	for i, src := range p.Sources {
+		stats := src.GetStats()
+		sourceStats[i] = map[string]any{
+			"type":            stats.Type,
+			"total_entries":   stats.TotalEntries,
+			"dropped_entries": stats.DroppedEntries,
+			"start_time":      stats.StartTime,
+			"last_entry_time": stats.LastEntryTime,
+			"details":         stats.Details,
+		}
+	}
+
+	// Collect sink stats
+	sinkStats := make([]map[string]any, len(p.Sinks))
+	for i, s := range p.Sinks {
+		stats := s.GetStats()
+		sinkStats[i] = map[string]any{
+			"type":               stats.Type,
+			"total_processed":    stats.TotalProcessed,
+			"active_connections": stats.ActiveConnections,
+			"start_time":         stats.StartTime,
+			"last_processed":     stats.LastProcessed,
+			"details":            stats.Details,
+		}
+	}
+
+	// Collect filter stats
+	var filterStats map[string]any
+	if p.FilterChain != nil {
+		filterStats = p.FilterChain.GetStats()
+	}
+
+	return map[string]any{
+		"name":            p.Name,
+		"uptime_seconds":  int(time.Since(p.Stats.StartTime).Seconds()),
+		"total_processed": p.Stats.TotalEntriesProcessed.Load(),
+		"total_filtered":  p.Stats.TotalEntriesFiltered.Load(),
+		"sources":         sourceStats,
+		"sinks":           sinkStats,
+		"filters":         filterStats,
+		"source_count":    len(p.Sources),
+		"sink_count":      len(p.Sinks),
+		"filter_count":    len(p.Config.Filters),
+	}
+}
+
+// startStatsUpdater runs periodic stats updates
+func (p *Pipeline) startStatsUpdater(ctx context.Context) {
+	go func() {
+		ticker := time.NewTicker(1 * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				// Periodic stats updates if needed
+			}
+		}
+	}()
+}

src/internal/service/routerserver.go

@@ -9,17 +9,25 @@ import (
 	"sync/atomic"
 	"time"
 
+	"logwisp/src/internal/sink"
 	"logwisp/src/internal/version"
 
 	"github.com/lixenwraith/log"
 	"github.com/valyala/fasthttp"
 )
 
+// routedSink represents a sink registered with the router
+type routedSink struct {
+	pipelineName string
+	httpSink     *sink.HTTPSink
+}
+
+// routerServer handles HTTP requests for a specific port
 type routerServer struct {
 	port      int
 	server    *fasthttp.Server
 	logger    *log.Logger
-	routes    map[string]*LogStream // path prefix -> transport
+	routes    map[string]*routedSink // path prefix -> sink
 	routeMu   sync.RWMutex
 	router    *HTTPRouter
 	startTime time.Time
@@ -36,7 +44,7 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
 	// Log request for debugging
 	rs.logger.Debug("msg", "Router request",
 		"component", "router_server",
-		"method", ctx.Method(),
+		"method", string(ctx.Method()),
 		"path", path,
 		"remote_addr", remoteAddr)
 
@@ -46,18 +54,18 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
 		return
 	}
 
-	// Find matching transport
+	// Find matching route
 	rs.routeMu.RLock()
-	var matchedStream *LogStream
+	var matchedSink *routedSink
 	var matchedPrefix string
 	var remainingPath string
 
-	for prefix, stream := range rs.routes {
+	for prefix, route := range rs.routes {
 		if strings.HasPrefix(path, prefix) {
 			// Use longest prefix match
 			if len(prefix) > len(matchedPrefix) {
 				matchedPrefix = prefix
-				matchedStream = stream
+				matchedSink = route
 				remainingPath = strings.TrimPrefix(path, prefix)
 				// Ensure remaining path starts with / or is empty
 				if remainingPath != "" && !strings.HasPrefix(remainingPath, "/") {
@@ -68,7 +76,7 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
 	}
 	rs.routeMu.RUnlock()
 
-	if matchedStream == nil {
+	if matchedSink == nil {
 		rs.router.failedRequests.Add(1)
 		rs.handleNotFound(ctx)
 		return
@@ -76,25 +84,25 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
 
 	rs.router.routedRequests.Add(1)
 
-	// Route to transport's handler
+	// Route to sink's handler
-	if matchedStream.HTTPServer != nil {
+	if matchedSink.httpSink != nil {
 		// Save original path
 		originalPath := string(ctx.URI().Path())
 
-		// Rewrite path to remove transport prefix
+		// Rewrite path to remove pipeline prefix
 		if remainingPath == "" {
-			// Default to transport path if no remaining path
+			// Default to stream path if no remaining path
-			remainingPath = matchedStream.Config.HTTPServer.StreamPath
+			remainingPath = matchedSink.httpSink.GetStreamPath()
 		}
 
-		rs.logger.Debug("msg", "Routing request to transport",
+		rs.logger.Debug("msg", "Routing request to pipeline",
 			"component", "router_server",
-			"transport", matchedStream.Name,
+			"pipeline", matchedSink.pipelineName,
 			"original_path", originalPath,
 			"remaining_path", remainingPath)
 
 		ctx.URI().SetPath(remainingPath)
-		matchedStream.HTTPServer.RouteRequest(ctx)
+		matchedSink.httpSink.RouteRequest(ctx)
 
 		// Restore original path
 		ctx.URI().SetPath(originalPath)
@@ -102,8 +110,8 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
 		ctx.SetStatusCode(fasthttp.StatusServiceUnavailable)
 		ctx.SetContentType("application/json")
 		json.NewEncoder(ctx).Encode(map[string]string{
-			"error":     "Stream HTTP server not available",
+			"error":    "Pipeline HTTP sink not available",
-			"transport": matchedStream.Name,
+			"pipeline": matchedSink.pipelineName,
 		})
 	}
 }
@@ -112,20 +120,26 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
 	ctx.SetContentType("application/json")
 
 	rs.routeMu.RLock()
-	streams := make(map[string]any)
+	pipelines := make(map[string]any)
-	for prefix, stream := range rs.routes {
+	for prefix, route := range rs.routes {
-		streamStats := stream.GetStats()
+		pipelineInfo := map[string]any{
-
-		// Add routing information
-		streamStats["routing"] = map[string]any{
 			"path_prefix": prefix,
 			"endpoints": map[string]string{
-				"transport": prefix + stream.Config.HTTPServer.StreamPath,
+				"stream": prefix + route.httpSink.GetStreamPath(),
-				"status":    prefix + stream.Config.HTTPServer.StatusPath,
+				"status": prefix + route.httpSink.GetStatusPath(),
 			},
 		}
 
-		streams[stream.Name] = streamStats
+		// Get sink stats
+		sinkStats := route.httpSink.GetStats()
+		pipelineInfo["sink"] = map[string]any{
+			"type":               sinkStats.Type,
+			"total_processed":    sinkStats.TotalProcessed,
+			"active_connections": sinkStats.ActiveConnections,
+			"details":            sinkStats.Details,
+		}
+
+		pipelines[route.pipelineName] = pipelineInfo
 	}
 	rs.routeMu.RUnlock()
 
@@ -136,8 +150,8 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
 		"service":         "LogWisp Router",
 		"version":         version.String(),
 		"port":            rs.port,
-		"streams":       streams,
+		"pipelines":       pipelines,
-		"total_streams": len(streams),
+		"total_pipelines": len(pipelines),
 		"router":          routerStats,
 		"endpoints": map[string]string{
 			"global_status": "/status",
@@ -156,11 +170,11 @@ func (rs *routerServer) handleNotFound(ctx *fasthttp.RequestCtx) {
 	availableRoutes := make([]string, 0, len(rs.routes)*2+1)
 	availableRoutes = append(availableRoutes, "/status (global status)")
 
-	for prefix, stream := range rs.routes {
+	for prefix, route := range rs.routes {
-		if stream.Config.HTTPServer != nil {
+		if route.httpSink != nil {
 			availableRoutes = append(availableRoutes,
-				fmt.Sprintf("%s%s (transport: %s)", prefix, stream.Config.HTTPServer.StreamPath, stream.Name),
+				fmt.Sprintf("%s%s (stream: %s)", prefix, route.httpSink.GetStreamPath(), route.pipelineName),
-				fmt.Sprintf("%s%s (status: %s)", prefix, stream.Config.HTTPServer.StatusPath, stream.Name),
+				fmt.Sprintf("%s%s (status: %s)", prefix, route.httpSink.GetStatusPath(), route.pipelineName),
 			)
 		}
 	}

src/internal/service/service.go

@@ -9,14 +9,15 @@ import (
 
 	"logwisp/src/internal/config"
 	"logwisp/src/internal/filter"
-	"logwisp/src/internal/monitor"
+	"logwisp/src/internal/sink"
-	"logwisp/src/internal/transport"
+	"logwisp/src/internal/source"
 
 	"github.com/lixenwraith/log"
 )
 
+// Service manages multiple pipelines
 type Service struct {
-	streams map[string]*LogStream
+	pipelines map[string]*Pipeline
 	mu        sync.RWMutex
 	ctx       context.Context
 	cancel    context.CancelFunc
@@ -24,239 +25,269 @@ type Service struct {
 	logger    *log.Logger
 }
 
+// New creates a new service
 func New(ctx context.Context, logger *log.Logger) *Service {
 	serviceCtx, cancel := context.WithCancel(ctx)
 	return &Service{
-		streams: make(map[string]*LogStream),
+		pipelines: make(map[string]*Pipeline),
 		ctx:       serviceCtx,
 		cancel:    cancel,
 		logger:    logger,
 	}
 }
 
-func (s *Service) CreateStream(cfg config.StreamConfig) error {
+// NewPipeline creates and starts a new pipeline
+func (s *Service) NewPipeline(cfg config.PipelineConfig) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	if _, exists := s.streams[cfg.Name]; exists {
+	if _, exists := s.pipelines[cfg.Name]; exists {
-		err := fmt.Errorf("transport '%s' already exists", cfg.Name)
+		err := fmt.Errorf("pipeline '%s' already exists", cfg.Name)
-		s.logger.Error("msg", "Failed to create stream - duplicate name",
+		s.logger.Error("msg", "Failed to create pipeline - duplicate name",
 			"component", "service",
-			"stream", cfg.Name,
+			"pipeline", cfg.Name,
 			"error", err)
 		return err
 	}
 
-	s.logger.Debug("msg", "Creating stream", "stream", cfg.Name)
+	s.logger.Debug("msg", "Creating pipeline", "pipeline", cfg.Name)
 
-	// Create transport context
+	// Create pipeline context
-	streamCtx, streamCancel := context.WithCancel(s.ctx)
+	pipelineCtx, pipelineCancel := context.WithCancel(s.ctx)
 
-	// Create monitor - pass the service logger directly
+	// Create pipeline instance
-	mon := monitor.New(s.logger)
+	pipeline := &Pipeline{
-	mon.SetCheckInterval(time.Duration(cfg.GetCheckInterval(100)) * time.Millisecond)
+		Name:   cfg.Name,
-
+		Config: cfg,
-	// Add targets
+		Stats: &PipelineStats{
-	for _, target := range cfg.GetTargets(nil) {
+			StartTime: time.Now(),
-		if err := mon.AddTarget(target.Path, target.Pattern, target.IsFile); err != nil {
+		},
-			streamCancel()
+		ctx:    pipelineCtx,
-			return fmt.Errorf("failed to add target %s: %w", target.Path, err)
+		cancel: pipelineCancel,
-		}
+		logger: s.logger,
 	}
 
-	// Start monitor
+	// Create sources
-	if err := mon.Start(streamCtx); err != nil {
+	for i, srcCfg := range cfg.Sources {
-		streamCancel()
+		src, err := s.createSource(srcCfg)
-		s.logger.Error("msg", "Failed to start monitor",
+		if err != nil {
-			"component", "service",
+			pipelineCancel()
-			"stream", cfg.Name,
+			return fmt.Errorf("failed to create source[%d]: %w", i, err)
-			"error", err)
+		}
-		return fmt.Errorf("failed to start monitor: %w", err)
+		pipeline.Sources = append(pipeline.Sources, src)
 	}
 
 	// Create filter chain
-	var filterChain *filter.Chain
 	if len(cfg.Filters) > 0 {
 		chain, err := filter.NewChain(cfg.Filters, s.logger)
 		if err != nil {
-			streamCancel()
+			pipelineCancel()
-			s.logger.Error("msg", "Failed to create filter chain",
-				"component", "service",
-				"stream", cfg.Name,
-				"filter_count", len(cfg.Filters),
-				"error", err)
 			return fmt.Errorf("failed to create filter chain: %w", err)
 		}
-		filterChain = chain
+		pipeline.FilterChain = chain
 	}
 
-	// Create log transport
+	// Create sinks
-	ls := &LogStream{
+	for i, sinkCfg := range cfg.Sinks {
-		Name:        cfg.Name,
+		sinkInst, err := s.createSink(sinkCfg)
-		Config:      cfg,
+		if err != nil {
-		Monitor:     mon,
+			pipelineCancel()
-		FilterChain: filterChain,
+			return fmt.Errorf("failed to create sink[%d]: %w", i, err)
-		Stats: &StreamStats{
-			StartTime: time.Now(),
-		},
-		ctx:    streamCtx,
-		cancel: streamCancel,
-		logger: s.logger, // Use parent logger
 		}
+		pipeline.Sinks = append(pipeline.Sinks, sinkInst)
 
-	// Start TCP server if configured
+		// Track HTTP/TCP sinks for router mode
-	if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
+		switch s := sinkInst.(type) {
-		// Create filtered channel
+		case *sink.HTTPSink:
-		rawChan := mon.Subscribe()
+			pipeline.HTTPSinks = append(pipeline.HTTPSinks, s)
-		tcpChan := make(chan monitor.LogEntry, cfg.TCPServer.BufferSize)
+		case *sink.TCPSink:
-
+			pipeline.TCPSinks = append(pipeline.TCPSinks, s)
-		// Start filter goroutine for TCP
-		s.wg.Add(1)
-		go func() {
-			defer s.wg.Done()
-			defer close(tcpChan)
-			s.filterLoop(streamCtx, rawChan, tcpChan, filterChain)
-		}()
-
-		ls.TCPServer = transport.NewTCPStreamer(
-			tcpChan,
-			*cfg.TCPServer,
-			s.logger) // Pass parent logger
-
-		if err := s.startTCPServer(ls); err != nil {
-			ls.Shutdown()
-			s.logger.Error("msg", "Failed to start TCP server",
-				"component", "service",
-				"stream", cfg.Name,
-				"port", cfg.TCPServer.Port,
-				"error", err)
-			return fmt.Errorf("TCP server failed: %w", err)
 		}
 	}
 
-	// Start HTTP server if configured
+	// Start all sources
-	if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
+	for i, src := range pipeline.Sources {
-		// Create filtered channel
+		if err := src.Start(); err != nil {
-		rawChan := mon.Subscribe()
+			pipeline.Shutdown()
-		httpChan := make(chan monitor.LogEntry, cfg.HTTPServer.BufferSize)
+			return fmt.Errorf("failed to start source[%d]: %w", i, err)
-
-		// Start filter goroutine for HTTP
-		s.wg.Add(1)
-		go func() {
-			defer s.wg.Done()
-			defer close(httpChan)
-			s.filterLoop(streamCtx, rawChan, httpChan, filterChain)
-		}()
-
-		ls.HTTPServer = transport.NewHTTPStreamer(
-			httpChan,
-			*cfg.HTTPServer,
-			s.logger) // Pass parent logger
-
-		if err := s.startHTTPServer(ls); err != nil {
-			ls.Shutdown()
-			s.logger.Error("msg", "Failed to start HTTP server",
-				"component", "service",
-				"stream", cfg.Name,
-				"port", cfg.HTTPServer.Port,
-				"error", err)
-			return fmt.Errorf("HTTP server failed: %w", err)
 		}
 	}
 
-	ls.startStatsUpdater(streamCtx)
+	// Start all sinks
+	for i, sinkInst := range pipeline.Sinks {
+		if err := sinkInst.Start(pipelineCtx); err != nil {
+			pipeline.Shutdown()
+			return fmt.Errorf("failed to start sink[%d]: %w", i, err)
+		}
+	}
 
-	s.streams[cfg.Name] = ls
+	// Wire sources to sinks through filters
-	s.logger.Info("msg", "Stream created successfully", "stream", cfg.Name)
+	s.wirePipeline(pipeline)
+
+	// Start stats updater
+	pipeline.startStatsUpdater(pipelineCtx)
+
+	s.pipelines[cfg.Name] = pipeline
+	s.logger.Info("msg", "Pipeline created successfully", "pipeline", cfg.Name)
 	return nil
 }
 
-// filterLoop applies filters to log entries
+// wirePipeline connects sources to sinks through filters
-func (s *Service) filterLoop(ctx context.Context, in <-chan monitor.LogEntry, out chan<- monitor.LogEntry, chain *filter.Chain) {
+func (s *Service) wirePipeline(p *Pipeline) {
+	// For each source, subscribe and process entries
+	for _, src := range p.Sources {
+		srcChan := src.Subscribe()
+
+		// Create a processing goroutine for this source
+		p.wg.Add(1)
+		go func(source source.Source, entries <-chan source.LogEntry) {
+			defer p.wg.Done()
+
 			for {
 				select {
-		case <-ctx.Done():
+				case <-p.ctx.Done():
 					return
-		case entry, ok := <-in:
+				case entry, ok := <-entries:
 					if !ok {
 						return
 					}
 
-			// Apply filter chain if configured
+					p.Stats.TotalEntriesProcessed.Add(1)
-			if chain == nil || chain.Apply(entry) {
+
+					// Apply filters if configured
+					if p.FilterChain != nil {
+						if !p.FilterChain.Apply(entry) {
+							p.Stats.TotalEntriesFiltered.Add(1)
+							continue
+						}
+					}
+
+					// Send to all sinks
+					for _, sinkInst := range p.Sinks {
 						select {
-				case out <- entry:
+						case sinkInst.Input() <- entry:
-				case <-ctx.Done():
+						case <-p.ctx.Done():
 							return
 						default:
-					// Drop if output buffer is full
+							// Drop if sink buffer is full
-					s.logger.Debug("msg", "Dropped log entry - buffer full")
+							s.logger.Debug("msg", "Dropped log entry - sink buffer full",
+								"pipeline", p.Name)
 						}
 					}
 				}
 			}
+		}(src, srcChan)
+	}
 }
 
-func (s *Service) GetStream(name string) (*LogStream, error) {
+// createSource creates a source instance based on configuration
+func (s *Service) createSource(cfg config.SourceConfig) (source.Source, error) {
+	switch cfg.Type {
+	case "directory":
+		return source.NewDirectorySource(cfg.Options, s.logger)
+	case "stdin":
+		return source.NewStdinSource(cfg.Options, s.logger)
+	default:
+		return nil, fmt.Errorf("unknown source type: %s", cfg.Type)
+	}
+}
+
+// createSink creates a sink instance based on configuration
+func (s *Service) createSink(cfg config.SinkConfig) (sink.Sink, error) {
+	switch cfg.Type {
+	case "http":
+		return sink.NewHTTPSink(cfg.Options, s.logger)
+	case "tcp":
+		return sink.NewTCPSink(cfg.Options, s.logger)
+	case "file":
+		return sink.NewFileSink(cfg.Options, s.logger)
+	case "stdout":
+		return sink.NewStdoutSink(cfg.Options, s.logger)
+	case "stderr":
+		return sink.NewStderrSink(cfg.Options, s.logger)
+	default:
+		return nil, fmt.Errorf("unknown sink type: %s", cfg.Type)
+	}
+}
+
+// GetPipeline returns a pipeline by name
+func (s *Service) GetPipeline(name string) (*Pipeline, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
-	stream, exists := s.streams[name]
+	pipeline, exists := s.pipelines[name]
 	if !exists {
-		return nil, fmt.Errorf("transport '%s' not found", name)
+		return nil, fmt.Errorf("pipeline '%s' not found", name)
 	}
-	return stream, nil
+	return pipeline, nil
 }
 
+// ListStreams is deprecated, use ListPipelines
 func (s *Service) ListStreams() []string {
+	s.logger.Warn("msg", "ListStreams is deprecated, use ListPipelines",
+		"component", "service")
+	return s.ListPipelines()
+}
+
+// ListPipelines returns all pipeline names
+func (s *Service) ListPipelines() []string {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
-	names := make([]string, 0, len(s.streams))
+	names := make([]string, 0, len(s.pipelines))
-	for name := range s.streams {
+	for name := range s.pipelines {
 		names = append(names, name)
 	}
 	return names
 }
 
+// RemoveStream is deprecated, use RemovePipeline
 func (s *Service) RemoveStream(name string) error {
+	s.logger.Warn("msg", "RemoveStream is deprecated, use RemovePipeline",
+		"component", "service")
+	return s.RemovePipeline(name)
+}
+
+// RemovePipeline stops and removes a pipeline
+func (s *Service) RemovePipeline(name string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	stream, exists := s.streams[name]
+	pipeline, exists := s.pipelines[name]
 	if !exists {
-		err := fmt.Errorf("transport '%s' not found", name)
+		err := fmt.Errorf("pipeline '%s' not found", name)
-		s.logger.Warn("msg", "Cannot remove non-existent stream",
+		s.logger.Warn("msg", "Cannot remove non-existent pipeline",
 			"component", "service",
-			"stream", name,
+			"pipeline", name,
 			"error", err)
 		return err
 	}
 
-	s.logger.Info("msg", "Removing stream", "stream", name)
+	s.logger.Info("msg", "Removing pipeline", "pipeline", name)
-	stream.Shutdown()
+	pipeline.Shutdown()
-	delete(s.streams, name)
+	delete(s.pipelines, name)
 	return nil
 }
 
+// Shutdown stops all pipelines
 func (s *Service) Shutdown() {
 	s.logger.Info("msg", "Service shutdown initiated")
 
 	s.mu.Lock()
-	streams := make([]*LogStream, 0, len(s.streams))
+	pipelines := make([]*Pipeline, 0, len(s.pipelines))
-	for _, stream := range s.streams {
+	for _, pipeline := range s.pipelines {
-		streams = append(streams, stream)
+		pipelines = append(pipelines, pipeline)
 	}
 	s.mu.Unlock()
 
-	// Stop all streams concurrently
+	// Stop all pipelines concurrently
 	var wg sync.WaitGroup
-	for _, stream := range streams {
+	for _, pipeline := range pipelines {
 		wg.Add(1)
-		go func(ls *LogStream) {
+		go func(p *Pipeline) {
 			defer wg.Done()
-			ls.Shutdown()
+			p.Shutdown()
-		}(stream)
+		}(pipeline)
 	}
 	wg.Wait()
 
@@ -266,68 +297,19 @@ func (s *Service) Shutdown() {
 	s.logger.Info("msg", "Service shutdown complete")
 }
 
+// GetGlobalStats returns statistics for all pipelines
 func (s *Service) GetGlobalStats() map[string]any {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
 	stats := map[string]any{
-		"streams":       make(map[string]any),
+		"pipelines":       make(map[string]any),
-		"total_streams": len(s.streams),
+		"total_pipelines": len(s.pipelines),
 	}
 
-	for name, stream := range s.streams {
+	for name, pipeline := range s.pipelines {
-		stats["streams"].(map[string]any)[name] = stream.GetStats()
+		stats["pipelines"].(map[string]any)[name] = pipeline.GetStats()
 	}
 
 	return stats
 }
-
-func (s *Service) startTCPServer(ls *LogStream) error {
-	errChan := make(chan error, 1)
-	s.wg.Add(1)
-
-	go func() {
-		defer s.wg.Done()
-		if err := ls.TCPServer.Start(); err != nil {
-			errChan <- err
-		}
-	}()
-
-	// Check startup
-	select {
-	case err := <-errChan:
-		s.logger.Error("msg", "TCP server startup failed immediately",
-			"component", "service",
-			"stream", ls.Name,
-			"error", err)
-		return err
-	case <-time.After(time.Second):
-		s.logger.Debug("msg", "TCP server started", "stream", ls.Name)
-		return nil
-	}
-}
-
-func (s *Service) startHTTPServer(ls *LogStream) error {
-	errChan := make(chan error, 1)
-	s.wg.Add(1)
-
-	go func() {
-		defer s.wg.Done()
-		if err := ls.HTTPServer.Start(); err != nil {
-			errChan <- err
-		}
-	}()
-
-	// Check startup
-	select {
-	case err := <-errChan:
-		s.logger.Error("msg", "HTTP server startup failed immediately",
-			"component", "service",
-			"stream", ls.Name,
-			"error", err)
-		return err
-	case <-time.After(time.Second):
-		s.logger.Debug("msg", "HTTP server started", "stream", ls.Name)
-		return nil
-	}
-}

src/internal/sink/console.go

@@ -0,0 +1,215 @@
+// FILE: src/internal/sink/console.go
+package sink
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"logwisp/src/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// StdoutSink writes log entries to stdout
+type StdoutSink struct {
+	input     chan source.LogEntry
+	writer    *log.Logger
+	done      chan struct{}
+	startTime time.Time
+	logger    *log.Logger
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+// NewStdoutSink creates a new stdout sink
+func NewStdoutSink(options map[string]any, logger *log.Logger) (*StdoutSink, error) {
+	// Create internal logger for stdout writing
+	writer := log.NewLogger()
+	if err := writer.InitWithDefaults(
+		"enable_stdout=true",
+		"disable_file=true",
+		"stdout_target=stdout",
+		"show_timestamp=false", // We format our own
+		"show_level=false",     // We format our own
+	); err != nil {
+		return nil, fmt.Errorf("failed to initialize stdout writer: %w", err)
+	}
+
+	bufferSize := 1000
+	if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
+		bufferSize = bufSize
+	}
+
+	s := &StdoutSink{
+		input:     make(chan source.LogEntry, bufferSize),
+		writer:    writer,
+		done:      make(chan struct{}),
+		startTime: time.Now(),
+		logger:    logger,
+	}
+	s.lastProcessed.Store(time.Time{})
+
+	return s, nil
+}
+
+func (s *StdoutSink) Input() chan<- source.LogEntry {
+	return s.input
+}
+
+func (s *StdoutSink) Start(ctx context.Context) error {
+	go s.processLoop(ctx)
+	s.logger.Info("msg", "Stdout sink started", "component", "stdout_sink")
+	return nil
+}
+
+func (s *StdoutSink) Stop() {
+	s.logger.Info("msg", "Stopping stdout sink")
+	close(s.done)
+	s.writer.Shutdown(1 * time.Second)
+	s.logger.Info("msg", "Stdout sink stopped")
+}
+
+func (s *StdoutSink) GetStats() SinkStats {
+	lastProc, _ := s.lastProcessed.Load().(time.Time)
+
+	return SinkStats{
+		Type:           "stdout",
+		TotalProcessed: s.totalProcessed.Load(),
+		StartTime:      s.startTime,
+		LastProcessed:  lastProc,
+		Details:        map[string]any{},
+	}
+}
+
+func (s *StdoutSink) processLoop(ctx context.Context) {
+	for {
+		select {
+		case entry, ok := <-s.input:
+			if !ok {
+				return
+			}
+
+			s.totalProcessed.Add(1)
+			s.lastProcessed.Store(time.Now())
+
+			// Format and write
+			timestamp := entry.Time.Format(time.RFC3339Nano)
+			level := entry.Level
+			if level == "" {
+				level = "INFO"
+			}
+
+			s.writer.Message(fmt.Sprintf("[%s] %s %s", timestamp, level, entry.Message))
+
+		case <-ctx.Done():
+			return
+		case <-s.done:
+			return
+		}
+	}
+}
+
+// StderrSink writes log entries to stderr
+type StderrSink struct {
+	input     chan source.LogEntry
+	writer    *log.Logger
+	done      chan struct{}
+	startTime time.Time
+	logger    *log.Logger
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+// NewStderrSink creates a new stderr sink
+func NewStderrSink(options map[string]any, logger *log.Logger) (*StderrSink, error) {
+	// Create internal logger for stderr writing
+	writer := log.NewLogger()
+	if err := writer.InitWithDefaults(
+		"enable_stdout=true",
+		"disable_file=true",
+		"stdout_target=stderr",
+		"show_timestamp=false", // We format our own
+		"show_level=false",     // We format our own
+	); err != nil {
+		return nil, fmt.Errorf("failed to initialize stderr writer: %w", err)
+	}
+
+	bufferSize := 1000
+	if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
+		bufferSize = bufSize
+	}
+
+	s := &StderrSink{
+		input:     make(chan source.LogEntry, bufferSize),
+		writer:    writer,
+		done:      make(chan struct{}),
+		startTime: time.Now(),
+		logger:    logger,
+	}
+	s.lastProcessed.Store(time.Time{})
+
+	return s, nil
+}
+
+func (s *StderrSink) Input() chan<- source.LogEntry {
+	return s.input
+}
+
+func (s *StderrSink) Start(ctx context.Context) error {
+	go s.processLoop(ctx)
+	s.logger.Info("msg", "Stderr sink started", "component", "stderr_sink")
+	return nil
+}
+
+func (s *StderrSink) Stop() {
+	s.logger.Info("msg", "Stopping stderr sink")
+	close(s.done)
+	s.writer.Shutdown(1 * time.Second)
+	s.logger.Info("msg", "Stderr sink stopped")
+}
+
+func (s *StderrSink) GetStats() SinkStats {
+	lastProc, _ := s.lastProcessed.Load().(time.Time)
+
+	return SinkStats{
+		Type:           "stderr",
+		TotalProcessed: s.totalProcessed.Load(),
+		StartTime:      s.startTime,
+		LastProcessed:  lastProc,
+		Details:        map[string]any{},
+	}
+}
+
+func (s *StderrSink) processLoop(ctx context.Context) {
+	for {
+		select {
+		case entry, ok := <-s.input:
+			if !ok {
+				return
+			}
+
+			s.totalProcessed.Add(1)
+			s.lastProcessed.Store(time.Now())
+
+			// Format and write
+			timestamp := entry.Time.Format(time.RFC3339Nano)
+			level := entry.Level
+			if level == "" {
+				level = "INFO"
+			}
+
+			s.writer.Message(fmt.Sprintf("[%s] %s %s", timestamp, level, entry.Message))
+
+		case <-ctx.Done():
+			return
+		case <-s.done:
+			return
+		}
+	}
+}

src/internal/sink/file.go

@@ -0,0 +1,155 @@
+// FILE: src/internal/sink/file.go
+package sink
+
+import (
+	"context"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"logwisp/src/internal/source"
+
+	"github.com/lixenwraith/log"
+)
+
+// FileSink writes log entries to files with rotation
+type FileSink struct {
+	input     chan source.LogEntry
+	writer    *log.Logger // Internal logger instance for file writing
+	done      chan struct{}
+	startTime time.Time
+	logger    *log.Logger // Application logger
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+// NewFileSink creates a new file sink
+func NewFileSink(options map[string]any, logger *log.Logger) (*FileSink, error) {
+	directory, ok := options["directory"].(string)
+	if !ok || directory == "" {
+		return nil, fmt.Errorf("file sink requires 'directory' option")
+	}
+
+	name, ok := options["name"].(string)
+	if !ok || name == "" {
+		return nil, fmt.Errorf("file sink requires 'name' option")
+	}
+
+	// Create configuration for the internal log writer
+	var configArgs []string
+	configArgs = append(configArgs,
+		fmt.Sprintf("directory=%s", directory),
+		fmt.Sprintf("name=%s", name),
+		"enable_stdout=false",  // File only
+		"show_timestamp=false", // We already have timestamps in entries
+		"show_level=false",     // We already have levels in entries
+	)
+
+	// Add optional configurations
+	if maxSize, ok := toInt(options["max_size_mb"]); ok && maxSize > 0 {
+		configArgs = append(configArgs, fmt.Sprintf("max_size_mb=%d", maxSize))
+	}
+
+	if maxTotalSize, ok := toInt(options["max_total_size_mb"]); ok && maxTotalSize >= 0 {
+		configArgs = append(configArgs, fmt.Sprintf("max_total_size_mb=%d", maxTotalSize))
+	}
+
+	if retention, ok := toFloat(options["retention_hours"]); ok && retention > 0 {
+		configArgs = append(configArgs, fmt.Sprintf("retention_period_hrs=%.1f", retention))
+	}
+
+	if minDiskFree, ok := toInt(options["min_disk_free_mb"]); ok && minDiskFree > 0 {
+		configArgs = append(configArgs, fmt.Sprintf("min_disk_free_mb=%d", minDiskFree))
+	}
+
+	// Create internal logger for file writing
+	writer := log.NewLogger()
+	if err := writer.InitWithDefaults(configArgs...); err != nil {
+		return nil, fmt.Errorf("failed to initialize file writer: %w", err)
+	}
+
+	// Buffer size for input channel
+	bufferSize := 1000
+	if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
+		bufferSize = bufSize
+	}
+
+	fs := &FileSink{
+		input:     make(chan source.LogEntry, bufferSize),
+		writer:    writer,
+		done:      make(chan struct{}),
+		startTime: time.Now(),
+		logger:    logger,
+	}
+	fs.lastProcessed.Store(time.Time{})
+
+	return fs, nil
+}
+
+func (fs *FileSink) Input() chan<- source.LogEntry {
+	return fs.input
+}
+
+func (fs *FileSink) Start(ctx context.Context) error {
+	go fs.processLoop(ctx)
+	fs.logger.Info("msg", "File sink started", "component", "file_sink")
+	return nil
+}
+
+func (fs *FileSink) Stop() {
+	fs.logger.Info("msg", "Stopping file sink")
+	close(fs.done)
+
+	// Shutdown the writer with timeout
+	if err := fs.writer.Shutdown(2 * time.Second); err != nil {
+		fs.logger.Error("msg", "Error shutting down file writer",
+			"component", "file_sink",
+			"error", err)
+	}
+
+	fs.logger.Info("msg", "File sink stopped")
+}
+
+func (fs *FileSink) GetStats() SinkStats {
+	lastProc, _ := fs.lastProcessed.Load().(time.Time)
+
+	return SinkStats{
+		Type:           "file",
+		TotalProcessed: fs.totalProcessed.Load(),
+		StartTime:      fs.startTime,
+		LastProcessed:  lastProc,
+		Details:        map[string]any{},
+	}
+}
+
+func (fs *FileSink) processLoop(ctx context.Context) {
+	for {
+		select {
+		case entry, ok := <-fs.input:
+			if !ok {
+				return
+			}
+
+			fs.totalProcessed.Add(1)
+			fs.lastProcessed.Store(time.Now())
+
+			// Format the log entry
+			// Include timestamp and level since we disabled them in the writer
+			timestamp := entry.Time.Format(time.RFC3339Nano)
+			level := entry.Level
+			if level == "" {
+				level = "INFO"
+			}
+
+			// Write to file using the internal logger
+			fs.writer.Message(fmt.Sprintf("[%s] %s %s", timestamp, level, entry.Message))
+
+		case <-ctx.Done():
+			return
+		case <-fs.done:
+			return
+		}
+	}
+}

src/internal/sink/http.go

@@ -1,5 +1,5 @@
-// FILE: src/internal/transport/httpstreamer.go
+// FILE: src/internal/sink/http.go
-package transport
+package sink
 
 import (
 	"bufio"
@@ -12,8 +12,8 @@ import (
 	"time"
 
 	"logwisp/src/internal/config"
-	"logwisp/src/internal/monitor"
 	"logwisp/src/internal/ratelimit"
+	"logwisp/src/internal/source"
 	"logwisp/src/internal/version"
 
 	"github.com/lixenwraith/log"
@@ -21,9 +21,10 @@ import (
 	"github.com/valyala/fasthttp"
 )
 
-type HTTPStreamer struct {
+// HTTPSink streams log entries via Server-Sent Events
-	logChan       chan monitor.LogEntry
+type HTTPSink struct {
-	config        config.HTTPConfig
+	input         chan source.LogEntry
+	config        HTTPConfig
 	server        *fasthttp.Server
 	activeClients atomic.Int32
 	mu            sync.RWMutex
@@ -41,50 +42,115 @@ type HTTPStreamer struct {
 
 	// Rate limiting
 	rateLimiter *ratelimit.Limiter
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
 }
 
-func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig, logger *log.Logger) *HTTPStreamer {
+// HTTPConfig holds HTTP sink configuration
-	// Set default paths if not configured
+type HTTPConfig struct {
-	streamPath := cfg.StreamPath
+	Port       int
-	if streamPath == "" {
+	BufferSize int
-		streamPath = "/transport"
+	StreamPath string
-	}
+	StatusPath string
-	statusPath := cfg.StatusPath
+	Heartbeat  config.HeartbeatConfig
-	if statusPath == "" {
+	SSL        *config.SSLConfig
-		statusPath = "/status"
+	RateLimit  *config.RateLimitConfig
 }
 
-	h := &HTTPStreamer{
+// NewHTTPSink creates a new HTTP streaming sink
-		logChan:    logChan,
+func NewHTTPSink(options map[string]any, logger *log.Logger) (*HTTPSink, error) {
+	cfg := HTTPConfig{
+		Port:       8080,
+		BufferSize: 1000,
+		StreamPath: "/transport",
+		StatusPath: "/status",
+	}
+
+	// Extract configuration from options
+	if port, ok := toInt(options["port"]); ok {
+		cfg.Port = port
+	}
+	if bufSize, ok := toInt(options["buffer_size"]); ok {
+		cfg.BufferSize = bufSize
+	}
+	if path, ok := options["stream_path"].(string); ok {
+		cfg.StreamPath = path
+	}
+	if path, ok := options["status_path"].(string); ok {
+		cfg.StatusPath = path
+	}
+
+	// Extract heartbeat config
+	if hb, ok := options["heartbeat"].(map[string]any); ok {
+		cfg.Heartbeat.Enabled, _ = hb["enabled"].(bool)
+		if interval, ok := toInt(hb["interval_seconds"]); ok {
+			cfg.Heartbeat.IntervalSeconds = interval
+		}
+		cfg.Heartbeat.IncludeTimestamp, _ = hb["include_timestamp"].(bool)
+		cfg.Heartbeat.IncludeStats, _ = hb["include_stats"].(bool)
+		if format, ok := hb["format"].(string); ok {
+			cfg.Heartbeat.Format = format
+		}
+	}
+
+	// Extract rate limit config
+	if rl, ok := options["rate_limit"].(map[string]any); ok {
+		cfg.RateLimit = &config.RateLimitConfig{}
+		cfg.RateLimit.Enabled, _ = rl["enabled"].(bool)
+		if rps, ok := toFloat(rl["requests_per_second"]); ok {
+			cfg.RateLimit.RequestsPerSecond = rps
+		}
+		if burst, ok := toInt(rl["burst_size"]); ok {
+			cfg.RateLimit.BurstSize = burst
+		}
+		if limitBy, ok := rl["limit_by"].(string); ok {
+			cfg.RateLimit.LimitBy = limitBy
+		}
+		if respCode, ok := toInt(rl["response_code"]); ok {
+			cfg.RateLimit.ResponseCode = respCode
+		}
+		if msg, ok := rl["response_message"].(string); ok {
+			cfg.RateLimit.ResponseMessage = msg
+		}
+		if maxPerIP, ok := toInt(rl["max_connections_per_ip"]); ok {
+			cfg.RateLimit.MaxConnectionsPerIP = maxPerIP
+		}
+		if maxTotal, ok := toInt(rl["max_total_connections"]); ok {
+			cfg.RateLimit.MaxTotalConnections = maxTotal
+		}
+	}
+
+	h := &HTTPSink{
+		input:      make(chan source.LogEntry, cfg.BufferSize),
 		config:     cfg,
 		startTime:  time.Now(),
 		done:       make(chan struct{}),
-		streamPath: streamPath,
+		streamPath: cfg.StreamPath,
-		statusPath: statusPath,
+		statusPath: cfg.StatusPath,
-		standalone: true, // Default to standalone mode
+		standalone: true,
 		logger:     logger,
 	}
+	h.lastProcessed.Store(time.Time{})
 
 	// Initialize rate limiter if configured
 	if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
 		h.rateLimiter = ratelimit.New(*cfg.RateLimit)
 	}
 
-	return h
+	return h, nil
 }
 
-// Configures the streamer for use with a router
+func (h *HTTPSink) Input() chan<- source.LogEntry {
-func (h *HTTPStreamer) SetRouterMode() {
+	return h.input
-	h.standalone = false
-	h.logger.Debug("msg", "HTTP streamer set to router mode",
-		"component", "http_streamer")
 }
 
-func (h *HTTPStreamer) Start() error {
+func (h *HTTPSink) Start(ctx context.Context) error {
 	if !h.standalone {
 		// In router mode, don't start our own server
-		h.logger.Debug("msg", "HTTP streamer in router mode, skipping server start",
+		h.logger.Debug("msg", "HTTP sink in router mode, skipping server start",
-			"component", "http_streamer")
+			"component", "http_sink")
 		return nil
 	}
 
@@ -104,7 +170,7 @@ func (h *HTTPStreamer) Start() error {
 	errChan := make(chan error, 1)
 	go func() {
 		h.logger.Info("msg", "HTTP server started",
-			"component", "http_streamer",
+			"component", "http_sink",
 			"port", h.config.Port,
 			"stream_path", h.streamPath,
 			"status_path", h.statusPath)
@@ -120,16 +186,12 @@ func (h *HTTPStreamer) Start() error {
 		return err
 	case <-time.After(100 * time.Millisecond):
 		// Server started successfully
-		h.logger.Info("msg", "HTTP server started",
-			"port", h.config.Port,
-			"stream_path", h.streamPath,
-			"status_path", h.statusPath)
 		return nil
 	}
 }
 
-func (h *HTTPStreamer) Stop() {
+func (h *HTTPSink) Stop() {
-	h.logger.Info("msg", "Stopping HTTP server")
+	h.logger.Info("msg", "Stopping HTTP sink")
 
 	// Signal all client handlers to stop
 	close(h.done)
@@ -144,14 +206,48 @@ func (h *HTTPStreamer) Stop() {
 	// Wait for all active client handlers to finish
 	h.wg.Wait()
 
-	h.logger.Info("msg", "HTTP server stopped")
+	h.logger.Info("msg", "HTTP sink stopped")
 }
 
-func (h *HTTPStreamer) RouteRequest(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) GetStats() SinkStats {
+	lastProc, _ := h.lastProcessed.Load().(time.Time)
+
+	var rateLimitStats map[string]any
+	if h.rateLimiter != nil {
+		rateLimitStats = h.rateLimiter.GetStats()
+	}
+
+	return SinkStats{
+		Type:              "http",
+		TotalProcessed:    h.totalProcessed.Load(),
+		ActiveConnections: h.activeClients.Load(),
+		StartTime:         h.startTime,
+		LastProcessed:     lastProc,
+		Details: map[string]any{
+			"port":        h.config.Port,
+			"buffer_size": h.config.BufferSize,
+			"endpoints": map[string]string{
+				"stream": h.streamPath,
+				"status": h.statusPath,
+			},
+			"rate_limit": rateLimitStats,
+		},
+	}
+}
+
+// SetRouterMode configures the sink for use with a router
+func (h *HTTPSink) SetRouterMode() {
+	h.standalone = false
+	h.logger.Debug("msg", "HTTP sink set to router mode",
+		"component", "http_sink")
+}
+
+// RouteRequest handles a request from the router
+func (h *HTTPSink) RouteRequest(ctx *fasthttp.RequestCtx) {
 	h.requestHandler(ctx)
 }
 
-func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) {
 	// Check rate limit first
 	remoteAddr := ctx.RemoteAddr().String()
 	if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed {
@@ -182,7 +278,7 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
 	}
 }
 
-func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
 	// Track connection for rate limiting
 	remoteAddr := ctx.RemoteAddr().String()
 	if h.rateLimiter != nil {
@@ -198,18 +294,21 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
 	ctx.Response.Header.Set("X-Accel-Buffering", "no")
 
 	// Create subscription for this client
-	clientChan := make(chan monitor.LogEntry, h.config.BufferSize)
+	clientChan := make(chan source.LogEntry, h.config.BufferSize)
 	clientDone := make(chan struct{})
 
-	// Subscribe to monitor's broadcast
+	// Subscribe to input channel
 	go func() {
 		defer close(clientChan)
 		for {
 			select {
-			case entry, ok := <-h.logChan:
+			case entry, ok := <-h.input:
 				if !ok {
 					return
 				}
+				h.totalProcessed.Add(1)
+				h.lastProcessed.Store(time.Now())
+
 				select {
 				case clientChan <- entry:
 				case <-clientDone:
@@ -219,7 +318,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
 				default:
 					// Drop if client buffer full
 					h.logger.Debug("msg", "Dropped entry for slow client",
-						"component", "http_streamer",
+						"component", "http_sink",
 						"remote_addr", remoteAddr)
 				}
 			case <-clientDone:
@@ -239,6 +338,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
 
 		h.wg.Add(1)
 		defer func() {
+			close(clientDone)
 			newCount := h.activeClients.Add(-1)
 			h.logger.Debug("msg", "HTTP client disconnected",
 				"remote_addr", remoteAddr,
@@ -277,7 +377,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
 				data, err := json.Marshal(entry)
 				if err != nil {
 					h.logger.Error("msg", "Failed to marshal log entry",
-						"component", "http_streamer",
+						"component", "http_sink",
 						"error", err,
 						"entry_source", entry.Source)
 					continue
@@ -308,7 +408,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
 	ctx.SetBodyStreamWriter(streamFunc)
 }
 
-func (h *HTTPStreamer) formatHeartbeat() string {
+func (h *HTTPSink) formatHeartbeat() string {
 	if !h.config.Heartbeat.Enabled {
 		return ""
 	}
@@ -346,7 +446,7 @@ func (h *HTTPStreamer) formatHeartbeat() string {
 	return fmt.Sprintf(": %s\n\n", strings.Join(parts, " "))
 }
 
-func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) handleStatus(ctx *fasthttp.RequestCtx) {
 	ctx.SetContentType("application/json")
 
 	var rateLimitStats any
@@ -390,17 +490,17 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
 	ctx.SetBody(data)
 }
 
-// Returns the current number of active clients
+// GetActiveConnections returns the current number of active clients
-func (h *HTTPStreamer) GetActiveConnections() int32 {
+func (h *HTTPSink) GetActiveConnections() int32 {
 	return h.activeClients.Load()
 }
 
-// Returns the configured transport endpoint path
+// GetStreamPath returns the configured transport endpoint path
-func (h *HTTPStreamer) GetStreamPath() string {
+func (h *HTTPSink) GetStreamPath() string {
 	return h.streamPath
 }
 
-// Returns the configured status endpoint path
+// GetStatusPath returns the configured status endpoint path
-func (h *HTTPStreamer) GetStatusPath() string {
+func (h *HTTPSink) GetStatusPath() string {
 	return h.statusPath
 }

src/internal/sink/sink.go

@@ -0,0 +1,61 @@
+// FILE: src/internal/sink/sink.go
+package sink
+
+import (
+	"context"
+	"time"
+
+	"logwisp/src/internal/source"
+)
+
+// Sink represents an output destination for log entries
+type Sink interface {
+	// Input returns the channel for sending log entries to this sink
+	Input() chan<- source.LogEntry
+
+	// Start begins processing log entries
+	Start(ctx context.Context) error
+
+	// Stop gracefully shuts down the sink
+	Stop()
+
+	// GetStats returns sink statistics
+	GetStats() SinkStats
+}
+
+// SinkStats contains statistics about a sink
+type SinkStats struct {
+	Type              string
+	TotalProcessed    uint64
+	ActiveConnections int32
+	StartTime         time.Time
+	LastProcessed     time.Time
+	Details           map[string]any
+}
+
+// Helper functions for type conversion
+func toInt(v any) (int, bool) {
+	switch val := v.(type) {
+	case int:
+		return val, true
+	case int64:
+		return int(val), true
+	case float64:
+		return int(val), true
+	default:
+		return 0, false
+	}
+}
+
+func toFloat(v any) (float64, bool) {
+	switch val := v.(type) {
+	case float64:
+		return val, true
+	case int:
+		return float64(val), true
+	case int64:
+		return float64(val), true
+	default:
+		return 0, false
+	}
+}

src/internal/sink/tcp.go

@@ -0,0 +1,380 @@
+// FILE: src/internal/sink/tcp.go
+package sink
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"logwisp/src/internal/config"
+	"logwisp/src/internal/ratelimit"
+	"logwisp/src/internal/source"
+
+	"github.com/lixenwraith/log"
+	"github.com/panjf2000/gnet/v2"
+)
+
+// TCPSink streams log entries via TCP
+type TCPSink struct {
+	input       chan source.LogEntry
+	config      TCPConfig
+	server      *tcpServer
+	done        chan struct{}
+	activeConns atomic.Int32
+	startTime   time.Time
+	engine      *gnet.Engine
+	engineMu    sync.Mutex
+	wg          sync.WaitGroup
+	rateLimiter *ratelimit.Limiter
+	logger      *log.Logger
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	lastProcessed  atomic.Value // time.Time
+}
+
+// TCPConfig holds TCP sink configuration
+type TCPConfig struct {
+	Port       int
+	BufferSize int
+	Heartbeat  config.HeartbeatConfig
+	SSL        *config.SSLConfig
+	RateLimit  *config.RateLimitConfig
+}
+
+// NewTCPSink creates a new TCP streaming sink
+func NewTCPSink(options map[string]any, logger *log.Logger) (*TCPSink, error) {
+	cfg := TCPConfig{
+		Port:       9090,
+		BufferSize: 1000,
+	}
+
+	// Extract configuration from options
+	if port, ok := toInt(options["port"]); ok {
+		cfg.Port = port
+	}
+	if bufSize, ok := toInt(options["buffer_size"]); ok {
+		cfg.BufferSize = bufSize
+	}
+
+	// Extract heartbeat config
+	if hb, ok := options["heartbeat"].(map[string]any); ok {
+		cfg.Heartbeat.Enabled, _ = hb["enabled"].(bool)
+		if interval, ok := toInt(hb["interval_seconds"]); ok {
+			cfg.Heartbeat.IntervalSeconds = interval
+		}
+		cfg.Heartbeat.IncludeTimestamp, _ = hb["include_timestamp"].(bool)
+		cfg.Heartbeat.IncludeStats, _ = hb["include_stats"].(bool)
+		if format, ok := hb["format"].(string); ok {
+			cfg.Heartbeat.Format = format
+		}
+	}
+
+	// Extract rate limit config
+	if rl, ok := options["rate_limit"].(map[string]any); ok {
+		cfg.RateLimit = &config.RateLimitConfig{}
+		cfg.RateLimit.Enabled, _ = rl["enabled"].(bool)
+		if rps, ok := toFloat(rl["requests_per_second"]); ok {
+			cfg.RateLimit.RequestsPerSecond = rps
+		}
+		if burst, ok := toInt(rl["burst_size"]); ok {
+			cfg.RateLimit.BurstSize = burst
+		}
+		if limitBy, ok := rl["limit_by"].(string); ok {
+			cfg.RateLimit.LimitBy = limitBy
+		}
+		if respCode, ok := toInt(rl["response_code"]); ok {
+			cfg.RateLimit.ResponseCode = respCode
+		}
+		if msg, ok := rl["response_message"].(string); ok {
+			cfg.RateLimit.ResponseMessage = msg
+		}
+		if maxPerIP, ok := toInt(rl["max_connections_per_ip"]); ok {
+			cfg.RateLimit.MaxConnectionsPerIP = maxPerIP
+		}
+		if maxTotal, ok := toInt(rl["max_total_connections"]); ok {
+			cfg.RateLimit.MaxTotalConnections = maxTotal
+		}
+	}
+
+	t := &TCPSink{
+		input:     make(chan source.LogEntry, cfg.BufferSize),
+		config:    cfg,
+		done:      make(chan struct{}),
+		startTime: time.Now(),
+		logger:    logger,
+	}
+	t.lastProcessed.Store(time.Time{})
+
+	if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
+		t.rateLimiter = ratelimit.New(*cfg.RateLimit)
+	}
+
+	return t, nil
+}
+
+func (t *TCPSink) Input() chan<- source.LogEntry {
+	return t.input
+}
+
+func (t *TCPSink) Start(ctx context.Context) error {
+	t.server = &tcpServer{sink: t}
+
+	// Start log broadcast loop
+	t.wg.Add(1)
+	go func() {
+		defer t.wg.Done()
+		t.broadcastLoop()
+	}()
+
+	// Configure gnet
+	addr := fmt.Sprintf("tcp://:%d", t.config.Port)
+
+	// Run gnet in separate goroutine to avoid blocking
+	errChan := make(chan error, 1)
+	go func() {
+		t.logger.Info("msg", "Starting TCP server",
+			"component", "tcp_sink",
+			"port", t.config.Port)
+
+		err := gnet.Run(t.server, addr,
+			gnet.WithLogger(noopLogger{}),
+			gnet.WithMulticore(true),
+			gnet.WithReusePort(true),
+		)
+		if err != nil {
+			t.logger.Error("msg", "TCP server failed",
+				"component", "tcp_sink",
+				"port", t.config.Port,
+				"error", err)
+		}
+		errChan <- err
+	}()
+
+	// Wait briefly for server to start or fail
+	select {
+	case err := <-errChan:
+		// Server failed immediately
+		close(t.done)
+		t.wg.Wait()
+		return err
+	case <-time.After(100 * time.Millisecond):
+		// Server started successfully
+		t.logger.Info("msg", "TCP server started", "port", t.config.Port)
+		return nil
+	}
+}
+
+func (t *TCPSink) Stop() {
+	t.logger.Info("msg", "Stopping TCP sink")
+	// Signal broadcast loop to stop
+	close(t.done)
+
+	// Stop gnet engine if running
+	t.engineMu.Lock()
+	engine := t.engine
+	t.engineMu.Unlock()
+
+	if engine != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		(*engine).Stop(ctx) // Dereference the pointer
+	}
+
+	// Wait for broadcast loop to finish
+	t.wg.Wait()
+
+	t.logger.Info("msg", "TCP sink stopped")
+}
+
+func (t *TCPSink) GetStats() SinkStats {
+	lastProc, _ := t.lastProcessed.Load().(time.Time)
+
+	var rateLimitStats map[string]any
+	if t.rateLimiter != nil {
+		rateLimitStats = t.rateLimiter.GetStats()
+	}
+
+	return SinkStats{
+		Type:              "tcp",
+		TotalProcessed:    t.totalProcessed.Load(),
+		ActiveConnections: t.activeConns.Load(),
+		StartTime:         t.startTime,
+		LastProcessed:     lastProc,
+		Details: map[string]any{
+			"port":        t.config.Port,
+			"buffer_size": t.config.BufferSize,
+			"rate_limit":  rateLimitStats,
+		},
+	}
+}
+
+func (t *TCPSink) broadcastLoop() {
+	var ticker *time.Ticker
+	var tickerChan <-chan time.Time
+
+	if t.config.Heartbeat.Enabled {
+		ticker = time.NewTicker(time.Duration(t.config.Heartbeat.IntervalSeconds) * time.Second)
+		tickerChan = ticker.C
+		defer ticker.Stop()
+	}
+
+	for {
+		select {
+		case entry, ok := <-t.input:
+			if !ok {
+				return
+			}
+			t.totalProcessed.Add(1)
+			t.lastProcessed.Store(time.Now())
+
+			data, err := json.Marshal(entry)
+			if err != nil {
+				t.logger.Error("msg", "Failed to marshal log entry",
+					"component", "tcp_sink",
+					"error", err,
+					"entry_source", entry.Source)
+				continue
+			}
+			data = append(data, '\n')
+
+			t.server.connections.Range(func(key, value any) bool {
+				conn := key.(gnet.Conn)
+				conn.AsyncWrite(data, nil)
+				return true
+			})
+
+		case <-tickerChan:
+			if heartbeat := t.formatHeartbeat(); heartbeat != nil {
+				t.server.connections.Range(func(key, value any) bool {
+					conn := key.(gnet.Conn)
+					conn.AsyncWrite(heartbeat, nil)
+					return true
+				})
+			}
+
+		case <-t.done:
+			return
+		}
+	}
+}
+
+func (t *TCPSink) formatHeartbeat() []byte {
+	if !t.config.Heartbeat.Enabled {
+		return nil
+	}
+
+	data := make(map[string]any)
+	data["type"] = "heartbeat"
+
+	if t.config.Heartbeat.IncludeTimestamp {
+		data["time"] = time.Now().UTC().Format(time.RFC3339Nano)
+	}
+
+	if t.config.Heartbeat.IncludeStats {
+		data["active_connections"] = t.activeConns.Load()
+		data["uptime_seconds"] = int(time.Since(t.startTime).Seconds())
+	}
+
+	// For TCP, always use JSON format
+	jsonData, _ := json.Marshal(data)
+	return append(jsonData, '\n')
+}
+
+// GetActiveConnections returns the current number of connections
+func (t *TCPSink) GetActiveConnections() int32 {
+	return t.activeConns.Load()
+}
+
+// tcpServer handles gnet events
+type tcpServer struct {
+	gnet.BuiltinEventEngine
+	sink        *TCPSink
+	connections sync.Map
+}
+
+func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
+	// Store engine reference for shutdown
+	s.sink.engineMu.Lock()
+	s.sink.engine = &eng
+	s.sink.engineMu.Unlock()
+
+	s.sink.logger.Debug("msg", "TCP server booted",
+		"component", "tcp_sink",
+		"port", s.sink.config.Port)
+	return gnet.None
+}
+
+func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
+	remoteAddr := c.RemoteAddr().String()
+	s.sink.logger.Debug("msg", "TCP connection attempt", "remote_addr", remoteAddr)
+
+	// Check rate limit
+	if s.sink.rateLimiter != nil {
+		// Parse the remote address to get proper net.Addr
+		remoteStr := c.RemoteAddr().String()
+		tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr)
+		if err != nil {
+			s.sink.logger.Warn("msg", "Failed to parse TCP address",
+				"remote_addr", remoteAddr,
+				"error", err)
+			return nil, gnet.Close
+		}
+
+		if !s.sink.rateLimiter.CheckTCP(tcpAddr) {
+			s.sink.logger.Warn("msg", "TCP connection rate limited",
+				"remote_addr", remoteAddr)
+			// Silently close connection when rate limited
+			return nil, gnet.Close
+		}
+
+		// Track connection
+		s.sink.rateLimiter.AddConnection(remoteStr)
+	}
+
+	s.connections.Store(c, struct{}{})
+
+	newCount := s.sink.activeConns.Add(1)
+	s.sink.logger.Debug("msg", "TCP connection opened",
+		"remote_addr", remoteAddr,
+		"active_connections", newCount)
+
+	return nil, gnet.None
+}
+
+func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
+	s.connections.Delete(c)
+
+	remoteAddr := c.RemoteAddr().String()
+
+	// Remove connection tracking
+	if s.sink.rateLimiter != nil {
+		s.sink.rateLimiter.RemoveConnection(c.RemoteAddr().String())
+	}
+
+	newCount := s.sink.activeConns.Add(-1)
+	s.sink.logger.Debug("msg", "TCP connection closed",
+		"remote_addr", remoteAddr,
+		"active_connections", newCount,
+		"error", err)
+	return gnet.None
+}
+
+func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
+	// We don't expect input from clients, just discard
+	c.Discard(-1)
+	return gnet.None
+}
+
+// noopLogger implements gnet's Logger interface but discards everything
+type noopLogger struct{}
+
+func (n noopLogger) Debugf(format string, args ...any) {}
+func (n noopLogger) Infof(format string, args ...any)  {}
+func (n noopLogger) Warnf(format string, args ...any)  {}
+func (n noopLogger) Errorf(format string, args ...any) {}
+func (n noopLogger) Fatalf(format string, args ...any) {}

src/internal/source/directory.go

@@ -0,0 +1,298 @@
+// FILE: src/internal/source/directory.go
+package source
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/lixenwraith/log"
+)
+
+// DirectorySource monitors a directory for log files
+type DirectorySource struct {
+	path           string
+	pattern        string
+	checkInterval  time.Duration
+	subscribers    []chan LogEntry
+	watchers       map[string]*fileWatcher
+	mu             sync.RWMutex
+	ctx            context.Context
+	cancel         context.CancelFunc
+	wg             sync.WaitGroup
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	startTime      time.Time
+	lastEntryTime  atomic.Value // time.Time
+	logger         *log.Logger
+}
+
+// NewDirectorySource creates a new directory monitoring source
+func NewDirectorySource(options map[string]any, logger *log.Logger) (*DirectorySource, error) {
+	path, ok := options["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("directory source requires 'path' option")
+	}
+
+	pattern, _ := options["pattern"].(string)
+	if pattern == "" {
+		pattern = "*"
+	}
+
+	checkInterval := 100 * time.Millisecond
+	if ms, ok := toInt(options["check_interval_ms"]); ok && ms > 0 {
+		checkInterval = time.Duration(ms) * time.Millisecond
+	}
+
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return nil, fmt.Errorf("invalid path %s: %w", path, err)
+	}
+
+	ds := &DirectorySource{
+		path:          absPath,
+		pattern:       pattern,
+		checkInterval: checkInterval,
+		watchers:      make(map[string]*fileWatcher),
+		startTime:     time.Now(),
+		logger:        logger,
+	}
+	ds.lastEntryTime.Store(time.Time{})
+
+	return ds, nil
+}
+
+func (ds *DirectorySource) Subscribe() <-chan LogEntry {
+	ds.mu.Lock()
+	defer ds.mu.Unlock()
+
+	ch := make(chan LogEntry, 1000)
+	ds.subscribers = append(ds.subscribers, ch)
+	return ch
+}
+
+func (ds *DirectorySource) Start() error {
+	ds.ctx, ds.cancel = context.WithCancel(context.Background())
+	ds.wg.Add(1)
+	go ds.monitorLoop()
+
+	ds.logger.Info("msg", "Directory source started",
+		"component", "directory_source",
+		"path", ds.path,
+		"pattern", ds.pattern,
+		"check_interval_ms", ds.checkInterval.Milliseconds())
+	return nil
+}
+
+func (ds *DirectorySource) Stop() {
+	if ds.cancel != nil {
+		ds.cancel()
+	}
+	ds.wg.Wait()
+
+	ds.mu.Lock()
+	for _, w := range ds.watchers {
+		w.close()
+	}
+	for _, ch := range ds.subscribers {
+		close(ch)
+	}
+	ds.mu.Unlock()
+
+	ds.logger.Info("msg", "Directory source stopped",
+		"component", "directory_source",
+		"path", ds.path)
+}
+
+func (ds *DirectorySource) GetStats() SourceStats {
+	lastEntry, _ := ds.lastEntryTime.Load().(time.Time)
+
+	ds.mu.RLock()
+	watcherCount := len(ds.watchers)
+	details := make(map[string]any)
+
+	// Add watcher details
+	watchers := make([]map[string]any, 0, watcherCount)
+	for _, w := range ds.watchers {
+		info := w.getInfo()
+		watchers = append(watchers, map[string]any{
+			"path":         info.Path,
+			"size":         info.Size,
+			"position":     info.Position,
+			"entries_read": info.EntriesRead,
+			"rotations":    info.Rotations,
+			"last_read":    info.LastReadTime,
+		})
+	}
+	details["watchers"] = watchers
+	details["active_watchers"] = watcherCount
+	ds.mu.RUnlock()
+
+	return SourceStats{
+		Type:           "directory",
+		TotalEntries:   ds.totalEntries.Load(),
+		DroppedEntries: ds.droppedEntries.Load(),
+		StartTime:      ds.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        details,
+	}
+}
+
+func (ds *DirectorySource) ApplyRateLimit(entry LogEntry) (LogEntry, bool) {
+	// TODO: Implement source-side rate limiting for aggregation/summarization
+	// For now, just pass through unchanged
+	return entry, true
+}
+
+func (ds *DirectorySource) publish(entry LogEntry) {
+	// Apply rate limiting (placeholder for now)
+	entry, allowed := ds.ApplyRateLimit(entry)
+	if !allowed {
+		return
+	}
+
+	ds.mu.RLock()
+	defer ds.mu.RUnlock()
+
+	ds.totalEntries.Add(1)
+	ds.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range ds.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			ds.droppedEntries.Add(1)
+			ds.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+				"component", "directory_source")
+		}
+	}
+}
+
+func (ds *DirectorySource) monitorLoop() {
+	defer ds.wg.Done()
+
+	ds.checkTargets()
+
+	ticker := time.NewTicker(ds.checkInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ds.ctx.Done():
+			return
+		case <-ticker.C:
+			ds.checkTargets()
+		}
+	}
+}
+
+func (ds *DirectorySource) checkTargets() {
+	files, err := ds.scanDirectory()
+	if err != nil {
+		ds.logger.Warn("msg", "Failed to scan directory",
+			"component", "directory_source",
+			"path", ds.path,
+			"pattern", ds.pattern,
+			"error", err)
+		return
+	}
+
+	for _, file := range files {
+		ds.ensureWatcher(file)
+	}
+
+	ds.cleanupWatchers()
+}
+
+func (ds *DirectorySource) scanDirectory() ([]string, error) {
+	entries, err := os.ReadDir(ds.path)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert glob pattern to regex
+	regexPattern := globToRegex(ds.pattern)
+	re, err := regexp.Compile(regexPattern)
+	if err != nil {
+		return nil, fmt.Errorf("invalid pattern regex: %w", err)
+	}
+
+	var files []string
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		name := entry.Name()
+		if re.MatchString(name) {
+			files = append(files, filepath.Join(ds.path, name))
+		}
+	}
+
+	return files, nil
+}
+
+func (ds *DirectorySource) ensureWatcher(path string) {
+	ds.mu.Lock()
+	defer ds.mu.Unlock()
+
+	if _, exists := ds.watchers[path]; exists {
+		return
+	}
+
+	w := newFileWatcher(path, ds.publish, ds.logger)
+	ds.watchers[path] = w
+
+	ds.logger.Debug("msg", "Created file watcher",
+		"component", "directory_source",
+		"path", path)
+
+	ds.wg.Add(1)
+	go func() {
+		defer ds.wg.Done()
+		if err := w.watch(ds.ctx); err != nil {
+			if err == context.Canceled {
+				ds.logger.Debug("msg", "Watcher cancelled",
+					"component", "directory_source",
+					"path", path)
+			} else {
+				ds.logger.Error("msg", "Watcher failed",
+					"component", "directory_source",
+					"path", path,
+					"error", err)
+			}
+		}
+
+		ds.mu.Lock()
+		delete(ds.watchers, path)
+		ds.mu.Unlock()
+	}()
+}
+
+func (ds *DirectorySource) cleanupWatchers() {
+	ds.mu.Lock()
+	defer ds.mu.Unlock()
+
+	for path, w := range ds.watchers {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			w.stop()
+			delete(ds.watchers, path)
+			ds.logger.Debug("msg", "Cleaned up watcher for non-existent file",
+				"component", "directory_source",
+				"path", path)
+		}
+	}
+}
+
+func globToRegex(glob string) string {
+	regex := regexp.QuoteMeta(glob)
+	regex = strings.ReplaceAll(regex, `\*`, `.*`)
+	regex = strings.ReplaceAll(regex, `\?`, `.`)
+	return "^" + regex + "$"
+}

src/internal/source/file_watcher.go

@@ -1,5 +1,5 @@
-// FILE: src/internal/monitor/file_watcher.go
+// FILE: src/internal/source/file_watcher.go
-package monitor
+package source
 
 import (
 	"bufio"
@@ -9,7 +9,6 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -19,6 +18,17 @@ import (
 	"github.com/lixenwraith/log"
 )
 
+// WatcherInfo contains information about a file watcher
+type WatcherInfo struct {
+	Path         string
+	Size         int64
+	Position     int64
+	ModTime      time.Time
+	EntriesRead  uint64
+	LastReadTime time.Time
+	Rotations    int
+}
+
 type fileWatcher struct {
 	path         string
 	callback     func(LogEntry)
@@ -333,13 +343,6 @@ func extractLogLevel(line string) string {
 	return ""
 }
 
-func globToRegex(glob string) string {
-	regex := regexp.QuoteMeta(glob)
-	regex = strings.ReplaceAll(regex, `\*`, `.*`)
-	regex = strings.ReplaceAll(regex, `\?`, `.`)
-	return "^" + regex + "$"
-}
-
 func (w *fileWatcher) getInfo() WatcherInfo {
 	w.mu.Lock()
 	info := WatcherInfo{

src/internal/source/source.go

@@ -0,0 +1,60 @@
+// FILE: src/internal/source/source.go
+package source
+
+import (
+	"encoding/json"
+	"time"
+)
+
+// LogEntry represents a single log record
+type LogEntry struct {
+	Time    time.Time       `json:"time"`
+	Source  string          `json:"source"`
+	Level   string          `json:"level,omitempty"`
+	Message string          `json:"message"`
+	Fields  json.RawMessage `json:"fields,omitempty"`
+}
+
+// Source represents an input data stream
+type Source interface {
+	// Subscribe returns a channel that receives log entries
+	Subscribe() <-chan LogEntry
+
+	// Start begins reading from the source
+	Start() error
+
+	// Stop gracefully shuts down the source
+	Stop()
+
+	// GetStats returns source statistics
+	GetStats() SourceStats
+
+	// ApplyRateLimit applies source-side rate limiting
+	// TODO: This is a placeholder for future features like aggregation and summarization
+	// Currently just returns the entry unchanged
+	ApplyRateLimit(entry LogEntry) (LogEntry, bool)
+}
+
+// SourceStats contains statistics about a source
+type SourceStats struct {
+	Type           string
+	TotalEntries   uint64
+	DroppedEntries uint64
+	StartTime      time.Time
+	LastEntryTime  time.Time
+	Details        map[string]any
+}
+
+// Helper function for type conversion
+func toInt(v any) (int, bool) {
+	switch val := v.(type) {
+	case int:
+		return val, true
+	case int64:
+		return int(val), true
+	case float64:
+		return int(val), true
+	default:
+		return 0, false
+	}
+}

src/internal/source/stdin.go

@@ -0,0 +1,123 @@
+// FILE: src/internal/source/stdin.go
+package source
+
+import (
+	"bufio"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"github.com/lixenwraith/log"
+)
+
+// StdinSource reads log entries from standard input
+type StdinSource struct {
+	subscribers    []chan LogEntry
+	done           chan struct{}
+	totalEntries   atomic.Uint64
+	droppedEntries atomic.Uint64
+	startTime      time.Time
+	lastEntryTime  atomic.Value // time.Time
+	logger         *log.Logger
+}
+
+// NewStdinSource creates a new stdin source
+func NewStdinSource(options map[string]any, logger *log.Logger) (*StdinSource, error) {
+	s := &StdinSource{
+		done:      make(chan struct{}),
+		startTime: time.Now(),
+		logger:    logger,
+	}
+	s.lastEntryTime.Store(time.Time{})
+	return s, nil
+}
+
+func (s *StdinSource) Subscribe() <-chan LogEntry {
+	ch := make(chan LogEntry, 1000)
+	s.subscribers = append(s.subscribers, ch)
+	return ch
+}
+
+func (s *StdinSource) Start() error {
+	go s.readLoop()
+	s.logger.Info("msg", "Stdin source started", "component", "stdin_source")
+	return nil
+}
+
+func (s *StdinSource) Stop() {
+	close(s.done)
+	for _, ch := range s.subscribers {
+		close(ch)
+	}
+	s.logger.Info("msg", "Stdin source stopped", "component", "stdin_source")
+}
+
+func (s *StdinSource) GetStats() SourceStats {
+	lastEntry, _ := s.lastEntryTime.Load().(time.Time)
+
+	return SourceStats{
+		Type:           "stdin",
+		TotalEntries:   s.totalEntries.Load(),
+		DroppedEntries: s.droppedEntries.Load(),
+		StartTime:      s.startTime,
+		LastEntryTime:  lastEntry,
+		Details:        map[string]any{},
+	}
+}
+
+func (s *StdinSource) ApplyRateLimit(entry LogEntry) (LogEntry, bool) {
+	// TODO: Implement source-side rate limiting for aggregation/summarization
+	// For now, just pass through unchanged
+	return entry, true
+}
+
+func (s *StdinSource) readLoop() {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		select {
+		case <-s.done:
+			return
+		default:
+			line := scanner.Text()
+			if line == "" {
+				continue
+			}
+
+			entry := LogEntry{
+				Time:    time.Now(),
+				Source:  "stdin",
+				Message: line,
+				Level:   extractLogLevel(line),
+			}
+
+			// Apply rate limiting
+			entry, allowed := s.ApplyRateLimit(entry)
+			if !allowed {
+				continue
+			}
+
+			s.publish(entry)
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		s.logger.Error("msg", "Scanner error reading stdin",
+			"component", "stdin_source",
+			"error", err)
+	}
+}
+
+func (s *StdinSource) publish(entry LogEntry) {
+	s.totalEntries.Add(1)
+	s.lastEntryTime.Store(entry.Time)
+
+	for _, ch := range s.subscribers {
+		select {
+		case ch <- entry:
+		default:
+			s.droppedEntries.Add(1)
+			s.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+				"component", "stdin_source")
+		}
+	}
+}

src/internal/transport/noop_logger.go

@@ -1,11 +0,0 @@
-// FILE: src/internal/transport/noop_logger.go
-package transport
-
-// noopLogger implements gnet's Logger interface but discards everything
-type noopLogger struct{}
-
-func (n noopLogger) Debugf(format string, args ...any) {}
-func (n noopLogger) Infof(format string, args ...any)  {}
-func (n noopLogger) Warnf(format string, args ...any)  {}
-func (n noopLogger) Errorf(format string, args ...any) {}
-func (n noopLogger) Fatalf(format string, args ...any) {}

src/internal/transport/tcpserver.go

@@ -1,87 +0,0 @@
-// FILE: src/internal/monitor/tcpserver.go
-package transport
-
-import (
-	"fmt"
-	"net"
-	"sync"
-
-	"github.com/panjf2000/gnet/v2"
-)
-
-type tcpServer struct {
-	gnet.BuiltinEventEngine
-	streamer    *TCPStreamer
-	connections sync.Map
-}
-
-func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
-	// Store engine reference for shutdown
-	s.streamer.engineMu.Lock()
-	s.streamer.engine = &eng
-	s.streamer.engineMu.Unlock()
-
-	fmt.Printf("[TCP DEBUG] Server booted on port %d\n", s.streamer.config.Port)
-	return gnet.None
-}
-
-func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
-	remoteAddr := c.RemoteAddr().String()
-	s.streamer.logger.Debug("msg", "TCP connection attempt", "remote_addr", remoteAddr)
-
-	// Check rate limit
-	if s.streamer.rateLimiter != nil {
-		// Parse the remote address to get proper net.Addr
-		remoteStr := c.RemoteAddr().String()
-		tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr)
-		if err != nil {
-			s.streamer.logger.Warn("msg", "Failed to parse TCP address",
-				"remote_addr", remoteAddr,
-				"error", err)
-			return nil, gnet.Close
-		}
-
-		if !s.streamer.rateLimiter.CheckTCP(tcpAddr) {
-			s.streamer.logger.Warn("msg", "TCP connection rate limited",
-				"remote_addr", remoteAddr)
-			// Silently close connection when rate limited
-			return nil, gnet.Close
-		}
-
-		// Track connection
-		s.streamer.rateLimiter.AddConnection(remoteStr)
-	}
-
-	s.connections.Store(c, struct{}{})
-
-	newCount := s.streamer.activeConns.Add(1)
-	s.streamer.logger.Debug("msg", "TCP connection opened",
-		"remote_addr", remoteAddr,
-		"active_connections", newCount)
-
-	return nil, gnet.None
-}
-
-func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
-	s.connections.Delete(c)
-
-	remoteAddr := c.RemoteAddr().String()
-
-	// Remove connection tracking
-	if s.streamer.rateLimiter != nil {
-		s.streamer.rateLimiter.RemoveConnection(c.RemoteAddr().String())
-	}
-
-	newCount := s.streamer.activeConns.Add(-1)
-	s.streamer.logger.Debug("msg", "TCP connection closed",
-		"remote_addr", remoteAddr,
-		"active_connections", newCount,
-		"error", err)
-	return gnet.None
-}
-
-func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
-	// We don't expect input from clients, just discard
-	c.Discard(-1)
-	return gnet.None
-}

src/internal/transport/tcpstreamer.go

@@ -1,191 +0,0 @@
-// FILE: src/internal/transport/tcpstreamer.go
-package transport
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"logwisp/src/internal/config"
-	"logwisp/src/internal/monitor"
-	"logwisp/src/internal/ratelimit"
-
-	"github.com/lixenwraith/log"
-	"github.com/panjf2000/gnet/v2"
-)
-
-type TCPStreamer struct {
-	logChan     chan monitor.LogEntry
-	config      config.TCPConfig
-	server      *tcpServer
-	done        chan struct{}
-	activeConns atomic.Int32
-	startTime   time.Time
-	engine      *gnet.Engine
-	engineMu    sync.Mutex
-	wg          sync.WaitGroup
-	rateLimiter *ratelimit.Limiter
-	logger      *log.Logger
-}
-
-func NewTCPStreamer(logChan chan monitor.LogEntry, cfg config.TCPConfig, logger *log.Logger) *TCPStreamer {
-	t := &TCPStreamer{
-		logChan:   logChan,
-		config:    cfg,
-		done:      make(chan struct{}),
-		startTime: time.Now(),
-		logger:    logger,
-	}
-
-	if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
-		t.rateLimiter = ratelimit.New(*cfg.RateLimit)
-	}
-
-	return t
-}
-
-func (t *TCPStreamer) Start() error {
-	t.server = &tcpServer{streamer: t}
-
-	// Start log broadcast loop
-	t.wg.Add(1)
-	go func() {
-		defer t.wg.Done()
-		t.broadcastLoop()
-	}()
-
-	// Configure gnet
-	addr := fmt.Sprintf("tcp://:%d", t.config.Port)
-
-	// Run gnet in separate goroutine to avoid blocking
-	errChan := make(chan error, 1)
-	go func() {
-		t.logger.Info("msg", "Starting TCP server",
-			"component", "tcp_streamer",
-			"port", t.config.Port)
-
-		err := gnet.Run(t.server, addr,
-			gnet.WithLogger(noopLogger{}),
-			gnet.WithMulticore(true),
-			gnet.WithReusePort(true),
-		)
-		if err != nil {
-			t.logger.Error("msg", "TCP server failed",
-				"component", "tcp_streamer",
-				"port", t.config.Port,
-				"error", err)
-		}
-		errChan <- err
-	}()
-
-	// Wait briefly for server to start or fail
-	select {
-	case err := <-errChan:
-		// Server failed immediately
-		close(t.done)
-		t.wg.Wait()
-		return err
-	case <-time.After(100 * time.Millisecond):
-		// Server started successfully
-		t.logger.Info("msg", "TCP server started", "port", t.config.Port)
-		return nil
-	}
-}
-
-func (t *TCPStreamer) Stop() {
-	t.logger.Info("msg", "Stopping TCP server")
-	// Signal broadcast loop to stop
-	close(t.done)
-
-	// Stop gnet engine if running
-	t.engineMu.Lock()
-	engine := t.engine
-	t.engineMu.Unlock()
-
-	if engine != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
-		defer cancel()
-		(*engine).Stop(ctx) // Dereference the pointer
-	}
-
-	// Wait for broadcast loop to finish
-	t.wg.Wait()
-
-	t.logger.Info("msg", "TCP server stopped")
-}
-
-func (t *TCPStreamer) broadcastLoop() {
-	var ticker *time.Ticker
-	var tickerChan <-chan time.Time
-
-	if t.config.Heartbeat.Enabled {
-		ticker = time.NewTicker(time.Duration(t.config.Heartbeat.IntervalSeconds) * time.Second)
-		tickerChan = ticker.C
-		defer ticker.Stop()
-	}
-
-	for {
-		select {
-		case entry, ok := <-t.logChan:
-			if !ok {
-				return
-			}
-			data, err := json.Marshal(entry)
-			if err != nil {
-				t.logger.Error("msg", "Failed to marshal log entry",
-					"component", "tcp_streamer",
-					"error", err,
-					"entry_source", entry.Source)
-				continue
-			}
-			data = append(data, '\n')
-
-			t.server.connections.Range(func(key, value any) bool {
-				conn := key.(gnet.Conn)
-				conn.AsyncWrite(data, nil)
-				return true
-			})
-
-		case <-tickerChan:
-			if heartbeat := t.formatHeartbeat(); heartbeat != nil {
-				t.server.connections.Range(func(key, value any) bool {
-					conn := key.(gnet.Conn)
-					conn.AsyncWrite(heartbeat, nil)
-					return true
-				})
-			}
-
-		case <-t.done:
-			return
-		}
-	}
-}
-
-func (t *TCPStreamer) formatHeartbeat() []byte {
-	if !t.config.Heartbeat.Enabled {
-		return nil
-	}
-
-	data := make(map[string]any)
-	data["type"] = "heartbeat"
-
-	if t.config.Heartbeat.IncludeTimestamp {
-		data["time"] = time.Now().UTC().Format(time.RFC3339Nano)
-	}
-
-	if t.config.Heartbeat.IncludeStats {
-		data["active_connections"] = t.activeConns.Load()
-		data["uptime_seconds"] = int(time.Since(t.startTime).Seconds())
-	}
-
-	// For TCP, always use JSON format
-	jsonData, _ := json.Marshal(data)
-	return append(jsonData, '\n')
-}
-
-func (t *TCPStreamer) GetActiveConnections() int32 {
-	return t.activeConns.Load()
-}