diff --git a/Makefile b/Makefile
index 0c4cdc1..a2f0821 100644
--- a/Makefile
+++ b/Makefile
@@ -1,31 +1,52 @@
-# FILE: Makefile
-BINARY_NAME := logwisp
-VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
-GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
-BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S')
+# LogWisp Makefile
+# Compatible with GNU Make (Linux) and BSD Make (FreeBSD)
-LDFLAGS := -ldflags "-X 'logwisp/src/internal/version.Version=$(VERSION)' \
- -X 'logwisp/src/internal/version.GitCommit=$(GIT_COMMIT)' \
- -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'"
+BINARY_NAME = logwisp
+BUILD_DIR = bin
+BINARY_PATH = $(BUILD_DIR)/$(BINARY_NAME)
+VERSION != git describe --tags --always --dirty 2>/dev/null || echo "dev"
+GIT_COMMIT != git rev-parse --short HEAD 2>/dev/null || echo "unknown"
+BUILD_TIME != date -u '+%Y-%m-%d_%H:%M:%S'
-.PHONY: build
+# Go build variables
+GO = go
+GOFLAGS =
+LDFLAGS = -X 'logwisp/src/internal/version.Version=$(VERSION)' \
+ -X 'logwisp/src/internal/version.GitCommit=$(GIT_COMMIT)' \
+ -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'
+
+# Installation directories
+PREFIX ?= /usr/local
+BINDIR = $(PREFIX)/bin
+
+# Default target
+all: build
+
+# Build the binary
build:
- go build $(LDFLAGS) -o $(BINARY_NAME) ./src/cmd/logwisp
+ mkdir -p $(BUILD_DIR)
+ $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./src/cmd/logwisp
-.PHONY: install
+# Install the binary
install: build
- install -m 755 $(BINARY_NAME) /usr/local/bin/
+ install -m 755 $(BINARY_PATH) $(BINDIR)/
-.PHONY: clean
+# Uninstall the binary
+uninstall:
+ rm -f $(BINDIR)/$(BINARY_PATH)
+
+# Clean build artifacts
clean:
- rm -f $(BINARY_NAME)
+ rm -f $(BINARY_PATH)
-.PHONY: test
-test:
- go test -v ./...
+# Development build with race detector
+dev:
+ $(GO) build $(GOFLAGS) -race -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./src/cmd/logwisp
-.PHONY: release
-release:
- @if [ -z "$(TAG)" ]; then echo "TAG is required: make release TAG=v1.0.0"; exit 1; fi
- git tag -a $(TAG) -m "Release $(TAG)"
- git push origin $(TAG)
+# Show current version
+version:
+ @echo "Version: $(VERSION)"
+ @echo "Commit: $(GIT_COMMIT)"
+ @echo "Build Time: $(BUILD_TIME)"
+
+.PHONY: all build install uninstall clean dev version
\ No newline at end of file
diff --git a/README.md b/README.md
index 5e392cb..8a29586 100644
--- a/README.md
+++ b/README.md
@@ -3,7 +3,7 @@
|
-
+ |
LogWisp
@@ -14,30 +14,29 @@
+**Flexible log monitoring with real-time streaming over HTTP/SSE and TCP**
-**Multi-stream log monitoring with real-time streaming over HTTP/SSE and TCP**
-
-LogWisp watches log files and streams updates to connected clients in real-time. Perfect for monitoring multiple applications, filtering noise, and centralizing log access.
+LogWisp watches log files and streams updates to connected clients in real-time using a pipeline architecture: **sources β filters β sinks**. Perfect for monitoring multiple applications, filtering noise, and routing logs to multiple destinations.
## π Quick Start
```bash
# Install
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+git clone https://github.com/lixenwraith/logwisp.git
+cd logwisp
+make install
# Run with defaults (monitors *.log in current directory)
logwisp
-
-# Stream logs (from another terminal)
-curl -N http://localhost:8080/stream
```
## β¨ Key Features
+- **π§ Pipeline Architecture** - Flexible source β filter β sink processing
- **π‘ Real-time Streaming** - SSE (HTTP) and TCP protocols
- **π Pattern Filtering** - Include/exclude logs with regex patterns
- **π‘οΈ Rate Limiting** - Protect against abuse with configurable limits
-- **π Multi-stream** - Monitor different log sources simultaneously
+- **π Multi-pipeline** - Process different log sources simultaneously
- **π Rotation Aware** - Handles log rotation seamlessly
- **β‘ High Performance** - Minimal CPU/memory footprint
@@ -50,33 +49,6 @@ Complete documentation is available in the [`doc/`](doc/) directory:
- [**CLI Reference**](doc/cli.md) - Command-line interface
- [**Examples**](doc/examples/) - Ready-to-use configurations
-## π» Basic Usage
-
-### Monitor application logs with filtering:
-
-```toml
-# ~/.config/logwisp.toml
-[[streams]]
-name = "myapp"
-
-[streams.monitor]
-targets = [{ path = "/var/log/myapp", pattern = "*.log" }]
-
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "WARN", "CRITICAL"]
-
-[streams.httpserver]
-enabled = true
-port = 8080
-```
-
-### Run multiple streams:
-
-```bash
-logwisp --router --config /etc/logwisp/multi-stream.toml
-```
-
## π License
BSD-3-Clause
\ No newline at end of file
diff --git a/config/logwisp.toml.defaults b/config/logwisp.toml.defaults
index 2419a08..12c1731 100644
--- a/config/logwisp.toml.defaults
+++ b/config/logwisp.toml.defaults
@@ -1,713 +1,221 @@
-# LogWisp Configuration File - Complete Reference
+# LogWisp Default Configuration and Guide
# Default path: ~/.config/logwisp.toml
-# Override with: ./logwisp --config /path/to/config.toml
-
-# This is a complete configuration reference showing all available options.
-# Default values are uncommented, alternatives and examples are commented.
-
-# ==============================================================================
-# LOGGING CONFIGURATION (LogWisp's own operational logs)
-# ==============================================================================
-# Controls where and how LogWisp logs its own operational messages.
-# This is separate from the logs being monitored and streamed.
+# Override: logwisp --config /path/to/config.toml
+# ============================================================================
+# LOGGING (LogWisp's operational logs)
+# ============================================================================
[logging]
-# Output mode: where to write LogWisp's operational logs
-# Options: "file", "stdout", "stderr", "both", "none"
-# - file: Write only to log files
-# - stdout: Write only to standard output
-# - stderr: Write only to standard error (default for containers)
-# - both: Write to both file and console
-# - none: Disable logging (β οΈ SECURITY: Not recommended)
+# Output mode: file, stdout, stderr, both, none
output = "stderr"
-# Minimum log level for operational logs
-# Options: "debug", "info", "warn", "error"
-# - debug: Maximum verbosity, includes internal state changes
-# - info: Normal operational messages (default)
-# - warn: Warnings and errors only
-# - error: Errors only
+# Log level: debug, info, warn, error
level = "info"
-# File output configuration (used when output includes "file" or "both")
+# File output settings (when output includes "file")
[logging.file]
-# Directory for log files
directory = "./logs"
-
-# Base name for log files (will append timestamp and .log)
name = "logwisp"
-
-# Maximum size per log file before rotation (megabytes)
max_size_mb = 100
-
-# Maximum total size of all log files (megabytes)
-# Oldest files are deleted when limit is reached
max_total_size_mb = 1000
-
-# How long to keep log files (hours)
-# 0 = no time-based deletion
retention_hours = 168.0 # 7 days
-# Console output configuration
+# Console output settings
[logging.console]
-# Target for console output
-# Options: "stdout", "stderr", "split"
-# - stdout: All logs to standard output
-# - stderr: All logs to standard error (default)
-# - split: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
-target = "stderr"
+target = "stderr" # stdout, stderr, split
+format = "txt" # txt, json
-# Output format
-# Options: "txt", "json"
-# - txt: Human-readable text format
-# - json: Structured JSON for log aggregation
-format = "txt"
+# ============================================================================
+# PIPELINE CONFIGURATION
+# ============================================================================
+# Each [[pipelines]] defines an independent log processing pipeline
+# Structure: sources β filters β sinks
-# ==============================================================================
-# STREAM CONFIGURATION
-# ==============================================================================
-# Each [[streams]] section defines an independent log monitoring stream.
-# You can have multiple streams, each with its own settings.
-
-# ------------------------------------------------------------------------------
-# Default Stream - Monitors current directory
-# ------------------------------------------------------------------------------
-[[streams]]
-# Stream identifier used in logs, metrics, and router paths
-# Must be unique across all streams
+[[pipelines]]
+# Unique pipeline identifier (used in router paths)
name = "default"
-# File monitoring configuration
-[streams.monitor]
-# How often to check for new log entries (milliseconds)
-# Lower = faster detection but more CPU usage
-# Range: 10-60000 (0.01 to 60 seconds)
-check_interval_ms = 100
+# ----------------------------------------------------------------------------
+# SOURCES - Input data sources
+# ----------------------------------------------------------------------------
+[[pipelines.sources]]
+# Source type: directory, file, stdin
+type = "directory"
-# Targets to monitor - can be files or directories
-# At least one target is required
-targets = [
- # Monitor all .log files in current directory
- { path = "./", pattern = "*.log", is_file = false },
+# Type-specific options
+options = {
+ path = "./",
+ pattern = "*.log",
+ check_interval_ms = 100 # How often to check for new entries (10-60000)
+}
- # Example: Monitor specific file
- # { path = "/var/log/app.log", is_file = true },
+# Additional source examples:
+# [[pipelines.sources]]
+# type = "file"
+# options = { path = "/var/log/app.log" }
+#
+# [[pipelines.sources]]
+# type = "stdin"
+# options = {}
- # Example: Multiple patterns in a directory
- # { path = "/logs", pattern = "*.log", is_file = false },
- # { path = "/logs", pattern = "*.txt", is_file = false },
-]
-
-# Filter configuration (optional) - controls which logs are streamed
+# ----------------------------------------------------------------------------
+# FILTERS - Log entry filtering (optional)
+# ----------------------------------------------------------------------------
# Multiple filters are applied sequentially - all must pass
-# Empty patterns array means "match everything"
-# Example: Include only errors and warnings
-# [[streams.filters]]
-# type = "include" # "include" (whitelist) or "exclude" (blacklist)
-# logic = "or" # "or" (match any) or "and" (match all)
+# [[pipelines.filters]]
+# type = "include" # include (whitelist) or exclude (blacklist)
+# logic = "or" # or (match any) or and (match all)
# patterns = [
-# "(?i)error", # Case-insensitive error matching
-# "(?i)warn" # Case-insensitive warning matching
+# "ERROR",
+# "(?i)warn", # Case-insensitive
+# "\\bfatal\\b" # Word boundary
# ]
-# Example: Exclude debug and trace logs
-# [[streams.filters]]
-# type = "exclude"
-# patterns = ["DEBUG", "TRACE", "VERBOSE"]
+# ----------------------------------------------------------------------------
+# SINKS - Output destinations
+# ----------------------------------------------------------------------------
+[[pipelines.sinks]]
+# Sink type: http, tcp, file, stdout, stderr
+type = "http"
-# HTTP Server configuration (SSE/Server-Sent Events)
-[streams.httpserver]
-# Enable/disable HTTP server for this stream
-enabled = true
+# Type-specific options
+options = {
+ port = 8080,
+ buffer_size = 1000,
+ stream_path = "/stream",
+ status_path = "/status",
-# Port to listen on (1-65535)
-# Each stream needs a unique port unless using router mode
-port = 8080
+ # Heartbeat configuration
+ heartbeat = {
+ enabled = true,
+ interval_seconds = 30,
+ format = "comment", # comment or json
+ include_timestamp = true,
+ include_stats = false
+ },
-# Per-client buffer size (number of messages)
-# Larger = handles bursts better, more memory per client
-buffer_size = 1000
+ # Rate limiting (optional)
+ rate_limit = {
+ enabled = false,
+ requests_per_second = 10.0,
+ burst_size = 20,
+ limit_by = "ip", # ip or global
+ max_connections_per_ip = 5,
+ max_total_connections = 100,
+ response_code = 429,
+ response_message = "Rate limit exceeded"
+ }
-# Endpoint paths (must start with /)
-stream_path = "/stream" # SSE stream endpoint
-status_path = "/status" # Statistics endpoint
+ # SSL/TLS (planned)
+ # ssl = {
+ # enabled = false,
+ # cert_file = "/path/to/cert.pem",
+ # key_file = "/path/to/key.pem"
+ # }
+}
-# Keep-alive heartbeat configuration
-# Prevents connection timeout on quiet logs
-[streams.httpserver.heartbeat]
-# Enable/disable heartbeat messages
-enabled = true
+# Additional sink examples:
-# Interval between heartbeats (seconds)
-# Range: 1-3600 (1 second to 1 hour)
-interval_seconds = 30
+# [[pipelines.sinks]]
+# type = "tcp"
+# options = {
+# port = 9090,
+# buffer_size = 5000,
+# heartbeat = { enabled = true, interval_seconds = 60 }
+# }
-# Heartbeat format
-# Options: "comment", "json"
-# - comment: SSE comment format (: heartbeat)
-# - json: JSON event format (data: {"type":"heartbeat"})
-format = "comment"
+# [[pipelines.sinks]]
+# type = "file"
+# options = {
+# directory = "/var/log/logwisp",
+# name = "app",
+# max_size_mb = 100,
+# retention_hours = 168.0
+# }
-# Include timestamp in heartbeat
-include_timestamp = true
+# [[pipelines.sinks]]
+# type = "stdout"
+# options = { buffer_size = 500 }
-# Include connection statistics
-include_stats = false
-
-# Rate limiting configuration (disabled by default)
-# Protects against abuse and resource exhaustion
-[streams.httpserver.rate_limit]
-# Enable/disable rate limiting
-enabled = false
-
-# Token refill rate (requests per second)
-# Float value, e.g., 0.5 = 1 request every 2 seconds
-# requests_per_second = 10.0
-
-# Maximum burst capacity (token bucket size)
-# Should be 2-3x requests_per_second for normal usage
-# burst_size = 20
-
-# Rate limit strategy
-# Options: "ip", "global"
-# - ip: Each client IP gets its own limit
-# - global: All clients share one limit
-# limit_by = "ip"
-
-# HTTP response code when rate limited
-# Common: 429 (Too Many Requests), 503 (Service Unavailable)
-# response_code = 429
-
-# Response message when rate limited
-# response_message = "Rate limit exceeded"
-
-# Maximum concurrent connections per IP address
-# 0 = unlimited
-# max_connections_per_ip = 5
-
-# Maximum total concurrent connections
-# 0 = unlimited
-# max_total_connections = 100
-
-# SSL/TLS configuration (planned feature)
-# [streams.httpserver.ssl]
-# enabled = false
-# cert_file = "/path/to/cert.pem"
-# key_file = "/path/to/key.pem"
-# min_version = "TLS1.2" # Minimum TLS version
-# client_auth = false # Require client certificates
-
-# TCP Server configuration (optional)
-# Raw TCP streaming for high-performance scenarios
-# [streams.tcpserver]
-# enabled = false
-# port = 9090
-# buffer_size = 5000 # Larger buffer for TCP
+# ----------------------------------------------------------------------------
+# AUTHENTICATION (optional, applies to network sinks)
+# ----------------------------------------------------------------------------
+# [pipelines.auth]
+# type = "none" # none, basic, bearer
#
-# [streams.tcpserver.heartbeat]
-# enabled = true
-# interval_seconds = 60
-# include_timestamp = true
-# include_stats = false
-#
-# [streams.tcpserver.rate_limit]
-# enabled = false
-# requests_per_second = 5.0
-# burst_size = 10
-# limit_by = "ip"
-
-# Authentication configuration (planned feature)
-# [streams.auth]
-# type = "none" # Options: "none", "basic", "bearer"
-#
-# # Basic authentication
-# [streams.auth.basic_auth]
-# users_file = "/etc/logwisp/users.htpasswd"
+# [pipelines.auth.basic_auth]
# realm = "LogWisp"
-#
-# # IP-based access control
-# ip_whitelist = ["192.168.1.0/24", "10.0.0.0/8"]
-# ip_blacklist = []
+# users = [
+# { username = "admin", password_hash = "$2a$10$..." }
+# ]
+# ip_whitelist = ["192.168.1.0/24"]
-# ------------------------------------------------------------------------------
-# Example: Application Logs Stream with Error Filtering
-# ------------------------------------------------------------------------------
-# [[streams]]
-# name = "app"
+# ============================================================================
+# COMPLETE EXAMPLES
+# ============================================================================
+
+# Example: Production logs with filtering and multiple outputs
+# [[pipelines]]
+# name = "production"
#
-# [streams.monitor]
-# check_interval_ms = 50 # Fast detection for active logs
-# targets = [
-# # Monitor specific application log directory
-# { path = "/var/log/myapp", pattern = "*.log", is_file = false },
-# # Also monitor specific file
-# { path = "/var/log/myapp/app.log", is_file = true },
-# ]
+# [[pipelines.sources]]
+# type = "directory"
+# options = { path = "/var/log/app", pattern = "*.log", check_interval_ms = 50 }
#
-# # Filter 1: Include only errors and warnings
-# [[streams.filters]]
+# [[pipelines.filters]]
# type = "include"
-# logic = "or" # Match ANY of these patterns
-# patterns = [
-# "(?i)\\berror\\b", # Word boundary error (case-insensitive)
-# "(?i)\\bwarn(ing)?\\b", # warn or warning
-# "(?i)\\bfatal\\b", # fatal
-# "(?i)\\bcritical\\b", # critical
-# "(?i)exception", # exception anywhere
-# "(?i)fail(ed|ure)?", # fail, failed, failure
-# "panic", # Go panics
-# "traceback", # Python tracebacks
-# ]
+# patterns = ["ERROR", "WARN", "CRITICAL"]
#
-# # Filter 2: Exclude health check noise
-# [[streams.filters]]
+# [[pipelines.filters]]
# type = "exclude"
-# patterns = [
-# "/health",
-# "/metrics",
-# "/ping",
-# "GET /favicon.ico",
-# "ELB-HealthChecker",
-# "kube-probe"
-# ]
+# patterns = ["/health", "/metrics"]
#
-# [streams.httpserver]
-# enabled = true
-# port = 8081 # Different port for each stream
-# buffer_size = 2000 # Larger buffer for busy logs
-# stream_path = "/logs" # Custom path
-# status_path = "/health" # Custom health endpoint
+# [[pipelines.sinks]]
+# type = "http"
+# options = {
+# port = 8080,
+# rate_limit = { enabled = true, requests_per_second = 25.0 }
+# }
#
-# # JSON heartbeat format for programmatic clients
-# [streams.httpserver.heartbeat]
-# enabled = true
-# interval_seconds = 20
-# format = "json" # JSON event format
-# include_timestamp = true
-# include_stats = true # Include active client count
-#
-# # Moderate rate limiting for public access
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 25.0
-# burst_size = 50
-# limit_by = "ip"
-# max_connections_per_ip = 10
-# max_total_connections = 200
+# [[pipelines.sinks]]
+# type = "file"
+# options = { directory = "/var/log/archive", name = "errors" }
-# ------------------------------------------------------------------------------
-# Example: System Logs Stream (TCP + HTTP) with Security Filtering
-# ------------------------------------------------------------------------------
-# [[streams]]
-# name = "system"
+# Example: Multi-source aggregation
+# [[pipelines]]
+# name = "aggregated"
#
-# [streams.monitor]
-# check_interval_ms = 1000 # Check every second (system logs update slowly)
-# targets = [
-# { path = "/var/log/syslog", is_file = true },
-# { path = "/var/log/auth.log", is_file = true },
-# { path = "/var/log/kern.log", is_file = true },
-# { path = "/var/log/messages", is_file = true },
-# ]
+# [[pipelines.sources]]
+# type = "directory"
+# options = { path = "/var/log/nginx", pattern = "*.log" }
#
-# # Include only security-relevant logs
-# [[streams.filters]]
-# type = "include"
-# logic = "or"
-# patterns = [
-# "(?i)auth",
-# "(?i)sudo",
-# "(?i)ssh",
-# "(?i)login",
-# "(?i)permission",
-# "(?i)denied",
-# "(?i)unauthorized",
-# "(?i)security",
-# "(?i)selinux",
-# "kernel:.*audit",
-# "COMMAND=", # sudo commands
-# "session opened",
-# "session closed"
-# ]
+# [[pipelines.sources]]
+# type = "directory"
+# options = { path = "/var/log/app", pattern = "*.log" }
#
-# # TCP Server for high-performance streaming
-# [streams.tcpserver]
-# enabled = true
-# port = 9090
-# buffer_size = 5000
-#
-# # TCP heartbeat (always JSON format)
-# [streams.tcpserver.heartbeat]
-# enabled = true
-# interval_seconds = 60 # Less frequent for TCP
-# include_timestamp = true
-# include_stats = false
-#
-# # TCP rate limiting
-# [streams.tcpserver.rate_limit]
-# enabled = true
-# requests_per_second = 5.0 # Limit TCP connections
-# burst_size = 10
-# limit_by = "ip"
-#
-# # Also expose via HTTP
-# [streams.httpserver]
-# enabled = true
-# port = 8082
-# buffer_size = 1000
-# stream_path = "/stream"
-# status_path = "/status"
-#
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 5.0
-# burst_size = 10
-# max_connections_per_ip = 2 # Strict for security logs
+# [[pipelines.sinks]]
+# type = "tcp"
+# options = { port = 9090 }
-# ------------------------------------------------------------------------------
-# Example: High-Volume Debug Logs with Performance Filtering
-# ------------------------------------------------------------------------------
-# [[streams]]
-# name = "debug"
-#
-# [streams.monitor]
-# check_interval_ms = 5000 # Check every 5 seconds (high volume)
-# targets = [
-# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
-# { path = "/var/log/debug", pattern = "debug-*.log", is_file = false },
-# ]
-#
-# # Exclude verbose debug output
-# [[streams.filters]]
-# type = "exclude"
-# patterns = [
-# "TRACE",
-# "VERBOSE",
-# "entering function",
-# "exiting function",
-# "memory dump",
-# "hex dump",
-# "stack trace",
-# "goroutine [0-9]+"
-# ]
-#
-# # Include only specific modules
-# [[streams.filters]]
-# type = "include"
-# patterns = [
-# "module=(api|database|auth)",
-# "component=(router|handler)",
-# "service=(payment|order|user)"
-# ]
-#
-# [streams.httpserver]
-# enabled = true
-# port = 8083
-# buffer_size = 10000 # Very large buffer
-# stream_path = "/debug"
-# status_path = "/stats"
-#
-# # Disable heartbeat for high-volume streams
-# [streams.httpserver.heartbeat]
-# enabled = false
-#
-# # Aggressive rate limiting
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 1.0 # Very restrictive
-# burst_size = 5
-# limit_by = "ip"
-# max_connections_per_ip = 1 # One connection per IP
-# response_code = 503 # Service Unavailable
-# response_message = "Debug stream overloaded"
-
-# ------------------------------------------------------------------------------
-# Example: Multi-Application with Router Mode
-# ------------------------------------------------------------------------------
+# ============================================================================
+# ROUTER MODE
+# ============================================================================
# Run with: logwisp --router
-#
-# [[streams]]
-# name = "frontend"
-# [streams.monitor]
-# targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
-# [[streams.filters]]
-# type = "exclude"
-# patterns = ["GET /static/", "GET /assets/"]
-# [streams.httpserver]
-# enabled = true
-# port = 8080 # Same port OK in router mode
-#
-# [[streams]]
-# name = "backend"
-# [streams.monitor]
-# targets = [{ path = "/var/log/api", pattern = "*.log" }]
-# [[streams.filters]]
-# type = "include"
-# patterns = ["ERROR", "WARN", "timeout", "failed"]
-# [streams.httpserver]
-# enabled = true
-# port = 8080 # Shared port in router mode
-#
-# [[streams]]
-# name = "database"
-# [streams.monitor]
-# targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
-# [streams.httpserver]
-# enabled = true
-# port = 8080
-#
-# # Access via:
-# # http://localhost:8080/frontend/stream
-# # http://localhost:8080/backend/stream
-# # http://localhost:8080/database/stream
-# # http://localhost:8080/status (global)
+# Allows multiple pipelines to share HTTP ports via path-based routing
+# Access: http://localhost:8080/{pipeline_name}/stream
+# Global status: http://localhost:8080/status
-# ==============================================================================
-# FILTER PATTERN REFERENCE
-# ==============================================================================
+# ============================================================================
+# QUICK REFERENCE
+# ============================================================================
+# Source types: directory, file, stdin
+# Sink types: http, tcp, file, stdout, stderr
+# Filter types: include, exclude
+# Filter logic: or, and
#
-# Basic Patterns:
-# - "ERROR" # Exact match (case sensitive)
-# - "(?i)error" # Case-insensitive
-# - "\\berror\\b" # Word boundary (won't match "errorCode")
-# - "error|warn|fatal" # Multiple options (OR)
-# - "(error|warn) level" # Group with context
-#
-# Position Patterns:
-# - "^\\[ERROR\\]" # Line starts with [ERROR]
-# - "ERROR:$" # Line ends with ERROR:
-# - "^\\d{4}-\\d{2}-\\d{2}" # Line starts with date
-#
-# Complex Patterns:
-# - "status=[4-5][0-9]{2}" # HTTP 4xx or 5xx status codes
-# - "duration>[0-9]{4}ms" # Duration over 999ms
-# - "user_id=\"[^\"]+\"" # Extract user_id values
-# - "\\[ERROR\\].*database" # ERROR followed by database
-# - "(?i)\\b(error|fail|critical)\\b" # Multiple error words
-#
-# Log Level Patterns:
-# - "\\[(ERROR|WARN|FATAL)\\]" # Common formats
-# - "level=(error|warning|critical)" # Key-value format
-# - "ERROR\\s*:" # ERROR with optional space
-# - "<(Error|Warning)>" # XML-style
-#
-# Application Patterns:
-# - "com\\.mycompany\\..*Exception" # Java exceptions
-# - "at .+\\(.+\\.java:[0-9]+\\)" # Java stack traces
-# - "File \".+\", line [0-9]+" # Python tracebacks
-# - "panic: .+" # Go panics
-# - "/api/v[0-9]+/" # API versioned paths
-#
-# Performance Patterns:
-# - "took [0-9]{4,}ms" # Operations over 999ms
-# - "memory usage: [8-9][0-9]%" # High memory usage
-# - "queue size: [0-9]{4,}" # Large queues
-# - "timeout|timed out" # Timeouts
-#
-# Security Patterns:
-# - "unauthorized|forbidden" # Access denied
-# - "invalid token|expired token" # Auth failures
-# - "SQL injection|XSS" # Security threats
-# - "failed login.*IP: ([0-9.]+)" # Failed logins with IP
-#
-# Performance Tips:
-# - Avoid nested quantifiers: "((a+)+)+" causes catastrophic backtracking
-# - Use anchors when possible: "^ERROR" is faster than "ERROR"
-# - Prefer character classes: "[0-9]" over "\\d" for clarity
-# - Use non-capturing groups: "(?:error|warn)" when not extracting
-# - Test complex patterns with sample data before deployment
-# - Consider using multiple simple patterns instead of one complex pattern
-#
-# Security Considerations:
-# - Be aware of ReDoS (Regular Expression Denial of Service)
-# - Limit pattern complexity for public-facing streams
-# - Monitor filter processing time in statistics
-# - Consider pre-filtering very high volume streams
-# - Use explicit allow-lists for sensitive logs
-
-# ==============================================================================
-# RATE LIMITING GUIDE
-# ==============================================================================
-#
-# Token Bucket Algorithm:
-# - Each client (IP) or global limit gets a bucket with 'burst_size' tokens
-# - Tokens refill at 'requests_per_second' rate
-# - Each request consumes one token
-# - Provides smooth rate limiting without hard cutoffs
-#
-# Configuration Examples:
-#
-# Light Protection (default for most streams):
-# requests_per_second = 10.0
-# burst_size = 20 # Handle short bursts
-#
-# Moderate Protection (public endpoints):
-# requests_per_second = 5.0
-# burst_size = 15
-# max_connections_per_ip = 5
-#
-# Strict Protection (sensitive logs):
-# requests_per_second = 1.0
-# burst_size = 3
-# max_connections_per_ip = 1
-# limit_by = "ip"
-#
-# Global Limiting (shared resource):
-# requests_per_second = 50.0 # Total for all clients
-# burst_size = 100
-# limit_by = "global"
-# max_total_connections = 50
-#
-# Behavior:
-# - HTTP: Returns response_code (default 429) with JSON error
-# - TCP: Silently drops connection (no error message)
-# - Cleanup: Inactive IPs removed after 5 minutes
-# - Statistics: Available in /status endpoint
-#
-# Best Practices:
-# - Set burst_size to 2-3x requests_per_second
-# - Use per-IP limiting for fairness
-# - Use global limiting for resource protection
-# - Monitor rate limit statistics for tuning
-# - Consider different limits for different streams
-# - Enable for any public-facing endpoints
-
-# ==============================================================================
-# PERFORMANCE TUNING
-# ==============================================================================
-#
-# Monitor Check Interval:
-# - 10-50ms: Real-time monitoring, higher CPU usage
-# - 100-500ms: Good balance for active logs
-# - 1000-5000ms: Low-activity logs, minimal CPU
-# - 10000ms+: Very slow changing logs
-#
-# Buffer Sizes:
-# - HTTP: 100-1000 for normal use, 5000+ for high volume
-# - TCP: 1000-5000 typical, 10000+ for bulk streaming
-# - Larger = more memory per client, handles bursts better
-#
-# Connection Limits:
-# - Development: No limits needed
-# - Production: 5-10 connections per IP typical
-# - Public: 1-3 connections per IP
-# - Total: Based on available memory (each uses ~1-5MB)
-#
-# Filter Performance:
-# - Simple patterns: ~1ΞΌs per check
-# - Complex patterns: ~10-100ΞΌs per check
-# - Many patterns: Consider multiple streams instead
-# - Use exclude filters to drop noise early
-#
-# Memory Usage (approximate):
-# - Base process: ~10-20MB
-# - Per stream: ~5-10MB
-# - Per HTTP client: ~1-2MB
-# - Per TCP client: ~0.5-1MB
-# - Filter chain: ~1-5MB depending on patterns
-
-# ==============================================================================
-# DEPLOYMENT SCENARIOS
-# ==============================================================================
-#
-# Single Application:
-# - One stream with basic filtering
-# - Moderate rate limiting
-# - Standard check interval (100ms)
-#
-# Microservices:
-# - Multiple streams, one per service
-# - Router mode for unified access
-# - Different filter rules per service
-# - Service-specific rate limits
-#
-# High Security:
-# - Strict include filters
-# - Low rate limits (1-2 req/sec)
-# - Single connection per IP
-# - TCP for internal, HTTP for external
-#
-# High Performance:
-# - TCP streaming preferred
-# - Large buffers (10000+)
-# - Minimal filtering
-# - Higher check intervals
-# - No heartbeats
-#
-# Development/Testing:
-# - Multiple streams for different log levels
-# - No rate limiting
-# - Debug level logging
-# - Fast check intervals
-# - All filters disabled
-
-# ==============================================================================
-# TROUBLESHOOTING
-# ==============================================================================
-#
-# Common Issues:
-#
-# "No logs appearing":
-# - Check file paths and permissions
-# - Verify pattern matches filenames
-# - Check filters aren't too restrictive
-# - Enable debug logging: --log-level debug
-#
-# "High CPU usage":
-# - Increase check_interval_ms
-# - Reduce number of filter patterns
-# - Use simpler regex patterns
-# - Check for runaway log growth
-#
-# "Clients disconnecting":
-# - Enable heartbeats
-# - Check rate limiting settings
-# - Verify network connectivity
-# - Increase buffer sizes
-#
-# "Memory growth":
-# - Check client connection count
-# - Verify buffer sizes are reasonable
-# - Look for memory leaks in filters
-# - Enable connection limits
-#
-# Debug Commands:
-# - Check status: curl http://localhost:8080/status
-# - Test stream: curl -N http://localhost:8080/stream
-# - View logs: logwisp --log-level debug --log-output stderr
-# - Test filters: Use simple patterns first
-
-# ==============================================================================
-# FUTURE FEATURES (Roadmap)
-# ==============================================================================
-#
-# Authentication:
-# - Basic auth with htpasswd files
-# - Bearer token authentication
-# - JWT validation
-# - mTLS client certificates
-#
-# SSL/TLS:
-# - HTTPS endpoints
-# - TLS for TCP streams
-# - Certificate management
-# - Let's Encrypt integration
-#
-# Advanced Filtering:
-# - Lua scripting for complex logic
-# - Rate-based filtering (N per minute)
-# - Statistical anomaly detection
-# - Multi-line pattern matching
-#
-# Output Formats:
-# - JSON transformation
-# - Field extraction
-# - Custom formatting templates
-# - Compression (gzip)
-#
-# Integrations:
-# - Prometheus metrics
-# - OpenTelemetry traces
-# - Webhook notifications
-# - Cloud storage backends
\ No newline at end of file
+# Common patterns:
+# "(?i)error" - Case-insensitive
+# "\\berror\\b" - Word boundary
+# "^ERROR" - Start of line
+# "status=[4-5]\\d{2}" - HTTP errors
\ No newline at end of file
diff --git a/config/logwisp.toml.minimal b/config/logwisp.toml.minimal
index a8f0ad5..920cdc2 100644
--- a/config/logwisp.toml.minimal
+++ b/config/logwisp.toml.minimal
@@ -1,40 +1,44 @@
# LogWisp Minimal Configuration
# Save as: ~/.config/logwisp.toml
-# Basic stream monitoring application logs
-[[streams]]
+# Basic pipeline monitoring application logs
+[[pipelines]]
name = "app"
-[streams.monitor]
-check_interval_ms = 100
-targets = [
- { path = "/var/log/myapp", pattern = "*.log", is_file = false }
-]
+# Source: Monitor log directory
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/myapp", pattern = "*.log", check_interval_ms = 100 }
-[streams.httpserver]
-enabled = true
-port = 8080
-stream_path = "/stream"
-status_path = "/status"
+# Sink: HTTP streaming
+[[pipelines.sinks]]
+type = "http"
+options = {
+ port = 8080,
+ buffer_size = 1000,
+ stream_path = "/stream",
+ status_path = "/status"
+}
# Optional additions:
# 1. Filter for errors only:
-# [[streams.filters]]
+# [[pipelines.filters]]
# type = "include"
# patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
# 2. Enable rate limiting:
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 10.0
-# burst_size = 20
-# limit_by = "ip"
+# Modify the sink options above:
+# options = {
+# port = 8080,
+# buffer_size = 1000,
+# rate_limit = { enabled = true, requests_per_second = 10.0, burst_size = 20 }
+# }
-# 3. Add heartbeat:
-# [streams.httpserver.heartbeat]
-# enabled = true
-# interval_seconds = 30
+# 3. Add file output:
+# [[pipelines.sinks]]
+# type = "file"
+# options = { directory = "/var/log/logwisp", name = "app" }
# 4. Change LogWisp's own logging:
# [logging]
diff --git a/doc/README.md b/doc/README.md
index 351aacf..c968861 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -1,75 +1,26 @@
# LogWisp Documentation
-Welcome to the LogWisp documentation. This guide covers all aspects of installing, configuring, and using LogWisp for multi-stream log monitoring.
+Documentation covers installation, configuration, and usage of LogWisp's pipeline-based log monitoring system.
## π Documentation Index
### Getting Started
-- **[Installation Guide](installation.md)** - How to install LogWisp on various platforms
-- **[Quick Start](quickstart.md)** - Get up and running in 5 minutes
-- **[Architecture Overview](architecture.md)** - System design and components
+- **[Installation Guide](installation.md)** - Platform-specific installation
+- **[Quick Start](quickstart.md)** - Get running in 5 minutes
+- **[Architecture Overview](architecture.md)** - Pipeline design
### Configuration
-- **[Configuration Guide](configuration.md)** - Complete configuration reference
-- **[Environment Variables](environment.md)** - Environment variable reference
-- **[Command Line Options](cli.md)** - CLI flags and parameters
+- **[Configuration Guide](configuration.md)** - Complete reference
+- **[Environment Variables](environment.md)** - Container configuration
+- **[Command Line Options](cli.md)** - CLI reference
+- **[Sample Configurations](../config/)** - Default & Minimal Config
### Features
-- **[Filters Guide](filters.md)** - Pattern-based log filtering
-- **[Rate Limiting](ratelimiting.md)** - Request and connection limiting
-- **[Router Mode](router.md)** - Path-based multi-stream routing
-- **[Authentication](authentication.md)** - Securing your log streams *(planned)*
-
-### Operations
-- **[Monitoring & Status](monitoring.md)** - Health checks and statistics
-- **[Performance Tuning](performance.md)** - Optimization guidelines
-- **[Troubleshooting](troubleshooting.md)** - Common issues and solutions
-
-### Advanced Topics
-- **[Security Best Practices](security.md)** - Hardening your deployment
-- **[Integration Examples](integrations.md)** - Working with other tools
-- **[Development Guide](development.md)** - Contributing to LogWisp
-
-## π Quick Links
-
-- **[Example Configurations](examples/)** - Ready-to-use config templates
-- **[API Reference](api.md)** - SSE/TCP protocol documentation
-- **[Changelog](../CHANGELOG.md)** - Version history and updates
-
-## π‘ Common Use Cases
-
-### Single Application Monitoring
-Monitor logs from one application with basic filtering:
-```toml
-[[streams]]
-name = "myapp"
-[streams.monitor]
-targets = [{ path = "/var/log/myapp", pattern = "*.log" }]
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "WARN"]
-```
-
-### Multi-Service Architecture
-Monitor multiple services with different configurations:
-```bash
-logwisp --router --config /etc/logwisp/services.toml
-```
-
-### High-Security Environments
-Enable authentication and rate limiting:
-```toml
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-max_connections_per_ip = 3
-```
-
-## π Finding Help
-
-- **GitHub Issues**: [Report bugs or request features](https://github.com/logwisp/logwisp/issues)
-- **Discussions**: [Ask questions and share ideas](https://github.com/logwisp/logwisp/discussions)
-- **Examples**: Check the [examples directory](examples/) for common scenarios
+- **[Status Monitoring](status.md)** - Health checks
+- **[Filters Guide](filters.md)** - Pattern-based filtering
+- **[Rate Limiting](ratelimiting.md)** - Connection protection
+- **[Router Mode](router.md)** - Multi-pipeline routing
+- **[Authentication](authentication.md)** - Access control *(planned)*
## π License
diff --git a/doc/architecture.md b/doc/architecture.md
new file mode 100644
index 0000000..852795c
--- /dev/null
+++ b/doc/architecture.md
@@ -0,0 +1,304 @@
+# Architecture Overview
+
+LogWisp implements a flexible pipeline architecture for real-time log processing and streaming.
+
+## Core Architecture
+
+```
+βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
+β LogWisp Service β
+βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ€
+β β
+β ββββββββββββββββββββββββββββ Pipeline 1 ββββββββββββββββββββββββββββ β
+β β β β
+β β Sources Filters Sinks β β
+β β ββββββββ ββββββββββ ββββββββ β β
+β β β Dir ββββ βInclude β ββββββ HTTP ββββ Client 1 β β
+β β ββββββββ β β ERROR β β ββββββββ β β
+β β ββββββΆβ WARN ββββββΆβββββββββββββ β β
+β β ββββββββ β ββββββββββ β β File β β β
+β β β File ββββ βΌ β ββββββββ β β
+β β ββββββββ ββββββββββ β ββββββββ β β
+β β βExclude β ββββββ TCP ββββ Client 2 β β
+β β β DEBUG β ββββββββ β β
+β β ββββββββββ β β
+β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ β
+β β
+β ββββββββββββββββββββββββββββ Pipeline 2 ββββββββββββββββββββββββββββ β
+β β β β
+β β ββββββββ ββββββββ β β
+β β βStdin ββββββββββββββββββββββββββββββStdoutβ β β
+β β ββββββββ (No Filters) ββββββββ β β
+β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ β
+β β
+β ββββββββββββββββββββββββββββ Pipeline N ββββββββββββββββββββββββββββ β
+β β Multiple Sources β Filter Chain β Multiple Sinks β β
+β ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ β
+βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
+```
+
+## Data Flow
+
+```
+Log Entry Flow:
+
+βββββββββββ βββββββββββ βββββββββββ βββββββββββ
+β File β β Parse β β Filter β β Format β
+β Watcher ββββββΆβ Entry ββββββΆβ Chain ββββββΆβ Send β
+βββββββββββ βββββββββββ βββββββββββ βββββββββββ
+ β β β β
+ βΌ βΌ βΌ βΌ
+ Detect Extract Include/ Deliver to
+ Changes Timestamp Exclude Clients
+ & Level Patterns
+
+
+Entry Processing:
+
+1. Source Detection 2. Entry Creation 3. Filter Application
+ ββββββββββββ ββββββββββββββ βββββββββββββββ
+ βNew Entry β β Timestamp β β Filter 1 β
+ βDetected ββββββββββββΆβ Level ββββββββββΆβ Include? β
+ ββββββββββββ β Message β ββββββββ¬βββββββ
+ ββββββββββββββ β
+ βΌ
+4. Sink Distribution βββββββββββββββ
+ ββββββββββββ β Filter 2 β
+ β HTTP ββββββ β Exclude? β
+ ββββββββββββ β ββββββββ¬βββββββ
+ ββββββββββββ β β
+ β TCP ββββββΌββββββββββ Entry ββββββββββββββββββββ
+ ββββββββββββ β (if passed)
+ ββββββββββββ β
+ β File ββββββ
+ ββββββββββββ
+```
+
+## Component Details
+
+### Sources
+
+Sources monitor inputs and generate log entries:
+
+```
+Directory Source:
+βββββββββββββββββββββββββββββββββββ
+β Directory Monitor β
+βββββββββββββββββββββββββββββββββββ€
+β β’ Pattern Matching (*.log) β
+β β’ File Rotation Detection β
+β β’ Position Tracking β
+β β’ Concurrent File Watching β
+βββββββββββββββββββββββββββββββββββ
+ β
+ βΌ
+ ββββββββββββββββ
+ β File Watcher β (per file)
+ ββββββββββββββββ€
+ β β’ Read New β
+ β β’ Track Pos β
+ β β’ Detect Rot β
+ ββββββββββββββββ
+```
+
+### Filters
+
+Filters process entries through pattern matching:
+
+```
+Filter Chain:
+ βββββββββββββββ
+Entry βββββββββββΆβ Filter 1 β
+ β (Include) β
+ ββββββββ¬βββββββ
+ β Pass?
+ βΌ
+ βββββββββββββββ
+ β Filter 2 β
+ β (Exclude) β
+ ββββββββ¬βββββββ
+ β Pass?
+ βΌ
+ βββββββββββββββ
+ β Filter N β
+ ββββββββ¬βββββββ
+ β
+ βΌ
+ To Sinks
+```
+
+### Sinks
+
+Sinks deliver processed entries to destinations:
+
+```
+HTTP Sink (SSE):
+βββββββββββββββββββββββββββββββββββββ
+β HTTP Server β
+βββββββββββββββββββββββββββββββββββββ€
+β βββββββββββ βββββββββββ β
+β β Stream β β Status β β
+β βEndpoint β βEndpoint β β
+β ββββββ¬βββββ ββββββ¬βββββ β
+β β β β
+β ββββββΌβββββββββββββββΌβββββ β
+β β Connection Manager β β
+β ββββββββββββββββββββββββββ€ β
+β β β’ Rate Limiting β β
+β β β’ Heartbeat β β
+β β β’ Buffer Management β β
+β ββββββββββββββββββββββββββ β
+βββββββββββββββββββββββββββββββββββββ
+
+TCP Sink:
+βββββββββββββββββββββββββββββββββββββ
+β TCP Server β
+βββββββββββββββββββββββββββββββββββββ€
+β ββββββββββββββββββββββββββ β
+β β gnet Event Loop β β
+β ββββββββββββββββββββββββββ€ β
+β β β’ Async I/O β β
+β β β’ Connection Pool β β
+β β β’ Rate Limiting β β
+β ββββββββββββββββββββββββββ β
+βββββββββββββββββββββββββββββββββββββ
+```
+
+## Router Mode
+
+In router mode, multiple pipelines share HTTP ports:
+
+```
+Router Architecture:
+ βββββββββββββββββββ
+ β HTTP Router β
+ β Port 8080 β
+ ββββββββββ¬βββββββββ
+ β
+ ββββββββββββββββββββββΌβββββββββββββββββββββ
+ β β β
+ /app/stream /db/stream /sys/stream
+ β β β
+ ββββββΌβββββ ββββββΌβββββ ββββββΌβββββ
+ βPipeline β βPipeline β βPipeline β
+ β "app" β β "db" β β "sys" β
+ βββββββββββ βββββββββββ βββββββββββ
+
+Path Routing:
+Client Request βββΆ Router βββΆ Parse Path βββΆ Find Pipeline βββΆ Route
+ β
+ βΌ
+ Extract Pipeline Name
+ from /pipeline/endpoint
+```
+
+## Memory Management
+
+```
+Buffer Flow:
+ββββββββββββ ββββββββββββ ββββββββββββ
+β Source β β Pipeline β β Sink β
+β Buffer ββββββΆβ Buffer ββββββΆβ Buffer β
+β (1000) β β (chan) β β (1000) β
+ββββββββββββ ββββββββββββ ββββββββββββ
+ β β β
+ βΌ βΌ βΌ
+ Drop if full Backpressure Drop if full
+ (counted) (blocking) (counted)
+```
+
+## Rate Limiting
+
+```
+Token Bucket Algorithm:
+βββββββββββββββββββββββββββββββ
+β Token Bucket β
+βββββββββββββββββββββββββββββββ€
+β Capacity: burst_size β
+β Refill: requests_per_second β
+β β
+β βββββββββββββββββββββββ β
+β β β β β β β β β β β β β β
+β βββββββββββββββββββββββ β
+β 6/10 tokens available β
+βββββββββββββββββββββββββββββββ
+ β
+ βΌ
+ Request arrives
+ β
+ βΌ
+ Token available? ββNoβββΆ Reject (429)
+ β
+ Yes
+ βΌ
+ Consume token βββΆ Allow request
+```
+
+## Concurrency Model
+
+```
+Goroutine Structure:
+
+Main βββββ¬ββββ Pipeline 1 βββββ¬ββββ Source Reader 1
+ β βββββ Source Reader 2
+ β βββββ Filter Processor
+ β βββββ HTTP Server
+ β βββββ TCP Server
+ β
+ βββββ Pipeline 2 βββββ¬ββββ Source Reader
+ β βββββ Sink Writers
+ β
+ βββββ HTTP Router (if enabled)
+
+Channel Communication:
+Source ββchanβββΆ Filter ββchanβββΆ Sink
+ β β
+ βββ Non-blocking send βββββββββββββ
+ (drop & count if full)
+```
+
+## Configuration Loading
+
+```
+Priority Order:
+1. CLI Flags ββββββββββ
+2. Environment Vars βββΌβββΆ Merge βββΆ Final Config
+3. Config File ββββββββ€
+4. Defaults βββββββββββ
+
+Example:
+CLI: --log-level debug
+Env: LOGWISP_PIPELINES_0_NAME=app
+File: pipelines.toml
+Default: buffer_size = 1000
+```
+
+## Security Architecture
+
+```
+Security Layers:
+
+βββββββββββββββββββββββββββββββββββββββ
+β Network Layer β
+βββββββββββββββββββββββββββββββββββββββ€
+β β’ Rate Limiting (per IP/global) β
+β β’ Connection Limits β
+β β’ TLS/SSL (planned) β
+ββββββββββββββββ¬βββββββββββββββββββββββ
+ β
+ββββββββββββββββΌβββββββββββββββββββββββ
+β Authentication Layer β
+βββββββββββββββββββββββββββββββββββββββ€
+β β’ Basic Auth (planned) β
+β β’ Bearer Tokens (planned) β
+β β’ IP Whitelisting (planned) β
+ββββββββββββββββ¬βββββββββββββββββββββββ
+ β
+ββββββββββββββββΌβββββββββββββββββββββββ
+β Application Layer β
+βββββββββββββββββββββββββββββββββββββββ€
+β β’ Input Validation β
+β β’ Path Traversal Prevention β
+β β’ Resource Limits β
+βββββββββββββββββββββββββββββββββββββββ
+```
\ No newline at end of file
diff --git a/doc/cli.md b/doc/cli.md
index 3983654..2c59062 100644
--- a/doc/cli.md
+++ b/doc/cli.md
@@ -1,6 +1,6 @@
# Command Line Interface
-LogWisp provides a comprehensive set of command-line options for controlling its behavior without modifying configuration files.
+LogWisp CLI options for controlling behavior without modifying configuration files.
## Synopsis
@@ -11,145 +11,92 @@ logwisp [options]
## General Options
### `--config `
-Specify the configuration file location.
+Configuration file location.
- **Default**: `~/.config/logwisp.toml`
- **Example**: `logwisp --config /etc/logwisp/production.toml`
### `--router`
-Enable HTTP router mode for path-based routing of multiple streams.
-- **Default**: `false` (standalone mode)
-- **Use case**: Consolidate multiple HTTP streams on shared ports
+Enable HTTP router mode for path-based routing.
+- **Default**: `false`
- **Example**: `logwisp --router`
### `--version`
-Display version information and exit.
-- **Example**: `logwisp --version`
+Display version information.
### `--background`
-Run LogWisp as a background process.
-- **Default**: `false` (foreground mode)
+Run as background process.
- **Example**: `logwisp --background`
## Logging Options
-These options override the corresponding configuration file settings.
+Override configuration file settings:
### `--log-output `
-Control where LogWisp writes its own operational logs.
+LogWisp's operational log output.
- **Values**: `file`, `stdout`, `stderr`, `both`, `none`
-- **Default**: Configured value or `stderr`
- **Example**: `logwisp --log-output both`
-#### Output Modes:
-- `file`: Write logs only to files
-- `stdout`: Write logs only to standard output
-- `stderr`: Write logs only to standard error
-- `both`: Write logs to both files and console
-- `none`: Disable logging (β οΈ SECURITY: Not recommended)
-
### `--log-level `
-Set the minimum log level for LogWisp's operational logs.
+Minimum log level.
- **Values**: `debug`, `info`, `warn`, `error`
-- **Default**: Configured value or `info`
- **Example**: `logwisp --log-level debug`
### `--log-file `
-Specify the log file path when using file output.
-- **Default**: Configured value or `./logs/logwisp.log`
-- **Example**: `logwisp --log-output file --log-file /var/log/logwisp/app.log`
+Log file path (with file output).
+- **Example**: `logwisp --log-file /var/log/logwisp/app.log`
### `--log-dir `
-Specify the log directory when using file output.
-- **Default**: Configured value or `./logs`
-- **Example**: `logwisp --log-output file --log-dir /var/log/logwisp`
+Log directory (with file output).
+- **Example**: `logwisp --log-dir /var/log/logwisp`
### `--log-console `
-Control console output destination when using `stdout`, `stderr`, or `both` modes.
+Console output destination.
- **Values**: `stdout`, `stderr`, `split`
-- **Default**: `stderr`
-- **Example**: `logwisp --log-output both --log-console split`
-
-#### Console Targets:
-- `stdout`: All logs to standard output
-- `stderr`: All logs to standard error
-- `split`: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
+- **Example**: `logwisp --log-console split`
## Examples
### Basic Usage
```bash
-# Start with default configuration
+# Default configuration
logwisp
-# Use a specific configuration file
+# Specific configuration
logwisp --config /etc/logwisp/production.toml
```
-### Development Mode
+### Development
```bash
-# Enable debug logging to console
+# Debug mode
logwisp --log-output stderr --log-level debug
-# Debug with file output
+# With file output
logwisp --log-output both --log-level debug --log-dir ./debug-logs
```
-### Production Deployment
+### Production
```bash
-# File logging with info level
-logwisp --log-output file --log-dir /var/log/logwisp --log-level info
+# File logging
+logwisp --log-output file --log-dir /var/log/logwisp
-# Background mode with custom config
-logwisp --background --config /etc/logwisp/prod.toml
-
-# Router mode for multiple services
-logwisp --router --config /etc/logwisp/services.toml
-```
-
-### Troubleshooting
-```bash
-# Maximum verbosity to stderr
-logwisp --log-output stderr --log-level debug
-
-# Check version
-logwisp --version
-
-# Test configuration without backgrounding
-logwisp --config test.toml --log-level debug
+# Background with router
+logwisp --background --router --config /etc/logwisp/prod.toml
```
## Priority Order
-Configuration values are applied in the following priority order (highest to lowest):
-
-1. **Command-line flags** - Explicitly specified options
-2. **Environment variables** - `LOGWISP_*` prefixed variables
-3. **Configuration file** - TOML configuration
-4. **Built-in defaults** - Hardcoded fallback values
+1. **Command-line flags** (highest)
+2. **Environment variables**
+3. **Configuration file**
+4. **Built-in defaults** (lowest)
## Exit Codes
-- `0`: Successful execution
-- `1`: General error (configuration, startup failure)
-- `2`: Invalid command-line arguments
+- `0`: Success
+- `1`: General error
+- `2`: Invalid arguments
## Signals
-LogWisp responds to the following signals:
-
- `SIGINT` (Ctrl+C): Graceful shutdown
-- `SIGTERM`: Graceful shutdown
-- `SIGKILL`: Immediate termination (not recommended)
-
-During graceful shutdown, LogWisp will:
-1. Stop accepting new connections
-2. Finish streaming to existing clients
-3. Flush all buffers
-4. Close all file handles
-5. Exit cleanly
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [Environment Variables](environment.md) - Environment variable options
-- [Router Mode](router.md) - Path-based routing details
\ No newline at end of file
+- `SIGTERM`: Graceful shutdown
\ No newline at end of file
diff --git a/doc/configuration.md b/doc/configuration.md
index d129b60..8b23cdb 100644
--- a/doc/configuration.md
+++ b/doc/configuration.md
@@ -1,10 +1,9 @@
# Configuration Guide
-LogWisp uses TOML format for configuration with sensible defaults for all settings.
+LogWisp uses TOML format with a flexible **source β filter β sink** pipeline architecture.
## Configuration File Location
-Default search order:
1. Command line: `--config /path/to/config.toml`
2. Environment: `$LOGWISP_CONFIG_FILE`
3. User config: `~/.config/logwisp.toml`
@@ -13,342 +12,281 @@ Default search order:
## Configuration Structure
```toml
-# Optional: LogWisp's own logging configuration
+# Optional: LogWisp's own logging
[logging]
-output = "stderr" # file, stdout, stderr, both, none
-level = "info" # debug, info, warn, error
+output = "stderr"
+level = "info"
-# Required: At least one stream
-[[streams]]
-name = "default" # Unique identifier
+# Required: At least one pipeline
+[[pipelines]]
+name = "default"
-[streams.monitor] # Required: What to monitor
-# ... monitor settings ...
+# Sources (required)
+[[pipelines.sources]]
+type = "directory"
+options = { ... }
-[streams.httpserver] # Optional: HTTP/SSE server
-# ... HTTP settings ...
+# Filters (optional)
+[[pipelines.filters]]
+type = "include"
+patterns = [...]
-[streams.tcpserver] # Optional: TCP server
-# ... TCP settings ...
-
-[[streams.filters]] # Optional: Log filtering
-# ... filter settings ...
+# Sinks (required)
+[[pipelines.sinks]]
+type = "http"
+options = { ... }
```
## Logging Configuration
-Controls LogWisp's operational logging (not the logs being monitored).
+Controls LogWisp's operational logging:
```toml
[logging]
-output = "stderr" # Where to write LogWisp's logs
-level = "info" # Minimum log level
+output = "stderr" # file, stdout, stderr, both, none
+level = "info" # debug, info, warn, error
-# File output settings (when output includes "file")
[logging.file]
-directory = "./logs" # Log directory
-name = "logwisp" # Base filename
-max_size_mb = 100 # Rotate at this size
-max_total_size_mb = 1000 # Total size limit
-retention_hours = 168 # Keep for 7 days
+directory = "./logs"
+name = "logwisp"
+max_size_mb = 100
+max_total_size_mb = 1000
+retention_hours = 168
-# Console output settings
[logging.console]
target = "stderr" # stdout, stderr, split
format = "txt" # txt or json
```
-## Stream Configuration
+## Pipeline Configuration
-Each `[[streams]]` section defines an independent log monitoring pipeline.
+Each `[[pipelines]]` section defines an independent processing pipeline.
-### Monitor Settings
+### Sources
-What files or directories to watch:
+Input data sources:
+#### Directory Source
```toml
-[streams.monitor]
-check_interval_ms = 100 # How often to check for new entries
-
-# Monitor targets (at least one required)
-targets = [
- # Watch all .log files in a directory
- { path = "/var/log/myapp", pattern = "*.log", is_file = false },
-
- # Watch a specific file
- { path = "/var/log/app.log", is_file = true },
-
- # Multiple patterns
- { path = "/logs", pattern = "app-*.log", is_file = false },
- { path = "/logs", pattern = "error-*.txt", is_file = false }
-]
+[[pipelines.sources]]
+type = "directory"
+options = {
+ path = "/var/log/myapp", # Directory to monitor
+ pattern = "*.log", # File pattern (glob)
+ check_interval_ms = 100 # Check interval (10-60000)
+}
```
-### HTTP Server (SSE)
-
-Server-Sent Events streaming over HTTP:
-
+#### File Source
```toml
-[streams.httpserver]
-enabled = true
-port = 8080
-buffer_size = 1000 # Per-client event buffer
-stream_path = "/stream" # SSE endpoint
-status_path = "/status" # Statistics endpoint
-
-# Keep-alive heartbeat
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-format = "comment" # "comment" or "json"
-include_timestamp = true
-include_stats = false
-
-# Rate limiting (optional)
-[streams.httpserver.rate_limit]
-enabled = false
-requests_per_second = 10.0
-burst_size = 20
-limit_by = "ip" # "ip" or "global"
-response_code = 429
-response_message = "Rate limit exceeded"
-max_connections_per_ip = 5
-max_total_connections = 100
+[[pipelines.sources]]
+type = "file"
+options = {
+ path = "/var/log/app.log" # Specific file
+}
```
-### TCP Server
-
-Raw TCP streaming for high performance:
-
+#### Stdin Source
```toml
-[streams.tcpserver]
-enabled = true
-port = 9090
-buffer_size = 5000 # Larger buffer for TCP
-
-# Heartbeat (always JSON format for TCP)
-[streams.tcpserver.heartbeat]
-enabled = true
-interval_seconds = 60
-include_timestamp = true
-include_stats = false
-
-# Rate limiting
-[streams.tcpserver.rate_limit]
-enabled = false
-requests_per_second = 5.0
-burst_size = 10
-limit_by = "ip"
+[[pipelines.sources]]
+type = "stdin"
+options = {}
```
### Filters
-Control which log entries are streamed:
+Control which log entries pass through:
```toml
# Include filter - only matching logs pass
-[[streams.filters]]
+[[pipelines.filters]]
type = "include"
-logic = "or" # "or" = match any, "and" = match all
+logic = "or" # or: match any, and: match all
patterns = [
"ERROR",
- "WARN",
- "CRITICAL"
+ "(?i)warn", # Case-insensitive
+ "\\bfatal\\b" # Word boundary
]
# Exclude filter - matching logs are dropped
-[[streams.filters]]
+[[pipelines.filters]]
type = "exclude"
-patterns = [
- "DEBUG",
- "health check"
-]
+patterns = ["DEBUG", "health-check"]
+```
+
+### Sinks
+
+Output destinations:
+
+#### HTTP Sink (SSE)
+```toml
+[[pipelines.sinks]]
+type = "http"
+options = {
+ port = 8080,
+ buffer_size = 1000,
+ stream_path = "/stream",
+ status_path = "/status",
+
+ # Heartbeat
+ heartbeat = {
+ enabled = true,
+ interval_seconds = 30,
+ format = "comment", # comment or json
+ include_timestamp = true
+ },
+
+ # Rate limiting
+ rate_limit = {
+ enabled = true,
+ requests_per_second = 10.0,
+ burst_size = 20,
+ limit_by = "ip", # ip or global
+ max_connections_per_ip = 5
+ }
+}
+```
+
+#### TCP Sink
+```toml
+[[pipelines.sinks]]
+type = "tcp"
+options = {
+ port = 9090,
+ buffer_size = 5000,
+ heartbeat = { enabled = true, interval_seconds = 60 },
+ rate_limit = { enabled = true, requests_per_second = 5.0 }
+}
+```
+
+#### File Sink
+```toml
+[[pipelines.sinks]]
+type = "file"
+options = {
+ directory = "/var/log/logwisp",
+ name = "app",
+ max_size_mb = 100,
+ retention_hours = 168.0,
+ buffer_size = 2000
+}
+```
+
+#### Console Sinks
+```toml
+[[pipelines.sinks]]
+type = "stdout" # or "stderr"
+options = { buffer_size = 500 }
```
## Complete Examples
-### Minimal Configuration
+### Basic Application Monitoring
```toml
-[[streams]]
-name = "simple"
-[streams.monitor]
-targets = [{ path = "./logs", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines]]
+name = "app"
+
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
+
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
```
-### Production Web Application
+### Production with Filtering
```toml
[logging]
output = "file"
level = "info"
-[logging.file]
-directory = "/var/log/logwisp"
-max_size_mb = 500
-retention_hours = 336 # 14 days
-[[streams]]
-name = "webapp"
+[[pipelines]]
+name = "production"
-[streams.monitor]
-check_interval_ms = 50
-targets = [
- { path = "/var/log/nginx", pattern = "access.log*" },
- { path = "/var/log/nginx", pattern = "error.log*" },
- { path = "/var/log/myapp", pattern = "*.log" }
-]
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log", check_interval_ms = 50 }
-# Only errors and warnings
-[[streams.filters]]
+[[pipelines.filters]]
type = "include"
-logic = "or"
-patterns = [
- "\\b(ERROR|error|Error)\\b",
- "\\b(WARN|WARNING|warn|warning)\\b",
- "\\b(CRITICAL|FATAL|critical|fatal)\\b",
- "status=[4-5][0-9][0-9]" # HTTP errors
-]
+patterns = ["ERROR", "WARN", "CRITICAL"]
-# Exclude noise
-[[streams.filters]]
+[[pipelines.filters]]
type = "exclude"
-patterns = [
- "/health",
- "/metrics",
- "ELB-HealthChecker"
-]
+patterns = ["/health", "/metrics"]
-[streams.httpserver]
-enabled = true
-port = 8080
-buffer_size = 2000
+[[pipelines.sinks]]
+type = "http"
+options = {
+ port = 8080,
+ rate_limit = { enabled = true, requests_per_second = 25.0 }
+}
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 25.0
-burst_size = 50
-max_connections_per_ip = 10
+[[pipelines.sinks]]
+type = "file"
+options = { directory = "/var/log/archive", name = "errors" }
```
-### Multi-Service with Router
+### Multi-Source Aggregation
+
+```toml
+[[pipelines]]
+name = "aggregated"
+
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/nginx", pattern = "*.log" }
+
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
+
+[[pipelines.sources]]
+type = "stdin"
+options = {}
+
+[[pipelines.sinks]]
+type = "tcp"
+options = { port = 9090 }
+```
+
+### Router Mode
```toml
# Run with: logwisp --router
-# Service 1: API
-[[streams]]
+[[pipelines]]
name = "api"
-[streams.monitor]
-targets = [{ path = "/var/log/api", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080 # All streams can use same port in router mode
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/api", pattern = "*.log" }
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 } # Same port OK in router mode
-# Service 2: Database
-[[streams]]
-name = "database"
-[streams.monitor]
-targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "FATAL", "deadlock", "timeout"]
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines]]
+name = "web"
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/nginx", pattern = "*.log" }
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 } # Shared port
-# Service 3: System
-[[streams]]
-name = "system"
-[streams.monitor]
-targets = [
- { path = "/var/log/syslog", is_file = true },
- { path = "/var/log/auth.log", is_file = true }
-]
-[streams.tcpserver]
-enabled = true
-port = 9090
+# Access:
+# http://localhost:8080/api/stream
+# http://localhost:8080/web/stream
+# http://localhost:8080/status
```
-### High-Security Configuration
-
-```toml
-[logging]
-output = "file"
-level = "warn" # Less verbose
-
-[[streams]]
-name = "secure"
-
-[streams.monitor]
-targets = [{ path = "/var/log/secure", pattern = "*.log" }]
-
-# Only security events
-[[streams.filters]]
-type = "include"
-patterns = [
- "auth",
- "sudo",
- "ssh",
- "login",
- "failed",
- "denied"
-]
-
-[streams.httpserver]
-enabled = true
-port = 8443
-
-# Strict rate limiting
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 2.0
-burst_size = 3
-limit_by = "ip"
-max_connections_per_ip = 1
-response_code = 403 # Forbidden instead of 429
-
-# Future: Authentication
-# [streams.auth]
-# type = "basic"
-# [streams.auth.basic_auth]
-# users_file = "/etc/logwisp/users.htpasswd"
-```
-
-## Configuration Tips
-
-### Performance Tuning
-
-- **check_interval_ms**: Higher values reduce CPU usage
-- **buffer_size**: Larger buffers handle bursts better
-- **rate_limit**: Essential for public-facing streams
-
-### Filter Patterns
-
-- Use word boundaries: `\\berror\\b` (won't match "errorCode")
-- Case-insensitive: `(?i)error`
-- Anchors for speed: `^ERROR` faster than `ERROR`
-- Test complex patterns before deployment
-
-### Resource Limits
-
-- Each stream uses ~10-50MB RAM (depending on buffers)
-- CPU usage scales with check_interval and file activity
-- Network bandwidth depends on log volume and client count
-
## Validation
-LogWisp validates configuration on startup:
-- Required fields (name, monitor targets)
-- Port conflicts between streams
-- Pattern syntax for filters
+LogWisp validates on startup:
+- Required fields (name, sources, sinks)
+- Port conflicts between pipelines
+- Pattern syntax
- Path accessibility
-
-## See Also
-
-- [Environment Variables](environment.md) - Override via environment
-- [CLI Options](cli.md) - Override via command line
-- [Filter Guide](filters.md) - Advanced filtering patterns
-- [Examples](examples/) - Ready-to-use configurations
\ No newline at end of file
+- Rate limit values
\ No newline at end of file
diff --git a/doc/environment.md b/doc/environment.md
index 6e6c9fb..7e8d48f 100644
--- a/doc/environment.md
+++ b/doc/environment.md
@@ -1,275 +1,148 @@
# Environment Variables
-LogWisp supports comprehensive configuration through environment variables, allowing deployment without configuration files or dynamic overrides in containerized environments.
+Configure LogWisp through environment variables for containerized deployments.
## Naming Convention
-Environment variables follow a structured pattern:
- **Prefix**: `LOGWISP_`
- **Path separator**: `_` (underscore)
- **Array indices**: Numeric suffix (0-based)
- **Case**: UPPERCASE
-### Examples:
-- Config file setting: `logging.level = "debug"`
-- Environment variable: `LOGWISP_LOGGING_LEVEL=debug`
-
-- Array element: `streams[0].name = "app"`
-- Environment variable: `LOGWISP_STREAMS_0_NAME=app`
+Examples:
+- `logging.level` β `LOGWISP_LOGGING_LEVEL`
+- `pipelines[0].name` β `LOGWISP_PIPELINES_0_NAME`
## General Variables
### `LOGWISP_CONFIG_FILE`
-Path to the configuration file.
-- **Default**: `~/.config/logwisp.toml`
-- **Example**: `LOGWISP_CONFIG_FILE=/etc/logwisp/config.toml`
+Configuration file path.
+```bash
+export LOGWISP_CONFIG_FILE=/etc/logwisp/config.toml
+```
### `LOGWISP_CONFIG_DIR`
-Directory containing configuration files.
-- **Usage**: Combined with `LOGWISP_CONFIG_FILE` for relative paths
-- **Example**:
- ```bash
- export LOGWISP_CONFIG_DIR=/etc/logwisp
- export LOGWISP_CONFIG_FILE=production.toml
- # Loads: /etc/logwisp/production.toml
- ```
+Configuration directory.
+```bash
+export LOGWISP_CONFIG_DIR=/etc/logwisp
+export LOGWISP_CONFIG_FILE=production.toml
+```
### `LOGWISP_DISABLE_STATUS_REPORTER`
-Disable the periodic status reporter.
-- **Values**: `1` (disable), `0` or unset (enable)
-- **Default**: `0` (enabled)
-- **Example**: `LOGWISP_DISABLE_STATUS_REPORTER=1`
-
-### `LOGWISP_BACKGROUND`
-Internal marker for background process detection.
-- **Note**: Set automatically by `--background` flag
-- **Values**: `1` (background), unset (foreground)
+Disable periodic status reporting.
+```bash
+export LOGWISP_DISABLE_STATUS_REPORTER=1
+```
## Logging Variables
-### `LOGWISP_LOGGING_OUTPUT`
-LogWisp's operational log output mode.
-- **Values**: `file`, `stdout`, `stderr`, `both`, `none`
-- **Example**: `LOGWISP_LOGGING_OUTPUT=both`
-
-### `LOGWISP_LOGGING_LEVEL`
-Minimum log level for operational logs.
-- **Values**: `debug`, `info`, `warn`, `error`
-- **Example**: `LOGWISP_LOGGING_LEVEL=debug`
-
-### File Logging
```bash
+# Output mode
+LOGWISP_LOGGING_OUTPUT=both
+
+# Log level
+LOGWISP_LOGGING_LEVEL=debug
+
+# File logging
LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
LOGWISP_LOGGING_FILE_NAME=logwisp
LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
-LOGWISP_LOGGING_FILE_MAX_TOTAL_SIZE_MB=1000
-LOGWISP_LOGGING_FILE_RETENTION_HOURS=168 # 7 days
+LOGWISP_LOGGING_FILE_RETENTION_HOURS=168
+
+# Console logging
+LOGWISP_LOGGING_CONSOLE_TARGET=stderr
+LOGWISP_LOGGING_CONSOLE_FORMAT=json
```
-### Console Logging
+## Pipeline Configuration
+
+### Basic Pipeline
```bash
-LOGWISP_LOGGING_CONSOLE_TARGET=stderr # stdout, stderr, split
-LOGWISP_LOGGING_CONSOLE_FORMAT=txt # txt, json
-```
+# Pipeline name
+LOGWISP_PIPELINES_0_NAME=app
-## Stream Configuration
+# Source configuration
+LOGWISP_PIPELINES_0_SOURCES_0_TYPE=directory
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATH=/var/log/app
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATTERN="*.log"
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_CHECK_INTERVAL_MS=100
-Streams are configured using array indices (0-based).
-
-### Basic Stream Settings
-```bash
-# First stream (index 0)
-LOGWISP_STREAMS_0_NAME=app
-LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=100
-
-# Second stream (index 1)
-LOGWISP_STREAMS_1_NAME=system
-LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
-```
-
-### Monitor Targets
-```bash
-# Single file target
-LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/app.log
-LOGWISP_STREAMS_0_MONITOR_TARGETS_0_IS_FILE=true
-
-# Directory with pattern
-LOGWISP_STREAMS_0_MONITOR_TARGETS_1_PATH=/var/log/myapp
-LOGWISP_STREAMS_0_MONITOR_TARGETS_1_PATTERN="*.log"
-LOGWISP_STREAMS_0_MONITOR_TARGETS_1_IS_FILE=false
+# Sink configuration
+LOGWISP_PIPELINES_0_SINKS_0_TYPE=http
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_PORT=8080
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_BUFFER_SIZE=1000
```
### Filters
```bash
# Include filter
-LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
-LOGWISP_STREAMS_0_FILTERS_0_LOGIC=or
-LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN","CRITICAL"]'
+LOGWISP_PIPELINES_0_FILTERS_0_TYPE=include
+LOGWISP_PIPELINES_0_FILTERS_0_LOGIC=or
+LOGWISP_PIPELINES_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
# Exclude filter
-LOGWISP_STREAMS_0_FILTERS_1_TYPE=exclude
-LOGWISP_STREAMS_0_FILTERS_1_PATTERNS='["DEBUG","TRACE"]'
+LOGWISP_PIPELINES_0_FILTERS_1_TYPE=exclude
+LOGWISP_PIPELINES_0_FILTERS_1_PATTERNS='["DEBUG"]'
```
-### HTTP Server
+### HTTP Sink Options
```bash
-LOGWISP_STREAMS_0_HTTPSERVER_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
-LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=1000
-LOGWISP_STREAMS_0_HTTPSERVER_STREAM_PATH=/stream
-LOGWISP_STREAMS_0_HTTPSERVER_STATUS_PATH=/status
+# Basic
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_STREAM_PATH=/stream
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_STATUS_PATH=/status
# Heartbeat
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INTERVAL_SECONDS=30
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=comment
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INCLUDE_TIMESTAMP=true
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INCLUDE_STATS=false
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_ENABLED=true
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_INTERVAL_SECONDS=30
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_FORMAT=comment
# Rate Limiting
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_BURST_SIZE=20
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_LIMIT_BY=ip
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_MAX_CONNECTIONS_PER_IP=5
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_ENABLED=true
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_BURST_SIZE=20
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_LIMIT_BY=ip
```
-### TCP Server
-```bash
-LOGWISP_STREAMS_0_TCPSERVER_ENABLED=true
-LOGWISP_STREAMS_0_TCPSERVER_PORT=9090
-LOGWISP_STREAMS_0_TCPSERVER_BUFFER_SIZE=5000
-
-# Rate Limiting
-LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_ENABLED=true
-LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=5.0
-LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_BURST_SIZE=10
-```
-
-## Complete Example
-
-Here's a complete example configuring two streams via environment variables:
+## Example
```bash
#!/bin/bash
-# Logging configuration
+# Logging
export LOGWISP_LOGGING_OUTPUT=both
export LOGWISP_LOGGING_LEVEL=info
-export LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
-export LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
-# Stream 0: Application logs
-export LOGWISP_STREAMS_0_NAME=app
-export LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
-export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/myapp
-export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATTERN="*.log"
-export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_IS_FILE=false
+# Pipeline 0: Application logs
+export LOGWISP_PIPELINES_0_NAME=app
+export LOGWISP_PIPELINES_0_SOURCES_0_TYPE=directory
+export LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATH=/var/log/myapp
+export LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PATTERN="*.log"
-# Stream 0: Filters
-export LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
-export LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
+# Filters
+export LOGWISP_PIPELINES_0_FILTERS_0_TYPE=include
+export LOGWISP_PIPELINES_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
-# Stream 0: HTTP server
-export LOGWISP_STREAMS_0_HTTPSERVER_ENABLED=true
-export LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
-export LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
-export LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=25.0
+# HTTP sink
+export LOGWISP_PIPELINES_0_SINKS_0_TYPE=http
+export LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_PORT=8080
+export LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_ENABLED=true
+export LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=25.0
-# Stream 1: System logs
-export LOGWISP_STREAMS_1_NAME=system
-export LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
-export LOGWISP_STREAMS_1_MONITOR_TARGETS_0_PATH=/var/log/syslog
-export LOGWISP_STREAMS_1_MONITOR_TARGETS_0_IS_FILE=true
+# Pipeline 1: System logs
+export LOGWISP_PIPELINES_1_NAME=system
+export LOGWISP_PIPELINES_1_SOURCES_0_TYPE=file
+export LOGWISP_PIPELINES_1_SOURCES_0_OPTIONS_PATH=/var/log/syslog
-# Stream 1: TCP server
-export LOGWISP_STREAMS_1_TCPSERVER_ENABLED=true
-export LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
+# TCP sink
+export LOGWISP_PIPELINES_1_SINKS_0_TYPE=tcp
+export LOGWISP_PIPELINES_1_SINKS_0_OPTIONS_PORT=9090
-# Start LogWisp
logwisp
```
-## Docker/Kubernetes Usage
+## Precedence
-Environment variables are ideal for containerized deployments:
-
-### Docker
-```dockerfile
-FROM logwisp:latest
-ENV LOGWISP_LOGGING_OUTPUT=stdout
-ENV LOGWISP_STREAMS_0_NAME=container
-ENV LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/app
-ENV LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
-```
-
-### Docker Compose
-```yaml
-version: '3'
-services:
- logwisp:
- image: logwisp:latest
- environment:
- - LOGWISP_LOGGING_OUTPUT=stdout
- - LOGWISP_STREAMS_0_NAME=webapp
- - LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/logs
- - LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
- volumes:
- - ./logs:/logs:ro
- ports:
- - "8080:8080"
-```
-
-### Kubernetes ConfigMap
-```yaml
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: logwisp-config
-data:
- LOGWISP_LOGGING_LEVEL: "info"
- LOGWISP_STREAMS_0_NAME: "k8s-app"
- LOGWISP_STREAMS_0_HTTPSERVER_PORT: "8080"
-```
-
-## Precedence Rules
-
-When the same setting is configured multiple ways, this precedence applies:
-
-1. **Command-line flags** (highest priority)
-2. **Environment variables**
-3. **Configuration file**
-4. **Default values** (lowest priority)
-
-Example:
-```bash
-# Config file has: logging.level = "info"
-export LOGWISP_LOGGING_LEVEL=warn
-logwisp --log-level debug
-
-# Result: log level will be "debug" (CLI flag wins)
-```
-
-## Debugging
-
-To see which environment variables LogWisp recognizes:
-```bash
-# List all LOGWISP variables
-env | grep ^LOGWISP_
-
-# Test configuration parsing
-LOGWISP_LOGGING_LEVEL=debug logwisp --version
-```
-
-## Security Considerations
-
-- **Sensitive Values**: Avoid putting passwords or tokens in environment variables
-- **Process Visibility**: Environment variables may be visible to other processes
-- **Container Security**: Use secrets management for sensitive configuration
-- **Logging**: Be careful not to log environment variable values
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [CLI Options](cli.md) - Command-line interface
-- [Docker Deployment](integrations.md#docker) - Container-specific guidance
\ No newline at end of file
+1. Command-line flags (highest)
+2. Environment variables
+3. Configuration file
+4. Defaults (lowest)
\ No newline at end of file
diff --git a/doc/filters.md b/doc/filters.md
index 23e9127..ce22df7 100644
--- a/doc/filters.md
+++ b/doc/filters.md
@@ -1,21 +1,17 @@
# Filter Guide
-LogWisp's filtering system allows you to control which log entries are streamed to clients, reducing noise and focusing on what matters.
+LogWisp filters control which log entries pass through pipelines using regular expressions.
## How Filters Work
-Filters use regular expressions to match log entries. Each filter can either:
-- **Include**: Only matching logs pass through (whitelist)
+- **Include**: Only matching logs pass (whitelist)
- **Exclude**: Matching logs are dropped (blacklist)
+- Multiple filters apply sequentially - all must pass
-Multiple filters are applied sequentially - a log entry must pass ALL filters to be streamed.
-
-## Filter Configuration
-
-### Basic Structure
+## Configuration
```toml
-[[streams.filters]]
+[[pipelines.filters]]
type = "include" # or "exclude"
logic = "or" # or "and"
patterns = [
@@ -26,115 +22,79 @@ patterns = [
### Filter Types
-#### Include Filter (Whitelist)
-Only logs matching the patterns are streamed:
-
+#### Include Filter
```toml
-[[streams.filters]]
+[[pipelines.filters]]
type = "include"
logic = "or"
-patterns = [
- "ERROR",
- "WARN",
- "CRITICAL"
-]
-# Result: Only ERROR, WARN, or CRITICAL logs are streamed
+patterns = ["ERROR", "WARN", "CRITICAL"]
+# Only ERROR, WARN, or CRITICAL logs pass
```
-#### Exclude Filter (Blacklist)
-Logs matching the patterns are dropped:
-
+#### Exclude Filter
```toml
-[[streams.filters]]
+[[pipelines.filters]]
type = "exclude"
-patterns = [
- "DEBUG",
- "TRACE",
- "/health"
-]
-# Result: DEBUG, TRACE, and health check logs are filtered out
+patterns = ["DEBUG", "TRACE", "/health"]
+# DEBUG, TRACE, and health checks are dropped
```
### Logic Operators
-#### OR Logic (Default)
-Log matches if ANY pattern matches:
+- **OR**: Match ANY pattern (default)
+- **AND**: Match ALL patterns
```toml
-[[streams.filters]]
-type = "include"
+# OR Logic
logic = "or"
-patterns = ["ERROR", "FAIL", "EXCEPTION"]
-# Matches: "ERROR: disk full" OR "FAIL: connection timeout" OR "NullPointerException"
-```
+patterns = ["ERROR", "FAIL"]
+# Matches: "ERROR: disk full" OR "FAIL: timeout"
-#### AND Logic
-Log matches only if ALL patterns match:
-
-```toml
-[[streams.filters]]
-type = "include"
+# AND Logic
logic = "and"
patterns = ["database", "timeout", "ERROR"]
# Matches: "ERROR: database connection timeout"
-# Doesn't match: "ERROR: file not found" (missing "database" and "timeout")
+# Not: "ERROR: file not found"
```
## Pattern Syntax
-LogWisp uses Go's regular expression syntax (RE2):
-
-### Basic Patterns
+Go regular expressions (RE2):
+```toml
+"ERROR" # Substring match
+"(?i)error" # Case-insensitive
+"\\berror\\b" # Word boundaries
+"^ERROR" # Start of line
+"ERROR$" # End of line
+"error|fail|warn" # Alternatives
+```
+
+## Common Patterns
+
+### Log Levels
```toml
patterns = [
- "ERROR", # Exact substring match
- "(?i)error", # Case-insensitive
- "\\berror\\b", # Word boundaries
- "^ERROR", # Start of line
- "ERROR$", # End of line
- "ERR(OR)?", # Optional group
- "error|fail|exception" # Alternatives
+ "\\[(ERROR|WARN|INFO)\\]", # [ERROR] format
+ "(?i)\\b(error|warning)\\b", # Word boundaries
+ "level=(error|warn)", # key=value format
]
```
-### Common Pattern Examples
-
-#### Log Levels
+### Application Errors
```toml
-# Standard log levels
-patterns = [
- "\\[(ERROR|WARN|INFO|DEBUG)\\]", # [ERROR] format
- "(?i)\\b(error|warning|info|debug)\\b", # Word boundaries
- "level=(error|warn|info|debug)", # key=value format
- "<(Error|Warning|Info|Debug)>" # XML-style
-]
-
-# Severity patterns
-patterns = [
- "(?i)(fatal|critical|severe)",
- "(?i)(error|fail|exception)",
- "(?i)(warn|warning|caution)",
- "panic:", # Go panics
- "Traceback", # Python errors
-]
-```
-
-#### Application Errors
-```toml
-# Java/JVM
+# Java
patterns = [
"Exception",
- "\\.java:[0-9]+", # Stack trace lines
- "at com\\.mycompany\\.", # Company packages
- "NullPointerException|ClassNotFoundException"
+ "at .+\\.java:[0-9]+",
+ "NullPointerException"
]
# Python
patterns = [
- "Traceback \\(most recent call last\\)",
+ "Traceback",
"File \".+\\.py\", line [0-9]+",
- "(ValueError|TypeError|KeyError)"
+ "ValueError|TypeError"
]
# Go
@@ -143,297 +103,73 @@ patterns = [
"goroutine [0-9]+",
"runtime error:"
]
+```
-# Node.js
+### Performance Issues
+```toml
patterns = [
- "Error:",
- "at .+ \\(.+\\.js:[0-9]+:[0-9]+\\)",
- "UnhandledPromiseRejection"
+ "took [0-9]{4,}ms", # >999ms operations
+ "timeout|timed out",
+ "slow query",
+ "high cpu|cpu usage: [8-9][0-9]%"
]
```
-#### Performance Issues
+### HTTP Patterns
```toml
patterns = [
- "took [0-9]{4,}ms", # Operations over 999ms
- "duration>[0-9]{3,}s", # Long durations
- "timeout|timed out", # Timeouts
- "slow query", # Database
- "memory pressure", # Memory issues
- "high cpu|cpu usage: [8-9][0-9]%" # CPU issues
-]
-```
-
-#### Security Patterns
-```toml
-patterns = [
- "(?i)(unauthorized|forbidden|denied)",
- "(?i)(auth|authentication) fail",
- "invalid (token|session|credentials)",
- "SQL injection|XSS|CSRF",
- "brute force|rate limit",
- "suspicious activity"
-]
-```
-
-#### HTTP Patterns
-```toml
-# Error status codes
-patterns = [
- "status[=:][4-5][0-9]{2}", # status=404, status:500
- "HTTP/[0-9.]+ [4-5][0-9]{2}", # HTTP/1.1 404
- "\"status\":\\s*[4-5][0-9]{2}" # JSON "status": 500
-]
-
-# Specific endpoints
-patterns = [
- "\"(GET|POST|PUT|DELETE) /api/",
- "/api/v[0-9]+/users",
- "path=\"/admin"
+ "status[=:][4-5][0-9]{2}", # 4xx/5xx codes
+ "HTTP/[0-9.]+ [4-5][0-9]{2}",
+ "\"/api/v[0-9]+/", # API paths
]
```
## Filter Chains
-Multiple filters create a processing chain. Each filter must pass for the log to be streamed.
-
-### Example: Error Monitoring
+### Error Monitoring
```toml
-# Step 1: Include only errors and warnings
-[[streams.filters]]
+# Include errors
+[[pipelines.filters]]
type = "include"
-logic = "or"
-patterns = [
- "(?i)\\b(error|fail|exception)\\b",
- "(?i)\\b(warn|warning)\\b",
- "(?i)\\b(critical|fatal|severe)\\b"
-]
+patterns = ["(?i)\\b(error|fail|critical)\\b"]
-# Step 2: Exclude known non-issues
-[[streams.filters]]
+# Exclude known non-issues
+[[pipelines.filters]]
type = "exclude"
-patterns = [
- "Error: Expected behavior",
- "Warning: Deprecated API",
- "INFO.*error in message" # INFO logs talking about errors
-]
-
-# Step 3: Exclude noisy sources
-[[streams.filters]]
-type = "exclude"
-patterns = [
- "/health",
- "/metrics",
- "ELB-HealthChecker",
- "Googlebot"
-]
+patterns = ["Error: Expected", "/health"]
```
-### Example: API Monitoring
+### API Monitoring
```toml
-# Include only API calls
-[[streams.filters]]
+# Include API calls
+[[pipelines.filters]]
type = "include"
-patterns = [
- "/api/",
- "/v[0-9]+/"
-]
+patterns = ["/api/", "/v[0-9]+/"]
-# Exclude successful requests
-[[streams.filters]]
+# Exclude successful
+[[pipelines.filters]]
type = "exclude"
-patterns = [
- "\" 200 ", # HTTP 200 OK
- "\" 201 ", # HTTP 201 Created
- "\" 204 ", # HTTP 204 No Content
- "\" 304 " # HTTP 304 Not Modified
-]
-
-# Exclude OPTIONS requests (CORS)
-[[streams.filters]]
-type = "exclude"
-patterns = [
- "OPTIONS "
-]
+patterns = ["\" 2[0-9]{2} "]
```
-### Example: Security Audit
-```toml
-# Include security-relevant events
-[[streams.filters]]
-type = "include"
-logic = "or"
-patterns = [
- "(?i)auth",
- "(?i)login|logout",
- "(?i)sudo|root",
- "(?i)ssh|sftp|ftp",
- "(?i)firewall|iptables",
- "COMMAND=", # sudo commands
- "USER=", # user actions
- "SELINUX"
-]
+## Performance Tips
-# Must also contain failure/success indicators
-[[streams.filters]]
-type = "include"
-logic = "or"
-patterns = [
- "(?i)(fail|denied|error)",
- "(?i)(success|accepted|granted)",
- "(?i)(invalid|unauthorized)"
-]
-```
-
-## Performance Considerations
-
-### Pattern Complexity
-
-Simple patterns are fast (~1ΞΌs per check):
-```toml
-patterns = ["ERROR", "WARN", "FATAL"]
-```
-
-Complex patterns are slower (~10-100ΞΌs per check):
-```toml
-patterns = [
- "^\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]\\s+\\[(ERROR|WARN)\\]\\s+\\[([^\\]]+)\\]\\s+(.+)$"
-]
-```
-
-### Optimization Tips
-
-1. **Use anchors when possible**:
- ```toml
- "^ERROR" # Faster than "ERROR"
- ```
-
-2. **Avoid nested quantifiers**:
- ```toml
- # BAD: Can cause exponential backtracking
- "((a+)+)+"
-
- # GOOD: Linear time
- "a+"
- ```
-
-3. **Use non-capturing groups**:
- ```toml
- "(?:error|warn)" # Instead of "(error|warn)"
- ```
-
-4. **Order patterns by frequency**:
- ```toml
- # Most common first
- patterns = ["ERROR", "WARN", "INFO", "DEBUG"]
- ```
-
-5. **Prefer character classes**:
- ```toml
- "[0-9]" # Instead of "\\d"
- "[a-zA-Z]" # Instead of "\\w"
- ```
+1. **Use anchors**: `^ERROR` faster than `ERROR`
+2. **Avoid nested quantifiers**: `((a+)+)+`
+3. **Non-capturing groups**: `(?:error|warn)`
+4. **Order by frequency**: Most common first
+5. **Simple patterns**: Faster than complex regex
## Testing Filters
-### Test Configuration
-Create a test configuration with sample logs:
-
-```toml
-[[streams]]
-name = "test"
-[streams.monitor]
-targets = [{ path = "./test-logs", pattern = "*.log" }]
-
-[[streams.filters]]
-type = "include"
-patterns = ["YOUR_PATTERN_HERE"]
-
-[streams.httpserver]
-enabled = true
-port = 8888
-```
-
-### Generate Test Logs
```bash
-# Create test log entries
-echo "[ERROR] Database connection failed" >> test-logs/app.log
-echo "[INFO] User logged in" >> test-logs/app.log
-echo "[WARN] High memory usage: 85%" >> test-logs/app.log
+# Test configuration
+echo "[ERROR] Test" >> test.log
+echo "[INFO] Test" >> test.log
-# Run LogWisp with debug logging
-logwisp --config test.toml --log-level debug
+# Run with debug
+logwisp --log-level debug
-# Check what passes through
-curl -N http://localhost:8888/stream
-```
-
-### Debug Filter Behavior
-Enable debug logging to see filter decisions:
-
-```bash
-logwisp --log-level debug --log-output stderr
-```
-
-Look for messages like:
-```
-Entry filtered out component=filter_chain filter_index=0 filter_type=include
-Entry passed all filters component=filter_chain
-```
-
-## Common Pitfalls
-
-### Case Sensitivity
-By default, patterns are case-sensitive:
-```toml
-# Won't match "error" or "Error"
-patterns = ["ERROR"]
-
-# Use case-insensitive flag
-patterns = ["(?i)error"]
-```
-
-### Partial Matches
-Patterns match substrings by default:
-```toml
-# Matches "ERROR", "ERRORS", "TERROR"
-patterns = ["ERROR"]
-
-# Use word boundaries for exact words
-patterns = ["\\bERROR\\b"]
-```
-
-### Special Characters
-Remember to escape regex special characters:
-```toml
-# Won't work as expected
-patterns = ["[ERROR]"]
-
-# Correct: escape brackets
-patterns = ["\\[ERROR\\]"]
-```
-
-### Performance Impact
-Too many complex patterns can impact performance:
-```toml
-# Consider splitting into multiple streams instead
-[[streams.filters]]
-patterns = [
- # 50+ complex patterns...
-]
-```
-
-## Best Practices
-
-1. **Start Simple**: Begin with basic patterns and refine as needed
-2. **Test Thoroughly**: Use test logs to verify filter behavior
-3. **Monitor Performance**: Check filter statistics in `/status`
-4. **Document Patterns**: Comment complex patterns for maintenance
-5. **Use Multiple Streams**: Instead of complex filters, consider separate streams
-6. **Regular Review**: Periodically review and optimize filter rules
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [Performance Tuning](performance.md) - Optimization guidelines
-- [Examples](examples/) - Real-world filter configurations
\ No newline at end of file
+# Check output
+curl -N http://localhost:8080/stream
+```
\ No newline at end of file
diff --git a/doc/installation.md b/doc/installation.md
index e238278..ccd4c00 100644
--- a/doc/installation.md
+++ b/doc/installation.md
@@ -1,137 +1,54 @@
# Installation Guide
-This guide covers installing LogWisp on various platforms and deployment scenarios.
+Installation process on tested platforms.
## Requirements
-### System Requirements
+- **OS**: Linux, FreeBSD
+- **Architecture**: amd64
+- **Go**: 1.23+ (for building)
-- **OS**: Linux, macOS, FreeBSD, Windows (with WSL)
-- **Architecture**: amd64, arm64
-- **Memory**: 64MB minimum, 256MB recommended
-- **Disk**: 10MB for binary, plus log storage
-- **Go**: 1.23+ (for building from source)
-
-### Runtime Dependencies
-
-LogWisp is a single static binary with no runtime dependencies. It only requires:
-- Read access to monitored log files
-- Network access for serving streams
-- Write access for operational logs (optional)
-
-## Installation Methods
+## Installation
### Pre-built Binaries
-Download the latest release:
-
```bash
-# Linux (amd64)
+# Linux amd64
wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
chmod +x logwisp-linux-amd64
sudo mv logwisp-linux-amd64 /usr/local/bin/logwisp
-# macOS (Intel)
+# macOS Intel
wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-amd64
chmod +x logwisp-darwin-amd64
sudo mv logwisp-darwin-amd64 /usr/local/bin/logwisp
-# macOS (Apple Silicon)
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-arm64
-chmod +x logwisp-darwin-arm64
-sudo mv logwisp-darwin-arm64 /usr/local/bin/logwisp
-```
-
-Verify installation:
-```bash
+# Verify
logwisp --version
```
### From Source
-Build from source code:
-
```bash
-# Clone repository
git clone https://github.com/yourusername/logwisp.git
cd logwisp
-
-# Build
make build
-
-# Install
sudo make install
-
-# Or install to custom location
-make install PREFIX=/opt/logwisp
```
-### Using Go Install
-
-Install directly with Go:
+### Go Install
```bash
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+go install github.com/lixenwraith/logwisp/src/cmd/logwisp@latest
```
+Note: Binary created with this method will not contain version information.
-Note: This installs to `$GOPATH/bin` (usually `~/go/bin`)
+## Platform-Specific
-### Docker
-
-Official Docker image:
+### Linux (systemd)
```bash
-# Pull image
-docker pull yourusername/logwisp:latest
-
-# Run with volume mount
-docker run -d \
- --name logwisp \
- -p 8080:8080 \
- -v /var/log:/logs:ro \
- -v $PWD/config.toml:/config/logwisp.toml:ro \
- yourusername/logwisp:latest \
- --config /config/logwisp.toml
-```
-
-Build your own image:
-
-```dockerfile
-FROM golang:1.23-alpine AS builder
-WORKDIR /build
-COPY . .
-RUN go build -o logwisp ./src/cmd/logwisp
-
-FROM alpine:latest
-RUN apk --no-cache add ca-certificates
-COPY --from=builder /build/logwisp /usr/local/bin/
-ENTRYPOINT ["logwisp"]
-```
-
-## Platform-Specific Instructions
-
-### Linux
-
-#### Debian/Ubuntu
-
-Create package (planned):
-```bash
-# Future feature
-sudo apt install logwisp
-```
-
-Manual installation:
-```bash
-# Download binary
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64 -O logwisp
-chmod +x logwisp
-sudo mv logwisp /usr/local/bin/
-
-# Create config directory
-sudo mkdir -p /etc/logwisp
-sudo cp config/logwisp.toml.example /etc/logwisp/logwisp.toml
-
-# Create systemd service
+# Create service
sudo tee /etc/systemd/system/logwisp.service << EOF
[Unit]
Description=LogWisp Log Monitoring Service
@@ -140,21 +57,10 @@ After=network.target
[Service]
Type=simple
User=logwisp
-Group=logwisp
ExecStart=/usr/local/bin/logwisp --config /etc/logwisp/logwisp.toml
Restart=always
-RestartSec=5
StandardOutput=journal
StandardError=journal
-SyslogIdentifier=logwisp
-
-# Security hardening
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadOnlyPaths=/var/log
-ReadWritePaths=/var/log/logwisp
[Install]
WantedBy=multi-user.target
@@ -163,9 +69,12 @@ EOF
# Create user
sudo useradd -r -s /bin/false logwisp
-# Create log directory
-sudo mkdir -p /var/log/logwisp
-sudo chown logwisp:logwisp /var/log/logwisp
+# Create service user
+sudo useradd -r -s /bin/false logwisp
+
+# Create configuration directory
+sudo mkdir -p /etc/logwisp
+sudo chown logwisp:logwisp /etc/logwisp
# Enable and start
sudo systemctl daemon-reload
@@ -173,93 +82,11 @@ sudo systemctl enable logwisp
sudo systemctl start logwisp
```
-#### Red Hat/CentOS/Fedora
+### FreeBSD (rc.d)
```bash
-# Similar to Debian, but use:
-sudo yum install wget # or dnf on newer versions
-
-# SELinux context (if enabled)
-sudo semanage fcontext -a -t bin_t /usr/local/bin/logwisp
-sudo restorecon -v /usr/local/bin/logwisp
-```
-
-#### Arch Linux
-
-AUR package (community maintained):
-```bash
-# Future feature
-yay -S logwisp
-```
-
-### macOS
-
-#### Homebrew
-
-Formula (planned):
-```bash
-# Future feature
-brew install logwisp
-```
-
-#### Manual Installation
-
-```bash
-# Download and install
-curl -L https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-$(uname -m) -o logwisp
-chmod +x logwisp
-sudo mv logwisp /usr/local/bin/
-
-# Create LaunchDaemon
-sudo tee /Library/LaunchDaemons/com.logwisp.plist << EOF
-
-
-
-
- Label
- com.logwisp
- ProgramArguments
-
- /usr/local/bin/logwisp
- --config
- /usr/local/etc/logwisp/logwisp.toml
-
- RunAtLoad
-
- KeepAlive
-
- StandardOutPath
- /usr/local/var/log/logwisp.log
- StandardErrorPath
- /usr/local/var/log/logwisp.error.log
-
-
-EOF
-
-# Load service
-sudo launchctl load /Library/LaunchDaemons/com.logwisp.plist
-```
-
-### FreeBSD
-
-#### Ports
-
-```bash
-# Future feature
-cd /usr/ports/sysutils/logwisp
-make install clean
-```
-
-#### Manual Installation
-
-```bash
-# Download
-fetch https://github.com/yourusername/logwisp/releases/latest/download/logwisp-freebsd-amd64
-chmod +x logwisp-freebsd-amd64
-mv logwisp-freebsd-amd64 /usr/local/bin/logwisp
-
-# RC script
-cat > /usr/local/etc/rc.d/logwisp << 'EOF'
+# Create service script
+sudo tee /usr/local/etc/rc.d/logwisp << 'EOF'
#!/bin/sh
# PROVIDE: logwisp
@@ -273,319 +100,125 @@ rcvar="${name}_enable"
command="/usr/local/bin/logwisp"
command_args="--config /usr/local/etc/logwisp/logwisp.toml"
pidfile="/var/run/${name}.pid"
+start_cmd="logwisp_start"
+stop_cmd="logwisp_stop"
+
+logwisp_start()
+{
+ echo "Starting logwisp service..."
+ /usr/sbin/daemon -c -f -p ${pidfile} ${command} ${command_args}
+}
+
+logwisp_stop()
+{
+ if [ -f ${pidfile} ]; then
+ echo "Stopping logwisp service..."
+ kill $(cat ${pidfile})
+ rm -f ${pidfile}
+ fi
+}
load_rc_config $name
: ${logwisp_enable:="NO"}
+: ${logwisp_config:="/usr/local/etc/logwisp/logwisp.toml"}
run_rc_command "$1"
EOF
-chmod +x /usr/local/etc/rc.d/logwisp
+# Make executable
+sudo chmod +x /usr/local/etc/rc.d/logwisp
-# Enable
-sysrc logwisp_enable="YES"
-service logwisp start
-```
+# Create service user
+sudo pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin
-### Windows
+# Create configuration directory
+sudo mkdir -p /usr/local/etc/logwisp
+sudo chown logwisp:logwisp /usr/local/etc/logwisp
-#### Windows Subsystem for Linux (WSL)
+# Enable service
+sudo sysrc logwisp_enable="YES"
-```bash
-# Inside WSL, follow Linux instructions
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
-chmod +x logwisp-linux-amd64
-./logwisp-linux-amd64
-```
-
-#### Native Windows (planned)
-
-Future support for native Windows service.
-
-## Container Deployment
-
-### Docker Compose
-
-```yaml
-version: '3.8'
-
-services:
- logwisp:
- image: yourusername/logwisp:latest
- container_name: logwisp
- restart: unless-stopped
- ports:
- - "8080:8080"
- - "9090:9090" # If using TCP
- volumes:
- - /var/log:/logs:ro
- - ./logwisp.toml:/config/logwisp.toml:ro
- command: ["--config", "/config/logwisp.toml"]
- environment:
- - LOGWISP_LOGGING_LEVEL=info
- healthcheck:
- test: ["CMD", "curl", "-f", "http://localhost:8080/status"]
- interval: 30s
- timeout: 3s
- retries: 3
-```
-
-### Kubernetes
-
-Deployment manifest:
-
-```yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: logwisp
- labels:
- app: logwisp
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: logwisp
- template:
- metadata:
- labels:
- app: logwisp
- spec:
- containers:
- - name: logwisp
- image: yourusername/logwisp:latest
- args:
- - --config
- - /config/logwisp.toml
- ports:
- - containerPort: 8080
- name: http
- - containerPort: 9090
- name: tcp
- volumeMounts:
- - name: logs
- mountPath: /logs
- readOnly: true
- - name: config
- mountPath: /config
- livenessProbe:
- httpGet:
- path: /status
- port: 8080
- initialDelaySeconds: 10
- periodSeconds: 30
- readinessProbe:
- httpGet:
- path: /status
- port: 8080
- initialDelaySeconds: 5
- periodSeconds: 10
- resources:
- requests:
- memory: "128Mi"
- cpu: "100m"
- limits:
- memory: "512Mi"
- cpu: "500m"
- volumes:
- - name: logs
- hostPath:
- path: /var/log
- - name: config
- configMap:
- name: logwisp-config
----
-apiVersion: v1
-kind: Service
-metadata:
- name: logwisp
-spec:
- selector:
- app: logwisp
- ports:
- - name: http
- port: 8080
- targetPort: 8080
- - name: tcp
- port: 9090
- targetPort: 9090
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: logwisp-config
-data:
- logwisp.toml: |
- [[streams]]
- name = "k8s"
- [streams.monitor]
- targets = [{ path = "/logs", pattern = "*.log" }]
- [streams.httpserver]
- enabled = true
- port = 8080
+# Start service
+sudo service logwisp start
```
## Post-Installation
### Verify Installation
+```bash
+# Check version
+logwisp --version
-1. Check version:
- ```bash
- logwisp --version
- ```
+# Test configuration
+logwisp --config /etc/logwisp/logwisp.toml --log-level debug
-2. Test configuration:
- ```bash
- logwisp --config /etc/logwisp/logwisp.toml --log-level debug
- ```
+# Check service
+sudo systemctl status logwisp
+```
-3. Check service status:
- ```bash
- # systemd
- sudo systemctl status logwisp
-
- # macOS
- sudo launchctl list | grep logwisp
-
- # FreeBSD
- service logwisp status
- ```
+### Linux Service Status
+```bash
+sudo systemctl status logwisp
+```
-4. Test streaming:
- ```bash
- curl -N http://localhost:8080/stream
- ```
-
-### Security Hardening
-
-1. **Create dedicated user**:
- ```bash
- sudo useradd -r -s /bin/false -d /var/lib/logwisp logwisp
- ```
-
-2. **Set file permissions**:
- ```bash
- sudo chown root:root /usr/local/bin/logwisp
- sudo chmod 755 /usr/local/bin/logwisp
- sudo chown -R logwisp:logwisp /etc/logwisp
- sudo chmod 640 /etc/logwisp/logwisp.toml
- ```
-
-3. **Configure firewall**:
- ```bash
- # UFW
- sudo ufw allow 8080/tcp comment "LogWisp HTTP"
-
- # firewalld
- sudo firewall-cmd --permanent --add-port=8080/tcp
- sudo firewall-cmd --reload
- ```
-
-4. **Enable SELinux/AppArmor** (if applicable)
+### FreeBSD Service Status
+```bash
+sudo service logwisp status
+```
### Initial Configuration
-1. Copy example configuration:
- ```bash
- sudo cp /usr/local/share/logwisp/examples/logwisp.toml.example /etc/logwisp/logwisp.toml
- ```
+Create a basic configuration file:
-2. Edit configuration:
- ```bash
- sudo nano /etc/logwisp/logwisp.toml
- ```
+```toml
+# /etc/logwisp/logwisp.toml (Linux)
+# /usr/local/etc/logwisp/logwisp.toml (FreeBSD)
-3. Set up log monitoring:
- ```toml
- [[streams]]
- name = "myapp"
- [streams.monitor]
- targets = [
- { path = "/var/log/myapp", pattern = "*.log" }
- ]
- ```
+[[pipelines]]
+name = "myapp"
-4. Restart service:
- ```bash
- sudo systemctl restart logwisp
- ```
+[[pipelines.sources]]
+type = "directory"
+options = {
+ path = "/path/to/application/logs",
+ pattern = "*.log"
+}
+
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
+```
+
+Restart service after configuration changes:
+
+**Linux:**
+```bash
+sudo systemctl restart logwisp
+```
+
+**FreeBSD:**
+```bash
+sudo service logwisp restart
+```
## Uninstallation
### Linux
```bash
-# Stop service
sudo systemctl stop logwisp
sudo systemctl disable logwisp
-
-# Remove files
sudo rm /usr/local/bin/logwisp
sudo rm /etc/systemd/system/logwisp.service
sudo rm -rf /etc/logwisp
-sudo rm -rf /var/log/logwisp
-
-# Remove user
sudo userdel logwisp
```
-### macOS
+### FreeBSD
```bash
-# Stop service
-sudo launchctl unload /Library/LaunchDaemons/com.logwisp.plist
-
-# Remove files
+sudo service logwisp stop
+sudo sysrc logwisp_enable="NO"
sudo rm /usr/local/bin/logwisp
-sudo rm /Library/LaunchDaemons/com.logwisp.plist
+sudo rm /usr/local/etc/rc.d/logwisp
sudo rm -rf /usr/local/etc/logwisp
-```
-
-### Docker
-```bash
-docker stop logwisp
-docker rm logwisp
-docker rmi yourusername/logwisp:latest
-```
-
-## Troubleshooting Installation
-
-### Permission Denied
-
-If you get permission errors:
-```bash
-# Check file ownership
-ls -la /usr/local/bin/logwisp
-
-# Fix permissions
-sudo chmod +x /usr/local/bin/logwisp
-
-# Check log directory
-sudo mkdir -p /var/log/logwisp
-sudo chown logwisp:logwisp /var/log/logwisp
-```
-
-### Service Won't Start
-
-Check logs:
-```bash
-# systemd
-sudo journalctl -u logwisp -f
-
-# Manual run
-sudo -u logwisp /usr/local/bin/logwisp --config /etc/logwisp/logwisp.toml
-```
-
-### Port Already in Use
-
-Find conflicting process:
-```bash
-sudo lsof -i :8080
-# or
-sudo netstat -tlnp | grep 8080
-```
-
-## See Also
-
-- [Quick Start](quickstart.md) - Get running quickly
-- [Configuration Guide](configuration.md) - Configure LogWisp
-- [Troubleshooting](troubleshooting.md) - Common issues
-- [Security Best Practices](security.md) - Hardening guide
\ No newline at end of file
+sudo pw userdel logwisp
+```
\ No newline at end of file
diff --git a/doc/monitoring.md b/doc/monitoring.md
deleted file mode 100644
index b2a96d8..0000000
--- a/doc/monitoring.md
+++ /dev/null
@@ -1,511 +0,0 @@
-# Monitoring & Status Guide
-
-LogWisp provides comprehensive monitoring capabilities through status endpoints, operational logs, and metrics.
-
-## Status Endpoints
-
-### Stream Status
-
-Each stream exposes its own status endpoint:
-
-```bash
-# Standalone mode
-curl http://localhost:8080/status
-
-# Router mode
-curl http://localhost:8080/streamname/status
-```
-
-Example response:
-```json
-{
- "service": "LogWisp",
- "version": "1.0.0",
- "server": {
- "type": "http",
- "port": 8080,
- "active_clients": 5,
- "buffer_size": 1000,
- "uptime_seconds": 3600,
- "mode": {
- "standalone": true,
- "router": false
- }
- },
- "monitor": {
- "active_watchers": 3,
- "total_entries": 152341,
- "dropped_entries": 12,
- "start_time": "2024-01-20T10:00:00Z",
- "last_entry_time": "2024-01-20T11:00:00Z"
- },
- "filters": {
- "filter_count": 2,
- "total_processed": 152341,
- "total_passed": 48234,
- "filters": [
- {
- "type": "include",
- "logic": "or",
- "pattern_count": 3,
- "total_processed": 152341,
- "total_matched": 48234,
- "total_dropped": 0
- }
- ]
- },
- "features": {
- "heartbeat": {
- "enabled": true,
- "interval": 30,
- "format": "comment"
- },
- "rate_limit": {
- "enabled": true,
- "total_requests": 8234,
- "blocked_requests": 89,
- "active_ips": 12,
- "total_connections": 5
- }
- }
-}
-```
-
-### Global Status (Router Mode)
-
-In router mode, a global status endpoint provides aggregated information:
-
-```bash
-curl http://localhost:8080/status
-```
-
-## Key Metrics
-
-### Monitor Metrics
-
-Track file watching performance:
-
-| Metric | Description | Healthy Range |
-|--------|-------------|---------------|
-| `active_watchers` | Number of files being watched | 1-1000 |
-| `total_entries` | Total log entries processed | Increasing |
-| `dropped_entries` | Entries dropped due to buffer full | < 1% of total |
-| `entries_per_second` | Current processing rate | Varies |
-
-### Connection Metrics
-
-Monitor client connections:
-
-| Metric | Description | Warning Signs |
-|--------|-------------|---------------|
-| `active_clients` | Current SSE connections | Near limit |
-| `tcp_connections` | Current TCP connections | Near limit |
-| `total_connections` | All active connections | > 80% of max |
-
-### Filter Metrics
-
-Understand filtering effectiveness:
-
-| Metric | Description | Optimization |
-|--------|-------------|--------------|
-| `total_processed` | Entries checked | - |
-| `total_passed` | Entries that passed | Very low = too restrictive |
-| `total_dropped` | Entries filtered out | Very high = review patterns |
-
-### Rate Limit Metrics
-
-Track rate limiting impact:
-
-| Metric | Description | Action Needed |
-|--------|-------------|---------------|
-| `blocked_requests` | Rejected requests | High = increase limits |
-| `active_ips` | Unique clients | High = scale out |
-| `blocked_percentage` | Rejection rate | > 10% = review |
-
-## Operational Logging
-
-### Log Levels
-
-Configure LogWisp's operational logging:
-
-```toml
-[logging]
-output = "both" # file and stderr
-level = "info" # info for production
-```
-
-Log levels and their use:
-- **DEBUG**: Detailed internal operations
-- **INFO**: Normal operations, connections
-- **WARN**: Recoverable issues
-- **ERROR**: Errors requiring attention
-
-### Important Log Messages
-
-#### Startup Messages
-```
-LogWisp starting version=1.0.0 config_file=/etc/logwisp.toml
-Stream registered with router stream=app
-TCP endpoint configured transport=system port=9090
-HTTP endpoints configured transport=app stream_url=http://localhost:8080/stream
-```
-
-#### Connection Events
-```
-HTTP client connected remote_addr=192.168.1.100:54231 active_clients=6
-HTTP client disconnected remote_addr=192.168.1.100:54231 active_clients=5
-TCP connection opened remote_addr=192.168.1.100:54232 active_connections=3
-```
-
-#### Error Conditions
-```
-Failed to open file for checking path=/var/log/app.log error=permission denied
-Scanner error while reading file path=/var/log/huge.log error=token too long
-Request rate limited ip=192.168.1.100
-Connection limit exceeded ip=192.168.1.100 connections=5 limit=5
-```
-
-#### Performance Warnings
-```
-Dropped log entry - subscriber buffer full
-Dropped entry for slow client remote_addr=192.168.1.100
-Check interval too small: 5ms (min: 10ms)
-```
-
-## Health Checks
-
-### Basic Health Check
-
-Simple up/down check:
-
-```bash
-#!/bin/bash
-# health_check.sh
-
-STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/status)
-
-if [ "$STATUS" -eq 200 ]; then
- echo "LogWisp is healthy"
- exit 0
-else
- echo "LogWisp is unhealthy (status: $STATUS)"
- exit 1
-fi
-```
-
-### Advanced Health Check
-
-Check specific conditions:
-
-```bash
-#!/bin/bash
-# advanced_health_check.sh
-
-RESPONSE=$(curl -s http://localhost:8080/status)
-
-# Check if processing logs
-ENTRIES=$(echo "$RESPONSE" | jq -r '.monitor.total_entries')
-if [ "$ENTRIES" -eq 0 ]; then
- echo "WARNING: No log entries processed"
- exit 1
-fi
-
-# Check dropped entries
-DROPPED=$(echo "$RESPONSE" | jq -r '.monitor.dropped_entries')
-TOTAL=$(echo "$RESPONSE" | jq -r '.monitor.total_entries')
-DROP_PERCENT=$(( DROPPED * 100 / TOTAL ))
-
-if [ "$DROP_PERCENT" -gt 5 ]; then
- echo "WARNING: High drop rate: ${DROP_PERCENT}%"
- exit 1
-fi
-
-# Check connections
-CONNECTIONS=$(echo "$RESPONSE" | jq -r '.server.active_clients')
-echo "OK: Processing logs, $CONNECTIONS active clients"
-exit 0
-```
-
-### Container Health Check
-
-Docker/Kubernetes configuration:
-
-```dockerfile
-# Dockerfile
-HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
- CMD curl -f http://localhost:8080/status || exit 1
-```
-
-```yaml
-# Kubernetes
-livenessProbe:
- httpGet:
- path: /status
- port: 8080
- initialDelaySeconds: 10
- periodSeconds: 30
-
-readinessProbe:
- httpGet:
- path: /status
- port: 8080
- initialDelaySeconds: 5
- periodSeconds: 10
-```
-
-## Monitoring Integration
-
-### Prometheus Metrics
-
-Export metrics in Prometheus format:
-
-```bash
-#!/bin/bash
-# prometheus_exporter.sh
-
-while true; do
- STATUS=$(curl -s http://localhost:8080/status)
-
- # Extract metrics
- CLIENTS=$(echo "$STATUS" | jq -r '.server.active_clients')
- ENTRIES=$(echo "$STATUS" | jq -r '.monitor.total_entries')
- DROPPED=$(echo "$STATUS" | jq -r '.monitor.dropped_entries')
-
- # Output Prometheus format
- cat << EOF
-# HELP logwisp_active_clients Number of active streaming clients
-# TYPE logwisp_active_clients gauge
-logwisp_active_clients $CLIENTS
-
-# HELP logwisp_total_entries Total log entries processed
-# TYPE logwisp_total_entries counter
-logwisp_total_entries $ENTRIES
-
-# HELP logwisp_dropped_entries Total log entries dropped
-# TYPE logwisp_dropped_entries counter
-logwisp_dropped_entries $DROPPED
-EOF
-
- sleep 60
-done
-```
-
-### Grafana Dashboard
-
-Key panels for Grafana:
-
-1. **Active Connections**
- - Query: `logwisp_active_clients`
- - Visualization: Graph
- - Alert: > 80% of max
-
-2. **Log Processing Rate**
- - Query: `rate(logwisp_total_entries[5m])`
- - Visualization: Graph
- - Alert: < 1 entry/min
-
-3. **Drop Rate**
- - Query: `rate(logwisp_dropped_entries[5m]) / rate(logwisp_total_entries[5m])`
- - Visualization: Gauge
- - Alert: > 5%
-
-4. **Rate Limit Rejections**
- - Query: `rate(logwisp_blocked_requests[5m])`
- - Visualization: Graph
- - Alert: > 10/min
-
-### Datadog Integration
-
-Send custom metrics:
-
-```bash
-#!/bin/bash
-# datadog_metrics.sh
-
-while true; do
- STATUS=$(curl -s http://localhost:8080/status)
-
- # Send metrics to Datadog
- echo "$STATUS" | jq -r '
- "logwisp.connections:\(.server.active_clients)|g",
- "logwisp.entries:\(.monitor.total_entries)|c",
- "logwisp.dropped:\(.monitor.dropped_entries)|c"
- ' | while read metric; do
- echo "$metric" | nc -u -w1 localhost 8125
- done
-
- sleep 60
-done
-```
-
-## Performance Monitoring
-
-### CPU Usage
-
-Monitor CPU usage by component:
-
-```bash
-# Check process CPU
-top -p $(pgrep logwisp) -b -n 1
-
-# Profile CPU usage
-go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
-```
-
-Common CPU consumers:
-- File watching (reduce check_interval_ms)
-- Regex filtering (simplify patterns)
-- JSON encoding (reduce clients)
-
-### Memory Usage
-
-Track memory consumption:
-
-```bash
-# Check process memory
-ps aux | grep logwisp
-
-# Detailed memory stats
-cat /proc/$(pgrep logwisp)/status | grep -E "Vm(RSS|Size)"
-```
-
-Memory optimization:
-- Reduce buffer sizes
-- Limit connections
-- Simplify filters
-
-### Network Bandwidth
-
-Monitor streaming bandwidth:
-
-```bash
-# Network statistics
-netstat -i
-iftop -i eth0 -f "port 8080"
-
-# Connection count
-ss -tan | grep :8080 | wc -l
-```
-
-## Alerting
-
-### Basic Alerts
-
-Essential alerts to configure:
-
-| Alert | Condition | Severity |
-|-------|-----------|----------|
-| Service Down | Status endpoint fails | Critical |
-| High Drop Rate | > 10% entries dropped | Warning |
-| No Log Activity | 0 entries/min for 5 min | Warning |
-| Connection Limit | > 90% of max connections | Warning |
-| Rate Limit High | > 20% requests blocked | Warning |
-
-### Alert Script
-
-Example monitoring script:
-
-```bash
-#!/bin/bash
-# monitor_alerts.sh
-
-check_alert() {
- local name=$1
- local condition=$2
- local message=$3
-
- if eval "$condition"; then
- echo "ALERT: $name - $message"
- # Send to alerting system
- # curl -X POST https://alerts.example.com/...
- fi
-}
-
-while true; do
- STATUS=$(curl -s http://localhost:8080/status)
-
- if [ -z "$STATUS" ]; then
- check_alert "SERVICE_DOWN" "true" "LogWisp not responding"
- sleep 60
- continue
- fi
-
- # Extract metrics
- DROPPED=$(echo "$STATUS" | jq -r '.monitor.dropped_entries')
- TOTAL=$(echo "$STATUS" | jq -r '.monitor.total_entries')
- CLIENTS=$(echo "$STATUS" | jq -r '.server.active_clients')
-
- # Check conditions
- check_alert "HIGH_DROP_RATE" \
- "[ $((DROPPED * 100 / TOTAL)) -gt 10 ]" \
- "Drop rate above 10%"
-
- check_alert "HIGH_CONNECTIONS" \
- "[ $CLIENTS -gt 90 ]" \
- "Near connection limit: $CLIENTS/100"
-
- sleep 60
-done
-```
-
-## Troubleshooting with Monitoring
-
-### No Logs Appearing
-
-Check monitor stats:
-```bash
-curl -s http://localhost:8080/status | jq '.monitor'
-```
-
-Look for:
-- `active_watchers` = 0 (no files found)
-- `total_entries` not increasing (files not updating)
-
-### High CPU Usage
-
-Enable debug logging:
-```bash
-logwisp --log-level debug --log-output stderr
-```
-
-Watch for:
-- Frequent "checkFile" messages (reduce check_interval)
-- Many filter operations (optimize patterns)
-
-### Memory Growth
-
-Monitor over time:
-```bash
-while true; do
- ps aux | grep logwisp | grep -v grep
- curl -s http://localhost:8080/status | jq '.server.active_clients'
- sleep 10
-done
-```
-
-### Connection Issues
-
-Check connection stats:
-```bash
-# Current connections
-curl -s http://localhost:8080/status | jq '.server'
-
-# Rate limit stats
-curl -s http://localhost:8080/status | jq '.features.rate_limit'
-```
-
-## Best Practices
-
-1. **Regular Monitoring**: Check status endpoints every 30-60 seconds
-2. **Set Alerts**: Configure alerts for critical conditions
-3. **Log Rotation**: Rotate LogWisp's own logs to prevent disk fill
-4. **Baseline Metrics**: Establish normal ranges for your environment
-5. **Capacity Planning**: Monitor trends for scaling decisions
-6. **Test Monitoring**: Verify alerts work before issues occur
-
-## See Also
-
-- [Performance Tuning](performance.md) - Optimization guide
-- [Troubleshooting](troubleshooting.md) - Common issues
-- [Configuration Guide](configuration.md) - Monitoring configuration
-- [Integration Examples](integrations.md) - Monitoring system integration
\ No newline at end of file
diff --git a/doc/quickstart.md b/doc/quickstart.md
index e133730..ca9a3bc 100644
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@@ -1,26 +1,20 @@
# Quick Start Guide
-Get LogWisp up and running in 5 minutes!
+Get LogWisp up and running in 5 minutes with the new pipeline architecture!
## Installation
### From Source
```bash
-# Clone the repository
-git clone https://github.com/yourusername/logwisp.git
+git clone https://github.com/lixenwraith/logwisp.git
cd logwisp
-
-# Build and install
make install
-
-# Or just build
-make build
-./logwisp --version
```
### Using Go Install
+
```bash
-go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+go install github.com/lixenwraith/logwisp/src/cmd/logwisp@latest
```
## Basic Usage
@@ -35,22 +29,19 @@ logwisp
### 2. Stream Logs
-In another terminal, connect to the log stream:
+Connect to the log stream:
```bash
-# Using curl (SSE stream)
+# SSE stream
curl -N http://localhost:8080/stream
# Check status
curl http://localhost:8080/status | jq .
```
-### 3. Create Some Logs
-
-Generate test logs to see streaming in action:
+### 3. Generate Test Logs
```bash
-# In a third terminal
echo "[ERROR] Something went wrong!" >> test.log
echo "[INFO] Application started" >> test.log
echo "[WARN] Low memory warning" >> test.log
@@ -60,88 +51,78 @@ echo "[WARN] Low memory warning" >> test.log
### Monitor Specific Directory
-Create a configuration file `~/.config/logwisp.toml`:
+Create `~/.config/logwisp.toml`:
```toml
-[[streams]]
+[[pipelines]]
name = "myapp"
-[streams.monitor]
-targets = [
- { path = "/var/log/myapp", pattern = "*.log", is_file = false }
-]
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/myapp", pattern = "*.log" }
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
```
-Run LogWisp:
-```bash
-logwisp
-```
-
-### Filter Only Errors and Warnings
-
-Add filters to your configuration:
+### Filter Only Errors
```toml
-[[streams]]
+[[pipelines]]
name = "errors"
-[streams.monitor]
-targets = [
- { path = "./", pattern = "*.log" }
-]
+[[pipelines.sources]]
+type = "directory"
+options = { path = "./", pattern = "*.log" }
-[[streams.filters]]
+[[pipelines.filters]]
type = "include"
-patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
+patterns = ["ERROR", "WARN", "CRITICAL"]
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
```
-### Multiple Log Sources
+### Multiple Outputs
-Monitor different applications on different ports:
+Send logs to both HTTP stream and file:
```toml
-# Stream 1: Web application
-[[streams]]
-name = "webapp"
-[streams.monitor]
-targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines]]
+name = "multi-output"
-# Stream 2: Database
-[[streams]]
-name = "database"
-[streams.monitor]
-targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8081
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
+
+# HTTP streaming
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
+
+# File archival
+[[pipelines.sinks]]
+type = "file"
+options = { directory = "/var/log/archive", name = "app" }
```
### TCP Streaming
-For high-performance streaming, use TCP:
+For high-performance streaming:
```toml
-[[streams]]
+[[pipelines]]
name = "highperf"
-[streams.monitor]
-targets = [{ path = "/var/log/app", pattern = "*.log" }]
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
-[streams.tcpserver]
-enabled = true
-port = 9090
-buffer_size = 5000
+[[pipelines.sinks]]
+type = "tcp"
+options = { port = 9090, buffer_size = 5000 }
```
Connect with netcat:
@@ -151,16 +132,15 @@ nc localhost 9090
### Router Mode
-Consolidate multiple streams on one port using router mode:
+Run multiple pipelines on shared ports:
```bash
-# With the multi-stream config above
logwisp --router
-# Access streams at:
-# http://localhost:8080/webapp/stream
-# http://localhost:8080/database/stream
-# http://localhost:8080/status (global status)
+# Access pipelines at:
+# http://localhost:8080/myapp/stream
+# http://localhost:8080/errors/stream
+# http://localhost:8080/status (global)
```
## Quick Tips
@@ -170,40 +150,23 @@ logwisp --router
logwisp --log-level debug --log-output stderr
```
-### Run in Background
-```bash
-logwisp --background --config /etc/logwisp/prod.toml
-```
-
### Rate Limiting
-Protect your streams from abuse:
-
```toml
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-burst_size = 20
-max_connections_per_ip = 5
+[[pipelines.sinks]]
+type = "http"
+options = {
+ port = 8080,
+ rate_limit = {
+ enabled = true,
+ requests_per_second = 10.0,
+ burst_size = 20
+ }
+}
```
-### JSON Output Format
-For structured logging:
-
+### Console Output
```toml
-[logging.console]
-format = "json"
-```
-
-## What's Next?
-
-- Read the [Configuration Guide](configuration.md) for all options
-- Learn about [Filters](filters.md) for advanced pattern matching
-- Explore [Rate Limiting](ratelimiting.md) for production deployments
-- Check out [Example Configurations](examples/) for more scenarios
-
-## Getting Help
-
-- Run `logwisp --help` for CLI options
-- Check `http://localhost:8080/status` for runtime statistics
-- Enable debug logging for troubleshooting
-- Visit our [GitHub repository](https://github.com/yourusername/logwisp) for issues and discussions
\ No newline at end of file
+[[pipelines.sinks]]
+type = "stdout" # or "stderr"
+options = {}
+```
\ No newline at end of file
diff --git a/doc/ratelimiting.md b/doc/ratelimiting.md
index 96243ca..a4bb597 100644
--- a/doc/ratelimiting.md
+++ b/doc/ratelimiting.md
@@ -1,136 +1,65 @@
# Rate Limiting Guide
-LogWisp provides configurable rate limiting to protect against abuse, prevent resource exhaustion, and ensure fair access to log streams.
+LogWisp provides configurable rate limiting to protect against abuse and ensure fair access.
-## How Rate Limiting Works
+## How It Works
-LogWisp uses a **token bucket algorithm** for smooth, burst-tolerant rate limiting:
-
-1. Each client (or globally) gets a bucket with a fixed capacity
-2. Tokens are added to the bucket at a configured rate
+Token bucket algorithm:
+1. Each client gets a bucket with fixed capacity
+2. Tokens refill at configured rate
3. Each request consumes one token
-4. If no tokens are available, the request is rejected
-5. The bucket can accumulate tokens up to its capacity for bursts
+4. No tokens = request rejected
## Configuration
-### Basic Configuration
-
```toml
-[streams.httpserver.rate_limit]
-enabled = true # Enable rate limiting
-requests_per_second = 10.0 # Token refill rate
-burst_size = 20 # Maximum tokens (bucket capacity)
-limit_by = "ip" # "ip" or "global"
+[[pipelines.sinks]]
+type = "http" # or "tcp"
+options = {
+ port = 8080,
+ rate_limit = {
+ enabled = true,
+ requests_per_second = 10.0,
+ burst_size = 20,
+ limit_by = "ip", # or "global"
+ max_connections_per_ip = 5,
+ max_total_connections = 100,
+ response_code = 429,
+ response_message = "Rate limit exceeded"
+ }
+}
```
-### Complete Options
-
-```toml
-[streams.httpserver.rate_limit]
-# Core settings
-enabled = true # Enable/disable rate limiting
-requests_per_second = 10.0 # Token generation rate (float)
-burst_size = 20 # Token bucket capacity
-
-# Limiting strategy
-limit_by = "ip" # "ip" or "global"
-
-# Connection limits
-max_connections_per_ip = 5 # Max concurrent connections per IP
-max_total_connections = 100 # Max total concurrent connections
-
-# Response configuration
-response_code = 429 # HTTP status code when limited
-response_message = "Rate limit exceeded" # Error message
-
-# Same options available for TCP
-[streams.tcpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0
-burst_size = 10
-limit_by = "ip"
-```
-
-## Limiting Strategies
+## Strategies
### Per-IP Limiting (Default)
-
-Each client IP address gets its own token bucket:
-
+Each IP gets its own bucket:
```toml
-[streams.httpserver.rate_limit]
-enabled = true
limit_by = "ip"
requests_per_second = 10.0
-burst_size = 20
+# Client A: 10 req/sec
+# Client B: 10 req/sec
```
-**Use cases:**
-- Fair access for multiple users
-- Prevent single client from monopolizing resources
-- Public-facing endpoints
-
-**Example behavior:**
-- Client A: Can make 10 req/sec
-- Client B: Also can make 10 req/sec
-- Total: Up to 10 Γ number of clients
-
### Global Limiting
-
-All clients share a single token bucket:
-
+All clients share one bucket:
```toml
-[streams.httpserver.rate_limit]
-enabled = true
limit_by = "global"
requests_per_second = 50.0
-burst_size = 100
+# All clients combined: 50 req/sec
```
-**Use cases:**
-- Protect backend resources
-- Control total system load
-- Internal services with known clients
-
-**Example behavior:**
-- All clients combined: 50 req/sec max
-- One aggressive client can consume all tokens
-
## Connection Limits
-In addition to request rate limiting, you can limit concurrent connections:
-
-### Per-IP Connection Limit
-
```toml
-[streams.httpserver.rate_limit]
-max_connections_per_ip = 5 # Each IP can have max 5 connections
+max_connections_per_ip = 5 # Per IP
+max_total_connections = 100 # Total
```
-**Behavior:**
-- Prevents connection exhaustion attacks
-- Limits resource usage per client
-- Checked before rate limits
-
-### Total Connection Limit
-
-```toml
-[streams.httpserver.rate_limit]
-max_total_connections = 100 # Max 100 connections total
-```
-
-**Behavior:**
-- Protects server resources
-- Prevents memory exhaustion
-- Global limit across all IPs
-
## Response Behavior
-### HTTP Responses
-
-When rate limited, HTTP clients receive:
-
+### HTTP
+Returns JSON with configured status:
```json
{
"error": "Rate limit exceeded",
@@ -138,389 +67,59 @@ When rate limited, HTTP clients receive:
}
```
-With these headers:
-- Status code: 429 (default) or configured value
-- Content-Type: application/json
+### TCP
+Connections silently dropped.
-Configure custom responses:
-
-```toml
-[streams.httpserver.rate_limit]
-response_code = 503 # Service Unavailable
-response_message = "Server overloaded, please retry later"
-```
-
-### TCP Behavior
-
-TCP connections are **silently dropped** when rate limited:
-- No error message sent
-- Connection immediately closed
-- Prevents information leakage
-
-## Configuration Examples
+## Examples
### Light Protection
-
-For internal or trusted environments:
-
```toml
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 50.0
-burst_size = 100
-limit_by = "ip"
+rate_limit = {
+ enabled = true,
+ requests_per_second = 50.0,
+ burst_size = 100
+}
```
### Moderate Protection
-
-For semi-public endpoints:
-
```toml
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-burst_size = 30
-limit_by = "ip"
-max_connections_per_ip = 5
-max_total_connections = 200
+rate_limit = {
+ enabled = true,
+ requests_per_second = 10.0,
+ burst_size = 30,
+ max_connections_per_ip = 5
+}
```
### Strict Protection
-
-For public or sensitive endpoints:
-
```toml
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 2.0
-burst_size = 5
-limit_by = "ip"
-max_connections_per_ip = 2
-max_total_connections = 50
-response_code = 503
-response_message = "Service temporarily unavailable"
-```
-
-### Debug/Development
-
-Disable for testing:
-
-```toml
-[streams.httpserver.rate_limit]
-enabled = false
-```
-
-## Use Case Scenarios
-
-### Public Log Viewer
-
-Prevent abuse while allowing legitimate use:
-
-```toml
-[[streams]]
-name = "public-logs"
-
-[streams.httpserver]
-enabled = true
-port = 8080
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0 # 5 new connections per second
-burst_size = 10 # Allow short bursts
-limit_by = "ip"
-max_connections_per_ip = 3 # Max 3 streams per user
-max_total_connections = 100
-```
-
-### Internal Monitoring
-
-Protect against accidental overload:
-
-```toml
-[[streams]]
-name = "internal-metrics"
-
-[streams.httpserver]
-enabled = true
-port = 8081
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 100.0 # High limit for internal use
-burst_size = 200
-limit_by = "global" # Total system limit
-max_total_connections = 500
-```
-
-### High-Security Audit Logs
-
-Very restrictive access:
-
-```toml
-[[streams]]
-name = "audit"
-
-[streams.httpserver]
-enabled = true
-port = 8443
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 0.5 # 1 request every 2 seconds
-burst_size = 2
-limit_by = "ip"
-max_connections_per_ip = 1 # Single connection only
-max_total_connections = 10
-response_code = 403 # Forbidden (hide rate limit)
-response_message = "Access denied"
-```
-
-### Multi-Tenant Service
-
-Different limits per stream:
-
-```toml
-# Free tier
-[[streams]]
-name = "logs-free"
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 1.0
-burst_size = 5
-max_connections_per_ip = 1
-
-# Premium tier
-[[streams]]
-name = "logs-premium"
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 50.0
-burst_size = 100
-max_connections_per_ip = 10
-```
-
-## Monitoring Rate Limits
-
-### Status Endpoint
-
-Check rate limit statistics:
-
-```bash
-curl http://localhost:8080/status | jq '.server.features.rate_limit'
-```
-
-Response includes:
-```json
-{
- "enabled": true,
- "total_requests": 15234,
- "blocked_requests": 89,
- "active_ips": 12,
- "total_connections": 8,
- "config": {
- "requests_per_second": 10,
- "burst_size": 20,
- "limit_by": "ip"
- }
+rate_limit = {
+ enabled = true,
+ requests_per_second = 2.0,
+ burst_size = 5,
+ max_connections_per_ip = 2,
+ response_code = 503
}
```
-### Debug Logging
+## Monitoring
-Enable debug logs to see rate limit decisions:
+Check statistics:
+```bash
+curl http://localhost:8080/status | jq '.sinks[0].details.rate_limit'
+```
+
+## Testing
```bash
-logwisp --log-level debug
-```
-
-Look for messages:
-```
-Request rate limited ip=192.168.1.100
-Connection limit exceeded ip=192.168.1.100 connections=5 limit=5
-Created new IP limiter ip=192.168.1.100 total_ips=3
-```
-
-## Testing Rate Limits
-
-### Test Script
-
-```bash
-#!/bin/bash
-# Test rate limiting behavior
-
-URL="http://localhost:8080/stream"
-PARALLEL=10
-DURATION=10
-
-echo "Testing rate limits..."
-echo "URL: $URL"
-echo "Parallel connections: $PARALLEL"
-echo "Duration: ${DURATION}s"
-echo
-
-# Function to connect and count lines
-test_connection() {
- local id=$1
- local count=0
- local start=$(date +%s)
-
- while (( $(date +%s) - start < DURATION )); do
- if curl -s -N --max-time 1 "$URL" >/dev/null 2>&1; then
- ((count++))
- echo "[$id] Connected successfully (total: $count)"
- else
- echo "[$id] Rate limited!"
- fi
- sleep 0.1
- done
-}
-
-# Run parallel connections
-for i in $(seq 1 $PARALLEL); do
- test_connection $i &
-done
-
-wait
-echo "Test complete"
-```
-
-### Load Testing
-
-Using Apache Bench (ab):
-
-```bash
-# Test burst handling
-ab -n 100 -c 20 http://localhost:8080/status
-
-# Test sustained load
-ab -n 1000 -c 5 -r http://localhost:8080/status
-```
-
-Using curl:
-
-```bash
-# Test connection limit
-for i in {1..10}; do
- curl -N http://localhost:8080/stream &
+# Test rate limits
+for i in {1..20}; do
+ curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8080/status
done
```
-## Tuning Guidelines
+## Tuning
-### Setting requests_per_second
-
-Consider:
-- Expected legitimate traffic
-- Server capacity
-- Client retry behavior
-
-**Formula**: `requests_per_second = expected_clients Γ requests_per_client`
-
-### Setting burst_size
-
-General rule: `burst_size = 2-3 Γ requests_per_second`
-
-Examples:
-- `10 req/s β burst_size = 20-30`
-- `1 req/s β burst_size = 3-5`
-- `100 req/s β burst_size = 200-300`
-
-### Connection Limits
-
-Based on available memory:
-- Each HTTP connection: ~1-2MB
-- Each TCP connection: ~0.5-1MB
-
-**Formula**: `max_connections = available_memory / memory_per_connection`
-
-## Common Issues
-
-### "All requests blocked"
-
-Check if:
-- Rate limits too strict
-- Burst size too small
-- Using global limiting with many clients
-
-### "Memory growth"
-
-Possible causes:
-- No connection limits set
-- Slow clients holding connections
-- Too high burst_size
-
-Solutions:
-```toml
-max_connections_per_ip = 5
-max_total_connections = 100
-```
-
-### "Legitimate users blocked"
-
-Consider:
-- Increasing burst_size for short spikes
-- Using per-IP instead of global limiting
-- Different streams for different user tiers
-
-## Security Considerations
-
-### Information Disclosure
-
-Rate limit responses can reveal information:
-
-```toml
-# Default - informative
-response_code = 429
-response_message = "Rate limit exceeded"
-
-# Security-focused - generic
-response_code = 503
-response_message = "Service unavailable"
-
-# High security - misleading
-response_code = 403
-response_message = "Forbidden"
-```
-
-### DDoS Protection
-
-Rate limiting helps but isn't complete DDoS protection:
-- Use with firewall rules
-- Consider CDN/proxy rate limiting
-- Monitor for distributed attacks
-
-### Resource Exhaustion
-
-Protect against:
-- Connection exhaustion
-- Memory exhaustion
-- CPU exhaustion
-
-```toml
-# Comprehensive protection
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-burst_size = 20
-max_connections_per_ip = 5
-max_total_connections = 100
-limit_by = "ip"
-```
-
-## Best Practices
-
-1. **Start Conservative**: Begin with strict limits and relax as needed
-2. **Monitor Statistics**: Use `/status` endpoint to track behavior
-3. **Test Thoroughly**: Verify limits work as expected under load
-4. **Document Limits**: Make rate limits clear to users
-5. **Provide Retry Info**: Help clients implement proper retry logic
-6. **Different Tiers**: Consider different limits for different user types
-7. **Regular Review**: Adjust limits based on usage patterns
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Complete configuration reference
-- [Security Best Practices](security.md) - Security hardening
-- [Performance Tuning](performance.md) - Optimization guidelines
-- [Troubleshooting](troubleshooting.md) - Common issues
\ No newline at end of file
+- **requests_per_second**: Expected load
+- **burst_size**: 2-3Γ requests_per_second
+- **Connection limits**: Based on memory
\ No newline at end of file
diff --git a/doc/router.md b/doc/router.md
index 8b49e8d..7da3937 100644
--- a/doc/router.md
+++ b/doc/router.md
@@ -1,520 +1,158 @@
# Router Mode Guide
-Router mode allows multiple LogWisp streams to share HTTP ports through path-based routing, simplifying deployment and access control.
+Router mode enables multiple pipelines to share HTTP ports through path-based routing.
## Overview
-In standard mode, each stream requires its own port:
-- Stream 1: `http://localhost:8080/stream`
-- Stream 2: `http://localhost:8081/stream`
-- Stream 3: `http://localhost:8082/stream`
+**Standard mode**: Each pipeline needs its own port
+- Pipeline 1: `http://localhost:8080/stream`
+- Pipeline 2: `http://localhost:8081/stream`
-In router mode, streams share ports via paths:
-- Stream 1: `http://localhost:8080/app/stream`
-- Stream 2: `http://localhost:8080/database/stream`
-- Stream 3: `http://localhost:8080/system/stream`
+**Router mode**: Pipelines share ports via paths
+- Pipeline 1: `http://localhost:8080/app/stream`
+- Pipeline 2: `http://localhost:8080/database/stream`
- Global status: `http://localhost:8080/status`
## Enabling Router Mode
-Start LogWisp with the `--router` flag:
-
```bash
-logwisp --router --config /etc/logwisp/multi-stream.toml
+logwisp --router --config /etc/logwisp/multi-pipeline.toml
```
## Configuration
-### Basic Router Configuration
-
```toml
-# All streams can use the same port in router mode
-[[streams]]
+# All pipelines can use the same port
+[[pipelines]]
name = "app"
-[streams.monitor]
-targets = [{ path = "/var/log/app", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080 # Same port OK
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 } # Same port OK
-[[streams]]
+[[pipelines]]
name = "database"
-[streams.monitor]
-targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080 # Shared port
-
-[[streams]]
-name = "nginx"
-[streams.monitor]
-targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080 # Shared port
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/postgresql", pattern = "*.log" }
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 } # Shared port
```
-### Path Structure
+## Path Structure
-In router mode, paths are automatically prefixed with the stream name:
+Paths are prefixed with pipeline name:
-| Stream Name | Configuration Path | Router Mode Path |
-|------------|-------------------|------------------|
+| Pipeline | Config Path | Router Path |
+|----------|-------------|-------------|
| `app` | `/stream` | `/app/stream` |
| `app` | `/status` | `/app/status` |
| `database` | `/stream` | `/database/stream` |
-| `database` | `/status` | `/database/status` |
### Custom Paths
-You can customize the paths in each stream:
-
```toml
-[[streams]]
-name = "api"
-[streams.httpserver]
-stream_path = "/logs" # Becomes /api/logs
-status_path = "/health" # Becomes /api/health
-```
-
-## URL Endpoints
-
-### Stream Endpoints
-
-Access individual streams:
-
-```bash
-# SSE stream for 'app' logs
-curl -N http://localhost:8080/app/stream
-
-# Status for 'database' stream
-curl http://localhost:8080/database/status
-
-# Custom path example
-curl -N http://localhost:8080/api/logs
-```
-
-### Global Status
-
-Router mode provides a global status endpoint:
-
-```bash
-curl http://localhost:8080/status | jq .
-```
-
-Returns aggregated information:
-```json
-{
- "service": "LogWisp Router",
- "version": "1.0.0",
- "port": 8080,
- "total_streams": 3,
- "streams": {
- "app": { /* stream stats */ },
- "database": { /* stream stats */ },
- "nginx": { /* stream stats */ }
- },
- "router": {
- "uptime_seconds": 3600,
- "total_requests": 15234,
- "routed_requests": 15220,
- "failed_requests": 14
- }
+[[pipelines.sinks]]
+type = "http"
+options = {
+ stream_path = "/logs", # Becomes /app/logs
+ status_path = "/health" # Becomes /app/health
}
```
-## Port Sharing
+## Endpoints
-### How It Works
+### Pipeline Endpoints
+```bash
+# SSE stream
+curl -N http://localhost:8080/app/stream
-1. Router server listens on configured ports
-2. Examines request path to determine target stream
-3. Routes request to appropriate stream handler
-4. Stream handles request as if standalone
+# Pipeline status
+curl http://localhost:8080/database/status
+```
-### Port Assignment Rules
+### Global Status
+```bash
+curl http://localhost:8080/status
+```
-In router mode:
-- Multiple streams can use the same port
-- Router detects and consolidates shared ports
-- Each unique port gets one router server
-- TCP servers remain independent (no routing)
-
-Example with multiple ports:
-
-```toml
-# Streams 1-3 share port 8080
-[[streams]]
-name = "app"
-[streams.httpserver]
-port = 8080
-
-[[streams]]
-name = "db"
-[streams.httpserver]
-port = 8080
-
-[[streams]]
-name = "web"
-[streams.httpserver]
-port = 8080
-
-# Stream 4 uses different port
-[[streams]]
-name = "admin"
-[streams.httpserver]
-port = 9090
-
-# Result: 2 router servers (8080 and 9090)
+Returns:
+```json
+{
+ "service": "LogWisp Router",
+ "pipelines": {
+ "app": { /* stats */ },
+ "database": { /* stats */ }
+ },
+ "total_pipelines": 2
+}
```
## Use Cases
-### Microservices Architecture
-
-Route logs from different services:
-
+### Microservices
```toml
-[[streams]]
+[[pipelines]]
name = "frontend"
-[streams.monitor]
-targets = [{ path = "/var/log/frontend", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/frontend", pattern = "*.log" }
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
-[[streams]]
+[[pipelines]]
name = "backend"
-[streams.monitor]
-targets = [{ path = "/var/log/backend", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/backend", pattern = "*.log" }
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
-[[streams]]
-name = "worker"
-[streams.monitor]
-targets = [{ path = "/var/log/worker", pattern = "*.log" }]
-[streams.httpserver]
-enabled = true
-port = 8080
+# Access:
+# http://localhost:8080/frontend/stream
+# http://localhost:8080/backend/stream
```
-Access via:
-- Frontend logs: `http://localhost:8080/frontend/stream`
-- Backend logs: `http://localhost:8080/backend/stream`
-- Worker logs: `http://localhost:8080/worker/stream`
-
-### Environment-Based Routing
-
-Different log levels per environment:
-
+### Environment-Based
```toml
-[[streams]]
+[[pipelines]]
name = "prod"
-[streams.monitor]
-targets = [{ path = "/logs/prod", pattern = "*.log" }]
-[[streams.filters]]
+[[pipelines.filters]]
type = "include"
patterns = ["ERROR", "WARN"]
-[streams.httpserver]
-port = 8080
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
-[[streams]]
-name = "staging"
-[streams.monitor]
-targets = [{ path = "/logs/staging", pattern = "*.log" }]
-[[streams.filters]]
-type = "include"
-patterns = ["ERROR", "WARN", "INFO"]
-[streams.httpserver]
-port = 8080
-
-[[streams]]
+[[pipelines]]
name = "dev"
-[streams.monitor]
-targets = [{ path = "/logs/dev", pattern = "*.log" }]
# No filters - all logs
-[streams.httpserver]
-port = 8080
-```
-
-### Department Access
-
-Separate streams for different teams:
-
-```toml
-[[streams]]
-name = "engineering"
-[streams.monitor]
-targets = [{ path = "/logs/apps", pattern = "*.log" }]
-[streams.httpserver]
-port = 8080
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 50.0
-
-[[streams]]
-name = "security"
-[streams.monitor]
-targets = [{ path = "/logs/audit", pattern = "*.log" }]
-[streams.httpserver]
-port = 8080
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0
-max_connections_per_ip = 1
-
-[[streams]]
-name = "support"
-[streams.monitor]
-targets = [{ path = "/logs/customer", pattern = "*.log" }]
-[[streams.filters]]
-type = "exclude"
-patterns = ["password", "token", "secret"]
-[streams.httpserver]
-port = 8080
-```
-
-## Advanced Features
-
-### Mixed Mode Deployment
-
-Combine router and standalone modes:
-
-```toml
-# Public streams via router
-[[streams]]
-name = "public-api"
-[streams.httpserver]
-enabled = true
-port = 8080 # Router mode
-
-[[streams]]
-name = "public-web"
-[streams.httpserver]
-enabled = true
-port = 8080 # Router mode
-
-# Internal stream standalone
-[[streams]]
-name = "internal"
-[streams.httpserver]
-enabled = true
-port = 9999 # Different port, standalone
-
-# High-performance TCP
-[[streams]]
-name = "metrics"
-[streams.tcpserver]
-enabled = true
-port = 9090 # TCP not affected by router
-```
-
-### Load Balancer Integration
-
-Router mode works well with load balancers:
-
-```nginx
-# Nginx configuration
-upstream logwisp {
- server logwisp1:8080;
- server logwisp2:8080;
- server logwisp3:8080;
-}
-
-location /logs/ {
- proxy_pass http://logwisp/;
- proxy_http_version 1.1;
- proxy_set_header Connection "";
- proxy_buffering off;
-}
-```
-
-Access becomes:
-- `https://example.com/logs/app/stream`
-- `https://example.com/logs/database/stream`
-- `https://example.com/logs/status`
-
-### Path-Based Access Control
-
-Use reverse proxy for authentication:
-
-```nginx
-# Require auth for security logs
-location /logs/security/ {
- auth_basic "Security Logs";
- auth_basic_user_file /etc/nginx/security.htpasswd;
- proxy_pass http://localhost:8080/security/;
-}
-
-# Public access for status
-location /logs/app/ {
- proxy_pass http://localhost:8080/app/;
-}
+[[pipelines.sinks]]
+type = "http"
+options = { port = 8080 }
```
## Limitations
-### Router Mode Limitations
+1. **HTTP Only**: Router mode only works for HTTP/SSE
+2. **No TCP Routing**: TCP remains on separate ports
+3. **Path Conflicts**: Pipeline names must be unique
-1. **HTTP Only**: Router mode only works for HTTP/SSE streams
-2. **No TCP Routing**: TCP streams remain on separate ports
-3. **Path Conflicts**: Stream names must be unique
-4. **Same Config**: All streams on a port share SSL/auth settings
+## Load Balancer Integration
-### When Not to Use Router Mode
-
-- High-performance scenarios (use TCP)
-- Streams need different SSL certificates
-- Complex authentication per stream
-- Network isolation requirements
-
-## Troubleshooting
-
-### "Path not found"
-
-Check available routes:
-```bash
-curl http://localhost:8080/invalid-path
-```
-
-Response shows available routes:
-```json
-{
- "error": "Not Found",
- "requested_path": "/invalid-path",
- "available_routes": [
- "/status (global status)",
- "/app/stream (stream: app)",
- "/app/status (status: app)",
- "/database/stream (stream: database)",
- "/database/status (status: database)"
- ]
+```nginx
+upstream logwisp {
+ server logwisp1:8080;
+ server logwisp2:8080;
}
-```
-### "Port conflict"
-
-If you see port conflicts:
-1. Ensure `--router` flag is used
-2. Check all streams have `httpserver.enabled = true`
-3. Verify no other services use the port
-
-### Debug Routing
-
-Enable debug logging:
-```bash
-logwisp --router --log-level debug
-```
-
-Look for routing decisions:
-```
-Router request method=GET path=/app/stream remote_addr=127.0.0.1:54321
-Routing request to stream stream=app original_path=/app/stream remaining_path=/stream
-```
-
-### Performance Impact
-
-Router mode adds minimal overhead:
-- ~100-200ns per request for path matching
-- Negligible memory overhead
-- No impact on streaming performance
-
-## Best Practices
-
-### Naming Conventions
-
-Use clear, consistent stream names:
-```toml
-# Good: Clear purpose
-name = "frontend-prod"
-name = "backend-staging"
-name = "worker-payments"
-
-# Bad: Ambiguous
-name = "logs1"
-name = "stream2"
-name = "test"
-```
-
-### Path Organization
-
-Group related streams:
-```
-/prod/frontend/stream
-/prod/backend/stream
-/staging/frontend/stream
-/staging/backend/stream
-```
-
-### Documentation
-
-Document your routing structure:
-```toml
-# Stream for production API logs
-# Access: https://logs.example.com/api-prod/stream
-[[streams]]
-name = "api-prod"
-```
-
-### Monitoring
-
-Use global status for overview:
-```bash
-# Monitor all streams
-watch -n 5 'curl -s localhost:8080/status | jq .streams'
-
-# Check specific stream
-curl -s localhost:8080/status | jq '.streams.app'
-```
-
-## Migration Guide
-
-### From Standalone to Router
-
-1. **Update configuration** - ensure consistent ports:
- ```toml
- # Change from different ports
- [streams.httpserver]
- port = 8080 # Was 8081, 8082, etc.
- ```
-
-2. **Start with router flag**:
- ```bash
- logwisp --router --config existing.toml
- ```
-
-3. **Update client URLs**:
- ```bash
- # Old: http://localhost:8081/stream
- # New: http://localhost:8080/streamname/stream
- ```
-
-4. **Update monitoring**:
- ```bash
- # Global status now available
- curl http://localhost:8080/status
- ```
-
-### Gradual Migration
-
-Run both modes during transition:
-```bash
-# Week 1: Run standalone (current)
-logwisp --config prod.toml
-
-# Week 2: Run both
-logwisp --config prod.toml & # Standalone
-logwisp --router --config prod-router.toml & # Router
-
-# Week 3: Router only
-logwisp --router --config prod.toml
-```
-
-## See Also
-
-- [Configuration Guide](configuration.md) - Stream configuration
-- [HTTP Streaming](api.md#http-sse) - SSE protocol details
-- [Load Balancing](integrations.md#load-balancers) - Integration patterns
-- [Security Best Practices](security.md) - Securing router deployments
\ No newline at end of file
+location /logs/ {
+ proxy_pass http://logwisp/;
+ proxy_buffering off;
+}
+```
\ No newline at end of file
diff --git a/doc/status.md b/doc/status.md
new file mode 100644
index 0000000..95b44ba
--- /dev/null
+++ b/doc/status.md
@@ -0,0 +1,186 @@
+# Status Monitoring
+
+LogWisp provides comprehensive monitoring through status endpoints and operational logs.
+
+## Status Endpoints
+
+### Pipeline Status
+
+```bash
+# Standalone mode
+curl http://localhost:8080/status
+
+# Router mode
+curl http://localhost:8080/pipelinename/status
+```
+
+Example response:
+```json
+{
+ "service": "LogWisp",
+ "version": "1.0.0",
+ "server": {
+ "type": "http",
+ "port": 8080,
+ "active_clients": 5,
+ "buffer_size": 1000,
+ "uptime_seconds": 3600
+ },
+ "sources": [{
+ "type": "directory",
+ "total_entries": 152341,
+ "dropped_entries": 12,
+ "active_watchers": 3
+ }],
+ "filters": {
+ "filter_count": 2,
+ "total_processed": 152341,
+ "total_passed": 48234
+ },
+ "sinks": [{
+ "type": "http",
+ "total_processed": 48234,
+ "active_connections": 5
+ }]
+}
+```
+
+## Key Metrics
+
+### Source Metrics
+| Metric | Description | Healthy Range |
+|--------|-------------|---------------|
+| `active_watchers` | Files being watched | 1-1000 |
+| `total_entries` | Entries processed | Increasing |
+| `dropped_entries` | Buffer overflows | < 1% of total |
+
+### Sink Metrics
+| Metric | Description | Warning Signs |
+|--------|-------------|---------------|
+| `active_connections` | Current clients | Near limit |
+| `total_processed` | Entries sent | Should match filter output |
+
+### Filter Metrics
+| Metric | Description | Notes |
+|--------|-------------|-------|
+| `total_processed` | Entries checked | All entries |
+| `total_passed` | Passed filters | Check if too low/high |
+
+## Operational Logging
+
+### Log Levels
+```toml
+[logging]
+level = "info" # debug, info, warn, error
+```
+
+### Important Messages
+
+**Startup**:
+```
+LogWisp starting version=1.0.0
+Pipeline created successfully pipeline=app
+HTTP server started port=8080
+```
+
+**Connections**:
+```
+HTTP client connected remote_addr=192.168.1.100 active_clients=6
+TCP connection opened active_connections=3
+```
+
+**Errors**:
+```
+Failed to open file path=/var/log/app.log error=permission denied
+Request rate limited ip=192.168.1.100
+```
+
+## Health Checks
+
+### Basic Check
+```bash
+#!/bin/bash
+if curl -s -f http://localhost:8080/status > /dev/null; then
+ echo "Healthy"
+else
+ echo "Unhealthy"
+ exit 1
+fi
+```
+
+### Advanced Check
+```bash
+#!/bin/bash
+STATUS=$(curl -s http://localhost:8080/status)
+DROPPED=$(echo "$STATUS" | jq '.sources[0].dropped_entries')
+TOTAL=$(echo "$STATUS" | jq '.sources[0].total_entries')
+
+if [ $((DROPPED * 100 / TOTAL)) -gt 5 ]; then
+ echo "High drop rate"
+ exit 1
+fi
+```
+
+### Docker
+```dockerfile
+HEALTHCHECK --interval=30s --timeout=3s \
+ CMD curl -f http://localhost:8080/status || exit 1
+```
+
+## Integration
+
+### Prometheus Export
+```bash
+#!/bin/bash
+STATUS=$(curl -s http://localhost:8080/status)
+
+cat << EOF
+# HELP logwisp_active_clients Active streaming clients
+# TYPE logwisp_active_clients gauge
+logwisp_active_clients $(echo "$STATUS" | jq '.server.active_clients')
+
+# HELP logwisp_total_entries Total log entries
+# TYPE logwisp_total_entries counter
+logwisp_total_entries $(echo "$STATUS" | jq '.sources[0].total_entries')
+EOF
+```
+
+### Alerts
+
+| Alert | Condition | Severity |
+|-------|-----------|----------|
+| Service Down | Status fails | Critical |
+| High Drops | >10% dropped | Warning |
+| No Activity | 0 entries/min | Warning |
+| Rate Limited | >20% blocked | Warning |
+
+## Performance Monitoring
+
+### CPU Usage
+```bash
+top -p $(pgrep logwisp)
+```
+
+### Memory Usage
+```bash
+ps aux | grep logwisp
+```
+
+### Connections
+```bash
+ss -tan | grep :8080 | wc -l
+```
+
+## Troubleshooting
+
+Enable debug logging:
+```bash
+logwisp --log-level debug --log-output stderr
+```
+
+Check specific components:
+```bash
+curl -s http://localhost:8080/status | jq '.sources'
+curl -s http://localhost:8080/status | jq '.filters'
+curl -s http://localhost:8080/status | jq '.sinks'
+```
\ No newline at end of file
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
deleted file mode 100644
index 873b7e3..0000000
--- a/doc/troubleshooting.md
+++ /dev/null
@@ -1,537 +0,0 @@
-# Troubleshooting Guide
-
-This guide helps diagnose and resolve common issues with LogWisp.
-
-## Diagnostic Tools
-
-### Enable Debug Logging
-
-The first step in troubleshooting is enabling debug logs:
-
-```bash
-# Via command line
-logwisp --log-level debug --log-output stderr
-
-# Via environment
-export LOGWISP_LOGGING_LEVEL=debug
-logwisp
-
-# Via config
-[logging]
-level = "debug"
-output = "stderr"
-```
-
-### Check Status Endpoint
-
-Verify LogWisp is running and processing:
-
-```bash
-# Basic check
-curl http://localhost:8080/status
-
-# Pretty print
-curl -s http://localhost:8080/status | jq .
-
-# Check specific metrics
-curl -s http://localhost:8080/status | jq '.monitor'
-```
-
-### Test Log Streaming
-
-Verify streams are working:
-
-```bash
-# Test SSE stream (should show heartbeats if enabled)
-curl -N http://localhost:8080/stream
-
-# Test with timeout
-timeout 5 curl -N http://localhost:8080/stream
-
-# Test TCP stream
-nc localhost 9090
-```
-
-## Common Issues
-
-### No Logs Appearing
-
-**Symptoms:**
-- Stream connects but no log entries appear
-- Status shows `total_entries: 0`
-
-**Diagnosis:**
-
-1. Check monitor configuration:
- ```bash
- curl -s http://localhost:8080/status | jq '.monitor'
- ```
-
-2. Verify file paths exist:
- ```bash
- # Check your configured paths
- ls -la /var/log/myapp/
- ```
-
-3. Check file permissions:
- ```bash
- # LogWisp user must have read access
- sudo -u logwisp ls /var/log/myapp/
- ```
-
-4. Verify files match pattern:
- ```bash
- # If pattern is "*.log"
- ls /var/log/myapp/*.log
- ```
-
-5. Check if files are being updated:
- ```bash
- # Should show recent timestamps
- ls -la /var/log/myapp/*.log
- tail -f /var/log/myapp/app.log
- ```
-
-**Solutions:**
-
-- Fix file permissions:
- ```bash
- sudo chmod 644 /var/log/myapp/*.log
- sudo usermod -a -G adm logwisp # Add to log group
- ```
-
-- Correct path configuration:
- ```toml
- targets = [
- { path = "/correct/path/to/logs", pattern = "*.log" }
- ]
- ```
-
-- Use absolute paths:
- ```toml
- # Bad: Relative path
- targets = [{ path = "./logs", pattern = "*.log" }]
-
- # Good: Absolute path
- targets = [{ path = "/var/log/app", pattern = "*.log" }]
- ```
-
-### High CPU Usage
-
-**Symptoms:**
-- LogWisp process using excessive CPU
-- System slowdown
-
-**Diagnosis:**
-
-1. Check process CPU:
- ```bash
- top -p $(pgrep logwisp)
- ```
-
-2. Review check intervals:
- ```bash
- grep check_interval /etc/logwisp/logwisp.toml
- ```
-
-3. Count active watchers:
- ```bash
- curl -s http://localhost:8080/status | jq '.monitor.active_watchers'
- ```
-
-4. Check filter complexity:
- ```bash
- curl -s http://localhost:8080/status | jq '.filters'
- ```
-
-**Solutions:**
-
-- Increase check interval:
- ```toml
- [streams.monitor]
- check_interval_ms = 1000 # Was 50ms
- ```
-
-- Reduce watched files:
- ```toml
- # Instead of watching entire directory
- targets = [
- { path = "/var/log/specific-app.log", is_file = true }
- ]
- ```
-
-- Simplify filter patterns:
- ```toml
- # Complex regex (slow)
- patterns = ["^\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]\\s+\\[(ERROR|WARN)\\]"]
-
- # Simple patterns (fast)
- patterns = ["ERROR", "WARN"]
- ```
-
-### Memory Growth
-
-**Symptoms:**
-- Increasing memory usage over time
-- Eventually runs out of memory
-
-**Diagnosis:**
-
-1. Monitor memory usage:
- ```bash
- watch -n 10 'ps aux | grep logwisp'
- ```
-
-2. Check connection count:
- ```bash
- curl -s http://localhost:8080/status | jq '.server.active_clients'
- ```
-
-3. Check for dropped entries:
- ```bash
- curl -s http://localhost:8080/status | jq '.monitor.dropped_entries'
- ```
-
-**Solutions:**
-
-- Limit connections:
- ```toml
- [streams.httpserver.rate_limit]
- enabled = true
- max_connections_per_ip = 5
- max_total_connections = 100
- ```
-
-- Reduce buffer sizes:
- ```toml
- [streams.httpserver]
- buffer_size = 500 # Was 5000
- ```
-
-- Enable rate limiting:
- ```toml
- [streams.httpserver.rate_limit]
- enabled = true
- requests_per_second = 10.0
- ```
-
-### Connection Refused
-
-**Symptoms:**
-- Cannot connect to LogWisp
-- `curl: (7) Failed to connect`
-
-**Diagnosis:**
-
-1. Check if LogWisp is running:
- ```bash
- ps aux | grep logwisp
- systemctl status logwisp
- ```
-
-2. Verify listening ports:
- ```bash
- sudo netstat -tlnp | grep logwisp
- # or
- sudo ss -tlnp | grep logwisp
- ```
-
-3. Check firewall:
- ```bash
- sudo iptables -L -n | grep 8080
- sudo ufw status
- ```
-
-**Solutions:**
-
-- Start the service:
- ```bash
- sudo systemctl start logwisp
- ```
-
-- Fix port configuration:
- ```toml
- [streams.httpserver]
- enabled = true # Must be true
- port = 8080 # Correct port
- ```
-
-- Open firewall:
- ```bash
- sudo ufw allow 8080/tcp
- ```
-
-### Rate Limit Errors
-
-**Symptoms:**
-- HTTP 429 responses
-- "Rate limit exceeded" errors
-
-**Diagnosis:**
-
-1. Check rate limit stats:
- ```bash
- curl -s http://localhost:8080/status | jq '.features.rate_limit'
- ```
-
-2. Test rate limits:
- ```bash
- # Rapid requests
- for i in {1..20}; do curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8080/status; done
- ```
-
-**Solutions:**
-
-- Increase rate limits:
- ```toml
- [streams.httpserver.rate_limit]
- requests_per_second = 50.0 # Was 10.0
- burst_size = 100 # Was 20
- ```
-
-- Use per-IP limiting:
- ```toml
- limit_by = "ip" # Instead of "global"
- ```
-
-- Disable for internal use:
- ```toml
- enabled = false
- ```
-
-### Filter Not Working
-
-**Symptoms:**
-- Unwanted logs still appearing
-- Wanted logs being filtered out
-
-**Diagnosis:**
-
-1. Check filter configuration:
- ```bash
- curl -s http://localhost:8080/status | jq '.filters'
- ```
-
-2. Test patterns:
- ```bash
- # Test regex pattern
- echo "ERROR: test message" | grep -E "your-pattern"
- ```
-
-3. Enable debug logging to see filter decisions:
- ```bash
- logwisp --log-level debug 2>&1 | grep filter
- ```
-
-**Solutions:**
-
-- Fix pattern syntax:
- ```toml
- # Word boundaries
- patterns = ["\\bERROR\\b"] # Not "ERROR" which matches "TERROR"
-
- # Case insensitive
- patterns = ["(?i)error"]
- ```
-
-- Check filter order:
- ```toml
- # Include filters run first
- [[streams.filters]]
- type = "include"
- patterns = ["ERROR", "WARN"]
-
- # Then exclude filters
- [[streams.filters]]
- type = "exclude"
- patterns = ["IGNORE_THIS"]
- ```
-
-- Use correct logic:
- ```toml
- logic = "or" # Match ANY pattern
- # not
- logic = "and" # Match ALL patterns
- ```
-
-### Logs Dropping
-
-**Symptoms:**
-- `dropped_entries` counter increasing
-- Missing log entries in stream
-
-**Diagnosis:**
-
-1. Check drop statistics:
- ```bash
- curl -s http://localhost:8080/status | jq '{
- dropped: .monitor.dropped_entries,
- total: .monitor.total_entries,
- percent: (.monitor.dropped_entries / .monitor.total_entries * 100)
- }'
- ```
-
-2. Monitor drop rate:
- ```bash
- watch -n 5 'curl -s http://localhost:8080/status | jq .monitor.dropped_entries'
- ```
-
-**Solutions:**
-
-- Increase buffer sizes:
- ```toml
- [streams.httpserver]
- buffer_size = 5000 # Was 1000
- ```
-
-- Add flow control:
- ```toml
- [streams.monitor]
- check_interval_ms = 500 # Slow down reading
- ```
-
-- Reduce clients:
- ```toml
- [streams.httpserver.rate_limit]
- max_total_connections = 50
- ```
-
-## Performance Issues
-
-### Slow Response Times
-
-**Diagnosis:**
-```bash
-# Measure response time
-time curl -s http://localhost:8080/status > /dev/null
-
-# Check system load
-uptime
-top
-```
-
-**Solutions:**
-- Reduce concurrent operations
-- Increase system resources
-- Use TCP instead of HTTP for high volume
-
-### Network Bandwidth
-
-**Diagnosis:**
-```bash
-# Monitor network usage
-iftop -i eth0 -f "port 8080"
-
-# Check connection count
-ss -tan | grep :8080 | wc -l
-```
-
-**Solutions:**
-- Enable compression (future feature)
-- Filter more aggressively
-- Use TCP for local connections
-
-## Debug Commands
-
-### System Information
-
-```bash
-# LogWisp version
-logwisp --version
-
-# System resources
-free -h
-df -h
-ulimit -a
-
-# Network state
-ss -tlnp
-netstat -anp | grep logwisp
-```
-
-### Process Inspection
-
-```bash
-# Process details
-ps aux | grep logwisp
-
-# Open files
-lsof -p $(pgrep logwisp)
-
-# System calls (Linux)
-strace -p $(pgrep logwisp) -e trace=open,read,write
-
-# File system activity
-inotifywait -m /var/log/myapp/
-```
-
-### Configuration Validation
-
-```bash
-# Test configuration
-logwisp --config test.toml --log-level debug --log-output stderr
-
-# Check file syntax
-cat /etc/logwisp/logwisp.toml | grep -E "^\s*\["
-
-# Validate TOML
-python3 -m pip install toml
-python3 -c "import toml; toml.load('/etc/logwisp/logwisp.toml'); print('Valid')"
-```
-
-## Getting Help
-
-### Collect Diagnostic Information
-
-Create a diagnostic bundle:
-
-```bash
-#!/bin/bash
-# diagnostic.sh
-
-DIAG_DIR="logwisp-diag-$(date +%Y%m%d-%H%M%S)"
-mkdir -p "$DIAG_DIR"
-
-# Version
-logwisp --version > "$DIAG_DIR/version.txt" 2>&1
-
-# Configuration (sanitized)
-grep -v "password\|secret\|token" /etc/logwisp/logwisp.toml > "$DIAG_DIR/config.toml"
-
-# Status
-curl -s http://localhost:8080/status > "$DIAG_DIR/status.json"
-
-# System info
-uname -a > "$DIAG_DIR/system.txt"
-free -h >> "$DIAG_DIR/system.txt"
-df -h >> "$DIAG_DIR/system.txt"
-
-# Process info
-ps aux | grep logwisp > "$DIAG_DIR/process.txt"
-lsof -p $(pgrep logwisp) > "$DIAG_DIR/files.txt" 2>&1
-
-# Recent logs
-journalctl -u logwisp -n 1000 > "$DIAG_DIR/logs.txt" 2>&1
-
-# Create archive
-tar -czf "$DIAG_DIR.tar.gz" "$DIAG_DIR"
-rm -rf "$DIAG_DIR"
-
-echo "Diagnostic bundle created: $DIAG_DIR.tar.gz"
-```
-
-### Report Issues
-
-When reporting issues, include:
-1. LogWisp version
-2. Configuration (sanitized)
-3. Error messages
-4. Steps to reproduce
-5. Diagnostic bundle
-
-## See Also
-
-- [Monitoring Guide](monitoring.md) - Status and metrics
-- [Performance Tuning](performance.md) - Optimization
-- [Configuration Guide](configuration.md) - Settings reference
-- [FAQ](faq.md) - Frequently asked questions
\ No newline at end of file
diff --git a/src/cmd/logwisp/bootstrap.go b/src/cmd/logwisp/bootstrap.go
index 582b066..c696444 100644
--- a/src/cmd/logwisp/bootstrap.go
+++ b/src/cmd/logwisp/bootstrap.go
@@ -18,7 +18,7 @@ import (
// bootstrapService creates and initializes the log transport service
func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service, *service.HTTPRouter, error) {
- // Create log transport service
+ // Create service
svc := service.New(ctx, logger)
// Create HTTP router if requested
@@ -28,75 +28,46 @@ func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service
logger.Info("msg", "HTTP router mode enabled")
}
- // Initialize streams
+ // Initialize pipelines
successCount := 0
- for _, streamCfg := range cfg.Streams {
- logger.Info("msg", "Initializing transport", "transport", streamCfg.Name)
+ for _, pipelineCfg := range cfg.Pipelines {
+ logger.Info("msg", "Initializing pipeline", "pipeline", pipelineCfg.Name)
- // Handle router mode configuration
- if *useRouter && streamCfg.HTTPServer != nil && streamCfg.HTTPServer.Enabled {
- if err := initializeRouterStream(svc, router, streamCfg); err != nil {
- logger.Error("msg", "Failed to initialize router stream",
- "transport", streamCfg.Name,
- "error", err)
- continue
- }
- } else {
- // Standard standalone mode
- if err := svc.CreateStream(streamCfg); err != nil {
- logger.Error("msg", "Failed to create transport",
- "transport", streamCfg.Name,
- "error", err)
- continue
+ // Create the pipeline
+ if err := svc.NewPipeline(pipelineCfg); err != nil {
+ logger.Error("msg", "Failed to create pipeline",
+ "pipeline", pipelineCfg.Name,
+ "error", err)
+ continue
+ }
+
+ // If using router mode, register HTTP sinks
+ if *useRouter {
+ pipeline, err := svc.GetPipeline(pipelineCfg.Name)
+ if err == nil && len(pipeline.HTTPSinks) > 0 {
+ if err := router.RegisterPipeline(pipeline); err != nil {
+ logger.Error("msg", "Failed to register pipeline with router",
+ "pipeline", pipelineCfg.Name,
+ "error", err)
+ }
}
}
successCount++
- displayStreamEndpoints(streamCfg, *useRouter)
+ displayPipelineEndpoints(pipelineCfg, *useRouter)
}
if successCount == 0 {
- return nil, nil, fmt.Errorf("no streams successfully started (attempted %d)", len(cfg.Streams))
+ return nil, nil, fmt.Errorf("no pipelines successfully started (attempted %d)", len(cfg.Pipelines))
}
logger.Info("msg", "LogWisp started",
"version", version.Short(),
- "transports", successCount)
+ "pipelines", successCount)
return svc, router, nil
}
-// initializeRouterStream sets up a stream for router mode
-func initializeRouterStream(svc *service.Service, router *service.HTTPRouter, streamCfg config.StreamConfig) error {
- // Temporarily disable standalone server startup
- originalEnabled := streamCfg.HTTPServer.Enabled
- streamCfg.HTTPServer.Enabled = false
-
- if err := svc.CreateStream(streamCfg); err != nil {
- return err
- }
-
- // Get the created transport and configure for router mode
- stream, err := svc.GetStream(streamCfg.Name)
- if err != nil {
- return err
- }
-
- if stream.HTTPServer != nil {
- stream.HTTPServer.SetRouterMode()
- // Restore enabled state
- stream.Config.HTTPServer.Enabled = originalEnabled
-
- if err := router.RegisterStream(stream); err != nil {
- return err
- }
-
- logger.Info("msg", "Stream registered with router", "stream", streamCfg.Name)
- }
-
- return nil
-}
-
// initializeLogger sets up the logger based on configuration and CLI flags
func initializeLogger(cfg *config.Config) error {
logger = log.NewLogger()
diff --git a/src/cmd/logwisp/flags.go b/src/cmd/logwisp/flags.go
index 6fcee22..3fc9315 100644
--- a/src/cmd/logwisp/flags.go
+++ b/src/cmd/logwisp/flags.go
@@ -31,7 +31,7 @@ func init() {
}
func customUsage() {
- fmt.Fprintf(os.Stderr, "LogWisp - Multi-Stream Log Monitoring Service\n\n")
+ fmt.Fprintf(os.Stderr, "LogWisp - Multi-Pipeline Log Processing Service\n\n")
fmt.Fprintf(os.Stderr, "Usage: %s [options]\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Options:\n")
@@ -63,8 +63,8 @@ func customUsage() {
fmt.Fprintf(os.Stderr, " # Run with custom config and override log level\n")
fmt.Fprintf(os.Stderr, " %s --config /etc/logwisp.toml --log-level warn\n\n", os.Args[0])
- fmt.Fprintf(os.Stderr, " # Run in router mode with multiple streams\n")
- fmt.Fprintf(os.Stderr, " %s --router --config /etc/logwisp/multi-stream.toml\n\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, " # Run in router mode with multiple pipelines\n")
+ fmt.Fprintf(os.Stderr, " %s --router --config /etc/logwisp/multi-pipeline.toml\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Environment Variables:\n")
fmt.Fprintf(os.Stderr, " LOGWISP_CONFIG_FILE Config file path\n")
diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go
index c686bc1..c53cfc9 100644
--- a/src/cmd/logwisp/main.go
+++ b/src/cmd/logwisp/main.go
@@ -81,7 +81,7 @@ func main() {
}
// Start status reporter if enabled
- if shouldEnableStatusReporter() {
+ if enableStatusReporter() {
go statusReporter(svc)
}
@@ -123,7 +123,7 @@ func shutdownLogger() {
}
}
-func shouldEnableStatusReporter() bool {
+func enableStatusReporter() bool {
// Status reporter can be disabled via environment variable
if os.Getenv("LOGWISP_DISABLE_STATUS_REPORTER") == "1" {
return false
diff --git a/src/cmd/logwisp/status.go b/src/cmd/logwisp/status.go
index f4ca0cd..5c9a5a4 100644
--- a/src/cmd/logwisp/status.go
+++ b/src/cmd/logwisp/status.go
@@ -16,9 +16,9 @@ func statusReporter(service *service.Service) {
for range ticker.C {
stats := service.GetGlobalStats()
- totalStreams := stats["total_streams"].(int)
- if totalStreams == 0 {
- logger.Warn("msg", "No active streams in status report",
+ totalPipelines := stats["total_pipelines"].(int)
+ if totalPipelines == 0 {
+ logger.Warn("msg", "No active pipelines in status report",
"component", "status_reporter")
return
}
@@ -26,94 +26,171 @@ func statusReporter(service *service.Service) {
// Log status at DEBUG level to avoid cluttering INFO logs
logger.Debug("msg", "Status report",
"component", "status_reporter",
- "active_streams", totalStreams,
+ "active_pipelines", totalPipelines,
"time", time.Now().Format("15:04:05"))
- // Log individual stream status
- for name, streamStats := range stats["streams"].(map[string]interface{}) {
- logStreamStatus(name, streamStats.(map[string]interface{}))
+ // Log individual pipeline status
+ pipelines := stats["pipelines"].(map[string]any)
+ for name, pipelineStats := range pipelines {
+ logPipelineStatus(name, pipelineStats.(map[string]any))
}
}
}
-// logStreamStatus logs the status of an individual stream
-func logStreamStatus(name string, stats map[string]interface{}) {
- statusFields := []interface{}{
- "msg", "Stream status",
- "stream", name,
+// logPipelineStatus logs the status of an individual pipeline
+func logPipelineStatus(name string, stats map[string]any) {
+ statusFields := []any{
+ "msg", "Pipeline status",
+ "pipeline", name,
}
- // Add monitor statistics
- if monitor, ok := stats["monitor"].(map[string]interface{}); ok {
- statusFields = append(statusFields,
- "watchers", monitor["active_watchers"],
- "entries", monitor["total_entries"])
+ // Add processing statistics
+ if totalProcessed, ok := stats["total_processed"].(uint64); ok {
+ statusFields = append(statusFields, "entries_processed", totalProcessed)
+ }
+ if totalFiltered, ok := stats["total_filtered"].(uint64); ok {
+ statusFields = append(statusFields, "entries_filtered", totalFiltered)
}
- // Add TCP server statistics
- if tcp, ok := stats["tcp"].(map[string]interface{}); ok && tcp["enabled"].(bool) {
- statusFields = append(statusFields, "tcp_conns", tcp["connections"])
+ // Add source count
+ if sourceCount, ok := stats["source_count"].(int); ok {
+ statusFields = append(statusFields, "sources", sourceCount)
}
- // Add HTTP server statistics
- if http, ok := stats["http"].(map[string]interface{}); ok && http["enabled"].(bool) {
- statusFields = append(statusFields, "http_conns", http["connections"])
+ // Add sink statistics
+ if sinks, ok := stats["sinks"].([]map[string]any); ok {
+ tcpConns := 0
+ httpConns := 0
+
+ for _, sink := range sinks {
+ sinkType := sink["type"].(string)
+ if activeConns, ok := sink["active_connections"].(int32); ok {
+ switch sinkType {
+ case "tcp":
+ tcpConns += int(activeConns)
+ case "http":
+ httpConns += int(activeConns)
+ }
+ }
+ }
+
+ if tcpConns > 0 {
+ statusFields = append(statusFields, "tcp_connections", tcpConns)
+ }
+ if httpConns > 0 {
+ statusFields = append(statusFields, "http_connections", httpConns)
+ }
}
logger.Debug(statusFields...)
}
-// displayStreamEndpoints logs the configured endpoints for a stream
-func displayStreamEndpoints(cfg config.StreamConfig, routerMode bool) {
- // Display TCP endpoints
- if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
- logger.Info("msg", "TCP endpoint configured",
- "component", "main",
- "transport", cfg.Name,
- "port", cfg.TCPServer.Port)
+// displayPipelineEndpoints logs the configured endpoints for a pipeline
+func displayPipelineEndpoints(cfg config.PipelineConfig, routerMode bool) {
+ // Display sink endpoints
+ for i, sinkCfg := range cfg.Sinks {
+ switch sinkCfg.Type {
+ case "tcp":
+ if port, ok := toInt(sinkCfg.Options["port"]); ok {
+ logger.Info("msg", "TCP endpoint configured",
+ "component", "main",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "port", port)
- if cfg.TCPServer.RateLimit != nil && cfg.TCPServer.RateLimit.Enabled {
- logger.Info("msg", "TCP rate limiting enabled",
- "transport", cfg.Name,
- "requests_per_second", cfg.TCPServer.RateLimit.RequestsPerSecond,
- "burst_size", cfg.TCPServer.RateLimit.BurstSize)
+ // Display rate limit info if configured
+ if rl, ok := sinkCfg.Options["rate_limit"].(map[string]any); ok {
+ if enabled, ok := rl["enabled"].(bool); ok && enabled {
+ logger.Info("msg", "TCP rate limiting enabled",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "requests_per_second", rl["requests_per_second"],
+ "burst_size", rl["burst_size"])
+ }
+ }
+ }
+
+ case "http":
+ if port, ok := toInt(sinkCfg.Options["port"]); ok {
+ streamPath := "/transport"
+ statusPath := "/status"
+ if path, ok := sinkCfg.Options["stream_path"].(string); ok {
+ streamPath = path
+ }
+ if path, ok := sinkCfg.Options["status_path"].(string); ok {
+ statusPath = path
+ }
+
+ if routerMode {
+ logger.Info("msg", "HTTP endpoints configured",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "stream_path", fmt.Sprintf("/%s%s", cfg.Name, streamPath),
+ "status_path", fmt.Sprintf("/%s%s", cfg.Name, statusPath))
+ } else {
+ logger.Info("msg", "HTTP endpoints configured",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "stream_url", fmt.Sprintf("http://localhost:%d%s", port, streamPath),
+ "status_url", fmt.Sprintf("http://localhost:%d%s", port, statusPath))
+ }
+
+ // Display rate limit info if configured
+ if rl, ok := sinkCfg.Options["rate_limit"].(map[string]any); ok {
+ if enabled, ok := rl["enabled"].(bool); ok && enabled {
+ logger.Info("msg", "HTTP rate limiting enabled",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "requests_per_second", rl["requests_per_second"],
+ "burst_size", rl["burst_size"],
+ "limit_by", rl["limit_by"])
+ }
+ }
+ }
+
+ case "file":
+ if dir, ok := sinkCfg.Options["directory"].(string); ok {
+ name, _ := sinkCfg.Options["name"].(string)
+ logger.Info("msg", "File sink configured",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "directory", dir,
+ "name", name)
+ }
+
+ case "stdout", "stderr":
+ logger.Info("msg", "Console sink configured",
+ "pipeline", cfg.Name,
+ "sink_index", i,
+ "type", sinkCfg.Type)
}
}
- // Display HTTP endpoints
- if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
- if routerMode {
- logger.Info("msg", "HTTP endpoints configured",
- "transport", cfg.Name,
- "stream_path", fmt.Sprintf("/%s%s", cfg.Name, cfg.HTTPServer.StreamPath),
- "status_path", fmt.Sprintf("/%s%s", cfg.Name, cfg.HTTPServer.StatusPath))
- } else {
- logger.Info("msg", "HTTP endpoints configured",
- "transport", cfg.Name,
- "stream_url", fmt.Sprintf("http://localhost:%d%s", cfg.HTTPServer.Port, cfg.HTTPServer.StreamPath),
- "status_url", fmt.Sprintf("http://localhost:%d%s", cfg.HTTPServer.Port, cfg.HTTPServer.StatusPath))
- }
-
- if cfg.HTTPServer.RateLimit != nil && cfg.HTTPServer.RateLimit.Enabled {
- logger.Info("msg", "HTTP rate limiting enabled",
- "transport", cfg.Name,
- "requests_per_second", cfg.HTTPServer.RateLimit.RequestsPerSecond,
- "burst_size", cfg.HTTPServer.RateLimit.BurstSize,
- "limit_by", cfg.HTTPServer.RateLimit.LimitBy)
- }
-
- // Display authentication information
- if cfg.Auth != nil && cfg.Auth.Type != "none" {
- logger.Info("msg", "Authentication enabled",
- "transport", cfg.Name,
- "auth_type", cfg.Auth.Type)
- }
+ // Display authentication information
+ if cfg.Auth != nil && cfg.Auth.Type != "none" {
+ logger.Info("msg", "Authentication enabled",
+ "pipeline", cfg.Name,
+ "auth_type", cfg.Auth.Type)
}
// Display filter information
if len(cfg.Filters) > 0 {
logger.Info("msg", "Filters configured",
- "transport", cfg.Name,
+ "pipeline", cfg.Name,
"filter_count", len(cfg.Filters))
}
+}
+
+// Helper function for type conversion
+func toInt(v any) (int, bool) {
+ switch val := v.(type) {
+ case int:
+ return val, true
+ case int64:
+ return int(val), true
+ case float64:
+ return int(val), true
+ default:
+ return 0, false
+ }
}
\ No newline at end of file
diff --git a/src/internal/config/auth.go b/src/internal/config/auth.go
index a37ba70..f4b7034 100644
--- a/src/internal/config/auth.go
+++ b/src/internal/config/auth.go
@@ -1,6 +1,8 @@
// FILE: src/internal/config/auth.go
package config
+import "fmt"
+
type AuthConfig struct {
// Authentication type: "none", "basic", "bearer", "mtls"
Type string `toml:"type"`
@@ -53,4 +55,25 @@ type JWTConfig struct {
// Expected audience
Audience string `toml:"audience"`
+}
+
+func validateAuth(pipelineName string, auth *AuthConfig) error {
+ if auth == nil {
+ return nil
+ }
+
+ validTypes := map[string]bool{"none": true, "basic": true, "bearer": true, "mtls": true}
+ if !validTypes[auth.Type] {
+ return fmt.Errorf("pipeline '%s': invalid auth type: %s", pipelineName, auth.Type)
+ }
+
+ if auth.Type == "basic" && auth.BasicAuth == nil {
+ return fmt.Errorf("pipeline '%s': basic auth type specified but config missing", pipelineName)
+ }
+
+ if auth.Type == "bearer" && auth.BearerAuth == nil {
+ return fmt.Errorf("pipeline '%s': bearer auth type specified but config missing", pipelineName)
+ }
+
+ return nil
}
\ No newline at end of file
diff --git a/src/internal/config/config.go b/src/internal/config/config.go
index b331c75..a4b901f 100644
--- a/src/internal/config/config.go
+++ b/src/internal/config/config.go
@@ -5,10 +5,33 @@ type Config struct {
// Logging configuration
Logging *LogConfig `toml:"logging"`
- // Stream configurations
- Streams []StreamConfig `toml:"streams"`
+ // Pipeline configurations
+ Pipelines []PipelineConfig `toml:"pipelines"`
}
-type MonitorConfig struct {
- CheckIntervalMs int `toml:"check_interval_ms"`
+// Helper functions to handle type conversions from any
+func toInt(v any) (int, bool) {
+ switch val := v.(type) {
+ case int:
+ return val, true
+ case int64:
+ return int(val), true
+ case float64:
+ return int(val), true
+ default:
+ return 0, false
+ }
+}
+
+func toFloat(v any) (float64, bool) {
+ switch val := v.(type) {
+ case float64:
+ return val, true
+ case int:
+ return float64(val), true
+ case int64:
+ return float64(val), true
+ default:
+ return 0, false
+ }
}
\ No newline at end of file
diff --git a/src/internal/config/filter.go b/src/internal/config/filter.go
new file mode 100644
index 0000000..178317f
--- /dev/null
+++ b/src/internal/config/filter.go
@@ -0,0 +1,44 @@
+// FILE: src/internal/config/filter.go
+package config
+
+import (
+ "fmt"
+ "regexp"
+
+ "logwisp/src/internal/filter"
+)
+
+func validateFilter(pipelineName string, filterIndex int, cfg *filter.Config) error {
+ // Validate filter type
+ switch cfg.Type {
+ case filter.TypeInclude, filter.TypeExclude, "":
+ // Valid types
+ default:
+ return fmt.Errorf("pipeline '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
+ pipelineName, filterIndex, cfg.Type)
+ }
+
+ // Validate filter logic
+ switch cfg.Logic {
+ case filter.LogicOr, filter.LogicAnd, "":
+ // Valid logic
+ default:
+ return fmt.Errorf("pipeline '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
+ pipelineName, filterIndex, cfg.Logic)
+ }
+
+ // Empty patterns is valid - passes everything
+ if len(cfg.Patterns) == 0 {
+ return nil
+ }
+
+ // Validate regex patterns
+ for i, pattern := range cfg.Patterns {
+ if _, err := regexp.Compile(pattern); err != nil {
+ return fmt.Errorf("pipeline '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
+ pipelineName, filterIndex, i, pattern, err)
+ }
+ }
+
+ return nil
+}
\ No newline at end of file
diff --git a/src/internal/config/loader.go b/src/internal/config/loader.go
index 8f02141..ccecd97 100644
--- a/src/internal/config/loader.go
+++ b/src/internal/config/loader.go
@@ -13,27 +13,35 @@ import (
func defaults() *Config {
return &Config{
Logging: DefaultLogConfig(),
- Streams: []StreamConfig{
+ Pipelines: []PipelineConfig{
{
Name: "default",
- Monitor: &StreamMonitorConfig{
- CheckIntervalMs: 100,
- Targets: []MonitorTarget{
- {Path: "./", Pattern: "*.log", IsFile: false},
+ Sources: []SourceConfig{
+ {
+ Type: "directory",
+ Options: map[string]any{
+ "path": "./",
+ "pattern": "*.log",
+ "check_interval_ms": 100,
+ },
},
},
- HTTPServer: &HTTPConfig{
- Enabled: true,
- Port: 8080,
- BufferSize: 1000,
- StreamPath: "/transport",
- StatusPath: "/status",
- Heartbeat: HeartbeatConfig{
- Enabled: true,
- IntervalSeconds: 30,
- IncludeTimestamp: true,
- IncludeStats: false,
- Format: "comment",
+ Sinks: []SinkConfig{
+ {
+ Type: "http",
+ Options: map[string]any{
+ "port": 8080,
+ "buffer_size": 1000,
+ "stream_path": "/transport",
+ "status_path": "/status",
+ "heartbeat": map[string]any{
+ "enabled": true,
+ "interval_seconds": 30,
+ "include_timestamp": true,
+ "include_stats": false,
+ "format": "comment",
+ },
+ },
},
},
},
diff --git a/src/internal/config/logging.go b/src/internal/config/logging.go
index e06ee9a..c39b481 100644
--- a/src/internal/config/logging.go
+++ b/src/internal/config/logging.go
@@ -1,6 +1,8 @@
// FILE: src/internal/config/logging.go
package config
+import "fmt"
+
// LogConfig represents logging configuration for LogWisp
type LogConfig struct {
// Output mode: "file", "stdout", "stderr", "both", "none"
@@ -59,4 +61,32 @@ func DefaultLogConfig() *LogConfig {
Format: "txt",
},
}
+}
+
+func validateLogConfig(cfg *LogConfig) error {
+ validOutputs := map[string]bool{
+ "file": true, "stdout": true, "stderr": true,
+ "both": true, "none": true,
+ }
+ if !validOutputs[cfg.Output] {
+ return fmt.Errorf("invalid log output mode: %s", cfg.Output)
+ }
+
+ validLevels := map[string]bool{
+ "debug": true, "info": true, "warn": true, "error": true,
+ }
+ if !validLevels[cfg.Level] {
+ return fmt.Errorf("invalid log level: %s", cfg.Level)
+ }
+
+ if cfg.Console != nil {
+ validTargets := map[string]bool{
+ "stdout": true, "stderr": true, "split": true,
+ }
+ if !validTargets[cfg.Console.Target] {
+ return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
+ }
+ }
+
+ return nil
}
\ No newline at end of file
diff --git a/src/internal/config/pipeline.go b/src/internal/config/pipeline.go
new file mode 100644
index 0000000..1896979
--- /dev/null
+++ b/src/internal/config/pipeline.go
@@ -0,0 +1,276 @@
+// FILE: src/internal/config/pipeline.go
+package config
+
+import (
+ "fmt"
+ "logwisp/src/internal/filter"
+ "path/filepath"
+ "strings"
+)
+
+// PipelineConfig represents a data processing pipeline
+type PipelineConfig struct {
+ // Pipeline identifier (used in logs and metrics)
+ Name string `toml:"name"`
+
+ // Data sources for this pipeline
+ Sources []SourceConfig `toml:"sources"`
+
+ // Filter configuration
+ Filters []filter.Config `toml:"filters"`
+
+ // Output sinks for this pipeline
+ Sinks []SinkConfig `toml:"sinks"`
+
+ // Authentication/Authorization (applies to network sinks)
+ Auth *AuthConfig `toml:"auth"`
+}
+
+// SourceConfig represents an input data source
+type SourceConfig struct {
+ // Source type: "directory", "file", "stdin", etc.
+ Type string `toml:"type"`
+
+ // Type-specific configuration options
+ Options map[string]any `toml:"options"`
+
+ // Placeholder for future source-side rate limiting
+ // This will be used for features like aggregation and summarization
+ RateLimit *RateLimitConfig `toml:"rate_limit"`
+}
+
+// SinkConfig represents an output destination
+type SinkConfig struct {
+ // Sink type: "http", "tcp", "file", "stdout", "stderr"
+ Type string `toml:"type"`
+
+ // Type-specific configuration options
+ Options map[string]any `toml:"options"`
+}
+
+func validateSource(pipelineName string, sourceIndex int, cfg *SourceConfig) error {
+ if cfg.Type == "" {
+ return fmt.Errorf("pipeline '%s' source[%d]: missing type", pipelineName, sourceIndex)
+ }
+
+ switch cfg.Type {
+ case "directory":
+ // Validate directory source options
+ path, ok := cfg.Options["path"].(string)
+ if !ok || path == "" {
+ return fmt.Errorf("pipeline '%s' source[%d]: directory source requires 'path' option",
+ pipelineName, sourceIndex)
+ }
+
+ // Check for directory traversal
+ if strings.Contains(path, "..") {
+ return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal",
+ pipelineName, sourceIndex)
+ }
+
+ // Validate pattern if provided
+ if pattern, ok := cfg.Options["pattern"].(string); ok && pattern != "" {
+ // Try to compile as glob pattern (will be converted to regex internally)
+ if strings.Count(pattern, "*") == 0 && strings.Count(pattern, "?") == 0 {
+ // If no wildcards, ensure it's a valid filename
+ if filepath.Base(pattern) != pattern {
+ return fmt.Errorf("pipeline '%s' source[%d]: pattern contains path separators",
+ pipelineName, sourceIndex)
+ }
+ }
+ }
+
+ // Validate check interval if provided
+ if interval, ok := cfg.Options["check_interval_ms"]; ok {
+ if intVal, ok := toInt(interval); ok {
+ if intVal < 10 {
+ return fmt.Errorf("pipeline '%s' source[%d]: check interval too small: %d ms (min: 10ms)",
+ pipelineName, sourceIndex, intVal)
+ }
+ } else {
+ return fmt.Errorf("pipeline '%s' source[%d]: invalid check_interval_ms type",
+ pipelineName, sourceIndex)
+ }
+ }
+
+ case "file":
+ // Validate file source options
+ path, ok := cfg.Options["path"].(string)
+ if !ok || path == "" {
+ return fmt.Errorf("pipeline '%s' source[%d]: file source requires 'path' option",
+ pipelineName, sourceIndex)
+ }
+
+ // Check for directory traversal
+ if strings.Contains(path, "..") {
+ return fmt.Errorf("pipeline '%s' source[%d]: path contains directory traversal",
+ pipelineName, sourceIndex)
+ }
+
+ case "stdin":
+ // No specific validation needed for stdin
+
+ default:
+ return fmt.Errorf("pipeline '%s' source[%d]: unknown source type '%s'",
+ pipelineName, sourceIndex, cfg.Type)
+ }
+
+ // Note: RateLimit field is ignored for now as it's a placeholder
+ return nil
+}
+
+func validateSink(pipelineName string, sinkIndex int, cfg *SinkConfig, allPorts map[int]string) error {
+ if cfg.Type == "" {
+ return fmt.Errorf("pipeline '%s' sink[%d]: missing type", pipelineName, sinkIndex)
+ }
+
+ switch cfg.Type {
+ case "http":
+ // Extract and validate HTTP configuration
+ port, ok := toInt(cfg.Options["port"])
+ if !ok || port < 1 || port > 65535 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: invalid or missing HTTP port",
+ pipelineName, sinkIndex)
+ }
+
+ // Check port conflicts
+ if existing, exists := allPorts[port]; exists {
+ return fmt.Errorf("pipeline '%s' sink[%d]: HTTP port %d already used by %s",
+ pipelineName, sinkIndex, port, existing)
+ }
+ allPorts[port] = fmt.Sprintf("%s-http[%d]", pipelineName, sinkIndex)
+
+ // Validate buffer size
+ if bufSize, ok := toInt(cfg.Options["buffer_size"]); ok {
+ if bufSize < 1 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: HTTP buffer size must be positive: %d",
+ pipelineName, sinkIndex, bufSize)
+ }
+ }
+
+ // Validate paths if provided
+ if streamPath, ok := cfg.Options["stream_path"].(string); ok {
+ if !strings.HasPrefix(streamPath, "/") {
+ return fmt.Errorf("pipeline '%s' sink[%d]: stream path must start with /: %s",
+ pipelineName, sinkIndex, streamPath)
+ }
+ }
+
+ if statusPath, ok := cfg.Options["status_path"].(string); ok {
+ if !strings.HasPrefix(statusPath, "/") {
+ return fmt.Errorf("pipeline '%s' sink[%d]: status path must start with /: %s",
+ pipelineName, sinkIndex, statusPath)
+ }
+ }
+
+ // Validate heartbeat if present
+ if hb, ok := cfg.Options["heartbeat"].(map[string]any); ok {
+ if err := validateHeartbeatOptions("HTTP", pipelineName, sinkIndex, hb); err != nil {
+ return err
+ }
+ }
+
+ // Validate SSL if present
+ if ssl, ok := cfg.Options["ssl"].(map[string]any); ok {
+ if err := validateSSLOptions("HTTP", pipelineName, sinkIndex, ssl); err != nil {
+ return err
+ }
+ }
+
+ // Validate rate limit if present
+ if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
+ if err := validateRateLimitOptions("HTTP", pipelineName, sinkIndex, rl); err != nil {
+ return err
+ }
+ }
+
+ case "tcp":
+ // Extract and validate TCP configuration
+ port, ok := toInt(cfg.Options["port"])
+ if !ok || port < 1 || port > 65535 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: invalid or missing TCP port",
+ pipelineName, sinkIndex)
+ }
+
+ // Check port conflicts
+ if existing, exists := allPorts[port]; exists {
+ return fmt.Errorf("pipeline '%s' sink[%d]: TCP port %d already used by %s",
+ pipelineName, sinkIndex, port, existing)
+ }
+ allPorts[port] = fmt.Sprintf("%s-tcp[%d]", pipelineName, sinkIndex)
+
+ // Validate buffer size
+ if bufSize, ok := toInt(cfg.Options["buffer_size"]); ok {
+ if bufSize < 1 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: TCP buffer size must be positive: %d",
+ pipelineName, sinkIndex, bufSize)
+ }
+ }
+
+ // Validate heartbeat if present
+ if hb, ok := cfg.Options["heartbeat"].(map[string]any); ok {
+ if err := validateHeartbeatOptions("TCP", pipelineName, sinkIndex, hb); err != nil {
+ return err
+ }
+ }
+
+ // Validate SSL if present
+ if ssl, ok := cfg.Options["ssl"].(map[string]any); ok {
+ if err := validateSSLOptions("TCP", pipelineName, sinkIndex, ssl); err != nil {
+ return err
+ }
+ }
+
+ // Validate rate limit if present
+ if rl, ok := cfg.Options["rate_limit"].(map[string]any); ok {
+ if err := validateRateLimitOptions("TCP", pipelineName, sinkIndex, rl); err != nil {
+ return err
+ }
+ }
+
+ case "file":
+ // Validate file sink options
+ directory, ok := cfg.Options["directory"].(string)
+ if !ok || directory == "" {
+ return fmt.Errorf("pipeline '%s' sink[%d]: file sink requires 'directory' option",
+ pipelineName, sinkIndex)
+ }
+
+ name, ok := cfg.Options["name"].(string)
+ if !ok || name == "" {
+ return fmt.Errorf("pipeline '%s' sink[%d]: file sink requires 'name' option",
+ pipelineName, sinkIndex)
+ }
+
+ // Validate numeric options
+ if maxSize, ok := toInt(cfg.Options["max_size_mb"]); ok {
+ if maxSize < 1 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: max_size_mb must be positive: %d",
+ pipelineName, sinkIndex, maxSize)
+ }
+ }
+
+ if maxTotalSize, ok := toInt(cfg.Options["max_total_size_mb"]); ok {
+ if maxTotalSize < 0 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: max_total_size_mb cannot be negative: %d",
+ pipelineName, sinkIndex, maxTotalSize)
+ }
+ }
+
+ if retention, ok := toFloat(cfg.Options["retention_hours"]); ok {
+ if retention < 0 {
+ return fmt.Errorf("pipeline '%s' sink[%d]: retention_hours cannot be negative: %f",
+ pipelineName, sinkIndex, retention)
+ }
+ }
+
+ case "stdout", "stderr":
+ // No specific validation needed for console sinks
+
+ default:
+ return fmt.Errorf("pipeline '%s' sink[%d]: unknown sink type '%s'",
+ pipelineName, sinkIndex, cfg.Type)
+ }
+
+ return nil
+}
\ No newline at end of file
diff --git a/src/internal/config/server.go b/src/internal/config/server.go
index e4868b4..a161d1c 100644
--- a/src/internal/config/server.go
+++ b/src/internal/config/server.go
@@ -1,6 +1,8 @@
// FILE: src/internal/config/server.go
package config
+import "fmt"
+
type TCPConfig struct {
Enabled bool `toml:"enabled"`
Port int `toml:"port"`
@@ -63,4 +65,72 @@ type RateLimitConfig struct {
// Connection limits
MaxConnectionsPerIP int `toml:"max_connections_per_ip"`
MaxTotalConnections int `toml:"max_total_connections"`
+}
+
+func validateHeartbeatOptions(serverType, pipelineName string, sinkIndex int, hb map[string]any) error {
+ if enabled, ok := hb["enabled"].(bool); ok && enabled {
+ interval, ok := toInt(hb["interval_seconds"])
+ if !ok || interval < 1 {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: heartbeat interval must be positive",
+ pipelineName, sinkIndex, serverType)
+ }
+
+ if format, ok := hb["format"].(string); ok {
+ if format != "json" && format != "comment" {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: heartbeat format must be 'json' or 'comment': %s",
+ pipelineName, sinkIndex, serverType, format)
+ }
+ }
+ }
+ return nil
+}
+
+func validateRateLimitOptions(serverType, pipelineName string, sinkIndex int, rl map[string]any) error {
+ if enabled, ok := rl["enabled"].(bool); !ok || !enabled {
+ return nil
+ }
+
+ // Validate requests per second
+ rps, ok := toFloat(rl["requests_per_second"])
+ if !ok || rps <= 0 {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: requests_per_second must be positive",
+ pipelineName, sinkIndex, serverType)
+ }
+
+ // Validate burst size
+ burst, ok := toInt(rl["burst_size"])
+ if !ok || burst < 1 {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: burst_size must be at least 1",
+ pipelineName, sinkIndex, serverType)
+ }
+
+ // Validate limit_by
+ if limitBy, ok := rl["limit_by"].(string); ok && limitBy != "" {
+ validLimitBy := map[string]bool{"ip": true, "global": true}
+ if !validLimitBy[limitBy] {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid limit_by value: %s (must be 'ip' or 'global')",
+ pipelineName, sinkIndex, serverType, limitBy)
+ }
+ }
+
+ // Validate response code
+ if respCode, ok := toInt(rl["response_code"]); ok {
+ if respCode > 0 && (respCode < 400 || respCode >= 600) {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: response_code must be 4xx or 5xx: %d",
+ pipelineName, sinkIndex, serverType, respCode)
+ }
+ }
+
+ // Validate connection limits
+ maxPerIP, perIPOk := toInt(rl["max_connections_per_ip"])
+ maxTotal, totalOk := toInt(rl["max_total_connections"])
+
+ if perIPOk && totalOk && maxPerIP > 0 && maxTotal > 0 {
+ if maxPerIP > maxTotal {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: max_connections_per_ip (%d) cannot exceed max_total_connections (%d)",
+ pipelineName, sinkIndex, serverType, maxPerIP, maxTotal)
+ }
+ }
+
+ return nil
}
\ No newline at end of file
diff --git a/src/internal/config/ssl.go b/src/internal/config/ssl.go
index 81af516..ff47ed7 100644
--- a/src/internal/config/ssl.go
+++ b/src/internal/config/ssl.go
@@ -1,6 +1,8 @@
// FILE: src/internal/config/ssl.go
package config
+import "fmt"
+
type SSLConfig struct {
Enabled bool `toml:"enabled"`
CertFile string `toml:"cert_file"`
@@ -17,4 +19,39 @@ type SSLConfig struct {
// Cipher suites (comma-separated list)
CipherSuites string `toml:"cipher_suites"`
+}
+
+func validateSSLOptions(serverType, pipelineName string, sinkIndex int, ssl map[string]any) error {
+ if enabled, ok := ssl["enabled"].(bool); ok && enabled {
+ certFile, certOk := ssl["cert_file"].(string)
+ keyFile, keyOk := ssl["key_file"].(string)
+
+ if !certOk || certFile == "" || !keyOk || keyFile == "" {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: SSL enabled but cert/key files not specified",
+ pipelineName, sinkIndex, serverType)
+ }
+
+ if clientAuth, ok := ssl["client_auth"].(bool); ok && clientAuth {
+ if caFile, ok := ssl["client_ca_file"].(string); !ok || caFile == "" {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: client auth enabled but CA file not specified",
+ pipelineName, sinkIndex, serverType)
+ }
+ }
+
+ // Validate TLS versions
+ validVersions := map[string]bool{"TLS1.0": true, "TLS1.1": true, "TLS1.2": true, "TLS1.3": true}
+ if minVer, ok := ssl["min_version"].(string); ok && minVer != "" {
+ if !validVersions[minVer] {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid min TLS version: %s",
+ pipelineName, sinkIndex, serverType, minVer)
+ }
+ }
+ if maxVer, ok := ssl["max_version"].(string); ok && maxVer != "" {
+ if !validVersions[maxVer] {
+ return fmt.Errorf("pipeline '%s' sink[%d] %s: invalid max TLS version: %s",
+ pipelineName, sinkIndex, serverType, maxVer)
+ }
+ }
+ }
+ return nil
}
\ No newline at end of file
diff --git a/src/internal/config/stream.go b/src/internal/config/stream.go
deleted file mode 100644
index 5efa354..0000000
--- a/src/internal/config/stream.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// FILE: src/internal/config/transport.go
-package config
-
-import (
- "logwisp/src/internal/filter"
-)
-
-type StreamConfig struct {
- // Stream identifier (used in logs and metrics)
- Name string `toml:"name"`
-
- // Monitor configuration for this transport
- Monitor *StreamMonitorConfig `toml:"monitor"`
-
- // Filter configuration
- Filters []filter.Config `toml:"filters"`
-
- // Server configurations
- TCPServer *TCPConfig `toml:"tcpserver"`
- HTTPServer *HTTPConfig `toml:"httpserver"`
-
- // Authentication/Authorization
- Auth *AuthConfig `toml:"auth"`
-}
-
-type StreamMonitorConfig struct {
- CheckIntervalMs int `toml:"check_interval_ms"`
- Targets []MonitorTarget `toml:"targets"`
-}
-
-type MonitorTarget struct {
- Path string `toml:"path"`
- Pattern string `toml:"pattern"`
- IsFile bool `toml:"is_file"`
-}
-
-func (s *StreamConfig) GetTargets(defaultTargets []MonitorTarget) []MonitorTarget {
- if s.Monitor != nil && len(s.Monitor.Targets) > 0 {
- return s.Monitor.Targets
- }
- return nil
-}
-
-func (s *StreamConfig) GetCheckInterval(defaultInterval int) int {
- if s.Monitor != nil && s.Monitor.CheckIntervalMs > 0 {
- return s.Monitor.CheckIntervalMs
- }
- return defaultInterval
-}
\ No newline at end of file
diff --git a/src/internal/config/validation.go b/src/internal/config/validation.go
index 52ec677..97f14a7 100644
--- a/src/internal/config/validation.go
+++ b/src/internal/config/validation.go
@@ -3,309 +3,67 @@ package config
import (
"fmt"
- "regexp"
- "strings"
-
- "logwisp/src/internal/filter"
)
func (c *Config) validate() error {
- if len(c.Streams) == 0 {
- return fmt.Errorf("no streams configured")
+ if len(c.Pipelines) == 0 {
+ return fmt.Errorf("no pipelines configured")
}
if err := validateLogConfig(c.Logging); err != nil {
return fmt.Errorf("logging config: %w", err)
}
- // Validate each transport
- streamNames := make(map[string]bool)
- streamPorts := make(map[int]string)
+ // Track used ports across all pipelines
+ allPorts := make(map[int]string)
+ pipelineNames := make(map[string]bool)
- for i, stream := range c.Streams {
- if stream.Name == "" {
- return fmt.Errorf("transport %d: missing name", i)
+ for i, pipeline := range c.Pipelines {
+ if pipeline.Name == "" {
+ return fmt.Errorf("pipeline %d: missing name", i)
}
- if streamNames[stream.Name] {
- return fmt.Errorf("transport %d: duplicate name '%s'", i, stream.Name)
+ if pipelineNames[pipeline.Name] {
+ return fmt.Errorf("pipeline %d: duplicate name '%s'", i, pipeline.Name)
}
- streamNames[stream.Name] = true
+ pipelineNames[pipeline.Name] = true
- // Stream must have monitor config with targets
- if stream.Monitor == nil || len(stream.Monitor.Targets) == 0 {
- return fmt.Errorf("transport '%s': no monitor targets specified", stream.Name)
+ // Pipeline must have at least one source
+ if len(pipeline.Sources) == 0 {
+ return fmt.Errorf("pipeline '%s': no sources specified", pipeline.Name)
}
- // Validate check interval
- if stream.Monitor.CheckIntervalMs < 10 {
- return fmt.Errorf("transport '%s': check interval too small: %d ms (min: 10ms)",
- stream.Name, stream.Monitor.CheckIntervalMs)
- }
-
- // Validate targets
- for j, target := range stream.Monitor.Targets {
- if target.Path == "" {
- return fmt.Errorf("transport '%s' target %d: empty path", stream.Name, j)
- }
- if strings.Contains(target.Path, "..") {
- return fmt.Errorf("transport '%s' target %d: path contains directory traversal", stream.Name, j)
+ // Validate sources
+ for j, source := range pipeline.Sources {
+ if err := validateSource(pipeline.Name, j, &source); err != nil {
+ return err
}
}
// Validate filters
- for j, filterCfg := range stream.Filters {
- if err := validateFilter(stream.Name, j, &filterCfg); err != nil {
+ for j, filterCfg := range pipeline.Filters {
+ if err := validateFilter(pipeline.Name, j, &filterCfg); err != nil {
return err
}
}
- // Validate TCP server
- if stream.TCPServer != nil && stream.TCPServer.Enabled {
- if stream.TCPServer.Port < 1 || stream.TCPServer.Port > 65535 {
- return fmt.Errorf("transport '%s': invalid TCP port: %d", stream.Name, stream.TCPServer.Port)
- }
- if existing, exists := streamPorts[stream.TCPServer.Port]; exists {
- return fmt.Errorf("transport '%s': TCP port %d already used by transport '%s'",
- stream.Name, stream.TCPServer.Port, existing)
- }
- streamPorts[stream.TCPServer.Port] = stream.Name + "-tcp"
-
- if stream.TCPServer.BufferSize < 1 {
- return fmt.Errorf("transport '%s': TCP buffer size must be positive: %d",
- stream.Name, stream.TCPServer.BufferSize)
- }
-
- if err := validateHeartbeat("TCP", stream.Name, &stream.TCPServer.Heartbeat); err != nil {
- return err
- }
-
- if err := validateSSL("TCP", stream.Name, stream.TCPServer.SSL); err != nil {
- return err
- }
-
- if err := validateRateLimit("TCP", stream.Name, stream.TCPServer.RateLimit); err != nil {
- return err
- }
+ // Pipeline must have at least one sink
+ if len(pipeline.Sinks) == 0 {
+ return fmt.Errorf("pipeline '%s': no sinks specified", pipeline.Name)
}
- // Validate HTTP server
- if stream.HTTPServer != nil && stream.HTTPServer.Enabled {
- if stream.HTTPServer.Port < 1 || stream.HTTPServer.Port > 65535 {
- return fmt.Errorf("transport '%s': invalid HTTP port: %d", stream.Name, stream.HTTPServer.Port)
- }
- if existing, exists := streamPorts[stream.HTTPServer.Port]; exists {
- return fmt.Errorf("transport '%s': HTTP port %d already used by transport '%s'",
- stream.Name, stream.HTTPServer.Port, existing)
- }
- streamPorts[stream.HTTPServer.Port] = stream.Name + "-http"
-
- if stream.HTTPServer.BufferSize < 1 {
- return fmt.Errorf("transport '%s': HTTP buffer size must be positive: %d",
- stream.Name, stream.HTTPServer.BufferSize)
- }
-
- // Validate paths
- if stream.HTTPServer.StreamPath == "" {
- stream.HTTPServer.StreamPath = "/transport"
- }
- if stream.HTTPServer.StatusPath == "" {
- stream.HTTPServer.StatusPath = "/status"
- }
- if !strings.HasPrefix(stream.HTTPServer.StreamPath, "/") {
- return fmt.Errorf("transport '%s': transport path must start with /: %s",
- stream.Name, stream.HTTPServer.StreamPath)
- }
- if !strings.HasPrefix(stream.HTTPServer.StatusPath, "/") {
- return fmt.Errorf("transport '%s': status path must start with /: %s",
- stream.Name, stream.HTTPServer.StatusPath)
- }
-
- if err := validateHeartbeat("HTTP", stream.Name, &stream.HTTPServer.Heartbeat); err != nil {
+ // Validate sinks and check for port conflicts
+ for j, sink := range pipeline.Sinks {
+ if err := validateSink(pipeline.Name, j, &sink, allPorts); err != nil {
return err
}
-
- if err := validateSSL("HTTP", stream.Name, stream.HTTPServer.SSL); err != nil {
- return err
- }
-
- if err := validateRateLimit("HTTP", stream.Name, stream.HTTPServer.RateLimit); err != nil {
- return err
- }
- }
-
- // At least one server must be enabled
- tcpEnabled := stream.TCPServer != nil && stream.TCPServer.Enabled
- httpEnabled := stream.HTTPServer != nil && stream.HTTPServer.Enabled
- if !tcpEnabled && !httpEnabled {
- return fmt.Errorf("transport '%s': no servers enabled", stream.Name)
}
// Validate auth if present
- if err := validateAuth(stream.Name, stream.Auth); err != nil {
+ if err := validateAuth(pipeline.Name, pipeline.Auth); err != nil {
return err
}
}
- return nil
-}
-
-func validateHeartbeat(serverType, streamName string, hb *HeartbeatConfig) error {
- if hb.Enabled {
- if hb.IntervalSeconds < 1 {
- return fmt.Errorf("transport '%s' %s: heartbeat interval must be positive: %d",
- streamName, serverType, hb.IntervalSeconds)
- }
- if hb.Format != "json" && hb.Format != "comment" {
- return fmt.Errorf("transport '%s' %s: heartbeat format must be 'json' or 'comment': %s",
- streamName, serverType, hb.Format)
- }
- }
- return nil
-}
-
-func validateSSL(serverType, streamName string, ssl *SSLConfig) error {
- if ssl != nil && ssl.Enabled {
- if ssl.CertFile == "" || ssl.KeyFile == "" {
- return fmt.Errorf("transport '%s' %s: SSL enabled but cert/key files not specified",
- streamName, serverType)
- }
-
- if ssl.ClientAuth && ssl.ClientCAFile == "" {
- return fmt.Errorf("transport '%s' %s: client auth enabled but CA file not specified",
- streamName, serverType)
- }
-
- // Validate TLS versions
- validVersions := map[string]bool{"TLS1.0": true, "TLS1.1": true, "TLS1.2": true, "TLS1.3": true}
- if ssl.MinVersion != "" && !validVersions[ssl.MinVersion] {
- return fmt.Errorf("transport '%s' %s: invalid min TLS version: %s",
- streamName, serverType, ssl.MinVersion)
- }
- if ssl.MaxVersion != "" && !validVersions[ssl.MaxVersion] {
- return fmt.Errorf("transport '%s' %s: invalid max TLS version: %s",
- streamName, serverType, ssl.MaxVersion)
- }
- }
- return nil
-}
-
-func validateAuth(streamName string, auth *AuthConfig) error {
- if auth == nil {
- return nil
- }
-
- validTypes := map[string]bool{"none": true, "basic": true, "bearer": true, "mtls": true}
- if !validTypes[auth.Type] {
- return fmt.Errorf("transport '%s': invalid auth type: %s", streamName, auth.Type)
- }
-
- if auth.Type == "basic" && auth.BasicAuth == nil {
- return fmt.Errorf("transport '%s': basic auth type specified but config missing", streamName)
- }
-
- if auth.Type == "bearer" && auth.BearerAuth == nil {
- return fmt.Errorf("transport '%s': bearer auth type specified but config missing", streamName)
- }
-
- return nil
-}
-
-func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error {
- if rl == nil || !rl.Enabled {
- return nil
- }
-
- if rl.RequestsPerSecond <= 0 {
- return fmt.Errorf("transport '%s' %s: requests_per_second must be positive: %f",
- streamName, serverType, rl.RequestsPerSecond)
- }
-
- if rl.BurstSize < 1 {
- return fmt.Errorf("transport '%s' %s: burst_size must be at least 1: %d",
- streamName, serverType, rl.BurstSize)
- }
-
- validLimitBy := map[string]bool{"ip": true, "global": true, "": true}
- if !validLimitBy[rl.LimitBy] {
- return fmt.Errorf("transport '%s' %s: invalid limit_by value: %s (must be 'ip' or 'global')",
- streamName, serverType, rl.LimitBy)
- }
-
- if rl.ResponseCode > 0 && (rl.ResponseCode < 400 || rl.ResponseCode >= 600) {
- return fmt.Errorf("transport '%s' %s: response_code must be 4xx or 5xx: %d",
- streamName, serverType, rl.ResponseCode)
- }
-
- if rl.MaxConnectionsPerIP > 0 && rl.MaxTotalConnections > 0 {
- if rl.MaxConnectionsPerIP > rl.MaxTotalConnections {
- return fmt.Errorf("stream '%s' %s: max_connections_per_ip (%d) cannot exceed max_total_connections (%d)",
- streamName, serverType, rl.MaxConnectionsPerIP, rl.MaxTotalConnections)
- }
- }
-
- return nil
-}
-
-func validateFilter(streamName string, filterIndex int, cfg *filter.Config) error {
- // Validate filter type
- switch cfg.Type {
- case filter.TypeInclude, filter.TypeExclude, "":
- // Valid types
- default:
- return fmt.Errorf("transport '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
- streamName, filterIndex, cfg.Type)
- }
-
- // Validate filter logic
- switch cfg.Logic {
- case filter.LogicOr, filter.LogicAnd, "":
- // Valid logic
- default:
- return fmt.Errorf("transport '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
- streamName, filterIndex, cfg.Logic)
- }
-
- // Empty patterns is valid - passes everything
- if len(cfg.Patterns) == 0 {
- return nil
- }
-
- // Validate regex patterns
- for i, pattern := range cfg.Patterns {
- if _, err := regexp.Compile(pattern); err != nil {
- return fmt.Errorf("transport '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
- streamName, filterIndex, i, pattern, err)
- }
- }
-
- return nil
-}
-
-func validateLogConfig(cfg *LogConfig) error {
- validOutputs := map[string]bool{
- "file": true, "stdout": true, "stderr": true,
- "both": true, "none": true,
- }
- if !validOutputs[cfg.Output] {
- return fmt.Errorf("invalid log output mode: %s", cfg.Output)
- }
-
- validLevels := map[string]bool{
- "debug": true, "info": true, "warn": true, "error": true,
- }
- if !validLevels[cfg.Level] {
- return fmt.Errorf("invalid log level: %s", cfg.Level)
- }
-
- if cfg.Console != nil {
- validTargets := map[string]bool{
- "stdout": true, "stderr": true, "split": true,
- }
- if !validTargets[cfg.Console.Target] {
- return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
- }
- }
-
return nil
}
\ No newline at end of file
diff --git a/src/internal/filter/chain.go b/src/internal/filter/chain.go
index a4b6324..9696d08 100644
--- a/src/internal/filter/chain.go
+++ b/src/internal/filter/chain.go
@@ -5,7 +5,7 @@ import (
"fmt"
"sync/atomic"
- "logwisp/src/internal/monitor"
+ "logwisp/src/internal/source"
"github.com/lixenwraith/log"
)
@@ -43,7 +43,7 @@ func NewChain(configs []Config, logger *log.Logger) (*Chain, error) {
// Apply runs all filters in sequence
// Returns true if the entry passes all filters
-func (c *Chain) Apply(entry monitor.LogEntry) bool {
+func (c *Chain) Apply(entry source.LogEntry) bool {
c.totalProcessed.Add(1)
// No filters means pass everything
@@ -68,13 +68,13 @@ func (c *Chain) Apply(entry monitor.LogEntry) bool {
}
// GetStats returns chain statistics
-func (c *Chain) GetStats() map[string]interface{} {
- filterStats := make([]map[string]interface{}, len(c.filters))
+func (c *Chain) GetStats() map[string]any {
+ filterStats := make([]map[string]any, len(c.filters))
for i, filter := range c.filters {
filterStats[i] = filter.GetStats()
}
- return map[string]interface{}{
+ return map[string]any{
"filter_count": len(c.filters),
"total_processed": c.totalProcessed.Load(),
"total_passed": c.totalPassed.Load(),
diff --git a/src/internal/filter/filter.go b/src/internal/filter/filter.go
index 9629f5a..0964c2e 100644
--- a/src/internal/filter/filter.go
+++ b/src/internal/filter/filter.go
@@ -7,7 +7,7 @@ import (
"sync"
"sync/atomic"
- "logwisp/src/internal/monitor"
+ "logwisp/src/internal/source"
"github.com/lixenwraith/log"
)
@@ -83,7 +83,7 @@ func New(cfg Config, logger *log.Logger) (*Filter, error) {
}
// Apply checks if a log entry should be passed through
-func (f *Filter) Apply(entry monitor.LogEntry) bool {
+func (f *Filter) Apply(entry source.LogEntry) bool {
f.totalProcessed.Add(1)
// No patterns means pass everything
@@ -152,8 +152,8 @@ func (f *Filter) matches(text string) bool {
}
// GetStats returns filter statistics
-func (f *Filter) GetStats() map[string]interface{} {
- return map[string]interface{}{
+func (f *Filter) GetStats() map[string]any {
+ return map[string]any{
"type": f.config.Type,
"logic": f.config.Logic,
"pattern_count": len(f.patterns),
diff --git a/src/internal/monitor/monitor.go b/src/internal/monitor/monitor.go
deleted file mode 100644
index 950e991..0000000
--- a/src/internal/monitor/monitor.go
+++ /dev/null
@@ -1,380 +0,0 @@
-// FILE: src/internal/monitor/monitor.go
-package monitor
-
-import (
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "os"
- "path/filepath"
- "regexp"
- "sync"
- "sync/atomic"
- "time"
-
- "github.com/lixenwraith/log"
-)
-
-type LogEntry struct {
- Time time.Time `json:"time"`
- Source string `json:"source"`
- Level string `json:"level,omitempty"`
- Message string `json:"message"`
- Fields json.RawMessage `json:"fields,omitempty"`
-}
-
-type Monitor interface {
- Start(ctx context.Context) error
- Stop()
- Subscribe() chan LogEntry
- AddTarget(path, pattern string, isFile bool) error
- RemoveTarget(path string) error
- SetCheckInterval(interval time.Duration)
- GetStats() Stats
- GetActiveWatchers() []WatcherInfo
-}
-
-type Stats struct {
- ActiveWatchers int
- TotalEntries uint64
- DroppedEntries uint64
- StartTime time.Time
- LastEntryTime time.Time
-}
-
-type WatcherInfo struct {
- Path string
- Size int64
- Position int64
- ModTime time.Time
- EntriesRead uint64
- LastReadTime time.Time
- Rotations int
-}
-
-type monitor struct {
- subscribers []chan LogEntry
- targets []target
- watchers map[string]*fileWatcher
- mu sync.RWMutex
- ctx context.Context
- cancel context.CancelFunc
- wg sync.WaitGroup
- checkInterval time.Duration
- totalEntries atomic.Uint64
- droppedEntries atomic.Uint64
- startTime time.Time
- lastEntryTime atomic.Value // time.Time
- logger *log.Logger
-}
-
-type target struct {
- path string
- pattern string
- isFile bool
- regex *regexp.Regexp
-}
-
-func New(logger *log.Logger) Monitor {
- m := &monitor{
- watchers: make(map[string]*fileWatcher),
- checkInterval: 100 * time.Millisecond,
- startTime: time.Now(),
- logger: logger,
- }
- m.lastEntryTime.Store(time.Time{})
- return m
-}
-
-func (m *monitor) Subscribe() chan LogEntry {
- m.mu.Lock()
- defer m.mu.Unlock()
-
- ch := make(chan LogEntry, 1000)
- m.subscribers = append(m.subscribers, ch)
- return ch
-}
-
-func (m *monitor) publish(entry LogEntry) {
- m.mu.RLock()
- defer m.mu.RUnlock()
-
- m.totalEntries.Add(1)
- m.lastEntryTime.Store(entry.Time)
-
- for _, ch := range m.subscribers {
- select {
- case ch <- entry:
- default:
- m.droppedEntries.Add(1)
- m.logger.Debug("msg", "Dropped log entry - subscriber buffer full")
- }
- }
-}
-
-func (m *monitor) SetCheckInterval(interval time.Duration) {
- m.mu.Lock()
- m.checkInterval = interval
- m.mu.Unlock()
-
- m.logger.Debug("msg", "Check interval updated", "interval_ms", interval.Milliseconds())
-}
-
-func (m *monitor) AddTarget(path, pattern string, isFile bool) error {
- absPath, err := filepath.Abs(path)
- if err != nil {
- m.logger.Error("msg", "Failed to resolve absolute path",
- "component", "monitor",
- "path", path,
- "error", err)
- return fmt.Errorf("invalid path %s: %w", path, err)
- }
-
- var compiledRegex *regexp.Regexp
- if !isFile && pattern != "" {
- regexPattern := globToRegex(pattern)
- compiledRegex, err = regexp.Compile(regexPattern)
- if err != nil {
- m.logger.Error("msg", "Failed to compile pattern regex",
- "component", "monitor",
- "pattern", pattern,
- "regex", regexPattern,
- "error", err)
- return fmt.Errorf("invalid pattern %s: %w", pattern, err)
- }
- }
-
- m.mu.Lock()
- m.targets = append(m.targets, target{
- path: absPath,
- pattern: pattern,
- isFile: isFile,
- regex: compiledRegex,
- })
- m.mu.Unlock()
-
- m.logger.Info("msg", "Added monitor target",
- "component", "monitor",
- "path", absPath,
- "pattern", pattern,
- "is_file", isFile)
-
- return nil
-}
-
-func (m *monitor) RemoveTarget(path string) error {
- absPath, err := filepath.Abs(path)
- if err != nil {
- return fmt.Errorf("invalid path %s: %w", path, err)
- }
-
- m.mu.Lock()
- defer m.mu.Unlock()
-
- // Remove from targets
- newTargets := make([]target, 0, len(m.targets))
- for _, t := range m.targets {
- if t.path != absPath {
- newTargets = append(newTargets, t)
- }
- }
- m.targets = newTargets
-
- // Stop any watchers for this path
- if w, exists := m.watchers[absPath]; exists {
- w.stop()
- delete(m.watchers, absPath)
- m.logger.Info("msg", "Monitor started",
- "component", "monitor",
- "check_interval_ms", m.checkInterval.Milliseconds())
- }
-
- return nil
-}
-
-func (m *monitor) Start(ctx context.Context) error {
- m.ctx, m.cancel = context.WithCancel(ctx)
- m.wg.Add(1)
- go m.monitorLoop()
-
- m.logger.Info("msg", "Monitor started", "check_interval_ms", m.checkInterval.Milliseconds())
- return nil
-}
-
-func (m *monitor) Stop() {
- if m.cancel != nil {
- m.cancel()
- }
- m.wg.Wait()
-
- m.mu.Lock()
- for _, w := range m.watchers {
- w.close()
- }
- for _, ch := range m.subscribers {
- close(ch)
- }
- m.mu.Unlock()
-
- m.logger.Info("msg", "Monitor stopped")
-}
-
-func (m *monitor) GetStats() Stats {
- lastEntry, _ := m.lastEntryTime.Load().(time.Time)
-
- m.mu.RLock()
- watcherCount := len(m.watchers)
- m.mu.RUnlock()
-
- return Stats{
- ActiveWatchers: watcherCount,
- TotalEntries: m.totalEntries.Load(),
- DroppedEntries: m.droppedEntries.Load(),
- StartTime: m.startTime,
- LastEntryTime: lastEntry,
- }
-}
-
-func (m *monitor) GetActiveWatchers() []WatcherInfo {
- m.mu.RLock()
- defer m.mu.RUnlock()
-
- info := make([]WatcherInfo, 0, len(m.watchers))
- for _, w := range m.watchers {
- info = append(info, w.getInfo())
- }
- return info
-}
-
-func (m *monitor) monitorLoop() {
- defer m.wg.Done()
-
- m.checkTargets()
-
- m.mu.RLock()
- interval := m.checkInterval
- m.mu.RUnlock()
-
- ticker := time.NewTicker(interval)
- defer ticker.Stop()
-
- for {
- select {
- case <-m.ctx.Done():
- return
- case <-ticker.C:
- m.checkTargets()
-
- m.mu.RLock()
- newInterval := m.checkInterval
- m.mu.RUnlock()
-
- if newInterval != interval {
- ticker.Stop()
- ticker = time.NewTicker(newInterval)
- interval = newInterval
- }
- }
- }
-}
-
-func (m *monitor) checkTargets() {
- m.mu.RLock()
- targets := make([]target, len(m.targets))
- copy(targets, m.targets)
- m.mu.RUnlock()
-
- for _, t := range targets {
- if t.isFile {
- m.ensureWatcher(t.path)
- } else {
- // Directory scanning for pattern matching
- files, err := m.scanDirectory(t.path, t.regex)
- if err != nil {
- m.logger.Warn("msg", "Failed to scan directory",
- "component", "monitor",
- "path", t.path,
- "pattern", t.pattern,
- "error", err)
- continue
- }
-
- for _, file := range files {
- m.ensureWatcher(file)
- }
- }
- }
-
- m.cleanupWatchers()
-}
-
-func (m *monitor) scanDirectory(dir string, pattern *regexp.Regexp) ([]string, error) {
- entries, err := os.ReadDir(dir)
- if err != nil {
- return nil, err
- }
-
- var files []string
- for _, entry := range entries {
- if entry.IsDir() {
- continue
- }
-
- name := entry.Name()
- if pattern == nil || pattern.MatchString(name) {
- files = append(files, filepath.Join(dir, name))
- }
- }
-
- return files, nil
-}
-
-func (m *monitor) ensureWatcher(path string) {
- m.mu.Lock()
- defer m.mu.Unlock()
-
- if _, exists := m.watchers[path]; exists {
- return
- }
-
- w := newFileWatcher(path, m.publish, m.logger)
- m.watchers[path] = w
-
- m.logger.Debug("msg", "Created watcher", "path", path)
-
- m.wg.Add(1)
- go func() {
- defer m.wg.Done()
- if err := w.watch(m.ctx); err != nil {
- // Log based on error type
- if errors.Is(err, context.Canceled) {
- m.logger.Debug("msg", "Watcher cancelled",
- "component", "monitor",
- "path", path)
- } else {
- m.logger.Error("msg", "Watcher failed",
- "component", "monitor",
- "path", path,
- "error", err)
- }
- }
-
- m.mu.Lock()
- delete(m.watchers, path)
- m.mu.Unlock()
- }()
-}
-
-func (m *monitor) cleanupWatchers() {
- m.mu.Lock()
- defer m.mu.Unlock()
-
- for path, w := range m.watchers {
- if _, err := os.Stat(path); os.IsNotExist(err) {
- w.stop()
- delete(m.watchers, path)
- m.logger.Debug("msg", "Cleaned up watcher for non-existent file", "path", path)
- }
- }
-}
\ No newline at end of file
diff --git a/src/internal/service/httprouter.go b/src/internal/service/httprouter.go
index 709e659..8c753c8 100644
--- a/src/internal/service/httprouter.go
+++ b/src/internal/service/httprouter.go
@@ -8,10 +8,13 @@ import (
"sync/atomic"
"time"
+ "logwisp/src/internal/sink"
+
"github.com/lixenwraith/log"
"github.com/valyala/fasthttp"
)
+// HTTPRouter manages HTTP routing for multiple pipelines
type HTTPRouter struct {
service *Service
servers map[int]*routerServer // port -> server
@@ -25,6 +28,7 @@ type HTTPRouter struct {
failedRequests atomic.Uint64
}
+// NewHTTPRouter creates a new HTTP router
func NewHTTPRouter(service *Service, logger *log.Logger) *HTTPRouter {
return &HTTPRouter{
service: service,
@@ -34,12 +38,23 @@ func NewHTTPRouter(service *Service, logger *log.Logger) *HTTPRouter {
}
}
-func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
- if stream.HTTPServer == nil || stream.Config.HTTPServer == nil {
- return nil // No HTTP server configured
+// RegisterPipeline registers a pipeline's HTTP sinks with the router
+func (r *HTTPRouter) RegisterPipeline(pipeline *Pipeline) error {
+ // Register all HTTP sinks in the pipeline
+ for _, httpSink := range pipeline.HTTPSinks {
+ if err := r.registerHTTPSink(pipeline.Name, httpSink); err != nil {
+ return err
+ }
}
+ return nil
+}
- port := stream.Config.HTTPServer.Port
+// registerHTTPSink registers a single HTTP sink
+func (r *HTTPRouter) registerHTTPSink(pipelineName string, httpSink *sink.HTTPSink) error {
+ // Get port from sink configuration
+ stats := httpSink.GetStats()
+ details := stats.Details
+ port := details["port"].(int)
r.mu.Lock()
rs, exists := r.servers[port]
@@ -47,7 +62,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
// Create new server for this port
rs = &routerServer{
port: port,
- routes: make(map[string]*LogStream),
+ routes: make(map[string]*routedSink),
router: r,
startTime: time.Now(),
logger: r.logger,
@@ -56,7 +71,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
Handler: rs.requestHandler,
DisableKeepalive: false,
StreamRequestBody: true,
- CloseOnShutdown: true, // Ensure connections close on shutdown
+ CloseOnShutdown: true,
}
r.servers[port] = rs
@@ -79,54 +94,74 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
}
r.mu.Unlock()
- // Register routes for this transport
+ // Register routes for this sink
rs.routeMu.Lock()
defer rs.routeMu.Unlock()
- // Use transport name as path prefix
- pathPrefix := "/" + stream.Name
+ // Use pipeline name as path prefix
+ pathPrefix := "/" + pipelineName
// Check for conflicts
- for existingPath, existingStream := range rs.routes {
+ for existingPath, existing := range rs.routes {
if strings.HasPrefix(pathPrefix, existingPath) || strings.HasPrefix(existingPath, pathPrefix) {
- return fmt.Errorf("path conflict: '%s' conflicts with existing transport '%s' at '%s'",
- pathPrefix, existingStream.Name, existingPath)
+ return fmt.Errorf("path conflict: '%s' conflicts with existing pipeline '%s' at '%s'",
+ pathPrefix, existing.pipelineName, existingPath)
}
}
- rs.routes[pathPrefix] = stream
- r.logger.Info("msg", "Registered transport route",
+ // Set the sink to router mode
+ httpSink.SetRouterMode()
+
+ rs.routes[pathPrefix] = &routedSink{
+ pipelineName: pipelineName,
+ httpSink: httpSink,
+ }
+
+ r.logger.Info("msg", "Registered pipeline route",
"component", "http_router",
- "transport", stream.Name,
+ "pipeline", pipelineName,
"path", pathPrefix,
"port", port)
return nil
}
+// UnregisterStream is deprecated
func (r *HTTPRouter) UnregisterStream(streamName string) {
+ r.logger.Warn("msg", "UnregisterStream is deprecated",
+ "component", "http_router")
+}
+
+// UnregisterPipeline removes a pipeline's routes
+func (r *HTTPRouter) UnregisterPipeline(pipelineName string) {
r.mu.RLock()
defer r.mu.RUnlock()
for port, rs := range r.servers {
rs.routeMu.Lock()
- for path, stream := range rs.routes {
- if stream.Name == streamName {
+ for path, route := range rs.routes {
+ if route.pipelineName == pipelineName {
delete(rs.routes, path)
- fmt.Printf("[ROUTER] Unregistered transport '%s' from path '%s' on port %d\n",
- streamName, path, port)
+ r.logger.Info("msg", "Unregistered pipeline route",
+ "component", "http_router",
+ "pipeline", pipelineName,
+ "path", path,
+ "port", port)
}
}
// Check if server has no more routes
if len(rs.routes) == 0 {
- fmt.Printf("[ROUTER] No routes left on port %d, considering shutdown\n", port)
+ r.logger.Info("msg", "No routes left on port, considering shutdown",
+ "component", "http_router",
+ "port", port)
}
rs.routeMu.Unlock()
}
}
+// Shutdown stops all router servers
func (r *HTTPRouter) Shutdown() {
- fmt.Println("[ROUTER] Starting router shutdown...")
+ r.logger.Info("msg", "Starting router shutdown...")
r.mu.Lock()
defer r.mu.Unlock()
@@ -136,17 +171,23 @@ func (r *HTTPRouter) Shutdown() {
wg.Add(1)
go func(p int, s *routerServer) {
defer wg.Done()
- fmt.Printf("[ROUTER] Shutting down server on port %d\n", p)
+ r.logger.Info("msg", "Shutting down server",
+ "component", "http_router",
+ "port", p)
if err := s.server.Shutdown(); err != nil {
- fmt.Printf("[ROUTER] Error shutting down server on port %d: %v\n", p, err)
+ r.logger.Error("msg", "Error shutting down server",
+ "component", "http_router",
+ "port", p,
+ "error", err)
}
}(port, rs)
}
wg.Wait()
- fmt.Println("[ROUTER] Router shutdown complete")
+ r.logger.Info("msg", "Router shutdown complete")
}
+// GetStats returns router statistics
func (r *HTTPRouter) GetStats() map[string]any {
r.mu.RLock()
defer r.mu.RUnlock()
diff --git a/src/internal/service/logstream.go b/src/internal/service/logstream.go
deleted file mode 100644
index 0c2f207..0000000
--- a/src/internal/service/logstream.go
+++ /dev/null
@@ -1,210 +0,0 @@
-// FILE: src/internal/service/logstream.go
-package service
-
-import (
- "context"
- "fmt"
- "path/filepath"
- "sync"
- "time"
-
- "logwisp/src/internal/config"
- "logwisp/src/internal/filter"
- "logwisp/src/internal/monitor"
- "logwisp/src/internal/transport"
-
- "github.com/lixenwraith/log"
-)
-
-type LogStream struct {
- Name string
- Config config.StreamConfig
- Monitor monitor.Monitor
- FilterChain *filter.Chain
- TCPServer *transport.TCPStreamer
- HTTPServer *transport.HTTPStreamer
- Stats *StreamStats
- logger *log.Logger
-
- ctx context.Context
- cancel context.CancelFunc
-}
-
-type StreamStats struct {
- StartTime time.Time
- MonitorStats monitor.Stats
- TCPConnections int32
- HTTPConnections int32
- TotalBytesServed uint64
- TotalEntriesServed uint64
- FilterStats map[string]any
-}
-
-func (ls *LogStream) Shutdown() {
- ls.logger.Info("msg", "Shutting down stream",
- "component", "logstream",
- "stream", ls.Name)
-
- // Stop servers first
- var wg sync.WaitGroup
-
- if ls.TCPServer != nil {
- wg.Add(1)
- go func() {
- defer wg.Done()
- ls.TCPServer.Stop()
- }()
- }
-
- if ls.HTTPServer != nil {
- wg.Add(1)
- go func() {
- defer wg.Done()
- ls.HTTPServer.Stop()
- }()
- }
-
- // Cancel context
- ls.cancel()
-
- // Wait for servers
- wg.Wait()
-
- // Stop monitor
- ls.Monitor.Stop()
-
- ls.logger.Info("msg", "Stream shutdown complete",
- "component", "logstream",
- "stream", ls.Name)
-}
-
-func (ls *LogStream) GetStats() map[string]any {
- monStats := ls.Monitor.GetStats()
-
- stats := map[string]any{
- "name": ls.Name,
- "uptime_seconds": int(time.Since(ls.Stats.StartTime).Seconds()),
- "monitor": monStats,
- }
-
- if ls.FilterChain != nil {
- stats["filters"] = ls.FilterChain.GetStats()
- }
-
- if ls.TCPServer != nil {
- currentConnections := ls.TCPServer.GetActiveConnections()
-
- stats["tcp"] = map[string]interface{}{
- "enabled": true,
- "port": ls.Config.TCPServer.Port,
- "connections": currentConnections,
- }
- }
-
- if ls.HTTPServer != nil {
- currentConnections := ls.HTTPServer.GetActiveConnections()
-
- stats["http"] = map[string]interface{}{
- "enabled": true,
- "port": ls.Config.HTTPServer.Port,
- "connections": currentConnections,
- "stream_path": ls.Config.HTTPServer.StreamPath,
- "status_path": ls.Config.HTTPServer.StatusPath,
- }
- }
-
- return stats
-}
-
-func (ls *LogStream) UpdateTargets(targets []config.MonitorTarget) error {
- // Validate new targets first
- validatedTargets := make([]config.MonitorTarget, 0, len(targets))
- for _, target := range targets {
- // Basic validation
- absPath, err := filepath.Abs(target.Path)
- if err != nil {
- ls.logger.Error("msg", "Invalid target path",
- "component", "logstream",
- "stream", ls.Name,
- "path", target.Path,
- "error", err)
- return fmt.Errorf("invalid target path %s: %w", target.Path, err)
- }
- target.Path = absPath
- validatedTargets = append(validatedTargets, target)
- }
-
- // Get current watchers
- oldWatchers := ls.Monitor.GetActiveWatchers()
-
- // Add new targets
- for _, target := range validatedTargets {
- if err := ls.Monitor.AddTarget(target.Path, target.Pattern, target.IsFile); err != nil {
- ls.logger.Error("msg", "Failed to add monitor target - rolling back",
- "component", "logstream",
- "stream", ls.Name,
- "target", target.Path,
- "pattern", target.Pattern,
- "error", err)
- // Rollback: restore old watchers
- for _, watcher := range oldWatchers {
- // Best effort restoration
- ls.Monitor.AddTarget(watcher.Path, "", false)
- }
- return fmt.Errorf("failed to add target %s: %w", target.Path, err)
- }
- }
-
- // Only remove old targets after new ones are successfully added
- for _, watcher := range oldWatchers {
- ls.Monitor.RemoveTarget(watcher.Path)
- }
-
- ls.logger.Info("msg", "Updated monitor targets",
- "component", "logstream",
- "stream", ls.Name,
- "old_count", len(oldWatchers),
- "new_count", len(validatedTargets))
-
- return nil
-}
-
-func (ls *LogStream) startStatsUpdater(ctx context.Context) {
- go func() {
- ticker := time.NewTicker(1 * time.Second)
- defer ticker.Stop()
-
- for {
- select {
- case <-ctx.Done():
- return
- case <-ticker.C:
- // Update cached values
- if ls.TCPServer != nil {
- oldTCP := ls.Stats.TCPConnections
- ls.Stats.TCPConnections = ls.TCPServer.GetActiveConnections()
- if oldTCP != ls.Stats.TCPConnections {
- // This debug should now show changes
- ls.logger.Debug("msg", "TCP connection count changed",
- "component", "logstream",
- "stream", ls.Name,
- "old", oldTCP,
- "new", ls.Stats.TCPConnections)
- }
- }
- if ls.HTTPServer != nil {
- oldHTTP := ls.Stats.HTTPConnections
- ls.Stats.HTTPConnections = ls.HTTPServer.GetActiveConnections()
- if oldHTTP != ls.Stats.HTTPConnections {
- // This debug should now show changes
- ls.logger.Debug("msg", "HTTP connection count changed",
- "component", "logstream",
- "stream", ls.Name,
- "old", oldHTTP,
- "new", ls.Stats.HTTPConnections)
- }
- }
- }
- }
- }()
-}
\ No newline at end of file
diff --git a/src/internal/service/pipeline.go b/src/internal/service/pipeline.go
new file mode 100644
index 0000000..5eaedb8
--- /dev/null
+++ b/src/internal/service/pipeline.go
@@ -0,0 +1,150 @@
+// FILE: src/internal/service/pipeline.go
+package service
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "logwisp/src/internal/config"
+ "logwisp/src/internal/filter"
+ "logwisp/src/internal/sink"
+ "logwisp/src/internal/source"
+
+ "github.com/lixenwraith/log"
+)
+
+// Pipeline manages the flow of data from sources through filters to sinks
+type Pipeline struct {
+ Name string
+ Config config.PipelineConfig
+ Sources []source.Source
+ FilterChain *filter.Chain
+ Sinks []sink.Sink
+ Stats *PipelineStats
+ logger *log.Logger
+
+ ctx context.Context
+ cancel context.CancelFunc
+ wg sync.WaitGroup
+
+ // For HTTP sinks in router mode
+ HTTPSinks []*sink.HTTPSink
+ TCPSinks []*sink.TCPSink
+}
+
+// PipelineStats contains statistics for a pipeline
+type PipelineStats struct {
+ StartTime time.Time
+ TotalEntriesProcessed atomic.Uint64
+ TotalEntriesFiltered atomic.Uint64
+ SourceStats []source.SourceStats
+ SinkStats []sink.SinkStats
+ FilterStats map[string]any
+}
+
+// Shutdown gracefully stops the pipeline
+func (p *Pipeline) Shutdown() {
+ p.logger.Info("msg", "Shutting down pipeline",
+ "component", "pipeline",
+ "pipeline", p.Name)
+
+ // Cancel context to stop processing
+ p.cancel()
+
+ // Stop all sinks first
+ var wg sync.WaitGroup
+ for _, s := range p.Sinks {
+ wg.Add(1)
+ go func(sink sink.Sink) {
+ defer wg.Done()
+ sink.Stop()
+ }(s)
+ }
+ wg.Wait()
+
+ // Stop all sources
+ for _, src := range p.Sources {
+ wg.Add(1)
+ go func(source source.Source) {
+ defer wg.Done()
+ source.Stop()
+ }(src)
+ }
+ wg.Wait()
+
+ // Wait for processing goroutines
+ p.wg.Wait()
+
+ p.logger.Info("msg", "Pipeline shutdown complete",
+ "component", "pipeline",
+ "pipeline", p.Name)
+}
+
+// GetStats returns pipeline statistics
+func (p *Pipeline) GetStats() map[string]any {
+ // Collect source stats
+ sourceStats := make([]map[string]any, len(p.Sources))
+ for i, src := range p.Sources {
+ stats := src.GetStats()
+ sourceStats[i] = map[string]any{
+ "type": stats.Type,
+ "total_entries": stats.TotalEntries,
+ "dropped_entries": stats.DroppedEntries,
+ "start_time": stats.StartTime,
+ "last_entry_time": stats.LastEntryTime,
+ "details": stats.Details,
+ }
+ }
+
+ // Collect sink stats
+ sinkStats := make([]map[string]any, len(p.Sinks))
+ for i, s := range p.Sinks {
+ stats := s.GetStats()
+ sinkStats[i] = map[string]any{
+ "type": stats.Type,
+ "total_processed": stats.TotalProcessed,
+ "active_connections": stats.ActiveConnections,
+ "start_time": stats.StartTime,
+ "last_processed": stats.LastProcessed,
+ "details": stats.Details,
+ }
+ }
+
+ // Collect filter stats
+ var filterStats map[string]any
+ if p.FilterChain != nil {
+ filterStats = p.FilterChain.GetStats()
+ }
+
+ return map[string]any{
+ "name": p.Name,
+ "uptime_seconds": int(time.Since(p.Stats.StartTime).Seconds()),
+ "total_processed": p.Stats.TotalEntriesProcessed.Load(),
+ "total_filtered": p.Stats.TotalEntriesFiltered.Load(),
+ "sources": sourceStats,
+ "sinks": sinkStats,
+ "filters": filterStats,
+ "source_count": len(p.Sources),
+ "sink_count": len(p.Sinks),
+ "filter_count": len(p.Config.Filters),
+ }
+}
+
+// startStatsUpdater runs periodic stats updates
+func (p *Pipeline) startStatsUpdater(ctx context.Context) {
+ go func() {
+ ticker := time.NewTicker(1 * time.Second)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ // Periodic stats updates if needed
+ }
+ }
+ }()
+}
\ No newline at end of file
diff --git a/src/internal/service/routerserver.go b/src/internal/service/routerserver.go
index 58dabcd..9e6e273 100644
--- a/src/internal/service/routerserver.go
+++ b/src/internal/service/routerserver.go
@@ -9,17 +9,25 @@ import (
"sync/atomic"
"time"
+ "logwisp/src/internal/sink"
"logwisp/src/internal/version"
"github.com/lixenwraith/log"
"github.com/valyala/fasthttp"
)
+// routedSink represents a sink registered with the router
+type routedSink struct {
+ pipelineName string
+ httpSink *sink.HTTPSink
+}
+
+// routerServer handles HTTP requests for a specific port
type routerServer struct {
port int
server *fasthttp.Server
logger *log.Logger
- routes map[string]*LogStream // path prefix -> transport
+ routes map[string]*routedSink // path prefix -> sink
routeMu sync.RWMutex
router *HTTPRouter
startTime time.Time
@@ -36,7 +44,7 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
// Log request for debugging
rs.logger.Debug("msg", "Router request",
"component", "router_server",
- "method", ctx.Method(),
+ "method", string(ctx.Method()),
"path", path,
"remote_addr", remoteAddr)
@@ -46,18 +54,18 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
return
}
- // Find matching transport
+ // Find matching route
rs.routeMu.RLock()
- var matchedStream *LogStream
+ var matchedSink *routedSink
var matchedPrefix string
var remainingPath string
- for prefix, stream := range rs.routes {
+ for prefix, route := range rs.routes {
if strings.HasPrefix(path, prefix) {
// Use longest prefix match
if len(prefix) > len(matchedPrefix) {
matchedPrefix = prefix
- matchedStream = stream
+ matchedSink = route
remainingPath = strings.TrimPrefix(path, prefix)
// Ensure remaining path starts with / or is empty
if remainingPath != "" && !strings.HasPrefix(remainingPath, "/") {
@@ -68,7 +76,7 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
}
rs.routeMu.RUnlock()
- if matchedStream == nil {
+ if matchedSink == nil {
rs.router.failedRequests.Add(1)
rs.handleNotFound(ctx)
return
@@ -76,25 +84,25 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
rs.router.routedRequests.Add(1)
- // Route to transport's handler
- if matchedStream.HTTPServer != nil {
+ // Route to sink's handler
+ if matchedSink.httpSink != nil {
// Save original path
originalPath := string(ctx.URI().Path())
- // Rewrite path to remove transport prefix
+ // Rewrite path to remove pipeline prefix
if remainingPath == "" {
- // Default to transport path if no remaining path
- remainingPath = matchedStream.Config.HTTPServer.StreamPath
+ // Default to stream path if no remaining path
+ remainingPath = matchedSink.httpSink.GetStreamPath()
}
- rs.logger.Debug("msg", "Routing request to transport",
+ rs.logger.Debug("msg", "Routing request to pipeline",
"component", "router_server",
- "transport", matchedStream.Name,
+ "pipeline", matchedSink.pipelineName,
"original_path", originalPath,
"remaining_path", remainingPath)
ctx.URI().SetPath(remainingPath)
- matchedStream.HTTPServer.RouteRequest(ctx)
+ matchedSink.httpSink.RouteRequest(ctx)
// Restore original path
ctx.URI().SetPath(originalPath)
@@ -102,8 +110,8 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
ctx.SetStatusCode(fasthttp.StatusServiceUnavailable)
ctx.SetContentType("application/json")
json.NewEncoder(ctx).Encode(map[string]string{
- "error": "Stream HTTP server not available",
- "transport": matchedStream.Name,
+ "error": "Pipeline HTTP sink not available",
+ "pipeline": matchedSink.pipelineName,
})
}
}
@@ -112,20 +120,26 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
ctx.SetContentType("application/json")
rs.routeMu.RLock()
- streams := make(map[string]any)
- for prefix, stream := range rs.routes {
- streamStats := stream.GetStats()
-
- // Add routing information
- streamStats["routing"] = map[string]any{
+ pipelines := make(map[string]any)
+ for prefix, route := range rs.routes {
+ pipelineInfo := map[string]any{
"path_prefix": prefix,
"endpoints": map[string]string{
- "transport": prefix + stream.Config.HTTPServer.StreamPath,
- "status": prefix + stream.Config.HTTPServer.StatusPath,
+ "stream": prefix + route.httpSink.GetStreamPath(),
+ "status": prefix + route.httpSink.GetStatusPath(),
},
}
- streams[stream.Name] = streamStats
+ // Get sink stats
+ sinkStats := route.httpSink.GetStats()
+ pipelineInfo["sink"] = map[string]any{
+ "type": sinkStats.Type,
+ "total_processed": sinkStats.TotalProcessed,
+ "active_connections": sinkStats.ActiveConnections,
+ "details": sinkStats.Details,
+ }
+
+ pipelines[route.pipelineName] = pipelineInfo
}
rs.routeMu.RUnlock()
@@ -133,12 +147,12 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
routerStats := rs.router.GetStats()
status := map[string]any{
- "service": "LogWisp Router",
- "version": version.String(),
- "port": rs.port,
- "streams": streams,
- "total_streams": len(streams),
- "router": routerStats,
+ "service": "LogWisp Router",
+ "version": version.String(),
+ "port": rs.port,
+ "pipelines": pipelines,
+ "total_pipelines": len(pipelines),
+ "router": routerStats,
"endpoints": map[string]string{
"global_status": "/status",
},
@@ -156,11 +170,11 @@ func (rs *routerServer) handleNotFound(ctx *fasthttp.RequestCtx) {
availableRoutes := make([]string, 0, len(rs.routes)*2+1)
availableRoutes = append(availableRoutes, "/status (global status)")
- for prefix, stream := range rs.routes {
- if stream.Config.HTTPServer != nil {
+ for prefix, route := range rs.routes {
+ if route.httpSink != nil {
availableRoutes = append(availableRoutes,
- fmt.Sprintf("%s%s (transport: %s)", prefix, stream.Config.HTTPServer.StreamPath, stream.Name),
- fmt.Sprintf("%s%s (status: %s)", prefix, stream.Config.HTTPServer.StatusPath, stream.Name),
+ fmt.Sprintf("%s%s (stream: %s)", prefix, route.httpSink.GetStreamPath(), route.pipelineName),
+ fmt.Sprintf("%s%s (status: %s)", prefix, route.httpSink.GetStatusPath(), route.pipelineName),
)
}
}
diff --git a/src/internal/service/service.go b/src/internal/service/service.go
index 82fc75e..70b6387 100644
--- a/src/internal/service/service.go
+++ b/src/internal/service/service.go
@@ -9,254 +9,285 @@ import (
"logwisp/src/internal/config"
"logwisp/src/internal/filter"
- "logwisp/src/internal/monitor"
- "logwisp/src/internal/transport"
+ "logwisp/src/internal/sink"
+ "logwisp/src/internal/source"
"github.com/lixenwraith/log"
)
+// Service manages multiple pipelines
type Service struct {
- streams map[string]*LogStream
- mu sync.RWMutex
- ctx context.Context
- cancel context.CancelFunc
- wg sync.WaitGroup
- logger *log.Logger
+ pipelines map[string]*Pipeline
+ mu sync.RWMutex
+ ctx context.Context
+ cancel context.CancelFunc
+ wg sync.WaitGroup
+ logger *log.Logger
}
+// New creates a new service
func New(ctx context.Context, logger *log.Logger) *Service {
serviceCtx, cancel := context.WithCancel(ctx)
return &Service{
- streams: make(map[string]*LogStream),
- ctx: serviceCtx,
- cancel: cancel,
- logger: logger,
+ pipelines: make(map[string]*Pipeline),
+ ctx: serviceCtx,
+ cancel: cancel,
+ logger: logger,
}
}
-func (s *Service) CreateStream(cfg config.StreamConfig) error {
+// NewPipeline creates and starts a new pipeline
+func (s *Service) NewPipeline(cfg config.PipelineConfig) error {
s.mu.Lock()
defer s.mu.Unlock()
- if _, exists := s.streams[cfg.Name]; exists {
- err := fmt.Errorf("transport '%s' already exists", cfg.Name)
- s.logger.Error("msg", "Failed to create stream - duplicate name",
+ if _, exists := s.pipelines[cfg.Name]; exists {
+ err := fmt.Errorf("pipeline '%s' already exists", cfg.Name)
+ s.logger.Error("msg", "Failed to create pipeline - duplicate name",
"component", "service",
- "stream", cfg.Name,
+ "pipeline", cfg.Name,
"error", err)
return err
}
- s.logger.Debug("msg", "Creating stream", "stream", cfg.Name)
+ s.logger.Debug("msg", "Creating pipeline", "pipeline", cfg.Name)
- // Create transport context
- streamCtx, streamCancel := context.WithCancel(s.ctx)
+ // Create pipeline context
+ pipelineCtx, pipelineCancel := context.WithCancel(s.ctx)
- // Create monitor - pass the service logger directly
- mon := monitor.New(s.logger)
- mon.SetCheckInterval(time.Duration(cfg.GetCheckInterval(100)) * time.Millisecond)
-
- // Add targets
- for _, target := range cfg.GetTargets(nil) {
- if err := mon.AddTarget(target.Path, target.Pattern, target.IsFile); err != nil {
- streamCancel()
- return fmt.Errorf("failed to add target %s: %w", target.Path, err)
- }
+ // Create pipeline instance
+ pipeline := &Pipeline{
+ Name: cfg.Name,
+ Config: cfg,
+ Stats: &PipelineStats{
+ StartTime: time.Now(),
+ },
+ ctx: pipelineCtx,
+ cancel: pipelineCancel,
+ logger: s.logger,
}
- // Start monitor
- if err := mon.Start(streamCtx); err != nil {
- streamCancel()
- s.logger.Error("msg", "Failed to start monitor",
- "component", "service",
- "stream", cfg.Name,
- "error", err)
- return fmt.Errorf("failed to start monitor: %w", err)
+ // Create sources
+ for i, srcCfg := range cfg.Sources {
+ src, err := s.createSource(srcCfg)
+ if err != nil {
+ pipelineCancel()
+ return fmt.Errorf("failed to create source[%d]: %w", i, err)
+ }
+ pipeline.Sources = append(pipeline.Sources, src)
}
// Create filter chain
- var filterChain *filter.Chain
if len(cfg.Filters) > 0 {
chain, err := filter.NewChain(cfg.Filters, s.logger)
if err != nil {
- streamCancel()
- s.logger.Error("msg", "Failed to create filter chain",
- "component", "service",
- "stream", cfg.Name,
- "filter_count", len(cfg.Filters),
- "error", err)
+ pipelineCancel()
return fmt.Errorf("failed to create filter chain: %w", err)
}
- filterChain = chain
+ pipeline.FilterChain = chain
}
- // Create log transport
- ls := &LogStream{
- Name: cfg.Name,
- Config: cfg,
- Monitor: mon,
- FilterChain: filterChain,
- Stats: &StreamStats{
- StartTime: time.Now(),
- },
- ctx: streamCtx,
- cancel: streamCancel,
- logger: s.logger, // Use parent logger
- }
+ // Create sinks
+ for i, sinkCfg := range cfg.Sinks {
+ sinkInst, err := s.createSink(sinkCfg)
+ if err != nil {
+ pipelineCancel()
+ return fmt.Errorf("failed to create sink[%d]: %w", i, err)
+ }
+ pipeline.Sinks = append(pipeline.Sinks, sinkInst)
- // Start TCP server if configured
- if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
- // Create filtered channel
- rawChan := mon.Subscribe()
- tcpChan := make(chan monitor.LogEntry, cfg.TCPServer.BufferSize)
-
- // Start filter goroutine for TCP
- s.wg.Add(1)
- go func() {
- defer s.wg.Done()
- defer close(tcpChan)
- s.filterLoop(streamCtx, rawChan, tcpChan, filterChain)
- }()
-
- ls.TCPServer = transport.NewTCPStreamer(
- tcpChan,
- *cfg.TCPServer,
- s.logger) // Pass parent logger
-
- if err := s.startTCPServer(ls); err != nil {
- ls.Shutdown()
- s.logger.Error("msg", "Failed to start TCP server",
- "component", "service",
- "stream", cfg.Name,
- "port", cfg.TCPServer.Port,
- "error", err)
- return fmt.Errorf("TCP server failed: %w", err)
+ // Track HTTP/TCP sinks for router mode
+ switch s := sinkInst.(type) {
+ case *sink.HTTPSink:
+ pipeline.HTTPSinks = append(pipeline.HTTPSinks, s)
+ case *sink.TCPSink:
+ pipeline.TCPSinks = append(pipeline.TCPSinks, s)
}
}
- // Start HTTP server if configured
- if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
- // Create filtered channel
- rawChan := mon.Subscribe()
- httpChan := make(chan monitor.LogEntry, cfg.HTTPServer.BufferSize)
-
- // Start filter goroutine for HTTP
- s.wg.Add(1)
- go func() {
- defer s.wg.Done()
- defer close(httpChan)
- s.filterLoop(streamCtx, rawChan, httpChan, filterChain)
- }()
-
- ls.HTTPServer = transport.NewHTTPStreamer(
- httpChan,
- *cfg.HTTPServer,
- s.logger) // Pass parent logger
-
- if err := s.startHTTPServer(ls); err != nil {
- ls.Shutdown()
- s.logger.Error("msg", "Failed to start HTTP server",
- "component", "service",
- "stream", cfg.Name,
- "port", cfg.HTTPServer.Port,
- "error", err)
- return fmt.Errorf("HTTP server failed: %w", err)
+ // Start all sources
+ for i, src := range pipeline.Sources {
+ if err := src.Start(); err != nil {
+ pipeline.Shutdown()
+ return fmt.Errorf("failed to start source[%d]: %w", i, err)
}
}
- ls.startStatsUpdater(streamCtx)
+ // Start all sinks
+ for i, sinkInst := range pipeline.Sinks {
+ if err := sinkInst.Start(pipelineCtx); err != nil {
+ pipeline.Shutdown()
+ return fmt.Errorf("failed to start sink[%d]: %w", i, err)
+ }
+ }
- s.streams[cfg.Name] = ls
- s.logger.Info("msg", "Stream created successfully", "stream", cfg.Name)
+ // Wire sources to sinks through filters
+ s.wirePipeline(pipeline)
+
+ // Start stats updater
+ pipeline.startStatsUpdater(pipelineCtx)
+
+ s.pipelines[cfg.Name] = pipeline
+ s.logger.Info("msg", "Pipeline created successfully", "pipeline", cfg.Name)
return nil
}
-// filterLoop applies filters to log entries
-func (s *Service) filterLoop(ctx context.Context, in <-chan monitor.LogEntry, out chan<- monitor.LogEntry, chain *filter.Chain) {
- for {
- select {
- case <-ctx.Done():
- return
- case entry, ok := <-in:
- if !ok {
- return
- }
+// wirePipeline connects sources to sinks through filters
+func (s *Service) wirePipeline(p *Pipeline) {
+ // For each source, subscribe and process entries
+ for _, src := range p.Sources {
+ srcChan := src.Subscribe()
- // Apply filter chain if configured
- if chain == nil || chain.Apply(entry) {
+ // Create a processing goroutine for this source
+ p.wg.Add(1)
+ go func(source source.Source, entries <-chan source.LogEntry) {
+ defer p.wg.Done()
+
+ for {
select {
- case out <- entry:
- case <-ctx.Done():
+ case <-p.ctx.Done():
return
- default:
- // Drop if output buffer is full
- s.logger.Debug("msg", "Dropped log entry - buffer full")
+ case entry, ok := <-entries:
+ if !ok {
+ return
+ }
+
+ p.Stats.TotalEntriesProcessed.Add(1)
+
+ // Apply filters if configured
+ if p.FilterChain != nil {
+ if !p.FilterChain.Apply(entry) {
+ p.Stats.TotalEntriesFiltered.Add(1)
+ continue
+ }
+ }
+
+ // Send to all sinks
+ for _, sinkInst := range p.Sinks {
+ select {
+ case sinkInst.Input() <- entry:
+ case <-p.ctx.Done():
+ return
+ default:
+ // Drop if sink buffer is full
+ s.logger.Debug("msg", "Dropped log entry - sink buffer full",
+ "pipeline", p.Name)
+ }
+ }
}
}
- }
+ }(src, srcChan)
}
}
-func (s *Service) GetStream(name string) (*LogStream, error) {
+// createSource creates a source instance based on configuration
+func (s *Service) createSource(cfg config.SourceConfig) (source.Source, error) {
+ switch cfg.Type {
+ case "directory":
+ return source.NewDirectorySource(cfg.Options, s.logger)
+ case "stdin":
+ return source.NewStdinSource(cfg.Options, s.logger)
+ default:
+ return nil, fmt.Errorf("unknown source type: %s", cfg.Type)
+ }
+}
+
+// createSink creates a sink instance based on configuration
+func (s *Service) createSink(cfg config.SinkConfig) (sink.Sink, error) {
+ switch cfg.Type {
+ case "http":
+ return sink.NewHTTPSink(cfg.Options, s.logger)
+ case "tcp":
+ return sink.NewTCPSink(cfg.Options, s.logger)
+ case "file":
+ return sink.NewFileSink(cfg.Options, s.logger)
+ case "stdout":
+ return sink.NewStdoutSink(cfg.Options, s.logger)
+ case "stderr":
+ return sink.NewStderrSink(cfg.Options, s.logger)
+ default:
+ return nil, fmt.Errorf("unknown sink type: %s", cfg.Type)
+ }
+}
+
+// GetPipeline returns a pipeline by name
+func (s *Service) GetPipeline(name string) (*Pipeline, error) {
s.mu.RLock()
defer s.mu.RUnlock()
- stream, exists := s.streams[name]
+ pipeline, exists := s.pipelines[name]
if !exists {
- return nil, fmt.Errorf("transport '%s' not found", name)
+ return nil, fmt.Errorf("pipeline '%s' not found", name)
}
- return stream, nil
+ return pipeline, nil
}
+// ListStreams is deprecated, use ListPipelines
func (s *Service) ListStreams() []string {
+ s.logger.Warn("msg", "ListStreams is deprecated, use ListPipelines",
+ "component", "service")
+ return s.ListPipelines()
+}
+
+// ListPipelines returns all pipeline names
+func (s *Service) ListPipelines() []string {
s.mu.RLock()
defer s.mu.RUnlock()
- names := make([]string, 0, len(s.streams))
- for name := range s.streams {
+ names := make([]string, 0, len(s.pipelines))
+ for name := range s.pipelines {
names = append(names, name)
}
return names
}
+// RemoveStream is deprecated, use RemovePipeline
func (s *Service) RemoveStream(name string) error {
+ s.logger.Warn("msg", "RemoveStream is deprecated, use RemovePipeline",
+ "component", "service")
+ return s.RemovePipeline(name)
+}
+
+// RemovePipeline stops and removes a pipeline
+func (s *Service) RemovePipeline(name string) error {
s.mu.Lock()
defer s.mu.Unlock()
- stream, exists := s.streams[name]
+ pipeline, exists := s.pipelines[name]
if !exists {
- err := fmt.Errorf("transport '%s' not found", name)
- s.logger.Warn("msg", "Cannot remove non-existent stream",
+ err := fmt.Errorf("pipeline '%s' not found", name)
+ s.logger.Warn("msg", "Cannot remove non-existent pipeline",
"component", "service",
- "stream", name,
+ "pipeline", name,
"error", err)
return err
}
- s.logger.Info("msg", "Removing stream", "stream", name)
- stream.Shutdown()
- delete(s.streams, name)
+ s.logger.Info("msg", "Removing pipeline", "pipeline", name)
+ pipeline.Shutdown()
+ delete(s.pipelines, name)
return nil
}
+// Shutdown stops all pipelines
func (s *Service) Shutdown() {
s.logger.Info("msg", "Service shutdown initiated")
s.mu.Lock()
- streams := make([]*LogStream, 0, len(s.streams))
- for _, stream := range s.streams {
- streams = append(streams, stream)
+ pipelines := make([]*Pipeline, 0, len(s.pipelines))
+ for _, pipeline := range s.pipelines {
+ pipelines = append(pipelines, pipeline)
}
s.mu.Unlock()
- // Stop all streams concurrently
+ // Stop all pipelines concurrently
var wg sync.WaitGroup
- for _, stream := range streams {
+ for _, pipeline := range pipelines {
wg.Add(1)
- go func(ls *LogStream) {
+ go func(p *Pipeline) {
defer wg.Done()
- ls.Shutdown()
- }(stream)
+ p.Shutdown()
+ }(pipeline)
}
wg.Wait()
@@ -266,68 +297,19 @@ func (s *Service) Shutdown() {
s.logger.Info("msg", "Service shutdown complete")
}
+// GetGlobalStats returns statistics for all pipelines
func (s *Service) GetGlobalStats() map[string]any {
s.mu.RLock()
defer s.mu.RUnlock()
stats := map[string]any{
- "streams": make(map[string]any),
- "total_streams": len(s.streams),
+ "pipelines": make(map[string]any),
+ "total_pipelines": len(s.pipelines),
}
- for name, stream := range s.streams {
- stats["streams"].(map[string]any)[name] = stream.GetStats()
+ for name, pipeline := range s.pipelines {
+ stats["pipelines"].(map[string]any)[name] = pipeline.GetStats()
}
return stats
-}
-
-func (s *Service) startTCPServer(ls *LogStream) error {
- errChan := make(chan error, 1)
- s.wg.Add(1)
-
- go func() {
- defer s.wg.Done()
- if err := ls.TCPServer.Start(); err != nil {
- errChan <- err
- }
- }()
-
- // Check startup
- select {
- case err := <-errChan:
- s.logger.Error("msg", "TCP server startup failed immediately",
- "component", "service",
- "stream", ls.Name,
- "error", err)
- return err
- case <-time.After(time.Second):
- s.logger.Debug("msg", "TCP server started", "stream", ls.Name)
- return nil
- }
-}
-
-func (s *Service) startHTTPServer(ls *LogStream) error {
- errChan := make(chan error, 1)
- s.wg.Add(1)
-
- go func() {
- defer s.wg.Done()
- if err := ls.HTTPServer.Start(); err != nil {
- errChan <- err
- }
- }()
-
- // Check startup
- select {
- case err := <-errChan:
- s.logger.Error("msg", "HTTP server startup failed immediately",
- "component", "service",
- "stream", ls.Name,
- "error", err)
- return err
- case <-time.After(time.Second):
- s.logger.Debug("msg", "HTTP server started", "stream", ls.Name)
- return nil
- }
}
\ No newline at end of file
diff --git a/src/internal/sink/console.go b/src/internal/sink/console.go
new file mode 100644
index 0000000..e4d7c86
--- /dev/null
+++ b/src/internal/sink/console.go
@@ -0,0 +1,215 @@
+// FILE: src/internal/sink/console.go
+package sink
+
+import (
+ "context"
+ "fmt"
+ "sync/atomic"
+ "time"
+
+ "logwisp/src/internal/source"
+
+ "github.com/lixenwraith/log"
+)
+
+// StdoutSink writes log entries to stdout
+type StdoutSink struct {
+ input chan source.LogEntry
+ writer *log.Logger
+ done chan struct{}
+ startTime time.Time
+ logger *log.Logger
+
+ // Statistics
+ totalProcessed atomic.Uint64
+ lastProcessed atomic.Value // time.Time
+}
+
+// NewStdoutSink creates a new stdout sink
+func NewStdoutSink(options map[string]any, logger *log.Logger) (*StdoutSink, error) {
+ // Create internal logger for stdout writing
+ writer := log.NewLogger()
+ if err := writer.InitWithDefaults(
+ "enable_stdout=true",
+ "disable_file=true",
+ "stdout_target=stdout",
+ "show_timestamp=false", // We format our own
+ "show_level=false", // We format our own
+ ); err != nil {
+ return nil, fmt.Errorf("failed to initialize stdout writer: %w", err)
+ }
+
+ bufferSize := 1000
+ if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
+ bufferSize = bufSize
+ }
+
+ s := &StdoutSink{
+ input: make(chan source.LogEntry, bufferSize),
+ writer: writer,
+ done: make(chan struct{}),
+ startTime: time.Now(),
+ logger: logger,
+ }
+ s.lastProcessed.Store(time.Time{})
+
+ return s, nil
+}
+
+func (s *StdoutSink) Input() chan<- source.LogEntry {
+ return s.input
+}
+
+func (s *StdoutSink) Start(ctx context.Context) error {
+ go s.processLoop(ctx)
+ s.logger.Info("msg", "Stdout sink started", "component", "stdout_sink")
+ return nil
+}
+
+func (s *StdoutSink) Stop() {
+ s.logger.Info("msg", "Stopping stdout sink")
+ close(s.done)
+ s.writer.Shutdown(1 * time.Second)
+ s.logger.Info("msg", "Stdout sink stopped")
+}
+
+func (s *StdoutSink) GetStats() SinkStats {
+ lastProc, _ := s.lastProcessed.Load().(time.Time)
+
+ return SinkStats{
+ Type: "stdout",
+ TotalProcessed: s.totalProcessed.Load(),
+ StartTime: s.startTime,
+ LastProcessed: lastProc,
+ Details: map[string]any{},
+ }
+}
+
+func (s *StdoutSink) processLoop(ctx context.Context) {
+ for {
+ select {
+ case entry, ok := <-s.input:
+ if !ok {
+ return
+ }
+
+ s.totalProcessed.Add(1)
+ s.lastProcessed.Store(time.Now())
+
+ // Format and write
+ timestamp := entry.Time.Format(time.RFC3339Nano)
+ level := entry.Level
+ if level == "" {
+ level = "INFO"
+ }
+
+ s.writer.Message(fmt.Sprintf("[%s] %s %s", timestamp, level, entry.Message))
+
+ case <-ctx.Done():
+ return
+ case <-s.done:
+ return
+ }
+ }
+}
+
+// StderrSink writes log entries to stderr
+type StderrSink struct {
+ input chan source.LogEntry
+ writer *log.Logger
+ done chan struct{}
+ startTime time.Time
+ logger *log.Logger
+
+ // Statistics
+ totalProcessed atomic.Uint64
+ lastProcessed atomic.Value // time.Time
+}
+
+// NewStderrSink creates a new stderr sink
+func NewStderrSink(options map[string]any, logger *log.Logger) (*StderrSink, error) {
+ // Create internal logger for stderr writing
+ writer := log.NewLogger()
+ if err := writer.InitWithDefaults(
+ "enable_stdout=true",
+ "disable_file=true",
+ "stdout_target=stderr",
+ "show_timestamp=false", // We format our own
+ "show_level=false", // We format our own
+ ); err != nil {
+ return nil, fmt.Errorf("failed to initialize stderr writer: %w", err)
+ }
+
+ bufferSize := 1000
+ if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
+ bufferSize = bufSize
+ }
+
+ s := &StderrSink{
+ input: make(chan source.LogEntry, bufferSize),
+ writer: writer,
+ done: make(chan struct{}),
+ startTime: time.Now(),
+ logger: logger,
+ }
+ s.lastProcessed.Store(time.Time{})
+
+ return s, nil
+}
+
+func (s *StderrSink) Input() chan<- source.LogEntry {
+ return s.input
+}
+
+func (s *StderrSink) Start(ctx context.Context) error {
+ go s.processLoop(ctx)
+ s.logger.Info("msg", "Stderr sink started", "component", "stderr_sink")
+ return nil
+}
+
+func (s *StderrSink) Stop() {
+ s.logger.Info("msg", "Stopping stderr sink")
+ close(s.done)
+ s.writer.Shutdown(1 * time.Second)
+ s.logger.Info("msg", "Stderr sink stopped")
+}
+
+func (s *StderrSink) GetStats() SinkStats {
+ lastProc, _ := s.lastProcessed.Load().(time.Time)
+
+ return SinkStats{
+ Type: "stderr",
+ TotalProcessed: s.totalProcessed.Load(),
+ StartTime: s.startTime,
+ LastProcessed: lastProc,
+ Details: map[string]any{},
+ }
+}
+
+func (s *StderrSink) processLoop(ctx context.Context) {
+ for {
+ select {
+ case entry, ok := <-s.input:
+ if !ok {
+ return
+ }
+
+ s.totalProcessed.Add(1)
+ s.lastProcessed.Store(time.Now())
+
+ // Format and write
+ timestamp := entry.Time.Format(time.RFC3339Nano)
+ level := entry.Level
+ if level == "" {
+ level = "INFO"
+ }
+
+ s.writer.Message(fmt.Sprintf("[%s] %s %s", timestamp, level, entry.Message))
+
+ case <-ctx.Done():
+ return
+ case <-s.done:
+ return
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/internal/sink/file.go b/src/internal/sink/file.go
new file mode 100644
index 0000000..a8d5ee2
--- /dev/null
+++ b/src/internal/sink/file.go
@@ -0,0 +1,155 @@
+// FILE: src/internal/sink/file.go
+package sink
+
+import (
+ "context"
+ "fmt"
+ "sync/atomic"
+ "time"
+
+ "logwisp/src/internal/source"
+
+ "github.com/lixenwraith/log"
+)
+
+// FileSink writes log entries to files with rotation
+type FileSink struct {
+ input chan source.LogEntry
+ writer *log.Logger // Internal logger instance for file writing
+ done chan struct{}
+ startTime time.Time
+ logger *log.Logger // Application logger
+
+ // Statistics
+ totalProcessed atomic.Uint64
+ lastProcessed atomic.Value // time.Time
+}
+
+// NewFileSink creates a new file sink
+func NewFileSink(options map[string]any, logger *log.Logger) (*FileSink, error) {
+ directory, ok := options["directory"].(string)
+ if !ok || directory == "" {
+ return nil, fmt.Errorf("file sink requires 'directory' option")
+ }
+
+ name, ok := options["name"].(string)
+ if !ok || name == "" {
+ return nil, fmt.Errorf("file sink requires 'name' option")
+ }
+
+ // Create configuration for the internal log writer
+ var configArgs []string
+ configArgs = append(configArgs,
+ fmt.Sprintf("directory=%s", directory),
+ fmt.Sprintf("name=%s", name),
+ "enable_stdout=false", // File only
+ "show_timestamp=false", // We already have timestamps in entries
+ "show_level=false", // We already have levels in entries
+ )
+
+ // Add optional configurations
+ if maxSize, ok := toInt(options["max_size_mb"]); ok && maxSize > 0 {
+ configArgs = append(configArgs, fmt.Sprintf("max_size_mb=%d", maxSize))
+ }
+
+ if maxTotalSize, ok := toInt(options["max_total_size_mb"]); ok && maxTotalSize >= 0 {
+ configArgs = append(configArgs, fmt.Sprintf("max_total_size_mb=%d", maxTotalSize))
+ }
+
+ if retention, ok := toFloat(options["retention_hours"]); ok && retention > 0 {
+ configArgs = append(configArgs, fmt.Sprintf("retention_period_hrs=%.1f", retention))
+ }
+
+ if minDiskFree, ok := toInt(options["min_disk_free_mb"]); ok && minDiskFree > 0 {
+ configArgs = append(configArgs, fmt.Sprintf("min_disk_free_mb=%d", minDiskFree))
+ }
+
+ // Create internal logger for file writing
+ writer := log.NewLogger()
+ if err := writer.InitWithDefaults(configArgs...); err != nil {
+ return nil, fmt.Errorf("failed to initialize file writer: %w", err)
+ }
+
+ // Buffer size for input channel
+ bufferSize := 1000
+ if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
+ bufferSize = bufSize
+ }
+
+ fs := &FileSink{
+ input: make(chan source.LogEntry, bufferSize),
+ writer: writer,
+ done: make(chan struct{}),
+ startTime: time.Now(),
+ logger: logger,
+ }
+ fs.lastProcessed.Store(time.Time{})
+
+ return fs, nil
+}
+
+func (fs *FileSink) Input() chan<- source.LogEntry {
+ return fs.input
+}
+
+func (fs *FileSink) Start(ctx context.Context) error {
+ go fs.processLoop(ctx)
+ fs.logger.Info("msg", "File sink started", "component", "file_sink")
+ return nil
+}
+
+func (fs *FileSink) Stop() {
+ fs.logger.Info("msg", "Stopping file sink")
+ close(fs.done)
+
+ // Shutdown the writer with timeout
+ if err := fs.writer.Shutdown(2 * time.Second); err != nil {
+ fs.logger.Error("msg", "Error shutting down file writer",
+ "component", "file_sink",
+ "error", err)
+ }
+
+ fs.logger.Info("msg", "File sink stopped")
+}
+
+func (fs *FileSink) GetStats() SinkStats {
+ lastProc, _ := fs.lastProcessed.Load().(time.Time)
+
+ return SinkStats{
+ Type: "file",
+ TotalProcessed: fs.totalProcessed.Load(),
+ StartTime: fs.startTime,
+ LastProcessed: lastProc,
+ Details: map[string]any{},
+ }
+}
+
+func (fs *FileSink) processLoop(ctx context.Context) {
+ for {
+ select {
+ case entry, ok := <-fs.input:
+ if !ok {
+ return
+ }
+
+ fs.totalProcessed.Add(1)
+ fs.lastProcessed.Store(time.Now())
+
+ // Format the log entry
+ // Include timestamp and level since we disabled them in the writer
+ timestamp := entry.Time.Format(time.RFC3339Nano)
+ level := entry.Level
+ if level == "" {
+ level = "INFO"
+ }
+
+ // Write to file using the internal logger
+ fs.writer.Message(fmt.Sprintf("[%s] %s %s", timestamp, level, entry.Message))
+
+ case <-ctx.Done():
+ return
+ case <-fs.done:
+ return
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/internal/transport/httpstreamer.go b/src/internal/sink/http.go
similarity index 62%
rename from src/internal/transport/httpstreamer.go
rename to src/internal/sink/http.go
index 3984ae5..8cec159 100644
--- a/src/internal/transport/httpstreamer.go
+++ b/src/internal/sink/http.go
@@ -1,5 +1,5 @@
-// FILE: src/internal/transport/httpstreamer.go
-package transport
+// FILE: src/internal/sink/http.go
+package sink
import (
"bufio"
@@ -12,8 +12,8 @@ import (
"time"
"logwisp/src/internal/config"
- "logwisp/src/internal/monitor"
"logwisp/src/internal/ratelimit"
+ "logwisp/src/internal/source"
"logwisp/src/internal/version"
"github.com/lixenwraith/log"
@@ -21,9 +21,10 @@ import (
"github.com/valyala/fasthttp"
)
-type HTTPStreamer struct {
- logChan chan monitor.LogEntry
- config config.HTTPConfig
+// HTTPSink streams log entries via Server-Sent Events
+type HTTPSink struct {
+ input chan source.LogEntry
+ config HTTPConfig
server *fasthttp.Server
activeClients atomic.Int32
mu sync.RWMutex
@@ -41,50 +42,115 @@ type HTTPStreamer struct {
// Rate limiting
rateLimiter *ratelimit.Limiter
+
+ // Statistics
+ totalProcessed atomic.Uint64
+ lastProcessed atomic.Value // time.Time
}
-func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig, logger *log.Logger) *HTTPStreamer {
- // Set default paths if not configured
- streamPath := cfg.StreamPath
- if streamPath == "" {
- streamPath = "/transport"
- }
- statusPath := cfg.StatusPath
- if statusPath == "" {
- statusPath = "/status"
+// HTTPConfig holds HTTP sink configuration
+type HTTPConfig struct {
+ Port int
+ BufferSize int
+ StreamPath string
+ StatusPath string
+ Heartbeat config.HeartbeatConfig
+ SSL *config.SSLConfig
+ RateLimit *config.RateLimitConfig
+}
+
+// NewHTTPSink creates a new HTTP streaming sink
+func NewHTTPSink(options map[string]any, logger *log.Logger) (*HTTPSink, error) {
+ cfg := HTTPConfig{
+ Port: 8080,
+ BufferSize: 1000,
+ StreamPath: "/transport",
+ StatusPath: "/status",
}
- h := &HTTPStreamer{
- logChan: logChan,
+ // Extract configuration from options
+ if port, ok := toInt(options["port"]); ok {
+ cfg.Port = port
+ }
+ if bufSize, ok := toInt(options["buffer_size"]); ok {
+ cfg.BufferSize = bufSize
+ }
+ if path, ok := options["stream_path"].(string); ok {
+ cfg.StreamPath = path
+ }
+ if path, ok := options["status_path"].(string); ok {
+ cfg.StatusPath = path
+ }
+
+ // Extract heartbeat config
+ if hb, ok := options["heartbeat"].(map[string]any); ok {
+ cfg.Heartbeat.Enabled, _ = hb["enabled"].(bool)
+ if interval, ok := toInt(hb["interval_seconds"]); ok {
+ cfg.Heartbeat.IntervalSeconds = interval
+ }
+ cfg.Heartbeat.IncludeTimestamp, _ = hb["include_timestamp"].(bool)
+ cfg.Heartbeat.IncludeStats, _ = hb["include_stats"].(bool)
+ if format, ok := hb["format"].(string); ok {
+ cfg.Heartbeat.Format = format
+ }
+ }
+
+ // Extract rate limit config
+ if rl, ok := options["rate_limit"].(map[string]any); ok {
+ cfg.RateLimit = &config.RateLimitConfig{}
+ cfg.RateLimit.Enabled, _ = rl["enabled"].(bool)
+ if rps, ok := toFloat(rl["requests_per_second"]); ok {
+ cfg.RateLimit.RequestsPerSecond = rps
+ }
+ if burst, ok := toInt(rl["burst_size"]); ok {
+ cfg.RateLimit.BurstSize = burst
+ }
+ if limitBy, ok := rl["limit_by"].(string); ok {
+ cfg.RateLimit.LimitBy = limitBy
+ }
+ if respCode, ok := toInt(rl["response_code"]); ok {
+ cfg.RateLimit.ResponseCode = respCode
+ }
+ if msg, ok := rl["response_message"].(string); ok {
+ cfg.RateLimit.ResponseMessage = msg
+ }
+ if maxPerIP, ok := toInt(rl["max_connections_per_ip"]); ok {
+ cfg.RateLimit.MaxConnectionsPerIP = maxPerIP
+ }
+ if maxTotal, ok := toInt(rl["max_total_connections"]); ok {
+ cfg.RateLimit.MaxTotalConnections = maxTotal
+ }
+ }
+
+ h := &HTTPSink{
+ input: make(chan source.LogEntry, cfg.BufferSize),
config: cfg,
startTime: time.Now(),
done: make(chan struct{}),
- streamPath: streamPath,
- statusPath: statusPath,
- standalone: true, // Default to standalone mode
+ streamPath: cfg.StreamPath,
+ statusPath: cfg.StatusPath,
+ standalone: true,
logger: logger,
}
+ h.lastProcessed.Store(time.Time{})
// Initialize rate limiter if configured
if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
h.rateLimiter = ratelimit.New(*cfg.RateLimit)
}
- return h
+ return h, nil
}
-// Configures the streamer for use with a router
-func (h *HTTPStreamer) SetRouterMode() {
- h.standalone = false
- h.logger.Debug("msg", "HTTP streamer set to router mode",
- "component", "http_streamer")
+func (h *HTTPSink) Input() chan<- source.LogEntry {
+ return h.input
}
-func (h *HTTPStreamer) Start() error {
+func (h *HTTPSink) Start(ctx context.Context) error {
if !h.standalone {
// In router mode, don't start our own server
- h.logger.Debug("msg", "HTTP streamer in router mode, skipping server start",
- "component", "http_streamer")
+ h.logger.Debug("msg", "HTTP sink in router mode, skipping server start",
+ "component", "http_sink")
return nil
}
@@ -104,7 +170,7 @@ func (h *HTTPStreamer) Start() error {
errChan := make(chan error, 1)
go func() {
h.logger.Info("msg", "HTTP server started",
- "component", "http_streamer",
+ "component", "http_sink",
"port", h.config.Port,
"stream_path", h.streamPath,
"status_path", h.statusPath)
@@ -120,16 +186,12 @@ func (h *HTTPStreamer) Start() error {
return err
case <-time.After(100 * time.Millisecond):
// Server started successfully
- h.logger.Info("msg", "HTTP server started",
- "port", h.config.Port,
- "stream_path", h.streamPath,
- "status_path", h.statusPath)
return nil
}
}
-func (h *HTTPStreamer) Stop() {
- h.logger.Info("msg", "Stopping HTTP server")
+func (h *HTTPSink) Stop() {
+ h.logger.Info("msg", "Stopping HTTP sink")
// Signal all client handlers to stop
close(h.done)
@@ -144,14 +206,48 @@ func (h *HTTPStreamer) Stop() {
// Wait for all active client handlers to finish
h.wg.Wait()
- h.logger.Info("msg", "HTTP server stopped")
+ h.logger.Info("msg", "HTTP sink stopped")
}
-func (h *HTTPStreamer) RouteRequest(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) GetStats() SinkStats {
+ lastProc, _ := h.lastProcessed.Load().(time.Time)
+
+ var rateLimitStats map[string]any
+ if h.rateLimiter != nil {
+ rateLimitStats = h.rateLimiter.GetStats()
+ }
+
+ return SinkStats{
+ Type: "http",
+ TotalProcessed: h.totalProcessed.Load(),
+ ActiveConnections: h.activeClients.Load(),
+ StartTime: h.startTime,
+ LastProcessed: lastProc,
+ Details: map[string]any{
+ "port": h.config.Port,
+ "buffer_size": h.config.BufferSize,
+ "endpoints": map[string]string{
+ "stream": h.streamPath,
+ "status": h.statusPath,
+ },
+ "rate_limit": rateLimitStats,
+ },
+ }
+}
+
+// SetRouterMode configures the sink for use with a router
+func (h *HTTPSink) SetRouterMode() {
+ h.standalone = false
+ h.logger.Debug("msg", "HTTP sink set to router mode",
+ "component", "http_sink")
+}
+
+// RouteRequest handles a request from the router
+func (h *HTTPSink) RouteRequest(ctx *fasthttp.RequestCtx) {
h.requestHandler(ctx)
}
-func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) {
// Check rate limit first
remoteAddr := ctx.RemoteAddr().String()
if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed {
@@ -182,7 +278,7 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
}
}
-func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
// Track connection for rate limiting
remoteAddr := ctx.RemoteAddr().String()
if h.rateLimiter != nil {
@@ -198,18 +294,21 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
ctx.Response.Header.Set("X-Accel-Buffering", "no")
// Create subscription for this client
- clientChan := make(chan monitor.LogEntry, h.config.BufferSize)
+ clientChan := make(chan source.LogEntry, h.config.BufferSize)
clientDone := make(chan struct{})
- // Subscribe to monitor's broadcast
+ // Subscribe to input channel
go func() {
defer close(clientChan)
for {
select {
- case entry, ok := <-h.logChan:
+ case entry, ok := <-h.input:
if !ok {
return
}
+ h.totalProcessed.Add(1)
+ h.lastProcessed.Store(time.Now())
+
select {
case clientChan <- entry:
case <-clientDone:
@@ -219,7 +318,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
default:
// Drop if client buffer full
h.logger.Debug("msg", "Dropped entry for slow client",
- "component", "http_streamer",
+ "component", "http_sink",
"remote_addr", remoteAddr)
}
case <-clientDone:
@@ -239,6 +338,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
h.wg.Add(1)
defer func() {
+ close(clientDone)
newCount := h.activeClients.Add(-1)
h.logger.Debug("msg", "HTTP client disconnected",
"remote_addr", remoteAddr,
@@ -277,7 +377,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
data, err := json.Marshal(entry)
if err != nil {
h.logger.Error("msg", "Failed to marshal log entry",
- "component", "http_streamer",
+ "component", "http_sink",
"error", err,
"entry_source", entry.Source)
continue
@@ -308,7 +408,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
ctx.SetBodyStreamWriter(streamFunc)
}
-func (h *HTTPStreamer) formatHeartbeat() string {
+func (h *HTTPSink) formatHeartbeat() string {
if !h.config.Heartbeat.Enabled {
return ""
}
@@ -346,7 +446,7 @@ func (h *HTTPStreamer) formatHeartbeat() string {
return fmt.Sprintf(": %s\n\n", strings.Join(parts, " "))
}
-func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
+func (h *HTTPSink) handleStatus(ctx *fasthttp.RequestCtx) {
ctx.SetContentType("application/json")
var rateLimitStats any
@@ -390,17 +490,17 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
ctx.SetBody(data)
}
-// Returns the current number of active clients
-func (h *HTTPStreamer) GetActiveConnections() int32 {
+// GetActiveConnections returns the current number of active clients
+func (h *HTTPSink) GetActiveConnections() int32 {
return h.activeClients.Load()
}
-// Returns the configured transport endpoint path
-func (h *HTTPStreamer) GetStreamPath() string {
+// GetStreamPath returns the configured transport endpoint path
+func (h *HTTPSink) GetStreamPath() string {
return h.streamPath
}
-// Returns the configured status endpoint path
-func (h *HTTPStreamer) GetStatusPath() string {
+// GetStatusPath returns the configured status endpoint path
+func (h *HTTPSink) GetStatusPath() string {
return h.statusPath
}
\ No newline at end of file
diff --git a/src/internal/sink/sink.go b/src/internal/sink/sink.go
new file mode 100644
index 0000000..85188d9
--- /dev/null
+++ b/src/internal/sink/sink.go
@@ -0,0 +1,61 @@
+// FILE: src/internal/sink/sink.go
+package sink
+
+import (
+ "context"
+ "time"
+
+ "logwisp/src/internal/source"
+)
+
+// Sink represents an output destination for log entries
+type Sink interface {
+ // Input returns the channel for sending log entries to this sink
+ Input() chan<- source.LogEntry
+
+ // Start begins processing log entries
+ Start(ctx context.Context) error
+
+ // Stop gracefully shuts down the sink
+ Stop()
+
+ // GetStats returns sink statistics
+ GetStats() SinkStats
+}
+
+// SinkStats contains statistics about a sink
+type SinkStats struct {
+ Type string
+ TotalProcessed uint64
+ ActiveConnections int32
+ StartTime time.Time
+ LastProcessed time.Time
+ Details map[string]any
+}
+
+// Helper functions for type conversion
+func toInt(v any) (int, bool) {
+ switch val := v.(type) {
+ case int:
+ return val, true
+ case int64:
+ return int(val), true
+ case float64:
+ return int(val), true
+ default:
+ return 0, false
+ }
+}
+
+func toFloat(v any) (float64, bool) {
+ switch val := v.(type) {
+ case float64:
+ return val, true
+ case int:
+ return float64(val), true
+ case int64:
+ return float64(val), true
+ default:
+ return 0, false
+ }
+}
\ No newline at end of file
diff --git a/src/internal/sink/tcp.go b/src/internal/sink/tcp.go
new file mode 100644
index 0000000..f7c5840
--- /dev/null
+++ b/src/internal/sink/tcp.go
@@ -0,0 +1,380 @@
+// FILE: src/internal/sink/tcp.go
+package sink
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "logwisp/src/internal/config"
+ "logwisp/src/internal/ratelimit"
+ "logwisp/src/internal/source"
+
+ "github.com/lixenwraith/log"
+ "github.com/panjf2000/gnet/v2"
+)
+
+// TCPSink streams log entries via TCP
+type TCPSink struct {
+ input chan source.LogEntry
+ config TCPConfig
+ server *tcpServer
+ done chan struct{}
+ activeConns atomic.Int32
+ startTime time.Time
+ engine *gnet.Engine
+ engineMu sync.Mutex
+ wg sync.WaitGroup
+ rateLimiter *ratelimit.Limiter
+ logger *log.Logger
+
+ // Statistics
+ totalProcessed atomic.Uint64
+ lastProcessed atomic.Value // time.Time
+}
+
+// TCPConfig holds TCP sink configuration
+type TCPConfig struct {
+ Port int
+ BufferSize int
+ Heartbeat config.HeartbeatConfig
+ SSL *config.SSLConfig
+ RateLimit *config.RateLimitConfig
+}
+
+// NewTCPSink creates a new TCP streaming sink
+func NewTCPSink(options map[string]any, logger *log.Logger) (*TCPSink, error) {
+ cfg := TCPConfig{
+ Port: 9090,
+ BufferSize: 1000,
+ }
+
+ // Extract configuration from options
+ if port, ok := toInt(options["port"]); ok {
+ cfg.Port = port
+ }
+ if bufSize, ok := toInt(options["buffer_size"]); ok {
+ cfg.BufferSize = bufSize
+ }
+
+ // Extract heartbeat config
+ if hb, ok := options["heartbeat"].(map[string]any); ok {
+ cfg.Heartbeat.Enabled, _ = hb["enabled"].(bool)
+ if interval, ok := toInt(hb["interval_seconds"]); ok {
+ cfg.Heartbeat.IntervalSeconds = interval
+ }
+ cfg.Heartbeat.IncludeTimestamp, _ = hb["include_timestamp"].(bool)
+ cfg.Heartbeat.IncludeStats, _ = hb["include_stats"].(bool)
+ if format, ok := hb["format"].(string); ok {
+ cfg.Heartbeat.Format = format
+ }
+ }
+
+ // Extract rate limit config
+ if rl, ok := options["rate_limit"].(map[string]any); ok {
+ cfg.RateLimit = &config.RateLimitConfig{}
+ cfg.RateLimit.Enabled, _ = rl["enabled"].(bool)
+ if rps, ok := toFloat(rl["requests_per_second"]); ok {
+ cfg.RateLimit.RequestsPerSecond = rps
+ }
+ if burst, ok := toInt(rl["burst_size"]); ok {
+ cfg.RateLimit.BurstSize = burst
+ }
+ if limitBy, ok := rl["limit_by"].(string); ok {
+ cfg.RateLimit.LimitBy = limitBy
+ }
+ if respCode, ok := toInt(rl["response_code"]); ok {
+ cfg.RateLimit.ResponseCode = respCode
+ }
+ if msg, ok := rl["response_message"].(string); ok {
+ cfg.RateLimit.ResponseMessage = msg
+ }
+ if maxPerIP, ok := toInt(rl["max_connections_per_ip"]); ok {
+ cfg.RateLimit.MaxConnectionsPerIP = maxPerIP
+ }
+ if maxTotal, ok := toInt(rl["max_total_connections"]); ok {
+ cfg.RateLimit.MaxTotalConnections = maxTotal
+ }
+ }
+
+ t := &TCPSink{
+ input: make(chan source.LogEntry, cfg.BufferSize),
+ config: cfg,
+ done: make(chan struct{}),
+ startTime: time.Now(),
+ logger: logger,
+ }
+ t.lastProcessed.Store(time.Time{})
+
+ if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
+ t.rateLimiter = ratelimit.New(*cfg.RateLimit)
+ }
+
+ return t, nil
+}
+
+func (t *TCPSink) Input() chan<- source.LogEntry {
+ return t.input
+}
+
+func (t *TCPSink) Start(ctx context.Context) error {
+ t.server = &tcpServer{sink: t}
+
+ // Start log broadcast loop
+ t.wg.Add(1)
+ go func() {
+ defer t.wg.Done()
+ t.broadcastLoop()
+ }()
+
+ // Configure gnet
+ addr := fmt.Sprintf("tcp://:%d", t.config.Port)
+
+ // Run gnet in separate goroutine to avoid blocking
+ errChan := make(chan error, 1)
+ go func() {
+ t.logger.Info("msg", "Starting TCP server",
+ "component", "tcp_sink",
+ "port", t.config.Port)
+
+ err := gnet.Run(t.server, addr,
+ gnet.WithLogger(noopLogger{}),
+ gnet.WithMulticore(true),
+ gnet.WithReusePort(true),
+ )
+ if err != nil {
+ t.logger.Error("msg", "TCP server failed",
+ "component", "tcp_sink",
+ "port", t.config.Port,
+ "error", err)
+ }
+ errChan <- err
+ }()
+
+ // Wait briefly for server to start or fail
+ select {
+ case err := <-errChan:
+ // Server failed immediately
+ close(t.done)
+ t.wg.Wait()
+ return err
+ case <-time.After(100 * time.Millisecond):
+ // Server started successfully
+ t.logger.Info("msg", "TCP server started", "port", t.config.Port)
+ return nil
+ }
+}
+
+func (t *TCPSink) Stop() {
+ t.logger.Info("msg", "Stopping TCP sink")
+ // Signal broadcast loop to stop
+ close(t.done)
+
+ // Stop gnet engine if running
+ t.engineMu.Lock()
+ engine := t.engine
+ t.engineMu.Unlock()
+
+ if engine != nil {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ (*engine).Stop(ctx) // Dereference the pointer
+ }
+
+ // Wait for broadcast loop to finish
+ t.wg.Wait()
+
+ t.logger.Info("msg", "TCP sink stopped")
+}
+
+func (t *TCPSink) GetStats() SinkStats {
+ lastProc, _ := t.lastProcessed.Load().(time.Time)
+
+ var rateLimitStats map[string]any
+ if t.rateLimiter != nil {
+ rateLimitStats = t.rateLimiter.GetStats()
+ }
+
+ return SinkStats{
+ Type: "tcp",
+ TotalProcessed: t.totalProcessed.Load(),
+ ActiveConnections: t.activeConns.Load(),
+ StartTime: t.startTime,
+ LastProcessed: lastProc,
+ Details: map[string]any{
+ "port": t.config.Port,
+ "buffer_size": t.config.BufferSize,
+ "rate_limit": rateLimitStats,
+ },
+ }
+}
+
+func (t *TCPSink) broadcastLoop() {
+ var ticker *time.Ticker
+ var tickerChan <-chan time.Time
+
+ if t.config.Heartbeat.Enabled {
+ ticker = time.NewTicker(time.Duration(t.config.Heartbeat.IntervalSeconds) * time.Second)
+ tickerChan = ticker.C
+ defer ticker.Stop()
+ }
+
+ for {
+ select {
+ case entry, ok := <-t.input:
+ if !ok {
+ return
+ }
+ t.totalProcessed.Add(1)
+ t.lastProcessed.Store(time.Now())
+
+ data, err := json.Marshal(entry)
+ if err != nil {
+ t.logger.Error("msg", "Failed to marshal log entry",
+ "component", "tcp_sink",
+ "error", err,
+ "entry_source", entry.Source)
+ continue
+ }
+ data = append(data, '\n')
+
+ t.server.connections.Range(func(key, value any) bool {
+ conn := key.(gnet.Conn)
+ conn.AsyncWrite(data, nil)
+ return true
+ })
+
+ case <-tickerChan:
+ if heartbeat := t.formatHeartbeat(); heartbeat != nil {
+ t.server.connections.Range(func(key, value any) bool {
+ conn := key.(gnet.Conn)
+ conn.AsyncWrite(heartbeat, nil)
+ return true
+ })
+ }
+
+ case <-t.done:
+ return
+ }
+ }
+}
+
+func (t *TCPSink) formatHeartbeat() []byte {
+ if !t.config.Heartbeat.Enabled {
+ return nil
+ }
+
+ data := make(map[string]any)
+ data["type"] = "heartbeat"
+
+ if t.config.Heartbeat.IncludeTimestamp {
+ data["time"] = time.Now().UTC().Format(time.RFC3339Nano)
+ }
+
+ if t.config.Heartbeat.IncludeStats {
+ data["active_connections"] = t.activeConns.Load()
+ data["uptime_seconds"] = int(time.Since(t.startTime).Seconds())
+ }
+
+ // For TCP, always use JSON format
+ jsonData, _ := json.Marshal(data)
+ return append(jsonData, '\n')
+}
+
+// GetActiveConnections returns the current number of connections
+func (t *TCPSink) GetActiveConnections() int32 {
+ return t.activeConns.Load()
+}
+
+// tcpServer handles gnet events
+type tcpServer struct {
+ gnet.BuiltinEventEngine
+ sink *TCPSink
+ connections sync.Map
+}
+
+func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
+ // Store engine reference for shutdown
+ s.sink.engineMu.Lock()
+ s.sink.engine = &eng
+ s.sink.engineMu.Unlock()
+
+ s.sink.logger.Debug("msg", "TCP server booted",
+ "component", "tcp_sink",
+ "port", s.sink.config.Port)
+ return gnet.None
+}
+
+func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
+ remoteAddr := c.RemoteAddr().String()
+ s.sink.logger.Debug("msg", "TCP connection attempt", "remote_addr", remoteAddr)
+
+ // Check rate limit
+ if s.sink.rateLimiter != nil {
+ // Parse the remote address to get proper net.Addr
+ remoteStr := c.RemoteAddr().String()
+ tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr)
+ if err != nil {
+ s.sink.logger.Warn("msg", "Failed to parse TCP address",
+ "remote_addr", remoteAddr,
+ "error", err)
+ return nil, gnet.Close
+ }
+
+ if !s.sink.rateLimiter.CheckTCP(tcpAddr) {
+ s.sink.logger.Warn("msg", "TCP connection rate limited",
+ "remote_addr", remoteAddr)
+ // Silently close connection when rate limited
+ return nil, gnet.Close
+ }
+
+ // Track connection
+ s.sink.rateLimiter.AddConnection(remoteStr)
+ }
+
+ s.connections.Store(c, struct{}{})
+
+ newCount := s.sink.activeConns.Add(1)
+ s.sink.logger.Debug("msg", "TCP connection opened",
+ "remote_addr", remoteAddr,
+ "active_connections", newCount)
+
+ return nil, gnet.None
+}
+
+func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
+ s.connections.Delete(c)
+
+ remoteAddr := c.RemoteAddr().String()
+
+ // Remove connection tracking
+ if s.sink.rateLimiter != nil {
+ s.sink.rateLimiter.RemoveConnection(c.RemoteAddr().String())
+ }
+
+ newCount := s.sink.activeConns.Add(-1)
+ s.sink.logger.Debug("msg", "TCP connection closed",
+ "remote_addr", remoteAddr,
+ "active_connections", newCount,
+ "error", err)
+ return gnet.None
+}
+
+func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
+ // We don't expect input from clients, just discard
+ c.Discard(-1)
+ return gnet.None
+}
+
+// noopLogger implements gnet's Logger interface but discards everything
+type noopLogger struct{}
+
+func (n noopLogger) Debugf(format string, args ...any) {}
+func (n noopLogger) Infof(format string, args ...any) {}
+func (n noopLogger) Warnf(format string, args ...any) {}
+func (n noopLogger) Errorf(format string, args ...any) {}
+func (n noopLogger) Fatalf(format string, args ...any) {}
\ No newline at end of file
diff --git a/src/internal/source/directory.go b/src/internal/source/directory.go
new file mode 100644
index 0000000..80d1a90
--- /dev/null
+++ b/src/internal/source/directory.go
@@ -0,0 +1,298 @@
+// FILE: src/internal/source/directory.go
+package source
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/lixenwraith/log"
+)
+
+// DirectorySource monitors a directory for log files
+type DirectorySource struct {
+ path string
+ pattern string
+ checkInterval time.Duration
+ subscribers []chan LogEntry
+ watchers map[string]*fileWatcher
+ mu sync.RWMutex
+ ctx context.Context
+ cancel context.CancelFunc
+ wg sync.WaitGroup
+ totalEntries atomic.Uint64
+ droppedEntries atomic.Uint64
+ startTime time.Time
+ lastEntryTime atomic.Value // time.Time
+ logger *log.Logger
+}
+
+// NewDirectorySource creates a new directory monitoring source
+func NewDirectorySource(options map[string]any, logger *log.Logger) (*DirectorySource, error) {
+ path, ok := options["path"].(string)
+ if !ok {
+ return nil, fmt.Errorf("directory source requires 'path' option")
+ }
+
+ pattern, _ := options["pattern"].(string)
+ if pattern == "" {
+ pattern = "*"
+ }
+
+ checkInterval := 100 * time.Millisecond
+ if ms, ok := toInt(options["check_interval_ms"]); ok && ms > 0 {
+ checkInterval = time.Duration(ms) * time.Millisecond
+ }
+
+ absPath, err := filepath.Abs(path)
+ if err != nil {
+ return nil, fmt.Errorf("invalid path %s: %w", path, err)
+ }
+
+ ds := &DirectorySource{
+ path: absPath,
+ pattern: pattern,
+ checkInterval: checkInterval,
+ watchers: make(map[string]*fileWatcher),
+ startTime: time.Now(),
+ logger: logger,
+ }
+ ds.lastEntryTime.Store(time.Time{})
+
+ return ds, nil
+}
+
+func (ds *DirectorySource) Subscribe() <-chan LogEntry {
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+
+ ch := make(chan LogEntry, 1000)
+ ds.subscribers = append(ds.subscribers, ch)
+ return ch
+}
+
+func (ds *DirectorySource) Start() error {
+ ds.ctx, ds.cancel = context.WithCancel(context.Background())
+ ds.wg.Add(1)
+ go ds.monitorLoop()
+
+ ds.logger.Info("msg", "Directory source started",
+ "component", "directory_source",
+ "path", ds.path,
+ "pattern", ds.pattern,
+ "check_interval_ms", ds.checkInterval.Milliseconds())
+ return nil
+}
+
+func (ds *DirectorySource) Stop() {
+ if ds.cancel != nil {
+ ds.cancel()
+ }
+ ds.wg.Wait()
+
+ ds.mu.Lock()
+ for _, w := range ds.watchers {
+ w.close()
+ }
+ for _, ch := range ds.subscribers {
+ close(ch)
+ }
+ ds.mu.Unlock()
+
+ ds.logger.Info("msg", "Directory source stopped",
+ "component", "directory_source",
+ "path", ds.path)
+}
+
+func (ds *DirectorySource) GetStats() SourceStats {
+ lastEntry, _ := ds.lastEntryTime.Load().(time.Time)
+
+ ds.mu.RLock()
+ watcherCount := len(ds.watchers)
+ details := make(map[string]any)
+
+ // Add watcher details
+ watchers := make([]map[string]any, 0, watcherCount)
+ for _, w := range ds.watchers {
+ info := w.getInfo()
+ watchers = append(watchers, map[string]any{
+ "path": info.Path,
+ "size": info.Size,
+ "position": info.Position,
+ "entries_read": info.EntriesRead,
+ "rotations": info.Rotations,
+ "last_read": info.LastReadTime,
+ })
+ }
+ details["watchers"] = watchers
+ details["active_watchers"] = watcherCount
+ ds.mu.RUnlock()
+
+ return SourceStats{
+ Type: "directory",
+ TotalEntries: ds.totalEntries.Load(),
+ DroppedEntries: ds.droppedEntries.Load(),
+ StartTime: ds.startTime,
+ LastEntryTime: lastEntry,
+ Details: details,
+ }
+}
+
+func (ds *DirectorySource) ApplyRateLimit(entry LogEntry) (LogEntry, bool) {
+ // TODO: Implement source-side rate limiting for aggregation/summarization
+ // For now, just pass through unchanged
+ return entry, true
+}
+
+func (ds *DirectorySource) publish(entry LogEntry) {
+ // Apply rate limiting (placeholder for now)
+ entry, allowed := ds.ApplyRateLimit(entry)
+ if !allowed {
+ return
+ }
+
+ ds.mu.RLock()
+ defer ds.mu.RUnlock()
+
+ ds.totalEntries.Add(1)
+ ds.lastEntryTime.Store(entry.Time)
+
+ for _, ch := range ds.subscribers {
+ select {
+ case ch <- entry:
+ default:
+ ds.droppedEntries.Add(1)
+ ds.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+ "component", "directory_source")
+ }
+ }
+}
+
+func (ds *DirectorySource) monitorLoop() {
+ defer ds.wg.Done()
+
+ ds.checkTargets()
+
+ ticker := time.NewTicker(ds.checkInterval)
+ defer ticker.Stop()
+
+ for {
+ select {
+ case <-ds.ctx.Done():
+ return
+ case <-ticker.C:
+ ds.checkTargets()
+ }
+ }
+}
+
+func (ds *DirectorySource) checkTargets() {
+ files, err := ds.scanDirectory()
+ if err != nil {
+ ds.logger.Warn("msg", "Failed to scan directory",
+ "component", "directory_source",
+ "path", ds.path,
+ "pattern", ds.pattern,
+ "error", err)
+ return
+ }
+
+ for _, file := range files {
+ ds.ensureWatcher(file)
+ }
+
+ ds.cleanupWatchers()
+}
+
+func (ds *DirectorySource) scanDirectory() ([]string, error) {
+ entries, err := os.ReadDir(ds.path)
+ if err != nil {
+ return nil, err
+ }
+
+ // Convert glob pattern to regex
+ regexPattern := globToRegex(ds.pattern)
+ re, err := regexp.Compile(regexPattern)
+ if err != nil {
+ return nil, fmt.Errorf("invalid pattern regex: %w", err)
+ }
+
+ var files []string
+ for _, entry := range entries {
+ if entry.IsDir() {
+ continue
+ }
+
+ name := entry.Name()
+ if re.MatchString(name) {
+ files = append(files, filepath.Join(ds.path, name))
+ }
+ }
+
+ return files, nil
+}
+
+func (ds *DirectorySource) ensureWatcher(path string) {
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+
+ if _, exists := ds.watchers[path]; exists {
+ return
+ }
+
+ w := newFileWatcher(path, ds.publish, ds.logger)
+ ds.watchers[path] = w
+
+ ds.logger.Debug("msg", "Created file watcher",
+ "component", "directory_source",
+ "path", path)
+
+ ds.wg.Add(1)
+ go func() {
+ defer ds.wg.Done()
+ if err := w.watch(ds.ctx); err != nil {
+ if err == context.Canceled {
+ ds.logger.Debug("msg", "Watcher cancelled",
+ "component", "directory_source",
+ "path", path)
+ } else {
+ ds.logger.Error("msg", "Watcher failed",
+ "component", "directory_source",
+ "path", path,
+ "error", err)
+ }
+ }
+
+ ds.mu.Lock()
+ delete(ds.watchers, path)
+ ds.mu.Unlock()
+ }()
+}
+
+func (ds *DirectorySource) cleanupWatchers() {
+ ds.mu.Lock()
+ defer ds.mu.Unlock()
+
+ for path, w := range ds.watchers {
+ if _, err := os.Stat(path); os.IsNotExist(err) {
+ w.stop()
+ delete(ds.watchers, path)
+ ds.logger.Debug("msg", "Cleaned up watcher for non-existent file",
+ "component", "directory_source",
+ "path", path)
+ }
+ }
+}
+
+func globToRegex(glob string) string {
+ regex := regexp.QuoteMeta(glob)
+ regex = strings.ReplaceAll(regex, `\*`, `.*`)
+ regex = strings.ReplaceAll(regex, `\?`, `.`)
+ return "^" + regex + "$"
+}
\ No newline at end of file
diff --git a/src/internal/monitor/file_watcher.go b/src/internal/source/file_watcher.go
similarity index 96%
rename from src/internal/monitor/file_watcher.go
rename to src/internal/source/file_watcher.go
index 4b52107..427b6fb 100644
--- a/src/internal/monitor/file_watcher.go
+++ b/src/internal/source/file_watcher.go
@@ -1,5 +1,5 @@
-// FILE: src/internal/monitor/file_watcher.go
-package monitor
+// FILE: src/internal/source/file_watcher.go
+package source
import (
"bufio"
@@ -9,7 +9,6 @@ import (
"io"
"os"
"path/filepath"
- "regexp"
"strings"
"sync"
"sync/atomic"
@@ -19,6 +18,17 @@ import (
"github.com/lixenwraith/log"
)
+// WatcherInfo contains information about a file watcher
+type WatcherInfo struct {
+ Path string
+ Size int64
+ Position int64
+ ModTime time.Time
+ EntriesRead uint64
+ LastReadTime time.Time
+ Rotations int
+}
+
type fileWatcher struct {
path string
callback func(LogEntry)
@@ -333,13 +343,6 @@ func extractLogLevel(line string) string {
return ""
}
-func globToRegex(glob string) string {
- regex := regexp.QuoteMeta(glob)
- regex = strings.ReplaceAll(regex, `\*`, `.*`)
- regex = strings.ReplaceAll(regex, `\?`, `.`)
- return "^" + regex + "$"
-}
-
func (w *fileWatcher) getInfo() WatcherInfo {
w.mu.Lock()
info := WatcherInfo{
diff --git a/src/internal/source/source.go b/src/internal/source/source.go
new file mode 100644
index 0000000..2a57953
--- /dev/null
+++ b/src/internal/source/source.go
@@ -0,0 +1,60 @@
+// FILE: src/internal/source/source.go
+package source
+
+import (
+ "encoding/json"
+ "time"
+)
+
+// LogEntry represents a single log record
+type LogEntry struct {
+ Time time.Time `json:"time"`
+ Source string `json:"source"`
+ Level string `json:"level,omitempty"`
+ Message string `json:"message"`
+ Fields json.RawMessage `json:"fields,omitempty"`
+}
+
+// Source represents an input data stream
+type Source interface {
+ // Subscribe returns a channel that receives log entries
+ Subscribe() <-chan LogEntry
+
+ // Start begins reading from the source
+ Start() error
+
+ // Stop gracefully shuts down the source
+ Stop()
+
+ // GetStats returns source statistics
+ GetStats() SourceStats
+
+ // ApplyRateLimit applies source-side rate limiting
+ // TODO: This is a placeholder for future features like aggregation and summarization
+ // Currently just returns the entry unchanged
+ ApplyRateLimit(entry LogEntry) (LogEntry, bool)
+}
+
+// SourceStats contains statistics about a source
+type SourceStats struct {
+ Type string
+ TotalEntries uint64
+ DroppedEntries uint64
+ StartTime time.Time
+ LastEntryTime time.Time
+ Details map[string]any
+}
+
+// Helper function for type conversion
+func toInt(v any) (int, bool) {
+ switch val := v.(type) {
+ case int:
+ return val, true
+ case int64:
+ return int(val), true
+ case float64:
+ return int(val), true
+ default:
+ return 0, false
+ }
+}
\ No newline at end of file
diff --git a/src/internal/source/stdin.go b/src/internal/source/stdin.go
new file mode 100644
index 0000000..ed47625
--- /dev/null
+++ b/src/internal/source/stdin.go
@@ -0,0 +1,123 @@
+// FILE: src/internal/source/stdin.go
+package source
+
+import (
+ "bufio"
+ "os"
+ "sync/atomic"
+ "time"
+
+ "github.com/lixenwraith/log"
+)
+
+// StdinSource reads log entries from standard input
+type StdinSource struct {
+ subscribers []chan LogEntry
+ done chan struct{}
+ totalEntries atomic.Uint64
+ droppedEntries atomic.Uint64
+ startTime time.Time
+ lastEntryTime atomic.Value // time.Time
+ logger *log.Logger
+}
+
+// NewStdinSource creates a new stdin source
+func NewStdinSource(options map[string]any, logger *log.Logger) (*StdinSource, error) {
+ s := &StdinSource{
+ done: make(chan struct{}),
+ startTime: time.Now(),
+ logger: logger,
+ }
+ s.lastEntryTime.Store(time.Time{})
+ return s, nil
+}
+
+func (s *StdinSource) Subscribe() <-chan LogEntry {
+ ch := make(chan LogEntry, 1000)
+ s.subscribers = append(s.subscribers, ch)
+ return ch
+}
+
+func (s *StdinSource) Start() error {
+ go s.readLoop()
+ s.logger.Info("msg", "Stdin source started", "component", "stdin_source")
+ return nil
+}
+
+func (s *StdinSource) Stop() {
+ close(s.done)
+ for _, ch := range s.subscribers {
+ close(ch)
+ }
+ s.logger.Info("msg", "Stdin source stopped", "component", "stdin_source")
+}
+
+func (s *StdinSource) GetStats() SourceStats {
+ lastEntry, _ := s.lastEntryTime.Load().(time.Time)
+
+ return SourceStats{
+ Type: "stdin",
+ TotalEntries: s.totalEntries.Load(),
+ DroppedEntries: s.droppedEntries.Load(),
+ StartTime: s.startTime,
+ LastEntryTime: lastEntry,
+ Details: map[string]any{},
+ }
+}
+
+func (s *StdinSource) ApplyRateLimit(entry LogEntry) (LogEntry, bool) {
+ // TODO: Implement source-side rate limiting for aggregation/summarization
+ // For now, just pass through unchanged
+ return entry, true
+}
+
+func (s *StdinSource) readLoop() {
+ scanner := bufio.NewScanner(os.Stdin)
+ for scanner.Scan() {
+ select {
+ case <-s.done:
+ return
+ default:
+ line := scanner.Text()
+ if line == "" {
+ continue
+ }
+
+ entry := LogEntry{
+ Time: time.Now(),
+ Source: "stdin",
+ Message: line,
+ Level: extractLogLevel(line),
+ }
+
+ // Apply rate limiting
+ entry, allowed := s.ApplyRateLimit(entry)
+ if !allowed {
+ continue
+ }
+
+ s.publish(entry)
+ }
+ }
+
+ if err := scanner.Err(); err != nil {
+ s.logger.Error("msg", "Scanner error reading stdin",
+ "component", "stdin_source",
+ "error", err)
+ }
+}
+
+func (s *StdinSource) publish(entry LogEntry) {
+ s.totalEntries.Add(1)
+ s.lastEntryTime.Store(entry.Time)
+
+ for _, ch := range s.subscribers {
+ select {
+ case ch <- entry:
+ default:
+ s.droppedEntries.Add(1)
+ s.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
+ "component", "stdin_source")
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/internal/transport/noop_logger.go b/src/internal/transport/noop_logger.go
deleted file mode 100644
index cd935c1..0000000
--- a/src/internal/transport/noop_logger.go
+++ /dev/null
@@ -1,11 +0,0 @@
-// FILE: src/internal/transport/noop_logger.go
-package transport
-
-// noopLogger implements gnet's Logger interface but discards everything
-type noopLogger struct{}
-
-func (n noopLogger) Debugf(format string, args ...any) {}
-func (n noopLogger) Infof(format string, args ...any) {}
-func (n noopLogger) Warnf(format string, args ...any) {}
-func (n noopLogger) Errorf(format string, args ...any) {}
-func (n noopLogger) Fatalf(format string, args ...any) {}
\ No newline at end of file
diff --git a/src/internal/transport/tcpserver.go b/src/internal/transport/tcpserver.go
deleted file mode 100644
index 652b8f5..0000000
--- a/src/internal/transport/tcpserver.go
+++ /dev/null
@@ -1,87 +0,0 @@
-// FILE: src/internal/monitor/tcpserver.go
-package transport
-
-import (
- "fmt"
- "net"
- "sync"
-
- "github.com/panjf2000/gnet/v2"
-)
-
-type tcpServer struct {
- gnet.BuiltinEventEngine
- streamer *TCPStreamer
- connections sync.Map
-}
-
-func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
- // Store engine reference for shutdown
- s.streamer.engineMu.Lock()
- s.streamer.engine = &eng
- s.streamer.engineMu.Unlock()
-
- fmt.Printf("[TCP DEBUG] Server booted on port %d\n", s.streamer.config.Port)
- return gnet.None
-}
-
-func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
- remoteAddr := c.RemoteAddr().String()
- s.streamer.logger.Debug("msg", "TCP connection attempt", "remote_addr", remoteAddr)
-
- // Check rate limit
- if s.streamer.rateLimiter != nil {
- // Parse the remote address to get proper net.Addr
- remoteStr := c.RemoteAddr().String()
- tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr)
- if err != nil {
- s.streamer.logger.Warn("msg", "Failed to parse TCP address",
- "remote_addr", remoteAddr,
- "error", err)
- return nil, gnet.Close
- }
-
- if !s.streamer.rateLimiter.CheckTCP(tcpAddr) {
- s.streamer.logger.Warn("msg", "TCP connection rate limited",
- "remote_addr", remoteAddr)
- // Silently close connection when rate limited
- return nil, gnet.Close
- }
-
- // Track connection
- s.streamer.rateLimiter.AddConnection(remoteStr)
- }
-
- s.connections.Store(c, struct{}{})
-
- newCount := s.streamer.activeConns.Add(1)
- s.streamer.logger.Debug("msg", "TCP connection opened",
- "remote_addr", remoteAddr,
- "active_connections", newCount)
-
- return nil, gnet.None
-}
-
-func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
- s.connections.Delete(c)
-
- remoteAddr := c.RemoteAddr().String()
-
- // Remove connection tracking
- if s.streamer.rateLimiter != nil {
- s.streamer.rateLimiter.RemoveConnection(c.RemoteAddr().String())
- }
-
- newCount := s.streamer.activeConns.Add(-1)
- s.streamer.logger.Debug("msg", "TCP connection closed",
- "remote_addr", remoteAddr,
- "active_connections", newCount,
- "error", err)
- return gnet.None
-}
-
-func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
- // We don't expect input from clients, just discard
- c.Discard(-1)
- return gnet.None
-}
\ No newline at end of file
diff --git a/src/internal/transport/tcpstreamer.go b/src/internal/transport/tcpstreamer.go
deleted file mode 100644
index 736d3fd..0000000
--- a/src/internal/transport/tcpstreamer.go
+++ /dev/null
@@ -1,191 +0,0 @@
-// FILE: src/internal/transport/tcpstreamer.go
-package transport
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "sync"
- "sync/atomic"
- "time"
-
- "logwisp/src/internal/config"
- "logwisp/src/internal/monitor"
- "logwisp/src/internal/ratelimit"
-
- "github.com/lixenwraith/log"
- "github.com/panjf2000/gnet/v2"
-)
-
-type TCPStreamer struct {
- logChan chan monitor.LogEntry
- config config.TCPConfig
- server *tcpServer
- done chan struct{}
- activeConns atomic.Int32
- startTime time.Time
- engine *gnet.Engine
- engineMu sync.Mutex
- wg sync.WaitGroup
- rateLimiter *ratelimit.Limiter
- logger *log.Logger
-}
-
-func NewTCPStreamer(logChan chan monitor.LogEntry, cfg config.TCPConfig, logger *log.Logger) *TCPStreamer {
- t := &TCPStreamer{
- logChan: logChan,
- config: cfg,
- done: make(chan struct{}),
- startTime: time.Now(),
- logger: logger,
- }
-
- if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
- t.rateLimiter = ratelimit.New(*cfg.RateLimit)
- }
-
- return t
-}
-
-func (t *TCPStreamer) Start() error {
- t.server = &tcpServer{streamer: t}
-
- // Start log broadcast loop
- t.wg.Add(1)
- go func() {
- defer t.wg.Done()
- t.broadcastLoop()
- }()
-
- // Configure gnet
- addr := fmt.Sprintf("tcp://:%d", t.config.Port)
-
- // Run gnet in separate goroutine to avoid blocking
- errChan := make(chan error, 1)
- go func() {
- t.logger.Info("msg", "Starting TCP server",
- "component", "tcp_streamer",
- "port", t.config.Port)
-
- err := gnet.Run(t.server, addr,
- gnet.WithLogger(noopLogger{}),
- gnet.WithMulticore(true),
- gnet.WithReusePort(true),
- )
- if err != nil {
- t.logger.Error("msg", "TCP server failed",
- "component", "tcp_streamer",
- "port", t.config.Port,
- "error", err)
- }
- errChan <- err
- }()
-
- // Wait briefly for server to start or fail
- select {
- case err := <-errChan:
- // Server failed immediately
- close(t.done)
- t.wg.Wait()
- return err
- case <-time.After(100 * time.Millisecond):
- // Server started successfully
- t.logger.Info("msg", "TCP server started", "port", t.config.Port)
- return nil
- }
-}
-
-func (t *TCPStreamer) Stop() {
- t.logger.Info("msg", "Stopping TCP server")
- // Signal broadcast loop to stop
- close(t.done)
-
- // Stop gnet engine if running
- t.engineMu.Lock()
- engine := t.engine
- t.engineMu.Unlock()
-
- if engine != nil {
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
- defer cancel()
- (*engine).Stop(ctx) // Dereference the pointer
- }
-
- // Wait for broadcast loop to finish
- t.wg.Wait()
-
- t.logger.Info("msg", "TCP server stopped")
-}
-
-func (t *TCPStreamer) broadcastLoop() {
- var ticker *time.Ticker
- var tickerChan <-chan time.Time
-
- if t.config.Heartbeat.Enabled {
- ticker = time.NewTicker(time.Duration(t.config.Heartbeat.IntervalSeconds) * time.Second)
- tickerChan = ticker.C
- defer ticker.Stop()
- }
-
- for {
- select {
- case entry, ok := <-t.logChan:
- if !ok {
- return
- }
- data, err := json.Marshal(entry)
- if err != nil {
- t.logger.Error("msg", "Failed to marshal log entry",
- "component", "tcp_streamer",
- "error", err,
- "entry_source", entry.Source)
- continue
- }
- data = append(data, '\n')
-
- t.server.connections.Range(func(key, value any) bool {
- conn := key.(gnet.Conn)
- conn.AsyncWrite(data, nil)
- return true
- })
-
- case <-tickerChan:
- if heartbeat := t.formatHeartbeat(); heartbeat != nil {
- t.server.connections.Range(func(key, value any) bool {
- conn := key.(gnet.Conn)
- conn.AsyncWrite(heartbeat, nil)
- return true
- })
- }
-
- case <-t.done:
- return
- }
- }
-}
-
-func (t *TCPStreamer) formatHeartbeat() []byte {
- if !t.config.Heartbeat.Enabled {
- return nil
- }
-
- data := make(map[string]any)
- data["type"] = "heartbeat"
-
- if t.config.Heartbeat.IncludeTimestamp {
- data["time"] = time.Now().UTC().Format(time.RFC3339Nano)
- }
-
- if t.config.Heartbeat.IncludeStats {
- data["active_connections"] = t.activeConns.Load()
- data["uptime_seconds"] = int(time.Since(t.startTime).Seconds())
- }
-
- // For TCP, always use JSON format
- jsonData, _ := json.Marshal(data)
- return append(jsonData, '\n')
-}
-
-func (t *TCPStreamer) GetActiveConnections() int32 {
- return t.activeConns.Load()
-}
\ No newline at end of file
|