491 lines
16 KiB
Plaintext
491 lines
16 KiB
Plaintext
# LogWisp Configuration File
|
|
# Default path: ~/.config/logwisp.toml
|
|
# Override with: ./logwisp --config /path/to/config.toml
|
|
|
|
# This is a complete configuration reference showing all available options.
|
|
# Default values are uncommented, alternatives and examples are commented.
|
|
|
|
# ==============================================================================
|
|
# STREAM CONFIGURATION
|
|
# ==============================================================================
|
|
# Each [[streams]] section defines an independent log monitoring stream.
|
|
# You can have multiple streams, each with its own settings.
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Default Stream - Monitors current directory
|
|
# ------------------------------------------------------------------------------
|
|
[[streams]]
|
|
# Stream identifier used in logs, metrics, and router paths
|
|
name = "default"
|
|
|
|
# File monitoring configuration
|
|
[streams.monitor]
|
|
# How often to check for new log entries (milliseconds)
|
|
# Lower = faster detection but more CPU usage
|
|
check_interval_ms = 100
|
|
|
|
# Targets to monitor - can be files or directories
|
|
targets = [
|
|
# Monitor all .log files in current directory
|
|
{ path = "./", pattern = "*.log", is_file = false },
|
|
]
|
|
|
|
# Filter configuration (optional) - controls which logs are streamed
|
|
# Multiple filters are applied sequentially - all must pass
|
|
# [[streams.filters]]
|
|
# type = "include" # "include" (whitelist) or "exclude" (blacklist)
|
|
# logic = "or" # "or" (match any) or "and" (match all)
|
|
# patterns = [
|
|
# "(?i)error", # Case-insensitive error matching
|
|
# "(?i)warn" # Case-insensitive warning matching
|
|
# ]
|
|
|
|
# HTTP Server configuration (SSE/Server-Sent Events)
|
|
[streams.httpserver]
|
|
enabled = true
|
|
port = 8080
|
|
buffer_size = 1000 # Per-client buffer size (messages)
|
|
stream_path = "/stream" # Endpoint for SSE stream
|
|
status_path = "/status" # Endpoint for statistics
|
|
|
|
# Keep-alive heartbeat configuration
|
|
[streams.httpserver.heartbeat]
|
|
enabled = true
|
|
interval_seconds = 30 # Send heartbeat every 30 seconds
|
|
format = "comment" # SSE comment format (: heartbeat)
|
|
include_timestamp = true # Include timestamp in heartbeat
|
|
include_stats = false # Include connection stats
|
|
|
|
# Rate limiting configuration (disabled by default)
|
|
[streams.httpserver.rate_limit]
|
|
enabled = false
|
|
# requests_per_second = 10.0 # Token refill rate
|
|
# burst_size = 20 # Max burst capacity
|
|
# limit_by = "ip" # "ip" or "global"
|
|
# response_code = 429 # HTTP Too Many Requests
|
|
# response_message = "Rate limit exceeded"
|
|
# max_connections_per_ip = 5 # Max SSE connections per IP
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: Application Logs Stream with Error Filtering
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "app"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 50 # Fast detection for active logs
|
|
# targets = [
|
|
# # Monitor specific application log directory
|
|
# { path = "/var/log/myapp", pattern = "*.log", is_file = false },
|
|
# # Also monitor specific file
|
|
# { path = "/var/log/myapp/app.log", is_file = true },
|
|
# ]
|
|
#
|
|
# # Filter 1: Include only errors and warnings
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# logic = "or" # Match ANY of these patterns
|
|
# patterns = [
|
|
# "(?i)\\berror\\b", # Word boundary error (case-insensitive)
|
|
# "(?i)\\bwarn(ing)?\\b", # warn or warning
|
|
# "(?i)\\bfatal\\b", # fatal
|
|
# "(?i)\\bcritical\\b", # critical
|
|
# "(?i)exception", # exception anywhere
|
|
# "(?i)fail(ed|ure)?", # fail, failed, failure
|
|
# ]
|
|
#
|
|
# # Filter 2: Exclude health check noise
|
|
# [[streams.filters]]
|
|
# type = "exclude"
|
|
# patterns = [
|
|
# "/health",
|
|
# "/metrics",
|
|
# "GET /ping"
|
|
# ]
|
|
#
|
|
# [streams.httpserver]
|
|
# enabled = true
|
|
# port = 8081 # Different port for each stream
|
|
# buffer_size = 2000 # Larger buffer for busy logs
|
|
# stream_path = "/logs" # Custom path
|
|
# status_path = "/health" # Custom health endpoint
|
|
#
|
|
# # JSON heartbeat format for programmatic clients
|
|
# [streams.httpserver.heartbeat]
|
|
# enabled = true
|
|
# interval_seconds = 20
|
|
# format = "json" # JSON event format
|
|
# include_timestamp = true
|
|
# include_stats = true # Include active client count
|
|
#
|
|
# # Moderate rate limiting for public access
|
|
# [streams.httpserver.rate_limit]
|
|
# enabled = true
|
|
# requests_per_second = 25.0
|
|
# burst_size = 50
|
|
# limit_by = "ip"
|
|
# max_connections_per_ip = 10
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: System Logs Stream (TCP + HTTP) with Security Filtering
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "system"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 1000 # Check every second (system logs update slowly)
|
|
# targets = [
|
|
# { path = "/var/log/syslog", is_file = true },
|
|
# { path = "/var/log/auth.log", is_file = true },
|
|
# { path = "/var/log/kern.log", is_file = true },
|
|
# ]
|
|
#
|
|
# # Include only security-relevant logs
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# logic = "or"
|
|
# patterns = [
|
|
# "(?i)auth",
|
|
# "(?i)sudo",
|
|
# "(?i)ssh",
|
|
# "(?i)login",
|
|
# "(?i)permission",
|
|
# "(?i)denied",
|
|
# "(?i)unauthorized",
|
|
# "kernel:.*audit"
|
|
# ]
|
|
#
|
|
# # TCP Server for high-performance streaming
|
|
# [streams.tcpserver]
|
|
# enabled = true
|
|
# port = 9090
|
|
# buffer_size = 5000
|
|
#
|
|
# # TCP heartbeat (always JSON format)
|
|
# [streams.tcpserver.heartbeat]
|
|
# enabled = true
|
|
# interval_seconds = 60 # Less frequent for TCP
|
|
# include_timestamp = true
|
|
# include_stats = false
|
|
#
|
|
# # TCP rate limiting
|
|
# [streams.tcpserver.rate_limit]
|
|
# enabled = true
|
|
# requests_per_second = 5.0 # Limit TCP connections
|
|
# burst_size = 10
|
|
# limit_by = "ip"
|
|
#
|
|
# # Also expose via HTTP
|
|
# [streams.httpserver]
|
|
# enabled = true
|
|
# port = 8082
|
|
# buffer_size = 1000
|
|
# stream_path = "/stream"
|
|
# status_path = "/status"
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: High-Volume Debug Logs with Filtering
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "debug"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 5000 # Check every 5 seconds (high volume)
|
|
# targets = [
|
|
# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
|
|
# ]
|
|
#
|
|
# # Exclude verbose debug output
|
|
# [[streams.filters]]
|
|
# type = "exclude"
|
|
# patterns = [
|
|
# "TRACE",
|
|
# "VERBOSE",
|
|
# "entering function",
|
|
# "exiting function",
|
|
# "memory dump"
|
|
# ]
|
|
#
|
|
# # Include only specific modules
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# patterns = [
|
|
# "module:(api|database|auth)",
|
|
# "component:(router|handler)"
|
|
# ]
|
|
#
|
|
# [streams.httpserver]
|
|
# enabled = true
|
|
# port = 8083
|
|
# buffer_size = 10000 # Very large buffer
|
|
# stream_path = "/debug"
|
|
# status_path = "/stats"
|
|
#
|
|
# # Disable heartbeat for high-volume streams
|
|
# [streams.httpserver.heartbeat]
|
|
# enabled = false
|
|
#
|
|
# # Aggressive rate limiting
|
|
# [streams.httpserver.rate_limit]
|
|
# enabled = true
|
|
# requests_per_second = 1.0 # Very restrictive
|
|
# burst_size = 5
|
|
# limit_by = "ip"
|
|
# max_connections_per_ip = 1 # One connection per IP
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: Database Logs with Complex Filtering
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "database"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 200
|
|
# targets = [
|
|
# { path = "/var/log/postgresql", pattern = "*.log", is_file = false },
|
|
# ]
|
|
#
|
|
# # Complex AND filter - must match all patterns
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# logic = "and" # Must match ALL patterns
|
|
# patterns = [
|
|
# "(?i)error|fail", # Must contain error or fail
|
|
# "(?i)connection|query", # AND must be about connections or queries
|
|
# "(?i)timeout|deadlock" # AND must involve timeout or deadlock
|
|
# ]
|
|
#
|
|
# # Exclude routine maintenance
|
|
# [[streams.filters]]
|
|
# type = "exclude"
|
|
# patterns = [
|
|
# "VACUUM",
|
|
# "ANALYZE",
|
|
# "checkpoint"
|
|
# ]
|
|
#
|
|
# [streams.tcpserver]
|
|
# enabled = true
|
|
# port = 9091
|
|
# buffer_size = 2000
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: API Access Logs with Pattern Extraction
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "api-access"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 100
|
|
# targets = [
|
|
# { path = "/var/log/nginx/access.log", is_file = true },
|
|
# ]
|
|
#
|
|
# # Include only API endpoints
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# patterns = [
|
|
# '"/api/v[0-9]+/', # API versioned endpoints
|
|
# '"(GET|POST|PUT|DELETE) /api/' # API requests
|
|
# ]
|
|
#
|
|
# # Exclude specific status codes
|
|
# [[streams.filters]]
|
|
# type = "exclude"
|
|
# patterns = [
|
|
# '" 200 ', # Success responses
|
|
# '" 204 ', # No content
|
|
# '" 304 ', # Not modified
|
|
# 'OPTIONS ' # CORS preflight
|
|
# ]
|
|
#
|
|
# [streams.httpserver]
|
|
# enabled = true
|
|
# port = 8084
|
|
# buffer_size = 3000
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: Security/Audit Logs with Strict Filtering
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "security"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 100
|
|
# targets = [
|
|
# { path = "/var/log/audit", pattern = "audit.log*", is_file = false },
|
|
# ]
|
|
#
|
|
# # Security-focused patterns
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# logic = "or"
|
|
# patterns = [
|
|
# "type=USER_AUTH",
|
|
# "type=USER_LOGIN",
|
|
# "type=USER_LOGOUT",
|
|
# "type=USER_ERR",
|
|
# "type=CRED_", # All credential operations
|
|
# "type=PRIV_", # All privilege operations
|
|
# "type=ANOM_", # All anomalies
|
|
# "type=RESP_", # All responses
|
|
# "failed|failure",
|
|
# "denied|unauthorized",
|
|
# "violation",
|
|
# "attack|intrusion"
|
|
# ]
|
|
#
|
|
# [streams.httpserver]
|
|
# enabled = true
|
|
# port = 8443 # HTTPS port (for future TLS)
|
|
# buffer_size = 1000
|
|
# stream_path = "/audit"
|
|
# status_path = "/health"
|
|
#
|
|
# # Strict rate limiting for security logs
|
|
# [streams.httpserver.rate_limit]
|
|
# enabled = true
|
|
# requests_per_second = 2.0 # Very limited access
|
|
# burst_size = 3
|
|
# limit_by = "ip"
|
|
# max_connections_per_ip = 1 # Single connection per IP
|
|
# response_code = 403 # Forbidden instead of rate limit
|
|
# response_message = "Access restricted"
|
|
#
|
|
# # Future: SSL/TLS configuration
|
|
# # [streams.httpserver.ssl]
|
|
# # enabled = true
|
|
# # cert_file = "/etc/logwisp/certs/server.crt"
|
|
# # key_file = "/etc/logwisp/certs/server.key"
|
|
# # min_version = "TLS1.2"
|
|
#
|
|
# # Future: Authentication
|
|
# # [streams.auth]
|
|
# # type = "basic"
|
|
# # [streams.auth.basic_auth]
|
|
# # users_file = "/etc/logwisp/security.users"
|
|
# # realm = "Security Logs"
|
|
|
|
# ------------------------------------------------------------------------------
|
|
# Example: Multi-Application Logs with Service Filtering
|
|
# ------------------------------------------------------------------------------
|
|
# [[streams]]
|
|
# name = "microservices"
|
|
#
|
|
# [streams.monitor]
|
|
# check_interval_ms = 100
|
|
# targets = [
|
|
# { path = "/var/log/containers", pattern = "*.log", is_file = false },
|
|
# ]
|
|
#
|
|
# # Filter by service name
|
|
# [[streams.filters]]
|
|
# type = "include"
|
|
# patterns = [
|
|
# "service=(api|auth|user|order)", # Specific services
|
|
# "pod=(api|auth|user|order)-" # Kubernetes pods
|
|
# ]
|
|
#
|
|
# # Exclude Kubernetes noise
|
|
# [[streams.filters]]
|
|
# type = "exclude"
|
|
# patterns = [
|
|
# "kube-system",
|
|
# "kube-proxy",
|
|
# "Readiness probe",
|
|
# "Liveness probe"
|
|
# ]
|
|
#
|
|
# [streams.httpserver]
|
|
# enabled = true
|
|
# port = 8085
|
|
# buffer_size = 5000
|
|
|
|
# ==============================================================================
|
|
# FILTER PATTERN EXAMPLES
|
|
# ==============================================================================
|
|
#
|
|
# Basic Patterns:
|
|
# - "ERROR" # Exact match
|
|
# - "(?i)error" # Case-insensitive
|
|
# - "\\berror\\b" # Word boundary (won't match "errorCode")
|
|
# - "error|warn|fatal" # Multiple options
|
|
#
|
|
# Complex Patterns:
|
|
# - "^\\[ERROR\\]" # Line starts with [ERROR]
|
|
# - "status=[4-5][0-9]{2}" # HTTP 4xx or 5xx status codes
|
|
# - "duration>[0-9]{4}ms" # Duration over 999ms
|
|
# - "user_id=\"[^\"]+\"" # Extract user_id values
|
|
#
|
|
# Performance Tips:
|
|
# - Avoid nested quantifiers: "((a+)+)+" can cause catastrophic backtracking
|
|
# - Use anchors when possible: "^ERROR" is faster than "ERROR"
|
|
# - Prefer character classes: "[0-9]" over "\\d" for clarity
|
|
# - Test complex patterns with sample data before deployment
|
|
#
|
|
# Security Considerations:
|
|
# - Be aware of ReDoS (Regular Expression Denial of Service)
|
|
# - Limit pattern complexity for public-facing streams
|
|
# - Monitor filter processing time in statistics
|
|
# - Consider pre-filtering very high volume streams
|
|
|
|
# ==============================================================================
|
|
# USAGE EXAMPLES
|
|
# ==============================================================================
|
|
|
|
# 1. Basic usage (single stream):
|
|
# ./logwisp
|
|
# - Monitors current directory for *.log files
|
|
# - Access logs at: http://localhost:8080/stream
|
|
# - View stats at: http://localhost:8080/status
|
|
|
|
# 2. Multi-stream configuration:
|
|
# - Uncomment additional [[streams]] sections above
|
|
# - Each stream runs independently on its own port
|
|
# - Different check intervals for different log types
|
|
# - Different filters for each stream
|
|
|
|
# 3. Router mode (consolidated access):
|
|
# ./logwisp --router
|
|
# - All streams accessible via paths: /streamname/stream
|
|
# - Global status at: /status
|
|
# - Example: http://localhost:8080/app/stream
|
|
|
|
# 4. Production deployment:
|
|
# - Enable filters to reduce noise and bandwidth
|
|
# - Enable rate limiting on public-facing streams
|
|
# - Use TCP for internal high-volume streams
|
|
# - Set appropriate check intervals (higher = less CPU)
|
|
# - Configure heartbeats for long-lived connections
|
|
|
|
# 5. Monitoring:
|
|
# curl http://localhost:8080/status | jq .
|
|
# - Check active connections
|
|
# - Monitor filter statistics (matched/dropped)
|
|
# - Monitor rate limit statistics
|
|
# - Track log entry counts
|
|
|
|
# ==============================================================================
|
|
# ENVIRONMENT VARIABLES
|
|
# ==============================================================================
|
|
# Configuration can be overridden via environment variables:
|
|
# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
|
|
# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
|
|
# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
|
# LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
|
|
# LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
|
|
|
|
# ==============================================================================
|
|
# NOTES
|
|
# ==============================================================================
|
|
# - Filters are processed sequentially - all must pass
|
|
# - Empty filter patterns means "pass everything"
|
|
# - Rate limiting is disabled by default for backward compatibility
|
|
# - Each stream can have different rate limit settings
|
|
# - TCP connections are silently dropped when rate limited
|
|
# - HTTP returns 429 (or configured code) with JSON error
|
|
# - IP tracking is cleaned up after 5 minutes of inactivity
|
|
# - Token bucket algorithm provides smooth rate limiting
|
|
# - Connection limits prevent resource exhaustion
|
|
# - Regex patterns are compiled once at startup for performance
|
|
# - Complex patterns can impact performance - monitor statistics |