v0.1.11 configurable logging added, minor refactoring, orgnized docs added
This commit is contained in:
@ -1,10 +1,68 @@
|
||||
# LogWisp Configuration File
|
||||
# LogWisp Configuration File - Complete Reference
|
||||
# Default path: ~/.config/logwisp.toml
|
||||
# Override with: ./logwisp --config /path/to/config.toml
|
||||
|
||||
# This is a complete configuration reference showing all available options.
|
||||
# Default values are uncommented, alternatives and examples are commented.
|
||||
|
||||
# ==============================================================================
|
||||
# LOGGING CONFIGURATION (LogWisp's own operational logs)
|
||||
# ==============================================================================
|
||||
# Controls where and how LogWisp logs its own operational messages.
|
||||
# This is separate from the logs being monitored and streamed.
|
||||
|
||||
[logging]
|
||||
# Output mode: where to write LogWisp's operational logs
|
||||
# Options: "file", "stdout", "stderr", "both", "none"
|
||||
# - file: Write only to log files
|
||||
# - stdout: Write only to standard output
|
||||
# - stderr: Write only to standard error (default for containers)
|
||||
# - both: Write to both file and console
|
||||
# - none: Disable logging (⚠️ SECURITY: Not recommended)
|
||||
output = "stderr"
|
||||
|
||||
# Minimum log level for operational logs
|
||||
# Options: "debug", "info", "warn", "error"
|
||||
# - debug: Maximum verbosity, includes internal state changes
|
||||
# - info: Normal operational messages (default)
|
||||
# - warn: Warnings and errors only
|
||||
# - error: Errors only
|
||||
level = "info"
|
||||
|
||||
# File output configuration (used when output includes "file" or "both")
|
||||
[logging.file]
|
||||
# Directory for log files
|
||||
directory = "./logs"
|
||||
|
||||
# Base name for log files (will append timestamp and .log)
|
||||
name = "logwisp"
|
||||
|
||||
# Maximum size per log file before rotation (megabytes)
|
||||
max_size_mb = 100
|
||||
|
||||
# Maximum total size of all log files (megabytes)
|
||||
# Oldest files are deleted when limit is reached
|
||||
max_total_size_mb = 1000
|
||||
|
||||
# How long to keep log files (hours)
|
||||
# 0 = no time-based deletion
|
||||
retention_hours = 168.0 # 7 days
|
||||
|
||||
# Console output configuration
|
||||
[logging.console]
|
||||
# Target for console output
|
||||
# Options: "stdout", "stderr", "split"
|
||||
# - stdout: All logs to standard output
|
||||
# - stderr: All logs to standard error (default)
|
||||
# - split: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
|
||||
target = "stderr"
|
||||
|
||||
# Output format
|
||||
# Options: "txt", "json"
|
||||
# - txt: Human-readable text format
|
||||
# - json: Structured JSON for log aggregation
|
||||
format = "txt"
|
||||
|
||||
# ==============================================================================
|
||||
# STREAM CONFIGURATION
|
||||
# ==============================================================================
|
||||
@ -16,22 +74,35 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
[[streams]]
|
||||
# Stream identifier used in logs, metrics, and router paths
|
||||
# Must be unique across all streams
|
||||
name = "default"
|
||||
|
||||
# File monitoring configuration
|
||||
[streams.monitor]
|
||||
# How often to check for new log entries (milliseconds)
|
||||
# Lower = faster detection but more CPU usage
|
||||
# Range: 10-60000 (0.01 to 60 seconds)
|
||||
check_interval_ms = 100
|
||||
|
||||
# Targets to monitor - can be files or directories
|
||||
# At least one target is required
|
||||
targets = [
|
||||
# Monitor all .log files in current directory
|
||||
{ path = "./", pattern = "*.log", is_file = false },
|
||||
|
||||
# Example: Monitor specific file
|
||||
# { path = "/var/log/app.log", is_file = true },
|
||||
|
||||
# Example: Multiple patterns in a directory
|
||||
# { path = "/logs", pattern = "*.log", is_file = false },
|
||||
# { path = "/logs", pattern = "*.txt", is_file = false },
|
||||
]
|
||||
|
||||
# Filter configuration (optional) - controls which logs are streamed
|
||||
# Multiple filters are applied sequentially - all must pass
|
||||
# Empty patterns array means "match everything"
|
||||
|
||||
# Example: Include only errors and warnings
|
||||
# [[streams.filters]]
|
||||
# type = "include" # "include" (whitelist) or "exclude" (blacklist)
|
||||
# logic = "or" # "or" (match any) or "and" (match all)
|
||||
@ -40,31 +111,124 @@ targets = [
|
||||
# "(?i)warn" # Case-insensitive warning matching
|
||||
# ]
|
||||
|
||||
# Example: Exclude debug and trace logs
|
||||
# [[streams.filters]]
|
||||
# type = "exclude"
|
||||
# patterns = ["DEBUG", "TRACE", "VERBOSE"]
|
||||
|
||||
# HTTP Server configuration (SSE/Server-Sent Events)
|
||||
[streams.httpserver]
|
||||
# Enable/disable HTTP server for this stream
|
||||
enabled = true
|
||||
|
||||
# Port to listen on (1-65535)
|
||||
# Each stream needs a unique port unless using router mode
|
||||
port = 8080
|
||||
buffer_size = 1000 # Per-client buffer size (messages)
|
||||
stream_path = "/stream" # Endpoint for SSE stream
|
||||
status_path = "/status" # Endpoint for statistics
|
||||
|
||||
# Per-client buffer size (number of messages)
|
||||
# Larger = handles bursts better, more memory per client
|
||||
buffer_size = 1000
|
||||
|
||||
# Endpoint paths (must start with /)
|
||||
stream_path = "/stream" # SSE stream endpoint
|
||||
status_path = "/status" # Statistics endpoint
|
||||
|
||||
# Keep-alive heartbeat configuration
|
||||
# Prevents connection timeout on quiet logs
|
||||
[streams.httpserver.heartbeat]
|
||||
# Enable/disable heartbeat messages
|
||||
enabled = true
|
||||
interval_seconds = 30 # Send heartbeat every 30 seconds
|
||||
format = "comment" # SSE comment format (: heartbeat)
|
||||
include_timestamp = true # Include timestamp in heartbeat
|
||||
include_stats = false # Include connection stats
|
||||
|
||||
# Interval between heartbeats (seconds)
|
||||
# Range: 1-3600 (1 second to 1 hour)
|
||||
interval_seconds = 30
|
||||
|
||||
# Heartbeat format
|
||||
# Options: "comment", "json"
|
||||
# - comment: SSE comment format (: heartbeat)
|
||||
# - json: JSON event format (data: {"type":"heartbeat"})
|
||||
format = "comment"
|
||||
|
||||
# Include timestamp in heartbeat
|
||||
include_timestamp = true
|
||||
|
||||
# Include connection statistics
|
||||
include_stats = false
|
||||
|
||||
# Rate limiting configuration (disabled by default)
|
||||
# Protects against abuse and resource exhaustion
|
||||
[streams.httpserver.rate_limit]
|
||||
# Enable/disable rate limiting
|
||||
enabled = false
|
||||
# requests_per_second = 10.0 # Token refill rate
|
||||
# burst_size = 20 # Max burst capacity
|
||||
# limit_by = "ip" # "ip" or "global"
|
||||
# response_code = 429 # HTTP Too Many Requests
|
||||
|
||||
# Token refill rate (requests per second)
|
||||
# Float value, e.g., 0.5 = 1 request every 2 seconds
|
||||
# requests_per_second = 10.0
|
||||
|
||||
# Maximum burst capacity (token bucket size)
|
||||
# Should be 2-3x requests_per_second for normal usage
|
||||
# burst_size = 20
|
||||
|
||||
# Rate limit strategy
|
||||
# Options: "ip", "global"
|
||||
# - ip: Each client IP gets its own limit
|
||||
# - global: All clients share one limit
|
||||
# limit_by = "ip"
|
||||
|
||||
# HTTP response code when rate limited
|
||||
# Common: 429 (Too Many Requests), 503 (Service Unavailable)
|
||||
# response_code = 429
|
||||
|
||||
# Response message when rate limited
|
||||
# response_message = "Rate limit exceeded"
|
||||
# max_connections_per_ip = 5 # Max SSE connections per IP
|
||||
|
||||
# Maximum concurrent connections per IP address
|
||||
# 0 = unlimited
|
||||
# max_connections_per_ip = 5
|
||||
|
||||
# Maximum total concurrent connections
|
||||
# 0 = unlimited
|
||||
# max_total_connections = 100
|
||||
|
||||
# SSL/TLS configuration (planned feature)
|
||||
# [streams.httpserver.ssl]
|
||||
# enabled = false
|
||||
# cert_file = "/path/to/cert.pem"
|
||||
# key_file = "/path/to/key.pem"
|
||||
# min_version = "TLS1.2" # Minimum TLS version
|
||||
# client_auth = false # Require client certificates
|
||||
|
||||
# TCP Server configuration (optional)
|
||||
# Raw TCP streaming for high-performance scenarios
|
||||
# [streams.tcpserver]
|
||||
# enabled = false
|
||||
# port = 9090
|
||||
# buffer_size = 5000 # Larger buffer for TCP
|
||||
#
|
||||
# [streams.tcpserver.heartbeat]
|
||||
# enabled = true
|
||||
# interval_seconds = 60
|
||||
# include_timestamp = true
|
||||
# include_stats = false
|
||||
#
|
||||
# [streams.tcpserver.rate_limit]
|
||||
# enabled = false
|
||||
# requests_per_second = 5.0
|
||||
# burst_size = 10
|
||||
# limit_by = "ip"
|
||||
|
||||
# Authentication configuration (planned feature)
|
||||
# [streams.auth]
|
||||
# type = "none" # Options: "none", "basic", "bearer"
|
||||
#
|
||||
# # Basic authentication
|
||||
# [streams.auth.basic_auth]
|
||||
# users_file = "/etc/logwisp/users.htpasswd"
|
||||
# realm = "LogWisp"
|
||||
#
|
||||
# # IP-based access control
|
||||
# ip_whitelist = ["192.168.1.0/24", "10.0.0.0/8"]
|
||||
# ip_blacklist = []
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Application Logs Stream with Error Filtering
|
||||
@ -92,6 +256,8 @@ enabled = false
|
||||
# "(?i)\\bcritical\\b", # critical
|
||||
# "(?i)exception", # exception anywhere
|
||||
# "(?i)fail(ed|ure)?", # fail, failed, failure
|
||||
# "panic", # Go panics
|
||||
# "traceback", # Python tracebacks
|
||||
# ]
|
||||
#
|
||||
# # Filter 2: Exclude health check noise
|
||||
@ -100,7 +266,10 @@ enabled = false
|
||||
# patterns = [
|
||||
# "/health",
|
||||
# "/metrics",
|
||||
# "GET /ping"
|
||||
# "/ping",
|
||||
# "GET /favicon.ico",
|
||||
# "ELB-HealthChecker",
|
||||
# "kube-probe"
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
@ -125,6 +294,7 @@ enabled = false
|
||||
# burst_size = 50
|
||||
# limit_by = "ip"
|
||||
# max_connections_per_ip = 10
|
||||
# max_total_connections = 200
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: System Logs Stream (TCP + HTTP) with Security Filtering
|
||||
@ -138,6 +308,7 @@ enabled = false
|
||||
# { path = "/var/log/syslog", is_file = true },
|
||||
# { path = "/var/log/auth.log", is_file = true },
|
||||
# { path = "/var/log/kern.log", is_file = true },
|
||||
# { path = "/var/log/messages", is_file = true },
|
||||
# ]
|
||||
#
|
||||
# # Include only security-relevant logs
|
||||
@ -152,7 +323,12 @@ enabled = false
|
||||
# "(?i)permission",
|
||||
# "(?i)denied",
|
||||
# "(?i)unauthorized",
|
||||
# "kernel:.*audit"
|
||||
# "(?i)security",
|
||||
# "(?i)selinux",
|
||||
# "kernel:.*audit",
|
||||
# "COMMAND=", # sudo commands
|
||||
# "session opened",
|
||||
# "session closed"
|
||||
# ]
|
||||
#
|
||||
# # TCP Server for high-performance streaming
|
||||
@ -182,9 +358,15 @@ enabled = false
|
||||
# buffer_size = 1000
|
||||
# stream_path = "/stream"
|
||||
# status_path = "/status"
|
||||
#
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 5.0
|
||||
# burst_size = 10
|
||||
# max_connections_per_ip = 2 # Strict for security logs
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: High-Volume Debug Logs with Filtering
|
||||
# Example: High-Volume Debug Logs with Performance Filtering
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "debug"
|
||||
@ -193,6 +375,7 @@ enabled = false
|
||||
# check_interval_ms = 5000 # Check every 5 seconds (high volume)
|
||||
# targets = [
|
||||
# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
|
||||
# { path = "/var/log/debug", pattern = "debug-*.log", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# # Exclude verbose debug output
|
||||
@ -203,15 +386,19 @@ enabled = false
|
||||
# "VERBOSE",
|
||||
# "entering function",
|
||||
# "exiting function",
|
||||
# "memory dump"
|
||||
# "memory dump",
|
||||
# "hex dump",
|
||||
# "stack trace",
|
||||
# "goroutine [0-9]+"
|
||||
# ]
|
||||
#
|
||||
# # Include only specific modules
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# patterns = [
|
||||
# "module:(api|database|auth)",
|
||||
# "component:(router|handler)"
|
||||
# "module=(api|database|auth)",
|
||||
# "component=(router|handler)",
|
||||
# "service=(payment|order|user)"
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
@ -232,260 +419,295 @@ enabled = false
|
||||
# burst_size = 5
|
||||
# limit_by = "ip"
|
||||
# max_connections_per_ip = 1 # One connection per IP
|
||||
# response_code = 503 # Service Unavailable
|
||||
# response_message = "Debug stream overloaded"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Database Logs with Complex Filtering
|
||||
# Example: Multi-Application with Router Mode
|
||||
# ------------------------------------------------------------------------------
|
||||
# Run with: logwisp --router
|
||||
#
|
||||
# [[streams]]
|
||||
# name = "frontend"
|
||||
# [streams.monitor]
|
||||
# targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
|
||||
# [[streams.filters]]
|
||||
# type = "exclude"
|
||||
# patterns = ["GET /static/", "GET /assets/"]
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8080 # Same port OK in router mode
|
||||
#
|
||||
# [[streams]]
|
||||
# name = "backend"
|
||||
# [streams.monitor]
|
||||
# targets = [{ path = "/var/log/api", pattern = "*.log" }]
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# patterns = ["ERROR", "WARN", "timeout", "failed"]
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8080 # Shared port in router mode
|
||||
#
|
||||
# [[streams]]
|
||||
# name = "database"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 200
|
||||
# targets = [
|
||||
# { path = "/var/log/postgresql", pattern = "*.log", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# # Complex AND filter - must match all patterns
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# logic = "and" # Must match ALL patterns
|
||||
# patterns = [
|
||||
# "(?i)error|fail", # Must contain error or fail
|
||||
# "(?i)connection|query", # AND must be about connections or queries
|
||||
# "(?i)timeout|deadlock" # AND must involve timeout or deadlock
|
||||
# ]
|
||||
#
|
||||
# # Exclude routine maintenance
|
||||
# [[streams.filters]]
|
||||
# type = "exclude"
|
||||
# patterns = [
|
||||
# "VACUUM",
|
||||
# "ANALYZE",
|
||||
# "checkpoint"
|
||||
# ]
|
||||
#
|
||||
# [streams.tcpserver]
|
||||
# enabled = true
|
||||
# port = 9091
|
||||
# buffer_size = 2000
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: API Access Logs with Pattern Extraction
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "api-access"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 100
|
||||
# targets = [
|
||||
# { path = "/var/log/nginx/access.log", is_file = true },
|
||||
# ]
|
||||
#
|
||||
# # Include only API endpoints
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# patterns = [
|
||||
# '"/api/v[0-9]+/', # API versioned endpoints
|
||||
# '"(GET|POST|PUT|DELETE) /api/' # API requests
|
||||
# ]
|
||||
#
|
||||
# # Exclude specific status codes
|
||||
# [[streams.filters]]
|
||||
# type = "exclude"
|
||||
# patterns = [
|
||||
# '" 200 ', # Success responses
|
||||
# '" 204 ', # No content
|
||||
# '" 304 ', # Not modified
|
||||
# 'OPTIONS ' # CORS preflight
|
||||
# ]
|
||||
#
|
||||
# targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8084
|
||||
# buffer_size = 3000
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Security/Audit Logs with Strict Filtering
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "security"
|
||||
# port = 8080
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 100
|
||||
# targets = [
|
||||
# { path = "/var/log/audit", pattern = "audit.log*", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# # Security-focused patterns
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# logic = "or"
|
||||
# patterns = [
|
||||
# "type=USER_AUTH",
|
||||
# "type=USER_LOGIN",
|
||||
# "type=USER_LOGOUT",
|
||||
# "type=USER_ERR",
|
||||
# "type=CRED_", # All credential operations
|
||||
# "type=PRIV_", # All privilege operations
|
||||
# "type=ANOM_", # All anomalies
|
||||
# "type=RESP_", # All responses
|
||||
# "failed|failure",
|
||||
# "denied|unauthorized",
|
||||
# "violation",
|
||||
# "attack|intrusion"
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8443 # HTTPS port (for future TLS)
|
||||
# buffer_size = 1000
|
||||
# stream_path = "/audit"
|
||||
# status_path = "/health"
|
||||
#
|
||||
# # Strict rate limiting for security logs
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 2.0 # Very limited access
|
||||
# burst_size = 3
|
||||
# limit_by = "ip"
|
||||
# max_connections_per_ip = 1 # Single connection per IP
|
||||
# response_code = 403 # Forbidden instead of rate limit
|
||||
# response_message = "Access restricted"
|
||||
#
|
||||
# # Future: SSL/TLS configuration
|
||||
# # [streams.httpserver.ssl]
|
||||
# # enabled = true
|
||||
# # cert_file = "/etc/logwisp/certs/server.crt"
|
||||
# # key_file = "/etc/logwisp/certs/server.key"
|
||||
# # min_version = "TLS1.2"
|
||||
#
|
||||
# # Future: Authentication
|
||||
# # [streams.auth]
|
||||
# # type = "basic"
|
||||
# # [streams.auth.basic_auth]
|
||||
# # users_file = "/etc/logwisp/security.users"
|
||||
# # realm = "Security Logs"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Multi-Application Logs with Service Filtering
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "microservices"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 100
|
||||
# targets = [
|
||||
# { path = "/var/log/containers", pattern = "*.log", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# # Filter by service name
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# patterns = [
|
||||
# "service=(api|auth|user|order)", # Specific services
|
||||
# "pod=(api|auth|user|order)-" # Kubernetes pods
|
||||
# ]
|
||||
#
|
||||
# # Exclude Kubernetes noise
|
||||
# [[streams.filters]]
|
||||
# type = "exclude"
|
||||
# patterns = [
|
||||
# "kube-system",
|
||||
# "kube-proxy",
|
||||
# "Readiness probe",
|
||||
# "Liveness probe"
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8085
|
||||
# buffer_size = 5000
|
||||
# # Access via:
|
||||
# # http://localhost:8080/frontend/stream
|
||||
# # http://localhost:8080/backend/stream
|
||||
# # http://localhost:8080/database/stream
|
||||
# # http://localhost:8080/status (global)
|
||||
|
||||
# ==============================================================================
|
||||
# FILTER PATTERN EXAMPLES
|
||||
# FILTER PATTERN REFERENCE
|
||||
# ==============================================================================
|
||||
#
|
||||
# Basic Patterns:
|
||||
# - "ERROR" # Exact match
|
||||
# - "ERROR" # Exact match (case sensitive)
|
||||
# - "(?i)error" # Case-insensitive
|
||||
# - "\\berror\\b" # Word boundary (won't match "errorCode")
|
||||
# - "error|warn|fatal" # Multiple options
|
||||
# - "error|warn|fatal" # Multiple options (OR)
|
||||
# - "(error|warn) level" # Group with context
|
||||
#
|
||||
# Position Patterns:
|
||||
# - "^\\[ERROR\\]" # Line starts with [ERROR]
|
||||
# - "ERROR:$" # Line ends with ERROR:
|
||||
# - "^\\d{4}-\\d{2}-\\d{2}" # Line starts with date
|
||||
#
|
||||
# Complex Patterns:
|
||||
# - "^\\[ERROR\\]" # Line starts with [ERROR]
|
||||
# - "status=[4-5][0-9]{2}" # HTTP 4xx or 5xx status codes
|
||||
# - "duration>[0-9]{4}ms" # Duration over 999ms
|
||||
# - "user_id=\"[^\"]+\"" # Extract user_id values
|
||||
# - "\\[ERROR\\].*database" # ERROR followed by database
|
||||
# - "(?i)\\b(error|fail|critical)\\b" # Multiple error words
|
||||
#
|
||||
# Log Level Patterns:
|
||||
# - "\\[(ERROR|WARN|FATAL)\\]" # Common formats
|
||||
# - "level=(error|warning|critical)" # Key-value format
|
||||
# - "ERROR\\s*:" # ERROR with optional space
|
||||
# - "<(Error|Warning)>" # XML-style
|
||||
#
|
||||
# Application Patterns:
|
||||
# - "com\\.mycompany\\..*Exception" # Java exceptions
|
||||
# - "at .+\\(.+\\.java:[0-9]+\\)" # Java stack traces
|
||||
# - "File \".+\", line [0-9]+" # Python tracebacks
|
||||
# - "panic: .+" # Go panics
|
||||
# - "/api/v[0-9]+/" # API versioned paths
|
||||
#
|
||||
# Performance Patterns:
|
||||
# - "took [0-9]{4,}ms" # Operations over 999ms
|
||||
# - "memory usage: [8-9][0-9]%" # High memory usage
|
||||
# - "queue size: [0-9]{4,}" # Large queues
|
||||
# - "timeout|timed out" # Timeouts
|
||||
#
|
||||
# Security Patterns:
|
||||
# - "unauthorized|forbidden" # Access denied
|
||||
# - "invalid token|expired token" # Auth failures
|
||||
# - "SQL injection|XSS" # Security threats
|
||||
# - "failed login.*IP: ([0-9.]+)" # Failed logins with IP
|
||||
#
|
||||
# Performance Tips:
|
||||
# - Avoid nested quantifiers: "((a+)+)+" can cause catastrophic backtracking
|
||||
# - Avoid nested quantifiers: "((a+)+)+" causes catastrophic backtracking
|
||||
# - Use anchors when possible: "^ERROR" is faster than "ERROR"
|
||||
# - Prefer character classes: "[0-9]" over "\\d" for clarity
|
||||
# - Use non-capturing groups: "(?:error|warn)" when not extracting
|
||||
# - Test complex patterns with sample data before deployment
|
||||
# - Consider using multiple simple patterns instead of one complex pattern
|
||||
#
|
||||
# Security Considerations:
|
||||
# - Be aware of ReDoS (Regular Expression Denial of Service)
|
||||
# - Limit pattern complexity for public-facing streams
|
||||
# - Monitor filter processing time in statistics
|
||||
# - Consider pre-filtering very high volume streams
|
||||
# - Use explicit allow-lists for sensitive logs
|
||||
|
||||
# ==============================================================================
|
||||
# USAGE EXAMPLES
|
||||
# RATE LIMITING GUIDE
|
||||
# ==============================================================================
|
||||
|
||||
# 1. Basic usage (single stream):
|
||||
# ./logwisp
|
||||
# - Monitors current directory for *.log files
|
||||
# - Access logs at: http://localhost:8080/stream
|
||||
# - View stats at: http://localhost:8080/status
|
||||
|
||||
# 2. Multi-stream configuration:
|
||||
# - Uncomment additional [[streams]] sections above
|
||||
# - Each stream runs independently on its own port
|
||||
# - Different check intervals for different log types
|
||||
# - Different filters for each stream
|
||||
|
||||
# 3. Router mode (consolidated access):
|
||||
# ./logwisp --router
|
||||
# - All streams accessible via paths: /streamname/stream
|
||||
# - Global status at: /status
|
||||
# - Example: http://localhost:8080/app/stream
|
||||
|
||||
# 4. Production deployment:
|
||||
# - Enable filters to reduce noise and bandwidth
|
||||
# - Enable rate limiting on public-facing streams
|
||||
# - Use TCP for internal high-volume streams
|
||||
# - Set appropriate check intervals (higher = less CPU)
|
||||
# - Configure heartbeats for long-lived connections
|
||||
|
||||
# 5. Monitoring:
|
||||
# curl http://localhost:8080/status | jq .
|
||||
# - Check active connections
|
||||
# - Monitor filter statistics (matched/dropped)
|
||||
# - Monitor rate limit statistics
|
||||
# - Track log entry counts
|
||||
#
|
||||
# Token Bucket Algorithm:
|
||||
# - Each client (IP) or global limit gets a bucket with 'burst_size' tokens
|
||||
# - Tokens refill at 'requests_per_second' rate
|
||||
# - Each request consumes one token
|
||||
# - Provides smooth rate limiting without hard cutoffs
|
||||
#
|
||||
# Configuration Examples:
|
||||
#
|
||||
# Light Protection (default for most streams):
|
||||
# requests_per_second = 10.0
|
||||
# burst_size = 20 # Handle short bursts
|
||||
#
|
||||
# Moderate Protection (public endpoints):
|
||||
# requests_per_second = 5.0
|
||||
# burst_size = 15
|
||||
# max_connections_per_ip = 5
|
||||
#
|
||||
# Strict Protection (sensitive logs):
|
||||
# requests_per_second = 1.0
|
||||
# burst_size = 3
|
||||
# max_connections_per_ip = 1
|
||||
# limit_by = "ip"
|
||||
#
|
||||
# Global Limiting (shared resource):
|
||||
# requests_per_second = 50.0 # Total for all clients
|
||||
# burst_size = 100
|
||||
# limit_by = "global"
|
||||
# max_total_connections = 50
|
||||
#
|
||||
# Behavior:
|
||||
# - HTTP: Returns response_code (default 429) with JSON error
|
||||
# - TCP: Silently drops connection (no error message)
|
||||
# - Cleanup: Inactive IPs removed after 5 minutes
|
||||
# - Statistics: Available in /status endpoint
|
||||
#
|
||||
# Best Practices:
|
||||
# - Set burst_size to 2-3x requests_per_second
|
||||
# - Use per-IP limiting for fairness
|
||||
# - Use global limiting for resource protection
|
||||
# - Monitor rate limit statistics for tuning
|
||||
# - Consider different limits for different streams
|
||||
# - Enable for any public-facing endpoints
|
||||
|
||||
# ==============================================================================
|
||||
# ENVIRONMENT VARIABLES
|
||||
# PERFORMANCE TUNING
|
||||
# ==============================================================================
|
||||
# Configuration can be overridden via environment variables:
|
||||
# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
|
||||
# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
|
||||
# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
||||
# LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
|
||||
# LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
|
||||
#
|
||||
# Monitor Check Interval:
|
||||
# - 10-50ms: Real-time monitoring, higher CPU usage
|
||||
# - 100-500ms: Good balance for active logs
|
||||
# - 1000-5000ms: Low-activity logs, minimal CPU
|
||||
# - 10000ms+: Very slow changing logs
|
||||
#
|
||||
# Buffer Sizes:
|
||||
# - HTTP: 100-1000 for normal use, 5000+ for high volume
|
||||
# - TCP: 1000-5000 typical, 10000+ for bulk streaming
|
||||
# - Larger = more memory per client, handles bursts better
|
||||
#
|
||||
# Connection Limits:
|
||||
# - Development: No limits needed
|
||||
# - Production: 5-10 connections per IP typical
|
||||
# - Public: 1-3 connections per IP
|
||||
# - Total: Based on available memory (each uses ~1-5MB)
|
||||
#
|
||||
# Filter Performance:
|
||||
# - Simple patterns: ~1μs per check
|
||||
# - Complex patterns: ~10-100μs per check
|
||||
# - Many patterns: Consider multiple streams instead
|
||||
# - Use exclude filters to drop noise early
|
||||
#
|
||||
# Memory Usage (approximate):
|
||||
# - Base process: ~10-20MB
|
||||
# - Per stream: ~5-10MB
|
||||
# - Per HTTP client: ~1-2MB
|
||||
# - Per TCP client: ~0.5-1MB
|
||||
# - Filter chain: ~1-5MB depending on patterns
|
||||
|
||||
# ==============================================================================
|
||||
# NOTES
|
||||
# DEPLOYMENT SCENARIOS
|
||||
# ==============================================================================
|
||||
# - Filters are processed sequentially - all must pass
|
||||
# - Empty filter patterns means "pass everything"
|
||||
# - Rate limiting is disabled by default for backward compatibility
|
||||
# - Each stream can have different rate limit settings
|
||||
# - TCP connections are silently dropped when rate limited
|
||||
# - HTTP returns 429 (or configured code) with JSON error
|
||||
# - IP tracking is cleaned up after 5 minutes of inactivity
|
||||
# - Token bucket algorithm provides smooth rate limiting
|
||||
# - Connection limits prevent resource exhaustion
|
||||
# - Regex patterns are compiled once at startup for performance
|
||||
# - Complex patterns can impact performance - monitor statistics
|
||||
#
|
||||
# Single Application:
|
||||
# - One stream with basic filtering
|
||||
# - Moderate rate limiting
|
||||
# - Standard check interval (100ms)
|
||||
#
|
||||
# Microservices:
|
||||
# - Multiple streams, one per service
|
||||
# - Router mode for unified access
|
||||
# - Different filter rules per service
|
||||
# - Service-specific rate limits
|
||||
#
|
||||
# High Security:
|
||||
# - Strict include filters
|
||||
# - Low rate limits (1-2 req/sec)
|
||||
# - Single connection per IP
|
||||
# - TCP for internal, HTTP for external
|
||||
#
|
||||
# High Performance:
|
||||
# - TCP streaming preferred
|
||||
# - Large buffers (10000+)
|
||||
# - Minimal filtering
|
||||
# - Higher check intervals
|
||||
# - No heartbeats
|
||||
#
|
||||
# Development/Testing:
|
||||
# - Multiple streams for different log levels
|
||||
# - No rate limiting
|
||||
# - Debug level logging
|
||||
# - Fast check intervals
|
||||
# - All filters disabled
|
||||
|
||||
# ==============================================================================
|
||||
# TROUBLESHOOTING
|
||||
# ==============================================================================
|
||||
#
|
||||
# Common Issues:
|
||||
#
|
||||
# "No logs appearing":
|
||||
# - Check file paths and permissions
|
||||
# - Verify pattern matches filenames
|
||||
# - Check filters aren't too restrictive
|
||||
# - Enable debug logging: --log-level debug
|
||||
#
|
||||
# "High CPU usage":
|
||||
# - Increase check_interval_ms
|
||||
# - Reduce number of filter patterns
|
||||
# - Use simpler regex patterns
|
||||
# - Check for runaway log growth
|
||||
#
|
||||
# "Clients disconnecting":
|
||||
# - Enable heartbeats
|
||||
# - Check rate limiting settings
|
||||
# - Verify network connectivity
|
||||
# - Increase buffer sizes
|
||||
#
|
||||
# "Memory growth":
|
||||
# - Check client connection count
|
||||
# - Verify buffer sizes are reasonable
|
||||
# - Look for memory leaks in filters
|
||||
# - Enable connection limits
|
||||
#
|
||||
# Debug Commands:
|
||||
# - Check status: curl http://localhost:8080/status
|
||||
# - Test stream: curl -N http://localhost:8080/stream
|
||||
# - View logs: logwisp --log-level debug --log-output stderr
|
||||
# - Test filters: Use simple patterns first
|
||||
|
||||
# ==============================================================================
|
||||
# FUTURE FEATURES (Roadmap)
|
||||
# ==============================================================================
|
||||
#
|
||||
# Authentication:
|
||||
# - Basic auth with htpasswd files
|
||||
# - Bearer token authentication
|
||||
# - JWT validation
|
||||
# - mTLS client certificates
|
||||
#
|
||||
# SSL/TLS:
|
||||
# - HTTPS endpoints
|
||||
# - TLS for TCP streams
|
||||
# - Certificate management
|
||||
# - Let's Encrypt integration
|
||||
#
|
||||
# Advanced Filtering:
|
||||
# - Lua scripting for complex logic
|
||||
# - Rate-based filtering (N per minute)
|
||||
# - Statistical anomaly detection
|
||||
# - Multi-line pattern matching
|
||||
#
|
||||
# Output Formats:
|
||||
# - JSON transformation
|
||||
# - Field extraction
|
||||
# - Custom formatting templates
|
||||
# - Compression (gzip)
|
||||
#
|
||||
# Integrations:
|
||||
# - Prometheus metrics
|
||||
# - OpenTelemetry traces
|
||||
# - Webhook notifications
|
||||
# - Cloud storage backends
|
||||
@ -1,120 +0,0 @@
|
||||
# LogWisp Configuration Example
|
||||
# Default path: ~/.config/logwisp.toml
|
||||
|
||||
# Application logs - public facing
|
||||
[[streams]]
|
||||
name = "app-public"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 100
|
||||
targets = [
|
||||
{ path = "/var/log/nginx", pattern = "access.log*", is_file = false },
|
||||
{ path = "/var/log/app", pattern = "production.log", is_file = true }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8080
|
||||
buffer_size = 2000
|
||||
stream_path = "/logs"
|
||||
status_path = "/health"
|
||||
|
||||
[streams.httpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 30
|
||||
format = "json"
|
||||
include_timestamp = true
|
||||
include_stats = true
|
||||
|
||||
# Rate limiting for public endpoint
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 50.0
|
||||
burst_size = 100
|
||||
limit_by = "ip"
|
||||
response_code = 429
|
||||
response_message = "Rate limit exceeded. Please retry after 60 seconds."
|
||||
max_connections_per_ip = 5
|
||||
max_total_connections = 100
|
||||
|
||||
# System logs - internal only
|
||||
[[streams]]
|
||||
name = "system"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 5000 # Check every 5 seconds
|
||||
targets = [
|
||||
{ path = "/var/log/syslog", is_file = true },
|
||||
{ path = "/var/log/auth.log", is_file = true },
|
||||
{ path = "/var/log/kern.log", is_file = true }
|
||||
]
|
||||
|
||||
# TCP for internal consumers
|
||||
[streams.tcpserver]
|
||||
enabled = true
|
||||
port = 9090
|
||||
buffer_size = 5000
|
||||
|
||||
[streams.tcpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 60
|
||||
include_timestamp = true
|
||||
|
||||
# Moderate rate limiting for internal use
|
||||
[streams.tcpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 10.0
|
||||
burst_size = 20
|
||||
limit_by = "ip"
|
||||
|
||||
# Security audit logs - restricted access
|
||||
[[streams]]
|
||||
name = "security"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 100
|
||||
targets = [
|
||||
{ path = "/var/log/audit", pattern = "*.log", is_file = false },
|
||||
{ path = "/var/log/fail2ban.log", is_file = true }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8443
|
||||
buffer_size = 1000
|
||||
stream_path = "/audit/stream"
|
||||
status_path = "/audit/status"
|
||||
|
||||
# Strict rate limiting
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 1.0
|
||||
burst_size = 3
|
||||
limit_by = "ip"
|
||||
max_connections_per_ip = 1
|
||||
response_code = 403
|
||||
response_message = "Access denied"
|
||||
|
||||
# Application debug logs - development team only
|
||||
[[streams]]
|
||||
name = "debug"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 1000
|
||||
targets = [
|
||||
{ path = "/var/log/app", pattern = "debug-*.log", is_file = false }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8090
|
||||
buffer_size = 5000
|
||||
stream_path = "/debug"
|
||||
status_path = "/debug/status"
|
||||
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 100.0 # Higher limit for internal use
|
||||
burst_size = 200
|
||||
limit_by = "ip"
|
||||
max_connections_per_ip = 10
|
||||
@ -1,7 +1,7 @@
|
||||
# LogWisp Minimal Configuration Example
|
||||
# LogWisp Minimal Configuration
|
||||
# Save as: ~/.config/logwisp.toml
|
||||
|
||||
# Monitor application logs
|
||||
# Basic stream monitoring application logs
|
||||
[[streams]]
|
||||
name = "app"
|
||||
|
||||
@ -11,20 +11,32 @@ targets = [
|
||||
{ path = "/var/log/myapp", pattern = "*.log", is_file = false }
|
||||
]
|
||||
|
||||
# Optional: Filter for errors and warnings only
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# patterns = ["ERROR", "WARN", "CRITICAL"]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8080
|
||||
stream_path = "/stream"
|
||||
status_path = "/status"
|
||||
|
||||
# Optional: Enable rate limiting
|
||||
# Optional additions:
|
||||
|
||||
# 1. Filter for errors only:
|
||||
# [[streams.filters]]
|
||||
# type = "include"
|
||||
# patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
|
||||
|
||||
# 2. Enable rate limiting:
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 10.0
|
||||
# burst_size = 20
|
||||
# limit_by = "ip"
|
||||
# limit_by = "ip"
|
||||
|
||||
# 3. Add heartbeat:
|
||||
# [streams.httpserver.heartbeat]
|
||||
# enabled = true
|
||||
# interval_seconds = 30
|
||||
|
||||
# 4. Change LogWisp's own logging:
|
||||
# [logging]
|
||||
# output = "file"
|
||||
# level = "info"
|
||||
Reference in New Issue
Block a user