diff --git a/.gitignore b/.gitignore
index 0a164b6..1d5f65f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
data
dev
log
+logs
cert
bin
script
diff --git a/README.md b/README.md
index 3015abe..5e392cb 100644
--- a/README.md
+++ b/README.md
@@ -1,750 +1,82 @@
-# LogWisp - Multi-Stream Log Monitoring Service
+
-
- 
-
-A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring, rotation detection, regex-based filtering, and rate limiting.
+**Multi-stream log monitoring with real-time streaming over HTTP/SSE and TCP**
-## Features
+LogWisp watches log files and streams updates to connected clients in real-time. Perfect for monitoring multiple applications, filtering noise, and centralizing log access.
-- **Multi-Stream Architecture**: Run multiple independent log streams, each with its own configuration
-- **Dual Protocol Support**: TCP (raw streaming) and HTTP/SSE (browser-friendly)
-- **Real-time Monitoring**: Instant updates with per-stream configurable check intervals
-- **File Rotation Detection**: Automatic detection and handling of log rotation
-- **Regex-based Filtering**: Include/exclude patterns with AND/OR logic per stream
-- **Path-based Routing**: Optional HTTP router for consolidated access
-- **Rate Limiting**: Per-IP or global rate limiting with token bucket algorithm
-- **Connection Limiting**: Configurable concurrent connection limits per IP
-- **Per-Stream Configuration**: Independent settings including check intervals, filters, and rate limits
-- **Connection Statistics**: Real-time monitoring of active connections, filter, and rate limit metrics
-- **Flexible Targets**: Monitor individual files or entire directories
-- **Version Management**: Git tag-based versioning with build information
-- **Configurable Heartbeats**: Keep connections alive with customizable formats
-- **Minimal Direct Dependencies**: panjf2000/gnet/v2, valyala/fasthttp, lixenwraith/config, and stdlib
-
-## Quick Start
+## ð Quick Start
```bash
-# Build with version information
-make build
+# Install
+go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
-# Run with default configuration if ~/.config/logwisp.toml doesn't exists
-./logwisp
+# Run with defaults (monitors *.log in current directory)
+logwisp
-# Run with custom config
-./logwisp --config /etc/logwisp/production.toml
-
-# Run with HTTP router (path-based routing)
-./logwisp --router
-
-# Show version information
-./logwisp --version
+# Stream logs (from another terminal)
+curl -N http://localhost:8080/stream
```
-## Architecture
+## âĻ Key Features
-LogWisp uses a service-oriented architecture where each stream is an independent pipeline:
+- **ðĄ Real-time Streaming** - SSE (HTTP) and TCP protocols
+- **ð Pattern Filtering** - Include/exclude logs with regex patterns
+- **ðĄïļ Rate Limiting** - Protect against abuse with configurable limits
+- **ð Multi-stream** - Monitor different log sources simultaneously
+- **ð Rotation Aware** - Handles log rotation seamlessly
+- **⥠High Performance** - Minimal CPU/memory footprint
-```
-LogStream Service
-âââ Stream["app-logs"]
-â âââ Monitor (watches files)
-â âââ Filter Chain (optional)
-â âââ Rate Limiter (optional)
-â âââ TCP Server (optional)
-â âââ HTTP Server (optional)
-âââ Stream["system-logs"]
-â âââ Monitor
-â âââ Filter Chain (optional)
-â âââ Rate Limiter (optional)
-â âââ HTTP Server
-âââ HTTP Router (optional, for path-based routing)
-```
+## ð Documentation
-## Configuration
+Complete documentation is available in the [`doc/`](doc/) directory:
-Default configuration file location: `~/.config/logwisp.toml`
+- [**Quick Start Guide**](doc/quickstart.md) - Get running in 5 minutes
+- [**Configuration**](doc/configuration.md) - All configuration options
+- [**CLI Reference**](doc/cli.md) - Command-line interface
+- [**Examples**](doc/examples/) - Ready-to-use configurations
-### Basic Multi-Stream Configuration
+## ðŧ Basic Usage
+
+### Monitor application logs with filtering:
```toml
-# Application logs transport
+# ~/.config/logwisp.toml
[[streams]]
-name = "app"
+name = "myapp"
[streams.monitor]
-# Per-transport check interval in milliseconds
-check_interval_ms = 100
-targets = [
- { path = "/var/log/myapp", pattern = "*.log", is_file = false },
- { path = "/var/log/myapp/app.log", is_file = true }
-]
+targets = [{ path = "/var/log/myapp", pattern = "*.log" }]
-# Filter configuration (optional)
[[streams.filters]]
-type = "include" # Only show matching logs
-logic = "or" # Match any pattern
-patterns = [
- "(?i)error", # Case-insensitive error
- "(?i)warn", # Case-insensitive warning
- "(?i)fatal" # Fatal errors
-]
+type = "include"
+patterns = ["ERROR", "WARN", "CRITICAL"]
[streams.httpserver]
enabled = true
port = 8080
-buffer_size = 2000
-stream_path = "/stream"
-status_path = "/status"
-
-# Heartbeat configuration
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-format = "comment" # or "json" for structured events
-include_timestamp = true
-include_stats = false
-
-# Rate limiting configuration
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-burst_size = 20
-limit_by = "ip"
-response_code = 429
-response_message = "Rate limit exceeded"
-max_connections_per_ip = 5
-
-# System logs transport with slower check interval
-[[streams]]
-name = "system"
-
-[streams.monitor]
-# Check every 60 seconds for slowly updating logs
-check_interval_ms = 60000
-targets = [
- { path = "/var/log/syslog", is_file = true },
- { path = "/var/log/auth.log", is_file = true }
-]
-
-# Exclude debug logs
-[[streams.filters]]
-type = "exclude"
-patterns = ["DEBUG", "TRACE"]
-
-[streams.tcpserver]
-enabled = true
-port = 9090
-buffer_size = 5000
-
-# TCP heartbeat (always JSON format)
-[streams.tcpserver.heartbeat]
-enabled = true
-interval_seconds = 300 # 5 minutes
-include_timestamp = true
-include_stats = true
-
-# TCP rate limiting
-[streams.tcpserver.rate_limit]
-enabled = true
-requests_per_second = 5.0
-burst_size = 10
-limit_by = "ip"
```
-### Target Configuration
-
-Monitor targets support both files and directories:
-
-```toml
-# Directory monitoring with pattern
-{ path = "/var/log", pattern = "*.log", is_file = false }
-
-# Specific file monitoring
-{ path = "/var/log/app.log", is_file = true }
-
-# All .log files in a directory
-{ path = "./logs", pattern = "*.log", is_file = false }
-```
-
-### Filter Configuration
-
-Control which logs are streamed using regex patterns:
-
-```toml
-# Include filter - only matching logs pass
-[[streams.filters]]
-type = "include"
-logic = "or" # Match ANY pattern
-patterns = [
- "ERROR",
- "WARN",
- "CRITICAL"
-]
-
-# Exclude filter - matching logs are dropped
-[[streams.filters]]
-type = "exclude"
-logic = "or" # Drop if ANY pattern matches
-patterns = [
- "DEBUG",
- "healthcheck",
- "/metrics"
-]
-
-# Complex filter with AND logic
-[[streams.filters]]
-type = "include"
-logic = "and" # Must match ALL patterns
-patterns = [
- "database", # Must contain "database"
- "error", # AND must contain "error"
- "connection" # AND must contain "connection"
-]
-```
-
-Multiple filters are applied sequentially - all must pass for a log to be streamed.
-
-### Check Interval Configuration
-
-Each stream can have its own check interval based on log update frequency:
-
-- **High-frequency logs**: 50-100ms (e.g., application debug logs)
-- **Normal logs**: 100-1000ms (e.g., application logs)
-- **Low-frequency logs**: 10000-60000ms (e.g., system logs, archives)
-
-### Rate Limiting Configuration
-
-Control request rates and connection limits per stream:
-
-```toml
-[streams.httpserver.rate_limit]
-enabled = true # Enable/disable rate limiting
-requests_per_second = 10.0 # Token refill rate
-burst_size = 20 # Maximum burst capacity
-limit_by = "ip" # "ip" or "global"
-response_code = 429 # HTTP response code when limited
-response_message = "Too many requests"
-max_connections_per_ip = 5 # Max concurrent connections per IP
-max_total_connections = 100 # Max total connections (global)
-```
-
-### Heartbeat Configuration
-
-Keep connections alive and detect stale clients with configurable heartbeats:
-
-```toml
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-format = "comment" # "comment" for SSE comments, "json" for events
-include_timestamp = true # Add timestamp to heartbeat
-include_stats = true # Include connection count and uptime
-```
-
-**Heartbeat Formats**:
-
-Comment format (SSE):
-```
-: heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s
-```
-
-JSON format (SSE):
-```
-event: heartbeat
-data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600}
-```
-
-TCP always uses JSON format with newline delimiter.
-
-## Usage Modes
-
-### 1. Standalone Mode (Default)
-
-Each stream runs on its configured ports:
+### Run multiple streams:
```bash
-./logwisp
-# Stream endpoints:
-# - app: http://localhost:8080/stream
-# - system: tcp://localhost:9090 and https://localhost:8443/logs
+logwisp --router --config /etc/logwisp/multi-stream.toml
```
-### 2. Router Mode
+## ð License
-All HTTP streams share ports with path-based routing:
-
-```bash
-./logwisp --router
-# Routed endpoints:
-# - app: http://localhost:8080/app/stream
-# - system: http://localhost:8080/system/logs
-# - global: http://localhost:8080/status
-```
-
-## Client Examples
-
-### HTTP/SSE Stream
-
-```bash
-# Connect to a transport
-curl -N http://localhost:8080/stream
-
-# Check transport status (includes filter and rate limit stats)
-curl http://localhost:8080/status
-
-# With authentication (when implemented)
-curl -u admin:password -N https://localhost:8443/logs
-```
-
-### TCP Stream
-
-```bash
-# Using netcat
-nc localhost 9090
-
-# Using telnet
-telnet localhost 9090
-
-# With TLS (when implemented)
-openssl s_client -connect localhost:9443
-```
-
-### JavaScript Client
-
-```javascript
-const eventSource = new EventSource('http://localhost:8080/stream');
-
-eventSource.addEventListener('connected', (e) => {
- const data = JSON.parse(e.data);
- console.log('Connected with ID:', data.client_id);
-});
-
-eventSource.addEventListener('message', (e) => {
- const logEntry = JSON.parse(e.data);
- console.log(`[${logEntry.time}] ${logEntry.level}: ${logEntry.message}`);
-});
-
-eventSource.addEventListener('heartbeat', (e) => {
- const heartbeat = JSON.parse(e.data);
- console.log('Heartbeat:', heartbeat);
-});
-
-eventSource.addEventListener('error', (e) => {
- if (e.status === 429) {
- console.error('Rate limited - backing off');
- // Implement exponential backoff
- }
-});
-```
-
-## Log Entry Format
-
-All log entries are streamed as JSON:
-
-```json
-{
- "time": "2024-01-01T12:00:00.123456Z",
- "source": "app.log",
- "level": "ERROR",
- "message": "Connection timeout",
- "fields": {
- "user_id": "12345",
- "request_id": "abc-def-ghi"
- }
-}
-```
-
-## API Endpoints
-
-### Stream Endpoints (per stream)
-
-- `GET {stream_path}` - SSE log stream
-- `GET {status_path}` - Stream statistics and configuration
-
-### Global Endpoints (router mode)
-
-- `GET /status` - Aggregated status for all streams
-- `GET /{stream_name}/{path}` - Stream-specific endpoints
-
-### Status Response
-
-```json
-{
- "service": "LogWisp",
- "version": "v1.0.0",
- "server": {
- "type": "http",
- "port": 8080,
- "active_clients": 5,
- "uptime_seconds": 3600
- },
- "monitor": {
- "active_watchers": 3,
- "total_entries": 15420,
- "dropped_entries": 0
- },
- "filters": {
- "filter_count": 2,
- "total_processed": 15420,
- "total_passed": 1234,
- "filters": [
- {
- "type": "include",
- "logic": "or",
- "pattern_count": 3,
- "total_processed": 15420,
- "total_matched": 1234,
- "total_dropped": 0
- }
- ]
- },
- "features": {
- "rate_limit": {
- "enabled": true,
- "total_requests": 45678,
- "blocked_requests": 234,
- "active_ips": 23,
- "total_connections": 5,
- "config": {
- "requests_per_second": 10,
- "burst_size": 20,
- "limit_by": "ip"
- }
- }
- }
-}
-```
-
-## Real-time Statistics
-
-LogWisp provides comprehensive statistics at multiple levels:
-
-- **Per-Stream Stats**: Monitor performance, connection counts, data throughput
-- **Per-Watcher Stats**: File size, position, entries read, rotation count
-- **Filter Stats**: Processed entries, matched patterns, dropped logs
-- **Rate Limit Stats**: Total requests, blocked requests, active IPs
-- **Global Stats**: Aggregated view of all streams (in router mode)
-
-Access statistics via status endpoints or watch the console output:
-
-```
-[15:04:05] Active streams: 2
- app: watchers=3 entries=1542 tcp_conns=2 http_conns=5
- system: watchers=2 entries=8901 tcp_conns=0 http_conns=3
-```
-
-## Advanced Features
-
-### Log Filtering
-
-LogWisp implements powerful regex-based filtering:
-- **Include Filters**: Whitelist patterns - only matching logs pass
-- **Exclude Filters**: Blacklist patterns - matching logs are dropped
-- **Logic Options**: OR (match any) or AND (match all) for pattern combinations
-- **Filter Chains**: Multiple filters applied sequentially
-- **Performance**: Patterns compiled once at startup for efficiency
-
-Filter statistics help monitor effectiveness:
-```bash
-# Watch filter statistics
-watch -n 1 'curl -s http://localhost:8080/status | jq .filters'
-```
-
-### Rate Limiting
-
-LogWisp implements token bucket rate limiting with:
-- **Per-IP limiting**: Each IP gets its own token bucket
-- **Global limiting**: All clients share a single token bucket
-- **Connection limits**: Restrict concurrent connections per IP
-- **Automatic cleanup**: Stale IP entries removed after 5 minutes
-- **Non-blocking**: Excess requests are immediately rejected with 429 status
-
-Monitor rate limiting effectiveness:
-```bash
-# Watch rate limit statistics
-watch -n 1 'curl -s http://localhost:8080/status | jq .features.rate_limit'
-```
-
-### File Rotation Detection
-
-LogWisp automatically detects log rotation through multiple methods:
-- Inode change detection
-- File size decrease
-- Modification time anomalies
-- Position beyond file size
-
-When rotation is detected, a special log entry is generated:
-```json
-{
- "level": "INFO",
- "message": "Log rotation detected (#1): inode change"
-}
-```
-
-### Buffer Management
-
-- **Non-blocking delivery**: Messages are dropped rather than blocking when buffers fill
-- **Per-client buffers**: Each client has independent buffer space
-- **Configurable sizes**: Adjust buffer sizes based on expected load
-
-### Per-Stream Check Intervals
-
-Optimize resource usage by configuring check intervals based on log update frequency:
-
-```toml
-# High-frequency application logs
-[streams.monitor]
-check_interval_ms = 50 # Check every 50ms
-
-# Low-frequency system logs
-[streams.monitor]
-check_interval_ms = 60000 # Check every minute
-```
-
-## Performance Tuning
-
-### Monitor Settings
-- `check_interval_ms`: Lower values = faster detection, higher CPU usage
-- Configure per-stream based on expected update frequency
-- Use 10000ms+ for archival or slowly updating logs
-
-### Filter Optimization
-- Place most selective filters first
-- Use simple patterns when possible
-- Consider combining patterns: `"ERROR|WARN"` vs separate patterns
-- Monitor filter statistics to identify bottlenecks
-
-### Rate Limiting
-- `requests_per_second`: Balance between protection and availability
-- `burst_size`: Set to 2-3x the per-second rate for traffic spikes
-- `max_connections_per_ip`: Prevent resource exhaustion from single IPs
-
-### File Watcher Optimization
-- Use specific file paths when possible (more efficient than directory scanning)
-- Adjust patterns to minimize unnecessary file checks
-- Consider separate streams for different update frequencies
-
-### Network Optimization
-- TCP: Best for high-volume, low-latency requirements
-- HTTP/SSE: Best for browser compatibility and firewall traversal
-- Router mode: Reduces port usage but adds slight routing overhead
-
-## Building from Source
-
-```bash
-# Clone repository
-git clone https://github.com/lixenwraith/logwisp
-cd logwisp
-
-# Install dependencies
-go mod init logwisp
-go get github.com/panjf2000/gnet/v2
-go get github.com/valyala/fasthttp
-go get github.com/lixenwraith/config
-
-# Build with version information
-make build
-
-# Run tests
-make test
-
-# Test rate limiting
-./test_ratelimit.sh
-
-# Test router functionality
-./test_router.sh
-
-# Create a release
-make release TAG=v1.0.0
-```
-
-### Makefile Targets
-
-- `make build` - Build binary with version information
-- `make install` - Install to /usr/local/bin
-- `make clean` - Remove built binary
-- `make test` - Run test suite
-- `make release TAG=vX.Y.Z` - Create and push git tag
-
-## Deployment
-
-### Systemd Service
-
-```ini
-[Unit]
-Description=LogWisp Multi-Stream Log Monitor
-After=network.target
-
-[Service]
-Type=simple
-ExecStart=/usr/local/bin/logwisp --config /etc/logwisp/production.toml
-Restart=always
-User=logwisp
-Group=logwisp
-
-# Security hardening
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadOnlyPaths=/var/log
-
-# Rate limiting at system level
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target
-```
-
-### Docker
-
-```dockerfile
-FROM golang:1.24 AS builder
-WORKDIR /app
-COPY . .
-RUN make build
-
-FROM debian:bookworm-slim
-RUN useradd -r -s /bin/false logwisp
-COPY --from=builder /app/logwisp /usr/local/bin/
-USER logwisp
-EXPOSE 8080 9090
-CMD ["logwisp"]
-```
-
-### Docker Compose
-
-```yaml
-version: '3.8'
-services:
- logwisp:
- build: .
- volumes:
- - /var/log:/var/log:ro
- - ./config.toml:/etc/logwisp/config.toml:ro
- ports:
- - "8080:8080"
- - "9090:9090"
- restart: unless-stopped
- command: ["logwisp", "--config", "/etc/logwisp/config.toml"]
- deploy:
- resources:
- limits:
- cpus: '1.0'
- memory: 512M
-```
-
-## Security Considerations
-
-### Current Implementation
-- Read-only file access
-- Regex pattern validation at startup
-- Rate limiting for DDoS protection
-- Connection limits to prevent resource exhaustion
-- No authentication (placeholder configuration only)
-- No TLS/SSL support (placeholder configuration only)
-
-### Filter Security
-â ïļ **SECURITY**: Be aware of potential ReDoS (Regular Expression Denial of Service) attacks:
-- Complex nested patterns can cause CPU spikes
-- Patterns are validated at startup but not for complexity
-- Monitor filter processing time in production
-- Consider pattern complexity limits for public-facing streams
-
-### Planned Security Features
-- **Authentication**: Basic, Bearer/JWT, mTLS
-- **TLS/SSL**: For both HTTP and TCP streams
-- **IP Filtering**: Whitelist/blacklist support
-- **Audit Logging**: Access and authentication events
-- **RBAC**: Role-based access control per stream
-
-### Best Practices
-1. Run with minimal privileges (read-only access to log files)
-2. Configure appropriate rate limits based on expected traffic
-3. Use network-level security until authentication is implemented
-4. Place behind a reverse proxy for production HTTPS
-5. Monitor rate limit statistics for potential attacks
-6. Regularly update dependencies
-7. Test filter patterns for performance impact
-8. Limit regex complexity in production environments
-
-### Rate Limiting Best Practices
-- Start with conservative limits and adjust based on monitoring
-- Use per-IP limiting for public endpoints
-- Use global limiting for resource protection
-- Set connection limits to prevent memory exhaustion
-- Monitor blocked request statistics for anomalies
-
-## Troubleshooting
-
-### Filter Issues
-1. Check filter statistics to see matched/dropped counts
-2. Test patterns with sample log entries
-3. Verify filter type (include vs exclude)
-4. Check filter logic (or vs and)
-5. Monitor CPU usage for complex patterns
-
-### Rate Limit Issues
-1. Check rate limit statistics in status endpoint
-2. Verify appropriate `requests_per_second` for your use case
-3. Ensure `burst_size` accommodates normal traffic spikes
-4. Monitor for distributed attacks if per-IP limiting isn't effective
-
-### No Log Entries Appearing
-1. Check file permissions (LogWisp needs read access)
-2. Verify file paths in configuration
-3. Ensure files match the specified patterns
-4. Check monitor statistics in status endpoint
-5. Verify check_interval_ms is appropriate for log update frequency
-6. Review filter configuration - logs might be filtered out
-
-### High Memory Usage
-1. Reduce buffer sizes in configuration
-2. Lower the number of concurrent watchers
-3. Enable rate limiting to prevent connection floods
-4. Increase check interval for less critical logs
-5. Use TCP instead of HTTP for high-volume streams
-6. Check for complex regex patterns causing backtracking
-
-### Connection Drops
-1. Check heartbeat configuration
-2. Verify network stability
-3. Monitor client-side errors
-4. Review dropped entry statistics
-5. Check if rate limits are too restrictive
-
-### Version Information
-Use `./logwisp --version` to see:
-- Version tag (from git tags)
-- Git commit hash
-- Build timestamp
-
-## License
-
-BSD-3-Clause
-
-## Contributing
-
-Contributions are welcome! Please read our contributing guidelines and submit pull requests to our repository.
-
-## Roadmap
-
-- [x] Multi-stream architecture
-- [x] File and directory monitoring
-- [x] TCP and HTTP/SSE streaming
-- [x] Path-based HTTP routing
-- [x] Per-stream check intervals
-- [x] Version management
-- [x] Configurable heartbeats
-- [x] Rate and connection limiting
-- [x] Regex-based log filtering
-- [ ] Log transformation (field extraction, formatting)
-- [ ] Configurable logging/stdout support
-- [ ] Service/non-interactive setup
-- [ ] Live config change support
-- [ ] Authentication (Basic, JWT, mTLS)
-- [ ] TLS/SSL support
-- [ ] Prometheus metrics export
-- [ ] WebSocket support
\ No newline at end of file
+BSD-3-Clause
\ No newline at end of file
diff --git a/assets/logwisp-logo.svg b/asset/logwisp-logo.svg
similarity index 100%
rename from assets/logwisp-logo.svg
rename to asset/logwisp-logo.svg
diff --git a/config/logwisp.toml.defaults b/config/logwisp.toml.defaults
index 3dae6a3..2419a08 100644
--- a/config/logwisp.toml.defaults
+++ b/config/logwisp.toml.defaults
@@ -1,10 +1,68 @@
-# LogWisp Configuration File
+# LogWisp Configuration File - Complete Reference
# Default path: ~/.config/logwisp.toml
# Override with: ./logwisp --config /path/to/config.toml
# This is a complete configuration reference showing all available options.
# Default values are uncommented, alternatives and examples are commented.
+# ==============================================================================
+# LOGGING CONFIGURATION (LogWisp's own operational logs)
+# ==============================================================================
+# Controls where and how LogWisp logs its own operational messages.
+# This is separate from the logs being monitored and streamed.
+
+[logging]
+# Output mode: where to write LogWisp's operational logs
+# Options: "file", "stdout", "stderr", "both", "none"
+# - file: Write only to log files
+# - stdout: Write only to standard output
+# - stderr: Write only to standard error (default for containers)
+# - both: Write to both file and console
+# - none: Disable logging (â ïļ SECURITY: Not recommended)
+output = "stderr"
+
+# Minimum log level for operational logs
+# Options: "debug", "info", "warn", "error"
+# - debug: Maximum verbosity, includes internal state changes
+# - info: Normal operational messages (default)
+# - warn: Warnings and errors only
+# - error: Errors only
+level = "info"
+
+# File output configuration (used when output includes "file" or "both")
+[logging.file]
+# Directory for log files
+directory = "./logs"
+
+# Base name for log files (will append timestamp and .log)
+name = "logwisp"
+
+# Maximum size per log file before rotation (megabytes)
+max_size_mb = 100
+
+# Maximum total size of all log files (megabytes)
+# Oldest files are deleted when limit is reached
+max_total_size_mb = 1000
+
+# How long to keep log files (hours)
+# 0 = no time-based deletion
+retention_hours = 168.0 # 7 days
+
+# Console output configuration
+[logging.console]
+# Target for console output
+# Options: "stdout", "stderr", "split"
+# - stdout: All logs to standard output
+# - stderr: All logs to standard error (default)
+# - split: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
+target = "stderr"
+
+# Output format
+# Options: "txt", "json"
+# - txt: Human-readable text format
+# - json: Structured JSON for log aggregation
+format = "txt"
+
# ==============================================================================
# STREAM CONFIGURATION
# ==============================================================================
@@ -16,22 +74,35 @@
# ------------------------------------------------------------------------------
[[streams]]
# Stream identifier used in logs, metrics, and router paths
+# Must be unique across all streams
name = "default"
# File monitoring configuration
[streams.monitor]
# How often to check for new log entries (milliseconds)
# Lower = faster detection but more CPU usage
+# Range: 10-60000 (0.01 to 60 seconds)
check_interval_ms = 100
# Targets to monitor - can be files or directories
+# At least one target is required
targets = [
# Monitor all .log files in current directory
{ path = "./", pattern = "*.log", is_file = false },
+
+ # Example: Monitor specific file
+ # { path = "/var/log/app.log", is_file = true },
+
+ # Example: Multiple patterns in a directory
+ # { path = "/logs", pattern = "*.log", is_file = false },
+ # { path = "/logs", pattern = "*.txt", is_file = false },
]
# Filter configuration (optional) - controls which logs are streamed
# Multiple filters are applied sequentially - all must pass
+# Empty patterns array means "match everything"
+
+# Example: Include only errors and warnings
# [[streams.filters]]
# type = "include" # "include" (whitelist) or "exclude" (blacklist)
# logic = "or" # "or" (match any) or "and" (match all)
@@ -40,31 +111,124 @@ targets = [
# "(?i)warn" # Case-insensitive warning matching
# ]
+# Example: Exclude debug and trace logs
+# [[streams.filters]]
+# type = "exclude"
+# patterns = ["DEBUG", "TRACE", "VERBOSE"]
+
# HTTP Server configuration (SSE/Server-Sent Events)
[streams.httpserver]
+# Enable/disable HTTP server for this stream
enabled = true
+
+# Port to listen on (1-65535)
+# Each stream needs a unique port unless using router mode
port = 8080
-buffer_size = 1000 # Per-client buffer size (messages)
-stream_path = "/stream" # Endpoint for SSE stream
-status_path = "/status" # Endpoint for statistics
+
+# Per-client buffer size (number of messages)
+# Larger = handles bursts better, more memory per client
+buffer_size = 1000
+
+# Endpoint paths (must start with /)
+stream_path = "/stream" # SSE stream endpoint
+status_path = "/status" # Statistics endpoint
# Keep-alive heartbeat configuration
+# Prevents connection timeout on quiet logs
[streams.httpserver.heartbeat]
+# Enable/disable heartbeat messages
enabled = true
-interval_seconds = 30 # Send heartbeat every 30 seconds
-format = "comment" # SSE comment format (: heartbeat)
-include_timestamp = true # Include timestamp in heartbeat
-include_stats = false # Include connection stats
+
+# Interval between heartbeats (seconds)
+# Range: 1-3600 (1 second to 1 hour)
+interval_seconds = 30
+
+# Heartbeat format
+# Options: "comment", "json"
+# - comment: SSE comment format (: heartbeat)
+# - json: JSON event format (data: {"type":"heartbeat"})
+format = "comment"
+
+# Include timestamp in heartbeat
+include_timestamp = true
+
+# Include connection statistics
+include_stats = false
# Rate limiting configuration (disabled by default)
+# Protects against abuse and resource exhaustion
[streams.httpserver.rate_limit]
+# Enable/disable rate limiting
enabled = false
-# requests_per_second = 10.0 # Token refill rate
-# burst_size = 20 # Max burst capacity
-# limit_by = "ip" # "ip" or "global"
-# response_code = 429 # HTTP Too Many Requests
+
+# Token refill rate (requests per second)
+# Float value, e.g., 0.5 = 1 request every 2 seconds
+# requests_per_second = 10.0
+
+# Maximum burst capacity (token bucket size)
+# Should be 2-3x requests_per_second for normal usage
+# burst_size = 20
+
+# Rate limit strategy
+# Options: "ip", "global"
+# - ip: Each client IP gets its own limit
+# - global: All clients share one limit
+# limit_by = "ip"
+
+# HTTP response code when rate limited
+# Common: 429 (Too Many Requests), 503 (Service Unavailable)
+# response_code = 429
+
+# Response message when rate limited
# response_message = "Rate limit exceeded"
-# max_connections_per_ip = 5 # Max SSE connections per IP
+
+# Maximum concurrent connections per IP address
+# 0 = unlimited
+# max_connections_per_ip = 5
+
+# Maximum total concurrent connections
+# 0 = unlimited
+# max_total_connections = 100
+
+# SSL/TLS configuration (planned feature)
+# [streams.httpserver.ssl]
+# enabled = false
+# cert_file = "/path/to/cert.pem"
+# key_file = "/path/to/key.pem"
+# min_version = "TLS1.2" # Minimum TLS version
+# client_auth = false # Require client certificates
+
+# TCP Server configuration (optional)
+# Raw TCP streaming for high-performance scenarios
+# [streams.tcpserver]
+# enabled = false
+# port = 9090
+# buffer_size = 5000 # Larger buffer for TCP
+#
+# [streams.tcpserver.heartbeat]
+# enabled = true
+# interval_seconds = 60
+# include_timestamp = true
+# include_stats = false
+#
+# [streams.tcpserver.rate_limit]
+# enabled = false
+# requests_per_second = 5.0
+# burst_size = 10
+# limit_by = "ip"
+
+# Authentication configuration (planned feature)
+# [streams.auth]
+# type = "none" # Options: "none", "basic", "bearer"
+#
+# # Basic authentication
+# [streams.auth.basic_auth]
+# users_file = "/etc/logwisp/users.htpasswd"
+# realm = "LogWisp"
+#
+# # IP-based access control
+# ip_whitelist = ["192.168.1.0/24", "10.0.0.0/8"]
+# ip_blacklist = []
# ------------------------------------------------------------------------------
# Example: Application Logs Stream with Error Filtering
@@ -92,6 +256,8 @@ enabled = false
# "(?i)\\bcritical\\b", # critical
# "(?i)exception", # exception anywhere
# "(?i)fail(ed|ure)?", # fail, failed, failure
+# "panic", # Go panics
+# "traceback", # Python tracebacks
# ]
#
# # Filter 2: Exclude health check noise
@@ -100,7 +266,10 @@ enabled = false
# patterns = [
# "/health",
# "/metrics",
-# "GET /ping"
+# "/ping",
+# "GET /favicon.ico",
+# "ELB-HealthChecker",
+# "kube-probe"
# ]
#
# [streams.httpserver]
@@ -125,6 +294,7 @@ enabled = false
# burst_size = 50
# limit_by = "ip"
# max_connections_per_ip = 10
+# max_total_connections = 200
# ------------------------------------------------------------------------------
# Example: System Logs Stream (TCP + HTTP) with Security Filtering
@@ -138,6 +308,7 @@ enabled = false
# { path = "/var/log/syslog", is_file = true },
# { path = "/var/log/auth.log", is_file = true },
# { path = "/var/log/kern.log", is_file = true },
+# { path = "/var/log/messages", is_file = true },
# ]
#
# # Include only security-relevant logs
@@ -152,7 +323,12 @@ enabled = false
# "(?i)permission",
# "(?i)denied",
# "(?i)unauthorized",
-# "kernel:.*audit"
+# "(?i)security",
+# "(?i)selinux",
+# "kernel:.*audit",
+# "COMMAND=", # sudo commands
+# "session opened",
+# "session closed"
# ]
#
# # TCP Server for high-performance streaming
@@ -182,9 +358,15 @@ enabled = false
# buffer_size = 1000
# stream_path = "/stream"
# status_path = "/status"
+#
+# [streams.httpserver.rate_limit]
+# enabled = true
+# requests_per_second = 5.0
+# burst_size = 10
+# max_connections_per_ip = 2 # Strict for security logs
# ------------------------------------------------------------------------------
-# Example: High-Volume Debug Logs with Filtering
+# Example: High-Volume Debug Logs with Performance Filtering
# ------------------------------------------------------------------------------
# [[streams]]
# name = "debug"
@@ -193,6 +375,7 @@ enabled = false
# check_interval_ms = 5000 # Check every 5 seconds (high volume)
# targets = [
# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
+# { path = "/var/log/debug", pattern = "debug-*.log", is_file = false },
# ]
#
# # Exclude verbose debug output
@@ -203,15 +386,19 @@ enabled = false
# "VERBOSE",
# "entering function",
# "exiting function",
-# "memory dump"
+# "memory dump",
+# "hex dump",
+# "stack trace",
+# "goroutine [0-9]+"
# ]
#
# # Include only specific modules
# [[streams.filters]]
# type = "include"
# patterns = [
-# "module:(api|database|auth)",
-# "component:(router|handler)"
+# "module=(api|database|auth)",
+# "component=(router|handler)",
+# "service=(payment|order|user)"
# ]
#
# [streams.httpserver]
@@ -232,260 +419,295 @@ enabled = false
# burst_size = 5
# limit_by = "ip"
# max_connections_per_ip = 1 # One connection per IP
+# response_code = 503 # Service Unavailable
+# response_message = "Debug stream overloaded"
# ------------------------------------------------------------------------------
-# Example: Database Logs with Complex Filtering
+# Example: Multi-Application with Router Mode
# ------------------------------------------------------------------------------
+# Run with: logwisp --router
+#
+# [[streams]]
+# name = "frontend"
+# [streams.monitor]
+# targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
+# [[streams.filters]]
+# type = "exclude"
+# patterns = ["GET /static/", "GET /assets/"]
+# [streams.httpserver]
+# enabled = true
+# port = 8080 # Same port OK in router mode
+#
+# [[streams]]
+# name = "backend"
+# [streams.monitor]
+# targets = [{ path = "/var/log/api", pattern = "*.log" }]
+# [[streams.filters]]
+# type = "include"
+# patterns = ["ERROR", "WARN", "timeout", "failed"]
+# [streams.httpserver]
+# enabled = true
+# port = 8080 # Shared port in router mode
+#
# [[streams]]
# name = "database"
-#
# [streams.monitor]
-# check_interval_ms = 200
-# targets = [
-# { path = "/var/log/postgresql", pattern = "*.log", is_file = false },
-# ]
-#
-# # Complex AND filter - must match all patterns
-# [[streams.filters]]
-# type = "include"
-# logic = "and" # Must match ALL patterns
-# patterns = [
-# "(?i)error|fail", # Must contain error or fail
-# "(?i)connection|query", # AND must be about connections or queries
-# "(?i)timeout|deadlock" # AND must involve timeout or deadlock
-# ]
-#
-# # Exclude routine maintenance
-# [[streams.filters]]
-# type = "exclude"
-# patterns = [
-# "VACUUM",
-# "ANALYZE",
-# "checkpoint"
-# ]
-#
-# [streams.tcpserver]
-# enabled = true
-# port = 9091
-# buffer_size = 2000
-
-# ------------------------------------------------------------------------------
-# Example: API Access Logs with Pattern Extraction
-# ------------------------------------------------------------------------------
-# [[streams]]
-# name = "api-access"
-#
-# [streams.monitor]
-# check_interval_ms = 100
-# targets = [
-# { path = "/var/log/nginx/access.log", is_file = true },
-# ]
-#
-# # Include only API endpoints
-# [[streams.filters]]
-# type = "include"
-# patterns = [
-# '"/api/v[0-9]+/', # API versioned endpoints
-# '"(GET|POST|PUT|DELETE) /api/' # API requests
-# ]
-#
-# # Exclude specific status codes
-# [[streams.filters]]
-# type = "exclude"
-# patterns = [
-# '" 200 ', # Success responses
-# '" 204 ', # No content
-# '" 304 ', # Not modified
-# 'OPTIONS ' # CORS preflight
-# ]
-#
+# targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
# [streams.httpserver]
# enabled = true
-# port = 8084
-# buffer_size = 3000
-
-# ------------------------------------------------------------------------------
-# Example: Security/Audit Logs with Strict Filtering
-# ------------------------------------------------------------------------------
-# [[streams]]
-# name = "security"
+# port = 8080
#
-# [streams.monitor]
-# check_interval_ms = 100
-# targets = [
-# { path = "/var/log/audit", pattern = "audit.log*", is_file = false },
-# ]
-#
-# # Security-focused patterns
-# [[streams.filters]]
-# type = "include"
-# logic = "or"
-# patterns = [
-# "type=USER_AUTH",
-# "type=USER_LOGIN",
-# "type=USER_LOGOUT",
-# "type=USER_ERR",
-# "type=CRED_", # All credential operations
-# "type=PRIV_", # All privilege operations
-# "type=ANOM_", # All anomalies
-# "type=RESP_", # All responses
-# "failed|failure",
-# "denied|unauthorized",
-# "violation",
-# "attack|intrusion"
-# ]
-#
-# [streams.httpserver]
-# enabled = true
-# port = 8443 # HTTPS port (for future TLS)
-# buffer_size = 1000
-# stream_path = "/audit"
-# status_path = "/health"
-#
-# # Strict rate limiting for security logs
-# [streams.httpserver.rate_limit]
-# enabled = true
-# requests_per_second = 2.0 # Very limited access
-# burst_size = 3
-# limit_by = "ip"
-# max_connections_per_ip = 1 # Single connection per IP
-# response_code = 403 # Forbidden instead of rate limit
-# response_message = "Access restricted"
-#
-# # Future: SSL/TLS configuration
-# # [streams.httpserver.ssl]
-# # enabled = true
-# # cert_file = "/etc/logwisp/certs/server.crt"
-# # key_file = "/etc/logwisp/certs/server.key"
-# # min_version = "TLS1.2"
-#
-# # Future: Authentication
-# # [streams.auth]
-# # type = "basic"
-# # [streams.auth.basic_auth]
-# # users_file = "/etc/logwisp/security.users"
-# # realm = "Security Logs"
-
-# ------------------------------------------------------------------------------
-# Example: Multi-Application Logs with Service Filtering
-# ------------------------------------------------------------------------------
-# [[streams]]
-# name = "microservices"
-#
-# [streams.monitor]
-# check_interval_ms = 100
-# targets = [
-# { path = "/var/log/containers", pattern = "*.log", is_file = false },
-# ]
-#
-# # Filter by service name
-# [[streams.filters]]
-# type = "include"
-# patterns = [
-# "service=(api|auth|user|order)", # Specific services
-# "pod=(api|auth|user|order)-" # Kubernetes pods
-# ]
-#
-# # Exclude Kubernetes noise
-# [[streams.filters]]
-# type = "exclude"
-# patterns = [
-# "kube-system",
-# "kube-proxy",
-# "Readiness probe",
-# "Liveness probe"
-# ]
-#
-# [streams.httpserver]
-# enabled = true
-# port = 8085
-# buffer_size = 5000
+# # Access via:
+# # http://localhost:8080/frontend/stream
+# # http://localhost:8080/backend/stream
+# # http://localhost:8080/database/stream
+# # http://localhost:8080/status (global)
# ==============================================================================
-# FILTER PATTERN EXAMPLES
+# FILTER PATTERN REFERENCE
# ==============================================================================
#
# Basic Patterns:
-# - "ERROR" # Exact match
+# - "ERROR" # Exact match (case sensitive)
# - "(?i)error" # Case-insensitive
# - "\\berror\\b" # Word boundary (won't match "errorCode")
-# - "error|warn|fatal" # Multiple options
+# - "error|warn|fatal" # Multiple options (OR)
+# - "(error|warn) level" # Group with context
+#
+# Position Patterns:
+# - "^\\[ERROR\\]" # Line starts with [ERROR]
+# - "ERROR:$" # Line ends with ERROR:
+# - "^\\d{4}-\\d{2}-\\d{2}" # Line starts with date
#
# Complex Patterns:
-# - "^\\[ERROR\\]" # Line starts with [ERROR]
# - "status=[4-5][0-9]{2}" # HTTP 4xx or 5xx status codes
# - "duration>[0-9]{4}ms" # Duration over 999ms
# - "user_id=\"[^\"]+\"" # Extract user_id values
+# - "\\[ERROR\\].*database" # ERROR followed by database
+# - "(?i)\\b(error|fail|critical)\\b" # Multiple error words
+#
+# Log Level Patterns:
+# - "\\[(ERROR|WARN|FATAL)\\]" # Common formats
+# - "level=(error|warning|critical)" # Key-value format
+# - "ERROR\\s*:" # ERROR with optional space
+# - "<(Error|Warning)>" # XML-style
+#
+# Application Patterns:
+# - "com\\.mycompany\\..*Exception" # Java exceptions
+# - "at .+\\(.+\\.java:[0-9]+\\)" # Java stack traces
+# - "File \".+\", line [0-9]+" # Python tracebacks
+# - "panic: .+" # Go panics
+# - "/api/v[0-9]+/" # API versioned paths
+#
+# Performance Patterns:
+# - "took [0-9]{4,}ms" # Operations over 999ms
+# - "memory usage: [8-9][0-9]%" # High memory usage
+# - "queue size: [0-9]{4,}" # Large queues
+# - "timeout|timed out" # Timeouts
+#
+# Security Patterns:
+# - "unauthorized|forbidden" # Access denied
+# - "invalid token|expired token" # Auth failures
+# - "SQL injection|XSS" # Security threats
+# - "failed login.*IP: ([0-9.]+)" # Failed logins with IP
#
# Performance Tips:
-# - Avoid nested quantifiers: "((a+)+)+" can cause catastrophic backtracking
+# - Avoid nested quantifiers: "((a+)+)+" causes catastrophic backtracking
# - Use anchors when possible: "^ERROR" is faster than "ERROR"
# - Prefer character classes: "[0-9]" over "\\d" for clarity
+# - Use non-capturing groups: "(?:error|warn)" when not extracting
# - Test complex patterns with sample data before deployment
+# - Consider using multiple simple patterns instead of one complex pattern
#
# Security Considerations:
# - Be aware of ReDoS (Regular Expression Denial of Service)
# - Limit pattern complexity for public-facing streams
# - Monitor filter processing time in statistics
# - Consider pre-filtering very high volume streams
+# - Use explicit allow-lists for sensitive logs
# ==============================================================================
-# USAGE EXAMPLES
+# RATE LIMITING GUIDE
# ==============================================================================
-
-# 1. Basic usage (single stream):
-# ./logwisp
-# - Monitors current directory for *.log files
-# - Access logs at: http://localhost:8080/stream
-# - View stats at: http://localhost:8080/status
-
-# 2. Multi-stream configuration:
-# - Uncomment additional [[streams]] sections above
-# - Each stream runs independently on its own port
-# - Different check intervals for different log types
-# - Different filters for each stream
-
-# 3. Router mode (consolidated access):
-# ./logwisp --router
-# - All streams accessible via paths: /streamname/stream
-# - Global status at: /status
-# - Example: http://localhost:8080/app/stream
-
-# 4. Production deployment:
-# - Enable filters to reduce noise and bandwidth
-# - Enable rate limiting on public-facing streams
-# - Use TCP for internal high-volume streams
-# - Set appropriate check intervals (higher = less CPU)
-# - Configure heartbeats for long-lived connections
-
-# 5. Monitoring:
-# curl http://localhost:8080/status | jq .
-# - Check active connections
-# - Monitor filter statistics (matched/dropped)
-# - Monitor rate limit statistics
-# - Track log entry counts
+#
+# Token Bucket Algorithm:
+# - Each client (IP) or global limit gets a bucket with 'burst_size' tokens
+# - Tokens refill at 'requests_per_second' rate
+# - Each request consumes one token
+# - Provides smooth rate limiting without hard cutoffs
+#
+# Configuration Examples:
+#
+# Light Protection (default for most streams):
+# requests_per_second = 10.0
+# burst_size = 20 # Handle short bursts
+#
+# Moderate Protection (public endpoints):
+# requests_per_second = 5.0
+# burst_size = 15
+# max_connections_per_ip = 5
+#
+# Strict Protection (sensitive logs):
+# requests_per_second = 1.0
+# burst_size = 3
+# max_connections_per_ip = 1
+# limit_by = "ip"
+#
+# Global Limiting (shared resource):
+# requests_per_second = 50.0 # Total for all clients
+# burst_size = 100
+# limit_by = "global"
+# max_total_connections = 50
+#
+# Behavior:
+# - HTTP: Returns response_code (default 429) with JSON error
+# - TCP: Silently drops connection (no error message)
+# - Cleanup: Inactive IPs removed after 5 minutes
+# - Statistics: Available in /status endpoint
+#
+# Best Practices:
+# - Set burst_size to 2-3x requests_per_second
+# - Use per-IP limiting for fairness
+# - Use global limiting for resource protection
+# - Monitor rate limit statistics for tuning
+# - Consider different limits for different streams
+# - Enable for any public-facing endpoints
# ==============================================================================
-# ENVIRONMENT VARIABLES
+# PERFORMANCE TUNING
# ==============================================================================
-# Configuration can be overridden via environment variables:
-# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
-# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
-# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
-# LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
-# LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
+#
+# Monitor Check Interval:
+# - 10-50ms: Real-time monitoring, higher CPU usage
+# - 100-500ms: Good balance for active logs
+# - 1000-5000ms: Low-activity logs, minimal CPU
+# - 10000ms+: Very slow changing logs
+#
+# Buffer Sizes:
+# - HTTP: 100-1000 for normal use, 5000+ for high volume
+# - TCP: 1000-5000 typical, 10000+ for bulk streaming
+# - Larger = more memory per client, handles bursts better
+#
+# Connection Limits:
+# - Development: No limits needed
+# - Production: 5-10 connections per IP typical
+# - Public: 1-3 connections per IP
+# - Total: Based on available memory (each uses ~1-5MB)
+#
+# Filter Performance:
+# - Simple patterns: ~1Ξs per check
+# - Complex patterns: ~10-100Ξs per check
+# - Many patterns: Consider multiple streams instead
+# - Use exclude filters to drop noise early
+#
+# Memory Usage (approximate):
+# - Base process: ~10-20MB
+# - Per stream: ~5-10MB
+# - Per HTTP client: ~1-2MB
+# - Per TCP client: ~0.5-1MB
+# - Filter chain: ~1-5MB depending on patterns
# ==============================================================================
-# NOTES
+# DEPLOYMENT SCENARIOS
# ==============================================================================
-# - Filters are processed sequentially - all must pass
-# - Empty filter patterns means "pass everything"
-# - Rate limiting is disabled by default for backward compatibility
-# - Each stream can have different rate limit settings
-# - TCP connections are silently dropped when rate limited
-# - HTTP returns 429 (or configured code) with JSON error
-# - IP tracking is cleaned up after 5 minutes of inactivity
-# - Token bucket algorithm provides smooth rate limiting
-# - Connection limits prevent resource exhaustion
-# - Regex patterns are compiled once at startup for performance
-# - Complex patterns can impact performance - monitor statistics
\ No newline at end of file
+#
+# Single Application:
+# - One stream with basic filtering
+# - Moderate rate limiting
+# - Standard check interval (100ms)
+#
+# Microservices:
+# - Multiple streams, one per service
+# - Router mode for unified access
+# - Different filter rules per service
+# - Service-specific rate limits
+#
+# High Security:
+# - Strict include filters
+# - Low rate limits (1-2 req/sec)
+# - Single connection per IP
+# - TCP for internal, HTTP for external
+#
+# High Performance:
+# - TCP streaming preferred
+# - Large buffers (10000+)
+# - Minimal filtering
+# - Higher check intervals
+# - No heartbeats
+#
+# Development/Testing:
+# - Multiple streams for different log levels
+# - No rate limiting
+# - Debug level logging
+# - Fast check intervals
+# - All filters disabled
+
+# ==============================================================================
+# TROUBLESHOOTING
+# ==============================================================================
+#
+# Common Issues:
+#
+# "No logs appearing":
+# - Check file paths and permissions
+# - Verify pattern matches filenames
+# - Check filters aren't too restrictive
+# - Enable debug logging: --log-level debug
+#
+# "High CPU usage":
+# - Increase check_interval_ms
+# - Reduce number of filter patterns
+# - Use simpler regex patterns
+# - Check for runaway log growth
+#
+# "Clients disconnecting":
+# - Enable heartbeats
+# - Check rate limiting settings
+# - Verify network connectivity
+# - Increase buffer sizes
+#
+# "Memory growth":
+# - Check client connection count
+# - Verify buffer sizes are reasonable
+# - Look for memory leaks in filters
+# - Enable connection limits
+#
+# Debug Commands:
+# - Check status: curl http://localhost:8080/status
+# - Test stream: curl -N http://localhost:8080/stream
+# - View logs: logwisp --log-level debug --log-output stderr
+# - Test filters: Use simple patterns first
+
+# ==============================================================================
+# FUTURE FEATURES (Roadmap)
+# ==============================================================================
+#
+# Authentication:
+# - Basic auth with htpasswd files
+# - Bearer token authentication
+# - JWT validation
+# - mTLS client certificates
+#
+# SSL/TLS:
+# - HTTPS endpoints
+# - TLS for TCP streams
+# - Certificate management
+# - Let's Encrypt integration
+#
+# Advanced Filtering:
+# - Lua scripting for complex logic
+# - Rate-based filtering (N per minute)
+# - Statistical anomaly detection
+# - Multi-line pattern matching
+#
+# Output Formats:
+# - JSON transformation
+# - Field extraction
+# - Custom formatting templates
+# - Compression (gzip)
+#
+# Integrations:
+# - Prometheus metrics
+# - OpenTelemetry traces
+# - Webhook notifications
+# - Cloud storage backends
\ No newline at end of file
diff --git a/config/logwisp.toml.example b/config/logwisp.toml.example
deleted file mode 100644
index f084878..0000000
--- a/config/logwisp.toml.example
+++ /dev/null
@@ -1,120 +0,0 @@
-# LogWisp Configuration Example
-# Default path: ~/.config/logwisp.toml
-
-# Application logs - public facing
-[[streams]]
-name = "app-public"
-
-[streams.monitor]
-check_interval_ms = 100
-targets = [
- { path = "/var/log/nginx", pattern = "access.log*", is_file = false },
- { path = "/var/log/app", pattern = "production.log", is_file = true }
-]
-
-[streams.httpserver]
-enabled = true
-port = 8080
-buffer_size = 2000
-stream_path = "/logs"
-status_path = "/health"
-
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-format = "json"
-include_timestamp = true
-include_stats = true
-
-# Rate limiting for public endpoint
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 50.0
-burst_size = 100
-limit_by = "ip"
-response_code = 429
-response_message = "Rate limit exceeded. Please retry after 60 seconds."
-max_connections_per_ip = 5
-max_total_connections = 100
-
-# System logs - internal only
-[[streams]]
-name = "system"
-
-[streams.monitor]
-check_interval_ms = 5000 # Check every 5 seconds
-targets = [
- { path = "/var/log/syslog", is_file = true },
- { path = "/var/log/auth.log", is_file = true },
- { path = "/var/log/kern.log", is_file = true }
-]
-
-# TCP for internal consumers
-[streams.tcpserver]
-enabled = true
-port = 9090
-buffer_size = 5000
-
-[streams.tcpserver.heartbeat]
-enabled = true
-interval_seconds = 60
-include_timestamp = true
-
-# Moderate rate limiting for internal use
-[streams.tcpserver.rate_limit]
-enabled = true
-requests_per_second = 10.0
-burst_size = 20
-limit_by = "ip"
-
-# Security audit logs - restricted access
-[[streams]]
-name = "security"
-
-[streams.monitor]
-check_interval_ms = 100
-targets = [
- { path = "/var/log/audit", pattern = "*.log", is_file = false },
- { path = "/var/log/fail2ban.log", is_file = true }
-]
-
-[streams.httpserver]
-enabled = true
-port = 8443
-buffer_size = 1000
-stream_path = "/audit/stream"
-status_path = "/audit/status"
-
-# Strict rate limiting
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 1.0
-burst_size = 3
-limit_by = "ip"
-max_connections_per_ip = 1
-response_code = 403
-response_message = "Access denied"
-
-# Application debug logs - development team only
-[[streams]]
-name = "debug"
-
-[streams.monitor]
-check_interval_ms = 1000
-targets = [
- { path = "/var/log/app", pattern = "debug-*.log", is_file = false }
-]
-
-[streams.httpserver]
-enabled = true
-port = 8090
-buffer_size = 5000
-stream_path = "/debug"
-status_path = "/debug/status"
-
-[streams.httpserver.rate_limit]
-enabled = true
-requests_per_second = 100.0 # Higher limit for internal use
-burst_size = 200
-limit_by = "ip"
-max_connections_per_ip = 10
\ No newline at end of file
diff --git a/config/logwisp.toml.minimal b/config/logwisp.toml.minimal
index e0d8327..a8f0ad5 100644
--- a/config/logwisp.toml.minimal
+++ b/config/logwisp.toml.minimal
@@ -1,7 +1,7 @@
-# LogWisp Minimal Configuration Example
+# LogWisp Minimal Configuration
# Save as: ~/.config/logwisp.toml
-# Monitor application logs
+# Basic stream monitoring application logs
[[streams]]
name = "app"
@@ -11,20 +11,32 @@ targets = [
{ path = "/var/log/myapp", pattern = "*.log", is_file = false }
]
-# Optional: Filter for errors and warnings only
-# [[streams.filters]]
-# type = "include"
-# patterns = ["ERROR", "WARN", "CRITICAL"]
-
[streams.httpserver]
enabled = true
port = 8080
stream_path = "/stream"
status_path = "/status"
-# Optional: Enable rate limiting
+# Optional additions:
+
+# 1. Filter for errors only:
+# [[streams.filters]]
+# type = "include"
+# patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
+
+# 2. Enable rate limiting:
# [streams.httpserver.rate_limit]
# enabled = true
# requests_per_second = 10.0
# burst_size = 20
-# limit_by = "ip"
\ No newline at end of file
+# limit_by = "ip"
+
+# 3. Add heartbeat:
+# [streams.httpserver.heartbeat]
+# enabled = true
+# interval_seconds = 30
+
+# 4. Change LogWisp's own logging:
+# [logging]
+# output = "file"
+# level = "info"
\ No newline at end of file
diff --git a/doc/README.md b/doc/README.md
new file mode 100644
index 0000000..351aacf
--- /dev/null
+++ b/doc/README.md
@@ -0,0 +1,76 @@
+# LogWisp Documentation
+
+Welcome to the LogWisp documentation. This guide covers all aspects of installing, configuring, and using LogWisp for multi-stream log monitoring.
+
+## ð Documentation Index
+
+### Getting Started
+- **[Installation Guide](installation.md)** - How to install LogWisp on various platforms
+- **[Quick Start](quickstart.md)** - Get up and running in 5 minutes
+- **[Architecture Overview](architecture.md)** - System design and components
+
+### Configuration
+- **[Configuration Guide](configuration.md)** - Complete configuration reference
+- **[Environment Variables](environment.md)** - Environment variable reference
+- **[Command Line Options](cli.md)** - CLI flags and parameters
+
+### Features
+- **[Filters Guide](filters.md)** - Pattern-based log filtering
+- **[Rate Limiting](ratelimiting.md)** - Request and connection limiting
+- **[Router Mode](router.md)** - Path-based multi-stream routing
+- **[Authentication](authentication.md)** - Securing your log streams *(planned)*
+
+### Operations
+- **[Monitoring & Status](monitoring.md)** - Health checks and statistics
+- **[Performance Tuning](performance.md)** - Optimization guidelines
+- **[Troubleshooting](troubleshooting.md)** - Common issues and solutions
+
+### Advanced Topics
+- **[Security Best Practices](security.md)** - Hardening your deployment
+- **[Integration Examples](integrations.md)** - Working with other tools
+- **[Development Guide](development.md)** - Contributing to LogWisp
+
+## ð Quick Links
+
+- **[Example Configurations](examples/)** - Ready-to-use config templates
+- **[API Reference](api.md)** - SSE/TCP protocol documentation
+- **[Changelog](../CHANGELOG.md)** - Version history and updates
+
+## ðĄ Common Use Cases
+
+### Single Application Monitoring
+Monitor logs from one application with basic filtering:
+```toml
+[[streams]]
+name = "myapp"
+[streams.monitor]
+targets = [{ path = "/var/log/myapp", pattern = "*.log" }]
+[[streams.filters]]
+type = "include"
+patterns = ["ERROR", "WARN"]
+```
+
+### Multi-Service Architecture
+Monitor multiple services with different configurations:
+```bash
+logwisp --router --config /etc/logwisp/services.toml
+```
+
+### High-Security Environments
+Enable authentication and rate limiting:
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 10.0
+max_connections_per_ip = 3
+```
+
+## ð Finding Help
+
+- **GitHub Issues**: [Report bugs or request features](https://github.com/logwisp/logwisp/issues)
+- **Discussions**: [Ask questions and share ideas](https://github.com/logwisp/logwisp/discussions)
+- **Examples**: Check the [examples directory](examples/) for common scenarios
+
+## ð License
+
+BSD-3-Clause
\ No newline at end of file
diff --git a/doc/architecture.md b/doc/architecture.md
deleted file mode 100644
index 6dc9c40..0000000
--- a/doc/architecture.md
+++ /dev/null
@@ -1,422 +0,0 @@
-# LogWisp Architecture and Project Structure
-
-## Directory Structure
-
-```
-logwisp/
-âââ Makefile # Build automation with version injection
-âââ go.mod # Go module definition
-âââ go.sum # Go module checksums
-âââ README.md # Project documentation
-âââ config/
-â âââ logwisp.toml.defaults # Default configuration and guide
-â âââ logwisp.toml.example # Example configuration
-â âââ logwisp.toml.minimal # Minimal configuration template
-âââ doc/
-â âââ architecture.md # This file - architecture documentation
-âââ src/
- âââ cmd/
- â âââ logwisp/
- â âââ main.go # Application entry point, CLI handling
- âââ internal/
- âââ config/
- â âââ auth.go # Authentication configuration structures
- â âââ config.go # Main configuration structures
- â âââ loader.go # Configuration loading with lixenwraith/config
- â âââ server.go # TCP/HTTP server configurations with rate limiting
- â âââ ssl.go # SSL/TLS configuration structures
- â âââ stream.go # Stream-specific configurations with filters
- â âââ validation.go # Configuration validation including filters and rate limits
- âââ filter/
- â âââ filter.go # Regex-based log filtering implementation
- â âââ chain.go # Sequential filter chain management
- âââ monitor/
- â âââ file_watcher.go # File watching and rotation detection
- â âââ monitor.go # Log monitoring interface and implementation
- âââ ratelimit/
- â âââ ratelimit.go # Token bucket algorithm implementation
- â âââ limiter.go # Per-stream rate limiter with IP tracking
- âââ service/
- â âââ httprouter.go # HTTP router for path-based routing
- â âââ logstream.go # Stream lifecycle management
- â âââ routerserver.go # Router server implementation
- â âââ service.go # Multi-stream service orchestration
- âââ transport/
- â âââ httpstreamer.go # HTTP/SSE streaming with rate limiting
- â âââ noop_logger.go # Silent logger for gnet
- â âââ tcpserver.go # TCP server with rate limiting (gnet)
- â âââ tcpstreamer.go # TCP streaming implementation
- âââ version/
- âââ version.go # Version information management
-```
-
-## Configuration System
-
-### Configuration Hierarchy (Highest to Lowest Priority)
-
-1. **CLI Arguments**: Direct command-line flags
-2. **Environment Variables**: `LOGWISP_` prefixed variables
-3. **Configuration File**: TOML format configuration
-4. **Built-in Defaults**: Hardcoded default values
-
-### Configuration Locations
-
-```bash
-# Default configuration file location
-~/.config/logwisp.toml
-
-# Override via environment variable
-export LOGWISP_CONFIG_FILE=/etc/logwisp/production.toml
-
-# Override config directory
-export LOGWISP_CONFIG_DIR=/etc/logwisp
-export LOGWISP_CONFIG_FILE=production.toml # Relative to CONFIG_DIR
-
-# Direct CLI override
-./logwisp --config /path/to/config.toml
-```
-
-### Environment Variable Mapping
-
-Environment variables follow a structured naming pattern:
-- Prefix: `LOGWISP_`
-- Path separator: `_` (underscore)
-- Array index: Numeric suffix (0-based)
-
-Examples:
-```bash
-# Stream-specific settings
-LOGWISP_STREAMS_0_NAME=app
-LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
-LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
-LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=2000
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=json
-
-# Filter configuration
-LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
-LOGWISP_STREAMS_0_FILTERS_0_LOGIC=or
-LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
-
-# Rate limiting configuration
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_BURST_SIZE=20
-LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_LIMIT_BY=ip
-
-# Multiple streams
-LOGWISP_STREAMS_1_NAME=system
-LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
-LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
-```
-
-## Component Architecture
-
-### Core Components
-
-1. **Service (`logstream.Service`)**
- - Manages multiple log streams
- - Handles lifecycle (creation, shutdown)
- - Provides global statistics
- - Thread-safe stream registry
-
-2. **LogStream (`logstream.LogStream`)**
- - Represents a single log monitoring pipeline
- - Contains: Monitor + Filter Chain + Rate Limiter + Servers (TCP/HTTP)
- - Independent configuration
- - Per-stream statistics with filter and rate limit metrics
-
-3. **Monitor (`monitor.Monitor`)**
- - Watches files and directories
- - Detects log rotation
- - Publishes log entries to subscribers
- - Configurable check intervals
-
-4. **Filter (`filter.Filter`)**
- - Regex-based log filtering
- - Include (whitelist) or Exclude (blacklist) modes
- - OR/AND logic for multiple patterns
- - Per-filter statistics (processed, matched, dropped)
-
-5. **Filter Chain (`filter.Chain`)**
- - Sequential application of multiple filters
- - All filters must pass for entry to be streamed
- - Aggregate statistics across filter chain
-
-6. **Rate Limiter (`ratelimit.Limiter`)**
- - Token bucket algorithm for smooth rate limiting
- - Per-IP or global limiting strategies
- - Connection tracking and limits
- - Automatic cleanup of stale entries
- - Non-blocking rejection of excess requests
-
-7. **Streamers**
- - **HTTPStreamer**: SSE-based streaming over HTTP
- - Rate limit enforcement before request handling
- - Connection tracking for per-IP limits
- - Configurable 429 responses
- - **TCPStreamer**: Raw JSON streaming over TCP
- - Silent connection drops when rate limited
- - Per-IP connection tracking
- - Both support configurable heartbeats
- - Non-blocking client management
-
-8. **HTTPRouter (`logstream.HTTPRouter`)**
- - Optional component for path-based routing
- - Consolidates multiple HTTP streams on shared ports
- - Provides global status endpoint
- - Longest-prefix path matching
- - Dynamic stream registration/deregistration
-
-### Data Flow
-
-```
-File System â Monitor â LogEntry Channel â Filter Chain â [Rate Limiter] â Streamer â Network Client
- â â â â
- âââ Rotation Detection Pattern Match Rate Limit Check
- â â
- Pass/Drop Accept/Reject
-```
-
-### Filter Architecture
-
-```
-Log Entry â Filter Chain â Filter 1 â Filter 2 â ... â Output
- â â
- Include? Exclude?
- â â
- OR/AND OR/AND
- Logic Logic
-```
-
-### Rate Limiting Architecture
-
-```
-Client Request â Rate Limiter â Token Bucket Check â Allow/Deny
- â â
- IP Tracking Refill Rate
- â
- Cleanup Timer
-```
-
-### Configuration Structure
-
-```toml
-[[streams]]
-name = "stream-name"
-
-[streams.monitor]
-check_interval_ms = 100 # Per-transport check interval
-targets = [
- { path = "/path/to/logs", pattern = "*.log", is_file = false },
- { path = "/path/to/file.log", is_file = true }
-]
-
-# Filter configuration (optional)
-[[streams.filters]]
-type = "include" # "include" or "exclude"
-logic = "or" # "or" or "and"
-patterns = [
- "(?i)error", # Case-insensitive error matching
- "(?i)warn" # Case-insensitive warning matching
-]
-
-[[streams.filters]]
-type = "exclude"
-patterns = ["DEBUG", "TRACE"]
-
-[streams.httpserver]
-enabled = true
-port = 8080
-buffer_size = 1000
-stream_path = "/stream"
-status_path = "/status"
-
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-format = "comment" # or "json"
-include_timestamp = true
-include_stats = false
-
-[streams.httpserver.rate_limit]
-enabled = false # Disabled by default
-requests_per_second = 10.0 # Token refill rate
-burst_size = 20 # Token bucket capacity
-limit_by = "ip" # "ip" or "global"
-response_code = 429 # HTTP response code
-response_message = "Rate limit exceeded"
-max_connections_per_ip = 5 # Concurrent connection limit
-max_total_connections = 100 # Global connection limit
-
-[streams.tcpserver]
-enabled = true
-port = 9090
-buffer_size = 5000
-
-[streams.tcpserver.heartbeat]
-enabled = true
-interval_seconds = 60
-include_timestamp = true
-include_stats = true
-
-[streams.tcpserver.rate_limit]
-enabled = false
-requests_per_second = 5.0
-burst_size = 10
-limit_by = "ip"
-```
-
-## Filter Implementation
-
-### Filter Types
-1. **Include Filter**: Only logs matching patterns are streamed (whitelist)
-2. **Exclude Filter**: Logs matching patterns are dropped (blacklist)
-
-### Pattern Logic
-- **OR Logic**: Log matches if ANY pattern matches
-- **AND Logic**: Log matches only if ALL patterns match
-
-### Filter Chain
-- Multiple filters are applied sequentially
-- All filters must pass for a log to be streamed
-- Efficient short-circuit evaluation
-
-### Performance Considerations
-- Regex patterns compiled once at startup
-- Cached for efficient matching
-- Statistics tracked without locks in hot path
-
-## Rate Limiting Implementation
-
-### Token Bucket Algorithm
-- Each IP (or global limiter) gets a bucket with configurable capacity
-- Tokens refill at `requests_per_second` rate
-- Each request/connection consumes one token
-- Smooth rate limiting without hard cutoffs
-
-### Limiting Strategies
-1. **Per-IP**: Each client IP gets its own token bucket
-2. **Global**: All clients share a single token bucket
-
-### Connection Limits
-- Per-IP connection limits prevent single client resource exhaustion
-- Global connection limits protect overall system resources
-- Checked before rate limits to prevent connection hanging
-
-### Cleanup
-- IP entries older than 5 minutes are automatically removed
-- Prevents unbounded memory growth
-- Runs every minute in background
-
-## Build System
-
-### Makefile Targets
-
-```bash
-make build # Build with version information
-make install # Install to /usr/local/bin
-make clean # Remove built binary
-make test # Run test suite
-make release TAG=v1.0.0 # Create and push git tag
-```
-
-### Version Management
-
-Version information is injected at compile time:
-```bash
-# Automatic version detection from git
-VERSION := $(shell git describe --tags --always --dirty)
-GIT_COMMIT := $(shell git rev-parse --short HEAD)
-BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S')
-
-# Manual build with version
-go build -ldflags "-X 'logwisp/src/internal/version.Version=v1.0.0'" \
- -o logwisp ./src/cmd/logwisp
-```
-
-## Operating Modes
-
-### 1. Standalone Mode (Default)
-- Each stream runs its own HTTP/TCP servers
-- Direct port access per stream
-- Simple configuration
-- Best for single-stream or distinct-port setups
-
-### 2. Router Mode (`--router`)
-- HTTP streams share ports via path-based routing
-- Consolidated access through URL paths
-- Global status endpoint with aggregated statistics
-- Best for multi-stream setups with limited ports
-- Streams accessible at `/{stream_name}/{path}`
-
-## Testing
-
-### Test Suites
-
-1. **Router Testing** (`test_router.sh`)
- - Path routing verification
- - Client isolation between streams
- - Statistics aggregation
- - Graceful shutdown
- - Port conflict handling
-
-2. **Rate Limiting Testing** (`test_ratelimit.sh`)
- - Per-IP rate limiting
- - Global rate limiting
- - Connection limits
- - Rate limit recovery
- - Statistics accuracy
- - Stress testing
-
-3. **Filter Testing** (recommended)
- - Pattern matching accuracy
- - Include/exclude logic
- - OR/AND combination logic
- - Performance with complex patterns
- - Filter chain behavior
-
-### Running Tests
-
-```bash
-# Test router functionality
-./test_router.sh
-
-# Test rate limiting
-./test_ratelimit.sh
-
-# Run all tests
-make test
-```
-
-## Performance Considerations
-
-### Filter Overhead
-- Regex compilation: One-time cost at startup
-- Pattern matching: O(n*m) where n=patterns, m=text length
-- Use simple patterns when possible
-- Consider pattern order (most likely matches first)
-
-### Rate Limiting Overhead
-- Token bucket checks: O(1) time complexity
-- Memory: ~100 bytes per tracked IP
-- Cleanup: Runs asynchronously every minute
-- Minimal impact when disabled
-
-### Optimization Guidelines
-- Use specific patterns to reduce regex complexity
-- Place most selective filters first in chain
-- Use per-IP limiting for fairness
-- Use global limiting for resource protection
-- Set burst size to 2-3x requests_per_second
-- Monitor rate limit statistics for tuning
-- Higher check_interval_ms for low-activity logs
-
-## Security Architecture
-
-### Current Security Features
-- Read-only file access
-- Rate limiting for DDoS protection
-- Connection limits for resource protection
-- Non-blocking request rejection
-- Regex pattern validation at startup
\ No newline at end of file
diff --git a/doc/cli.md b/doc/cli.md
new file mode 100644
index 0000000..3983654
--- /dev/null
+++ b/doc/cli.md
@@ -0,0 +1,155 @@
+# Command Line Interface
+
+LogWisp provides a comprehensive set of command-line options for controlling its behavior without modifying configuration files.
+
+## Synopsis
+
+```bash
+logwisp [options]
+```
+
+## General Options
+
+### `--config `
+Specify the configuration file location.
+- **Default**: `~/.config/logwisp.toml`
+- **Example**: `logwisp --config /etc/logwisp/production.toml`
+
+### `--router`
+Enable HTTP router mode for path-based routing of multiple streams.
+- **Default**: `false` (standalone mode)
+- **Use case**: Consolidate multiple HTTP streams on shared ports
+- **Example**: `logwisp --router`
+
+### `--version`
+Display version information and exit.
+- **Example**: `logwisp --version`
+
+### `--background`
+Run LogWisp as a background process.
+- **Default**: `false` (foreground mode)
+- **Example**: `logwisp --background`
+
+## Logging Options
+
+These options override the corresponding configuration file settings.
+
+### `--log-output `
+Control where LogWisp writes its own operational logs.
+- **Values**: `file`, `stdout`, `stderr`, `both`, `none`
+- **Default**: Configured value or `stderr`
+- **Example**: `logwisp --log-output both`
+
+#### Output Modes:
+- `file`: Write logs only to files
+- `stdout`: Write logs only to standard output
+- `stderr`: Write logs only to standard error
+- `both`: Write logs to both files and console
+- `none`: Disable logging (â ïļ SECURITY: Not recommended)
+
+### `--log-level `
+Set the minimum log level for LogWisp's operational logs.
+- **Values**: `debug`, `info`, `warn`, `error`
+- **Default**: Configured value or `info`
+- **Example**: `logwisp --log-level debug`
+
+### `--log-file `
+Specify the log file path when using file output.
+- **Default**: Configured value or `./logs/logwisp.log`
+- **Example**: `logwisp --log-output file --log-file /var/log/logwisp/app.log`
+
+### `--log-dir `
+Specify the log directory when using file output.
+- **Default**: Configured value or `./logs`
+- **Example**: `logwisp --log-output file --log-dir /var/log/logwisp`
+
+### `--log-console `
+Control console output destination when using `stdout`, `stderr`, or `both` modes.
+- **Values**: `stdout`, `stderr`, `split`
+- **Default**: `stderr`
+- **Example**: `logwisp --log-output both --log-console split`
+
+#### Console Targets:
+- `stdout`: All logs to standard output
+- `stderr`: All logs to standard error
+- `split`: INFO/DEBUG to stdout, WARN/ERROR to stderr (planned)
+
+## Examples
+
+### Basic Usage
+```bash
+# Start with default configuration
+logwisp
+
+# Use a specific configuration file
+logwisp --config /etc/logwisp/production.toml
+```
+
+### Development Mode
+```bash
+# Enable debug logging to console
+logwisp --log-output stderr --log-level debug
+
+# Debug with file output
+logwisp --log-output both --log-level debug --log-dir ./debug-logs
+```
+
+### Production Deployment
+```bash
+# File logging with info level
+logwisp --log-output file --log-dir /var/log/logwisp --log-level info
+
+# Background mode with custom config
+logwisp --background --config /etc/logwisp/prod.toml
+
+# Router mode for multiple services
+logwisp --router --config /etc/logwisp/services.toml
+```
+
+### Troubleshooting
+```bash
+# Maximum verbosity to stderr
+logwisp --log-output stderr --log-level debug
+
+# Check version
+logwisp --version
+
+# Test configuration without backgrounding
+logwisp --config test.toml --log-level debug
+```
+
+## Priority Order
+
+Configuration values are applied in the following priority order (highest to lowest):
+
+1. **Command-line flags** - Explicitly specified options
+2. **Environment variables** - `LOGWISP_*` prefixed variables
+3. **Configuration file** - TOML configuration
+4. **Built-in defaults** - Hardcoded fallback values
+
+## Exit Codes
+
+- `0`: Successful execution
+- `1`: General error (configuration, startup failure)
+- `2`: Invalid command-line arguments
+
+## Signals
+
+LogWisp responds to the following signals:
+
+- `SIGINT` (Ctrl+C): Graceful shutdown
+- `SIGTERM`: Graceful shutdown
+- `SIGKILL`: Immediate termination (not recommended)
+
+During graceful shutdown, LogWisp will:
+1. Stop accepting new connections
+2. Finish streaming to existing clients
+3. Flush all buffers
+4. Close all file handles
+5. Exit cleanly
+
+## See Also
+
+- [Configuration Guide](configuration.md) - Complete configuration reference
+- [Environment Variables](environment.md) - Environment variable options
+- [Router Mode](router.md) - Path-based routing details
\ No newline at end of file
diff --git a/doc/configuration.md b/doc/configuration.md
new file mode 100644
index 0000000..d129b60
--- /dev/null
+++ b/doc/configuration.md
@@ -0,0 +1,354 @@
+# Configuration Guide
+
+LogWisp uses TOML format for configuration with sensible defaults for all settings.
+
+## Configuration File Location
+
+Default search order:
+1. Command line: `--config /path/to/config.toml`
+2. Environment: `$LOGWISP_CONFIG_FILE`
+3. User config: `~/.config/logwisp.toml`
+4. Current directory: `./logwisp.toml`
+
+## Configuration Structure
+
+```toml
+# Optional: LogWisp's own logging configuration
+[logging]
+output = "stderr" # file, stdout, stderr, both, none
+level = "info" # debug, info, warn, error
+
+# Required: At least one stream
+[[streams]]
+name = "default" # Unique identifier
+
+[streams.monitor] # Required: What to monitor
+# ... monitor settings ...
+
+[streams.httpserver] # Optional: HTTP/SSE server
+# ... HTTP settings ...
+
+[streams.tcpserver] # Optional: TCP server
+# ... TCP settings ...
+
+[[streams.filters]] # Optional: Log filtering
+# ... filter settings ...
+```
+
+## Logging Configuration
+
+Controls LogWisp's operational logging (not the logs being monitored).
+
+```toml
+[logging]
+output = "stderr" # Where to write LogWisp's logs
+level = "info" # Minimum log level
+
+# File output settings (when output includes "file")
+[logging.file]
+directory = "./logs" # Log directory
+name = "logwisp" # Base filename
+max_size_mb = 100 # Rotate at this size
+max_total_size_mb = 1000 # Total size limit
+retention_hours = 168 # Keep for 7 days
+
+# Console output settings
+[logging.console]
+target = "stderr" # stdout, stderr, split
+format = "txt" # txt or json
+```
+
+## Stream Configuration
+
+Each `[[streams]]` section defines an independent log monitoring pipeline.
+
+### Monitor Settings
+
+What files or directories to watch:
+
+```toml
+[streams.monitor]
+check_interval_ms = 100 # How often to check for new entries
+
+# Monitor targets (at least one required)
+targets = [
+ # Watch all .log files in a directory
+ { path = "/var/log/myapp", pattern = "*.log", is_file = false },
+
+ # Watch a specific file
+ { path = "/var/log/app.log", is_file = true },
+
+ # Multiple patterns
+ { path = "/logs", pattern = "app-*.log", is_file = false },
+ { path = "/logs", pattern = "error-*.txt", is_file = false }
+]
+```
+
+### HTTP Server (SSE)
+
+Server-Sent Events streaming over HTTP:
+
+```toml
+[streams.httpserver]
+enabled = true
+port = 8080
+buffer_size = 1000 # Per-client event buffer
+stream_path = "/stream" # SSE endpoint
+status_path = "/status" # Statistics endpoint
+
+# Keep-alive heartbeat
+[streams.httpserver.heartbeat]
+enabled = true
+interval_seconds = 30
+format = "comment" # "comment" or "json"
+include_timestamp = true
+include_stats = false
+
+# Rate limiting (optional)
+[streams.httpserver.rate_limit]
+enabled = false
+requests_per_second = 10.0
+burst_size = 20
+limit_by = "ip" # "ip" or "global"
+response_code = 429
+response_message = "Rate limit exceeded"
+max_connections_per_ip = 5
+max_total_connections = 100
+```
+
+### TCP Server
+
+Raw TCP streaming for high performance:
+
+```toml
+[streams.tcpserver]
+enabled = true
+port = 9090
+buffer_size = 5000 # Larger buffer for TCP
+
+# Heartbeat (always JSON format for TCP)
+[streams.tcpserver.heartbeat]
+enabled = true
+interval_seconds = 60
+include_timestamp = true
+include_stats = false
+
+# Rate limiting
+[streams.tcpserver.rate_limit]
+enabled = false
+requests_per_second = 5.0
+burst_size = 10
+limit_by = "ip"
+```
+
+### Filters
+
+Control which log entries are streamed:
+
+```toml
+# Include filter - only matching logs pass
+[[streams.filters]]
+type = "include"
+logic = "or" # "or" = match any, "and" = match all
+patterns = [
+ "ERROR",
+ "WARN",
+ "CRITICAL"
+]
+
+# Exclude filter - matching logs are dropped
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "DEBUG",
+ "health check"
+]
+```
+
+## Complete Examples
+
+### Minimal Configuration
+
+```toml
+[[streams]]
+name = "simple"
+[streams.monitor]
+targets = [{ path = "./logs", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080
+```
+
+### Production Web Application
+
+```toml
+[logging]
+output = "file"
+level = "info"
+[logging.file]
+directory = "/var/log/logwisp"
+max_size_mb = 500
+retention_hours = 336 # 14 days
+
+[[streams]]
+name = "webapp"
+
+[streams.monitor]
+check_interval_ms = 50
+targets = [
+ { path = "/var/log/nginx", pattern = "access.log*" },
+ { path = "/var/log/nginx", pattern = "error.log*" },
+ { path = "/var/log/myapp", pattern = "*.log" }
+]
+
+# Only errors and warnings
+[[streams.filters]]
+type = "include"
+logic = "or"
+patterns = [
+ "\\b(ERROR|error|Error)\\b",
+ "\\b(WARN|WARNING|warn|warning)\\b",
+ "\\b(CRITICAL|FATAL|critical|fatal)\\b",
+ "status=[4-5][0-9][0-9]" # HTTP errors
+]
+
+# Exclude noise
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "/health",
+ "/metrics",
+ "ELB-HealthChecker"
+]
+
+[streams.httpserver]
+enabled = true
+port = 8080
+buffer_size = 2000
+
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 25.0
+burst_size = 50
+max_connections_per_ip = 10
+```
+
+### Multi-Service with Router
+
+```toml
+# Run with: logwisp --router
+
+# Service 1: API
+[[streams]]
+name = "api"
+[streams.monitor]
+targets = [{ path = "/var/log/api", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080 # All streams can use same port in router mode
+
+# Service 2: Database
+[[streams]]
+name = "database"
+[streams.monitor]
+targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
+[[streams.filters]]
+type = "include"
+patterns = ["ERROR", "FATAL", "deadlock", "timeout"]
+[streams.httpserver]
+enabled = true
+port = 8080
+
+# Service 3: System
+[[streams]]
+name = "system"
+[streams.monitor]
+targets = [
+ { path = "/var/log/syslog", is_file = true },
+ { path = "/var/log/auth.log", is_file = true }
+]
+[streams.tcpserver]
+enabled = true
+port = 9090
+```
+
+### High-Security Configuration
+
+```toml
+[logging]
+output = "file"
+level = "warn" # Less verbose
+
+[[streams]]
+name = "secure"
+
+[streams.monitor]
+targets = [{ path = "/var/log/secure", pattern = "*.log" }]
+
+# Only security events
+[[streams.filters]]
+type = "include"
+patterns = [
+ "auth",
+ "sudo",
+ "ssh",
+ "login",
+ "failed",
+ "denied"
+]
+
+[streams.httpserver]
+enabled = true
+port = 8443
+
+# Strict rate limiting
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 2.0
+burst_size = 3
+limit_by = "ip"
+max_connections_per_ip = 1
+response_code = 403 # Forbidden instead of 429
+
+# Future: Authentication
+# [streams.auth]
+# type = "basic"
+# [streams.auth.basic_auth]
+# users_file = "/etc/logwisp/users.htpasswd"
+```
+
+## Configuration Tips
+
+### Performance Tuning
+
+- **check_interval_ms**: Higher values reduce CPU usage
+- **buffer_size**: Larger buffers handle bursts better
+- **rate_limit**: Essential for public-facing streams
+
+### Filter Patterns
+
+- Use word boundaries: `\\berror\\b` (won't match "errorCode")
+- Case-insensitive: `(?i)error`
+- Anchors for speed: `^ERROR` faster than `ERROR`
+- Test complex patterns before deployment
+
+### Resource Limits
+
+- Each stream uses ~10-50MB RAM (depending on buffers)
+- CPU usage scales with check_interval and file activity
+- Network bandwidth depends on log volume and client count
+
+## Validation
+
+LogWisp validates configuration on startup:
+- Required fields (name, monitor targets)
+- Port conflicts between streams
+- Pattern syntax for filters
+- Path accessibility
+
+## See Also
+
+- [Environment Variables](environment.md) - Override via environment
+- [CLI Options](cli.md) - Override via command line
+- [Filter Guide](filters.md) - Advanced filtering patterns
+- [Examples](examples/) - Ready-to-use configurations
\ No newline at end of file
diff --git a/doc/environment.md b/doc/environment.md
new file mode 100644
index 0000000..6e6c9fb
--- /dev/null
+++ b/doc/environment.md
@@ -0,0 +1,275 @@
+# Environment Variables
+
+LogWisp supports comprehensive configuration through environment variables, allowing deployment without configuration files or dynamic overrides in containerized environments.
+
+## Naming Convention
+
+Environment variables follow a structured pattern:
+- **Prefix**: `LOGWISP_`
+- **Path separator**: `_` (underscore)
+- **Array indices**: Numeric suffix (0-based)
+- **Case**: UPPERCASE
+
+### Examples:
+- Config file setting: `logging.level = "debug"`
+- Environment variable: `LOGWISP_LOGGING_LEVEL=debug`
+
+- Array element: `streams[0].name = "app"`
+- Environment variable: `LOGWISP_STREAMS_0_NAME=app`
+
+## General Variables
+
+### `LOGWISP_CONFIG_FILE`
+Path to the configuration file.
+- **Default**: `~/.config/logwisp.toml`
+- **Example**: `LOGWISP_CONFIG_FILE=/etc/logwisp/config.toml`
+
+### `LOGWISP_CONFIG_DIR`
+Directory containing configuration files.
+- **Usage**: Combined with `LOGWISP_CONFIG_FILE` for relative paths
+- **Example**:
+ ```bash
+ export LOGWISP_CONFIG_DIR=/etc/logwisp
+ export LOGWISP_CONFIG_FILE=production.toml
+ # Loads: /etc/logwisp/production.toml
+ ```
+
+### `LOGWISP_DISABLE_STATUS_REPORTER`
+Disable the periodic status reporter.
+- **Values**: `1` (disable), `0` or unset (enable)
+- **Default**: `0` (enabled)
+- **Example**: `LOGWISP_DISABLE_STATUS_REPORTER=1`
+
+### `LOGWISP_BACKGROUND`
+Internal marker for background process detection.
+- **Note**: Set automatically by `--background` flag
+- **Values**: `1` (background), unset (foreground)
+
+## Logging Variables
+
+### `LOGWISP_LOGGING_OUTPUT`
+LogWisp's operational log output mode.
+- **Values**: `file`, `stdout`, `stderr`, `both`, `none`
+- **Example**: `LOGWISP_LOGGING_OUTPUT=both`
+
+### `LOGWISP_LOGGING_LEVEL`
+Minimum log level for operational logs.
+- **Values**: `debug`, `info`, `warn`, `error`
+- **Example**: `LOGWISP_LOGGING_LEVEL=debug`
+
+### File Logging
+```bash
+LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
+LOGWISP_LOGGING_FILE_NAME=logwisp
+LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
+LOGWISP_LOGGING_FILE_MAX_TOTAL_SIZE_MB=1000
+LOGWISP_LOGGING_FILE_RETENTION_HOURS=168 # 7 days
+```
+
+### Console Logging
+```bash
+LOGWISP_LOGGING_CONSOLE_TARGET=stderr # stdout, stderr, split
+LOGWISP_LOGGING_CONSOLE_FORMAT=txt # txt, json
+```
+
+## Stream Configuration
+
+Streams are configured using array indices (0-based).
+
+### Basic Stream Settings
+```bash
+# First stream (index 0)
+LOGWISP_STREAMS_0_NAME=app
+LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=100
+
+# Second stream (index 1)
+LOGWISP_STREAMS_1_NAME=system
+LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
+```
+
+### Monitor Targets
+```bash
+# Single file target
+LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/app.log
+LOGWISP_STREAMS_0_MONITOR_TARGETS_0_IS_FILE=true
+
+# Directory with pattern
+LOGWISP_STREAMS_0_MONITOR_TARGETS_1_PATH=/var/log/myapp
+LOGWISP_STREAMS_0_MONITOR_TARGETS_1_PATTERN="*.log"
+LOGWISP_STREAMS_0_MONITOR_TARGETS_1_IS_FILE=false
+```
+
+### Filters
+```bash
+# Include filter
+LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
+LOGWISP_STREAMS_0_FILTERS_0_LOGIC=or
+LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN","CRITICAL"]'
+
+# Exclude filter
+LOGWISP_STREAMS_0_FILTERS_1_TYPE=exclude
+LOGWISP_STREAMS_0_FILTERS_1_PATTERNS='["DEBUG","TRACE"]'
+```
+
+### HTTP Server
+```bash
+LOGWISP_STREAMS_0_HTTPSERVER_ENABLED=true
+LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
+LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=1000
+LOGWISP_STREAMS_0_HTTPSERVER_STREAM_PATH=/stream
+LOGWISP_STREAMS_0_HTTPSERVER_STATUS_PATH=/status
+
+# Heartbeat
+LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
+LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INTERVAL_SECONDS=30
+LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=comment
+LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INCLUDE_TIMESTAMP=true
+LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_INCLUDE_STATS=false
+
+# Rate Limiting
+LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
+LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
+LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_BURST_SIZE=20
+LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_LIMIT_BY=ip
+LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_MAX_CONNECTIONS_PER_IP=5
+```
+
+### TCP Server
+```bash
+LOGWISP_STREAMS_0_TCPSERVER_ENABLED=true
+LOGWISP_STREAMS_0_TCPSERVER_PORT=9090
+LOGWISP_STREAMS_0_TCPSERVER_BUFFER_SIZE=5000
+
+# Rate Limiting
+LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_ENABLED=true
+LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=5.0
+LOGWISP_STREAMS_0_TCPSERVER_RATE_LIMIT_BURST_SIZE=10
+```
+
+## Complete Example
+
+Here's a complete example configuring two streams via environment variables:
+
+```bash
+#!/bin/bash
+
+# Logging configuration
+export LOGWISP_LOGGING_OUTPUT=both
+export LOGWISP_LOGGING_LEVEL=info
+export LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
+export LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
+
+# Stream 0: Application logs
+export LOGWISP_STREAMS_0_NAME=app
+export LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
+export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/myapp
+export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATTERN="*.log"
+export LOGWISP_STREAMS_0_MONITOR_TARGETS_0_IS_FILE=false
+
+# Stream 0: Filters
+export LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
+export LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
+
+# Stream 0: HTTP server
+export LOGWISP_STREAMS_0_HTTPSERVER_ENABLED=true
+export LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
+export LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
+export LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=25.0
+
+# Stream 1: System logs
+export LOGWISP_STREAMS_1_NAME=system
+export LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
+export LOGWISP_STREAMS_1_MONITOR_TARGETS_0_PATH=/var/log/syslog
+export LOGWISP_STREAMS_1_MONITOR_TARGETS_0_IS_FILE=true
+
+# Stream 1: TCP server
+export LOGWISP_STREAMS_1_TCPSERVER_ENABLED=true
+export LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
+
+# Start LogWisp
+logwisp
+```
+
+## Docker/Kubernetes Usage
+
+Environment variables are ideal for containerized deployments:
+
+### Docker
+```dockerfile
+FROM logwisp:latest
+ENV LOGWISP_LOGGING_OUTPUT=stdout
+ENV LOGWISP_STREAMS_0_NAME=container
+ENV LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/var/log/app
+ENV LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
+```
+
+### Docker Compose
+```yaml
+version: '3'
+services:
+ logwisp:
+ image: logwisp:latest
+ environment:
+ - LOGWISP_LOGGING_OUTPUT=stdout
+ - LOGWISP_STREAMS_0_NAME=webapp
+ - LOGWISP_STREAMS_0_MONITOR_TARGETS_0_PATH=/logs
+ - LOGWISP_STREAMS_0_HTTPSERVER_PORT=8080
+ volumes:
+ - ./logs:/logs:ro
+ ports:
+ - "8080:8080"
+```
+
+### Kubernetes ConfigMap
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logwisp-config
+data:
+ LOGWISP_LOGGING_LEVEL: "info"
+ LOGWISP_STREAMS_0_NAME: "k8s-app"
+ LOGWISP_STREAMS_0_HTTPSERVER_PORT: "8080"
+```
+
+## Precedence Rules
+
+When the same setting is configured multiple ways, this precedence applies:
+
+1. **Command-line flags** (highest priority)
+2. **Environment variables**
+3. **Configuration file**
+4. **Default values** (lowest priority)
+
+Example:
+```bash
+# Config file has: logging.level = "info"
+export LOGWISP_LOGGING_LEVEL=warn
+logwisp --log-level debug
+
+# Result: log level will be "debug" (CLI flag wins)
+```
+
+## Debugging
+
+To see which environment variables LogWisp recognizes:
+```bash
+# List all LOGWISP variables
+env | grep ^LOGWISP_
+
+# Test configuration parsing
+LOGWISP_LOGGING_LEVEL=debug logwisp --version
+```
+
+## Security Considerations
+
+- **Sensitive Values**: Avoid putting passwords or tokens in environment variables
+- **Process Visibility**: Environment variables may be visible to other processes
+- **Container Security**: Use secrets management for sensitive configuration
+- **Logging**: Be careful not to log environment variable values
+
+## See Also
+
+- [Configuration Guide](configuration.md) - Complete configuration reference
+- [CLI Options](cli.md) - Command-line interface
+- [Docker Deployment](integrations.md#docker) - Container-specific guidance
\ No newline at end of file
diff --git a/doc/filters.md b/doc/filters.md
new file mode 100644
index 0000000..23e9127
--- /dev/null
+++ b/doc/filters.md
@@ -0,0 +1,439 @@
+# Filter Guide
+
+LogWisp's filtering system allows you to control which log entries are streamed to clients, reducing noise and focusing on what matters.
+
+## How Filters Work
+
+Filters use regular expressions to match log entries. Each filter can either:
+- **Include**: Only matching logs pass through (whitelist)
+- **Exclude**: Matching logs are dropped (blacklist)
+
+Multiple filters are applied sequentially - a log entry must pass ALL filters to be streamed.
+
+## Filter Configuration
+
+### Basic Structure
+
+```toml
+[[streams.filters]]
+type = "include" # or "exclude"
+logic = "or" # or "and"
+patterns = [
+ "pattern1",
+ "pattern2"
+]
+```
+
+### Filter Types
+
+#### Include Filter (Whitelist)
+Only logs matching the patterns are streamed:
+
+```toml
+[[streams.filters]]
+type = "include"
+logic = "or"
+patterns = [
+ "ERROR",
+ "WARN",
+ "CRITICAL"
+]
+# Result: Only ERROR, WARN, or CRITICAL logs are streamed
+```
+
+#### Exclude Filter (Blacklist)
+Logs matching the patterns are dropped:
+
+```toml
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "DEBUG",
+ "TRACE",
+ "/health"
+]
+# Result: DEBUG, TRACE, and health check logs are filtered out
+```
+
+### Logic Operators
+
+#### OR Logic (Default)
+Log matches if ANY pattern matches:
+
+```toml
+[[streams.filters]]
+type = "include"
+logic = "or"
+patterns = ["ERROR", "FAIL", "EXCEPTION"]
+# Matches: "ERROR: disk full" OR "FAIL: connection timeout" OR "NullPointerException"
+```
+
+#### AND Logic
+Log matches only if ALL patterns match:
+
+```toml
+[[streams.filters]]
+type = "include"
+logic = "and"
+patterns = ["database", "timeout", "ERROR"]
+# Matches: "ERROR: database connection timeout"
+# Doesn't match: "ERROR: file not found" (missing "database" and "timeout")
+```
+
+## Pattern Syntax
+
+LogWisp uses Go's regular expression syntax (RE2):
+
+### Basic Patterns
+
+```toml
+patterns = [
+ "ERROR", # Exact substring match
+ "(?i)error", # Case-insensitive
+ "\\berror\\b", # Word boundaries
+ "^ERROR", # Start of line
+ "ERROR$", # End of line
+ "ERR(OR)?", # Optional group
+ "error|fail|exception" # Alternatives
+]
+```
+
+### Common Pattern Examples
+
+#### Log Levels
+```toml
+# Standard log levels
+patterns = [
+ "\\[(ERROR|WARN|INFO|DEBUG)\\]", # [ERROR] format
+ "(?i)\\b(error|warning|info|debug)\\b", # Word boundaries
+ "level=(error|warn|info|debug)", # key=value format
+ "<(Error|Warning|Info|Debug)>" # XML-style
+]
+
+# Severity patterns
+patterns = [
+ "(?i)(fatal|critical|severe)",
+ "(?i)(error|fail|exception)",
+ "(?i)(warn|warning|caution)",
+ "panic:", # Go panics
+ "Traceback", # Python errors
+]
+```
+
+#### Application Errors
+```toml
+# Java/JVM
+patterns = [
+ "Exception",
+ "\\.java:[0-9]+", # Stack trace lines
+ "at com\\.mycompany\\.", # Company packages
+ "NullPointerException|ClassNotFoundException"
+]
+
+# Python
+patterns = [
+ "Traceback \\(most recent call last\\)",
+ "File \".+\\.py\", line [0-9]+",
+ "(ValueError|TypeError|KeyError)"
+]
+
+# Go
+patterns = [
+ "panic:",
+ "goroutine [0-9]+",
+ "runtime error:"
+]
+
+# Node.js
+patterns = [
+ "Error:",
+ "at .+ \\(.+\\.js:[0-9]+:[0-9]+\\)",
+ "UnhandledPromiseRejection"
+]
+```
+
+#### Performance Issues
+```toml
+patterns = [
+ "took [0-9]{4,}ms", # Operations over 999ms
+ "duration>[0-9]{3,}s", # Long durations
+ "timeout|timed out", # Timeouts
+ "slow query", # Database
+ "memory pressure", # Memory issues
+ "high cpu|cpu usage: [8-9][0-9]%" # CPU issues
+]
+```
+
+#### Security Patterns
+```toml
+patterns = [
+ "(?i)(unauthorized|forbidden|denied)",
+ "(?i)(auth|authentication) fail",
+ "invalid (token|session|credentials)",
+ "SQL injection|XSS|CSRF",
+ "brute force|rate limit",
+ "suspicious activity"
+]
+```
+
+#### HTTP Patterns
+```toml
+# Error status codes
+patterns = [
+ "status[=:][4-5][0-9]{2}", # status=404, status:500
+ "HTTP/[0-9.]+ [4-5][0-9]{2}", # HTTP/1.1 404
+ "\"status\":\\s*[4-5][0-9]{2}" # JSON "status": 500
+]
+
+# Specific endpoints
+patterns = [
+ "\"(GET|POST|PUT|DELETE) /api/",
+ "/api/v[0-9]+/users",
+ "path=\"/admin"
+]
+```
+
+## Filter Chains
+
+Multiple filters create a processing chain. Each filter must pass for the log to be streamed.
+
+### Example: Error Monitoring
+```toml
+# Step 1: Include only errors and warnings
+[[streams.filters]]
+type = "include"
+logic = "or"
+patterns = [
+ "(?i)\\b(error|fail|exception)\\b",
+ "(?i)\\b(warn|warning)\\b",
+ "(?i)\\b(critical|fatal|severe)\\b"
+]
+
+# Step 2: Exclude known non-issues
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "Error: Expected behavior",
+ "Warning: Deprecated API",
+ "INFO.*error in message" # INFO logs talking about errors
+]
+
+# Step 3: Exclude noisy sources
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "/health",
+ "/metrics",
+ "ELB-HealthChecker",
+ "Googlebot"
+]
+```
+
+### Example: API Monitoring
+```toml
+# Include only API calls
+[[streams.filters]]
+type = "include"
+patterns = [
+ "/api/",
+ "/v[0-9]+/"
+]
+
+# Exclude successful requests
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "\" 200 ", # HTTP 200 OK
+ "\" 201 ", # HTTP 201 Created
+ "\" 204 ", # HTTP 204 No Content
+ "\" 304 " # HTTP 304 Not Modified
+]
+
+# Exclude OPTIONS requests (CORS)
+[[streams.filters]]
+type = "exclude"
+patterns = [
+ "OPTIONS "
+]
+```
+
+### Example: Security Audit
+```toml
+# Include security-relevant events
+[[streams.filters]]
+type = "include"
+logic = "or"
+patterns = [
+ "(?i)auth",
+ "(?i)login|logout",
+ "(?i)sudo|root",
+ "(?i)ssh|sftp|ftp",
+ "(?i)firewall|iptables",
+ "COMMAND=", # sudo commands
+ "USER=", # user actions
+ "SELINUX"
+]
+
+# Must also contain failure/success indicators
+[[streams.filters]]
+type = "include"
+logic = "or"
+patterns = [
+ "(?i)(fail|denied|error)",
+ "(?i)(success|accepted|granted)",
+ "(?i)(invalid|unauthorized)"
+]
+```
+
+## Performance Considerations
+
+### Pattern Complexity
+
+Simple patterns are fast (~1Ξs per check):
+```toml
+patterns = ["ERROR", "WARN", "FATAL"]
+```
+
+Complex patterns are slower (~10-100Ξs per check):
+```toml
+patterns = [
+ "^\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]\\s+\\[(ERROR|WARN)\\]\\s+\\[([^\\]]+)\\]\\s+(.+)$"
+]
+```
+
+### Optimization Tips
+
+1. **Use anchors when possible**:
+ ```toml
+ "^ERROR" # Faster than "ERROR"
+ ```
+
+2. **Avoid nested quantifiers**:
+ ```toml
+ # BAD: Can cause exponential backtracking
+ "((a+)+)+"
+
+ # GOOD: Linear time
+ "a+"
+ ```
+
+3. **Use non-capturing groups**:
+ ```toml
+ "(?:error|warn)" # Instead of "(error|warn)"
+ ```
+
+4. **Order patterns by frequency**:
+ ```toml
+ # Most common first
+ patterns = ["ERROR", "WARN", "INFO", "DEBUG"]
+ ```
+
+5. **Prefer character classes**:
+ ```toml
+ "[0-9]" # Instead of "\\d"
+ "[a-zA-Z]" # Instead of "\\w"
+ ```
+
+## Testing Filters
+
+### Test Configuration
+Create a test configuration with sample logs:
+
+```toml
+[[streams]]
+name = "test"
+[streams.monitor]
+targets = [{ path = "./test-logs", pattern = "*.log" }]
+
+[[streams.filters]]
+type = "include"
+patterns = ["YOUR_PATTERN_HERE"]
+
+[streams.httpserver]
+enabled = true
+port = 8888
+```
+
+### Generate Test Logs
+```bash
+# Create test log entries
+echo "[ERROR] Database connection failed" >> test-logs/app.log
+echo "[INFO] User logged in" >> test-logs/app.log
+echo "[WARN] High memory usage: 85%" >> test-logs/app.log
+
+# Run LogWisp with debug logging
+logwisp --config test.toml --log-level debug
+
+# Check what passes through
+curl -N http://localhost:8888/stream
+```
+
+### Debug Filter Behavior
+Enable debug logging to see filter decisions:
+
+```bash
+logwisp --log-level debug --log-output stderr
+```
+
+Look for messages like:
+```
+Entry filtered out component=filter_chain filter_index=0 filter_type=include
+Entry passed all filters component=filter_chain
+```
+
+## Common Pitfalls
+
+### Case Sensitivity
+By default, patterns are case-sensitive:
+```toml
+# Won't match "error" or "Error"
+patterns = ["ERROR"]
+
+# Use case-insensitive flag
+patterns = ["(?i)error"]
+```
+
+### Partial Matches
+Patterns match substrings by default:
+```toml
+# Matches "ERROR", "ERRORS", "TERROR"
+patterns = ["ERROR"]
+
+# Use word boundaries for exact words
+patterns = ["\\bERROR\\b"]
+```
+
+### Special Characters
+Remember to escape regex special characters:
+```toml
+# Won't work as expected
+patterns = ["[ERROR]"]
+
+# Correct: escape brackets
+patterns = ["\\[ERROR\\]"]
+```
+
+### Performance Impact
+Too many complex patterns can impact performance:
+```toml
+# Consider splitting into multiple streams instead
+[[streams.filters]]
+patterns = [
+ # 50+ complex patterns...
+]
+```
+
+## Best Practices
+
+1. **Start Simple**: Begin with basic patterns and refine as needed
+2. **Test Thoroughly**: Use test logs to verify filter behavior
+3. **Monitor Performance**: Check filter statistics in `/status`
+4. **Document Patterns**: Comment complex patterns for maintenance
+5. **Use Multiple Streams**: Instead of complex filters, consider separate streams
+6. **Regular Review**: Periodically review and optimize filter rules
+
+## See Also
+
+- [Configuration Guide](configuration.md) - Complete configuration reference
+- [Performance Tuning](performance.md) - Optimization guidelines
+- [Examples](examples/) - Real-world filter configurations
\ No newline at end of file
diff --git a/doc/installation.md b/doc/installation.md
new file mode 100644
index 0000000..e238278
--- /dev/null
+++ b/doc/installation.md
@@ -0,0 +1,591 @@
+# Installation Guide
+
+This guide covers installing LogWisp on various platforms and deployment scenarios.
+
+## Requirements
+
+### System Requirements
+
+- **OS**: Linux, macOS, FreeBSD, Windows (with WSL)
+- **Architecture**: amd64, arm64
+- **Memory**: 64MB minimum, 256MB recommended
+- **Disk**: 10MB for binary, plus log storage
+- **Go**: 1.23+ (for building from source)
+
+### Runtime Dependencies
+
+LogWisp is a single static binary with no runtime dependencies. It only requires:
+- Read access to monitored log files
+- Network access for serving streams
+- Write access for operational logs (optional)
+
+## Installation Methods
+
+### Pre-built Binaries
+
+Download the latest release:
+
+```bash
+# Linux (amd64)
+wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
+chmod +x logwisp-linux-amd64
+sudo mv logwisp-linux-amd64 /usr/local/bin/logwisp
+
+# macOS (Intel)
+wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-amd64
+chmod +x logwisp-darwin-amd64
+sudo mv logwisp-darwin-amd64 /usr/local/bin/logwisp
+
+# macOS (Apple Silicon)
+wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-arm64
+chmod +x logwisp-darwin-arm64
+sudo mv logwisp-darwin-arm64 /usr/local/bin/logwisp
+```
+
+Verify installation:
+```bash
+logwisp --version
+```
+
+### From Source
+
+Build from source code:
+
+```bash
+# Clone repository
+git clone https://github.com/yourusername/logwisp.git
+cd logwisp
+
+# Build
+make build
+
+# Install
+sudo make install
+
+# Or install to custom location
+make install PREFIX=/opt/logwisp
+```
+
+### Using Go Install
+
+Install directly with Go:
+
+```bash
+go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+```
+
+Note: This installs to `$GOPATH/bin` (usually `~/go/bin`)
+
+### Docker
+
+Official Docker image:
+
+```bash
+# Pull image
+docker pull yourusername/logwisp:latest
+
+# Run with volume mount
+docker run -d \
+ --name logwisp \
+ -p 8080:8080 \
+ -v /var/log:/logs:ro \
+ -v $PWD/config.toml:/config/logwisp.toml:ro \
+ yourusername/logwisp:latest \
+ --config /config/logwisp.toml
+```
+
+Build your own image:
+
+```dockerfile
+FROM golang:1.23-alpine AS builder
+WORKDIR /build
+COPY . .
+RUN go build -o logwisp ./src/cmd/logwisp
+
+FROM alpine:latest
+RUN apk --no-cache add ca-certificates
+COPY --from=builder /build/logwisp /usr/local/bin/
+ENTRYPOINT ["logwisp"]
+```
+
+## Platform-Specific Instructions
+
+### Linux
+
+#### Debian/Ubuntu
+
+Create package (planned):
+```bash
+# Future feature
+sudo apt install logwisp
+```
+
+Manual installation:
+```bash
+# Download binary
+wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64 -O logwisp
+chmod +x logwisp
+sudo mv logwisp /usr/local/bin/
+
+# Create config directory
+sudo mkdir -p /etc/logwisp
+sudo cp config/logwisp.toml.example /etc/logwisp/logwisp.toml
+
+# Create systemd service
+sudo tee /etc/systemd/system/logwisp.service << EOF
+[Unit]
+Description=LogWisp Log Monitoring Service
+After=network.target
+
+[Service]
+Type=simple
+User=logwisp
+Group=logwisp
+ExecStart=/usr/local/bin/logwisp --config /etc/logwisp/logwisp.toml
+Restart=always
+RestartSec=5
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=logwisp
+
+# Security hardening
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=strict
+ProtectHome=true
+ReadOnlyPaths=/var/log
+ReadWritePaths=/var/log/logwisp
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+# Create user
+sudo useradd -r -s /bin/false logwisp
+
+# Create log directory
+sudo mkdir -p /var/log/logwisp
+sudo chown logwisp:logwisp /var/log/logwisp
+
+# Enable and start
+sudo systemctl daemon-reload
+sudo systemctl enable logwisp
+sudo systemctl start logwisp
+```
+
+#### Red Hat/CentOS/Fedora
+
+```bash
+# Similar to Debian, but use:
+sudo yum install wget # or dnf on newer versions
+
+# SELinux context (if enabled)
+sudo semanage fcontext -a -t bin_t /usr/local/bin/logwisp
+sudo restorecon -v /usr/local/bin/logwisp
+```
+
+#### Arch Linux
+
+AUR package (community maintained):
+```bash
+# Future feature
+yay -S logwisp
+```
+
+### macOS
+
+#### Homebrew
+
+Formula (planned):
+```bash
+# Future feature
+brew install logwisp
+```
+
+#### Manual Installation
+
+```bash
+# Download and install
+curl -L https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-$(uname -m) -o logwisp
+chmod +x logwisp
+sudo mv logwisp /usr/local/bin/
+
+# Create LaunchDaemon
+sudo tee /Library/LaunchDaemons/com.logwisp.plist << EOF
+
+
+
+
+ Label
+ com.logwisp
+ ProgramArguments
+
+ /usr/local/bin/logwisp
+ --config
+ /usr/local/etc/logwisp/logwisp.toml
+
+ RunAtLoad
+
+ KeepAlive
+
+ StandardOutPath
+ /usr/local/var/log/logwisp.log
+ StandardErrorPath
+ /usr/local/var/log/logwisp.error.log
+
+
+EOF
+
+# Load service
+sudo launchctl load /Library/LaunchDaemons/com.logwisp.plist
+```
+
+### FreeBSD
+
+#### Ports
+
+```bash
+# Future feature
+cd /usr/ports/sysutils/logwisp
+make install clean
+```
+
+#### Manual Installation
+
+```bash
+# Download
+fetch https://github.com/yourusername/logwisp/releases/latest/download/logwisp-freebsd-amd64
+chmod +x logwisp-freebsd-amd64
+mv logwisp-freebsd-amd64 /usr/local/bin/logwisp
+
+# RC script
+cat > /usr/local/etc/rc.d/logwisp << 'EOF'
+#!/bin/sh
+
+# PROVIDE: logwisp
+# REQUIRE: DAEMON
+# KEYWORD: shutdown
+
+. /etc/rc.subr
+
+name="logwisp"
+rcvar="${name}_enable"
+command="/usr/local/bin/logwisp"
+command_args="--config /usr/local/etc/logwisp/logwisp.toml"
+pidfile="/var/run/${name}.pid"
+
+load_rc_config $name
+: ${logwisp_enable:="NO"}
+
+run_rc_command "$1"
+EOF
+
+chmod +x /usr/local/etc/rc.d/logwisp
+
+# Enable
+sysrc logwisp_enable="YES"
+service logwisp start
+```
+
+### Windows
+
+#### Windows Subsystem for Linux (WSL)
+
+```bash
+# Inside WSL, follow Linux instructions
+wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
+chmod +x logwisp-linux-amd64
+./logwisp-linux-amd64
+```
+
+#### Native Windows (planned)
+
+Future support for native Windows service.
+
+## Container Deployment
+
+### Docker Compose
+
+```yaml
+version: '3.8'
+
+services:
+ logwisp:
+ image: yourusername/logwisp:latest
+ container_name: logwisp
+ restart: unless-stopped
+ ports:
+ - "8080:8080"
+ - "9090:9090" # If using TCP
+ volumes:
+ - /var/log:/logs:ro
+ - ./logwisp.toml:/config/logwisp.toml:ro
+ command: ["--config", "/config/logwisp.toml"]
+ environment:
+ - LOGWISP_LOGGING_LEVEL=info
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8080/status"]
+ interval: 30s
+ timeout: 3s
+ retries: 3
+```
+
+### Kubernetes
+
+Deployment manifest:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: logwisp
+ labels:
+ app: logwisp
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: logwisp
+ template:
+ metadata:
+ labels:
+ app: logwisp
+ spec:
+ containers:
+ - name: logwisp
+ image: yourusername/logwisp:latest
+ args:
+ - --config
+ - /config/logwisp.toml
+ ports:
+ - containerPort: 8080
+ name: http
+ - containerPort: 9090
+ name: tcp
+ volumeMounts:
+ - name: logs
+ mountPath: /logs
+ readOnly: true
+ - name: config
+ mountPath: /config
+ livenessProbe:
+ httpGet:
+ path: /status
+ port: 8080
+ initialDelaySeconds: 10
+ periodSeconds: 30
+ readinessProbe:
+ httpGet:
+ path: /status
+ port: 8080
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ resources:
+ requests:
+ memory: "128Mi"
+ cpu: "100m"
+ limits:
+ memory: "512Mi"
+ cpu: "500m"
+ volumes:
+ - name: logs
+ hostPath:
+ path: /var/log
+ - name: config
+ configMap:
+ name: logwisp-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: logwisp
+spec:
+ selector:
+ app: logwisp
+ ports:
+ - name: http
+ port: 8080
+ targetPort: 8080
+ - name: tcp
+ port: 9090
+ targetPort: 9090
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: logwisp-config
+data:
+ logwisp.toml: |
+ [[streams]]
+ name = "k8s"
+ [streams.monitor]
+ targets = [{ path = "/logs", pattern = "*.log" }]
+ [streams.httpserver]
+ enabled = true
+ port = 8080
+```
+
+## Post-Installation
+
+### Verify Installation
+
+1. Check version:
+ ```bash
+ logwisp --version
+ ```
+
+2. Test configuration:
+ ```bash
+ logwisp --config /etc/logwisp/logwisp.toml --log-level debug
+ ```
+
+3. Check service status:
+ ```bash
+ # systemd
+ sudo systemctl status logwisp
+
+ # macOS
+ sudo launchctl list | grep logwisp
+
+ # FreeBSD
+ service logwisp status
+ ```
+
+4. Test streaming:
+ ```bash
+ curl -N http://localhost:8080/stream
+ ```
+
+### Security Hardening
+
+1. **Create dedicated user**:
+ ```bash
+ sudo useradd -r -s /bin/false -d /var/lib/logwisp logwisp
+ ```
+
+2. **Set file permissions**:
+ ```bash
+ sudo chown root:root /usr/local/bin/logwisp
+ sudo chmod 755 /usr/local/bin/logwisp
+ sudo chown -R logwisp:logwisp /etc/logwisp
+ sudo chmod 640 /etc/logwisp/logwisp.toml
+ ```
+
+3. **Configure firewall**:
+ ```bash
+ # UFW
+ sudo ufw allow 8080/tcp comment "LogWisp HTTP"
+
+ # firewalld
+ sudo firewall-cmd --permanent --add-port=8080/tcp
+ sudo firewall-cmd --reload
+ ```
+
+4. **Enable SELinux/AppArmor** (if applicable)
+
+### Initial Configuration
+
+1. Copy example configuration:
+ ```bash
+ sudo cp /usr/local/share/logwisp/examples/logwisp.toml.example /etc/logwisp/logwisp.toml
+ ```
+
+2. Edit configuration:
+ ```bash
+ sudo nano /etc/logwisp/logwisp.toml
+ ```
+
+3. Set up log monitoring:
+ ```toml
+ [[streams]]
+ name = "myapp"
+ [streams.monitor]
+ targets = [
+ { path = "/var/log/myapp", pattern = "*.log" }
+ ]
+ ```
+
+4. Restart service:
+ ```bash
+ sudo systemctl restart logwisp
+ ```
+
+## Uninstallation
+
+### Linux
+```bash
+# Stop service
+sudo systemctl stop logwisp
+sudo systemctl disable logwisp
+
+# Remove files
+sudo rm /usr/local/bin/logwisp
+sudo rm /etc/systemd/system/logwisp.service
+sudo rm -rf /etc/logwisp
+sudo rm -rf /var/log/logwisp
+
+# Remove user
+sudo userdel logwisp
+```
+
+### macOS
+```bash
+# Stop service
+sudo launchctl unload /Library/LaunchDaemons/com.logwisp.plist
+
+# Remove files
+sudo rm /usr/local/bin/logwisp
+sudo rm /Library/LaunchDaemons/com.logwisp.plist
+sudo rm -rf /usr/local/etc/logwisp
+```
+
+### Docker
+```bash
+docker stop logwisp
+docker rm logwisp
+docker rmi yourusername/logwisp:latest
+```
+
+## Troubleshooting Installation
+
+### Permission Denied
+
+If you get permission errors:
+```bash
+# Check file ownership
+ls -la /usr/local/bin/logwisp
+
+# Fix permissions
+sudo chmod +x /usr/local/bin/logwisp
+
+# Check log directory
+sudo mkdir -p /var/log/logwisp
+sudo chown logwisp:logwisp /var/log/logwisp
+```
+
+### Service Won't Start
+
+Check logs:
+```bash
+# systemd
+sudo journalctl -u logwisp -f
+
+# Manual run
+sudo -u logwisp /usr/local/bin/logwisp --config /etc/logwisp/logwisp.toml
+```
+
+### Port Already in Use
+
+Find conflicting process:
+```bash
+sudo lsof -i :8080
+# or
+sudo netstat -tlnp | grep 8080
+```
+
+## See Also
+
+- [Quick Start](quickstart.md) - Get running quickly
+- [Configuration Guide](configuration.md) - Configure LogWisp
+- [Troubleshooting](troubleshooting.md) - Common issues
+- [Security Best Practices](security.md) - Hardening guide
\ No newline at end of file
diff --git a/doc/monitoring.md b/doc/monitoring.md
new file mode 100644
index 0000000..b2a96d8
--- /dev/null
+++ b/doc/monitoring.md
@@ -0,0 +1,511 @@
+# Monitoring & Status Guide
+
+LogWisp provides comprehensive monitoring capabilities through status endpoints, operational logs, and metrics.
+
+## Status Endpoints
+
+### Stream Status
+
+Each stream exposes its own status endpoint:
+
+```bash
+# Standalone mode
+curl http://localhost:8080/status
+
+# Router mode
+curl http://localhost:8080/streamname/status
+```
+
+Example response:
+```json
+{
+ "service": "LogWisp",
+ "version": "1.0.0",
+ "server": {
+ "type": "http",
+ "port": 8080,
+ "active_clients": 5,
+ "buffer_size": 1000,
+ "uptime_seconds": 3600,
+ "mode": {
+ "standalone": true,
+ "router": false
+ }
+ },
+ "monitor": {
+ "active_watchers": 3,
+ "total_entries": 152341,
+ "dropped_entries": 12,
+ "start_time": "2024-01-20T10:00:00Z",
+ "last_entry_time": "2024-01-20T11:00:00Z"
+ },
+ "filters": {
+ "filter_count": 2,
+ "total_processed": 152341,
+ "total_passed": 48234,
+ "filters": [
+ {
+ "type": "include",
+ "logic": "or",
+ "pattern_count": 3,
+ "total_processed": 152341,
+ "total_matched": 48234,
+ "total_dropped": 0
+ }
+ ]
+ },
+ "features": {
+ "heartbeat": {
+ "enabled": true,
+ "interval": 30,
+ "format": "comment"
+ },
+ "rate_limit": {
+ "enabled": true,
+ "total_requests": 8234,
+ "blocked_requests": 89,
+ "active_ips": 12,
+ "total_connections": 5
+ }
+ }
+}
+```
+
+### Global Status (Router Mode)
+
+In router mode, a global status endpoint provides aggregated information:
+
+```bash
+curl http://localhost:8080/status
+```
+
+## Key Metrics
+
+### Monitor Metrics
+
+Track file watching performance:
+
+| Metric | Description | Healthy Range |
+|--------|-------------|---------------|
+| `active_watchers` | Number of files being watched | 1-1000 |
+| `total_entries` | Total log entries processed | Increasing |
+| `dropped_entries` | Entries dropped due to buffer full | < 1% of total |
+| `entries_per_second` | Current processing rate | Varies |
+
+### Connection Metrics
+
+Monitor client connections:
+
+| Metric | Description | Warning Signs |
+|--------|-------------|---------------|
+| `active_clients` | Current SSE connections | Near limit |
+| `tcp_connections` | Current TCP connections | Near limit |
+| `total_connections` | All active connections | > 80% of max |
+
+### Filter Metrics
+
+Understand filtering effectiveness:
+
+| Metric | Description | Optimization |
+|--------|-------------|--------------|
+| `total_processed` | Entries checked | - |
+| `total_passed` | Entries that passed | Very low = too restrictive |
+| `total_dropped` | Entries filtered out | Very high = review patterns |
+
+### Rate Limit Metrics
+
+Track rate limiting impact:
+
+| Metric | Description | Action Needed |
+|--------|-------------|---------------|
+| `blocked_requests` | Rejected requests | High = increase limits |
+| `active_ips` | Unique clients | High = scale out |
+| `blocked_percentage` | Rejection rate | > 10% = review |
+
+## Operational Logging
+
+### Log Levels
+
+Configure LogWisp's operational logging:
+
+```toml
+[logging]
+output = "both" # file and stderr
+level = "info" # info for production
+```
+
+Log levels and their use:
+- **DEBUG**: Detailed internal operations
+- **INFO**: Normal operations, connections
+- **WARN**: Recoverable issues
+- **ERROR**: Errors requiring attention
+
+### Important Log Messages
+
+#### Startup Messages
+```
+LogWisp starting version=1.0.0 config_file=/etc/logwisp.toml
+Stream registered with router stream=app
+TCP endpoint configured transport=system port=9090
+HTTP endpoints configured transport=app stream_url=http://localhost:8080/stream
+```
+
+#### Connection Events
+```
+HTTP client connected remote_addr=192.168.1.100:54231 active_clients=6
+HTTP client disconnected remote_addr=192.168.1.100:54231 active_clients=5
+TCP connection opened remote_addr=192.168.1.100:54232 active_connections=3
+```
+
+#### Error Conditions
+```
+Failed to open file for checking path=/var/log/app.log error=permission denied
+Scanner error while reading file path=/var/log/huge.log error=token too long
+Request rate limited ip=192.168.1.100
+Connection limit exceeded ip=192.168.1.100 connections=5 limit=5
+```
+
+#### Performance Warnings
+```
+Dropped log entry - subscriber buffer full
+Dropped entry for slow client remote_addr=192.168.1.100
+Check interval too small: 5ms (min: 10ms)
+```
+
+## Health Checks
+
+### Basic Health Check
+
+Simple up/down check:
+
+```bash
+#!/bin/bash
+# health_check.sh
+
+STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/status)
+
+if [ "$STATUS" -eq 200 ]; then
+ echo "LogWisp is healthy"
+ exit 0
+else
+ echo "LogWisp is unhealthy (status: $STATUS)"
+ exit 1
+fi
+```
+
+### Advanced Health Check
+
+Check specific conditions:
+
+```bash
+#!/bin/bash
+# advanced_health_check.sh
+
+RESPONSE=$(curl -s http://localhost:8080/status)
+
+# Check if processing logs
+ENTRIES=$(echo "$RESPONSE" | jq -r '.monitor.total_entries')
+if [ "$ENTRIES" -eq 0 ]; then
+ echo "WARNING: No log entries processed"
+ exit 1
+fi
+
+# Check dropped entries
+DROPPED=$(echo "$RESPONSE" | jq -r '.monitor.dropped_entries')
+TOTAL=$(echo "$RESPONSE" | jq -r '.monitor.total_entries')
+DROP_PERCENT=$(( DROPPED * 100 / TOTAL ))
+
+if [ "$DROP_PERCENT" -gt 5 ]; then
+ echo "WARNING: High drop rate: ${DROP_PERCENT}%"
+ exit 1
+fi
+
+# Check connections
+CONNECTIONS=$(echo "$RESPONSE" | jq -r '.server.active_clients')
+echo "OK: Processing logs, $CONNECTIONS active clients"
+exit 0
+```
+
+### Container Health Check
+
+Docker/Kubernetes configuration:
+
+```dockerfile
+# Dockerfile
+HEALTHCHECK --interval=30s --timeout=3s --retries=3 \
+ CMD curl -f http://localhost:8080/status || exit 1
+```
+
+```yaml
+# Kubernetes
+livenessProbe:
+ httpGet:
+ path: /status
+ port: 8080
+ initialDelaySeconds: 10
+ periodSeconds: 30
+
+readinessProbe:
+ httpGet:
+ path: /status
+ port: 8080
+ initialDelaySeconds: 5
+ periodSeconds: 10
+```
+
+## Monitoring Integration
+
+### Prometheus Metrics
+
+Export metrics in Prometheus format:
+
+```bash
+#!/bin/bash
+# prometheus_exporter.sh
+
+while true; do
+ STATUS=$(curl -s http://localhost:8080/status)
+
+ # Extract metrics
+ CLIENTS=$(echo "$STATUS" | jq -r '.server.active_clients')
+ ENTRIES=$(echo "$STATUS" | jq -r '.monitor.total_entries')
+ DROPPED=$(echo "$STATUS" | jq -r '.monitor.dropped_entries')
+
+ # Output Prometheus format
+ cat << EOF
+# HELP logwisp_active_clients Number of active streaming clients
+# TYPE logwisp_active_clients gauge
+logwisp_active_clients $CLIENTS
+
+# HELP logwisp_total_entries Total log entries processed
+# TYPE logwisp_total_entries counter
+logwisp_total_entries $ENTRIES
+
+# HELP logwisp_dropped_entries Total log entries dropped
+# TYPE logwisp_dropped_entries counter
+logwisp_dropped_entries $DROPPED
+EOF
+
+ sleep 60
+done
+```
+
+### Grafana Dashboard
+
+Key panels for Grafana:
+
+1. **Active Connections**
+ - Query: `logwisp_active_clients`
+ - Visualization: Graph
+ - Alert: > 80% of max
+
+2. **Log Processing Rate**
+ - Query: `rate(logwisp_total_entries[5m])`
+ - Visualization: Graph
+ - Alert: < 1 entry/min
+
+3. **Drop Rate**
+ - Query: `rate(logwisp_dropped_entries[5m]) / rate(logwisp_total_entries[5m])`
+ - Visualization: Gauge
+ - Alert: > 5%
+
+4. **Rate Limit Rejections**
+ - Query: `rate(logwisp_blocked_requests[5m])`
+ - Visualization: Graph
+ - Alert: > 10/min
+
+### Datadog Integration
+
+Send custom metrics:
+
+```bash
+#!/bin/bash
+# datadog_metrics.sh
+
+while true; do
+ STATUS=$(curl -s http://localhost:8080/status)
+
+ # Send metrics to Datadog
+ echo "$STATUS" | jq -r '
+ "logwisp.connections:\(.server.active_clients)|g",
+ "logwisp.entries:\(.monitor.total_entries)|c",
+ "logwisp.dropped:\(.monitor.dropped_entries)|c"
+ ' | while read metric; do
+ echo "$metric" | nc -u -w1 localhost 8125
+ done
+
+ sleep 60
+done
+```
+
+## Performance Monitoring
+
+### CPU Usage
+
+Monitor CPU usage by component:
+
+```bash
+# Check process CPU
+top -p $(pgrep logwisp) -b -n 1
+
+# Profile CPU usage
+go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30
+```
+
+Common CPU consumers:
+- File watching (reduce check_interval_ms)
+- Regex filtering (simplify patterns)
+- JSON encoding (reduce clients)
+
+### Memory Usage
+
+Track memory consumption:
+
+```bash
+# Check process memory
+ps aux | grep logwisp
+
+# Detailed memory stats
+cat /proc/$(pgrep logwisp)/status | grep -E "Vm(RSS|Size)"
+```
+
+Memory optimization:
+- Reduce buffer sizes
+- Limit connections
+- Simplify filters
+
+### Network Bandwidth
+
+Monitor streaming bandwidth:
+
+```bash
+# Network statistics
+netstat -i
+iftop -i eth0 -f "port 8080"
+
+# Connection count
+ss -tan | grep :8080 | wc -l
+```
+
+## Alerting
+
+### Basic Alerts
+
+Essential alerts to configure:
+
+| Alert | Condition | Severity |
+|-------|-----------|----------|
+| Service Down | Status endpoint fails | Critical |
+| High Drop Rate | > 10% entries dropped | Warning |
+| No Log Activity | 0 entries/min for 5 min | Warning |
+| Connection Limit | > 90% of max connections | Warning |
+| Rate Limit High | > 20% requests blocked | Warning |
+
+### Alert Script
+
+Example monitoring script:
+
+```bash
+#!/bin/bash
+# monitor_alerts.sh
+
+check_alert() {
+ local name=$1
+ local condition=$2
+ local message=$3
+
+ if eval "$condition"; then
+ echo "ALERT: $name - $message"
+ # Send to alerting system
+ # curl -X POST https://alerts.example.com/...
+ fi
+}
+
+while true; do
+ STATUS=$(curl -s http://localhost:8080/status)
+
+ if [ -z "$STATUS" ]; then
+ check_alert "SERVICE_DOWN" "true" "LogWisp not responding"
+ sleep 60
+ continue
+ fi
+
+ # Extract metrics
+ DROPPED=$(echo "$STATUS" | jq -r '.monitor.dropped_entries')
+ TOTAL=$(echo "$STATUS" | jq -r '.monitor.total_entries')
+ CLIENTS=$(echo "$STATUS" | jq -r '.server.active_clients')
+
+ # Check conditions
+ check_alert "HIGH_DROP_RATE" \
+ "[ $((DROPPED * 100 / TOTAL)) -gt 10 ]" \
+ "Drop rate above 10%"
+
+ check_alert "HIGH_CONNECTIONS" \
+ "[ $CLIENTS -gt 90 ]" \
+ "Near connection limit: $CLIENTS/100"
+
+ sleep 60
+done
+```
+
+## Troubleshooting with Monitoring
+
+### No Logs Appearing
+
+Check monitor stats:
+```bash
+curl -s http://localhost:8080/status | jq '.monitor'
+```
+
+Look for:
+- `active_watchers` = 0 (no files found)
+- `total_entries` not increasing (files not updating)
+
+### High CPU Usage
+
+Enable debug logging:
+```bash
+logwisp --log-level debug --log-output stderr
+```
+
+Watch for:
+- Frequent "checkFile" messages (reduce check_interval)
+- Many filter operations (optimize patterns)
+
+### Memory Growth
+
+Monitor over time:
+```bash
+while true; do
+ ps aux | grep logwisp | grep -v grep
+ curl -s http://localhost:8080/status | jq '.server.active_clients'
+ sleep 10
+done
+```
+
+### Connection Issues
+
+Check connection stats:
+```bash
+# Current connections
+curl -s http://localhost:8080/status | jq '.server'
+
+# Rate limit stats
+curl -s http://localhost:8080/status | jq '.features.rate_limit'
+```
+
+## Best Practices
+
+1. **Regular Monitoring**: Check status endpoints every 30-60 seconds
+2. **Set Alerts**: Configure alerts for critical conditions
+3. **Log Rotation**: Rotate LogWisp's own logs to prevent disk fill
+4. **Baseline Metrics**: Establish normal ranges for your environment
+5. **Capacity Planning**: Monitor trends for scaling decisions
+6. **Test Monitoring**: Verify alerts work before issues occur
+
+## See Also
+
+- [Performance Tuning](performance.md) - Optimization guide
+- [Troubleshooting](troubleshooting.md) - Common issues
+- [Configuration Guide](configuration.md) - Monitoring configuration
+- [Integration Examples](integrations.md) - Monitoring system integration
\ No newline at end of file
diff --git a/doc/quickstart.md b/doc/quickstart.md
new file mode 100644
index 0000000..e133730
--- /dev/null
+++ b/doc/quickstart.md
@@ -0,0 +1,209 @@
+# Quick Start Guide
+
+Get LogWisp up and running in 5 minutes!
+
+## Installation
+
+### From Source
+```bash
+# Clone the repository
+git clone https://github.com/yourusername/logwisp.git
+cd logwisp
+
+# Build and install
+make install
+
+# Or just build
+make build
+./logwisp --version
+```
+
+### Using Go Install
+```bash
+go install github.com/yourusername/logwisp/src/cmd/logwisp@latest
+```
+
+## Basic Usage
+
+### 1. Monitor Current Directory
+
+Start LogWisp with defaults (monitors `*.log` files in current directory):
+
+```bash
+logwisp
+```
+
+### 2. Stream Logs
+
+In another terminal, connect to the log stream:
+
+```bash
+# Using curl (SSE stream)
+curl -N http://localhost:8080/stream
+
+# Check status
+curl http://localhost:8080/status | jq .
+```
+
+### 3. Create Some Logs
+
+Generate test logs to see streaming in action:
+
+```bash
+# In a third terminal
+echo "[ERROR] Something went wrong!" >> test.log
+echo "[INFO] Application started" >> test.log
+echo "[WARN] Low memory warning" >> test.log
+```
+
+## Common Scenarios
+
+### Monitor Specific Directory
+
+Create a configuration file `~/.config/logwisp.toml`:
+
+```toml
+[[streams]]
+name = "myapp"
+
+[streams.monitor]
+targets = [
+ { path = "/var/log/myapp", pattern = "*.log", is_file = false }
+]
+
+[streams.httpserver]
+enabled = true
+port = 8080
+```
+
+Run LogWisp:
+```bash
+logwisp
+```
+
+### Filter Only Errors and Warnings
+
+Add filters to your configuration:
+
+```toml
+[[streams]]
+name = "errors"
+
+[streams.monitor]
+targets = [
+ { path = "./", pattern = "*.log" }
+]
+
+[[streams.filters]]
+type = "include"
+patterns = ["ERROR", "WARN", "CRITICAL", "FATAL"]
+
+[streams.httpserver]
+enabled = true
+port = 8080
+```
+
+### Multiple Log Sources
+
+Monitor different applications on different ports:
+
+```toml
+# Stream 1: Web application
+[[streams]]
+name = "webapp"
+[streams.monitor]
+targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080
+
+# Stream 2: Database
+[[streams]]
+name = "database"
+[streams.monitor]
+targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8081
+```
+
+### TCP Streaming
+
+For high-performance streaming, use TCP:
+
+```toml
+[[streams]]
+name = "highperf"
+
+[streams.monitor]
+targets = [{ path = "/var/log/app", pattern = "*.log" }]
+
+[streams.tcpserver]
+enabled = true
+port = 9090
+buffer_size = 5000
+```
+
+Connect with netcat:
+```bash
+nc localhost 9090
+```
+
+### Router Mode
+
+Consolidate multiple streams on one port using router mode:
+
+```bash
+# With the multi-stream config above
+logwisp --router
+
+# Access streams at:
+# http://localhost:8080/webapp/stream
+# http://localhost:8080/database/stream
+# http://localhost:8080/status (global status)
+```
+
+## Quick Tips
+
+### Enable Debug Logging
+```bash
+logwisp --log-level debug --log-output stderr
+```
+
+### Run in Background
+```bash
+logwisp --background --config /etc/logwisp/prod.toml
+```
+
+### Rate Limiting
+Protect your streams from abuse:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 10.0
+burst_size = 20
+max_connections_per_ip = 5
+```
+
+### JSON Output Format
+For structured logging:
+
+```toml
+[logging.console]
+format = "json"
+```
+
+## What's Next?
+
+- Read the [Configuration Guide](configuration.md) for all options
+- Learn about [Filters](filters.md) for advanced pattern matching
+- Explore [Rate Limiting](ratelimiting.md) for production deployments
+- Check out [Example Configurations](examples/) for more scenarios
+
+## Getting Help
+
+- Run `logwisp --help` for CLI options
+- Check `http://localhost:8080/status` for runtime statistics
+- Enable debug logging for troubleshooting
+- Visit our [GitHub repository](https://github.com/yourusername/logwisp) for issues and discussions
\ No newline at end of file
diff --git a/doc/ratelimiting.md b/doc/ratelimiting.md
new file mode 100644
index 0000000..96243ca
--- /dev/null
+++ b/doc/ratelimiting.md
@@ -0,0 +1,526 @@
+# Rate Limiting Guide
+
+LogWisp provides configurable rate limiting to protect against abuse, prevent resource exhaustion, and ensure fair access to log streams.
+
+## How Rate Limiting Works
+
+LogWisp uses a **token bucket algorithm** for smooth, burst-tolerant rate limiting:
+
+1. Each client (or globally) gets a bucket with a fixed capacity
+2. Tokens are added to the bucket at a configured rate
+3. Each request consumes one token
+4. If no tokens are available, the request is rejected
+5. The bucket can accumulate tokens up to its capacity for bursts
+
+## Configuration
+
+### Basic Configuration
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true # Enable rate limiting
+requests_per_second = 10.0 # Token refill rate
+burst_size = 20 # Maximum tokens (bucket capacity)
+limit_by = "ip" # "ip" or "global"
+```
+
+### Complete Options
+
+```toml
+[streams.httpserver.rate_limit]
+# Core settings
+enabled = true # Enable/disable rate limiting
+requests_per_second = 10.0 # Token generation rate (float)
+burst_size = 20 # Token bucket capacity
+
+# Limiting strategy
+limit_by = "ip" # "ip" or "global"
+
+# Connection limits
+max_connections_per_ip = 5 # Max concurrent connections per IP
+max_total_connections = 100 # Max total concurrent connections
+
+# Response configuration
+response_code = 429 # HTTP status code when limited
+response_message = "Rate limit exceeded" # Error message
+
+# Same options available for TCP
+[streams.tcpserver.rate_limit]
+enabled = true
+requests_per_second = 5.0
+burst_size = 10
+limit_by = "ip"
+```
+
+## Limiting Strategies
+
+### Per-IP Limiting (Default)
+
+Each client IP address gets its own token bucket:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+limit_by = "ip"
+requests_per_second = 10.0
+burst_size = 20
+```
+
+**Use cases:**
+- Fair access for multiple users
+- Prevent single client from monopolizing resources
+- Public-facing endpoints
+
+**Example behavior:**
+- Client A: Can make 10 req/sec
+- Client B: Also can make 10 req/sec
+- Total: Up to 10 Ã number of clients
+
+### Global Limiting
+
+All clients share a single token bucket:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+limit_by = "global"
+requests_per_second = 50.0
+burst_size = 100
+```
+
+**Use cases:**
+- Protect backend resources
+- Control total system load
+- Internal services with known clients
+
+**Example behavior:**
+- All clients combined: 50 req/sec max
+- One aggressive client can consume all tokens
+
+## Connection Limits
+
+In addition to request rate limiting, you can limit concurrent connections:
+
+### Per-IP Connection Limit
+
+```toml
+[streams.httpserver.rate_limit]
+max_connections_per_ip = 5 # Each IP can have max 5 connections
+```
+
+**Behavior:**
+- Prevents connection exhaustion attacks
+- Limits resource usage per client
+- Checked before rate limits
+
+### Total Connection Limit
+
+```toml
+[streams.httpserver.rate_limit]
+max_total_connections = 100 # Max 100 connections total
+```
+
+**Behavior:**
+- Protects server resources
+- Prevents memory exhaustion
+- Global limit across all IPs
+
+## Response Behavior
+
+### HTTP Responses
+
+When rate limited, HTTP clients receive:
+
+```json
+{
+ "error": "Rate limit exceeded",
+ "retry_after": "60"
+}
+```
+
+With these headers:
+- Status code: 429 (default) or configured value
+- Content-Type: application/json
+
+Configure custom responses:
+
+```toml
+[streams.httpserver.rate_limit]
+response_code = 503 # Service Unavailable
+response_message = "Server overloaded, please retry later"
+```
+
+### TCP Behavior
+
+TCP connections are **silently dropped** when rate limited:
+- No error message sent
+- Connection immediately closed
+- Prevents information leakage
+
+## Configuration Examples
+
+### Light Protection
+
+For internal or trusted environments:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 50.0
+burst_size = 100
+limit_by = "ip"
+```
+
+### Moderate Protection
+
+For semi-public endpoints:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 10.0
+burst_size = 30
+limit_by = "ip"
+max_connections_per_ip = 5
+max_total_connections = 200
+```
+
+### Strict Protection
+
+For public or sensitive endpoints:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 2.0
+burst_size = 5
+limit_by = "ip"
+max_connections_per_ip = 2
+max_total_connections = 50
+response_code = 503
+response_message = "Service temporarily unavailable"
+```
+
+### Debug/Development
+
+Disable for testing:
+
+```toml
+[streams.httpserver.rate_limit]
+enabled = false
+```
+
+## Use Case Scenarios
+
+### Public Log Viewer
+
+Prevent abuse while allowing legitimate use:
+
+```toml
+[[streams]]
+name = "public-logs"
+
+[streams.httpserver]
+enabled = true
+port = 8080
+
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 5.0 # 5 new connections per second
+burst_size = 10 # Allow short bursts
+limit_by = "ip"
+max_connections_per_ip = 3 # Max 3 streams per user
+max_total_connections = 100
+```
+
+### Internal Monitoring
+
+Protect against accidental overload:
+
+```toml
+[[streams]]
+name = "internal-metrics"
+
+[streams.httpserver]
+enabled = true
+port = 8081
+
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 100.0 # High limit for internal use
+burst_size = 200
+limit_by = "global" # Total system limit
+max_total_connections = 500
+```
+
+### High-Security Audit Logs
+
+Very restrictive access:
+
+```toml
+[[streams]]
+name = "audit"
+
+[streams.httpserver]
+enabled = true
+port = 8443
+
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 0.5 # 1 request every 2 seconds
+burst_size = 2
+limit_by = "ip"
+max_connections_per_ip = 1 # Single connection only
+max_total_connections = 10
+response_code = 403 # Forbidden (hide rate limit)
+response_message = "Access denied"
+```
+
+### Multi-Tenant Service
+
+Different limits per stream:
+
+```toml
+# Free tier
+[[streams]]
+name = "logs-free"
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 1.0
+burst_size = 5
+max_connections_per_ip = 1
+
+# Premium tier
+[[streams]]
+name = "logs-premium"
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 50.0
+burst_size = 100
+max_connections_per_ip = 10
+```
+
+## Monitoring Rate Limits
+
+### Status Endpoint
+
+Check rate limit statistics:
+
+```bash
+curl http://localhost:8080/status | jq '.server.features.rate_limit'
+```
+
+Response includes:
+```json
+{
+ "enabled": true,
+ "total_requests": 15234,
+ "blocked_requests": 89,
+ "active_ips": 12,
+ "total_connections": 8,
+ "config": {
+ "requests_per_second": 10,
+ "burst_size": 20,
+ "limit_by": "ip"
+ }
+}
+```
+
+### Debug Logging
+
+Enable debug logs to see rate limit decisions:
+
+```bash
+logwisp --log-level debug
+```
+
+Look for messages:
+```
+Request rate limited ip=192.168.1.100
+Connection limit exceeded ip=192.168.1.100 connections=5 limit=5
+Created new IP limiter ip=192.168.1.100 total_ips=3
+```
+
+## Testing Rate Limits
+
+### Test Script
+
+```bash
+#!/bin/bash
+# Test rate limiting behavior
+
+URL="http://localhost:8080/stream"
+PARALLEL=10
+DURATION=10
+
+echo "Testing rate limits..."
+echo "URL: $URL"
+echo "Parallel connections: $PARALLEL"
+echo "Duration: ${DURATION}s"
+echo
+
+# Function to connect and count lines
+test_connection() {
+ local id=$1
+ local count=0
+ local start=$(date +%s)
+
+ while (( $(date +%s) - start < DURATION )); do
+ if curl -s -N --max-time 1 "$URL" >/dev/null 2>&1; then
+ ((count++))
+ echo "[$id] Connected successfully (total: $count)"
+ else
+ echo "[$id] Rate limited!"
+ fi
+ sleep 0.1
+ done
+}
+
+# Run parallel connections
+for i in $(seq 1 $PARALLEL); do
+ test_connection $i &
+done
+
+wait
+echo "Test complete"
+```
+
+### Load Testing
+
+Using Apache Bench (ab):
+
+```bash
+# Test burst handling
+ab -n 100 -c 20 http://localhost:8080/status
+
+# Test sustained load
+ab -n 1000 -c 5 -r http://localhost:8080/status
+```
+
+Using curl:
+
+```bash
+# Test connection limit
+for i in {1..10}; do
+ curl -N http://localhost:8080/stream &
+done
+```
+
+## Tuning Guidelines
+
+### Setting requests_per_second
+
+Consider:
+- Expected legitimate traffic
+- Server capacity
+- Client retry behavior
+
+**Formula**: `requests_per_second = expected_clients à requests_per_client`
+
+### Setting burst_size
+
+General rule: `burst_size = 2-3 Ã requests_per_second`
+
+Examples:
+- `10 req/s â burst_size = 20-30`
+- `1 req/s â burst_size = 3-5`
+- `100 req/s â burst_size = 200-300`
+
+### Connection Limits
+
+Based on available memory:
+- Each HTTP connection: ~1-2MB
+- Each TCP connection: ~0.5-1MB
+
+**Formula**: `max_connections = available_memory / memory_per_connection`
+
+## Common Issues
+
+### "All requests blocked"
+
+Check if:
+- Rate limits too strict
+- Burst size too small
+- Using global limiting with many clients
+
+### "Memory growth"
+
+Possible causes:
+- No connection limits set
+- Slow clients holding connections
+- Too high burst_size
+
+Solutions:
+```toml
+max_connections_per_ip = 5
+max_total_connections = 100
+```
+
+### "Legitimate users blocked"
+
+Consider:
+- Increasing burst_size for short spikes
+- Using per-IP instead of global limiting
+- Different streams for different user tiers
+
+## Security Considerations
+
+### Information Disclosure
+
+Rate limit responses can reveal information:
+
+```toml
+# Default - informative
+response_code = 429
+response_message = "Rate limit exceeded"
+
+# Security-focused - generic
+response_code = 503
+response_message = "Service unavailable"
+
+# High security - misleading
+response_code = 403
+response_message = "Forbidden"
+```
+
+### DDoS Protection
+
+Rate limiting helps but isn't complete DDoS protection:
+- Use with firewall rules
+- Consider CDN/proxy rate limiting
+- Monitor for distributed attacks
+
+### Resource Exhaustion
+
+Protect against:
+- Connection exhaustion
+- Memory exhaustion
+- CPU exhaustion
+
+```toml
+# Comprehensive protection
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 10.0
+burst_size = 20
+max_connections_per_ip = 5
+max_total_connections = 100
+limit_by = "ip"
+```
+
+## Best Practices
+
+1. **Start Conservative**: Begin with strict limits and relax as needed
+2. **Monitor Statistics**: Use `/status` endpoint to track behavior
+3. **Test Thoroughly**: Verify limits work as expected under load
+4. **Document Limits**: Make rate limits clear to users
+5. **Provide Retry Info**: Help clients implement proper retry logic
+6. **Different Tiers**: Consider different limits for different user types
+7. **Regular Review**: Adjust limits based on usage patterns
+
+## See Also
+
+- [Configuration Guide](configuration.md) - Complete configuration reference
+- [Security Best Practices](security.md) - Security hardening
+- [Performance Tuning](performance.md) - Optimization guidelines
+- [Troubleshooting](troubleshooting.md) - Common issues
\ No newline at end of file
diff --git a/doc/router.md b/doc/router.md
new file mode 100644
index 0000000..8b49e8d
--- /dev/null
+++ b/doc/router.md
@@ -0,0 +1,520 @@
+# Router Mode Guide
+
+Router mode allows multiple LogWisp streams to share HTTP ports through path-based routing, simplifying deployment and access control.
+
+## Overview
+
+In standard mode, each stream requires its own port:
+- Stream 1: `http://localhost:8080/stream`
+- Stream 2: `http://localhost:8081/stream`
+- Stream 3: `http://localhost:8082/stream`
+
+In router mode, streams share ports via paths:
+- Stream 1: `http://localhost:8080/app/stream`
+- Stream 2: `http://localhost:8080/database/stream`
+- Stream 3: `http://localhost:8080/system/stream`
+- Global status: `http://localhost:8080/status`
+
+## Enabling Router Mode
+
+Start LogWisp with the `--router` flag:
+
+```bash
+logwisp --router --config /etc/logwisp/multi-stream.toml
+```
+
+## Configuration
+
+### Basic Router Configuration
+
+```toml
+# All streams can use the same port in router mode
+[[streams]]
+name = "app"
+[streams.monitor]
+targets = [{ path = "/var/log/app", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080 # Same port OK
+
+[[streams]]
+name = "database"
+[streams.monitor]
+targets = [{ path = "/var/log/postgresql", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080 # Shared port
+
+[[streams]]
+name = "nginx"
+[streams.monitor]
+targets = [{ path = "/var/log/nginx", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080 # Shared port
+```
+
+### Path Structure
+
+In router mode, paths are automatically prefixed with the stream name:
+
+| Stream Name | Configuration Path | Router Mode Path |
+|------------|-------------------|------------------|
+| `app` | `/stream` | `/app/stream` |
+| `app` | `/status` | `/app/status` |
+| `database` | `/stream` | `/database/stream` |
+| `database` | `/status` | `/database/status` |
+
+### Custom Paths
+
+You can customize the paths in each stream:
+
+```toml
+[[streams]]
+name = "api"
+[streams.httpserver]
+stream_path = "/logs" # Becomes /api/logs
+status_path = "/health" # Becomes /api/health
+```
+
+## URL Endpoints
+
+### Stream Endpoints
+
+Access individual streams:
+
+```bash
+# SSE stream for 'app' logs
+curl -N http://localhost:8080/app/stream
+
+# Status for 'database' stream
+curl http://localhost:8080/database/status
+
+# Custom path example
+curl -N http://localhost:8080/api/logs
+```
+
+### Global Status
+
+Router mode provides a global status endpoint:
+
+```bash
+curl http://localhost:8080/status | jq .
+```
+
+Returns aggregated information:
+```json
+{
+ "service": "LogWisp Router",
+ "version": "1.0.0",
+ "port": 8080,
+ "total_streams": 3,
+ "streams": {
+ "app": { /* stream stats */ },
+ "database": { /* stream stats */ },
+ "nginx": { /* stream stats */ }
+ },
+ "router": {
+ "uptime_seconds": 3600,
+ "total_requests": 15234,
+ "routed_requests": 15220,
+ "failed_requests": 14
+ }
+}
+```
+
+## Port Sharing
+
+### How It Works
+
+1. Router server listens on configured ports
+2. Examines request path to determine target stream
+3. Routes request to appropriate stream handler
+4. Stream handles request as if standalone
+
+### Port Assignment Rules
+
+In router mode:
+- Multiple streams can use the same port
+- Router detects and consolidates shared ports
+- Each unique port gets one router server
+- TCP servers remain independent (no routing)
+
+Example with multiple ports:
+
+```toml
+# Streams 1-3 share port 8080
+[[streams]]
+name = "app"
+[streams.httpserver]
+port = 8080
+
+[[streams]]
+name = "db"
+[streams.httpserver]
+port = 8080
+
+[[streams]]
+name = "web"
+[streams.httpserver]
+port = 8080
+
+# Stream 4 uses different port
+[[streams]]
+name = "admin"
+[streams.httpserver]
+port = 9090
+
+# Result: 2 router servers (8080 and 9090)
+```
+
+## Use Cases
+
+### Microservices Architecture
+
+Route logs from different services:
+
+```toml
+[[streams]]
+name = "frontend"
+[streams.monitor]
+targets = [{ path = "/var/log/frontend", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080
+
+[[streams]]
+name = "backend"
+[streams.monitor]
+targets = [{ path = "/var/log/backend", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080
+
+[[streams]]
+name = "worker"
+[streams.monitor]
+targets = [{ path = "/var/log/worker", pattern = "*.log" }]
+[streams.httpserver]
+enabled = true
+port = 8080
+```
+
+Access via:
+- Frontend logs: `http://localhost:8080/frontend/stream`
+- Backend logs: `http://localhost:8080/backend/stream`
+- Worker logs: `http://localhost:8080/worker/stream`
+
+### Environment-Based Routing
+
+Different log levels per environment:
+
+```toml
+[[streams]]
+name = "prod"
+[streams.monitor]
+targets = [{ path = "/logs/prod", pattern = "*.log" }]
+[[streams.filters]]
+type = "include"
+patterns = ["ERROR", "WARN"]
+[streams.httpserver]
+port = 8080
+
+[[streams]]
+name = "staging"
+[streams.monitor]
+targets = [{ path = "/logs/staging", pattern = "*.log" }]
+[[streams.filters]]
+type = "include"
+patterns = ["ERROR", "WARN", "INFO"]
+[streams.httpserver]
+port = 8080
+
+[[streams]]
+name = "dev"
+[streams.monitor]
+targets = [{ path = "/logs/dev", pattern = "*.log" }]
+# No filters - all logs
+[streams.httpserver]
+port = 8080
+```
+
+### Department Access
+
+Separate streams for different teams:
+
+```toml
+[[streams]]
+name = "engineering"
+[streams.monitor]
+targets = [{ path = "/logs/apps", pattern = "*.log" }]
+[streams.httpserver]
+port = 8080
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 50.0
+
+[[streams]]
+name = "security"
+[streams.monitor]
+targets = [{ path = "/logs/audit", pattern = "*.log" }]
+[streams.httpserver]
+port = 8080
+[streams.httpserver.rate_limit]
+enabled = true
+requests_per_second = 5.0
+max_connections_per_ip = 1
+
+[[streams]]
+name = "support"
+[streams.monitor]
+targets = [{ path = "/logs/customer", pattern = "*.log" }]
+[[streams.filters]]
+type = "exclude"
+patterns = ["password", "token", "secret"]
+[streams.httpserver]
+port = 8080
+```
+
+## Advanced Features
+
+### Mixed Mode Deployment
+
+Combine router and standalone modes:
+
+```toml
+# Public streams via router
+[[streams]]
+name = "public-api"
+[streams.httpserver]
+enabled = true
+port = 8080 # Router mode
+
+[[streams]]
+name = "public-web"
+[streams.httpserver]
+enabled = true
+port = 8080 # Router mode
+
+# Internal stream standalone
+[[streams]]
+name = "internal"
+[streams.httpserver]
+enabled = true
+port = 9999 # Different port, standalone
+
+# High-performance TCP
+[[streams]]
+name = "metrics"
+[streams.tcpserver]
+enabled = true
+port = 9090 # TCP not affected by router
+```
+
+### Load Balancer Integration
+
+Router mode works well with load balancers:
+
+```nginx
+# Nginx configuration
+upstream logwisp {
+ server logwisp1:8080;
+ server logwisp2:8080;
+ server logwisp3:8080;
+}
+
+location /logs/ {
+ proxy_pass http://logwisp/;
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ proxy_buffering off;
+}
+```
+
+Access becomes:
+- `https://example.com/logs/app/stream`
+- `https://example.com/logs/database/stream`
+- `https://example.com/logs/status`
+
+### Path-Based Access Control
+
+Use reverse proxy for authentication:
+
+```nginx
+# Require auth for security logs
+location /logs/security/ {
+ auth_basic "Security Logs";
+ auth_basic_user_file /etc/nginx/security.htpasswd;
+ proxy_pass http://localhost:8080/security/;
+}
+
+# Public access for status
+location /logs/app/ {
+ proxy_pass http://localhost:8080/app/;
+}
+```
+
+## Limitations
+
+### Router Mode Limitations
+
+1. **HTTP Only**: Router mode only works for HTTP/SSE streams
+2. **No TCP Routing**: TCP streams remain on separate ports
+3. **Path Conflicts**: Stream names must be unique
+4. **Same Config**: All streams on a port share SSL/auth settings
+
+### When Not to Use Router Mode
+
+- High-performance scenarios (use TCP)
+- Streams need different SSL certificates
+- Complex authentication per stream
+- Network isolation requirements
+
+## Troubleshooting
+
+### "Path not found"
+
+Check available routes:
+```bash
+curl http://localhost:8080/invalid-path
+```
+
+Response shows available routes:
+```json
+{
+ "error": "Not Found",
+ "requested_path": "/invalid-path",
+ "available_routes": [
+ "/status (global status)",
+ "/app/stream (stream: app)",
+ "/app/status (status: app)",
+ "/database/stream (stream: database)",
+ "/database/status (status: database)"
+ ]
+}
+```
+
+### "Port conflict"
+
+If you see port conflicts:
+1. Ensure `--router` flag is used
+2. Check all streams have `httpserver.enabled = true`
+3. Verify no other services use the port
+
+### Debug Routing
+
+Enable debug logging:
+```bash
+logwisp --router --log-level debug
+```
+
+Look for routing decisions:
+```
+Router request method=GET path=/app/stream remote_addr=127.0.0.1:54321
+Routing request to stream stream=app original_path=/app/stream remaining_path=/stream
+```
+
+### Performance Impact
+
+Router mode adds minimal overhead:
+- ~100-200ns per request for path matching
+- Negligible memory overhead
+- No impact on streaming performance
+
+## Best Practices
+
+### Naming Conventions
+
+Use clear, consistent stream names:
+```toml
+# Good: Clear purpose
+name = "frontend-prod"
+name = "backend-staging"
+name = "worker-payments"
+
+# Bad: Ambiguous
+name = "logs1"
+name = "stream2"
+name = "test"
+```
+
+### Path Organization
+
+Group related streams:
+```
+/prod/frontend/stream
+/prod/backend/stream
+/staging/frontend/stream
+/staging/backend/stream
+```
+
+### Documentation
+
+Document your routing structure:
+```toml
+# Stream for production API logs
+# Access: https://logs.example.com/api-prod/stream
+[[streams]]
+name = "api-prod"
+```
+
+### Monitoring
+
+Use global status for overview:
+```bash
+# Monitor all streams
+watch -n 5 'curl -s localhost:8080/status | jq .streams'
+
+# Check specific stream
+curl -s localhost:8080/status | jq '.streams.app'
+```
+
+## Migration Guide
+
+### From Standalone to Router
+
+1. **Update configuration** - ensure consistent ports:
+ ```toml
+ # Change from different ports
+ [streams.httpserver]
+ port = 8080 # Was 8081, 8082, etc.
+ ```
+
+2. **Start with router flag**:
+ ```bash
+ logwisp --router --config existing.toml
+ ```
+
+3. **Update client URLs**:
+ ```bash
+ # Old: http://localhost:8081/stream
+ # New: http://localhost:8080/streamname/stream
+ ```
+
+4. **Update monitoring**:
+ ```bash
+ # Global status now available
+ curl http://localhost:8080/status
+ ```
+
+### Gradual Migration
+
+Run both modes during transition:
+```bash
+# Week 1: Run standalone (current)
+logwisp --config prod.toml
+
+# Week 2: Run both
+logwisp --config prod.toml & # Standalone
+logwisp --router --config prod-router.toml & # Router
+
+# Week 3: Router only
+logwisp --router --config prod.toml
+```
+
+## See Also
+
+- [Configuration Guide](configuration.md) - Stream configuration
+- [HTTP Streaming](api.md#http-sse) - SSE protocol details
+- [Load Balancing](integrations.md#load-balancers) - Integration patterns
+- [Security Best Practices](security.md) - Securing router deployments
\ No newline at end of file
diff --git a/doc/troubleshooting.md b/doc/troubleshooting.md
new file mode 100644
index 0000000..873b7e3
--- /dev/null
+++ b/doc/troubleshooting.md
@@ -0,0 +1,537 @@
+# Troubleshooting Guide
+
+This guide helps diagnose and resolve common issues with LogWisp.
+
+## Diagnostic Tools
+
+### Enable Debug Logging
+
+The first step in troubleshooting is enabling debug logs:
+
+```bash
+# Via command line
+logwisp --log-level debug --log-output stderr
+
+# Via environment
+export LOGWISP_LOGGING_LEVEL=debug
+logwisp
+
+# Via config
+[logging]
+level = "debug"
+output = "stderr"
+```
+
+### Check Status Endpoint
+
+Verify LogWisp is running and processing:
+
+```bash
+# Basic check
+curl http://localhost:8080/status
+
+# Pretty print
+curl -s http://localhost:8080/status | jq .
+
+# Check specific metrics
+curl -s http://localhost:8080/status | jq '.monitor'
+```
+
+### Test Log Streaming
+
+Verify streams are working:
+
+```bash
+# Test SSE stream (should show heartbeats if enabled)
+curl -N http://localhost:8080/stream
+
+# Test with timeout
+timeout 5 curl -N http://localhost:8080/stream
+
+# Test TCP stream
+nc localhost 9090
+```
+
+## Common Issues
+
+### No Logs Appearing
+
+**Symptoms:**
+- Stream connects but no log entries appear
+- Status shows `total_entries: 0`
+
+**Diagnosis:**
+
+1. Check monitor configuration:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.monitor'
+ ```
+
+2. Verify file paths exist:
+ ```bash
+ # Check your configured paths
+ ls -la /var/log/myapp/
+ ```
+
+3. Check file permissions:
+ ```bash
+ # LogWisp user must have read access
+ sudo -u logwisp ls /var/log/myapp/
+ ```
+
+4. Verify files match pattern:
+ ```bash
+ # If pattern is "*.log"
+ ls /var/log/myapp/*.log
+ ```
+
+5. Check if files are being updated:
+ ```bash
+ # Should show recent timestamps
+ ls -la /var/log/myapp/*.log
+ tail -f /var/log/myapp/app.log
+ ```
+
+**Solutions:**
+
+- Fix file permissions:
+ ```bash
+ sudo chmod 644 /var/log/myapp/*.log
+ sudo usermod -a -G adm logwisp # Add to log group
+ ```
+
+- Correct path configuration:
+ ```toml
+ targets = [
+ { path = "/correct/path/to/logs", pattern = "*.log" }
+ ]
+ ```
+
+- Use absolute paths:
+ ```toml
+ # Bad: Relative path
+ targets = [{ path = "./logs", pattern = "*.log" }]
+
+ # Good: Absolute path
+ targets = [{ path = "/var/log/app", pattern = "*.log" }]
+ ```
+
+### High CPU Usage
+
+**Symptoms:**
+- LogWisp process using excessive CPU
+- System slowdown
+
+**Diagnosis:**
+
+1. Check process CPU:
+ ```bash
+ top -p $(pgrep logwisp)
+ ```
+
+2. Review check intervals:
+ ```bash
+ grep check_interval /etc/logwisp/logwisp.toml
+ ```
+
+3. Count active watchers:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.monitor.active_watchers'
+ ```
+
+4. Check filter complexity:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.filters'
+ ```
+
+**Solutions:**
+
+- Increase check interval:
+ ```toml
+ [streams.monitor]
+ check_interval_ms = 1000 # Was 50ms
+ ```
+
+- Reduce watched files:
+ ```toml
+ # Instead of watching entire directory
+ targets = [
+ { path = "/var/log/specific-app.log", is_file = true }
+ ]
+ ```
+
+- Simplify filter patterns:
+ ```toml
+ # Complex regex (slow)
+ patterns = ["^\\[\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\]\\s+\\[(ERROR|WARN)\\]"]
+
+ # Simple patterns (fast)
+ patterns = ["ERROR", "WARN"]
+ ```
+
+### Memory Growth
+
+**Symptoms:**
+- Increasing memory usage over time
+- Eventually runs out of memory
+
+**Diagnosis:**
+
+1. Monitor memory usage:
+ ```bash
+ watch -n 10 'ps aux | grep logwisp'
+ ```
+
+2. Check connection count:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.server.active_clients'
+ ```
+
+3. Check for dropped entries:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.monitor.dropped_entries'
+ ```
+
+**Solutions:**
+
+- Limit connections:
+ ```toml
+ [streams.httpserver.rate_limit]
+ enabled = true
+ max_connections_per_ip = 5
+ max_total_connections = 100
+ ```
+
+- Reduce buffer sizes:
+ ```toml
+ [streams.httpserver]
+ buffer_size = 500 # Was 5000
+ ```
+
+- Enable rate limiting:
+ ```toml
+ [streams.httpserver.rate_limit]
+ enabled = true
+ requests_per_second = 10.0
+ ```
+
+### Connection Refused
+
+**Symptoms:**
+- Cannot connect to LogWisp
+- `curl: (7) Failed to connect`
+
+**Diagnosis:**
+
+1. Check if LogWisp is running:
+ ```bash
+ ps aux | grep logwisp
+ systemctl status logwisp
+ ```
+
+2. Verify listening ports:
+ ```bash
+ sudo netstat -tlnp | grep logwisp
+ # or
+ sudo ss -tlnp | grep logwisp
+ ```
+
+3. Check firewall:
+ ```bash
+ sudo iptables -L -n | grep 8080
+ sudo ufw status
+ ```
+
+**Solutions:**
+
+- Start the service:
+ ```bash
+ sudo systemctl start logwisp
+ ```
+
+- Fix port configuration:
+ ```toml
+ [streams.httpserver]
+ enabled = true # Must be true
+ port = 8080 # Correct port
+ ```
+
+- Open firewall:
+ ```bash
+ sudo ufw allow 8080/tcp
+ ```
+
+### Rate Limit Errors
+
+**Symptoms:**
+- HTTP 429 responses
+- "Rate limit exceeded" errors
+
+**Diagnosis:**
+
+1. Check rate limit stats:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.features.rate_limit'
+ ```
+
+2. Test rate limits:
+ ```bash
+ # Rapid requests
+ for i in {1..20}; do curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8080/status; done
+ ```
+
+**Solutions:**
+
+- Increase rate limits:
+ ```toml
+ [streams.httpserver.rate_limit]
+ requests_per_second = 50.0 # Was 10.0
+ burst_size = 100 # Was 20
+ ```
+
+- Use per-IP limiting:
+ ```toml
+ limit_by = "ip" # Instead of "global"
+ ```
+
+- Disable for internal use:
+ ```toml
+ enabled = false
+ ```
+
+### Filter Not Working
+
+**Symptoms:**
+- Unwanted logs still appearing
+- Wanted logs being filtered out
+
+**Diagnosis:**
+
+1. Check filter configuration:
+ ```bash
+ curl -s http://localhost:8080/status | jq '.filters'
+ ```
+
+2. Test patterns:
+ ```bash
+ # Test regex pattern
+ echo "ERROR: test message" | grep -E "your-pattern"
+ ```
+
+3. Enable debug logging to see filter decisions:
+ ```bash
+ logwisp --log-level debug 2>&1 | grep filter
+ ```
+
+**Solutions:**
+
+- Fix pattern syntax:
+ ```toml
+ # Word boundaries
+ patterns = ["\\bERROR\\b"] # Not "ERROR" which matches "TERROR"
+
+ # Case insensitive
+ patterns = ["(?i)error"]
+ ```
+
+- Check filter order:
+ ```toml
+ # Include filters run first
+ [[streams.filters]]
+ type = "include"
+ patterns = ["ERROR", "WARN"]
+
+ # Then exclude filters
+ [[streams.filters]]
+ type = "exclude"
+ patterns = ["IGNORE_THIS"]
+ ```
+
+- Use correct logic:
+ ```toml
+ logic = "or" # Match ANY pattern
+ # not
+ logic = "and" # Match ALL patterns
+ ```
+
+### Logs Dropping
+
+**Symptoms:**
+- `dropped_entries` counter increasing
+- Missing log entries in stream
+
+**Diagnosis:**
+
+1. Check drop statistics:
+ ```bash
+ curl -s http://localhost:8080/status | jq '{
+ dropped: .monitor.dropped_entries,
+ total: .monitor.total_entries,
+ percent: (.monitor.dropped_entries / .monitor.total_entries * 100)
+ }'
+ ```
+
+2. Monitor drop rate:
+ ```bash
+ watch -n 5 'curl -s http://localhost:8080/status | jq .monitor.dropped_entries'
+ ```
+
+**Solutions:**
+
+- Increase buffer sizes:
+ ```toml
+ [streams.httpserver]
+ buffer_size = 5000 # Was 1000
+ ```
+
+- Add flow control:
+ ```toml
+ [streams.monitor]
+ check_interval_ms = 500 # Slow down reading
+ ```
+
+- Reduce clients:
+ ```toml
+ [streams.httpserver.rate_limit]
+ max_total_connections = 50
+ ```
+
+## Performance Issues
+
+### Slow Response Times
+
+**Diagnosis:**
+```bash
+# Measure response time
+time curl -s http://localhost:8080/status > /dev/null
+
+# Check system load
+uptime
+top
+```
+
+**Solutions:**
+- Reduce concurrent operations
+- Increase system resources
+- Use TCP instead of HTTP for high volume
+
+### Network Bandwidth
+
+**Diagnosis:**
+```bash
+# Monitor network usage
+iftop -i eth0 -f "port 8080"
+
+# Check connection count
+ss -tan | grep :8080 | wc -l
+```
+
+**Solutions:**
+- Enable compression (future feature)
+- Filter more aggressively
+- Use TCP for local connections
+
+## Debug Commands
+
+### System Information
+
+```bash
+# LogWisp version
+logwisp --version
+
+# System resources
+free -h
+df -h
+ulimit -a
+
+# Network state
+ss -tlnp
+netstat -anp | grep logwisp
+```
+
+### Process Inspection
+
+```bash
+# Process details
+ps aux | grep logwisp
+
+# Open files
+lsof -p $(pgrep logwisp)
+
+# System calls (Linux)
+strace -p $(pgrep logwisp) -e trace=open,read,write
+
+# File system activity
+inotifywait -m /var/log/myapp/
+```
+
+### Configuration Validation
+
+```bash
+# Test configuration
+logwisp --config test.toml --log-level debug --log-output stderr
+
+# Check file syntax
+cat /etc/logwisp/logwisp.toml | grep -E "^\s*\["
+
+# Validate TOML
+python3 -m pip install toml
+python3 -c "import toml; toml.load('/etc/logwisp/logwisp.toml'); print('Valid')"
+```
+
+## Getting Help
+
+### Collect Diagnostic Information
+
+Create a diagnostic bundle:
+
+```bash
+#!/bin/bash
+# diagnostic.sh
+
+DIAG_DIR="logwisp-diag-$(date +%Y%m%d-%H%M%S)"
+mkdir -p "$DIAG_DIR"
+
+# Version
+logwisp --version > "$DIAG_DIR/version.txt" 2>&1
+
+# Configuration (sanitized)
+grep -v "password\|secret\|token" /etc/logwisp/logwisp.toml > "$DIAG_DIR/config.toml"
+
+# Status
+curl -s http://localhost:8080/status > "$DIAG_DIR/status.json"
+
+# System info
+uname -a > "$DIAG_DIR/system.txt"
+free -h >> "$DIAG_DIR/system.txt"
+df -h >> "$DIAG_DIR/system.txt"
+
+# Process info
+ps aux | grep logwisp > "$DIAG_DIR/process.txt"
+lsof -p $(pgrep logwisp) > "$DIAG_DIR/files.txt" 2>&1
+
+# Recent logs
+journalctl -u logwisp -n 1000 > "$DIAG_DIR/logs.txt" 2>&1
+
+# Create archive
+tar -czf "$DIAG_DIR.tar.gz" "$DIAG_DIR"
+rm -rf "$DIAG_DIR"
+
+echo "Diagnostic bundle created: $DIAG_DIR.tar.gz"
+```
+
+### Report Issues
+
+When reporting issues, include:
+1. LogWisp version
+2. Configuration (sanitized)
+3. Error messages
+4. Steps to reproduce
+5. Diagnostic bundle
+
+## See Also
+
+- [Monitoring Guide](monitoring.md) - Status and metrics
+- [Performance Tuning](performance.md) - Optimization
+- [Configuration Guide](configuration.md) - Settings reference
+- [FAQ](faq.md) - Frequently asked questions
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 510cc77..df9f187 100644
--- a/go.mod
+++ b/go.mod
@@ -1,11 +1,10 @@
module logwisp
-go 1.24.2
-
-toolchain go1.24.4
+go 1.24.5
require (
github.com/lixenwraith/config v0.0.0-20250701170607-8515fa0543b6
+ github.com/lixenwraith/log v0.0.0-20250710012114-049926224b0e
github.com/panjf2000/gnet/v2 v2.9.1
github.com/valyala/fasthttp v1.63.0
)
@@ -19,7 +18,7 @@ require (
github.com/valyala/bytebufferpool v1.0.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
- golang.org/x/sync v0.15.0 // indirect
- golang.org/x/sys v0.33.0 // indirect
+ golang.org/x/sync v0.16.0 // indirect
+ golang.org/x/sys v0.34.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
)
diff --git a/go.sum b/go.sum
index 0e750ea..4573c76 100644
--- a/go.sum
+++ b/go.sum
@@ -8,6 +8,8 @@ github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zt
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/lixenwraith/config v0.0.0-20250701170607-8515fa0543b6 h1:qE4SpAJWFaLkdRyE0FjTPBBRYE7LOvcmRCB5p86W73Q=
github.com/lixenwraith/config v0.0.0-20250701170607-8515fa0543b6/go.mod h1:4wPJ3HnLrYrtUwTinngCsBgtdIXsnxkLa7q4KAIbwY8=
+github.com/lixenwraith/log v0.0.0-20250710012114-049926224b0e h1:WjYl/OIKxDCFA1In2W0bJbCGJ/Ub9X9DL+avZRNjXIQ=
+github.com/lixenwraith/log v0.0.0-20250710012114-049926224b0e/go.mod h1:KFE7B7m2pu5kAl0olDCvywlOqFJhanogAhTlVvlp8JE=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
@@ -30,10 +32,10 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
-golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/src/cmd/logwisp/bootstrap.go b/src/cmd/logwisp/bootstrap.go
new file mode 100644
index 0000000..582b066
--- /dev/null
+++ b/src/cmd/logwisp/bootstrap.go
@@ -0,0 +1,229 @@
+// FILE: src/cmd/logwisp/bootstrap.go
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "logwisp/src/internal/config"
+ "logwisp/src/internal/service"
+ "logwisp/src/internal/version"
+
+ "github.com/lixenwraith/log"
+)
+
+// bootstrapService creates and initializes the log transport service
+func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service, *service.HTTPRouter, error) {
+ // Create log transport service
+ svc := service.New(ctx, logger)
+
+ // Create HTTP router if requested
+ var router *service.HTTPRouter
+ if *useRouter {
+ router = service.NewHTTPRouter(svc, logger)
+ logger.Info("msg", "HTTP router mode enabled")
+ }
+
+ // Initialize streams
+ successCount := 0
+ for _, streamCfg := range cfg.Streams {
+ logger.Info("msg", "Initializing transport", "transport", streamCfg.Name)
+
+ // Handle router mode configuration
+ if *useRouter && streamCfg.HTTPServer != nil && streamCfg.HTTPServer.Enabled {
+ if err := initializeRouterStream(svc, router, streamCfg); err != nil {
+ logger.Error("msg", "Failed to initialize router stream",
+ "transport", streamCfg.Name,
+ "error", err)
+ continue
+ }
+ } else {
+ // Standard standalone mode
+ if err := svc.CreateStream(streamCfg); err != nil {
+ logger.Error("msg", "Failed to create transport",
+ "transport", streamCfg.Name,
+ "error", err)
+ continue
+ }
+ }
+
+ successCount++
+ displayStreamEndpoints(streamCfg, *useRouter)
+ }
+
+ if successCount == 0 {
+ return nil, nil, fmt.Errorf("no streams successfully started (attempted %d)", len(cfg.Streams))
+ }
+
+ logger.Info("msg", "LogWisp started",
+ "version", version.Short(),
+ "transports", successCount)
+
+ return svc, router, nil
+}
+
+// initializeRouterStream sets up a stream for router mode
+func initializeRouterStream(svc *service.Service, router *service.HTTPRouter, streamCfg config.StreamConfig) error {
+ // Temporarily disable standalone server startup
+ originalEnabled := streamCfg.HTTPServer.Enabled
+ streamCfg.HTTPServer.Enabled = false
+
+ if err := svc.CreateStream(streamCfg); err != nil {
+ return err
+ }
+
+ // Get the created transport and configure for router mode
+ stream, err := svc.GetStream(streamCfg.Name)
+ if err != nil {
+ return err
+ }
+
+ if stream.HTTPServer != nil {
+ stream.HTTPServer.SetRouterMode()
+ // Restore enabled state
+ stream.Config.HTTPServer.Enabled = originalEnabled
+
+ if err := router.RegisterStream(stream); err != nil {
+ return err
+ }
+
+ logger.Info("msg", "Stream registered with router", "stream", streamCfg.Name)
+ }
+
+ return nil
+}
+
+// initializeLogger sets up the logger based on configuration and CLI flags
+func initializeLogger(cfg *config.Config) error {
+ logger = log.NewLogger()
+
+ var configArgs []string
+
+ // Determine output mode from CLI or config
+ outputMode := cfg.Logging.Output
+ if *logOutput != "" {
+ outputMode = *logOutput
+ }
+
+ // Determine log level
+ level := cfg.Logging.Level
+ if *logLevel != "" {
+ level = *logLevel
+ }
+ levelValue, err := parseLogLevel(level)
+ if err != nil {
+ return fmt.Errorf("invalid log level: %w", err)
+ }
+ configArgs = append(configArgs, fmt.Sprintf("level=%d", levelValue))
+
+ // Configure based on output mode
+ switch outputMode {
+ case "none":
+ // â ïļ SECURITY: Disabling logs may hide security events
+ configArgs = append(configArgs, "disable_file=true", "enable_stdout=false")
+
+ case "stdout":
+ configArgs = append(configArgs,
+ "disable_file=true",
+ "enable_stdout=true",
+ "stdout_target=stdout")
+
+ case "stderr":
+ configArgs = append(configArgs,
+ "disable_file=true",
+ "enable_stdout=true",
+ "stdout_target=stderr")
+
+ case "file":
+ configArgs = append(configArgs, "enable_stdout=false")
+ configureFileLogging(&configArgs, cfg)
+
+ case "both":
+ configArgs = append(configArgs, "enable_stdout=true")
+ configureFileLogging(&configArgs, cfg)
+ configureConsoleTarget(&configArgs, cfg)
+
+ default:
+ return fmt.Errorf("invalid log output mode: %s", outputMode)
+ }
+
+ // Apply format if specified
+ if cfg.Logging.Console != nil && cfg.Logging.Console.Format != "" {
+ configArgs = append(configArgs, fmt.Sprintf("format=%s", cfg.Logging.Console.Format))
+ }
+
+ return logger.InitWithDefaults(configArgs...)
+}
+
+// configureFileLogging sets up file-based logging parameters
+func configureFileLogging(configArgs *[]string, cfg *config.Config) {
+ // CLI overrides
+ if *logFile != "" {
+ dir := filepath.Dir(*logFile)
+ name := strings.TrimSuffix(filepath.Base(*logFile), filepath.Ext(*logFile))
+ *configArgs = append(*configArgs,
+ fmt.Sprintf("directory=%s", dir),
+ fmt.Sprintf("name=%s", name))
+ } else if *logDir != "" {
+ *configArgs = append(*configArgs,
+ fmt.Sprintf("directory=%s", *logDir),
+ fmt.Sprintf("name=%s", cfg.Logging.File.Name))
+ } else if cfg.Logging.File != nil {
+ // Use config file settings
+ *configArgs = append(*configArgs,
+ fmt.Sprintf("directory=%s", cfg.Logging.File.Directory),
+ fmt.Sprintf("name=%s", cfg.Logging.File.Name),
+ fmt.Sprintf("max_size_mb=%d", cfg.Logging.File.MaxSizeMB),
+ fmt.Sprintf("max_total_size_mb=%d", cfg.Logging.File.MaxTotalSizeMB))
+
+ if cfg.Logging.File.RetentionHours > 0 {
+ *configArgs = append(*configArgs,
+ fmt.Sprintf("retention_period_hrs=%.1f", cfg.Logging.File.RetentionHours))
+ }
+ }
+}
+
+// configureConsoleTarget sets up console output parameters
+func configureConsoleTarget(configArgs *[]string, cfg *config.Config) {
+ target := "stderr" // default
+
+ if *logConsole != "" {
+ target = *logConsole
+ } else if cfg.Logging.Console != nil && cfg.Logging.Console.Target != "" {
+ target = cfg.Logging.Console.Target
+ }
+
+ // Handle "split" mode at application level since log package doesn't support it natively
+ if target == "split" {
+ // For now, default to stderr for all since log package doesn't support split
+ // TODO: Future enhancement - route ERROR/WARN to stderr, INFO/DEBUG to stdout
+ target = "stderr"
+ }
+
+ *configArgs = append(*configArgs, fmt.Sprintf("stdout_target=%s", target))
+}
+
+// isBackgroundProcess checks if we're already running in background
+func isBackgroundProcess() bool {
+ return os.Getenv("LOGWISP_BACKGROUND") == "1"
+}
+
+// runInBackground starts the process in background
+func runInBackground() error {
+ cmd := exec.Command(os.Args[0], os.Args[1:]...)
+ cmd.Env = append(os.Environ(), "LOGWISP_BACKGROUND=1")
+ cmd.Stdin = nil
+ cmd.Stdout = os.Stdout // Keep stdout for logging
+ cmd.Stderr = os.Stderr // Keep stderr for logging
+
+ if err := cmd.Start(); err != nil {
+ return err
+ }
+
+ fmt.Printf("Started LogWisp in background (PID: %d)\n", cmd.Process.Pid)
+ return nil
+}
\ No newline at end of file
diff --git a/src/cmd/logwisp/flags.go b/src/cmd/logwisp/flags.go
new file mode 100644
index 0000000..6fcee22
--- /dev/null
+++ b/src/cmd/logwisp/flags.go
@@ -0,0 +1,124 @@
+// FILE: src/cmd/logwisp/flags.go
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "strings"
+
+ "github.com/lixenwraith/log"
+)
+
+// Command-line flags
+var (
+ // General flags
+ configFile = flag.String("config", "", "Config file path")
+ useRouter = flag.Bool("router", false, "Use HTTP router for path-based routing")
+ showVersion = flag.Bool("version", false, "Show version information")
+ background = flag.Bool("background", false, "Run as background process")
+
+ // Logging flags
+ logOutput = flag.String("log-output", "", "Log output: file, stdout, stderr, both, none (overrides config)")
+ logLevel = flag.String("log-level", "", "Log level: debug, info, warn, error (overrides config)")
+ logFile = flag.String("log-file", "", "Log file path (when using file output)")
+ logDir = flag.String("log-dir", "", "Log directory (when using file output)")
+ logConsole = flag.String("log-console", "", "Console target: stdout, stderr, split (overrides config)")
+)
+
+func init() {
+ flag.Usage = customUsage
+}
+
+func customUsage() {
+ fmt.Fprintf(os.Stderr, "LogWisp - Multi-Stream Log Monitoring Service\n\n")
+ fmt.Fprintf(os.Stderr, "Usage: %s [options]\n\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, "Options:\n")
+
+ // General options
+ fmt.Fprintf(os.Stderr, "\nGeneral:\n")
+ fmt.Fprintf(os.Stderr, " -config string\n\tConfig file path\n")
+ fmt.Fprintf(os.Stderr, " -router\n\tUse HTTP router for path-based routing\n")
+ fmt.Fprintf(os.Stderr, " -version\n\tShow version information\n")
+ fmt.Fprintf(os.Stderr, " -background\n\tRun as background process\n")
+
+ // Logging options
+ fmt.Fprintf(os.Stderr, "\nLogging:\n")
+ fmt.Fprintf(os.Stderr, " -log-output string\n\tLog output: file, stdout, stderr, both, none (overrides config)\n")
+ fmt.Fprintf(os.Stderr, " -log-level string\n\tLog level: debug, info, warn, error (overrides config)\n")
+ fmt.Fprintf(os.Stderr, " -log-file string\n\tLog file path (when using file output)\n")
+ fmt.Fprintf(os.Stderr, " -log-dir string\n\tLog directory (when using file output)\n")
+ fmt.Fprintf(os.Stderr, " -log-console string\n\tConsole target: stdout, stderr, split (overrides config)\n")
+
+ fmt.Fprintf(os.Stderr, "\nExamples:\n")
+ fmt.Fprintf(os.Stderr, " # Run with default config (logs to stderr)\n")
+ fmt.Fprintf(os.Stderr, " %s\n\n", os.Args[0])
+
+ fmt.Fprintf(os.Stderr, " # Run with file logging\n")
+ fmt.Fprintf(os.Stderr, " %s --log-output file --log-dir /var/log/logwisp\n\n", os.Args[0])
+
+ fmt.Fprintf(os.Stderr, " # Run with debug logging to both file and console\n")
+ fmt.Fprintf(os.Stderr, " %s --log-output both --log-level debug\n\n", os.Args[0])
+
+ fmt.Fprintf(os.Stderr, " # Run with custom config and override log level\n")
+ fmt.Fprintf(os.Stderr, " %s --config /etc/logwisp.toml --log-level warn\n\n", os.Args[0])
+
+ fmt.Fprintf(os.Stderr, " # Run in router mode with multiple streams\n")
+ fmt.Fprintf(os.Stderr, " %s --router --config /etc/logwisp/multi-stream.toml\n\n", os.Args[0])
+
+ fmt.Fprintf(os.Stderr, "Environment Variables:\n")
+ fmt.Fprintf(os.Stderr, " LOGWISP_CONFIG_FILE Config file path\n")
+ fmt.Fprintf(os.Stderr, " LOGWISP_CONFIG_DIR Config directory\n")
+ fmt.Fprintf(os.Stderr, " LOGWISP_DISABLE_STATUS_REPORTER Disable periodic status reports (set to 1)\n")
+ fmt.Fprintf(os.Stderr, " LOGWISP_BACKGROUND Internal use - background process marker\n")
+ fmt.Fprintf(os.Stderr, "\nFor complete documentation, see: https://github.com/logwisp/logwisp/tree/main/doc\n")
+}
+
+func parseFlags() error {
+ flag.Parse()
+
+ // Validate log-output flag if provided
+ if *logOutput != "" {
+ validOutputs := map[string]bool{
+ "file": true, "stdout": true, "stderr": true,
+ "both": true, "none": true,
+ }
+ if !validOutputs[*logOutput] {
+ return fmt.Errorf("invalid log-output: %s (valid: file, stdout, stderr, both, none)", *logOutput)
+ }
+ }
+
+ // Validate log-level flag if provided
+ if *logLevel != "" {
+ if _, err := parseLogLevel(*logLevel); err != nil {
+ return fmt.Errorf("invalid log-level: %s (valid: debug, info, warn, error)", *logLevel)
+ }
+ }
+
+ // Validate log-console flag if provided
+ if *logConsole != "" {
+ validTargets := map[string]bool{
+ "stdout": true, "stderr": true, "split": true,
+ }
+ if !validTargets[*logConsole] {
+ return fmt.Errorf("invalid log-console: %s (valid: stdout, stderr, split)", *logConsole)
+ }
+ }
+
+ return nil
+}
+
+func parseLogLevel(level string) (int, error) {
+ switch strings.ToLower(level) {
+ case "debug":
+ return int(log.LevelDebug), nil
+ case "info":
+ return int(log.LevelInfo), nil
+ case "warn", "warning":
+ return int(log.LevelWarn), nil
+ case "error":
+ return int(log.LevelError), nil
+ default:
+ return 0, fmt.Errorf("unknown log level: %s", level)
+ }
+}
\ No newline at end of file
diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go
index 3d86991..c686bc1 100644
--- a/src/cmd/logwisp/main.go
+++ b/src/cmd/logwisp/main.go
@@ -3,7 +3,6 @@ package main
import (
"context"
- "flag"
"fmt"
"os"
"os/signal"
@@ -11,25 +10,36 @@ import (
"time"
"logwisp/src/internal/config"
- "logwisp/src/internal/service"
"logwisp/src/internal/version"
+
+ "github.com/lixenwraith/log"
)
-func main() {
- // Parse CLI flags
- var (
- configFile = flag.String("config", "", "Config file path")
- useRouter = flag.Bool("router", false, "Use HTTP router for path-based routing")
- // routerPort = flag.Int("router-port", 0, "Override router port (default: first HTTP port)")
- showVersion = flag.Bool("version", false, "Show version information")
- )
- flag.Parse()
+var logger *log.Logger
+func main() {
+ // Parse and validate flags
+ if err := parseFlags(); err != nil {
+ fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+ os.Exit(1)
+ }
+
+ // Handle version flag
if *showVersion {
fmt.Println(version.String())
os.Exit(0)
}
+ // Handle background mode
+ if *background && !isBackgroundProcess() {
+ if err := runInBackground(); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to start background process: %v\n", err)
+ os.Exit(1)
+ }
+ os.Exit(0)
+ }
+
+ // Set config file environment if specified
if *configFile != "" {
os.Setenv("LOGWISP_CONFIG_FILE", *configFile)
}
@@ -41,6 +51,20 @@ func main() {
os.Exit(1)
}
+ // Initialize logger
+ if err := initializeLogger(cfg); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to initialize logger: %v\n", err)
+ os.Exit(1)
+ }
+ defer shutdownLogger()
+
+ // Log startup information
+ logger.Info("msg", "LogWisp starting",
+ "version", version.String(),
+ "config_file", *configFile,
+ "log_output", cfg.Logging.Output,
+ "router_mode", *useRouter)
+
// Create context for shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -49,82 +73,29 @@ func main() {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
- // Create log transport service
- svc := service.New(ctx)
-
- // Create HTTP router if requested
- var router *service.HTTPRouter
- if *useRouter {
- router = service.NewHTTPRouter(svc)
- fmt.Println("HTTP router mode enabled")
- }
-
- // Initialize streams
- successCount := 0
- for _, streamCfg := range cfg.Streams {
- fmt.Printf("Initializing transport '%s'...\n", streamCfg.Name)
-
- // Set router mode BEFORE creating transport
- if *useRouter && streamCfg.HTTPServer != nil && streamCfg.HTTPServer.Enabled {
- // Temporarily disable standalone server startup
- originalEnabled := streamCfg.HTTPServer.Enabled
- streamCfg.HTTPServer.Enabled = false
-
- if err := svc.CreateStream(streamCfg); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to create transport '%s': %v\n", streamCfg.Name, err)
- continue
- }
-
- // Get the created transport and configure for router mode
- stream, _ := svc.GetStream(streamCfg.Name)
- if stream.HTTPServer != nil {
- stream.HTTPServer.SetRouterMode()
- // Restore enabled state
- stream.Config.HTTPServer.Enabled = originalEnabled
-
- if err := router.RegisterStream(stream); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to register transport '%s' with router: %v\n",
- streamCfg.Name, err)
- } else {
- fmt.Printf("Stream '%s' registered with router\n", streamCfg.Name)
- }
- }
- } else {
- // Standard standalone mode
- if err := svc.CreateStream(streamCfg); err != nil {
- fmt.Fprintf(os.Stderr, "Failed to create transport '%s': %v\n", streamCfg.Name, err)
- continue
- }
- }
-
- successCount++
-
- // Display endpoints
- displayStreamEndpoints(streamCfg, *useRouter)
- }
-
- if successCount == 0 {
- fmt.Fprintln(os.Stderr, "No streams successfully started")
+ // Bootstrap the service
+ svc, router, err := bootstrapService(ctx, cfg)
+ if err != nil {
+ logger.Error("msg", "Failed to bootstrap service", "error", err)
os.Exit(1)
}
- fmt.Printf("LogWisp %s\n", version.Short())
- fmt.Printf("\n%d transport(s) running. Press Ctrl+C to stop.\n", successCount)
+ // Start status reporter if enabled
+ if shouldEnableStatusReporter() {
+ go statusReporter(svc)
+ }
- // Start periodic status display
- go statusReporter(svc)
-
- // Wait for shutdown
+ // Wait for shutdown signal
<-sigChan
- fmt.Println("\nShutting down...")
+ logger.Info("msg", "Shutdown signal received, starting graceful shutdown...")
// Shutdown router first if using it
if router != nil {
- fmt.Println("Shutting down HTTP router...")
+ logger.Info("msg", "Shutting down HTTP router...")
router.Shutdown()
}
- // Shutdown service (handles all streams)
+ // Shutdown service with timeout
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel()
@@ -136,68 +107,26 @@ func main() {
select {
case <-done:
- fmt.Println("Shutdown complete")
+ logger.Info("msg", "Shutdown complete")
case <-shutdownCtx.Done():
- fmt.Println("Shutdown timeout - forcing exit")
+ logger.Error("msg", "Shutdown timeout exceeded - forcing exit")
os.Exit(1)
}
}
-func displayStreamEndpoints(cfg config.StreamConfig, routerMode bool) {
- if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
- fmt.Printf(" TCP: port %d\n", cfg.TCPServer.Port)
- }
-
- if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
- if routerMode {
- fmt.Printf(" HTTP: /%s%s (transport), /%s%s (status)\n",
- cfg.Name, cfg.HTTPServer.StreamPath,
- cfg.Name, cfg.HTTPServer.StatusPath)
- } else {
- fmt.Printf(" HTTP: http://localhost:%d%s (transport), http://localhost:%d%s (status)\n",
- cfg.HTTPServer.Port, cfg.HTTPServer.StreamPath,
- cfg.HTTPServer.Port, cfg.HTTPServer.StatusPath)
- }
-
- if cfg.Auth != nil && cfg.Auth.Type != "none" {
- fmt.Printf(" Auth: %s\n", cfg.Auth.Type)
+func shutdownLogger() {
+ if logger != nil {
+ if err := logger.Shutdown(2 * time.Second); err != nil {
+ // Best effort - can't log the shutdown error
+ fmt.Fprintf(os.Stderr, "Logger shutdown error: %v\n", err)
}
}
}
-func statusReporter(service *service.Service) {
- ticker := time.NewTicker(30 * time.Second)
- defer ticker.Stop()
-
- for range ticker.C {
- stats := service.GetGlobalStats()
- totalStreams := stats["total_streams"].(int)
- if totalStreams == 0 {
- return
- }
-
- fmt.Printf("\n[%s] Active streams: %d\n",
- time.Now().Format("15:04:05"), totalStreams)
-
- for name, streamStats := range stats["streams"].(map[string]interface{}) {
- s := streamStats.(map[string]interface{})
- fmt.Printf(" %s: ", name)
-
- if monitor, ok := s["monitor"].(map[string]interface{}); ok {
- fmt.Printf("watchers=%d entries=%d ",
- monitor["active_watchers"],
- monitor["total_entries"])
- }
-
- if tcp, ok := s["tcp"].(map[string]interface{}); ok && tcp["enabled"].(bool) {
- fmt.Printf("tcp_conns=%d ", tcp["connections"])
- }
-
- if http, ok := s["http"].(map[string]interface{}); ok && http["enabled"].(bool) {
- fmt.Printf("http_conns=%d ", http["connections"])
- }
-
- fmt.Println()
- }
+func shouldEnableStatusReporter() bool {
+ // Status reporter can be disabled via environment variable
+ if os.Getenv("LOGWISP_DISABLE_STATUS_REPORTER") == "1" {
+ return false
}
+ return true
}
\ No newline at end of file
diff --git a/src/cmd/logwisp/status.go b/src/cmd/logwisp/status.go
new file mode 100644
index 0000000..f4ca0cd
--- /dev/null
+++ b/src/cmd/logwisp/status.go
@@ -0,0 +1,119 @@
+// FILE: src/cmd/logwisp/status.go
+package main
+
+import (
+ "fmt"
+ "time"
+
+ "logwisp/src/internal/config"
+ "logwisp/src/internal/service"
+)
+
+// statusReporter periodically logs service status
+func statusReporter(service *service.Service) {
+ ticker := time.NewTicker(30 * time.Second)
+ defer ticker.Stop()
+
+ for range ticker.C {
+ stats := service.GetGlobalStats()
+ totalStreams := stats["total_streams"].(int)
+ if totalStreams == 0 {
+ logger.Warn("msg", "No active streams in status report",
+ "component", "status_reporter")
+ return
+ }
+
+ // Log status at DEBUG level to avoid cluttering INFO logs
+ logger.Debug("msg", "Status report",
+ "component", "status_reporter",
+ "active_streams", totalStreams,
+ "time", time.Now().Format("15:04:05"))
+
+ // Log individual stream status
+ for name, streamStats := range stats["streams"].(map[string]interface{}) {
+ logStreamStatus(name, streamStats.(map[string]interface{}))
+ }
+ }
+}
+
+// logStreamStatus logs the status of an individual stream
+func logStreamStatus(name string, stats map[string]interface{}) {
+ statusFields := []interface{}{
+ "msg", "Stream status",
+ "stream", name,
+ }
+
+ // Add monitor statistics
+ if monitor, ok := stats["monitor"].(map[string]interface{}); ok {
+ statusFields = append(statusFields,
+ "watchers", monitor["active_watchers"],
+ "entries", monitor["total_entries"])
+ }
+
+ // Add TCP server statistics
+ if tcp, ok := stats["tcp"].(map[string]interface{}); ok && tcp["enabled"].(bool) {
+ statusFields = append(statusFields, "tcp_conns", tcp["connections"])
+ }
+
+ // Add HTTP server statistics
+ if http, ok := stats["http"].(map[string]interface{}); ok && http["enabled"].(bool) {
+ statusFields = append(statusFields, "http_conns", http["connections"])
+ }
+
+ logger.Debug(statusFields...)
+}
+
+// displayStreamEndpoints logs the configured endpoints for a stream
+func displayStreamEndpoints(cfg config.StreamConfig, routerMode bool) {
+ // Display TCP endpoints
+ if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
+ logger.Info("msg", "TCP endpoint configured",
+ "component", "main",
+ "transport", cfg.Name,
+ "port", cfg.TCPServer.Port)
+
+ if cfg.TCPServer.RateLimit != nil && cfg.TCPServer.RateLimit.Enabled {
+ logger.Info("msg", "TCP rate limiting enabled",
+ "transport", cfg.Name,
+ "requests_per_second", cfg.TCPServer.RateLimit.RequestsPerSecond,
+ "burst_size", cfg.TCPServer.RateLimit.BurstSize)
+ }
+ }
+
+ // Display HTTP endpoints
+ if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
+ if routerMode {
+ logger.Info("msg", "HTTP endpoints configured",
+ "transport", cfg.Name,
+ "stream_path", fmt.Sprintf("/%s%s", cfg.Name, cfg.HTTPServer.StreamPath),
+ "status_path", fmt.Sprintf("/%s%s", cfg.Name, cfg.HTTPServer.StatusPath))
+ } else {
+ logger.Info("msg", "HTTP endpoints configured",
+ "transport", cfg.Name,
+ "stream_url", fmt.Sprintf("http://localhost:%d%s", cfg.HTTPServer.Port, cfg.HTTPServer.StreamPath),
+ "status_url", fmt.Sprintf("http://localhost:%d%s", cfg.HTTPServer.Port, cfg.HTTPServer.StatusPath))
+ }
+
+ if cfg.HTTPServer.RateLimit != nil && cfg.HTTPServer.RateLimit.Enabled {
+ logger.Info("msg", "HTTP rate limiting enabled",
+ "transport", cfg.Name,
+ "requests_per_second", cfg.HTTPServer.RateLimit.RequestsPerSecond,
+ "burst_size", cfg.HTTPServer.RateLimit.BurstSize,
+ "limit_by", cfg.HTTPServer.RateLimit.LimitBy)
+ }
+
+ // Display authentication information
+ if cfg.Auth != nil && cfg.Auth.Type != "none" {
+ logger.Info("msg", "Authentication enabled",
+ "transport", cfg.Name,
+ "auth_type", cfg.Auth.Type)
+ }
+ }
+
+ // Display filter information
+ if len(cfg.Filters) > 0 {
+ logger.Info("msg", "Filters configured",
+ "transport", cfg.Name,
+ "filter_count", len(cfg.Filters))
+ }
+}
\ No newline at end of file
diff --git a/src/internal/config/config.go b/src/internal/config/config.go
index 7b66407..b331c75 100644
--- a/src/internal/config/config.go
+++ b/src/internal/config/config.go
@@ -2,6 +2,9 @@
package config
type Config struct {
+ // Logging configuration
+ Logging *LogConfig `toml:"logging"`
+
// Stream configurations
Streams []StreamConfig `toml:"streams"`
}
diff --git a/src/internal/config/loader.go b/src/internal/config/loader.go
index 582439f..8f02141 100644
--- a/src/internal/config/loader.go
+++ b/src/internal/config/loader.go
@@ -12,6 +12,7 @@ import (
func defaults() *Config {
return &Config{
+ Logging: DefaultLogConfig(),
Streams: []StreamConfig{
{
Name: "default",
diff --git a/src/internal/config/logging.go b/src/internal/config/logging.go
new file mode 100644
index 0000000..e06ee9a
--- /dev/null
+++ b/src/internal/config/logging.go
@@ -0,0 +1,62 @@
+// FILE: src/internal/config/logging.go
+package config
+
+// LogConfig represents logging configuration for LogWisp
+type LogConfig struct {
+ // Output mode: "file", "stdout", "stderr", "both", "none"
+ Output string `toml:"output"`
+
+ // Log level: "debug", "info", "warn", "error"
+ Level string `toml:"level"`
+
+ // File output settings (when Output includes "file" or "both")
+ File *LogFileConfig `toml:"file"`
+
+ // Console output settings
+ Console *LogConsoleConfig `toml:"console"`
+}
+
+type LogFileConfig struct {
+ // Directory for log files
+ Directory string `toml:"directory"`
+
+ // Base name for log files
+ Name string `toml:"name"`
+
+ // Maximum size per log file in MB
+ MaxSizeMB int64 `toml:"max_size_mb"`
+
+ // Maximum total size of all logs in MB
+ MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
+
+ // Log retention in hours (0 = disabled)
+ RetentionHours float64 `toml:"retention_hours"`
+}
+
+type LogConsoleConfig struct {
+ // Target for console output: "stdout", "stderr", "split"
+ // "split" means info/debug to stdout, warn/error to stderr
+ Target string `toml:"target"`
+
+ // Format: "txt" or "json"
+ Format string `toml:"format"`
+}
+
+// DefaultLogConfig returns sensible logging defaults
+func DefaultLogConfig() *LogConfig {
+ return &LogConfig{
+ Output: "stderr", // Default to stderr for containerized environments
+ Level: "info",
+ File: &LogFileConfig{
+ Directory: "./logs",
+ Name: "logwisp",
+ MaxSizeMB: 100,
+ MaxTotalSizeMB: 1000,
+ RetentionHours: 168, // 7 days
+ },
+ Console: &LogConsoleConfig{
+ Target: "stderr",
+ Format: "txt",
+ },
+ }
+}
\ No newline at end of file
diff --git a/src/internal/config/validation.go b/src/internal/config/validation.go
index 91ecc21..52ec677 100644
--- a/src/internal/config/validation.go
+++ b/src/internal/config/validation.go
@@ -14,6 +14,10 @@ func (c *Config) validate() error {
return fmt.Errorf("no streams configured")
}
+ if err := validateLogConfig(c.Logging); err != nil {
+ return fmt.Errorf("logging config: %w", err)
+ }
+
// Validate each transport
streamNames := make(map[string]bool)
streamPorts := make(map[int]string)
@@ -275,5 +279,33 @@ func validateFilter(streamName string, filterIndex int, cfg *filter.Config) erro
}
}
+ return nil
+}
+
+func validateLogConfig(cfg *LogConfig) error {
+ validOutputs := map[string]bool{
+ "file": true, "stdout": true, "stderr": true,
+ "both": true, "none": true,
+ }
+ if !validOutputs[cfg.Output] {
+ return fmt.Errorf("invalid log output mode: %s", cfg.Output)
+ }
+
+ validLevels := map[string]bool{
+ "debug": true, "info": true, "warn": true, "error": true,
+ }
+ if !validLevels[cfg.Level] {
+ return fmt.Errorf("invalid log level: %s", cfg.Level)
+ }
+
+ if cfg.Console != nil {
+ validTargets := map[string]bool{
+ "stdout": true, "stderr": true, "split": true,
+ }
+ if !validTargets[cfg.Console.Target] {
+ return fmt.Errorf("invalid console target: %s", cfg.Console.Target)
+ }
+ }
+
return nil
}
\ No newline at end of file
diff --git a/src/internal/filter/chain.go b/src/internal/filter/chain.go
index d0784d3..a4b6324 100644
--- a/src/internal/filter/chain.go
+++ b/src/internal/filter/chain.go
@@ -6,11 +6,14 @@ import (
"sync/atomic"
"logwisp/src/internal/monitor"
+
+ "github.com/lixenwraith/log"
)
// Chain manages multiple filters in sequence
type Chain struct {
filters []*Filter
+ logger *log.Logger
// Statistics
totalProcessed atomic.Uint64
@@ -18,19 +21,23 @@ type Chain struct {
}
// NewChain creates a new filter chain from configurations
-func NewChain(configs []Config) (*Chain, error) {
+func NewChain(configs []Config, logger *log.Logger) (*Chain, error) {
chain := &Chain{
filters: make([]*Filter, 0, len(configs)),
+ logger: logger,
}
for i, cfg := range configs {
- filter, err := New(cfg)
+ filter, err := New(cfg, logger)
if err != nil {
return nil, fmt.Errorf("filter[%d]: %w", i, err)
}
chain.filters = append(chain.filters, filter)
}
+ logger.Info("msg", "Filter chain created",
+ "component", "filter_chain",
+ "filter_count", len(configs))
return chain, nil
}
@@ -46,8 +53,12 @@ func (c *Chain) Apply(entry monitor.LogEntry) bool {
}
// All filters must pass
- for _, filter := range c.filters {
+ for i, filter := range c.filters {
if !filter.Apply(entry) {
+ c.logger.Debug("msg", "Entry filtered out",
+ "component", "filter_chain",
+ "filter_index", i,
+ "filter_type", filter.config.Type)
return false
}
}
diff --git a/src/internal/filter/filter.go b/src/internal/filter/filter.go
index 5c2326e..9629f5a 100644
--- a/src/internal/filter/filter.go
+++ b/src/internal/filter/filter.go
@@ -8,6 +8,8 @@ import (
"sync/atomic"
"logwisp/src/internal/monitor"
+
+ "github.com/lixenwraith/log"
)
// Type represents the filter type
@@ -38,6 +40,7 @@ type Filter struct {
config Config
patterns []*regexp.Regexp
mu sync.RWMutex
+ logger *log.Logger
// Statistics
totalProcessed atomic.Uint64
@@ -46,7 +49,7 @@ type Filter struct {
}
// New creates a new filter from configuration
-func New(cfg Config) (*Filter, error) {
+func New(cfg Config, logger *log.Logger) (*Filter, error) {
// Set defaults
if cfg.Type == "" {
cfg.Type = TypeInclude
@@ -58,6 +61,7 @@ func New(cfg Config) (*Filter, error) {
f := &Filter{
config: cfg,
patterns: make([]*regexp.Regexp, 0, len(cfg.Patterns)),
+ logger: logger,
}
// Compile patterns
@@ -69,6 +73,12 @@ func New(cfg Config) (*Filter, error) {
f.patterns = append(f.patterns, re)
}
+ logger.Debug("msg", "Filter created",
+ "component", "filter",
+ "type", cfg.Type,
+ "logic", cfg.Logic,
+ "pattern_count", len(cfg.Patterns))
+
return f, nil
}
@@ -134,6 +144,9 @@ func (f *Filter) matches(text string) bool {
default:
// Shouldn't happen after validation
+ f.logger.Warn("msg", "Unknown filter logic",
+ "component", "filter",
+ "logic", f.config.Logic)
return false
}
}
@@ -169,5 +182,8 @@ func (f *Filter) UpdatePatterns(patterns []string) error {
f.config.Patterns = patterns
f.mu.Unlock()
+ f.logger.Info("msg", "Filter patterns updated",
+ "component", "filter",
+ "pattern_count", len(patterns))
return nil
}
\ No newline at end of file
diff --git a/src/internal/monitor/file_watcher.go b/src/internal/monitor/file_watcher.go
index 4ca13b6..4b52107 100644
--- a/src/internal/monitor/file_watcher.go
+++ b/src/internal/monitor/file_watcher.go
@@ -15,6 +15,8 @@ import (
"sync/atomic"
"syscall"
"time"
+
+ "github.com/lixenwraith/log"
)
type fileWatcher struct {
@@ -29,13 +31,15 @@ type fileWatcher struct {
rotationSeq int
entriesRead atomic.Uint64
lastReadTime atomic.Value // time.Time
+ logger *log.Logger
}
-func newFileWatcher(path string, callback func(LogEntry)) *fileWatcher {
+func newFileWatcher(path string, callback func(LogEntry), logger *log.Logger) *fileWatcher {
w := &fileWatcher{
path: path,
callback: callback,
position: -1,
+ logger: logger,
}
w.lastReadTime.Store(time.Time{})
return w
@@ -59,7 +63,7 @@ func (w *fileWatcher) watch(ctx context.Context) error {
}
if err := w.checkFile(); err != nil {
// Log error but continue watching
- fmt.Printf("[WARN] checkFile error for %s: %v\n", w.path, err)
+ w.logger.Warn("msg", "checkFile error", "error", err)
}
}
}
@@ -118,12 +122,20 @@ func (w *fileWatcher) checkFile() error {
// File doesn't exist yet, keep watching
return nil
}
+ w.logger.Error("msg", "Failed to open file for checking",
+ "component", "file_watcher",
+ "path", w.path,
+ "error", err)
return err
}
defer file.Close()
info, err := file.Stat()
if err != nil {
+ w.logger.Error("msg", "Failed to stat file",
+ "component", "file_watcher",
+ "path", w.path,
+ "error", err)
return err
}
@@ -193,6 +205,12 @@ func (w *fileWatcher) checkFile() error {
Level: "INFO",
Message: fmt.Sprintf("Log rotation detected (#%d): %s", seq, rotationReason),
})
+
+ w.logger.Info("msg", "Log rotation detected",
+ "component", "file_watcher",
+ "path", w.path,
+ "sequence", seq,
+ "reason", rotationReason)
}
// Only read if there's new content
@@ -216,11 +234,20 @@ func (w *fileWatcher) checkFile() error {
w.lastReadTime.Store(time.Now())
}
+ if err := scanner.Err(); err != nil {
+ w.logger.Error("msg", "Scanner error while reading file",
+ "component", "file_watcher",
+ "path", w.path,
+ "position", startPos,
+ "error", err)
+ return err
+ }
+
// Update position after successful read
currentPos, err := file.Seek(0, io.SeekCurrent)
if err != nil {
// Log error but don't fail - position tracking is best effort
- fmt.Printf("[WARN] Failed to get file position for %s: %v\n", w.path, err)
+ w.logger.Warn("msg", "Failed to get file position", "error", err)
// Use size as fallback position
currentPos = currentSize
}
diff --git a/src/internal/monitor/monitor.go b/src/internal/monitor/monitor.go
index 6332741..950e991 100644
--- a/src/internal/monitor/monitor.go
+++ b/src/internal/monitor/monitor.go
@@ -4,6 +4,7 @@ package monitor
import (
"context"
"encoding/json"
+ "errors"
"fmt"
"os"
"path/filepath"
@@ -11,6 +12,8 @@ import (
"sync"
"sync/atomic"
"time"
+
+ "github.com/lixenwraith/log"
)
type LogEntry struct {
@@ -63,6 +66,7 @@ type monitor struct {
droppedEntries atomic.Uint64
startTime time.Time
lastEntryTime atomic.Value // time.Time
+ logger *log.Logger
}
type target struct {
@@ -72,11 +76,12 @@ type target struct {
regex *regexp.Regexp
}
-func New() Monitor {
+func New(logger *log.Logger) Monitor {
m := &monitor{
watchers: make(map[string]*fileWatcher),
checkInterval: 100 * time.Millisecond,
startTime: time.Now(),
+ logger: logger,
}
m.lastEntryTime.Store(time.Time{})
return m
@@ -103,6 +108,7 @@ func (m *monitor) publish(entry LogEntry) {
case ch <- entry:
default:
m.droppedEntries.Add(1)
+ m.logger.Debug("msg", "Dropped log entry - subscriber buffer full")
}
}
}
@@ -111,11 +117,17 @@ func (m *monitor) SetCheckInterval(interval time.Duration) {
m.mu.Lock()
m.checkInterval = interval
m.mu.Unlock()
+
+ m.logger.Debug("msg", "Check interval updated", "interval_ms", interval.Milliseconds())
}
func (m *monitor) AddTarget(path, pattern string, isFile bool) error {
absPath, err := filepath.Abs(path)
if err != nil {
+ m.logger.Error("msg", "Failed to resolve absolute path",
+ "component", "monitor",
+ "path", path,
+ "error", err)
return fmt.Errorf("invalid path %s: %w", path, err)
}
@@ -124,6 +136,11 @@ func (m *monitor) AddTarget(path, pattern string, isFile bool) error {
regexPattern := globToRegex(pattern)
compiledRegex, err = regexp.Compile(regexPattern)
if err != nil {
+ m.logger.Error("msg", "Failed to compile pattern regex",
+ "component", "monitor",
+ "pattern", pattern,
+ "regex", regexPattern,
+ "error", err)
return fmt.Errorf("invalid pattern %s: %w", pattern, err)
}
}
@@ -137,6 +154,12 @@ func (m *monitor) AddTarget(path, pattern string, isFile bool) error {
})
m.mu.Unlock()
+ m.logger.Info("msg", "Added monitor target",
+ "component", "monitor",
+ "path", absPath,
+ "pattern", pattern,
+ "is_file", isFile)
+
return nil
}
@@ -162,6 +185,9 @@ func (m *monitor) RemoveTarget(path string) error {
if w, exists := m.watchers[absPath]; exists {
w.stop()
delete(m.watchers, absPath)
+ m.logger.Info("msg", "Monitor started",
+ "component", "monitor",
+ "check_interval_ms", m.checkInterval.Milliseconds())
}
return nil
@@ -171,6 +197,8 @@ func (m *monitor) Start(ctx context.Context) error {
m.ctx, m.cancel = context.WithCancel(ctx)
m.wg.Add(1)
go m.monitorLoop()
+
+ m.logger.Info("msg", "Monitor started", "check_interval_ms", m.checkInterval.Milliseconds())
return nil
}
@@ -188,6 +216,8 @@ func (m *monitor) Stop() {
close(ch)
}
m.mu.Unlock()
+
+ m.logger.Info("msg", "Monitor stopped")
}
func (m *monitor) GetStats() Stats {
@@ -262,7 +292,11 @@ func (m *monitor) checkTargets() {
// Directory scanning for pattern matching
files, err := m.scanDirectory(t.path, t.regex)
if err != nil {
- fmt.Printf("[DEBUG] Error scanning directory %s: %v\n", t.path, err)
+ m.logger.Warn("msg", "Failed to scan directory",
+ "component", "monitor",
+ "path", t.path,
+ "pattern", t.pattern,
+ "error", err)
continue
}
@@ -304,16 +338,26 @@ func (m *monitor) ensureWatcher(path string) {
return
}
- w := newFileWatcher(path, m.publish)
+ w := newFileWatcher(path, m.publish, m.logger)
m.watchers[path] = w
- fmt.Printf("[DEBUG] Created watcher for: %s\n", path)
+ m.logger.Debug("msg", "Created watcher", "path", path)
m.wg.Add(1)
go func() {
defer m.wg.Done()
if err := w.watch(m.ctx); err != nil {
- fmt.Printf("[ERROR] Watcher for %s failed: %v\n", path, err)
+ // Log based on error type
+ if errors.Is(err, context.Canceled) {
+ m.logger.Debug("msg", "Watcher cancelled",
+ "component", "monitor",
+ "path", path)
+ } else {
+ m.logger.Error("msg", "Watcher failed",
+ "component", "monitor",
+ "path", path,
+ "error", err)
+ }
}
m.mu.Lock()
@@ -330,6 +374,7 @@ func (m *monitor) cleanupWatchers() {
if _, err := os.Stat(path); os.IsNotExist(err) {
w.stop()
delete(m.watchers, path)
+ m.logger.Debug("msg", "Cleaned up watcher for non-existent file", "path", path)
}
}
}
\ No newline at end of file
diff --git a/src/internal/ratelimit/limiter.go b/src/internal/ratelimit/limiter.go
index 0a331ab..3c6c775 100644
--- a/src/internal/ratelimit/limiter.go
+++ b/src/internal/ratelimit/limiter.go
@@ -4,16 +4,20 @@ package ratelimit
import (
"fmt"
"net"
+ "os"
"sync"
"sync/atomic"
"time"
"logwisp/src/internal/config"
+
+ "github.com/lixenwraith/log"
)
// Manages rate limiting for a transport
type Limiter struct {
config config.RateLimitConfig
+ logger *log.Logger
// Per-IP limiters
ipLimiters map[string]*ipLimiter
@@ -53,6 +57,13 @@ func New(cfg config.RateLimitConfig) *Limiter {
ipLimiters: make(map[string]*ipLimiter),
ipConnections: make(map[string]*atomic.Int32),
lastCleanup: time.Now(),
+ logger: log.NewLogger(),
+ }
+
+ // Initialize the logger with defaults
+ if err := l.logger.InitWithDefaults(); err != nil {
+ // Fall back to stderr logging if logger init fails
+ fmt.Fprintf(os.Stderr, "ratelimit: failed to initialize logger: %v\n", err)
}
// Create global limiter if not using per-IP limiting
@@ -66,6 +77,12 @@ func New(cfg config.RateLimitConfig) *Limiter {
// Start cleanup goroutine
go l.cleanupLoop()
+ l.logger.Info("msg", "Rate limiter initialized",
+ "component", "ratelimit",
+ "requests_per_second", cfg.RequestsPerSecond,
+ "burst_size", cfg.BurstSize,
+ "limit_by", cfg.LimitBy)
+
return l
}
@@ -80,7 +97,10 @@ func (l *Limiter) CheckHTTP(remoteAddr string) (allowed bool, statusCode int, me
ip, _, err := net.SplitHostPort(remoteAddr)
if err != nil {
// If we can't parse the IP, allow the request but log
- fmt.Printf("[RATELIMIT] Failed to parse remote addr %s: %v\n", remoteAddr, err)
+ l.logger.Warn("msg", "Failed to parse remote addr",
+ "component", "ratelimit",
+ "remote_addr", remoteAddr,
+ "error", err)
return true, 0, ""
}
@@ -97,6 +117,13 @@ func (l *Limiter) CheckHTTP(remoteAddr string) (allowed bool, statusCode int, me
statusCode = 429
}
message = "Connection limit exceeded"
+
+ l.logger.Warn("msg", "Connection limit exceeded",
+ "component", "ratelimit",
+ "ip", ip,
+ "connections", counter.Load(),
+ "limit", l.config.MaxConnectionsPerIP)
+
return false, statusCode, message
}
}
@@ -113,6 +140,7 @@ func (l *Limiter) CheckHTTP(remoteAddr string) (allowed bool, statusCode int, me
if message == "" {
message = "Rate limit exceeded"
}
+ l.logger.Debug("msg", "Request rate limited", "ip", ip)
}
return allowed, statusCode, message
@@ -136,6 +164,7 @@ func (l *Limiter) CheckTCP(remoteAddr net.Addr) bool {
allowed := l.checkLimit(ip)
if !allowed {
l.blockedRequests.Add(1)
+ l.logger.Debug("msg", "TCP connection rate limited", "ip", ip)
}
return allowed
@@ -160,7 +189,10 @@ func (l *Limiter) AddConnection(remoteAddr string) {
}
l.connMu.Unlock()
- counter.Add(1)
+ newCount := counter.Add(1)
+ l.logger.Debug("msg", "Connection added",
+ "ip", ip,
+ "connections", newCount)
}
// Removes a connection for an IP
@@ -180,6 +212,10 @@ func (l *Limiter) RemoveConnection(remoteAddr string) {
if exists {
newCount := counter.Add(-1)
+ l.logger.Debug("msg", "Connection removed",
+ "ip", ip,
+ "connections", newCount)
+
if newCount <= 0 {
// Clean up if no more connections
l.connMu.Lock()
@@ -248,6 +284,10 @@ func (l *Limiter) checkLimit(ip string) bool {
}
l.ipLimiters[ip] = limiter
l.uniqueIPs.Add(1)
+
+ l.logger.Debug("msg", "Created new IP limiter",
+ "ip", ip,
+ "total_ips", l.uniqueIPs.Load())
} else {
limiter.lastSeen = time.Now()
}
@@ -268,6 +308,8 @@ func (l *Limiter) checkLimit(ip string) bool {
default:
// Unknown limit_by value, allow by default
+ l.logger.Warn("msg", "Unknown limit_by value",
+ "limit_by", l.config.LimitBy)
return true
}
}
@@ -293,11 +335,19 @@ func (l *Limiter) cleanup() {
l.ipMu.Lock()
defer l.ipMu.Unlock()
+ cleaned := 0
for ip, limiter := range l.ipLimiters {
if now.Sub(limiter.lastSeen) > staleTimeout {
delete(l.ipLimiters, ip)
+ cleaned++
}
}
+
+ if cleaned > 0 {
+ l.logger.Debug("msg", "Cleaned up stale IP limiters",
+ "cleaned", cleaned,
+ "remaining", len(l.ipLimiters))
+ }
}
// Runs periodic cleanup
diff --git a/src/internal/service/httprouter.go b/src/internal/service/httprouter.go
index 79bbb0d..709e659 100644
--- a/src/internal/service/httprouter.go
+++ b/src/internal/service/httprouter.go
@@ -8,6 +8,7 @@ import (
"sync/atomic"
"time"
+ "github.com/lixenwraith/log"
"github.com/valyala/fasthttp"
)
@@ -15,6 +16,7 @@ type HTTPRouter struct {
service *Service
servers map[int]*routerServer // port -> server
mu sync.RWMutex
+ logger *log.Logger
// Statistics
startTime time.Time
@@ -23,11 +25,12 @@ type HTTPRouter struct {
failedRequests atomic.Uint64
}
-func NewHTTPRouter(service *Service) *HTTPRouter {
+func NewHTTPRouter(service *Service, logger *log.Logger) *HTTPRouter {
return &HTTPRouter{
service: service,
servers: make(map[int]*routerServer),
startTime: time.Now(),
+ logger: logger,
}
}
@@ -47,6 +50,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
routes: make(map[string]*LogStream),
router: r,
startTime: time.Now(),
+ logger: r.logger,
}
rs.server = &fasthttp.Server{
Handler: rs.requestHandler,
@@ -59,10 +63,14 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
// Start server in background
go func() {
addr := fmt.Sprintf(":%d", port)
- fmt.Printf("[ROUTER] Starting server on port %d\n", port)
+ r.logger.Info("msg", "Starting router server",
+ "component", "http_router",
+ "port", port)
if err := rs.server.ListenAndServe(addr); err != nil {
- // Log error but don't crash
- fmt.Printf("[ROUTER] Server on port %d failed: %v\n", port, err)
+ r.logger.Error("msg", "Router server failed",
+ "component", "http_router",
+ "port", port,
+ "error", err)
}
}()
@@ -87,7 +95,11 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
}
rs.routes[pathPrefix] = stream
- fmt.Printf("[ROUTER] Registered transport '%s' at path '%s' on port %d\n", stream.Name, pathPrefix, port)
+ r.logger.Info("msg", "Registered transport route",
+ "component", "http_router",
+ "transport", stream.Name,
+ "path", pathPrefix,
+ "port", port)
return nil
}
diff --git a/src/internal/service/logstream.go b/src/internal/service/logstream.go
index d22f7c0..0c2f207 100644
--- a/src/internal/service/logstream.go
+++ b/src/internal/service/logstream.go
@@ -12,6 +12,8 @@ import (
"logwisp/src/internal/filter"
"logwisp/src/internal/monitor"
"logwisp/src/internal/transport"
+
+ "github.com/lixenwraith/log"
)
type LogStream struct {
@@ -22,6 +24,7 @@ type LogStream struct {
TCPServer *transport.TCPStreamer
HTTPServer *transport.HTTPStreamer
Stats *StreamStats
+ logger *log.Logger
ctx context.Context
cancel context.CancelFunc
@@ -38,6 +41,10 @@ type StreamStats struct {
}
func (ls *LogStream) Shutdown() {
+ ls.logger.Info("msg", "Shutting down stream",
+ "component", "logstream",
+ "stream", ls.Name)
+
// Stop servers first
var wg sync.WaitGroup
@@ -65,6 +72,10 @@ func (ls *LogStream) Shutdown() {
// Stop monitor
ls.Monitor.Stop()
+
+ ls.logger.Info("msg", "Stream shutdown complete",
+ "component", "logstream",
+ "stream", ls.Name)
}
func (ls *LogStream) GetStats() map[string]any {
@@ -112,6 +123,11 @@ func (ls *LogStream) UpdateTargets(targets []config.MonitorTarget) error {
// Basic validation
absPath, err := filepath.Abs(target.Path)
if err != nil {
+ ls.logger.Error("msg", "Invalid target path",
+ "component", "logstream",
+ "stream", ls.Name,
+ "path", target.Path,
+ "error", err)
return fmt.Errorf("invalid target path %s: %w", target.Path, err)
}
target.Path = absPath
@@ -124,6 +140,12 @@ func (ls *LogStream) UpdateTargets(targets []config.MonitorTarget) error {
// Add new targets
for _, target := range validatedTargets {
if err := ls.Monitor.AddTarget(target.Path, target.Pattern, target.IsFile); err != nil {
+ ls.logger.Error("msg", "Failed to add monitor target - rolling back",
+ "component", "logstream",
+ "stream", ls.Name,
+ "target", target.Path,
+ "pattern", target.Pattern,
+ "error", err)
// Rollback: restore old watchers
for _, watcher := range oldWatchers {
// Best effort restoration
@@ -138,6 +160,12 @@ func (ls *LogStream) UpdateTargets(targets []config.MonitorTarget) error {
ls.Monitor.RemoveTarget(watcher.Path)
}
+ ls.logger.Info("msg", "Updated monitor targets",
+ "component", "logstream",
+ "stream", ls.Name,
+ "old_count", len(oldWatchers),
+ "new_count", len(validatedTargets))
+
return nil
}
@@ -157,8 +185,11 @@ func (ls *LogStream) startStatsUpdater(ctx context.Context) {
ls.Stats.TCPConnections = ls.TCPServer.GetActiveConnections()
if oldTCP != ls.Stats.TCPConnections {
// This debug should now show changes
- fmt.Printf("[STATS DEBUG] %s TCP: %d -> %d\n",
- ls.Name, oldTCP, ls.Stats.TCPConnections)
+ ls.logger.Debug("msg", "TCP connection count changed",
+ "component", "logstream",
+ "stream", ls.Name,
+ "old", oldTCP,
+ "new", ls.Stats.TCPConnections)
}
}
if ls.HTTPServer != nil {
@@ -166,8 +197,11 @@ func (ls *LogStream) startStatsUpdater(ctx context.Context) {
ls.Stats.HTTPConnections = ls.HTTPServer.GetActiveConnections()
if oldHTTP != ls.Stats.HTTPConnections {
// This debug should now show changes
- fmt.Printf("[STATS DEBUG] %s HTTP: %d -> %d\n",
- ls.Name, oldHTTP, ls.Stats.HTTPConnections)
+ ls.logger.Debug("msg", "HTTP connection count changed",
+ "component", "logstream",
+ "stream", ls.Name,
+ "old", oldHTTP,
+ "new", ls.Stats.HTTPConnections)
}
}
}
diff --git a/src/internal/service/routerserver.go b/src/internal/service/routerserver.go
index c49d6e2..58dabcd 100644
--- a/src/internal/service/routerserver.go
+++ b/src/internal/service/routerserver.go
@@ -9,13 +9,16 @@ import (
"sync/atomic"
"time"
- "github.com/valyala/fasthttp"
"logwisp/src/internal/version"
+
+ "github.com/lixenwraith/log"
+ "github.com/valyala/fasthttp"
)
type routerServer struct {
port int
server *fasthttp.Server
+ logger *log.Logger
routes map[string]*LogStream // path prefix -> transport
routeMu sync.RWMutex
router *HTTPRouter
@@ -28,9 +31,14 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
rs.router.totalRequests.Add(1)
path := string(ctx.Path())
+ remoteAddr := ctx.RemoteAddr().String()
// Log request for debugging
- fmt.Printf("[ROUTER] Request: %s %s from %s\n", ctx.Method(), path, ctx.RemoteAddr())
+ rs.logger.Debug("msg", "Router request",
+ "component", "router_server",
+ "method", ctx.Method(),
+ "path", path,
+ "remote_addr", remoteAddr)
// Special case: global status at /status
if path == "/status" {
@@ -79,8 +87,11 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
remainingPath = matchedStream.Config.HTTPServer.StreamPath
}
- fmt.Printf("[ROUTER] Routing to transport '%s': %s -> %s\n",
- matchedStream.Name, originalPath, remainingPath)
+ rs.logger.Debug("msg", "Routing request to transport",
+ "component", "router_server",
+ "transport", matchedStream.Name,
+ "original_path", originalPath,
+ "remaining_path", remainingPath)
ctx.URI().SetPath(remainingPath)
matchedStream.HTTPServer.RouteRequest(ctx)
diff --git a/src/internal/service/service.go b/src/internal/service/service.go
index ac2c3ef..82fc75e 100644
--- a/src/internal/service/service.go
+++ b/src/internal/service/service.go
@@ -11,6 +11,8 @@ import (
"logwisp/src/internal/filter"
"logwisp/src/internal/monitor"
"logwisp/src/internal/transport"
+
+ "github.com/lixenwraith/log"
)
type Service struct {
@@ -19,14 +21,16 @@ type Service struct {
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
+ logger *log.Logger
}
-func New(ctx context.Context) *Service {
+func New(ctx context.Context, logger *log.Logger) *Service {
serviceCtx, cancel := context.WithCancel(ctx)
return &Service{
streams: make(map[string]*LogStream),
ctx: serviceCtx,
cancel: cancel,
+ logger: logger,
}
}
@@ -35,14 +39,21 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
defer s.mu.Unlock()
if _, exists := s.streams[cfg.Name]; exists {
- return fmt.Errorf("transport '%s' already exists", cfg.Name)
+ err := fmt.Errorf("transport '%s' already exists", cfg.Name)
+ s.logger.Error("msg", "Failed to create stream - duplicate name",
+ "component", "service",
+ "stream", cfg.Name,
+ "error", err)
+ return err
}
+ s.logger.Debug("msg", "Creating stream", "stream", cfg.Name)
+
// Create transport context
streamCtx, streamCancel := context.WithCancel(s.ctx)
- // Create monitor
- mon := monitor.New()
+ // Create monitor - pass the service logger directly
+ mon := monitor.New(s.logger)
mon.SetCheckInterval(time.Duration(cfg.GetCheckInterval(100)) * time.Millisecond)
// Add targets
@@ -56,15 +67,24 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
// Start monitor
if err := mon.Start(streamCtx); err != nil {
streamCancel()
+ s.logger.Error("msg", "Failed to start monitor",
+ "component", "service",
+ "stream", cfg.Name,
+ "error", err)
return fmt.Errorf("failed to start monitor: %w", err)
}
// Create filter chain
var filterChain *filter.Chain
if len(cfg.Filters) > 0 {
- chain, err := filter.NewChain(cfg.Filters)
+ chain, err := filter.NewChain(cfg.Filters, s.logger)
if err != nil {
streamCancel()
+ s.logger.Error("msg", "Failed to create filter chain",
+ "component", "service",
+ "stream", cfg.Name,
+ "filter_count", len(cfg.Filters),
+ "error", err)
return fmt.Errorf("failed to create filter chain: %w", err)
}
filterChain = chain
@@ -81,6 +101,7 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
},
ctx: streamCtx,
cancel: streamCancel,
+ logger: s.logger, // Use parent logger
}
// Start TCP server if configured
@@ -97,10 +118,18 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
s.filterLoop(streamCtx, rawChan, tcpChan, filterChain)
}()
- ls.TCPServer = transport.NewTCPStreamer(tcpChan, *cfg.TCPServer)
+ ls.TCPServer = transport.NewTCPStreamer(
+ tcpChan,
+ *cfg.TCPServer,
+ s.logger) // Pass parent logger
if err := s.startTCPServer(ls); err != nil {
ls.Shutdown()
+ s.logger.Error("msg", "Failed to start TCP server",
+ "component", "service",
+ "stream", cfg.Name,
+ "port", cfg.TCPServer.Port,
+ "error", err)
return fmt.Errorf("TCP server failed: %w", err)
}
}
@@ -119,10 +148,18 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
s.filterLoop(streamCtx, rawChan, httpChan, filterChain)
}()
- ls.HTTPServer = transport.NewHTTPStreamer(httpChan, *cfg.HTTPServer)
+ ls.HTTPServer = transport.NewHTTPStreamer(
+ httpChan,
+ *cfg.HTTPServer,
+ s.logger) // Pass parent logger
if err := s.startHTTPServer(ls); err != nil {
ls.Shutdown()
+ s.logger.Error("msg", "Failed to start HTTP server",
+ "component", "service",
+ "stream", cfg.Name,
+ "port", cfg.HTTPServer.Port,
+ "error", err)
return fmt.Errorf("HTTP server failed: %w", err)
}
}
@@ -130,6 +167,7 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
ls.startStatsUpdater(streamCtx)
s.streams[cfg.Name] = ls
+ s.logger.Info("msg", "Stream created successfully", "stream", cfg.Name)
return nil
}
@@ -152,6 +190,7 @@ func (s *Service) filterLoop(ctx context.Context, in <-chan monitor.LogEntry, ou
return
default:
// Drop if output buffer is full
+ s.logger.Debug("msg", "Dropped log entry - buffer full")
}
}
}
@@ -186,15 +225,23 @@ func (s *Service) RemoveStream(name string) error {
stream, exists := s.streams[name]
if !exists {
- return fmt.Errorf("transport '%s' not found", name)
+ err := fmt.Errorf("transport '%s' not found", name)
+ s.logger.Warn("msg", "Cannot remove non-existent stream",
+ "component", "service",
+ "stream", name,
+ "error", err)
+ return err
}
+ s.logger.Info("msg", "Removing stream", "stream", name)
stream.Shutdown()
delete(s.streams, name)
return nil
}
func (s *Service) Shutdown() {
+ s.logger.Info("msg", "Service shutdown initiated")
+
s.mu.Lock()
streams := make([]*LogStream, 0, len(s.streams))
for _, stream := range s.streams {
@@ -215,6 +262,8 @@ func (s *Service) Shutdown() {
s.cancel()
s.wg.Wait()
+
+ s.logger.Info("msg", "Service shutdown complete")
}
func (s *Service) GetGlobalStats() map[string]any {
@@ -247,8 +296,13 @@ func (s *Service) startTCPServer(ls *LogStream) error {
// Check startup
select {
case err := <-errChan:
+ s.logger.Error("msg", "TCP server startup failed immediately",
+ "component", "service",
+ "stream", ls.Name,
+ "error", err)
return err
case <-time.After(time.Second):
+ s.logger.Debug("msg", "TCP server started", "stream", ls.Name)
return nil
}
}
@@ -267,8 +321,13 @@ func (s *Service) startHTTPServer(ls *LogStream) error {
// Check startup
select {
case err := <-errChan:
+ s.logger.Error("msg", "HTTP server startup failed immediately",
+ "component", "service",
+ "stream", ls.Name,
+ "error", err)
return err
case <-time.After(time.Second):
+ s.logger.Debug("msg", "HTTP server started", "stream", ls.Name)
return nil
}
}
\ No newline at end of file
diff --git a/src/internal/transport/httpstreamer.go b/src/internal/transport/httpstreamer.go
index 7e4e47f..3984ae5 100644
--- a/src/internal/transport/httpstreamer.go
+++ b/src/internal/transport/httpstreamer.go
@@ -11,11 +11,14 @@ import (
"sync/atomic"
"time"
- "github.com/valyala/fasthttp"
"logwisp/src/internal/config"
"logwisp/src/internal/monitor"
"logwisp/src/internal/ratelimit"
"logwisp/src/internal/version"
+
+ "github.com/lixenwraith/log"
+ "github.com/lixenwraith/log/compat"
+ "github.com/valyala/fasthttp"
)
type HTTPStreamer struct {
@@ -27,6 +30,7 @@ type HTTPStreamer struct {
startTime time.Time
done chan struct{}
wg sync.WaitGroup
+ logger *log.Logger
// Path configuration
streamPath string
@@ -39,7 +43,7 @@ type HTTPStreamer struct {
rateLimiter *ratelimit.Limiter
}
-func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTPStreamer {
+func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig, logger *log.Logger) *HTTPStreamer {
// Set default paths if not configured
streamPath := cfg.StreamPath
if streamPath == "" {
@@ -58,6 +62,7 @@ func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTP
streamPath: streamPath,
statusPath: statusPath,
standalone: true, // Default to standalone mode
+ logger: logger,
}
// Initialize rate limiter if configured
@@ -71,19 +76,26 @@ func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTP
// Configures the streamer for use with a router
func (h *HTTPStreamer) SetRouterMode() {
h.standalone = false
+ h.logger.Debug("msg", "HTTP streamer set to router mode",
+ "component", "http_streamer")
}
func (h *HTTPStreamer) Start() error {
if !h.standalone {
// In router mode, don't start our own server
+ h.logger.Debug("msg", "HTTP streamer in router mode, skipping server start",
+ "component", "http_streamer")
return nil
}
+ // Create fasthttp adapter for logging
+ fasthttpLogger := compat.NewFastHTTPAdapter(h.logger)
+
h.server = &fasthttp.Server{
Handler: h.requestHandler,
DisableKeepalive: false,
StreamRequestBody: true,
- Logger: nil,
+ Logger: fasthttpLogger,
}
addr := fmt.Sprintf(":%d", h.config.Port)
@@ -91,6 +103,11 @@ func (h *HTTPStreamer) Start() error {
// Run server in separate goroutine to avoid blocking
errChan := make(chan error, 1)
go func() {
+ h.logger.Info("msg", "HTTP server started",
+ "component", "http_streamer",
+ "port", h.config.Port,
+ "stream_path", h.streamPath,
+ "status_path", h.statusPath)
err := h.server.ListenAndServe(addr)
if err != nil {
errChan <- err
@@ -103,11 +120,17 @@ func (h *HTTPStreamer) Start() error {
return err
case <-time.After(100 * time.Millisecond):
// Server started successfully
+ h.logger.Info("msg", "HTTP server started",
+ "port", h.config.Port,
+ "stream_path", h.streamPath,
+ "status_path", h.statusPath)
return nil
}
}
func (h *HTTPStreamer) Stop() {
+ h.logger.Info("msg", "Stopping HTTP server")
+
// Signal all client handlers to stop
close(h.done)
@@ -120,6 +143,8 @@ func (h *HTTPStreamer) Stop() {
// Wait for all active client handlers to finish
h.wg.Wait()
+
+ h.logger.Info("msg", "HTTP server stopped")
}
func (h *HTTPStreamer) RouteRequest(ctx *fasthttp.RequestCtx) {
@@ -193,6 +218,9 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
return
default:
// Drop if client buffer full
+ h.logger.Debug("msg", "Dropped entry for slow client",
+ "component", "http_streamer",
+ "remote_addr", remoteAddr)
}
case <-clientDone:
return
@@ -205,14 +233,16 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
// Define the transport writer function
streamFunc := func(w *bufio.Writer) {
newCount := h.activeClients.Add(1)
- fmt.Printf("[HTTP DEBUG] Client connected on port %d. Count now: %d\n",
- h.config.Port, newCount)
+ h.logger.Debug("msg", "HTTP client connected",
+ "remote_addr", remoteAddr,
+ "active_clients", newCount)
h.wg.Add(1)
defer func() {
newCount := h.activeClients.Add(-1)
- fmt.Printf("[HTTP DEBUG] Client disconnected on port %d. Count now: %d\n",
- h.config.Port, newCount)
+ h.logger.Debug("msg", "HTTP client disconnected",
+ "remote_addr", remoteAddr,
+ "active_clients", newCount)
h.wg.Done()
}()
@@ -246,6 +276,10 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
data, err := json.Marshal(entry)
if err != nil {
+ h.logger.Error("msg", "Failed to marshal log entry",
+ "component", "http_streamer",
+ "error", err,
+ "entry_source", entry.Source)
continue
}
diff --git a/src/internal/transport/tcpserver.go b/src/internal/transport/tcpserver.go
index a88ee91..652b8f5 100644
--- a/src/internal/transport/tcpserver.go
+++ b/src/internal/transport/tcpserver.go
@@ -26,8 +26,8 @@ func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
}
func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
- // Debug: Log all connection attempts
- fmt.Printf("[TCP DEBUG] Connection attempt from %s\n", c.RemoteAddr())
+ remoteAddr := c.RemoteAddr().String()
+ s.streamer.logger.Debug("msg", "TCP connection attempt", "remote_addr", remoteAddr)
// Check rate limit
if s.streamer.rateLimiter != nil {
@@ -35,12 +35,15 @@ func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
remoteStr := c.RemoteAddr().String()
tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr)
if err != nil {
- fmt.Printf("[TCP DEBUG] Failed to parse address %s: %v\n", remoteStr, err)
+ s.streamer.logger.Warn("msg", "Failed to parse TCP address",
+ "remote_addr", remoteAddr,
+ "error", err)
return nil, gnet.Close
}
if !s.streamer.rateLimiter.CheckTCP(tcpAddr) {
- fmt.Printf("[TCP DEBUG] Rate limited connection from %s\n", remoteStr)
+ s.streamer.logger.Warn("msg", "TCP connection rate limited",
+ "remote_addr", remoteAddr)
// Silently close connection when rate limited
return nil, gnet.Close
}
@@ -51,27 +54,29 @@ func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
s.connections.Store(c, struct{}{})
- oldCount := s.streamer.activeConns.Load()
newCount := s.streamer.activeConns.Add(1)
- fmt.Printf("[TCP ATOMIC] OnOpen: %d -> %d (expected: %d)\n", oldCount, newCount, oldCount+1)
+ s.streamer.logger.Debug("msg", "TCP connection opened",
+ "remote_addr", remoteAddr,
+ "active_connections", newCount)
- fmt.Printf("[TCP DEBUG] Connection opened. Count now: %d\n", newCount)
return nil, gnet.None
}
func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
s.connections.Delete(c)
+ remoteAddr := c.RemoteAddr().String()
+
// Remove connection tracking
if s.streamer.rateLimiter != nil {
s.streamer.rateLimiter.RemoveConnection(c.RemoteAddr().String())
}
- oldCount := s.streamer.activeConns.Load()
newCount := s.streamer.activeConns.Add(-1)
- fmt.Printf("[TCP ATOMIC] OnClose: %d -> %d (expected: %d)\n", oldCount, newCount, oldCount-1)
-
- fmt.Printf("[TCP DEBUG] Connection closed. Count now: %d (err: %v)\n", newCount, err)
+ s.streamer.logger.Debug("msg", "TCP connection closed",
+ "remote_addr", remoteAddr,
+ "active_connections", newCount,
+ "error", err)
return gnet.None
}
@@ -79,8 +84,4 @@ func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
// We don't expect input from clients, just discard
c.Discard(-1)
return gnet.None
-}
-
-func (t *TCPStreamer) GetActiveConnections() int32 {
- return t.activeConns.Load()
}
\ No newline at end of file
diff --git a/src/internal/transport/tcpstreamer.go b/src/internal/transport/tcpstreamer.go
index c7c9e7a..736d3fd 100644
--- a/src/internal/transport/tcpstreamer.go
+++ b/src/internal/transport/tcpstreamer.go
@@ -9,10 +9,12 @@ import (
"sync/atomic"
"time"
- "github.com/panjf2000/gnet/v2"
"logwisp/src/internal/config"
"logwisp/src/internal/monitor"
"logwisp/src/internal/ratelimit"
+
+ "github.com/lixenwraith/log"
+ "github.com/panjf2000/gnet/v2"
)
type TCPStreamer struct {
@@ -26,14 +28,16 @@ type TCPStreamer struct {
engineMu sync.Mutex
wg sync.WaitGroup
rateLimiter *ratelimit.Limiter
+ logger *log.Logger
}
-func NewTCPStreamer(logChan chan monitor.LogEntry, cfg config.TCPConfig) *TCPStreamer {
+func NewTCPStreamer(logChan chan monitor.LogEntry, cfg config.TCPConfig, logger *log.Logger) *TCPStreamer {
t := &TCPStreamer{
logChan: logChan,
config: cfg,
done: make(chan struct{}),
startTime: time.Now(),
+ logger: logger,
}
if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
@@ -59,11 +63,21 @@ func (t *TCPStreamer) Start() error {
// Run gnet in separate goroutine to avoid blocking
errChan := make(chan error, 1)
go func() {
+ t.logger.Info("msg", "Starting TCP server",
+ "component", "tcp_streamer",
+ "port", t.config.Port)
+
err := gnet.Run(t.server, addr,
gnet.WithLogger(noopLogger{}),
gnet.WithMulticore(true),
gnet.WithReusePort(true),
)
+ if err != nil {
+ t.logger.Error("msg", "TCP server failed",
+ "component", "tcp_streamer",
+ "port", t.config.Port,
+ "error", err)
+ }
errChan <- err
}()
@@ -76,11 +90,13 @@ func (t *TCPStreamer) Start() error {
return err
case <-time.After(100 * time.Millisecond):
// Server started successfully
+ t.logger.Info("msg", "TCP server started", "port", t.config.Port)
return nil
}
}
func (t *TCPStreamer) Stop() {
+ t.logger.Info("msg", "Stopping TCP server")
// Signal broadcast loop to stop
close(t.done)
@@ -97,6 +113,8 @@ func (t *TCPStreamer) Stop() {
// Wait for broadcast loop to finish
t.wg.Wait()
+
+ t.logger.Info("msg", "TCP server stopped")
}
func (t *TCPStreamer) broadcastLoop() {
@@ -117,6 +135,10 @@ func (t *TCPStreamer) broadcastLoop() {
}
data, err := json.Marshal(entry)
if err != nil {
+ t.logger.Error("msg", "Failed to marshal log entry",
+ "component", "tcp_streamer",
+ "error", err,
+ "entry_source", entry.Source)
continue
}
data = append(data, '\n')
@@ -162,4 +184,8 @@ func (t *TCPStreamer) formatHeartbeat() []byte {
// For TCP, always use JSON format
jsonData, _ := json.Marshal(data)
return append(jsonData, '\n')
+}
+
+func (t *TCPStreamer) GetActiveConnections() int32 {
+ return t.activeConns.Load()
}
\ No newline at end of file
+ 
+ 
+