From d7f2c0d54dde381633d10dc37dd76c308bd967cd4d8e56987f37b874c81f52f2 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Tue, 8 Jul 2025 00:57:21 -0400 Subject: [PATCH] v0.1.8 rate limiter added, improved http router, config templates added, docs updated --- README.md | 135 ++++++++++- config/logwisp.toml | 181 -------------- config/logwisp.toml.defaults | 312 +++++++++++++++++++++++++ config/logwisp.toml.example | 120 ++++++++++ config/logwisp.toml.minimal | 25 ++ doc/architecture.md | 209 ++++++++++++++--- src/internal/config/server.go | 6 +- src/internal/config/validation.go | 37 +++ src/internal/logstream/httprouter.go | 76 +++++- src/internal/logstream/routerserver.go | 72 ++++-- src/internal/ratelimit/limiter.go | 311 ++++++++++++++++++++++++ src/internal/ratelimit/ratelimit.go | 53 +++++ src/internal/stream/httpstreamer.go | 47 +++- src/internal/stream/tcpserver.go | 30 +++ src/internal/stream/tcpstreamer.go | 10 +- 15 files changed, 1373 insertions(+), 251 deletions(-) delete mode 100644 config/logwisp.toml create mode 100644 config/logwisp.toml.defaults create mode 100644 config/logwisp.toml.example create mode 100644 config/logwisp.toml.minimal create mode 100644 src/internal/ratelimit/limiter.go create mode 100644 src/internal/ratelimit/ratelimit.go diff --git a/README.md b/README.md index 98ec688..9116ab8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ LogWisp Logo

-A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring and rotation detection. +A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring, rotation detection, and rate limiting. ## Features @@ -13,8 +13,10 @@ A high-performance log streaming service with multi-stream architecture, support - **Real-time Monitoring**: Instant updates with per-stream configurable check intervals - **File Rotation Detection**: Automatic detection and handling of log rotation - **Path-based Routing**: Optional HTTP router for consolidated access -- **Per-Stream Configuration**: Independent settings including check intervals for each log stream -- **Connection Statistics**: Real-time monitoring of active connections +- **Rate Limiting**: Per-IP or global rate limiting with token bucket algorithm +- **Connection Limiting**: Configurable concurrent connection limits per IP +- **Per-Stream Configuration**: Independent settings including check intervals and rate limits +- **Connection Statistics**: Real-time monitoring of active connections and rate limit metrics - **Flexible Targets**: Monitor individual files or entire directories - **Version Management**: Git tag-based versioning with build information - **Configurable Heartbeats**: Keep connections alive with customizable formats @@ -47,10 +49,12 @@ LogWisp uses a service-oriented architecture where each stream is an independent LogStream Service ├── Stream["app-logs"] │ ├── Monitor (watches files) +│ ├── Rate Limiter (optional) │ ├── TCP Server (optional) │ └── HTTP Server (optional) ├── Stream["system-logs"] │ ├── Monitor +│ ├── Rate Limiter (optional) │ └── HTTP Server └── HTTP Router (optional, for path-based routing) ``` @@ -89,6 +93,16 @@ format = "comment" # or "json" for structured events include_timestamp = true include_stats = false +# Rate limiting configuration +[streams.httpserver.rate_limit] +enabled = true +requests_per_second = 10.0 +burst_size = 20 +limit_by = "ip" +response_code = 429 +response_message = "Rate limit exceeded" +max_connections_per_ip = 5 + # System logs stream with slower check interval [[streams]] name = "system" @@ -112,6 +126,13 @@ enabled = true interval_seconds = 300 # 5 minutes include_timestamp = true include_stats = true + +# TCP rate limiting +[streams.tcpserver.rate_limit] +enabled = true +requests_per_second = 5.0 +burst_size = 10 +limit_by = "ip" ``` ### Target Configuration @@ -137,6 +158,22 @@ Each stream can have its own check interval based on log update frequency: - **Normal logs**: 100-1000ms (e.g., application logs) - **Low-frequency logs**: 10000-60000ms (e.g., system logs, archives) +### Rate Limiting Configuration + +Control request rates and connection limits per stream: + +```toml +[streams.httpserver.rate_limit] +enabled = true # Enable/disable rate limiting +requests_per_second = 10.0 # Token refill rate +burst_size = 20 # Maximum burst capacity +limit_by = "ip" # "ip" or "global" +response_code = 429 # HTTP response code when limited +response_message = "Too many requests" +max_connections_per_ip = 5 # Max concurrent connections per IP +max_total_connections = 100 # Max total connections (global) +``` + ### Heartbeat Configuration Keep connections alive and detect stale clients with configurable heartbeats: @@ -198,7 +235,7 @@ All HTTP streams share ports with path-based routing: # Connect to a stream curl -N http://localhost:8080/stream -# Check stream status +# Check stream status (includes rate limit stats) curl http://localhost:8080/status # With authentication (when implemented) @@ -237,6 +274,13 @@ eventSource.addEventListener('heartbeat', (e) => { const heartbeat = JSON.parse(e.data); console.log('Heartbeat:', heartbeat); }); + +eventSource.addEventListener('error', (e) => { + if (e.status === 429) { + console.error('Rate limited - backing off'); + // Implement exponential backoff + } +}); ``` ## Log Entry Format @@ -284,6 +328,20 @@ All log entries are streamed as JSON: "active_watchers": 3, "total_entries": 15420, "dropped_entries": 0 + }, + "features": { + "rate_limit": { + "enabled": true, + "total_requests": 45678, + "blocked_requests": 234, + "active_ips": 23, + "total_connections": 5, + "config": { + "requests_per_second": 10, + "burst_size": 20, + "limit_by": "ip" + } + } } } ``` @@ -294,6 +352,7 @@ LogWisp provides comprehensive statistics at multiple levels: - **Per-Stream Stats**: Monitor performance, connection counts, data throughput - **Per-Watcher Stats**: File size, position, entries read, rotation count +- **Rate Limit Stats**: Total requests, blocked requests, active IPs - **Global Stats**: Aggregated view of all streams (in router mode) Access statistics via status endpoints or watch the console output: @@ -306,6 +365,21 @@ Access statistics via status endpoints or watch the console output: ## Advanced Features +### Rate Limiting + +LogWisp implements token bucket rate limiting with: +- **Per-IP limiting**: Each IP gets its own token bucket +- **Global limiting**: All clients share a single token bucket +- **Connection limits**: Restrict concurrent connections per IP +- **Automatic cleanup**: Stale IP entries removed after 5 minutes +- **Non-blocking**: Excess requests are immediately rejected with 429 status + +Monitor rate limiting effectiveness: +```bash +# Watch rate limit statistics +watch -n 1 'curl -s http://localhost:8080/status | jq .features.rate_limit' +``` + ### File Rotation Detection LogWisp automatically detects log rotation through multiple methods: @@ -349,6 +423,11 @@ check_interval_ms = 60000 # Check every minute - Configure per-stream based on expected update frequency - Use 10000ms+ for archival or slowly updating logs +### Rate Limiting +- `requests_per_second`: Balance between protection and availability +- `burst_size`: Set to 2-3x the per-second rate for traffic spikes +- `max_connections_per_ip`: Prevent resource exhaustion from single IPs + ### File Watcher Optimization - Use specific file paths when possible (more efficient than directory scanning) - Adjust patterns to minimize unnecessary file checks @@ -378,6 +457,12 @@ make build # Run tests make test +# Test rate limiting +./test_ratelimit.sh + +# Test router functionality +./test_router.sh + # Create a release make release TAG=v1.0.0 ``` @@ -413,6 +498,9 @@ ProtectSystem=strict ProtectHome=true ReadOnlyPaths=/var/log +# Rate limiting at system level +LimitNOFILE=65536 + [Install] WantedBy=multi-user.target ``` @@ -448,31 +536,52 @@ services: - "9090:9090" restart: unless-stopped command: ["logwisp", "--config", "/etc/logwisp/config.toml"] + deploy: + resources: + limits: + cpus: '1.0' + memory: 512M ``` ## Security Considerations ### Current Implementation - Read-only file access +- Rate limiting for DDoS protection +- Connection limits to prevent resource exhaustion - No authentication (placeholder configuration only) - No TLS/SSL support (placeholder configuration only) ### Planned Security Features - **Authentication**: Basic, Bearer/JWT, mTLS - **TLS/SSL**: For both HTTP and TCP streams -- **Rate Limiting**: Per-client request limits - **IP Filtering**: Whitelist/blacklist support - **Audit Logging**: Access and authentication events +- **RBAC**: Role-based access control per stream ### Best Practices 1. Run with minimal privileges (read-only access to log files) -2. Use network-level security until authentication is implemented -3. Place behind a reverse proxy for production HTTPS -4. Monitor access logs for unusual patterns -5. Regularly update dependencies +2. Configure appropriate rate limits based on expected traffic +3. Use network-level security until authentication is implemented +4. Place behind a reverse proxy for production HTTPS +5. Monitor rate limit statistics for potential attacks +6. Regularly update dependencies + +### Rate Limiting Best Practices +- Start with conservative limits and adjust based on monitoring +- Use per-IP limiting for public endpoints +- Use global limiting for resource protection +- Set connection limits to prevent memory exhaustion +- Monitor blocked request statistics for anomalies ## Troubleshooting +### Rate Limit Issues +1. Check rate limit statistics in status endpoint +2. Verify appropriate `requests_per_second` for your use case +3. Ensure `burst_size` accommodates normal traffic spikes +4. Monitor for distributed attacks if per-IP limiting isn't effective + ### No Log Entries Appearing 1. Check file permissions (LogWisp needs read access) 2. Verify file paths in configuration @@ -483,14 +592,16 @@ services: ### High Memory Usage 1. Reduce buffer sizes in configuration 2. Lower the number of concurrent watchers -3. Increase check interval for less critical logs -4. Use TCP instead of HTTP for high-volume streams +3. Enable rate limiting to prevent connection floods +4. Increase check interval for less critical logs +5. Use TCP instead of HTTP for high-volume streams ### Connection Drops 1. Check heartbeat configuration 2. Verify network stability 3. Monitor client-side errors 4. Review dropped entry statistics +5. Check if rate limits are too restrictive ### Version Information Use `./logwisp --version` to see: @@ -515,7 +626,7 @@ Contributions are welcome! Please read our contributing guidelines and submit pu - [x] Per-stream check intervals - [x] Version management - [x] Configurable heartbeats -- [ ] Rate and connection limiting +- [x] Rate and connection limiting - [ ] Log filtering and transformation - [ ] Configurable logging support - [ ] Authentication (Basic, JWT, mTLS) diff --git a/config/logwisp.toml b/config/logwisp.toml deleted file mode 100644 index 0e03e54..0000000 --- a/config/logwisp.toml +++ /dev/null @@ -1,181 +0,0 @@ -# LogWisp Multi-Stream Configuration -# Location: ~/.config/logwisp.toml - -# Stream 1: Application logs (public access) -[[streams]] -name = "app" - -[streams.monitor] -# Check interval in milliseconds (per-stream configuration) -check_interval_ms = 100 -# Array of folders and files to be monitored -# For file targets, pattern is ignored and can be omitted -targets = [ - { path = "/var/log/myapp", pattern = "*.log", is_file = false }, - { path = "/var/log/myapp/app.log", pattern = "", is_file = true } -] - -[streams.httpserver] -enabled = true -port = 8080 -buffer_size = 2000 -stream_path = "/stream" -status_path = "/status" - -# HTTP SSE Heartbeat Configuration -[streams.httpserver.heartbeat] -enabled = true -interval_seconds = 30 -# Format options: "comment" (SSE comments) or "json" (JSON events) -format = "comment" -# Include timestamp in heartbeat -include_timestamp = true -# Include server statistics (client count, uptime) -include_stats = false - -# Stream 2: System logs (authenticated) -[[streams]] -name = "system" - -[streams.monitor] -# More frequent checks for critical system logs -check_interval_ms = 50 -targets = [ - { path = "/var/log", pattern = "syslog*", is_file = false }, - { path = "/var/log/auth.log", pattern = "", is_file = true } -] - -[streams.httpserver] -enabled = true -port = 8443 -buffer_size = 5000 -stream_path = "/logs" -status_path = "/health" - -# JSON format heartbeat with full stats -[streams.httpserver.heartbeat] -enabled = true -interval_seconds = 20 -format = "json" -include_timestamp = true -include_stats = true - -# SSL placeholder -[streams.httpserver.ssl] -enabled = true -cert_file = "/etc/logwisp/certs/server.crt" -key_file = "/etc/logwisp/certs/server.key" -min_version = "TLS1.2" - -# Authentication placeholder -[streams.auth] -type = "basic" - -[streams.auth.basic_auth] -realm = "System Logs" -users = [ - { username = "admin", password_hash = "$2y$10$..." } -] -ip_whitelist = ["10.0.0.0/8", "192.168.0.0/16"] - -# TCP server also available -[streams.tcpserver] -enabled = true -port = 9443 -buffer_size = 5000 - -# TCP heartbeat (always JSON format) -[streams.tcpserver.heartbeat] -enabled = true -interval_seconds = 60 -include_timestamp = true -include_stats = true - -# Stream 3: Debug logs (high-volume, less frequent checks) -[[streams]] -name = "debug" - -[streams.monitor] -# Check every 10 seconds for debug logs -check_interval_ms = 10000 -targets = [ - { path = "./debug", pattern = "*.debug", is_file = false } -] - -[streams.httpserver] -enabled = true -port = 8082 -buffer_size = 10000 -stream_path = "/stream" -status_path = "/status" - -# Disable heartbeat for high-volume stream -[streams.httpserver.heartbeat] -enabled = false - -# Rate limiting placeholder -[streams.httpserver.rate_limit] -enabled = true -requests_per_second = 100.0 -burst_size = 1000 -limit_by = "ip" - -# Stream 4: Slow logs (infrequent updates) -[[streams]] -name = "archive" - -[streams.monitor] -# Check once per minute for archival logs -check_interval_ms = 60000 -targets = [ - { path = "/var/log/archive", pattern = "*.log.gz", is_file = false } -] - -[streams.tcpserver] -enabled = true -port = 9091 -buffer_size = 1000 - -# Minimal heartbeat for connection keep-alive -[streams.tcpserver.heartbeat] -enabled = true -interval_seconds = 300 # 5 minutes -include_timestamp = false -include_stats = false - -# Heartbeat Format Examples: -# -# Comment format (SSE): -# : heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s -# -# JSON format (SSE): -# event: heartbeat -# data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600} -# -# TCP always uses JSON format with newline delimiter - -# Usage Examples: -# -# 1. Standard mode (each stream on its own port): -# ./logwisp -# - App logs: http://localhost:8080/stream -# - System logs: https://localhost:8443/logs (with auth) -# - Debug logs: http://localhost:8082/stream -# - Archive logs: tcp://localhost:9091 -# -# 2. Router mode (shared port with path routing): -# ./logwisp --router -# - App logs: http://localhost:8080/app/stream -# - System logs: http://localhost:8080/system/logs -# - Debug logs: http://localhost:8080/debug/stream -# - Global status: http://localhost:8080/status -# -# 3. Override config file: -# ./logwisp --config /etc/logwisp/production.toml -# -# 4. Environment variables: -# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50 -# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090 -# -# 5. Show version: -# ./logwisp --version diff --git a/config/logwisp.toml.defaults b/config/logwisp.toml.defaults new file mode 100644 index 0000000..ead4531 --- /dev/null +++ b/config/logwisp.toml.defaults @@ -0,0 +1,312 @@ +# LogWisp Configuration File +# Default path: ~/.config/logwisp.toml +# Override with: ./logwisp --config /path/to/config.toml + +# This is a complete configuration reference showing all available options. +# Default values are uncommented, alternatives and examples are commented. + +# ============================================================================== +# STREAM CONFIGURATION +# ============================================================================== +# Each [[streams]] section defines an independent log monitoring stream. +# You can have multiple streams, each with its own settings. + +# ------------------------------------------------------------------------------ +# Default Stream - Monitors current directory +# ------------------------------------------------------------------------------ +[[streams]] +# Stream identifier used in logs, metrics, and router paths +name = "default" + +# File monitoring configuration +[streams.monitor] +# How often to check for new log entries (milliseconds) +# Lower = faster detection but more CPU usage +check_interval_ms = 100 + +# Targets to monitor - can be files or directories +targets = [ + # Monitor all .log files in current directory + { path = "./", pattern = "*.log", is_file = false }, +] + +# HTTP Server configuration (SSE/Server-Sent Events) +[streams.httpserver] +enabled = true +port = 8080 +buffer_size = 1000 # Per-client buffer size (messages) +stream_path = "/stream" # Endpoint for SSE stream +status_path = "/status" # Endpoint for statistics + +# Keep-alive heartbeat configuration +[streams.httpserver.heartbeat] +enabled = true +interval_seconds = 30 # Send heartbeat every 30 seconds +format = "comment" # SSE comment format (: heartbeat) +include_timestamp = true # Include timestamp in heartbeat +include_stats = false # Include connection stats + +# Rate limiting configuration (disabled by default) +[streams.httpserver.rate_limit] +enabled = false +# requests_per_second = 10.0 # Token refill rate +# burst_size = 20 # Max burst capacity +# limit_by = "ip" # "ip" or "global" +# response_code = 429 # HTTP Too Many Requests +# response_message = "Rate limit exceeded" +# max_connections_per_ip = 5 # Max SSE connections per IP + +# ------------------------------------------------------------------------------ +# Example: Application Logs Stream +# ------------------------------------------------------------------------------ +# [[streams]] +# name = "app" +# +# [streams.monitor] +# check_interval_ms = 50 # Fast detection for active logs +# targets = [ +# # Monitor specific application log directory +# { path = "/var/log/myapp", pattern = "*.log", is_file = false }, +# # Also monitor specific file +# { path = "/var/log/myapp/app.log", is_file = true }, +# ] +# +# [streams.httpserver] +# enabled = true +# port = 8081 # Different port for each stream +# buffer_size = 2000 # Larger buffer for busy logs +# stream_path = "/logs" # Custom path +# status_path = "/health" # Custom health endpoint +# +# # JSON heartbeat format for programmatic clients +# [streams.httpserver.heartbeat] +# enabled = true +# interval_seconds = 20 +# format = "json" # JSON event format +# include_timestamp = true +# include_stats = true # Include active client count +# +# # Moderate rate limiting for public access +# [streams.httpserver.rate_limit] +# enabled = true +# requests_per_second = 25.0 +# burst_size = 50 +# limit_by = "ip" +# max_connections_per_ip = 10 + +# ------------------------------------------------------------------------------ +# Example: System Logs Stream (TCP + HTTP) +# ------------------------------------------------------------------------------ +# [[streams]] +# name = "system" +# +# [streams.monitor] +# check_interval_ms = 1000 # Check every second (system logs update slowly) +# targets = [ +# { path = "/var/log/syslog", is_file = true }, +# { path = "/var/log/auth.log", is_file = true }, +# { path = "/var/log/kern.log", is_file = true }, +# ] +# +# # TCP Server for high-performance streaming +# [streams.tcpserver] +# enabled = true +# port = 9090 +# buffer_size = 5000 +# +# # TCP heartbeat (always JSON format) +# [streams.tcpserver.heartbeat] +# enabled = true +# interval_seconds = 60 # Less frequent for TCP +# include_timestamp = true +# include_stats = false +# +# # TCP rate limiting +# [streams.tcpserver.rate_limit] +# enabled = true +# requests_per_second = 5.0 # Limit TCP connections +# burst_size = 10 +# limit_by = "ip" +# +# # Also expose via HTTP +# [streams.httpserver] +# enabled = true +# port = 8082 +# buffer_size = 1000 +# stream_path = "/stream" +# status_path = "/status" + +# ------------------------------------------------------------------------------ +# Example: High-Volume Debug Logs +# ------------------------------------------------------------------------------ +# [[streams]] +# name = "debug" +# +# [streams.monitor] +# check_interval_ms = 5000 # Check every 5 seconds (high volume) +# targets = [ +# { path = "/tmp/debug", pattern = "*.debug", is_file = false }, +# ] +# +# [streams.httpserver] +# enabled = true +# port = 8083 +# buffer_size = 10000 # Very large buffer +# stream_path = "/debug" +# status_path = "/stats" +# +# # Disable heartbeat for high-volume streams +# [streams.httpserver.heartbeat] +# enabled = false +# +# # Aggressive rate limiting +# [streams.httpserver.rate_limit] +# enabled = true +# requests_per_second = 1.0 # Very restrictive +# burst_size = 5 +# limit_by = "ip" +# max_connections_per_ip = 1 # One connection per IP + +# ------------------------------------------------------------------------------ +# Example: Archived Logs (Slow Monitoring) +# ------------------------------------------------------------------------------ +# [[streams]] +# name = "archive" +# +# [streams.monitor] +# check_interval_ms = 60000 # Check once per minute +# targets = [ +# { path = "/var/log/archive", pattern = "*.gz", is_file = false }, +# ] +# +# [streams.tcpserver] +# enabled = true +# port = 9091 +# buffer_size = 500 # Small buffer for archived logs +# +# # Infrequent heartbeat +# [streams.tcpserver.heartbeat] +# enabled = true +# interval_seconds = 300 # Every 5 minutes +# include_timestamp = false +# include_stats = false + +# ------------------------------------------------------------------------------ +# Example: Security/Audit Logs with Strict Limits +# ------------------------------------------------------------------------------ +# [[streams]] +# name = "security" +# +# [streams.monitor] +# check_interval_ms = 100 +# targets = [ +# { path = "/var/log/audit", pattern = "audit.log*", is_file = false }, +# ] +# +# [streams.httpserver] +# enabled = true +# port = 8443 # HTTPS port (for future TLS) +# buffer_size = 1000 +# stream_path = "/audit" +# status_path = "/health" +# +# # Strict rate limiting for security logs +# [streams.httpserver.rate_limit] +# enabled = true +# requests_per_second = 2.0 # Very limited access +# burst_size = 3 +# limit_by = "ip" +# max_connections_per_ip = 1 # Single connection per IP +# response_code = 403 # Forbidden instead of rate limit +# response_message = "Access restricted" +# +# # Future: SSL/TLS configuration +# # [streams.httpserver.ssl] +# # enabled = true +# # cert_file = "/etc/logwisp/certs/server.crt" +# # key_file = "/etc/logwisp/certs/server.key" +# # min_version = "TLS1.2" +# +# # Future: Authentication +# # [streams.auth] +# # type = "basic" +# # [streams.auth.basic_auth] +# # users_file = "/etc/logwisp/security.users" +# # realm = "Security Logs" + +# ------------------------------------------------------------------------------ +# Example: Public API Logs with Global Rate Limiting +# ------------------------------------------------------------------------------ +# [[streams]] +# name = "api-public" +# +# [streams.monitor] +# check_interval_ms = 100 +# targets = [ +# { path = "/var/log/api", pattern = "access.log*", is_file = false }, +# ] +# +# [streams.httpserver] +# enabled = true +# port = 8084 +# buffer_size = 2000 +# +# # Global rate limiting (all clients share limit) +# [streams.httpserver.rate_limit] +# enabled = true +# requests_per_second = 100.0 # 100 req/s total +# burst_size = 200 +# limit_by = "global" # All clients share this limit +# max_total_connections = 50 # Max 50 connections total + +# ============================================================================== +# USAGE EXAMPLES +# ============================================================================== + +# 1. Basic usage (single stream): +# ./logwisp +# - Monitors current directory for *.log files +# - Access logs at: http://localhost:8080/stream +# - View stats at: http://localhost:8080/status + +# 2. Multi-stream configuration: +# - Uncomment additional [[streams]] sections above +# - Each stream runs independently on its own port +# - Different check intervals for different log types + +# 3. Router mode (consolidated access): +# ./logwisp --router +# - All streams accessible via paths: /streamname/stream +# - Global status at: /status +# - Example: http://localhost:8080/app/stream + +# 4. Production deployment: +# - Enable rate limiting on public-facing streams +# - Use TCP for internal high-volume streams +# - Set appropriate check intervals (higher = less CPU) +# - Configure heartbeats for long-lived connections + +# 5. Monitoring: +# curl http://localhost:8080/status | jq . +# - Check active connections +# - Monitor rate limit statistics +# - Track log entry counts + +# ============================================================================== +# ENVIRONMENT VARIABLES +# ============================================================================== +# Configuration can be overridden via environment variables: +# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50 +# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090 +# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true + +# ============================================================================== +# NOTES +# ============================================================================== +# - Rate limiting is disabled by default for backward compatibility +# - Each stream can have different rate limit settings +# - TCP connections are silently dropped when rate limited +# - HTTP returns 429 (or configured code) with JSON error +# - IP tracking is cleaned up after 5 minutes of inactivity +# - Token bucket algorithm provides smooth rate limiting +# - Connection limits prevent resource exhaustion \ No newline at end of file diff --git a/config/logwisp.toml.example b/config/logwisp.toml.example new file mode 100644 index 0000000..f084878 --- /dev/null +++ b/config/logwisp.toml.example @@ -0,0 +1,120 @@ +# LogWisp Configuration Example +# Default path: ~/.config/logwisp.toml + +# Application logs - public facing +[[streams]] +name = "app-public" + +[streams.monitor] +check_interval_ms = 100 +targets = [ + { path = "/var/log/nginx", pattern = "access.log*", is_file = false }, + { path = "/var/log/app", pattern = "production.log", is_file = true } +] + +[streams.httpserver] +enabled = true +port = 8080 +buffer_size = 2000 +stream_path = "/logs" +status_path = "/health" + +[streams.httpserver.heartbeat] +enabled = true +interval_seconds = 30 +format = "json" +include_timestamp = true +include_stats = true + +# Rate limiting for public endpoint +[streams.httpserver.rate_limit] +enabled = true +requests_per_second = 50.0 +burst_size = 100 +limit_by = "ip" +response_code = 429 +response_message = "Rate limit exceeded. Please retry after 60 seconds." +max_connections_per_ip = 5 +max_total_connections = 100 + +# System logs - internal only +[[streams]] +name = "system" + +[streams.monitor] +check_interval_ms = 5000 # Check every 5 seconds +targets = [ + { path = "/var/log/syslog", is_file = true }, + { path = "/var/log/auth.log", is_file = true }, + { path = "/var/log/kern.log", is_file = true } +] + +# TCP for internal consumers +[streams.tcpserver] +enabled = true +port = 9090 +buffer_size = 5000 + +[streams.tcpserver.heartbeat] +enabled = true +interval_seconds = 60 +include_timestamp = true + +# Moderate rate limiting for internal use +[streams.tcpserver.rate_limit] +enabled = true +requests_per_second = 10.0 +burst_size = 20 +limit_by = "ip" + +# Security audit logs - restricted access +[[streams]] +name = "security" + +[streams.monitor] +check_interval_ms = 100 +targets = [ + { path = "/var/log/audit", pattern = "*.log", is_file = false }, + { path = "/var/log/fail2ban.log", is_file = true } +] + +[streams.httpserver] +enabled = true +port = 8443 +buffer_size = 1000 +stream_path = "/audit/stream" +status_path = "/audit/status" + +# Strict rate limiting +[streams.httpserver.rate_limit] +enabled = true +requests_per_second = 1.0 +burst_size = 3 +limit_by = "ip" +max_connections_per_ip = 1 +response_code = 403 +response_message = "Access denied" + +# Application debug logs - development team only +[[streams]] +name = "debug" + +[streams.monitor] +check_interval_ms = 1000 +targets = [ + { path = "/var/log/app", pattern = "debug-*.log", is_file = false } +] + +[streams.httpserver] +enabled = true +port = 8090 +buffer_size = 5000 +stream_path = "/debug" +status_path = "/debug/status" + +[streams.httpserver.rate_limit] +enabled = true +requests_per_second = 100.0 # Higher limit for internal use +burst_size = 200 +limit_by = "ip" +max_connections_per_ip = 10 \ No newline at end of file diff --git a/config/logwisp.toml.minimal b/config/logwisp.toml.minimal new file mode 100644 index 0000000..7534034 --- /dev/null +++ b/config/logwisp.toml.minimal @@ -0,0 +1,25 @@ +# LogWisp Minimal Configuration Example +# Save as: ~/.config/logwisp.toml + +# Monitor application logs +[[streams]] +name = "app" + +[streams.monitor] +check_interval_ms = 100 +targets = [ + { path = "/var/log/myapp", pattern = "*.log", is_file = false } +] + +[streams.httpserver] +enabled = true +port = 8080 +stream_path = "/stream" +status_path = "/status" + +# Optional: Enable rate limiting +# [streams.httpserver.rate_limit] +# enabled = true +# requests_per_second = 10.0 +# burst_size = 20 +# limit_by = "ip" \ No newline at end of file diff --git a/doc/architecture.md b/doc/architecture.md index 969e276..6c96ee1 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -9,9 +9,13 @@ logwisp/ ├── go.sum # Go module checksums ├── README.md # Project documentation ├── config/ -│ └── logwisp.toml # Example configuration template +│ ├── logwisp.toml.defaults # Default configuration and guide +│ ├── logwisp.toml.example # Example configuration +│ └── logwisp.toml.minimal # Minimal configuration template ├── doc/ │ └── architecture.md # This file - architecture documentation +├── test_router.sh # Router functionality test suite +├── test_ratelimit.sh # Rate limiting test suite └── src/ ├── cmd/ │ └── logwisp/ @@ -21,10 +25,10 @@ logwisp/ │ ├── auth.go # Authentication configuration structures │ ├── config.go # Main configuration structures │ ├── loader.go # Configuration loading with lixenwraith/config - │ ├── server.go # TCP/HTTP server configurations + │ ├── server.go # TCP/HTTP server configurations with rate limiting │ ├── ssl.go # SSL/TLS configuration structures │ ├── stream.go # Stream-specific configurations - │ └── validation.go # Configuration validation logic + │ └── validation.go # Configuration validation including rate limits ├── logstream/ │ ├── httprouter.go # HTTP router for path-based routing │ ├── logstream.go # Stream lifecycle management @@ -33,10 +37,13 @@ logwisp/ ├── monitor/ │ ├── file_watcher.go # File watching and rotation detection │ └── monitor.go # Log monitoring interface and implementation + ├── ratelimit/ + │ ├── ratelimit.go # Token bucket algorithm implementation + │ └── limiter.go # Per-stream rate limiter with IP tracking ├── stream/ - │ ├── httpstreamer.go # HTTP/SSE streaming server + │ ├── httpstreamer.go # HTTP/SSE streaming with rate limiting │ ├── noop_logger.go # Silent logger for gnet - │ ├── tcpserver.go # TCP server using gnet + │ ├── tcpserver.go # TCP server with rate limiting (gnet) │ └── tcpstreamer.go # TCP streaming implementation └── version/ └── version.go # Version information management @@ -85,6 +92,12 @@ LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=2000 LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=json +# Rate limiting configuration +LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true +LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0 +LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_BURST_SIZE=20 +LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_LIMIT_BY=ip + # Multiple streams LOGWISP_STREAMS_1_NAME=system LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000 @@ -96,40 +109,66 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090 ### Core Components 1. **Service (`logstream.Service`)** - - Manages multiple log streams - - Handles lifecycle (creation, shutdown) - - Provides global statistics - - Thread-safe stream registry + - Manages multiple log streams + - Handles lifecycle (creation, shutdown) + - Provides global statistics + - Thread-safe stream registry 2. **LogStream (`logstream.LogStream`)** - - Represents a single log monitoring pipeline - - Contains: Monitor + Servers (TCP/HTTP) - - Independent configuration - - Per-stream statistics + - Represents a single log monitoring pipeline + - Contains: Monitor + Rate Limiter + Servers (TCP/HTTP) + - Independent configuration + - Per-stream statistics with rate limit metrics 3. **Monitor (`monitor.Monitor`)** - - Watches files and directories - - Detects log rotation - - Publishes log entries to subscribers - - Configurable check intervals + - Watches files and directories + - Detects log rotation + - Publishes log entries to subscribers + - Configurable check intervals -4. **Streamers** - - **HTTPStreamer**: SSE-based streaming over HTTP - - **TCPStreamer**: Raw JSON streaming over TCP - - Both support configurable heartbeats - - Non-blocking client management +4. **Rate Limiter (`ratelimit.Limiter`)** + - Token bucket algorithm for smooth rate limiting + - Per-IP or global limiting strategies + - Connection tracking and limits + - Automatic cleanup of stale entries + - Non-blocking rejection of excess requests -5. **HTTPRouter (`logstream.HTTPRouter`)** - - Optional component for path-based routing - - Consolidates multiple HTTP streams on shared ports - - Provides global status endpoint +5. **Streamers** + - **HTTPStreamer**: SSE-based streaming over HTTP + - Rate limit enforcement before request handling + - Connection tracking for per-IP limits + - Configurable 429 responses + - **TCPStreamer**: Raw JSON streaming over TCP + - Silent connection drops when rate limited + - Per-IP connection tracking + - Both support configurable heartbeats + - Non-blocking client management + +6. **HTTPRouter (`logstream.HTTPRouter`)** + - Optional component for path-based routing + - Consolidates multiple HTTP streams on shared ports + - Provides global status endpoint + - Longest-prefix path matching + - Dynamic stream registration/deregistration ### Data Flow ``` -File System → Monitor → LogEntry Channel → Streamer → Network Client - ↑ ↓ - └── Rotation Detection +File System → Monitor → LogEntry Channel → [Rate Limiter] → Streamer → Network Client + ↑ ↓ ↓ + └── Rotation Detection Rate Limit Check + ↓ + Accept/Reject +``` + +### Rate Limiting Architecture + +``` +Client Request → Rate Limiter → Token Bucket Check → Allow/Deny + ↓ ↓ + IP Tracking Refill Rate + ↓ + Cleanup Timer ``` ### Configuration Structure @@ -159,6 +198,16 @@ format = "comment" # or "json" include_timestamp = true include_stats = false +[streams.httpserver.rate_limit] +enabled = false # Disabled by default +requests_per_second = 10.0 # Token refill rate +burst_size = 20 # Token bucket capacity +limit_by = "ip" # "ip" or "global" +response_code = 429 # HTTP response code +response_message = "Rate limit exceeded" +max_connections_per_ip = 5 # Concurrent connection limit +max_total_connections = 100 # Global connection limit + [streams.tcpserver] enabled = true port = 9090 @@ -169,8 +218,36 @@ enabled = true interval_seconds = 60 include_timestamp = true include_stats = true + +[streams.tcpserver.rate_limit] +enabled = false +requests_per_second = 5.0 +burst_size = 10 +limit_by = "ip" ``` +## Rate Limiting Implementation + +### Token Bucket Algorithm +- Each IP (or global limiter) gets a bucket with configurable capacity +- Tokens refill at `requests_per_second` rate +- Each request/connection consumes one token +- Smooth rate limiting without hard cutoffs + +### Limiting Strategies +1. **Per-IP**: Each client IP gets its own token bucket +2. **Global**: All clients share a single token bucket + +### Connection Limits +- Per-IP connection limits prevent single client resource exhaustion +- Global connection limits protect overall system resources +- Checked before rate limits to prevent connection hanging + +### Cleanup +- IP entries older than 5 minutes are automatically removed +- Prevents unbounded memory growth +- Runs every minute in background + ## Build System ### Makefile Targets @@ -208,5 +285,75 @@ go build -ldflags "-X 'logwisp/src/internal/version.Version=v1.0.0'" \ ### 2. Router Mode (`--router`) - HTTP streams share ports via path-based routing - Consolidated access through URL paths -- Global status endpoint -- Best for multi-stream setups with limited ports \ No newline at end of file +- Global status endpoint with aggregated statistics +- Best for multi-stream setups with limited ports +- Streams accessible at `/{stream_name}/{path}` + +## Testing + +### Test Suites + +1. **Router Testing** (`test_router.sh`) + - Path routing verification + - Client isolation between streams + - Statistics aggregation + - Graceful shutdown + - Port conflict handling + +2. **Rate Limiting Testing** (`test_ratelimit.sh`) + - Per-IP rate limiting + - Global rate limiting + - Connection limits + - Rate limit recovery + - Statistics accuracy + - Stress testing + +### Running Tests + +```bash +# Test router functionality +./test_router.sh + +# Test rate limiting +./test_ratelimit.sh + +# Run all tests +make test +``` + +## Performance Considerations + +### Rate Limiting Overhead +- Token bucket checks: O(1) time complexity +- Memory: ~100 bytes per tracked IP +- Cleanup: Runs asynchronously every minute +- Minimal impact when disabled + +### Optimization Guidelines +- Use per-IP limiting for fairness +- Use global limiting for resource protection +- Set burst size to 2-3x requests_per_second +- Monitor rate limit statistics for tuning +- Higher check_interval_ms for low-activity logs + +## Security Architecture + +### Current Security Features +- Read-only file access +- Rate limiting for DDoS protection +- Connection limits for resource protection +- Non-blocking request rejection + +### Future Security Roadmap +- Authentication (Basic, JWT, mTLS) +- TLS/SSL support +- IP whitelisting/blacklisting +- Audit logging +- RBAC per stream + +### Security Best Practices +- Run with minimal privileges +- Enable rate limiting on public endpoints +- Use connection limits to prevent exhaustion +- Deploy behind reverse proxy for HTTPS +- Monitor rate limit statistics for attacks \ No newline at end of file diff --git a/src/internal/config/server.go b/src/internal/config/server.go index 1d0b435..e4868b4 100644 --- a/src/internal/config/server.go +++ b/src/internal/config/server.go @@ -53,10 +53,14 @@ type RateLimitConfig struct { // Burst size (token bucket) BurstSize int `toml:"burst_size"` - // Rate limit by: "ip", "user", "token" + // Rate limit by: "ip", "user", "token", "global" LimitBy string `toml:"limit_by"` // Response when rate limited ResponseCode int `toml:"response_code"` // Default: 429 ResponseMessage string `toml:"response_message"` // Default: "Rate limit exceeded" + + // Connection limits + MaxConnectionsPerIP int `toml:"max_connections_per_ip"` + MaxTotalConnections int `toml:"max_total_connections"` } \ No newline at end of file diff --git a/src/internal/config/validation.go b/src/internal/config/validation.go index 5c07a4e..112ad2f 100644 --- a/src/internal/config/validation.go +++ b/src/internal/config/validation.go @@ -68,6 +68,10 @@ func (c *Config) validate() error { if err := validateSSL("TCP", stream.Name, stream.TCPServer.SSL); err != nil { return err } + + if err := validateRateLimit("TCP", stream.Name, stream.TCPServer.RateLimit); err != nil { + return err + } } // Validate HTTP server @@ -109,6 +113,10 @@ func (c *Config) validate() error { if err := validateSSL("HTTP", stream.Name, stream.HTTPServer.SSL); err != nil { return err } + + if err := validateRateLimit("HTTP", stream.Name, stream.HTTPServer.RateLimit); err != nil { + return err + } } // At least one server must be enabled @@ -185,5 +193,34 @@ func validateAuth(streamName string, auth *AuthConfig) error { return fmt.Errorf("stream '%s': bearer auth type specified but config missing", streamName) } + return nil +} + +func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error { + if rl == nil || !rl.Enabled { + return nil + } + + if rl.RequestsPerSecond <= 0 { + return fmt.Errorf("stream '%s' %s: requests_per_second must be positive: %f", + streamName, serverType, rl.RequestsPerSecond) + } + + if rl.BurstSize < 1 { + return fmt.Errorf("stream '%s' %s: burst_size must be at least 1: %d", + streamName, serverType, rl.BurstSize) + } + + validLimitBy := map[string]bool{"ip": true, "global": true, "": true} + if !validLimitBy[rl.LimitBy] { + return fmt.Errorf("stream '%s' %s: invalid limit_by value: %s (must be 'ip' or 'global')", + streamName, serverType, rl.LimitBy) + } + + if rl.ResponseCode > 0 && (rl.ResponseCode < 400 || rl.ResponseCode >= 600) { + return fmt.Errorf("stream '%s' %s: response_code must be 4xx or 5xx: %d", + streamName, serverType, rl.ResponseCode) + } + return nil } \ No newline at end of file diff --git a/src/internal/logstream/httprouter.go b/src/internal/logstream/httprouter.go index 3b02324..997b55e 100644 --- a/src/internal/logstream/httprouter.go +++ b/src/internal/logstream/httprouter.go @@ -5,6 +5,8 @@ import ( "fmt" "strings" "sync" + "sync/atomic" + "time" "github.com/valyala/fasthttp" ) @@ -13,12 +15,19 @@ type HTTPRouter struct { service *Service servers map[int]*routerServer // port -> server mu sync.RWMutex + + // Statistics + startTime time.Time + totalRequests atomic.Uint64 + routedRequests atomic.Uint64 + failedRequests atomic.Uint64 } func NewHTTPRouter(service *Service) *HTTPRouter { return &HTTPRouter{ - service: service, - servers: make(map[int]*routerServer), + service: service, + servers: make(map[int]*routerServer), + startTime: time.Now(), } } @@ -34,24 +43,31 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error { if !exists { // Create new server for this port rs = &routerServer{ - port: port, - routes: make(map[string]*LogStream), + port: port, + routes: make(map[string]*LogStream), + router: r, + startTime: time.Now(), } rs.server = &fasthttp.Server{ Handler: rs.requestHandler, DisableKeepalive: false, StreamRequestBody: true, + CloseOnShutdown: true, // Ensure connections close on shutdown } r.servers[port] = rs // Start server in background go func() { addr := fmt.Sprintf(":%d", port) + fmt.Printf("[ROUTER] Starting server on port %d\n", port) if err := rs.server.ListenAndServe(addr); err != nil { // Log error but don't crash - fmt.Printf("Router server on port %d failed: %v\n", port, err) + fmt.Printf("[ROUTER] Server on port %d failed: %v\n", port, err) } }() + + // Wait briefly to ensure server starts + time.Sleep(100 * time.Millisecond) } r.mu.Unlock() @@ -71,6 +87,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error { } rs.routes[pathPrefix] = stream + fmt.Printf("[ROUTER] Registered stream '%s' at path '%s' on port %d\n", stream.Name, pathPrefix, port) return nil } @@ -78,18 +95,27 @@ func (r *HTTPRouter) UnregisterStream(streamName string) { r.mu.RLock() defer r.mu.RUnlock() - for _, rs := range r.servers { + for port, rs := range r.servers { rs.routeMu.Lock() for path, stream := range rs.routes { if stream.Name == streamName { delete(rs.routes, path) + fmt.Printf("[ROUTER] Unregistered stream '%s' from path '%s' on port %d\n", + streamName, path, port) } } + + // Check if server has no more routes + if len(rs.routes) == 0 { + fmt.Printf("[ROUTER] No routes left on port %d, considering shutdown\n", port) + } rs.routeMu.Unlock() } } func (r *HTTPRouter) Shutdown() { + fmt.Println("[ROUTER] Starting router shutdown...") + r.mu.Lock() defer r.mu.Unlock() @@ -98,10 +124,46 @@ func (r *HTTPRouter) Shutdown() { wg.Add(1) go func(p int, s *routerServer) { defer wg.Done() + fmt.Printf("[ROUTER] Shutting down server on port %d\n", p) if err := s.server.Shutdown(); err != nil { - fmt.Printf("Error shutting down router server on port %d: %v\n", p, err) + fmt.Printf("[ROUTER] Error shutting down server on port %d: %v\n", p, err) } }(port, rs) } wg.Wait() + + fmt.Println("[ROUTER] Router shutdown complete") +} + +func (r *HTTPRouter) GetStats() map[string]interface{} { + r.mu.RLock() + defer r.mu.RUnlock() + + serverStats := make(map[int]interface{}) + totalRoutes := 0 + + for port, rs := range r.servers { + rs.routeMu.RLock() + routes := make([]string, 0, len(rs.routes)) + for path := range rs.routes { + routes = append(routes, path) + totalRoutes++ + } + rs.routeMu.RUnlock() + + serverStats[port] = map[string]interface{}{ + "routes": routes, + "requests": rs.requests.Load(), + "uptime": int(time.Since(rs.startTime).Seconds()), + } + } + + return map[string]interface{}{ + "uptime_seconds": int(time.Since(r.startTime).Seconds()), + "total_requests": r.totalRequests.Load(), + "routed_requests": r.routedRequests.Load(), + "failed_requests": r.failedRequests.Load(), + "servers": serverStats, + "total_routes": totalRoutes, + } } \ No newline at end of file diff --git a/src/internal/logstream/routerserver.go b/src/internal/logstream/routerserver.go index f2a4dc8..fef4b6f 100644 --- a/src/internal/logstream/routerserver.go +++ b/src/internal/logstream/routerserver.go @@ -6,21 +6,32 @@ import ( "fmt" "strings" "sync" + "sync/atomic" + "time" "github.com/valyala/fasthttp" "logwisp/src/internal/version" ) type routerServer struct { - port int - server *fasthttp.Server - routes map[string]*LogStream // path prefix -> stream - routeMu sync.RWMutex + port int + server *fasthttp.Server + routes map[string]*LogStream // path prefix -> stream + routeMu sync.RWMutex + router *HTTPRouter + startTime time.Time + requests atomic.Uint64 } func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) { + rs.requests.Add(1) + rs.router.totalRequests.Add(1) + path := string(ctx.Path()) + // Log request for debugging + fmt.Printf("[ROUTER] Request: %s %s from %s\n", ctx.Method(), path, ctx.RemoteAddr()) + // Special case: global status at /status if path == "/status" { rs.handleGlobalStatus(ctx) @@ -40,26 +51,48 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) { matchedPrefix = prefix matchedStream = stream remainingPath = strings.TrimPrefix(path, prefix) + // Ensure remaining path starts with / or is empty + if remainingPath != "" && !strings.HasPrefix(remainingPath, "/") { + remainingPath = "/" + remainingPath + } } } } rs.routeMu.RUnlock() if matchedStream == nil { + rs.router.failedRequests.Add(1) rs.handleNotFound(ctx) return } + rs.router.routedRequests.Add(1) + // Route to stream's handler if matchedStream.HTTPServer != nil { + // Save original path + originalPath := string(ctx.URI().Path()) + // Rewrite path to remove stream prefix + if remainingPath == "" { + // Default to stream path if no remaining path + remainingPath = matchedStream.Config.HTTPServer.StreamPath + } + + fmt.Printf("[ROUTER] Routing to stream '%s': %s -> %s\n", + matchedStream.Name, originalPath, remainingPath) + ctx.URI().SetPath(remainingPath) matchedStream.HTTPServer.RouteRequest(ctx) + + // Restore original path + ctx.URI().SetPath(originalPath) } else { ctx.SetStatusCode(fasthttp.StatusServiceUnavailable) ctx.SetContentType("application/json") json.NewEncoder(ctx).Encode(map[string]string{ - "error": "Stream HTTP server not available", + "error": "Stream HTTP server not available", + "stream": matchedStream.Name, }) } } @@ -70,26 +103,37 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) { rs.routeMu.RLock() streams := make(map[string]interface{}) for prefix, stream := range rs.routes { - streams[stream.Name] = map[string]interface{}{ + streamStats := stream.GetStats() + + // Add routing information + streamStats["routing"] = map[string]interface{}{ "path_prefix": prefix, - "config": map[string]interface{}{ - "stream_path": stream.Config.HTTPServer.StreamPath, - "status_path": stream.Config.HTTPServer.StatusPath, + "endpoints": map[string]string{ + "stream": prefix + stream.Config.HTTPServer.StreamPath, + "status": prefix + stream.Config.HTTPServer.StatusPath, }, - "stats": stream.GetStats(), } + + streams[stream.Name] = streamStats } rs.routeMu.RUnlock() + // Get router stats + routerStats := rs.router.GetStats() + status := map[string]interface{}{ "service": "LogWisp Router", - "version": version.Short(), + "version": version.String(), "port": rs.port, "streams": streams, "total_streams": len(streams), + "router": routerStats, + "endpoints": map[string]string{ + "global_status": "/status", + }, } - data, _ := json.Marshal(status) + data, _ := json.MarshalIndent(status, "", " ") ctx.SetBody(data) } @@ -113,9 +157,11 @@ func (rs *routerServer) handleNotFound(ctx *fasthttp.RequestCtx) { response := map[string]interface{}{ "error": "Not Found", + "requested_path": string(ctx.Path()), "available_routes": availableRoutes, + "hint": "Use /status for global router status", } - data, _ := json.Marshal(response) + data, _ := json.MarshalIndent(response, "", " ") ctx.SetBody(data) } \ No newline at end of file diff --git a/src/internal/ratelimit/limiter.go b/src/internal/ratelimit/limiter.go new file mode 100644 index 0000000..28972e4 --- /dev/null +++ b/src/internal/ratelimit/limiter.go @@ -0,0 +1,311 @@ +// FILE: src/internal/ratelimit/limiter.go +package ratelimit + +import ( + "fmt" + "net" + "sync" + "sync/atomic" + "time" + + "logwisp/src/internal/config" +) + +// Manages rate limiting for a stream +type Limiter struct { + config config.RateLimitConfig + + // Per-IP limiters + ipLimiters map[string]*ipLimiter + ipMu sync.RWMutex + + // Global limiter for the stream + globalLimiter *TokenBucket + + // Connection tracking + ipConnections map[string]*atomic.Int32 + connMu sync.RWMutex + + // Statistics + totalRequests atomic.Uint64 + blockedRequests atomic.Uint64 + uniqueIPs atomic.Uint64 + + // Cleanup + lastCleanup time.Time + cleanupMu sync.Mutex +} + +type ipLimiter struct { + bucket *TokenBucket + lastSeen time.Time + connections atomic.Int32 +} + +// Creates a new rate limiter +func New(cfg config.RateLimitConfig) *Limiter { + if !cfg.Enabled { + return nil + } + + l := &Limiter{ + config: cfg, + ipLimiters: make(map[string]*ipLimiter), + ipConnections: make(map[string]*atomic.Int32), + lastCleanup: time.Now(), + } + + // Create global limiter if not using per-IP limiting + if cfg.LimitBy == "global" { + l.globalLimiter = NewTokenBucket( + float64(cfg.BurstSize), + cfg.RequestsPerSecond, + ) + } + + // Start cleanup goroutine + go l.cleanupLoop() + + return l +} + +// Checks if an HTTP request should be allowed +func (l *Limiter) CheckHTTP(remoteAddr string) (allowed bool, statusCode int, message string) { + if l == nil { + return true, 0, "" + } + + l.totalRequests.Add(1) + + ip, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + // If we can't parse the IP, allow the request but log + fmt.Printf("[RATELIMIT] Failed to parse remote addr %s: %v\n", remoteAddr, err) + return true, 0, "" + } + + // Check connection limit for streaming endpoint + if l.config.MaxConnectionsPerIP > 0 { + l.connMu.RLock() + counter, exists := l.ipConnections[ip] + l.connMu.RUnlock() + + if exists && counter.Load() >= int32(l.config.MaxConnectionsPerIP) { + l.blockedRequests.Add(1) + statusCode = l.config.ResponseCode + if statusCode == 0 { + statusCode = 429 + } + message = "Connection limit exceeded" + return false, statusCode, message + } + } + + // Check rate limit + allowed = l.checkLimit(ip) + if !allowed { + l.blockedRequests.Add(1) + statusCode = l.config.ResponseCode + if statusCode == 0 { + statusCode = 429 + } + message = l.config.ResponseMessage + if message == "" { + message = "Rate limit exceeded" + } + } + + return allowed, statusCode, message +} + +// Checks if a TCP connection should be allowed +func (l *Limiter) CheckTCP(remoteAddr net.Addr) bool { + if l == nil { + return true + } + + l.totalRequests.Add(1) + + // Extract IP from TCP addr + tcpAddr, ok := remoteAddr.(*net.TCPAddr) + if !ok { + return true + } + + ip := tcpAddr.IP.String() + allowed := l.checkLimit(ip) + if !allowed { + l.blockedRequests.Add(1) + } + + return allowed +} + +// Tracks a new connection for an IP +func (l *Limiter) AddConnection(remoteAddr string) { + if l == nil { + return + } + + ip, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return + } + + l.connMu.Lock() + counter, exists := l.ipConnections[ip] + if !exists { + counter = &atomic.Int32{} + l.ipConnections[ip] = counter + } + l.connMu.Unlock() + + counter.Add(1) +} + +// Removes a connection for an IP +func (l *Limiter) RemoveConnection(remoteAddr string) { + if l == nil { + return + } + + ip, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return + } + + l.connMu.RLock() + counter, exists := l.ipConnections[ip] + l.connMu.RUnlock() + + if exists { + newCount := counter.Add(-1) + if newCount <= 0 { + // Clean up if no more connections + l.connMu.Lock() + if counter.Load() <= 0 { + delete(l.ipConnections, ip) + } + l.connMu.Unlock() + } + } +} + +// Returns rate limiter statistics +func (l *Limiter) GetStats() map[string]interface{} { + if l == nil { + return map[string]interface{}{ + "enabled": false, + } + } + + l.ipMu.RLock() + activeIPs := len(l.ipLimiters) + l.ipMu.RUnlock() + + l.connMu.RLock() + totalConnections := 0 + for _, counter := range l.ipConnections { + totalConnections += int(counter.Load()) + } + l.connMu.RUnlock() + + return map[string]interface{}{ + "enabled": true, + "total_requests": l.totalRequests.Load(), + "blocked_requests": l.blockedRequests.Load(), + "active_ips": activeIPs, + "total_connections": totalConnections, + "config": map[string]interface{}{ + "requests_per_second": l.config.RequestsPerSecond, + "burst_size": l.config.BurstSize, + "limit_by": l.config.LimitBy, + }, + } +} + +// Performs the actual rate limit check +func (l *Limiter) checkLimit(ip string) bool { + // Maybe run cleanup + l.maybeCleanup() + + switch l.config.LimitBy { + case "global": + return l.globalLimiter.Allow() + + case "ip", "": + // Default to per-IP limiting + l.ipMu.Lock() + limiter, exists := l.ipLimiters[ip] + if !exists { + // Create new limiter for this IP + limiter = &ipLimiter{ + bucket: NewTokenBucket( + float64(l.config.BurstSize), + l.config.RequestsPerSecond, + ), + lastSeen: time.Now(), + } + l.ipLimiters[ip] = limiter + l.uniqueIPs.Add(1) + } else { + limiter.lastSeen = time.Now() + } + l.ipMu.Unlock() + + // Check connection limit if configured + if l.config.MaxConnectionsPerIP > 0 { + l.connMu.RLock() + counter, exists := l.ipConnections[ip] + l.connMu.RUnlock() + + if exists && counter.Load() >= int32(l.config.MaxConnectionsPerIP) { + return false + } + } + + return limiter.bucket.Allow() + + default: + // Unknown limit_by value, allow by default + return true + } +} + +// Runs cleanup if enough time has passed +func (l *Limiter) maybeCleanup() { + l.cleanupMu.Lock() + defer l.cleanupMu.Unlock() + + if time.Since(l.lastCleanup) < 30*time.Second { + return + } + + l.lastCleanup = time.Now() + go l.cleanup() +} + +// Removes stale IP limiters +func (l *Limiter) cleanup() { + staleTimeout := 5 * time.Minute + now := time.Now() + + l.ipMu.Lock() + defer l.ipMu.Unlock() + + for ip, limiter := range l.ipLimiters { + if now.Sub(limiter.lastSeen) > staleTimeout { + delete(l.ipLimiters, ip) + } + } +} + +// Runs periodic cleanup +func (l *Limiter) cleanupLoop() { + ticker := time.NewTicker(1 * time.Minute) + defer ticker.Stop() + + for range ticker.C { + l.cleanup() + } +} \ No newline at end of file diff --git a/src/internal/ratelimit/ratelimit.go b/src/internal/ratelimit/ratelimit.go new file mode 100644 index 0000000..ce13c3c --- /dev/null +++ b/src/internal/ratelimit/ratelimit.go @@ -0,0 +1,53 @@ +// FILE: src/internal/ratelimit/ratelimit.go +package ratelimit + +import ( + "sync" + "time" +) + +// TokenBucket implements a token bucket rate limiter +type TokenBucket struct { + capacity float64 + tokens float64 + refillRate float64 + lastRefill time.Time + mu sync.Mutex +} + +// NewTokenBucket creates a new token bucket with given capacity and refill rate +func NewTokenBucket(capacity float64, refillRate float64) *TokenBucket { + return &TokenBucket{ + capacity: capacity, + tokens: capacity, + refillRate: refillRate, + lastRefill: time.Now(), + } +} + +// Allow attempts to consume one token, returns true if allowed +func (tb *TokenBucket) Allow() bool { + return tb.AllowN(1) +} + +// AllowN attempts to consume n tokens, returns true if allowed +func (tb *TokenBucket) AllowN(n float64) bool { + tb.mu.Lock() + defer tb.mu.Unlock() + + // Refill tokens based on time elapsed + now := time.Now() + elapsed := now.Sub(tb.lastRefill).Seconds() + tb.tokens += elapsed * tb.refillRate + if tb.tokens > tb.capacity { + tb.tokens = tb.capacity + } + tb.lastRefill = now + + // Check if we have enough tokens + if tb.tokens >= n { + tb.tokens -= n + return true + } + return false +} \ No newline at end of file diff --git a/src/internal/stream/httpstreamer.go b/src/internal/stream/httpstreamer.go index bd7ca91..30b91e8 100644 --- a/src/internal/stream/httpstreamer.go +++ b/src/internal/stream/httpstreamer.go @@ -14,6 +14,7 @@ import ( "github.com/valyala/fasthttp" "logwisp/src/internal/config" "logwisp/src/internal/monitor" + "logwisp/src/internal/ratelimit" "logwisp/src/internal/version" ) @@ -33,6 +34,9 @@ type HTTPStreamer struct { // For router integration standalone bool + + // Rate limiting + rateLimiter *ratelimit.Limiter } func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTPStreamer { @@ -46,7 +50,7 @@ func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTP statusPath = "/status" } - return &HTTPStreamer{ + h := &HTTPStreamer{ logChan: logChan, config: cfg, startTime: time.Now(), @@ -55,9 +59,16 @@ func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTP statusPath: statusPath, standalone: true, // Default to standalone mode } + + // Initialize rate limiter if configured + if cfg.RateLimit != nil && cfg.RateLimit.Enabled { + h.rateLimiter = ratelimit.New(*cfg.RateLimit) + } + + return h } -// SetRouterMode configures the streamer for use with a router +// Configures the streamer for use with a router func (h *HTTPStreamer) SetRouterMode() { h.standalone = false } @@ -116,6 +127,18 @@ func (h *HTTPStreamer) RouteRequest(ctx *fasthttp.RequestCtx) { } func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) { + // Check rate limit first + remoteAddr := ctx.RemoteAddr().String() + if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed { + ctx.SetStatusCode(statusCode) + ctx.SetContentType("application/json") + json.NewEncoder(ctx).Encode(map[string]interface{}{ + "error": message, + "retry_after": "60", // seconds + }) + return + } + path := string(ctx.Path()) switch path { @@ -135,6 +158,13 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) { } func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) { + // Track connection for rate limiting + remoteAddr := ctx.RemoteAddr().String() + if h.rateLimiter != nil { + h.rateLimiter.AddConnection(remoteAddr) + defer h.rateLimiter.RemoveConnection(remoteAddr) + } + // Set SSE headers ctx.Response.Header.Set("Content-Type", "text/event-stream") ctx.Response.Header.Set("Cache-Control", "no-cache") @@ -285,6 +315,15 @@ func (h *HTTPStreamer) formatHeartbeat() string { func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) { ctx.SetContentType("application/json") + var rateLimitStats interface{} + if h.rateLimiter != nil { + rateLimitStats = h.rateLimiter.GetStats() + } else { + rateLimitStats = map[string]interface{}{ + "enabled": false, + } + } + status := map[string]interface{}{ "service": "LogWisp", "version": version.Short(), @@ -309,9 +348,7 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) { "ssl": map[string]bool{ "enabled": h.config.SSL != nil && h.config.SSL.Enabled, }, - "rate_limit": map[string]bool{ - "enabled": h.config.RateLimit != nil && h.config.RateLimit.Enabled, - }, + "rate_limit": rateLimitStats, }, } diff --git a/src/internal/stream/tcpserver.go b/src/internal/stream/tcpserver.go index 4e71dca..74c628a 100644 --- a/src/internal/stream/tcpserver.go +++ b/src/internal/stream/tcpserver.go @@ -3,6 +3,7 @@ package stream import ( "fmt" + "net" "sync" "github.com/panjf2000/gnet/v2" @@ -17,10 +18,34 @@ type tcpServer struct { func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action { // Store engine reference for shutdown s.streamer.engine = &eng + fmt.Printf("[TCP DEBUG] Server booted on port %d\n", s.streamer.config.Port) return gnet.None } func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) { + // Debug: Log all connection attempts + fmt.Printf("[TCP DEBUG] Connection attempt from %s\n", c.RemoteAddr()) + + // Check rate limit + if s.streamer.rateLimiter != nil { + // Parse the remote address to get proper net.Addr + remoteStr := c.RemoteAddr().String() + tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr) + if err != nil { + fmt.Printf("[TCP DEBUG] Failed to parse address %s: %v\n", remoteStr, err) + return nil, gnet.Close + } + + if !s.streamer.rateLimiter.CheckTCP(tcpAddr) { + fmt.Printf("[TCP DEBUG] Rate limited connection from %s\n", remoteStr) + // Silently close connection when rate limited + return nil, gnet.Close + } + + // Track connection + s.streamer.rateLimiter.AddConnection(remoteStr) + } + s.connections.Store(c, struct{}{}) oldCount := s.streamer.activeConns.Load() @@ -34,6 +59,11 @@ func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) { func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action { s.connections.Delete(c) + // Remove connection tracking + if s.streamer.rateLimiter != nil { + s.streamer.rateLimiter.RemoveConnection(c.RemoteAddr().String()) + } + oldCount := s.streamer.activeConns.Load() newCount := s.streamer.activeConns.Add(-1) fmt.Printf("[TCP ATOMIC] OnClose: %d -> %d (expected: %d)\n", oldCount, newCount, oldCount-1) diff --git a/src/internal/stream/tcpstreamer.go b/src/internal/stream/tcpstreamer.go index 1fb0458..4cdddfe 100644 --- a/src/internal/stream/tcpstreamer.go +++ b/src/internal/stream/tcpstreamer.go @@ -12,6 +12,7 @@ import ( "github.com/panjf2000/gnet/v2" "logwisp/src/internal/config" "logwisp/src/internal/monitor" + "logwisp/src/internal/ratelimit" ) type TCPStreamer struct { @@ -23,15 +24,22 @@ type TCPStreamer struct { startTime time.Time engine *gnet.Engine wg sync.WaitGroup + rateLimiter *ratelimit.Limiter } func NewTCPStreamer(logChan chan monitor.LogEntry, cfg config.TCPConfig) *TCPStreamer { - return &TCPStreamer{ + t := &TCPStreamer{ logChan: logChan, config: cfg, done: make(chan struct{}), startTime: time.Now(), } + + if cfg.RateLimit != nil && cfg.RateLimit.Enabled { + t.rateLimiter = ratelimit.New(*cfg.RateLimit) + } + + return t } func (t *TCPStreamer) Start() error {