v0.1.8 rate limiter added, improved http router, config templates added, docs updated
This commit is contained in:
135
README.md
135
README.md
@ -4,7 +4,7 @@
|
||||
<img src="assets/logwisp-logo.svg" alt="LogWisp Logo" width="200"/>
|
||||
</p>
|
||||
|
||||
A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring and rotation detection.
|
||||
A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring, rotation detection, and rate limiting.
|
||||
|
||||
## Features
|
||||
|
||||
@ -13,8 +13,10 @@ A high-performance log streaming service with multi-stream architecture, support
|
||||
- **Real-time Monitoring**: Instant updates with per-stream configurable check intervals
|
||||
- **File Rotation Detection**: Automatic detection and handling of log rotation
|
||||
- **Path-based Routing**: Optional HTTP router for consolidated access
|
||||
- **Per-Stream Configuration**: Independent settings including check intervals for each log stream
|
||||
- **Connection Statistics**: Real-time monitoring of active connections
|
||||
- **Rate Limiting**: Per-IP or global rate limiting with token bucket algorithm
|
||||
- **Connection Limiting**: Configurable concurrent connection limits per IP
|
||||
- **Per-Stream Configuration**: Independent settings including check intervals and rate limits
|
||||
- **Connection Statistics**: Real-time monitoring of active connections and rate limit metrics
|
||||
- **Flexible Targets**: Monitor individual files or entire directories
|
||||
- **Version Management**: Git tag-based versioning with build information
|
||||
- **Configurable Heartbeats**: Keep connections alive with customizable formats
|
||||
@ -47,10 +49,12 @@ LogWisp uses a service-oriented architecture where each stream is an independent
|
||||
LogStream Service
|
||||
├── Stream["app-logs"]
|
||||
│ ├── Monitor (watches files)
|
||||
│ ├── Rate Limiter (optional)
|
||||
│ ├── TCP Server (optional)
|
||||
│ └── HTTP Server (optional)
|
||||
├── Stream["system-logs"]
|
||||
│ ├── Monitor
|
||||
│ ├── Rate Limiter (optional)
|
||||
│ └── HTTP Server
|
||||
└── HTTP Router (optional, for path-based routing)
|
||||
```
|
||||
@ -89,6 +93,16 @@ format = "comment" # or "json" for structured events
|
||||
include_timestamp = true
|
||||
include_stats = false
|
||||
|
||||
# Rate limiting configuration
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 10.0
|
||||
burst_size = 20
|
||||
limit_by = "ip"
|
||||
response_code = 429
|
||||
response_message = "Rate limit exceeded"
|
||||
max_connections_per_ip = 5
|
||||
|
||||
# System logs stream with slower check interval
|
||||
[[streams]]
|
||||
name = "system"
|
||||
@ -112,6 +126,13 @@ enabled = true
|
||||
interval_seconds = 300 # 5 minutes
|
||||
include_timestamp = true
|
||||
include_stats = true
|
||||
|
||||
# TCP rate limiting
|
||||
[streams.tcpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 5.0
|
||||
burst_size = 10
|
||||
limit_by = "ip"
|
||||
```
|
||||
|
||||
### Target Configuration
|
||||
@ -137,6 +158,22 @@ Each stream can have its own check interval based on log update frequency:
|
||||
- **Normal logs**: 100-1000ms (e.g., application logs)
|
||||
- **Low-frequency logs**: 10000-60000ms (e.g., system logs, archives)
|
||||
|
||||
### Rate Limiting Configuration
|
||||
|
||||
Control request rates and connection limits per stream:
|
||||
|
||||
```toml
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true # Enable/disable rate limiting
|
||||
requests_per_second = 10.0 # Token refill rate
|
||||
burst_size = 20 # Maximum burst capacity
|
||||
limit_by = "ip" # "ip" or "global"
|
||||
response_code = 429 # HTTP response code when limited
|
||||
response_message = "Too many requests"
|
||||
max_connections_per_ip = 5 # Max concurrent connections per IP
|
||||
max_total_connections = 100 # Max total connections (global)
|
||||
```
|
||||
|
||||
### Heartbeat Configuration
|
||||
|
||||
Keep connections alive and detect stale clients with configurable heartbeats:
|
||||
@ -198,7 +235,7 @@ All HTTP streams share ports with path-based routing:
|
||||
# Connect to a stream
|
||||
curl -N http://localhost:8080/stream
|
||||
|
||||
# Check stream status
|
||||
# Check stream status (includes rate limit stats)
|
||||
curl http://localhost:8080/status
|
||||
|
||||
# With authentication (when implemented)
|
||||
@ -237,6 +274,13 @@ eventSource.addEventListener('heartbeat', (e) => {
|
||||
const heartbeat = JSON.parse(e.data);
|
||||
console.log('Heartbeat:', heartbeat);
|
||||
});
|
||||
|
||||
eventSource.addEventListener('error', (e) => {
|
||||
if (e.status === 429) {
|
||||
console.error('Rate limited - backing off');
|
||||
// Implement exponential backoff
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Log Entry Format
|
||||
@ -284,6 +328,20 @@ All log entries are streamed as JSON:
|
||||
"active_watchers": 3,
|
||||
"total_entries": 15420,
|
||||
"dropped_entries": 0
|
||||
},
|
||||
"features": {
|
||||
"rate_limit": {
|
||||
"enabled": true,
|
||||
"total_requests": 45678,
|
||||
"blocked_requests": 234,
|
||||
"active_ips": 23,
|
||||
"total_connections": 5,
|
||||
"config": {
|
||||
"requests_per_second": 10,
|
||||
"burst_size": 20,
|
||||
"limit_by": "ip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
@ -294,6 +352,7 @@ LogWisp provides comprehensive statistics at multiple levels:
|
||||
|
||||
- **Per-Stream Stats**: Monitor performance, connection counts, data throughput
|
||||
- **Per-Watcher Stats**: File size, position, entries read, rotation count
|
||||
- **Rate Limit Stats**: Total requests, blocked requests, active IPs
|
||||
- **Global Stats**: Aggregated view of all streams (in router mode)
|
||||
|
||||
Access statistics via status endpoints or watch the console output:
|
||||
@ -306,6 +365,21 @@ Access statistics via status endpoints or watch the console output:
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Rate Limiting
|
||||
|
||||
LogWisp implements token bucket rate limiting with:
|
||||
- **Per-IP limiting**: Each IP gets its own token bucket
|
||||
- **Global limiting**: All clients share a single token bucket
|
||||
- **Connection limits**: Restrict concurrent connections per IP
|
||||
- **Automatic cleanup**: Stale IP entries removed after 5 minutes
|
||||
- **Non-blocking**: Excess requests are immediately rejected with 429 status
|
||||
|
||||
Monitor rate limiting effectiveness:
|
||||
```bash
|
||||
# Watch rate limit statistics
|
||||
watch -n 1 'curl -s http://localhost:8080/status | jq .features.rate_limit'
|
||||
```
|
||||
|
||||
### File Rotation Detection
|
||||
|
||||
LogWisp automatically detects log rotation through multiple methods:
|
||||
@ -349,6 +423,11 @@ check_interval_ms = 60000 # Check every minute
|
||||
- Configure per-stream based on expected update frequency
|
||||
- Use 10000ms+ for archival or slowly updating logs
|
||||
|
||||
### Rate Limiting
|
||||
- `requests_per_second`: Balance between protection and availability
|
||||
- `burst_size`: Set to 2-3x the per-second rate for traffic spikes
|
||||
- `max_connections_per_ip`: Prevent resource exhaustion from single IPs
|
||||
|
||||
### File Watcher Optimization
|
||||
- Use specific file paths when possible (more efficient than directory scanning)
|
||||
- Adjust patterns to minimize unnecessary file checks
|
||||
@ -378,6 +457,12 @@ make build
|
||||
# Run tests
|
||||
make test
|
||||
|
||||
# Test rate limiting
|
||||
./test_ratelimit.sh
|
||||
|
||||
# Test router functionality
|
||||
./test_router.sh
|
||||
|
||||
# Create a release
|
||||
make release TAG=v1.0.0
|
||||
```
|
||||
@ -413,6 +498,9 @@ ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadOnlyPaths=/var/log
|
||||
|
||||
# Rate limiting at system level
|
||||
LimitNOFILE=65536
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
@ -448,31 +536,52 @@ services:
|
||||
- "9090:9090"
|
||||
restart: unless-stopped
|
||||
command: ["logwisp", "--config", "/etc/logwisp/config.toml"]
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1.0'
|
||||
memory: 512M
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Current Implementation
|
||||
- Read-only file access
|
||||
- Rate limiting for DDoS protection
|
||||
- Connection limits to prevent resource exhaustion
|
||||
- No authentication (placeholder configuration only)
|
||||
- No TLS/SSL support (placeholder configuration only)
|
||||
|
||||
### Planned Security Features
|
||||
- **Authentication**: Basic, Bearer/JWT, mTLS
|
||||
- **TLS/SSL**: For both HTTP and TCP streams
|
||||
- **Rate Limiting**: Per-client request limits
|
||||
- **IP Filtering**: Whitelist/blacklist support
|
||||
- **Audit Logging**: Access and authentication events
|
||||
- **RBAC**: Role-based access control per stream
|
||||
|
||||
### Best Practices
|
||||
1. Run with minimal privileges (read-only access to log files)
|
||||
2. Use network-level security until authentication is implemented
|
||||
3. Place behind a reverse proxy for production HTTPS
|
||||
4. Monitor access logs for unusual patterns
|
||||
5. Regularly update dependencies
|
||||
2. Configure appropriate rate limits based on expected traffic
|
||||
3. Use network-level security until authentication is implemented
|
||||
4. Place behind a reverse proxy for production HTTPS
|
||||
5. Monitor rate limit statistics for potential attacks
|
||||
6. Regularly update dependencies
|
||||
|
||||
### Rate Limiting Best Practices
|
||||
- Start with conservative limits and adjust based on monitoring
|
||||
- Use per-IP limiting for public endpoints
|
||||
- Use global limiting for resource protection
|
||||
- Set connection limits to prevent memory exhaustion
|
||||
- Monitor blocked request statistics for anomalies
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Rate Limit Issues
|
||||
1. Check rate limit statistics in status endpoint
|
||||
2. Verify appropriate `requests_per_second` for your use case
|
||||
3. Ensure `burst_size` accommodates normal traffic spikes
|
||||
4. Monitor for distributed attacks if per-IP limiting isn't effective
|
||||
|
||||
### No Log Entries Appearing
|
||||
1. Check file permissions (LogWisp needs read access)
|
||||
2. Verify file paths in configuration
|
||||
@ -483,14 +592,16 @@ services:
|
||||
### High Memory Usage
|
||||
1. Reduce buffer sizes in configuration
|
||||
2. Lower the number of concurrent watchers
|
||||
3. Increase check interval for less critical logs
|
||||
4. Use TCP instead of HTTP for high-volume streams
|
||||
3. Enable rate limiting to prevent connection floods
|
||||
4. Increase check interval for less critical logs
|
||||
5. Use TCP instead of HTTP for high-volume streams
|
||||
|
||||
### Connection Drops
|
||||
1. Check heartbeat configuration
|
||||
2. Verify network stability
|
||||
3. Monitor client-side errors
|
||||
4. Review dropped entry statistics
|
||||
5. Check if rate limits are too restrictive
|
||||
|
||||
### Version Information
|
||||
Use `./logwisp --version` to see:
|
||||
@ -515,7 +626,7 @@ Contributions are welcome! Please read our contributing guidelines and submit pu
|
||||
- [x] Per-stream check intervals
|
||||
- [x] Version management
|
||||
- [x] Configurable heartbeats
|
||||
- [ ] Rate and connection limiting
|
||||
- [x] Rate and connection limiting
|
||||
- [ ] Log filtering and transformation
|
||||
- [ ] Configurable logging support
|
||||
- [ ] Authentication (Basic, JWT, mTLS)
|
||||
|
||||
@ -1,181 +0,0 @@
|
||||
# LogWisp Multi-Stream Configuration
|
||||
# Location: ~/.config/logwisp.toml
|
||||
|
||||
# Stream 1: Application logs (public access)
|
||||
[[streams]]
|
||||
name = "app"
|
||||
|
||||
[streams.monitor]
|
||||
# Check interval in milliseconds (per-stream configuration)
|
||||
check_interval_ms = 100
|
||||
# Array of folders and files to be monitored
|
||||
# For file targets, pattern is ignored and can be omitted
|
||||
targets = [
|
||||
{ path = "/var/log/myapp", pattern = "*.log", is_file = false },
|
||||
{ path = "/var/log/myapp/app.log", pattern = "", is_file = true }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8080
|
||||
buffer_size = 2000
|
||||
stream_path = "/stream"
|
||||
status_path = "/status"
|
||||
|
||||
# HTTP SSE Heartbeat Configuration
|
||||
[streams.httpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 30
|
||||
# Format options: "comment" (SSE comments) or "json" (JSON events)
|
||||
format = "comment"
|
||||
# Include timestamp in heartbeat
|
||||
include_timestamp = true
|
||||
# Include server statistics (client count, uptime)
|
||||
include_stats = false
|
||||
|
||||
# Stream 2: System logs (authenticated)
|
||||
[[streams]]
|
||||
name = "system"
|
||||
|
||||
[streams.monitor]
|
||||
# More frequent checks for critical system logs
|
||||
check_interval_ms = 50
|
||||
targets = [
|
||||
{ path = "/var/log", pattern = "syslog*", is_file = false },
|
||||
{ path = "/var/log/auth.log", pattern = "", is_file = true }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8443
|
||||
buffer_size = 5000
|
||||
stream_path = "/logs"
|
||||
status_path = "/health"
|
||||
|
||||
# JSON format heartbeat with full stats
|
||||
[streams.httpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 20
|
||||
format = "json"
|
||||
include_timestamp = true
|
||||
include_stats = true
|
||||
|
||||
# SSL placeholder
|
||||
[streams.httpserver.ssl]
|
||||
enabled = true
|
||||
cert_file = "/etc/logwisp/certs/server.crt"
|
||||
key_file = "/etc/logwisp/certs/server.key"
|
||||
min_version = "TLS1.2"
|
||||
|
||||
# Authentication placeholder
|
||||
[streams.auth]
|
||||
type = "basic"
|
||||
|
||||
[streams.auth.basic_auth]
|
||||
realm = "System Logs"
|
||||
users = [
|
||||
{ username = "admin", password_hash = "$2y$10$..." }
|
||||
]
|
||||
ip_whitelist = ["10.0.0.0/8", "192.168.0.0/16"]
|
||||
|
||||
# TCP server also available
|
||||
[streams.tcpserver]
|
||||
enabled = true
|
||||
port = 9443
|
||||
buffer_size = 5000
|
||||
|
||||
# TCP heartbeat (always JSON format)
|
||||
[streams.tcpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 60
|
||||
include_timestamp = true
|
||||
include_stats = true
|
||||
|
||||
# Stream 3: Debug logs (high-volume, less frequent checks)
|
||||
[[streams]]
|
||||
name = "debug"
|
||||
|
||||
[streams.monitor]
|
||||
# Check every 10 seconds for debug logs
|
||||
check_interval_ms = 10000
|
||||
targets = [
|
||||
{ path = "./debug", pattern = "*.debug", is_file = false }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8082
|
||||
buffer_size = 10000
|
||||
stream_path = "/stream"
|
||||
status_path = "/status"
|
||||
|
||||
# Disable heartbeat for high-volume stream
|
||||
[streams.httpserver.heartbeat]
|
||||
enabled = false
|
||||
|
||||
# Rate limiting placeholder
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 100.0
|
||||
burst_size = 1000
|
||||
limit_by = "ip"
|
||||
|
||||
# Stream 4: Slow logs (infrequent updates)
|
||||
[[streams]]
|
||||
name = "archive"
|
||||
|
||||
[streams.monitor]
|
||||
# Check once per minute for archival logs
|
||||
check_interval_ms = 60000
|
||||
targets = [
|
||||
{ path = "/var/log/archive", pattern = "*.log.gz", is_file = false }
|
||||
]
|
||||
|
||||
[streams.tcpserver]
|
||||
enabled = true
|
||||
port = 9091
|
||||
buffer_size = 1000
|
||||
|
||||
# Minimal heartbeat for connection keep-alive
|
||||
[streams.tcpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 300 # 5 minutes
|
||||
include_timestamp = false
|
||||
include_stats = false
|
||||
|
||||
# Heartbeat Format Examples:
|
||||
#
|
||||
# Comment format (SSE):
|
||||
# : heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s
|
||||
#
|
||||
# JSON format (SSE):
|
||||
# event: heartbeat
|
||||
# data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600}
|
||||
#
|
||||
# TCP always uses JSON format with newline delimiter
|
||||
|
||||
# Usage Examples:
|
||||
#
|
||||
# 1. Standard mode (each stream on its own port):
|
||||
# ./logwisp
|
||||
# - App logs: http://localhost:8080/stream
|
||||
# - System logs: https://localhost:8443/logs (with auth)
|
||||
# - Debug logs: http://localhost:8082/stream
|
||||
# - Archive logs: tcp://localhost:9091
|
||||
#
|
||||
# 2. Router mode (shared port with path routing):
|
||||
# ./logwisp --router
|
||||
# - App logs: http://localhost:8080/app/stream
|
||||
# - System logs: http://localhost:8080/system/logs
|
||||
# - Debug logs: http://localhost:8080/debug/stream
|
||||
# - Global status: http://localhost:8080/status
|
||||
#
|
||||
# 3. Override config file:
|
||||
# ./logwisp --config /etc/logwisp/production.toml
|
||||
#
|
||||
# 4. Environment variables:
|
||||
# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
|
||||
# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
|
||||
#
|
||||
# 5. Show version:
|
||||
# ./logwisp --version
|
||||
312
config/logwisp.toml.defaults
Normal file
312
config/logwisp.toml.defaults
Normal file
@ -0,0 +1,312 @@
|
||||
# LogWisp Configuration File
|
||||
# Default path: ~/.config/logwisp.toml
|
||||
# Override with: ./logwisp --config /path/to/config.toml
|
||||
|
||||
# This is a complete configuration reference showing all available options.
|
||||
# Default values are uncommented, alternatives and examples are commented.
|
||||
|
||||
# ==============================================================================
|
||||
# STREAM CONFIGURATION
|
||||
# ==============================================================================
|
||||
# Each [[streams]] section defines an independent log monitoring stream.
|
||||
# You can have multiple streams, each with its own settings.
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Default Stream - Monitors current directory
|
||||
# ------------------------------------------------------------------------------
|
||||
[[streams]]
|
||||
# Stream identifier used in logs, metrics, and router paths
|
||||
name = "default"
|
||||
|
||||
# File monitoring configuration
|
||||
[streams.monitor]
|
||||
# How often to check for new log entries (milliseconds)
|
||||
# Lower = faster detection but more CPU usage
|
||||
check_interval_ms = 100
|
||||
|
||||
# Targets to monitor - can be files or directories
|
||||
targets = [
|
||||
# Monitor all .log files in current directory
|
||||
{ path = "./", pattern = "*.log", is_file = false },
|
||||
]
|
||||
|
||||
# HTTP Server configuration (SSE/Server-Sent Events)
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8080
|
||||
buffer_size = 1000 # Per-client buffer size (messages)
|
||||
stream_path = "/stream" # Endpoint for SSE stream
|
||||
status_path = "/status" # Endpoint for statistics
|
||||
|
||||
# Keep-alive heartbeat configuration
|
||||
[streams.httpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 30 # Send heartbeat every 30 seconds
|
||||
format = "comment" # SSE comment format (: heartbeat)
|
||||
include_timestamp = true # Include timestamp in heartbeat
|
||||
include_stats = false # Include connection stats
|
||||
|
||||
# Rate limiting configuration (disabled by default)
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = false
|
||||
# requests_per_second = 10.0 # Token refill rate
|
||||
# burst_size = 20 # Max burst capacity
|
||||
# limit_by = "ip" # "ip" or "global"
|
||||
# response_code = 429 # HTTP Too Many Requests
|
||||
# response_message = "Rate limit exceeded"
|
||||
# max_connections_per_ip = 5 # Max SSE connections per IP
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Application Logs Stream
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "app"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 50 # Fast detection for active logs
|
||||
# targets = [
|
||||
# # Monitor specific application log directory
|
||||
# { path = "/var/log/myapp", pattern = "*.log", is_file = false },
|
||||
# # Also monitor specific file
|
||||
# { path = "/var/log/myapp/app.log", is_file = true },
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8081 # Different port for each stream
|
||||
# buffer_size = 2000 # Larger buffer for busy logs
|
||||
# stream_path = "/logs" # Custom path
|
||||
# status_path = "/health" # Custom health endpoint
|
||||
#
|
||||
# # JSON heartbeat format for programmatic clients
|
||||
# [streams.httpserver.heartbeat]
|
||||
# enabled = true
|
||||
# interval_seconds = 20
|
||||
# format = "json" # JSON event format
|
||||
# include_timestamp = true
|
||||
# include_stats = true # Include active client count
|
||||
#
|
||||
# # Moderate rate limiting for public access
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 25.0
|
||||
# burst_size = 50
|
||||
# limit_by = "ip"
|
||||
# max_connections_per_ip = 10
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: System Logs Stream (TCP + HTTP)
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "system"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 1000 # Check every second (system logs update slowly)
|
||||
# targets = [
|
||||
# { path = "/var/log/syslog", is_file = true },
|
||||
# { path = "/var/log/auth.log", is_file = true },
|
||||
# { path = "/var/log/kern.log", is_file = true },
|
||||
# ]
|
||||
#
|
||||
# # TCP Server for high-performance streaming
|
||||
# [streams.tcpserver]
|
||||
# enabled = true
|
||||
# port = 9090
|
||||
# buffer_size = 5000
|
||||
#
|
||||
# # TCP heartbeat (always JSON format)
|
||||
# [streams.tcpserver.heartbeat]
|
||||
# enabled = true
|
||||
# interval_seconds = 60 # Less frequent for TCP
|
||||
# include_timestamp = true
|
||||
# include_stats = false
|
||||
#
|
||||
# # TCP rate limiting
|
||||
# [streams.tcpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 5.0 # Limit TCP connections
|
||||
# burst_size = 10
|
||||
# limit_by = "ip"
|
||||
#
|
||||
# # Also expose via HTTP
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8082
|
||||
# buffer_size = 1000
|
||||
# stream_path = "/stream"
|
||||
# status_path = "/status"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: High-Volume Debug Logs
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "debug"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 5000 # Check every 5 seconds (high volume)
|
||||
# targets = [
|
||||
# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8083
|
||||
# buffer_size = 10000 # Very large buffer
|
||||
# stream_path = "/debug"
|
||||
# status_path = "/stats"
|
||||
#
|
||||
# # Disable heartbeat for high-volume streams
|
||||
# [streams.httpserver.heartbeat]
|
||||
# enabled = false
|
||||
#
|
||||
# # Aggressive rate limiting
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 1.0 # Very restrictive
|
||||
# burst_size = 5
|
||||
# limit_by = "ip"
|
||||
# max_connections_per_ip = 1 # One connection per IP
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Archived Logs (Slow Monitoring)
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "archive"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 60000 # Check once per minute
|
||||
# targets = [
|
||||
# { path = "/var/log/archive", pattern = "*.gz", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# [streams.tcpserver]
|
||||
# enabled = true
|
||||
# port = 9091
|
||||
# buffer_size = 500 # Small buffer for archived logs
|
||||
#
|
||||
# # Infrequent heartbeat
|
||||
# [streams.tcpserver.heartbeat]
|
||||
# enabled = true
|
||||
# interval_seconds = 300 # Every 5 minutes
|
||||
# include_timestamp = false
|
||||
# include_stats = false
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Security/Audit Logs with Strict Limits
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "security"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 100
|
||||
# targets = [
|
||||
# { path = "/var/log/audit", pattern = "audit.log*", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8443 # HTTPS port (for future TLS)
|
||||
# buffer_size = 1000
|
||||
# stream_path = "/audit"
|
||||
# status_path = "/health"
|
||||
#
|
||||
# # Strict rate limiting for security logs
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 2.0 # Very limited access
|
||||
# burst_size = 3
|
||||
# limit_by = "ip"
|
||||
# max_connections_per_ip = 1 # Single connection per IP
|
||||
# response_code = 403 # Forbidden instead of rate limit
|
||||
# response_message = "Access restricted"
|
||||
#
|
||||
# # Future: SSL/TLS configuration
|
||||
# # [streams.httpserver.ssl]
|
||||
# # enabled = true
|
||||
# # cert_file = "/etc/logwisp/certs/server.crt"
|
||||
# # key_file = "/etc/logwisp/certs/server.key"
|
||||
# # min_version = "TLS1.2"
|
||||
#
|
||||
# # Future: Authentication
|
||||
# # [streams.auth]
|
||||
# # type = "basic"
|
||||
# # [streams.auth.basic_auth]
|
||||
# # users_file = "/etc/logwisp/security.users"
|
||||
# # realm = "Security Logs"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Example: Public API Logs with Global Rate Limiting
|
||||
# ------------------------------------------------------------------------------
|
||||
# [[streams]]
|
||||
# name = "api-public"
|
||||
#
|
||||
# [streams.monitor]
|
||||
# check_interval_ms = 100
|
||||
# targets = [
|
||||
# { path = "/var/log/api", pattern = "access.log*", is_file = false },
|
||||
# ]
|
||||
#
|
||||
# [streams.httpserver]
|
||||
# enabled = true
|
||||
# port = 8084
|
||||
# buffer_size = 2000
|
||||
#
|
||||
# # Global rate limiting (all clients share limit)
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 100.0 # 100 req/s total
|
||||
# burst_size = 200
|
||||
# limit_by = "global" # All clients share this limit
|
||||
# max_total_connections = 50 # Max 50 connections total
|
||||
|
||||
# ==============================================================================
|
||||
# USAGE EXAMPLES
|
||||
# ==============================================================================
|
||||
|
||||
# 1. Basic usage (single stream):
|
||||
# ./logwisp
|
||||
# - Monitors current directory for *.log files
|
||||
# - Access logs at: http://localhost:8080/stream
|
||||
# - View stats at: http://localhost:8080/status
|
||||
|
||||
# 2. Multi-stream configuration:
|
||||
# - Uncomment additional [[streams]] sections above
|
||||
# - Each stream runs independently on its own port
|
||||
# - Different check intervals for different log types
|
||||
|
||||
# 3. Router mode (consolidated access):
|
||||
# ./logwisp --router
|
||||
# - All streams accessible via paths: /streamname/stream
|
||||
# - Global status at: /status
|
||||
# - Example: http://localhost:8080/app/stream
|
||||
|
||||
# 4. Production deployment:
|
||||
# - Enable rate limiting on public-facing streams
|
||||
# - Use TCP for internal high-volume streams
|
||||
# - Set appropriate check intervals (higher = less CPU)
|
||||
# - Configure heartbeats for long-lived connections
|
||||
|
||||
# 5. Monitoring:
|
||||
# curl http://localhost:8080/status | jq .
|
||||
# - Check active connections
|
||||
# - Monitor rate limit statistics
|
||||
# - Track log entry counts
|
||||
|
||||
# ==============================================================================
|
||||
# ENVIRONMENT VARIABLES
|
||||
# ==============================================================================
|
||||
# Configuration can be overridden via environment variables:
|
||||
# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
|
||||
# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
|
||||
# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
||||
|
||||
# ==============================================================================
|
||||
# NOTES
|
||||
# ==============================================================================
|
||||
# - Rate limiting is disabled by default for backward compatibility
|
||||
# - Each stream can have different rate limit settings
|
||||
# - TCP connections are silently dropped when rate limited
|
||||
# - HTTP returns 429 (or configured code) with JSON error
|
||||
# - IP tracking is cleaned up after 5 minutes of inactivity
|
||||
# - Token bucket algorithm provides smooth rate limiting
|
||||
# - Connection limits prevent resource exhaustion
|
||||
120
config/logwisp.toml.example
Normal file
120
config/logwisp.toml.example
Normal file
@ -0,0 +1,120 @@
|
||||
# LogWisp Configuration Example
|
||||
# Default path: ~/.config/logwisp.toml
|
||||
|
||||
# Application logs - public facing
|
||||
[[streams]]
|
||||
name = "app-public"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 100
|
||||
targets = [
|
||||
{ path = "/var/log/nginx", pattern = "access.log*", is_file = false },
|
||||
{ path = "/var/log/app", pattern = "production.log", is_file = true }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8080
|
||||
buffer_size = 2000
|
||||
stream_path = "/logs"
|
||||
status_path = "/health"
|
||||
|
||||
[streams.httpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 30
|
||||
format = "json"
|
||||
include_timestamp = true
|
||||
include_stats = true
|
||||
|
||||
# Rate limiting for public endpoint
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 50.0
|
||||
burst_size = 100
|
||||
limit_by = "ip"
|
||||
response_code = 429
|
||||
response_message = "Rate limit exceeded. Please retry after 60 seconds."
|
||||
max_connections_per_ip = 5
|
||||
max_total_connections = 100
|
||||
|
||||
# System logs - internal only
|
||||
[[streams]]
|
||||
name = "system"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 5000 # Check every 5 seconds
|
||||
targets = [
|
||||
{ path = "/var/log/syslog", is_file = true },
|
||||
{ path = "/var/log/auth.log", is_file = true },
|
||||
{ path = "/var/log/kern.log", is_file = true }
|
||||
]
|
||||
|
||||
# TCP for internal consumers
|
||||
[streams.tcpserver]
|
||||
enabled = true
|
||||
port = 9090
|
||||
buffer_size = 5000
|
||||
|
||||
[streams.tcpserver.heartbeat]
|
||||
enabled = true
|
||||
interval_seconds = 60
|
||||
include_timestamp = true
|
||||
|
||||
# Moderate rate limiting for internal use
|
||||
[streams.tcpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 10.0
|
||||
burst_size = 20
|
||||
limit_by = "ip"
|
||||
|
||||
# Security audit logs - restricted access
|
||||
[[streams]]
|
||||
name = "security"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 100
|
||||
targets = [
|
||||
{ path = "/var/log/audit", pattern = "*.log", is_file = false },
|
||||
{ path = "/var/log/fail2ban.log", is_file = true }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8443
|
||||
buffer_size = 1000
|
||||
stream_path = "/audit/stream"
|
||||
status_path = "/audit/status"
|
||||
|
||||
# Strict rate limiting
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 1.0
|
||||
burst_size = 3
|
||||
limit_by = "ip"
|
||||
max_connections_per_ip = 1
|
||||
response_code = 403
|
||||
response_message = "Access denied"
|
||||
|
||||
# Application debug logs - development team only
|
||||
[[streams]]
|
||||
name = "debug"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 1000
|
||||
targets = [
|
||||
{ path = "/var/log/app", pattern = "debug-*.log", is_file = false }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8090
|
||||
buffer_size = 5000
|
||||
stream_path = "/debug"
|
||||
status_path = "/debug/status"
|
||||
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = true
|
||||
requests_per_second = 100.0 # Higher limit for internal use
|
||||
burst_size = 200
|
||||
limit_by = "ip"
|
||||
max_connections_per_ip = 10
|
||||
25
config/logwisp.toml.minimal
Normal file
25
config/logwisp.toml.minimal
Normal file
@ -0,0 +1,25 @@
|
||||
# LogWisp Minimal Configuration Example
|
||||
# Save as: ~/.config/logwisp.toml
|
||||
|
||||
# Monitor application logs
|
||||
[[streams]]
|
||||
name = "app"
|
||||
|
||||
[streams.monitor]
|
||||
check_interval_ms = 100
|
||||
targets = [
|
||||
{ path = "/var/log/myapp", pattern = "*.log", is_file = false }
|
||||
]
|
||||
|
||||
[streams.httpserver]
|
||||
enabled = true
|
||||
port = 8080
|
||||
stream_path = "/stream"
|
||||
status_path = "/status"
|
||||
|
||||
# Optional: Enable rate limiting
|
||||
# [streams.httpserver.rate_limit]
|
||||
# enabled = true
|
||||
# requests_per_second = 10.0
|
||||
# burst_size = 20
|
||||
# limit_by = "ip"
|
||||
@ -9,9 +9,13 @@ logwisp/
|
||||
├── go.sum # Go module checksums
|
||||
├── README.md # Project documentation
|
||||
├── config/
|
||||
│ └── logwisp.toml # Example configuration template
|
||||
│ ├── logwisp.toml.defaults # Default configuration and guide
|
||||
│ ├── logwisp.toml.example # Example configuration
|
||||
│ └── logwisp.toml.minimal # Minimal configuration template
|
||||
├── doc/
|
||||
│ └── architecture.md # This file - architecture documentation
|
||||
├── test_router.sh # Router functionality test suite
|
||||
├── test_ratelimit.sh # Rate limiting test suite
|
||||
└── src/
|
||||
├── cmd/
|
||||
│ └── logwisp/
|
||||
@ -21,10 +25,10 @@ logwisp/
|
||||
│ ├── auth.go # Authentication configuration structures
|
||||
│ ├── config.go # Main configuration structures
|
||||
│ ├── loader.go # Configuration loading with lixenwraith/config
|
||||
│ ├── server.go # TCP/HTTP server configurations
|
||||
│ ├── server.go # TCP/HTTP server configurations with rate limiting
|
||||
│ ├── ssl.go # SSL/TLS configuration structures
|
||||
│ ├── stream.go # Stream-specific configurations
|
||||
│ └── validation.go # Configuration validation logic
|
||||
│ └── validation.go # Configuration validation including rate limits
|
||||
├── logstream/
|
||||
│ ├── httprouter.go # HTTP router for path-based routing
|
||||
│ ├── logstream.go # Stream lifecycle management
|
||||
@ -33,10 +37,13 @@ logwisp/
|
||||
├── monitor/
|
||||
│ ├── file_watcher.go # File watching and rotation detection
|
||||
│ └── monitor.go # Log monitoring interface and implementation
|
||||
├── ratelimit/
|
||||
│ ├── ratelimit.go # Token bucket algorithm implementation
|
||||
│ └── limiter.go # Per-stream rate limiter with IP tracking
|
||||
├── stream/
|
||||
│ ├── httpstreamer.go # HTTP/SSE streaming server
|
||||
│ ├── httpstreamer.go # HTTP/SSE streaming with rate limiting
|
||||
│ ├── noop_logger.go # Silent logger for gnet
|
||||
│ ├── tcpserver.go # TCP server using gnet
|
||||
│ ├── tcpserver.go # TCP server with rate limiting (gnet)
|
||||
│ └── tcpstreamer.go # TCP streaming implementation
|
||||
└── version/
|
||||
└── version.go # Version information management
|
||||
@ -85,6 +92,12 @@ LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=2000
|
||||
LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
|
||||
LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=json
|
||||
|
||||
# Rate limiting configuration
|
||||
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
||||
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
|
||||
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_BURST_SIZE=20
|
||||
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_LIMIT_BY=ip
|
||||
|
||||
# Multiple streams
|
||||
LOGWISP_STREAMS_1_NAME=system
|
||||
LOGWISP_STREAMS_1_MONITOR_CHECK_INTERVAL_MS=1000
|
||||
@ -103,9 +116,9 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
|
||||
|
||||
2. **LogStream (`logstream.LogStream`)**
|
||||
- Represents a single log monitoring pipeline
|
||||
- Contains: Monitor + Servers (TCP/HTTP)
|
||||
- Contains: Monitor + Rate Limiter + Servers (TCP/HTTP)
|
||||
- Independent configuration
|
||||
- Per-stream statistics
|
||||
- Per-stream statistics with rate limit metrics
|
||||
|
||||
3. **Monitor (`monitor.Monitor`)**
|
||||
- Watches files and directories
|
||||
@ -113,23 +126,49 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
|
||||
- Publishes log entries to subscribers
|
||||
- Configurable check intervals
|
||||
|
||||
4. **Streamers**
|
||||
4. **Rate Limiter (`ratelimit.Limiter`)**
|
||||
- Token bucket algorithm for smooth rate limiting
|
||||
- Per-IP or global limiting strategies
|
||||
- Connection tracking and limits
|
||||
- Automatic cleanup of stale entries
|
||||
- Non-blocking rejection of excess requests
|
||||
|
||||
5. **Streamers**
|
||||
- **HTTPStreamer**: SSE-based streaming over HTTP
|
||||
- Rate limit enforcement before request handling
|
||||
- Connection tracking for per-IP limits
|
||||
- Configurable 429 responses
|
||||
- **TCPStreamer**: Raw JSON streaming over TCP
|
||||
- Silent connection drops when rate limited
|
||||
- Per-IP connection tracking
|
||||
- Both support configurable heartbeats
|
||||
- Non-blocking client management
|
||||
|
||||
5. **HTTPRouter (`logstream.HTTPRouter`)**
|
||||
6. **HTTPRouter (`logstream.HTTPRouter`)**
|
||||
- Optional component for path-based routing
|
||||
- Consolidates multiple HTTP streams on shared ports
|
||||
- Provides global status endpoint
|
||||
- Longest-prefix path matching
|
||||
- Dynamic stream registration/deregistration
|
||||
|
||||
### Data Flow
|
||||
|
||||
```
|
||||
File System → Monitor → LogEntry Channel → Streamer → Network Client
|
||||
↑ ↓
|
||||
└── Rotation Detection
|
||||
File System → Monitor → LogEntry Channel → [Rate Limiter] → Streamer → Network Client
|
||||
↑ ↓ ↓
|
||||
└── Rotation Detection Rate Limit Check
|
||||
↓
|
||||
Accept/Reject
|
||||
```
|
||||
|
||||
### Rate Limiting Architecture
|
||||
|
||||
```
|
||||
Client Request → Rate Limiter → Token Bucket Check → Allow/Deny
|
||||
↓ ↓
|
||||
IP Tracking Refill Rate
|
||||
↓
|
||||
Cleanup Timer
|
||||
```
|
||||
|
||||
### Configuration Structure
|
||||
@ -159,6 +198,16 @@ format = "comment" # or "json"
|
||||
include_timestamp = true
|
||||
include_stats = false
|
||||
|
||||
[streams.httpserver.rate_limit]
|
||||
enabled = false # Disabled by default
|
||||
requests_per_second = 10.0 # Token refill rate
|
||||
burst_size = 20 # Token bucket capacity
|
||||
limit_by = "ip" # "ip" or "global"
|
||||
response_code = 429 # HTTP response code
|
||||
response_message = "Rate limit exceeded"
|
||||
max_connections_per_ip = 5 # Concurrent connection limit
|
||||
max_total_connections = 100 # Global connection limit
|
||||
|
||||
[streams.tcpserver]
|
||||
enabled = true
|
||||
port = 9090
|
||||
@ -169,8 +218,36 @@ enabled = true
|
||||
interval_seconds = 60
|
||||
include_timestamp = true
|
||||
include_stats = true
|
||||
|
||||
[streams.tcpserver.rate_limit]
|
||||
enabled = false
|
||||
requests_per_second = 5.0
|
||||
burst_size = 10
|
||||
limit_by = "ip"
|
||||
```
|
||||
|
||||
## Rate Limiting Implementation
|
||||
|
||||
### Token Bucket Algorithm
|
||||
- Each IP (or global limiter) gets a bucket with configurable capacity
|
||||
- Tokens refill at `requests_per_second` rate
|
||||
- Each request/connection consumes one token
|
||||
- Smooth rate limiting without hard cutoffs
|
||||
|
||||
### Limiting Strategies
|
||||
1. **Per-IP**: Each client IP gets its own token bucket
|
||||
2. **Global**: All clients share a single token bucket
|
||||
|
||||
### Connection Limits
|
||||
- Per-IP connection limits prevent single client resource exhaustion
|
||||
- Global connection limits protect overall system resources
|
||||
- Checked before rate limits to prevent connection hanging
|
||||
|
||||
### Cleanup
|
||||
- IP entries older than 5 minutes are automatically removed
|
||||
- Prevents unbounded memory growth
|
||||
- Runs every minute in background
|
||||
|
||||
## Build System
|
||||
|
||||
### Makefile Targets
|
||||
@ -208,5 +285,75 @@ go build -ldflags "-X 'logwisp/src/internal/version.Version=v1.0.0'" \
|
||||
### 2. Router Mode (`--router`)
|
||||
- HTTP streams share ports via path-based routing
|
||||
- Consolidated access through URL paths
|
||||
- Global status endpoint
|
||||
- Global status endpoint with aggregated statistics
|
||||
- Best for multi-stream setups with limited ports
|
||||
- Streams accessible at `/{stream_name}/{path}`
|
||||
|
||||
## Testing
|
||||
|
||||
### Test Suites
|
||||
|
||||
1. **Router Testing** (`test_router.sh`)
|
||||
- Path routing verification
|
||||
- Client isolation between streams
|
||||
- Statistics aggregation
|
||||
- Graceful shutdown
|
||||
- Port conflict handling
|
||||
|
||||
2. **Rate Limiting Testing** (`test_ratelimit.sh`)
|
||||
- Per-IP rate limiting
|
||||
- Global rate limiting
|
||||
- Connection limits
|
||||
- Rate limit recovery
|
||||
- Statistics accuracy
|
||||
- Stress testing
|
||||
|
||||
### Running Tests
|
||||
|
||||
```bash
|
||||
# Test router functionality
|
||||
./test_router.sh
|
||||
|
||||
# Test rate limiting
|
||||
./test_ratelimit.sh
|
||||
|
||||
# Run all tests
|
||||
make test
|
||||
```
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### Rate Limiting Overhead
|
||||
- Token bucket checks: O(1) time complexity
|
||||
- Memory: ~100 bytes per tracked IP
|
||||
- Cleanup: Runs asynchronously every minute
|
||||
- Minimal impact when disabled
|
||||
|
||||
### Optimization Guidelines
|
||||
- Use per-IP limiting for fairness
|
||||
- Use global limiting for resource protection
|
||||
- Set burst size to 2-3x requests_per_second
|
||||
- Monitor rate limit statistics for tuning
|
||||
- Higher check_interval_ms for low-activity logs
|
||||
|
||||
## Security Architecture
|
||||
|
||||
### Current Security Features
|
||||
- Read-only file access
|
||||
- Rate limiting for DDoS protection
|
||||
- Connection limits for resource protection
|
||||
- Non-blocking request rejection
|
||||
|
||||
### Future Security Roadmap
|
||||
- Authentication (Basic, JWT, mTLS)
|
||||
- TLS/SSL support
|
||||
- IP whitelisting/blacklisting
|
||||
- Audit logging
|
||||
- RBAC per stream
|
||||
|
||||
### Security Best Practices
|
||||
- Run with minimal privileges
|
||||
- Enable rate limiting on public endpoints
|
||||
- Use connection limits to prevent exhaustion
|
||||
- Deploy behind reverse proxy for HTTPS
|
||||
- Monitor rate limit statistics for attacks
|
||||
@ -53,10 +53,14 @@ type RateLimitConfig struct {
|
||||
// Burst size (token bucket)
|
||||
BurstSize int `toml:"burst_size"`
|
||||
|
||||
// Rate limit by: "ip", "user", "token"
|
||||
// Rate limit by: "ip", "user", "token", "global"
|
||||
LimitBy string `toml:"limit_by"`
|
||||
|
||||
// Response when rate limited
|
||||
ResponseCode int `toml:"response_code"` // Default: 429
|
||||
ResponseMessage string `toml:"response_message"` // Default: "Rate limit exceeded"
|
||||
|
||||
// Connection limits
|
||||
MaxConnectionsPerIP int `toml:"max_connections_per_ip"`
|
||||
MaxTotalConnections int `toml:"max_total_connections"`
|
||||
}
|
||||
@ -68,6 +68,10 @@ func (c *Config) validate() error {
|
||||
if err := validateSSL("TCP", stream.Name, stream.TCPServer.SSL); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateRateLimit("TCP", stream.Name, stream.TCPServer.RateLimit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Validate HTTP server
|
||||
@ -109,6 +113,10 @@ func (c *Config) validate() error {
|
||||
if err := validateSSL("HTTP", stream.Name, stream.HTTPServer.SSL); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateRateLimit("HTTP", stream.Name, stream.HTTPServer.RateLimit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// At least one server must be enabled
|
||||
@ -187,3 +195,32 @@ func validateAuth(streamName string, auth *AuthConfig) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error {
|
||||
if rl == nil || !rl.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
if rl.RequestsPerSecond <= 0 {
|
||||
return fmt.Errorf("stream '%s' %s: requests_per_second must be positive: %f",
|
||||
streamName, serverType, rl.RequestsPerSecond)
|
||||
}
|
||||
|
||||
if rl.BurstSize < 1 {
|
||||
return fmt.Errorf("stream '%s' %s: burst_size must be at least 1: %d",
|
||||
streamName, serverType, rl.BurstSize)
|
||||
}
|
||||
|
||||
validLimitBy := map[string]bool{"ip": true, "global": true, "": true}
|
||||
if !validLimitBy[rl.LimitBy] {
|
||||
return fmt.Errorf("stream '%s' %s: invalid limit_by value: %s (must be 'ip' or 'global')",
|
||||
streamName, serverType, rl.LimitBy)
|
||||
}
|
||||
|
||||
if rl.ResponseCode > 0 && (rl.ResponseCode < 400 || rl.ResponseCode >= 600) {
|
||||
return fmt.Errorf("stream '%s' %s: response_code must be 4xx or 5xx: %d",
|
||||
streamName, serverType, rl.ResponseCode)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
@ -13,12 +15,19 @@ type HTTPRouter struct {
|
||||
service *Service
|
||||
servers map[int]*routerServer // port -> server
|
||||
mu sync.RWMutex
|
||||
|
||||
// Statistics
|
||||
startTime time.Time
|
||||
totalRequests atomic.Uint64
|
||||
routedRequests atomic.Uint64
|
||||
failedRequests atomic.Uint64
|
||||
}
|
||||
|
||||
func NewHTTPRouter(service *Service) *HTTPRouter {
|
||||
return &HTTPRouter{
|
||||
service: service,
|
||||
servers: make(map[int]*routerServer),
|
||||
startTime: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,22 +45,29 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
|
||||
rs = &routerServer{
|
||||
port: port,
|
||||
routes: make(map[string]*LogStream),
|
||||
router: r,
|
||||
startTime: time.Now(),
|
||||
}
|
||||
rs.server = &fasthttp.Server{
|
||||
Handler: rs.requestHandler,
|
||||
DisableKeepalive: false,
|
||||
StreamRequestBody: true,
|
||||
CloseOnShutdown: true, // Ensure connections close on shutdown
|
||||
}
|
||||
r.servers[port] = rs
|
||||
|
||||
// Start server in background
|
||||
go func() {
|
||||
addr := fmt.Sprintf(":%d", port)
|
||||
fmt.Printf("[ROUTER] Starting server on port %d\n", port)
|
||||
if err := rs.server.ListenAndServe(addr); err != nil {
|
||||
// Log error but don't crash
|
||||
fmt.Printf("Router server on port %d failed: %v\n", port, err)
|
||||
fmt.Printf("[ROUTER] Server on port %d failed: %v\n", port, err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait briefly to ensure server starts
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
@ -71,6 +87,7 @@ func (r *HTTPRouter) RegisterStream(stream *LogStream) error {
|
||||
}
|
||||
|
||||
rs.routes[pathPrefix] = stream
|
||||
fmt.Printf("[ROUTER] Registered stream '%s' at path '%s' on port %d\n", stream.Name, pathPrefix, port)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -78,18 +95,27 @@ func (r *HTTPRouter) UnregisterStream(streamName string) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
for _, rs := range r.servers {
|
||||
for port, rs := range r.servers {
|
||||
rs.routeMu.Lock()
|
||||
for path, stream := range rs.routes {
|
||||
if stream.Name == streamName {
|
||||
delete(rs.routes, path)
|
||||
fmt.Printf("[ROUTER] Unregistered stream '%s' from path '%s' on port %d\n",
|
||||
streamName, path, port)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if server has no more routes
|
||||
if len(rs.routes) == 0 {
|
||||
fmt.Printf("[ROUTER] No routes left on port %d, considering shutdown\n", port)
|
||||
}
|
||||
rs.routeMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *HTTPRouter) Shutdown() {
|
||||
fmt.Println("[ROUTER] Starting router shutdown...")
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@ -98,10 +124,46 @@ func (r *HTTPRouter) Shutdown() {
|
||||
wg.Add(1)
|
||||
go func(p int, s *routerServer) {
|
||||
defer wg.Done()
|
||||
fmt.Printf("[ROUTER] Shutting down server on port %d\n", p)
|
||||
if err := s.server.Shutdown(); err != nil {
|
||||
fmt.Printf("Error shutting down router server on port %d: %v\n", p, err)
|
||||
fmt.Printf("[ROUTER] Error shutting down server on port %d: %v\n", p, err)
|
||||
}
|
||||
}(port, rs)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
fmt.Println("[ROUTER] Router shutdown complete")
|
||||
}
|
||||
|
||||
func (r *HTTPRouter) GetStats() map[string]interface{} {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
serverStats := make(map[int]interface{})
|
||||
totalRoutes := 0
|
||||
|
||||
for port, rs := range r.servers {
|
||||
rs.routeMu.RLock()
|
||||
routes := make([]string, 0, len(rs.routes))
|
||||
for path := range rs.routes {
|
||||
routes = append(routes, path)
|
||||
totalRoutes++
|
||||
}
|
||||
rs.routeMu.RUnlock()
|
||||
|
||||
serverStats[port] = map[string]interface{}{
|
||||
"routes": routes,
|
||||
"requests": rs.requests.Load(),
|
||||
"uptime": int(time.Since(rs.startTime).Seconds()),
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"uptime_seconds": int(time.Since(r.startTime).Seconds()),
|
||||
"total_requests": r.totalRequests.Load(),
|
||||
"routed_requests": r.routedRequests.Load(),
|
||||
"failed_requests": r.failedRequests.Load(),
|
||||
"servers": serverStats,
|
||||
"total_routes": totalRoutes,
|
||||
}
|
||||
}
|
||||
@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/valyala/fasthttp"
|
||||
"logwisp/src/internal/version"
|
||||
@ -16,11 +18,20 @@ type routerServer struct {
|
||||
server *fasthttp.Server
|
||||
routes map[string]*LogStream // path prefix -> stream
|
||||
routeMu sync.RWMutex
|
||||
router *HTTPRouter
|
||||
startTime time.Time
|
||||
requests atomic.Uint64
|
||||
}
|
||||
|
||||
func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
rs.requests.Add(1)
|
||||
rs.router.totalRequests.Add(1)
|
||||
|
||||
path := string(ctx.Path())
|
||||
|
||||
// Log request for debugging
|
||||
fmt.Printf("[ROUTER] Request: %s %s from %s\n", ctx.Method(), path, ctx.RemoteAddr())
|
||||
|
||||
// Special case: global status at /status
|
||||
if path == "/status" {
|
||||
rs.handleGlobalStatus(ctx)
|
||||
@ -40,26 +51,48 @@ func (rs *routerServer) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
matchedPrefix = prefix
|
||||
matchedStream = stream
|
||||
remainingPath = strings.TrimPrefix(path, prefix)
|
||||
// Ensure remaining path starts with / or is empty
|
||||
if remainingPath != "" && !strings.HasPrefix(remainingPath, "/") {
|
||||
remainingPath = "/" + remainingPath
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rs.routeMu.RUnlock()
|
||||
|
||||
if matchedStream == nil {
|
||||
rs.router.failedRequests.Add(1)
|
||||
rs.handleNotFound(ctx)
|
||||
return
|
||||
}
|
||||
|
||||
rs.router.routedRequests.Add(1)
|
||||
|
||||
// Route to stream's handler
|
||||
if matchedStream.HTTPServer != nil {
|
||||
// Save original path
|
||||
originalPath := string(ctx.URI().Path())
|
||||
|
||||
// Rewrite path to remove stream prefix
|
||||
if remainingPath == "" {
|
||||
// Default to stream path if no remaining path
|
||||
remainingPath = matchedStream.Config.HTTPServer.StreamPath
|
||||
}
|
||||
|
||||
fmt.Printf("[ROUTER] Routing to stream '%s': %s -> %s\n",
|
||||
matchedStream.Name, originalPath, remainingPath)
|
||||
|
||||
ctx.URI().SetPath(remainingPath)
|
||||
matchedStream.HTTPServer.RouteRequest(ctx)
|
||||
|
||||
// Restore original path
|
||||
ctx.URI().SetPath(originalPath)
|
||||
} else {
|
||||
ctx.SetStatusCode(fasthttp.StatusServiceUnavailable)
|
||||
ctx.SetContentType("application/json")
|
||||
json.NewEncoder(ctx).Encode(map[string]string{
|
||||
"error": "Stream HTTP server not available",
|
||||
"stream": matchedStream.Name,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -70,26 +103,37 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
|
||||
rs.routeMu.RLock()
|
||||
streams := make(map[string]interface{})
|
||||
for prefix, stream := range rs.routes {
|
||||
streams[stream.Name] = map[string]interface{}{
|
||||
streamStats := stream.GetStats()
|
||||
|
||||
// Add routing information
|
||||
streamStats["routing"] = map[string]interface{}{
|
||||
"path_prefix": prefix,
|
||||
"config": map[string]interface{}{
|
||||
"stream_path": stream.Config.HTTPServer.StreamPath,
|
||||
"status_path": stream.Config.HTTPServer.StatusPath,
|
||||
"endpoints": map[string]string{
|
||||
"stream": prefix + stream.Config.HTTPServer.StreamPath,
|
||||
"status": prefix + stream.Config.HTTPServer.StatusPath,
|
||||
},
|
||||
"stats": stream.GetStats(),
|
||||
}
|
||||
|
||||
streams[stream.Name] = streamStats
|
||||
}
|
||||
rs.routeMu.RUnlock()
|
||||
|
||||
// Get router stats
|
||||
routerStats := rs.router.GetStats()
|
||||
|
||||
status := map[string]interface{}{
|
||||
"service": "LogWisp Router",
|
||||
"version": version.Short(),
|
||||
"version": version.String(),
|
||||
"port": rs.port,
|
||||
"streams": streams,
|
||||
"total_streams": len(streams),
|
||||
"router": routerStats,
|
||||
"endpoints": map[string]string{
|
||||
"global_status": "/status",
|
||||
},
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(status)
|
||||
data, _ := json.MarshalIndent(status, "", " ")
|
||||
ctx.SetBody(data)
|
||||
}
|
||||
|
||||
@ -113,9 +157,11 @@ func (rs *routerServer) handleNotFound(ctx *fasthttp.RequestCtx) {
|
||||
|
||||
response := map[string]interface{}{
|
||||
"error": "Not Found",
|
||||
"requested_path": string(ctx.Path()),
|
||||
"available_routes": availableRoutes,
|
||||
"hint": "Use /status for global router status",
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(response)
|
||||
data, _ := json.MarshalIndent(response, "", " ")
|
||||
ctx.SetBody(data)
|
||||
}
|
||||
311
src/internal/ratelimit/limiter.go
Normal file
311
src/internal/ratelimit/limiter.go
Normal file
@ -0,0 +1,311 @@
|
||||
// FILE: src/internal/ratelimit/limiter.go
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/config"
|
||||
)
|
||||
|
||||
// Manages rate limiting for a stream
|
||||
type Limiter struct {
|
||||
config config.RateLimitConfig
|
||||
|
||||
// Per-IP limiters
|
||||
ipLimiters map[string]*ipLimiter
|
||||
ipMu sync.RWMutex
|
||||
|
||||
// Global limiter for the stream
|
||||
globalLimiter *TokenBucket
|
||||
|
||||
// Connection tracking
|
||||
ipConnections map[string]*atomic.Int32
|
||||
connMu sync.RWMutex
|
||||
|
||||
// Statistics
|
||||
totalRequests atomic.Uint64
|
||||
blockedRequests atomic.Uint64
|
||||
uniqueIPs atomic.Uint64
|
||||
|
||||
// Cleanup
|
||||
lastCleanup time.Time
|
||||
cleanupMu sync.Mutex
|
||||
}
|
||||
|
||||
type ipLimiter struct {
|
||||
bucket *TokenBucket
|
||||
lastSeen time.Time
|
||||
connections atomic.Int32
|
||||
}
|
||||
|
||||
// Creates a new rate limiter
|
||||
func New(cfg config.RateLimitConfig) *Limiter {
|
||||
if !cfg.Enabled {
|
||||
return nil
|
||||
}
|
||||
|
||||
l := &Limiter{
|
||||
config: cfg,
|
||||
ipLimiters: make(map[string]*ipLimiter),
|
||||
ipConnections: make(map[string]*atomic.Int32),
|
||||
lastCleanup: time.Now(),
|
||||
}
|
||||
|
||||
// Create global limiter if not using per-IP limiting
|
||||
if cfg.LimitBy == "global" {
|
||||
l.globalLimiter = NewTokenBucket(
|
||||
float64(cfg.BurstSize),
|
||||
cfg.RequestsPerSecond,
|
||||
)
|
||||
}
|
||||
|
||||
// Start cleanup goroutine
|
||||
go l.cleanupLoop()
|
||||
|
||||
return l
|
||||
}
|
||||
|
||||
// Checks if an HTTP request should be allowed
|
||||
func (l *Limiter) CheckHTTP(remoteAddr string) (allowed bool, statusCode int, message string) {
|
||||
if l == nil {
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
l.totalRequests.Add(1)
|
||||
|
||||
ip, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err != nil {
|
||||
// If we can't parse the IP, allow the request but log
|
||||
fmt.Printf("[RATELIMIT] Failed to parse remote addr %s: %v\n", remoteAddr, err)
|
||||
return true, 0, ""
|
||||
}
|
||||
|
||||
// Check connection limit for streaming endpoint
|
||||
if l.config.MaxConnectionsPerIP > 0 {
|
||||
l.connMu.RLock()
|
||||
counter, exists := l.ipConnections[ip]
|
||||
l.connMu.RUnlock()
|
||||
|
||||
if exists && counter.Load() >= int32(l.config.MaxConnectionsPerIP) {
|
||||
l.blockedRequests.Add(1)
|
||||
statusCode = l.config.ResponseCode
|
||||
if statusCode == 0 {
|
||||
statusCode = 429
|
||||
}
|
||||
message = "Connection limit exceeded"
|
||||
return false, statusCode, message
|
||||
}
|
||||
}
|
||||
|
||||
// Check rate limit
|
||||
allowed = l.checkLimit(ip)
|
||||
if !allowed {
|
||||
l.blockedRequests.Add(1)
|
||||
statusCode = l.config.ResponseCode
|
||||
if statusCode == 0 {
|
||||
statusCode = 429
|
||||
}
|
||||
message = l.config.ResponseMessage
|
||||
if message == "" {
|
||||
message = "Rate limit exceeded"
|
||||
}
|
||||
}
|
||||
|
||||
return allowed, statusCode, message
|
||||
}
|
||||
|
||||
// Checks if a TCP connection should be allowed
|
||||
func (l *Limiter) CheckTCP(remoteAddr net.Addr) bool {
|
||||
if l == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
l.totalRequests.Add(1)
|
||||
|
||||
// Extract IP from TCP addr
|
||||
tcpAddr, ok := remoteAddr.(*net.TCPAddr)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
|
||||
ip := tcpAddr.IP.String()
|
||||
allowed := l.checkLimit(ip)
|
||||
if !allowed {
|
||||
l.blockedRequests.Add(1)
|
||||
}
|
||||
|
||||
return allowed
|
||||
}
|
||||
|
||||
// Tracks a new connection for an IP
|
||||
func (l *Limiter) AddConnection(remoteAddr string) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ip, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
l.connMu.Lock()
|
||||
counter, exists := l.ipConnections[ip]
|
||||
if !exists {
|
||||
counter = &atomic.Int32{}
|
||||
l.ipConnections[ip] = counter
|
||||
}
|
||||
l.connMu.Unlock()
|
||||
|
||||
counter.Add(1)
|
||||
}
|
||||
|
||||
// Removes a connection for an IP
|
||||
func (l *Limiter) RemoveConnection(remoteAddr string) {
|
||||
if l == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ip, _, err := net.SplitHostPort(remoteAddr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
l.connMu.RLock()
|
||||
counter, exists := l.ipConnections[ip]
|
||||
l.connMu.RUnlock()
|
||||
|
||||
if exists {
|
||||
newCount := counter.Add(-1)
|
||||
if newCount <= 0 {
|
||||
// Clean up if no more connections
|
||||
l.connMu.Lock()
|
||||
if counter.Load() <= 0 {
|
||||
delete(l.ipConnections, ip)
|
||||
}
|
||||
l.connMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Returns rate limiter statistics
|
||||
func (l *Limiter) GetStats() map[string]interface{} {
|
||||
if l == nil {
|
||||
return map[string]interface{}{
|
||||
"enabled": false,
|
||||
}
|
||||
}
|
||||
|
||||
l.ipMu.RLock()
|
||||
activeIPs := len(l.ipLimiters)
|
||||
l.ipMu.RUnlock()
|
||||
|
||||
l.connMu.RLock()
|
||||
totalConnections := 0
|
||||
for _, counter := range l.ipConnections {
|
||||
totalConnections += int(counter.Load())
|
||||
}
|
||||
l.connMu.RUnlock()
|
||||
|
||||
return map[string]interface{}{
|
||||
"enabled": true,
|
||||
"total_requests": l.totalRequests.Load(),
|
||||
"blocked_requests": l.blockedRequests.Load(),
|
||||
"active_ips": activeIPs,
|
||||
"total_connections": totalConnections,
|
||||
"config": map[string]interface{}{
|
||||
"requests_per_second": l.config.RequestsPerSecond,
|
||||
"burst_size": l.config.BurstSize,
|
||||
"limit_by": l.config.LimitBy,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Performs the actual rate limit check
|
||||
func (l *Limiter) checkLimit(ip string) bool {
|
||||
// Maybe run cleanup
|
||||
l.maybeCleanup()
|
||||
|
||||
switch l.config.LimitBy {
|
||||
case "global":
|
||||
return l.globalLimiter.Allow()
|
||||
|
||||
case "ip", "":
|
||||
// Default to per-IP limiting
|
||||
l.ipMu.Lock()
|
||||
limiter, exists := l.ipLimiters[ip]
|
||||
if !exists {
|
||||
// Create new limiter for this IP
|
||||
limiter = &ipLimiter{
|
||||
bucket: NewTokenBucket(
|
||||
float64(l.config.BurstSize),
|
||||
l.config.RequestsPerSecond,
|
||||
),
|
||||
lastSeen: time.Now(),
|
||||
}
|
||||
l.ipLimiters[ip] = limiter
|
||||
l.uniqueIPs.Add(1)
|
||||
} else {
|
||||
limiter.lastSeen = time.Now()
|
||||
}
|
||||
l.ipMu.Unlock()
|
||||
|
||||
// Check connection limit if configured
|
||||
if l.config.MaxConnectionsPerIP > 0 {
|
||||
l.connMu.RLock()
|
||||
counter, exists := l.ipConnections[ip]
|
||||
l.connMu.RUnlock()
|
||||
|
||||
if exists && counter.Load() >= int32(l.config.MaxConnectionsPerIP) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return limiter.bucket.Allow()
|
||||
|
||||
default:
|
||||
// Unknown limit_by value, allow by default
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Runs cleanup if enough time has passed
|
||||
func (l *Limiter) maybeCleanup() {
|
||||
l.cleanupMu.Lock()
|
||||
defer l.cleanupMu.Unlock()
|
||||
|
||||
if time.Since(l.lastCleanup) < 30*time.Second {
|
||||
return
|
||||
}
|
||||
|
||||
l.lastCleanup = time.Now()
|
||||
go l.cleanup()
|
||||
}
|
||||
|
||||
// Removes stale IP limiters
|
||||
func (l *Limiter) cleanup() {
|
||||
staleTimeout := 5 * time.Minute
|
||||
now := time.Now()
|
||||
|
||||
l.ipMu.Lock()
|
||||
defer l.ipMu.Unlock()
|
||||
|
||||
for ip, limiter := range l.ipLimiters {
|
||||
if now.Sub(limiter.lastSeen) > staleTimeout {
|
||||
delete(l.ipLimiters, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Runs periodic cleanup
|
||||
func (l *Limiter) cleanupLoop() {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
|
||||
for range ticker.C {
|
||||
l.cleanup()
|
||||
}
|
||||
}
|
||||
53
src/internal/ratelimit/ratelimit.go
Normal file
53
src/internal/ratelimit/ratelimit.go
Normal file
@ -0,0 +1,53 @@
|
||||
// FILE: src/internal/ratelimit/ratelimit.go
|
||||
package ratelimit
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TokenBucket implements a token bucket rate limiter
|
||||
type TokenBucket struct {
|
||||
capacity float64
|
||||
tokens float64
|
||||
refillRate float64
|
||||
lastRefill time.Time
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewTokenBucket creates a new token bucket with given capacity and refill rate
|
||||
func NewTokenBucket(capacity float64, refillRate float64) *TokenBucket {
|
||||
return &TokenBucket{
|
||||
capacity: capacity,
|
||||
tokens: capacity,
|
||||
refillRate: refillRate,
|
||||
lastRefill: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// Allow attempts to consume one token, returns true if allowed
|
||||
func (tb *TokenBucket) Allow() bool {
|
||||
return tb.AllowN(1)
|
||||
}
|
||||
|
||||
// AllowN attempts to consume n tokens, returns true if allowed
|
||||
func (tb *TokenBucket) AllowN(n float64) bool {
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
|
||||
// Refill tokens based on time elapsed
|
||||
now := time.Now()
|
||||
elapsed := now.Sub(tb.lastRefill).Seconds()
|
||||
tb.tokens += elapsed * tb.refillRate
|
||||
if tb.tokens > tb.capacity {
|
||||
tb.tokens = tb.capacity
|
||||
}
|
||||
tb.lastRefill = now
|
||||
|
||||
// Check if we have enough tokens
|
||||
if tb.tokens >= n {
|
||||
tb.tokens -= n
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@ -14,6 +14,7 @@ import (
|
||||
"github.com/valyala/fasthttp"
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/monitor"
|
||||
"logwisp/src/internal/ratelimit"
|
||||
"logwisp/src/internal/version"
|
||||
)
|
||||
|
||||
@ -33,6 +34,9 @@ type HTTPStreamer struct {
|
||||
|
||||
// For router integration
|
||||
standalone bool
|
||||
|
||||
// Rate limiting
|
||||
rateLimiter *ratelimit.Limiter
|
||||
}
|
||||
|
||||
func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTPStreamer {
|
||||
@ -46,7 +50,7 @@ func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTP
|
||||
statusPath = "/status"
|
||||
}
|
||||
|
||||
return &HTTPStreamer{
|
||||
h := &HTTPStreamer{
|
||||
logChan: logChan,
|
||||
config: cfg,
|
||||
startTime: time.Now(),
|
||||
@ -55,9 +59,16 @@ func NewHTTPStreamer(logChan chan monitor.LogEntry, cfg config.HTTPConfig) *HTTP
|
||||
statusPath: statusPath,
|
||||
standalone: true, // Default to standalone mode
|
||||
}
|
||||
|
||||
// Initialize rate limiter if configured
|
||||
if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
|
||||
h.rateLimiter = ratelimit.New(*cfg.RateLimit)
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
// SetRouterMode configures the streamer for use with a router
|
||||
// Configures the streamer for use with a router
|
||||
func (h *HTTPStreamer) SetRouterMode() {
|
||||
h.standalone = false
|
||||
}
|
||||
@ -116,6 +127,18 @@ func (h *HTTPStreamer) RouteRequest(ctx *fasthttp.RequestCtx) {
|
||||
}
|
||||
|
||||
func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
// Check rate limit first
|
||||
remoteAddr := ctx.RemoteAddr().String()
|
||||
if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed {
|
||||
ctx.SetStatusCode(statusCode)
|
||||
ctx.SetContentType("application/json")
|
||||
json.NewEncoder(ctx).Encode(map[string]interface{}{
|
||||
"error": message,
|
||||
"retry_after": "60", // seconds
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
path := string(ctx.Path())
|
||||
|
||||
switch path {
|
||||
@ -135,6 +158,13 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
}
|
||||
|
||||
func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
|
||||
// Track connection for rate limiting
|
||||
remoteAddr := ctx.RemoteAddr().String()
|
||||
if h.rateLimiter != nil {
|
||||
h.rateLimiter.AddConnection(remoteAddr)
|
||||
defer h.rateLimiter.RemoveConnection(remoteAddr)
|
||||
}
|
||||
|
||||
// Set SSE headers
|
||||
ctx.Response.Header.Set("Content-Type", "text/event-stream")
|
||||
ctx.Response.Header.Set("Cache-Control", "no-cache")
|
||||
@ -285,6 +315,15 @@ func (h *HTTPStreamer) formatHeartbeat() string {
|
||||
func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
var rateLimitStats interface{}
|
||||
if h.rateLimiter != nil {
|
||||
rateLimitStats = h.rateLimiter.GetStats()
|
||||
} else {
|
||||
rateLimitStats = map[string]interface{}{
|
||||
"enabled": false,
|
||||
}
|
||||
}
|
||||
|
||||
status := map[string]interface{}{
|
||||
"service": "LogWisp",
|
||||
"version": version.Short(),
|
||||
@ -309,9 +348,7 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
|
||||
"ssl": map[string]bool{
|
||||
"enabled": h.config.SSL != nil && h.config.SSL.Enabled,
|
||||
},
|
||||
"rate_limit": map[string]bool{
|
||||
"enabled": h.config.RateLimit != nil && h.config.RateLimit.Enabled,
|
||||
},
|
||||
"rate_limit": rateLimitStats,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,7 @@ package stream
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
|
||||
"github.com/panjf2000/gnet/v2"
|
||||
@ -17,10 +18,34 @@ type tcpServer struct {
|
||||
func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
|
||||
// Store engine reference for shutdown
|
||||
s.streamer.engine = &eng
|
||||
fmt.Printf("[TCP DEBUG] Server booted on port %d\n", s.streamer.config.Port)
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
|
||||
// Debug: Log all connection attempts
|
||||
fmt.Printf("[TCP DEBUG] Connection attempt from %s\n", c.RemoteAddr())
|
||||
|
||||
// Check rate limit
|
||||
if s.streamer.rateLimiter != nil {
|
||||
// Parse the remote address to get proper net.Addr
|
||||
remoteStr := c.RemoteAddr().String()
|
||||
tcpAddr, err := net.ResolveTCPAddr("tcp", remoteStr)
|
||||
if err != nil {
|
||||
fmt.Printf("[TCP DEBUG] Failed to parse address %s: %v\n", remoteStr, err)
|
||||
return nil, gnet.Close
|
||||
}
|
||||
|
||||
if !s.streamer.rateLimiter.CheckTCP(tcpAddr) {
|
||||
fmt.Printf("[TCP DEBUG] Rate limited connection from %s\n", remoteStr)
|
||||
// Silently close connection when rate limited
|
||||
return nil, gnet.Close
|
||||
}
|
||||
|
||||
// Track connection
|
||||
s.streamer.rateLimiter.AddConnection(remoteStr)
|
||||
}
|
||||
|
||||
s.connections.Store(c, struct{}{})
|
||||
|
||||
oldCount := s.streamer.activeConns.Load()
|
||||
@ -34,6 +59,11 @@ func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
|
||||
func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
|
||||
s.connections.Delete(c)
|
||||
|
||||
// Remove connection tracking
|
||||
if s.streamer.rateLimiter != nil {
|
||||
s.streamer.rateLimiter.RemoveConnection(c.RemoteAddr().String())
|
||||
}
|
||||
|
||||
oldCount := s.streamer.activeConns.Load()
|
||||
newCount := s.streamer.activeConns.Add(-1)
|
||||
fmt.Printf("[TCP ATOMIC] OnClose: %d -> %d (expected: %d)\n", oldCount, newCount, oldCount-1)
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
"github.com/panjf2000/gnet/v2"
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/monitor"
|
||||
"logwisp/src/internal/ratelimit"
|
||||
)
|
||||
|
||||
type TCPStreamer struct {
|
||||
@ -23,15 +24,22 @@ type TCPStreamer struct {
|
||||
startTime time.Time
|
||||
engine *gnet.Engine
|
||||
wg sync.WaitGroup
|
||||
rateLimiter *ratelimit.Limiter
|
||||
}
|
||||
|
||||
func NewTCPStreamer(logChan chan monitor.LogEntry, cfg config.TCPConfig) *TCPStreamer {
|
||||
return &TCPStreamer{
|
||||
t := &TCPStreamer{
|
||||
logChan: logChan,
|
||||
config: cfg,
|
||||
done: make(chan struct{}),
|
||||
startTime: time.Now(),
|
||||
}
|
||||
|
||||
if cfg.RateLimit != nil && cfg.RateLimit.Enabled {
|
||||
t.rateLimiter = ratelimit.New(*cfg.RateLimit)
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *TCPStreamer) Start() error {
|
||||
|
||||
Reference in New Issue
Block a user