v0.1.9 pre-stream log regex filtering added
This commit is contained in:
125
README.md
125
README.md
@ -4,7 +4,7 @@
|
|||||||
<img src="assets/logwisp-logo.svg" alt="LogWisp Logo" width="200"/>
|
<img src="assets/logwisp-logo.svg" alt="LogWisp Logo" width="200"/>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring, rotation detection, and rate limiting.
|
A high-performance log streaming service with multi-stream architecture, supporting both TCP and HTTP/SSE protocols with real-time file monitoring, rotation detection, regex-based filtering, and rate limiting.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
@ -12,11 +12,12 @@ A high-performance log streaming service with multi-stream architecture, support
|
|||||||
- **Dual Protocol Support**: TCP (raw streaming) and HTTP/SSE (browser-friendly)
|
- **Dual Protocol Support**: TCP (raw streaming) and HTTP/SSE (browser-friendly)
|
||||||
- **Real-time Monitoring**: Instant updates with per-stream configurable check intervals
|
- **Real-time Monitoring**: Instant updates with per-stream configurable check intervals
|
||||||
- **File Rotation Detection**: Automatic detection and handling of log rotation
|
- **File Rotation Detection**: Automatic detection and handling of log rotation
|
||||||
|
- **Regex-based Filtering**: Include/exclude patterns with AND/OR logic per stream
|
||||||
- **Path-based Routing**: Optional HTTP router for consolidated access
|
- **Path-based Routing**: Optional HTTP router for consolidated access
|
||||||
- **Rate Limiting**: Per-IP or global rate limiting with token bucket algorithm
|
- **Rate Limiting**: Per-IP or global rate limiting with token bucket algorithm
|
||||||
- **Connection Limiting**: Configurable concurrent connection limits per IP
|
- **Connection Limiting**: Configurable concurrent connection limits per IP
|
||||||
- **Per-Stream Configuration**: Independent settings including check intervals and rate limits
|
- **Per-Stream Configuration**: Independent settings including check intervals, filters, and rate limits
|
||||||
- **Connection Statistics**: Real-time monitoring of active connections and rate limit metrics
|
- **Connection Statistics**: Real-time monitoring of active connections, filter, and rate limit metrics
|
||||||
- **Flexible Targets**: Monitor individual files or entire directories
|
- **Flexible Targets**: Monitor individual files or entire directories
|
||||||
- **Version Management**: Git tag-based versioning with build information
|
- **Version Management**: Git tag-based versioning with build information
|
||||||
- **Configurable Heartbeats**: Keep connections alive with customizable formats
|
- **Configurable Heartbeats**: Keep connections alive with customizable formats
|
||||||
@ -49,11 +50,13 @@ LogWisp uses a service-oriented architecture where each stream is an independent
|
|||||||
LogStream Service
|
LogStream Service
|
||||||
├── Stream["app-logs"]
|
├── Stream["app-logs"]
|
||||||
│ ├── Monitor (watches files)
|
│ ├── Monitor (watches files)
|
||||||
|
│ ├── Filter Chain (optional)
|
||||||
│ ├── Rate Limiter (optional)
|
│ ├── Rate Limiter (optional)
|
||||||
│ ├── TCP Server (optional)
|
│ ├── TCP Server (optional)
|
||||||
│ └── HTTP Server (optional)
|
│ └── HTTP Server (optional)
|
||||||
├── Stream["system-logs"]
|
├── Stream["system-logs"]
|
||||||
│ ├── Monitor
|
│ ├── Monitor
|
||||||
|
│ ├── Filter Chain (optional)
|
||||||
│ ├── Rate Limiter (optional)
|
│ ├── Rate Limiter (optional)
|
||||||
│ └── HTTP Server
|
│ └── HTTP Server
|
||||||
└── HTTP Router (optional, for path-based routing)
|
└── HTTP Router (optional, for path-based routing)
|
||||||
@ -78,6 +81,16 @@ targets = [
|
|||||||
{ path = "/var/log/myapp/app.log", is_file = true }
|
{ path = "/var/log/myapp/app.log", is_file = true }
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Filter configuration (optional)
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "include" # Only show matching logs
|
||||||
|
logic = "or" # Match any pattern
|
||||||
|
patterns = [
|
||||||
|
"(?i)error", # Case-insensitive error
|
||||||
|
"(?i)warn", # Case-insensitive warning
|
||||||
|
"(?i)fatal" # Fatal errors
|
||||||
|
]
|
||||||
|
|
||||||
[streams.httpserver]
|
[streams.httpserver]
|
||||||
enabled = true
|
enabled = true
|
||||||
port = 8080
|
port = 8080
|
||||||
@ -115,6 +128,11 @@ targets = [
|
|||||||
{ path = "/var/log/auth.log", is_file = true }
|
{ path = "/var/log/auth.log", is_file = true }
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Exclude debug logs
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "exclude"
|
||||||
|
patterns = ["DEBUG", "TRACE"]
|
||||||
|
|
||||||
[streams.tcpserver]
|
[streams.tcpserver]
|
||||||
enabled = true
|
enabled = true
|
||||||
port = 9090
|
port = 9090
|
||||||
@ -150,6 +168,44 @@ Monitor targets support both files and directories:
|
|||||||
{ path = "./logs", pattern = "*.log", is_file = false }
|
{ path = "./logs", pattern = "*.log", is_file = false }
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Filter Configuration
|
||||||
|
|
||||||
|
Control which logs are streamed using regex patterns:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
# Include filter - only matching logs pass
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "include"
|
||||||
|
logic = "or" # Match ANY pattern
|
||||||
|
patterns = [
|
||||||
|
"ERROR",
|
||||||
|
"WARN",
|
||||||
|
"CRITICAL"
|
||||||
|
]
|
||||||
|
|
||||||
|
# Exclude filter - matching logs are dropped
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "exclude"
|
||||||
|
logic = "or" # Drop if ANY pattern matches
|
||||||
|
patterns = [
|
||||||
|
"DEBUG",
|
||||||
|
"healthcheck",
|
||||||
|
"/metrics"
|
||||||
|
]
|
||||||
|
|
||||||
|
# Complex filter with AND logic
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "include"
|
||||||
|
logic = "and" # Must match ALL patterns
|
||||||
|
patterns = [
|
||||||
|
"database", # Must contain "database"
|
||||||
|
"error", # AND must contain "error"
|
||||||
|
"connection" # AND must contain "connection"
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Multiple filters are applied sequentially - all must pass for a log to be streamed.
|
||||||
|
|
||||||
### Check Interval Configuration
|
### Check Interval Configuration
|
||||||
|
|
||||||
Each stream can have its own check interval based on log update frequency:
|
Each stream can have its own check interval based on log update frequency:
|
||||||
@ -235,7 +291,7 @@ All HTTP streams share ports with path-based routing:
|
|||||||
# Connect to a stream
|
# Connect to a stream
|
||||||
curl -N http://localhost:8080/stream
|
curl -N http://localhost:8080/stream
|
||||||
|
|
||||||
# Check stream status (includes rate limit stats)
|
# Check stream status (includes filter and rate limit stats)
|
||||||
curl http://localhost:8080/status
|
curl http://localhost:8080/status
|
||||||
|
|
||||||
# With authentication (when implemented)
|
# With authentication (when implemented)
|
||||||
@ -329,6 +385,21 @@ All log entries are streamed as JSON:
|
|||||||
"total_entries": 15420,
|
"total_entries": 15420,
|
||||||
"dropped_entries": 0
|
"dropped_entries": 0
|
||||||
},
|
},
|
||||||
|
"filters": {
|
||||||
|
"filter_count": 2,
|
||||||
|
"total_processed": 15420,
|
||||||
|
"total_passed": 1234,
|
||||||
|
"filters": [
|
||||||
|
{
|
||||||
|
"type": "include",
|
||||||
|
"logic": "or",
|
||||||
|
"pattern_count": 3,
|
||||||
|
"total_processed": 15420,
|
||||||
|
"total_matched": 1234,
|
||||||
|
"total_dropped": 0
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"features": {
|
"features": {
|
||||||
"rate_limit": {
|
"rate_limit": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
@ -352,6 +423,7 @@ LogWisp provides comprehensive statistics at multiple levels:
|
|||||||
|
|
||||||
- **Per-Stream Stats**: Monitor performance, connection counts, data throughput
|
- **Per-Stream Stats**: Monitor performance, connection counts, data throughput
|
||||||
- **Per-Watcher Stats**: File size, position, entries read, rotation count
|
- **Per-Watcher Stats**: File size, position, entries read, rotation count
|
||||||
|
- **Filter Stats**: Processed entries, matched patterns, dropped logs
|
||||||
- **Rate Limit Stats**: Total requests, blocked requests, active IPs
|
- **Rate Limit Stats**: Total requests, blocked requests, active IPs
|
||||||
- **Global Stats**: Aggregated view of all streams (in router mode)
|
- **Global Stats**: Aggregated view of all streams (in router mode)
|
||||||
|
|
||||||
@ -365,6 +437,21 @@ Access statistics via status endpoints or watch the console output:
|
|||||||
|
|
||||||
## Advanced Features
|
## Advanced Features
|
||||||
|
|
||||||
|
### Log Filtering
|
||||||
|
|
||||||
|
LogWisp implements powerful regex-based filtering:
|
||||||
|
- **Include Filters**: Whitelist patterns - only matching logs pass
|
||||||
|
- **Exclude Filters**: Blacklist patterns - matching logs are dropped
|
||||||
|
- **Logic Options**: OR (match any) or AND (match all) for pattern combinations
|
||||||
|
- **Filter Chains**: Multiple filters applied sequentially
|
||||||
|
- **Performance**: Patterns compiled once at startup for efficiency
|
||||||
|
|
||||||
|
Filter statistics help monitor effectiveness:
|
||||||
|
```bash
|
||||||
|
# Watch filter statistics
|
||||||
|
watch -n 1 'curl -s http://localhost:8080/status | jq .filters'
|
||||||
|
```
|
||||||
|
|
||||||
### Rate Limiting
|
### Rate Limiting
|
||||||
|
|
||||||
LogWisp implements token bucket rate limiting with:
|
LogWisp implements token bucket rate limiting with:
|
||||||
@ -423,6 +510,12 @@ check_interval_ms = 60000 # Check every minute
|
|||||||
- Configure per-stream based on expected update frequency
|
- Configure per-stream based on expected update frequency
|
||||||
- Use 10000ms+ for archival or slowly updating logs
|
- Use 10000ms+ for archival or slowly updating logs
|
||||||
|
|
||||||
|
### Filter Optimization
|
||||||
|
- Place most selective filters first
|
||||||
|
- Use simple patterns when possible
|
||||||
|
- Consider combining patterns: `"ERROR|WARN"` vs separate patterns
|
||||||
|
- Monitor filter statistics to identify bottlenecks
|
||||||
|
|
||||||
### Rate Limiting
|
### Rate Limiting
|
||||||
- `requests_per_second`: Balance between protection and availability
|
- `requests_per_second`: Balance between protection and availability
|
||||||
- `burst_size`: Set to 2-3x the per-second rate for traffic spikes
|
- `burst_size`: Set to 2-3x the per-second rate for traffic spikes
|
||||||
@ -547,11 +640,19 @@ services:
|
|||||||
|
|
||||||
### Current Implementation
|
### Current Implementation
|
||||||
- Read-only file access
|
- Read-only file access
|
||||||
|
- Regex pattern validation at startup
|
||||||
- Rate limiting for DDoS protection
|
- Rate limiting for DDoS protection
|
||||||
- Connection limits to prevent resource exhaustion
|
- Connection limits to prevent resource exhaustion
|
||||||
- No authentication (placeholder configuration only)
|
- No authentication (placeholder configuration only)
|
||||||
- No TLS/SSL support (placeholder configuration only)
|
- No TLS/SSL support (placeholder configuration only)
|
||||||
|
|
||||||
|
### Filter Security
|
||||||
|
⚠️ **SECURITY**: Be aware of potential ReDoS (Regular Expression Denial of Service) attacks:
|
||||||
|
- Complex nested patterns can cause CPU spikes
|
||||||
|
- Patterns are validated at startup but not for complexity
|
||||||
|
- Monitor filter processing time in production
|
||||||
|
- Consider pattern complexity limits for public-facing streams
|
||||||
|
|
||||||
### Planned Security Features
|
### Planned Security Features
|
||||||
- **Authentication**: Basic, Bearer/JWT, mTLS
|
- **Authentication**: Basic, Bearer/JWT, mTLS
|
||||||
- **TLS/SSL**: For both HTTP and TCP streams
|
- **TLS/SSL**: For both HTTP and TCP streams
|
||||||
@ -566,6 +667,8 @@ services:
|
|||||||
4. Place behind a reverse proxy for production HTTPS
|
4. Place behind a reverse proxy for production HTTPS
|
||||||
5. Monitor rate limit statistics for potential attacks
|
5. Monitor rate limit statistics for potential attacks
|
||||||
6. Regularly update dependencies
|
6. Regularly update dependencies
|
||||||
|
7. Test filter patterns for performance impact
|
||||||
|
8. Limit regex complexity in production environments
|
||||||
|
|
||||||
### Rate Limiting Best Practices
|
### Rate Limiting Best Practices
|
||||||
- Start with conservative limits and adjust based on monitoring
|
- Start with conservative limits and adjust based on monitoring
|
||||||
@ -576,6 +679,13 @@ services:
|
|||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Filter Issues
|
||||||
|
1. Check filter statistics to see matched/dropped counts
|
||||||
|
2. Test patterns with sample log entries
|
||||||
|
3. Verify filter type (include vs exclude)
|
||||||
|
4. Check filter logic (or vs and)
|
||||||
|
5. Monitor CPU usage for complex patterns
|
||||||
|
|
||||||
### Rate Limit Issues
|
### Rate Limit Issues
|
||||||
1. Check rate limit statistics in status endpoint
|
1. Check rate limit statistics in status endpoint
|
||||||
2. Verify appropriate `requests_per_second` for your use case
|
2. Verify appropriate `requests_per_second` for your use case
|
||||||
@ -588,6 +698,7 @@ services:
|
|||||||
3. Ensure files match the specified patterns
|
3. Ensure files match the specified patterns
|
||||||
4. Check monitor statistics in status endpoint
|
4. Check monitor statistics in status endpoint
|
||||||
5. Verify check_interval_ms is appropriate for log update frequency
|
5. Verify check_interval_ms is appropriate for log update frequency
|
||||||
|
6. Review filter configuration - logs might be filtered out
|
||||||
|
|
||||||
### High Memory Usage
|
### High Memory Usage
|
||||||
1. Reduce buffer sizes in configuration
|
1. Reduce buffer sizes in configuration
|
||||||
@ -595,6 +706,7 @@ services:
|
|||||||
3. Enable rate limiting to prevent connection floods
|
3. Enable rate limiting to prevent connection floods
|
||||||
4. Increase check interval for less critical logs
|
4. Increase check interval for less critical logs
|
||||||
5. Use TCP instead of HTTP for high-volume streams
|
5. Use TCP instead of HTTP for high-volume streams
|
||||||
|
6. Check for complex regex patterns causing backtracking
|
||||||
|
|
||||||
### Connection Drops
|
### Connection Drops
|
||||||
1. Check heartbeat configuration
|
1. Check heartbeat configuration
|
||||||
@ -627,8 +739,9 @@ Contributions are welcome! Please read our contributing guidelines and submit pu
|
|||||||
- [x] Version management
|
- [x] Version management
|
||||||
- [x] Configurable heartbeats
|
- [x] Configurable heartbeats
|
||||||
- [x] Rate and connection limiting
|
- [x] Rate and connection limiting
|
||||||
- [ ] Log filtering and transformation
|
- [x] Regex-based log filtering
|
||||||
- [ ] Configurable logging support
|
- [ ] Log transformation (field extraction, formatting)
|
||||||
|
- [ ] Configurable logging/stdout support
|
||||||
- [ ] Authentication (Basic, JWT, mTLS)
|
- [ ] Authentication (Basic, JWT, mTLS)
|
||||||
- [ ] TLS/SSL support
|
- [ ] TLS/SSL support
|
||||||
- [ ] Prometheus metrics export
|
- [ ] Prometheus metrics export
|
||||||
|
|||||||
@ -30,6 +30,16 @@ targets = [
|
|||||||
{ path = "./", pattern = "*.log", is_file = false },
|
{ path = "./", pattern = "*.log", is_file = false },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Filter configuration (optional) - controls which logs are streamed
|
||||||
|
# Multiple filters are applied sequentially - all must pass
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include" # "include" (whitelist) or "exclude" (blacklist)
|
||||||
|
# logic = "or" # "or" (match any) or "and" (match all)
|
||||||
|
# patterns = [
|
||||||
|
# "(?i)error", # Case-insensitive error matching
|
||||||
|
# "(?i)warn" # Case-insensitive warning matching
|
||||||
|
# ]
|
||||||
|
|
||||||
# HTTP Server configuration (SSE/Server-Sent Events)
|
# HTTP Server configuration (SSE/Server-Sent Events)
|
||||||
[streams.httpserver]
|
[streams.httpserver]
|
||||||
enabled = true
|
enabled = true
|
||||||
@ -57,7 +67,7 @@ enabled = false
|
|||||||
# max_connections_per_ip = 5 # Max SSE connections per IP
|
# max_connections_per_ip = 5 # Max SSE connections per IP
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Example: Application Logs Stream
|
# Example: Application Logs Stream with Error Filtering
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# [[streams]]
|
# [[streams]]
|
||||||
# name = "app"
|
# name = "app"
|
||||||
@ -71,6 +81,28 @@ enabled = false
|
|||||||
# { path = "/var/log/myapp/app.log", is_file = true },
|
# { path = "/var/log/myapp/app.log", is_file = true },
|
||||||
# ]
|
# ]
|
||||||
#
|
#
|
||||||
|
# # Filter 1: Include only errors and warnings
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# logic = "or" # Match ANY of these patterns
|
||||||
|
# patterns = [
|
||||||
|
# "(?i)\\berror\\b", # Word boundary error (case-insensitive)
|
||||||
|
# "(?i)\\bwarn(ing)?\\b", # warn or warning
|
||||||
|
# "(?i)\\bfatal\\b", # fatal
|
||||||
|
# "(?i)\\bcritical\\b", # critical
|
||||||
|
# "(?i)exception", # exception anywhere
|
||||||
|
# "(?i)fail(ed|ure)?", # fail, failed, failure
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Filter 2: Exclude health check noise
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "exclude"
|
||||||
|
# patterns = [
|
||||||
|
# "/health",
|
||||||
|
# "/metrics",
|
||||||
|
# "GET /ping"
|
||||||
|
# ]
|
||||||
|
#
|
||||||
# [streams.httpserver]
|
# [streams.httpserver]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
# port = 8081 # Different port for each stream
|
# port = 8081 # Different port for each stream
|
||||||
@ -95,7 +127,7 @@ enabled = false
|
|||||||
# max_connections_per_ip = 10
|
# max_connections_per_ip = 10
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Example: System Logs Stream (TCP + HTTP)
|
# Example: System Logs Stream (TCP + HTTP) with Security Filtering
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# [[streams]]
|
# [[streams]]
|
||||||
# name = "system"
|
# name = "system"
|
||||||
@ -108,6 +140,21 @@ enabled = false
|
|||||||
# { path = "/var/log/kern.log", is_file = true },
|
# { path = "/var/log/kern.log", is_file = true },
|
||||||
# ]
|
# ]
|
||||||
#
|
#
|
||||||
|
# # Include only security-relevant logs
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# logic = "or"
|
||||||
|
# patterns = [
|
||||||
|
# "(?i)auth",
|
||||||
|
# "(?i)sudo",
|
||||||
|
# "(?i)ssh",
|
||||||
|
# "(?i)login",
|
||||||
|
# "(?i)permission",
|
||||||
|
# "(?i)denied",
|
||||||
|
# "(?i)unauthorized",
|
||||||
|
# "kernel:.*audit"
|
||||||
|
# ]
|
||||||
|
#
|
||||||
# # TCP Server for high-performance streaming
|
# # TCP Server for high-performance streaming
|
||||||
# [streams.tcpserver]
|
# [streams.tcpserver]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
@ -137,7 +184,7 @@ enabled = false
|
|||||||
# status_path = "/status"
|
# status_path = "/status"
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Example: High-Volume Debug Logs
|
# Example: High-Volume Debug Logs with Filtering
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# [[streams]]
|
# [[streams]]
|
||||||
# name = "debug"
|
# name = "debug"
|
||||||
@ -148,6 +195,25 @@ enabled = false
|
|||||||
# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
|
# { path = "/tmp/debug", pattern = "*.debug", is_file = false },
|
||||||
# ]
|
# ]
|
||||||
#
|
#
|
||||||
|
# # Exclude verbose debug output
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "exclude"
|
||||||
|
# patterns = [
|
||||||
|
# "TRACE",
|
||||||
|
# "VERBOSE",
|
||||||
|
# "entering function",
|
||||||
|
# "exiting function",
|
||||||
|
# "memory dump"
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Include only specific modules
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# patterns = [
|
||||||
|
# "module:(api|database|auth)",
|
||||||
|
# "component:(router|handler)"
|
||||||
|
# ]
|
||||||
|
#
|
||||||
# [streams.httpserver]
|
# [streams.httpserver]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
# port = 8083
|
# port = 8083
|
||||||
@ -168,31 +234,78 @@ enabled = false
|
|||||||
# max_connections_per_ip = 1 # One connection per IP
|
# max_connections_per_ip = 1 # One connection per IP
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Example: Archived Logs (Slow Monitoring)
|
# Example: Database Logs with Complex Filtering
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# [[streams]]
|
# [[streams]]
|
||||||
# name = "archive"
|
# name = "database"
|
||||||
#
|
#
|
||||||
# [streams.monitor]
|
# [streams.monitor]
|
||||||
# check_interval_ms = 60000 # Check once per minute
|
# check_interval_ms = 200
|
||||||
# targets = [
|
# targets = [
|
||||||
# { path = "/var/log/archive", pattern = "*.gz", is_file = false },
|
# { path = "/var/log/postgresql", pattern = "*.log", is_file = false },
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Complex AND filter - must match all patterns
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# logic = "and" # Must match ALL patterns
|
||||||
|
# patterns = [
|
||||||
|
# "(?i)error|fail", # Must contain error or fail
|
||||||
|
# "(?i)connection|query", # AND must be about connections or queries
|
||||||
|
# "(?i)timeout|deadlock" # AND must involve timeout or deadlock
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Exclude routine maintenance
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "exclude"
|
||||||
|
# patterns = [
|
||||||
|
# "VACUUM",
|
||||||
|
# "ANALYZE",
|
||||||
|
# "checkpoint"
|
||||||
# ]
|
# ]
|
||||||
#
|
#
|
||||||
# [streams.tcpserver]
|
# [streams.tcpserver]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
# port = 9091
|
# port = 9091
|
||||||
# buffer_size = 500 # Small buffer for archived logs
|
# buffer_size = 2000
|
||||||
#
|
|
||||||
# # Infrequent heartbeat
|
|
||||||
# [streams.tcpserver.heartbeat]
|
|
||||||
# enabled = true
|
|
||||||
# interval_seconds = 300 # Every 5 minutes
|
|
||||||
# include_timestamp = false
|
|
||||||
# include_stats = false
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Example: Security/Audit Logs with Strict Limits
|
# Example: API Access Logs with Pattern Extraction
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# [[streams]]
|
||||||
|
# name = "api-access"
|
||||||
|
#
|
||||||
|
# [streams.monitor]
|
||||||
|
# check_interval_ms = 100
|
||||||
|
# targets = [
|
||||||
|
# { path = "/var/log/nginx/access.log", is_file = true },
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Include only API endpoints
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# patterns = [
|
||||||
|
# '"/api/v[0-9]+/', # API versioned endpoints
|
||||||
|
# '"(GET|POST|PUT|DELETE) /api/' # API requests
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Exclude specific status codes
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "exclude"
|
||||||
|
# patterns = [
|
||||||
|
# '" 200 ', # Success responses
|
||||||
|
# '" 204 ', # No content
|
||||||
|
# '" 304 ', # Not modified
|
||||||
|
# 'OPTIONS ' # CORS preflight
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# [streams.httpserver]
|
||||||
|
# enabled = true
|
||||||
|
# port = 8084
|
||||||
|
# buffer_size = 3000
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------------------
|
||||||
|
# Example: Security/Audit Logs with Strict Filtering
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# [[streams]]
|
# [[streams]]
|
||||||
# name = "security"
|
# name = "security"
|
||||||
@ -203,6 +316,25 @@ enabled = false
|
|||||||
# { path = "/var/log/audit", pattern = "audit.log*", is_file = false },
|
# { path = "/var/log/audit", pattern = "audit.log*", is_file = false },
|
||||||
# ]
|
# ]
|
||||||
#
|
#
|
||||||
|
# # Security-focused patterns
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# logic = "or"
|
||||||
|
# patterns = [
|
||||||
|
# "type=USER_AUTH",
|
||||||
|
# "type=USER_LOGIN",
|
||||||
|
# "type=USER_LOGOUT",
|
||||||
|
# "type=USER_ERR",
|
||||||
|
# "type=CRED_", # All credential operations
|
||||||
|
# "type=PRIV_", # All privilege operations
|
||||||
|
# "type=ANOM_", # All anomalies
|
||||||
|
# "type=RESP_", # All responses
|
||||||
|
# "failed|failure",
|
||||||
|
# "denied|unauthorized",
|
||||||
|
# "violation",
|
||||||
|
# "attack|intrusion"
|
||||||
|
# ]
|
||||||
|
#
|
||||||
# [streams.httpserver]
|
# [streams.httpserver]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
# port = 8443 # HTTPS port (for future TLS)
|
# port = 8443 # HTTPS port (for future TLS)
|
||||||
@ -235,29 +367,67 @@ enabled = false
|
|||||||
# # realm = "Security Logs"
|
# # realm = "Security Logs"
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# Example: Public API Logs with Global Rate Limiting
|
# Example: Multi-Application Logs with Service Filtering
|
||||||
# ------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------
|
||||||
# [[streams]]
|
# [[streams]]
|
||||||
# name = "api-public"
|
# name = "microservices"
|
||||||
#
|
#
|
||||||
# [streams.monitor]
|
# [streams.monitor]
|
||||||
# check_interval_ms = 100
|
# check_interval_ms = 100
|
||||||
# targets = [
|
# targets = [
|
||||||
# { path = "/var/log/api", pattern = "access.log*", is_file = false },
|
# { path = "/var/log/containers", pattern = "*.log", is_file = false },
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Filter by service name
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# patterns = [
|
||||||
|
# "service=(api|auth|user|order)", # Specific services
|
||||||
|
# "pod=(api|auth|user|order)-" # Kubernetes pods
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# # Exclude Kubernetes noise
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "exclude"
|
||||||
|
# patterns = [
|
||||||
|
# "kube-system",
|
||||||
|
# "kube-proxy",
|
||||||
|
# "Readiness probe",
|
||||||
|
# "Liveness probe"
|
||||||
# ]
|
# ]
|
||||||
#
|
#
|
||||||
# [streams.httpserver]
|
# [streams.httpserver]
|
||||||
# enabled = true
|
# enabled = true
|
||||||
# port = 8084
|
# port = 8085
|
||||||
# buffer_size = 2000
|
# buffer_size = 5000
|
||||||
|
|
||||||
|
# ==============================================================================
|
||||||
|
# FILTER PATTERN EXAMPLES
|
||||||
|
# ==============================================================================
|
||||||
#
|
#
|
||||||
# # Global rate limiting (all clients share limit)
|
# Basic Patterns:
|
||||||
# [streams.httpserver.rate_limit]
|
# - "ERROR" # Exact match
|
||||||
# enabled = true
|
# - "(?i)error" # Case-insensitive
|
||||||
# requests_per_second = 100.0 # 100 req/s total
|
# - "\\berror\\b" # Word boundary (won't match "errorCode")
|
||||||
# burst_size = 200
|
# - "error|warn|fatal" # Multiple options
|
||||||
# limit_by = "global" # All clients share this limit
|
#
|
||||||
# max_total_connections = 50 # Max 50 connections total
|
# Complex Patterns:
|
||||||
|
# - "^\\[ERROR\\]" # Line starts with [ERROR]
|
||||||
|
# - "status=[4-5][0-9]{2}" # HTTP 4xx or 5xx status codes
|
||||||
|
# - "duration>[0-9]{4}ms" # Duration over 999ms
|
||||||
|
# - "user_id=\"[^\"]+\"" # Extract user_id values
|
||||||
|
#
|
||||||
|
# Performance Tips:
|
||||||
|
# - Avoid nested quantifiers: "((a+)+)+" can cause catastrophic backtracking
|
||||||
|
# - Use anchors when possible: "^ERROR" is faster than "ERROR"
|
||||||
|
# - Prefer character classes: "[0-9]" over "\\d" for clarity
|
||||||
|
# - Test complex patterns with sample data before deployment
|
||||||
|
#
|
||||||
|
# Security Considerations:
|
||||||
|
# - Be aware of ReDoS (Regular Expression Denial of Service)
|
||||||
|
# - Limit pattern complexity for public-facing streams
|
||||||
|
# - Monitor filter processing time in statistics
|
||||||
|
# - Consider pre-filtering very high volume streams
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# USAGE EXAMPLES
|
# USAGE EXAMPLES
|
||||||
@ -273,6 +443,7 @@ enabled = false
|
|||||||
# - Uncomment additional [[streams]] sections above
|
# - Uncomment additional [[streams]] sections above
|
||||||
# - Each stream runs independently on its own port
|
# - Each stream runs independently on its own port
|
||||||
# - Different check intervals for different log types
|
# - Different check intervals for different log types
|
||||||
|
# - Different filters for each stream
|
||||||
|
|
||||||
# 3. Router mode (consolidated access):
|
# 3. Router mode (consolidated access):
|
||||||
# ./logwisp --router
|
# ./logwisp --router
|
||||||
@ -281,6 +452,7 @@ enabled = false
|
|||||||
# - Example: http://localhost:8080/app/stream
|
# - Example: http://localhost:8080/app/stream
|
||||||
|
|
||||||
# 4. Production deployment:
|
# 4. Production deployment:
|
||||||
|
# - Enable filters to reduce noise and bandwidth
|
||||||
# - Enable rate limiting on public-facing streams
|
# - Enable rate limiting on public-facing streams
|
||||||
# - Use TCP for internal high-volume streams
|
# - Use TCP for internal high-volume streams
|
||||||
# - Set appropriate check intervals (higher = less CPU)
|
# - Set appropriate check intervals (higher = less CPU)
|
||||||
@ -289,6 +461,7 @@ enabled = false
|
|||||||
# 5. Monitoring:
|
# 5. Monitoring:
|
||||||
# curl http://localhost:8080/status | jq .
|
# curl http://localhost:8080/status | jq .
|
||||||
# - Check active connections
|
# - Check active connections
|
||||||
|
# - Monitor filter statistics (matched/dropped)
|
||||||
# - Monitor rate limit statistics
|
# - Monitor rate limit statistics
|
||||||
# - Track log entry counts
|
# - Track log entry counts
|
||||||
|
|
||||||
@ -299,10 +472,14 @@ enabled = false
|
|||||||
# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
|
# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
|
||||||
# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
|
# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
|
||||||
# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
# LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
||||||
|
# LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
|
||||||
|
# LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
# NOTES
|
# NOTES
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
# - Filters are processed sequentially - all must pass
|
||||||
|
# - Empty filter patterns means "pass everything"
|
||||||
# - Rate limiting is disabled by default for backward compatibility
|
# - Rate limiting is disabled by default for backward compatibility
|
||||||
# - Each stream can have different rate limit settings
|
# - Each stream can have different rate limit settings
|
||||||
# - TCP connections are silently dropped when rate limited
|
# - TCP connections are silently dropped when rate limited
|
||||||
@ -310,3 +487,5 @@ enabled = false
|
|||||||
# - IP tracking is cleaned up after 5 minutes of inactivity
|
# - IP tracking is cleaned up after 5 minutes of inactivity
|
||||||
# - Token bucket algorithm provides smooth rate limiting
|
# - Token bucket algorithm provides smooth rate limiting
|
||||||
# - Connection limits prevent resource exhaustion
|
# - Connection limits prevent resource exhaustion
|
||||||
|
# - Regex patterns are compiled once at startup for performance
|
||||||
|
# - Complex patterns can impact performance - monitor statistics
|
||||||
@ -11,6 +11,11 @@ targets = [
|
|||||||
{ path = "/var/log/myapp", pattern = "*.log", is_file = false }
|
{ path = "/var/log/myapp", pattern = "*.log", is_file = false }
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Optional: Filter for errors and warnings only
|
||||||
|
# [[streams.filters]]
|
||||||
|
# type = "include"
|
||||||
|
# patterns = ["ERROR", "WARN", "CRITICAL"]
|
||||||
|
|
||||||
[streams.httpserver]
|
[streams.httpserver]
|
||||||
enabled = true
|
enabled = true
|
||||||
port = 8080
|
port = 8080
|
||||||
|
|||||||
@ -14,8 +14,6 @@ logwisp/
|
|||||||
│ └── logwisp.toml.minimal # Minimal configuration template
|
│ └── logwisp.toml.minimal # Minimal configuration template
|
||||||
├── doc/
|
├── doc/
|
||||||
│ └── architecture.md # This file - architecture documentation
|
│ └── architecture.md # This file - architecture documentation
|
||||||
├── test_router.sh # Router functionality test suite
|
|
||||||
├── test_ratelimit.sh # Rate limiting test suite
|
|
||||||
└── src/
|
└── src/
|
||||||
├── cmd/
|
├── cmd/
|
||||||
│ └── logwisp/
|
│ └── logwisp/
|
||||||
@ -27,8 +25,11 @@ logwisp/
|
|||||||
│ ├── loader.go # Configuration loading with lixenwraith/config
|
│ ├── loader.go # Configuration loading with lixenwraith/config
|
||||||
│ ├── server.go # TCP/HTTP server configurations with rate limiting
|
│ ├── server.go # TCP/HTTP server configurations with rate limiting
|
||||||
│ ├── ssl.go # SSL/TLS configuration structures
|
│ ├── ssl.go # SSL/TLS configuration structures
|
||||||
│ ├── stream.go # Stream-specific configurations
|
│ ├── stream.go # Stream-specific configurations with filters
|
||||||
│ └── validation.go # Configuration validation including rate limits
|
│ └── validation.go # Configuration validation including filters and rate limits
|
||||||
|
├── filter/
|
||||||
|
│ ├── filter.go # Regex-based log filtering implementation
|
||||||
|
│ └── chain.go # Sequential filter chain management
|
||||||
├── logstream/
|
├── logstream/
|
||||||
│ ├── httprouter.go # HTTP router for path-based routing
|
│ ├── httprouter.go # HTTP router for path-based routing
|
||||||
│ ├── logstream.go # Stream lifecycle management
|
│ ├── logstream.go # Stream lifecycle management
|
||||||
@ -92,6 +93,11 @@ LOGWISP_STREAMS_0_HTTPSERVER_BUFFER_SIZE=2000
|
|||||||
LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
|
LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_ENABLED=true
|
||||||
LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=json
|
LOGWISP_STREAMS_0_HTTPSERVER_HEARTBEAT_FORMAT=json
|
||||||
|
|
||||||
|
# Filter configuration
|
||||||
|
LOGWISP_STREAMS_0_FILTERS_0_TYPE=include
|
||||||
|
LOGWISP_STREAMS_0_FILTERS_0_LOGIC=or
|
||||||
|
LOGWISP_STREAMS_0_FILTERS_0_PATTERNS='["ERROR","WARN"]'
|
||||||
|
|
||||||
# Rate limiting configuration
|
# Rate limiting configuration
|
||||||
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_ENABLED=true
|
||||||
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
|
LOGWISP_STREAMS_0_HTTPSERVER_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
|
||||||
@ -116,9 +122,9 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
|
|||||||
|
|
||||||
2. **LogStream (`logstream.LogStream`)**
|
2. **LogStream (`logstream.LogStream`)**
|
||||||
- Represents a single log monitoring pipeline
|
- Represents a single log monitoring pipeline
|
||||||
- Contains: Monitor + Rate Limiter + Servers (TCP/HTTP)
|
- Contains: Monitor + Filter Chain + Rate Limiter + Servers (TCP/HTTP)
|
||||||
- Independent configuration
|
- Independent configuration
|
||||||
- Per-stream statistics with rate limit metrics
|
- Per-stream statistics with filter and rate limit metrics
|
||||||
|
|
||||||
3. **Monitor (`monitor.Monitor`)**
|
3. **Monitor (`monitor.Monitor`)**
|
||||||
- Watches files and directories
|
- Watches files and directories
|
||||||
@ -126,14 +132,25 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
|
|||||||
- Publishes log entries to subscribers
|
- Publishes log entries to subscribers
|
||||||
- Configurable check intervals
|
- Configurable check intervals
|
||||||
|
|
||||||
4. **Rate Limiter (`ratelimit.Limiter`)**
|
4. **Filter (`filter.Filter`)**
|
||||||
|
- Regex-based log filtering
|
||||||
|
- Include (whitelist) or Exclude (blacklist) modes
|
||||||
|
- OR/AND logic for multiple patterns
|
||||||
|
- Per-filter statistics (processed, matched, dropped)
|
||||||
|
|
||||||
|
5. **Filter Chain (`filter.Chain`)**
|
||||||
|
- Sequential application of multiple filters
|
||||||
|
- All filters must pass for entry to be streamed
|
||||||
|
- Aggregate statistics across filter chain
|
||||||
|
|
||||||
|
6. **Rate Limiter (`ratelimit.Limiter`)**
|
||||||
- Token bucket algorithm for smooth rate limiting
|
- Token bucket algorithm for smooth rate limiting
|
||||||
- Per-IP or global limiting strategies
|
- Per-IP or global limiting strategies
|
||||||
- Connection tracking and limits
|
- Connection tracking and limits
|
||||||
- Automatic cleanup of stale entries
|
- Automatic cleanup of stale entries
|
||||||
- Non-blocking rejection of excess requests
|
- Non-blocking rejection of excess requests
|
||||||
|
|
||||||
5. **Streamers**
|
7. **Streamers**
|
||||||
- **HTTPStreamer**: SSE-based streaming over HTTP
|
- **HTTPStreamer**: SSE-based streaming over HTTP
|
||||||
- Rate limit enforcement before request handling
|
- Rate limit enforcement before request handling
|
||||||
- Connection tracking for per-IP limits
|
- Connection tracking for per-IP limits
|
||||||
@ -144,7 +161,7 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
|
|||||||
- Both support configurable heartbeats
|
- Both support configurable heartbeats
|
||||||
- Non-blocking client management
|
- Non-blocking client management
|
||||||
|
|
||||||
6. **HTTPRouter (`logstream.HTTPRouter`)**
|
8. **HTTPRouter (`logstream.HTTPRouter`)**
|
||||||
- Optional component for path-based routing
|
- Optional component for path-based routing
|
||||||
- Consolidates multiple HTTP streams on shared ports
|
- Consolidates multiple HTTP streams on shared ports
|
||||||
- Provides global status endpoint
|
- Provides global status endpoint
|
||||||
@ -154,11 +171,22 @@ LOGWISP_STREAMS_1_TCPSERVER_PORT=9090
|
|||||||
### Data Flow
|
### Data Flow
|
||||||
|
|
||||||
```
|
```
|
||||||
File System → Monitor → LogEntry Channel → [Rate Limiter] → Streamer → Network Client
|
File System → Monitor → LogEntry Channel → Filter Chain → [Rate Limiter] → Streamer → Network Client
|
||||||
↑ ↓ ↓
|
↑ ↓ ↓ ↓
|
||||||
└── Rotation Detection Rate Limit Check
|
└── Rotation Detection Pattern Match Rate Limit Check
|
||||||
↓
|
↓ ↓
|
||||||
Accept/Reject
|
Pass/Drop Accept/Reject
|
||||||
|
```
|
||||||
|
|
||||||
|
### Filter Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Log Entry → Filter Chain → Filter 1 → Filter 2 → ... → Output
|
||||||
|
↓ ↓
|
||||||
|
Include? Exclude?
|
||||||
|
↓ ↓
|
||||||
|
OR/AND OR/AND
|
||||||
|
Logic Logic
|
||||||
```
|
```
|
||||||
|
|
||||||
### Rate Limiting Architecture
|
### Rate Limiting Architecture
|
||||||
@ -184,6 +212,19 @@ targets = [
|
|||||||
{ path = "/path/to/file.log", is_file = true }
|
{ path = "/path/to/file.log", is_file = true }
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Filter configuration (optional)
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "include" # "include" or "exclude"
|
||||||
|
logic = "or" # "or" or "and"
|
||||||
|
patterns = [
|
||||||
|
"(?i)error", # Case-insensitive error matching
|
||||||
|
"(?i)warn" # Case-insensitive warning matching
|
||||||
|
]
|
||||||
|
|
||||||
|
[[streams.filters]]
|
||||||
|
type = "exclude"
|
||||||
|
patterns = ["DEBUG", "TRACE"]
|
||||||
|
|
||||||
[streams.httpserver]
|
[streams.httpserver]
|
||||||
enabled = true
|
enabled = true
|
||||||
port = 8080
|
port = 8080
|
||||||
@ -226,6 +267,26 @@ burst_size = 10
|
|||||||
limit_by = "ip"
|
limit_by = "ip"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Filter Implementation
|
||||||
|
|
||||||
|
### Filter Types
|
||||||
|
1. **Include Filter**: Only logs matching patterns are streamed (whitelist)
|
||||||
|
2. **Exclude Filter**: Logs matching patterns are dropped (blacklist)
|
||||||
|
|
||||||
|
### Pattern Logic
|
||||||
|
- **OR Logic**: Log matches if ANY pattern matches
|
||||||
|
- **AND Logic**: Log matches only if ALL patterns match
|
||||||
|
|
||||||
|
### Filter Chain
|
||||||
|
- Multiple filters are applied sequentially
|
||||||
|
- All filters must pass for a log to be streamed
|
||||||
|
- Efficient short-circuit evaluation
|
||||||
|
|
||||||
|
### Performance Considerations
|
||||||
|
- Regex patterns compiled once at startup
|
||||||
|
- Cached for efficient matching
|
||||||
|
- Statistics tracked without locks in hot path
|
||||||
|
|
||||||
## Rate Limiting Implementation
|
## Rate Limiting Implementation
|
||||||
|
|
||||||
### Token Bucket Algorithm
|
### Token Bucket Algorithm
|
||||||
@ -308,6 +369,13 @@ go build -ldflags "-X 'logwisp/src/internal/version.Version=v1.0.0'" \
|
|||||||
- Statistics accuracy
|
- Statistics accuracy
|
||||||
- Stress testing
|
- Stress testing
|
||||||
|
|
||||||
|
3. **Filter Testing** (recommended)
|
||||||
|
- Pattern matching accuracy
|
||||||
|
- Include/exclude logic
|
||||||
|
- OR/AND combination logic
|
||||||
|
- Performance with complex patterns
|
||||||
|
- Filter chain behavior
|
||||||
|
|
||||||
### Running Tests
|
### Running Tests
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@ -323,6 +391,12 @@ make test
|
|||||||
|
|
||||||
## Performance Considerations
|
## Performance Considerations
|
||||||
|
|
||||||
|
### Filter Overhead
|
||||||
|
- Regex compilation: One-time cost at startup
|
||||||
|
- Pattern matching: O(n*m) where n=patterns, m=text length
|
||||||
|
- Use simple patterns when possible
|
||||||
|
- Consider pattern order (most likely matches first)
|
||||||
|
|
||||||
### Rate Limiting Overhead
|
### Rate Limiting Overhead
|
||||||
- Token bucket checks: O(1) time complexity
|
- Token bucket checks: O(1) time complexity
|
||||||
- Memory: ~100 bytes per tracked IP
|
- Memory: ~100 bytes per tracked IP
|
||||||
@ -330,6 +404,8 @@ make test
|
|||||||
- Minimal impact when disabled
|
- Minimal impact when disabled
|
||||||
|
|
||||||
### Optimization Guidelines
|
### Optimization Guidelines
|
||||||
|
- Use specific patterns to reduce regex complexity
|
||||||
|
- Place most selective filters first in chain
|
||||||
- Use per-IP limiting for fairness
|
- Use per-IP limiting for fairness
|
||||||
- Use global limiting for resource protection
|
- Use global limiting for resource protection
|
||||||
- Set burst size to 2-3x requests_per_second
|
- Set burst size to 2-3x requests_per_second
|
||||||
@ -343,17 +419,4 @@ make test
|
|||||||
- Rate limiting for DDoS protection
|
- Rate limiting for DDoS protection
|
||||||
- Connection limits for resource protection
|
- Connection limits for resource protection
|
||||||
- Non-blocking request rejection
|
- Non-blocking request rejection
|
||||||
|
- Regex pattern validation at startup
|
||||||
### Future Security Roadmap
|
|
||||||
- Authentication (Basic, JWT, mTLS)
|
|
||||||
- TLS/SSL support
|
|
||||||
- IP whitelisting/blacklisting
|
|
||||||
- Audit logging
|
|
||||||
- RBAC per stream
|
|
||||||
|
|
||||||
### Security Best Practices
|
|
||||||
- Run with minimal privileges
|
|
||||||
- Enable rate limiting on public endpoints
|
|
||||||
- Use connection limits to prevent exhaustion
|
|
||||||
- Deploy behind reverse proxy for HTTPS
|
|
||||||
- Monitor rate limit statistics for attacks
|
|
||||||
@ -1,6 +1,8 @@
|
|||||||
// FILE: src/internal/config/stream.go
|
// FILE: src/internal/config/stream.go
|
||||||
package config
|
package config
|
||||||
|
|
||||||
|
import "logwisp/src/internal/filter"
|
||||||
|
|
||||||
type StreamConfig struct {
|
type StreamConfig struct {
|
||||||
// Stream identifier (used in logs and metrics)
|
// Stream identifier (used in logs and metrics)
|
||||||
Name string `toml:"name"`
|
Name string `toml:"name"`
|
||||||
@ -8,6 +10,9 @@ type StreamConfig struct {
|
|||||||
// Monitor configuration for this stream
|
// Monitor configuration for this stream
|
||||||
Monitor *StreamMonitorConfig `toml:"monitor"`
|
Monitor *StreamMonitorConfig `toml:"monitor"`
|
||||||
|
|
||||||
|
// Filter configuration
|
||||||
|
Filters []filter.Config `toml:"filters"`
|
||||||
|
|
||||||
// Server configurations
|
// Server configurations
|
||||||
TCPServer *TCPConfig `toml:"tcpserver"`
|
TCPServer *TCPConfig `toml:"tcpserver"`
|
||||||
HTTPServer *HTTPConfig `toml:"httpserver"`
|
HTTPServer *HTTPConfig `toml:"httpserver"`
|
||||||
|
|||||||
@ -3,6 +3,8 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"logwisp/src/internal/filter"
|
||||||
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -36,6 +38,7 @@ func (c *Config) validate() error {
|
|||||||
stream.Name, stream.Monitor.CheckIntervalMs)
|
stream.Name, stream.Monitor.CheckIntervalMs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate targets
|
||||||
for j, target := range stream.Monitor.Targets {
|
for j, target := range stream.Monitor.Targets {
|
||||||
if target.Path == "" {
|
if target.Path == "" {
|
||||||
return fmt.Errorf("stream '%s' target %d: empty path", stream.Name, j)
|
return fmt.Errorf("stream '%s' target %d: empty path", stream.Name, j)
|
||||||
@ -45,6 +48,13 @@ func (c *Config) validate() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Validate filters
|
||||||
|
for j, filterCfg := range stream.Filters {
|
||||||
|
if err := validateFilter(stream.Name, j, &filterCfg); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Validate TCP server
|
// Validate TCP server
|
||||||
if stream.TCPServer != nil && stream.TCPServer.Enabled {
|
if stream.TCPServer != nil && stream.TCPServer.Enabled {
|
||||||
if stream.TCPServer.Port < 1 || stream.TCPServer.Port > 65535 {
|
if stream.TCPServer.Port < 1 || stream.TCPServer.Port > 65535 {
|
||||||
@ -224,3 +234,38 @@ func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateFilter(streamName string, filterIndex int, cfg *filter.Config) error {
|
||||||
|
// Validate filter type
|
||||||
|
switch cfg.Type {
|
||||||
|
case filter.TypeInclude, filter.TypeExclude, "":
|
||||||
|
// Valid types
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("stream '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
|
||||||
|
streamName, filterIndex, cfg.Type)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate filter logic
|
||||||
|
switch cfg.Logic {
|
||||||
|
case filter.LogicOr, filter.LogicAnd, "":
|
||||||
|
// Valid logic
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("stream '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
|
||||||
|
streamName, filterIndex, cfg.Logic)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty patterns is valid - passes everything
|
||||||
|
if len(cfg.Patterns) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate regex patterns
|
||||||
|
for i, pattern := range cfg.Patterns {
|
||||||
|
if _, err := regexp.Compile(pattern); err != nil {
|
||||||
|
return fmt.Errorf("stream '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
|
||||||
|
streamName, filterIndex, i, pattern, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
72
src/internal/filter/chain.go
Normal file
72
src/internal/filter/chain.go
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
// FILE: src/internal/filter/chain.go
|
||||||
|
package filter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"logwisp/src/internal/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Chain manages multiple filters in sequence
|
||||||
|
type Chain struct {
|
||||||
|
filters []*Filter
|
||||||
|
|
||||||
|
// Statistics
|
||||||
|
totalProcessed atomic.Uint64
|
||||||
|
totalPassed atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewChain creates a new filter chain from configurations
|
||||||
|
func NewChain(configs []Config) (*Chain, error) {
|
||||||
|
chain := &Chain{
|
||||||
|
filters: make([]*Filter, 0, len(configs)),
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, cfg := range configs {
|
||||||
|
filter, err := New(cfg)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("filter[%d]: %w", i, err)
|
||||||
|
}
|
||||||
|
chain.filters = append(chain.filters, filter)
|
||||||
|
}
|
||||||
|
|
||||||
|
return chain, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply runs all filters in sequence
|
||||||
|
// Returns true if the entry passes all filters
|
||||||
|
func (c *Chain) Apply(entry monitor.LogEntry) bool {
|
||||||
|
c.totalProcessed.Add(1)
|
||||||
|
|
||||||
|
// No filters means pass everything
|
||||||
|
if len(c.filters) == 0 {
|
||||||
|
c.totalPassed.Add(1)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// All filters must pass
|
||||||
|
for _, filter := range c.filters {
|
||||||
|
if !filter.Apply(entry) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.totalPassed.Add(1)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns chain statistics
|
||||||
|
func (c *Chain) GetStats() map[string]interface{} {
|
||||||
|
filterStats := make([]map[string]interface{}, len(c.filters))
|
||||||
|
for i, filter := range c.filters {
|
||||||
|
filterStats[i] = filter.GetStats()
|
||||||
|
}
|
||||||
|
|
||||||
|
return map[string]interface{}{
|
||||||
|
"filter_count": len(c.filters),
|
||||||
|
"total_processed": c.totalProcessed.Load(),
|
||||||
|
"total_passed": c.totalPassed.Load(),
|
||||||
|
"filters": filterStats,
|
||||||
|
}
|
||||||
|
}
|
||||||
173
src/internal/filter/filter.go
Normal file
173
src/internal/filter/filter.go
Normal file
@ -0,0 +1,173 @@
|
|||||||
|
// FILE: src/internal/filter/filter.go
|
||||||
|
package filter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"logwisp/src/internal/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Type represents the filter type
|
||||||
|
type Type string
|
||||||
|
|
||||||
|
const (
|
||||||
|
TypeInclude Type = "include" // Whitelist - only matching logs pass
|
||||||
|
TypeExclude Type = "exclude" // Blacklist - matching logs are dropped
|
||||||
|
)
|
||||||
|
|
||||||
|
// Logic represents how multiple patterns are combined
|
||||||
|
type Logic string
|
||||||
|
|
||||||
|
const (
|
||||||
|
LogicOr Logic = "or" // Match any pattern
|
||||||
|
LogicAnd Logic = "and" // Match all patterns
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config represents filter configuration
|
||||||
|
type Config struct {
|
||||||
|
Type Type `toml:"type"`
|
||||||
|
Logic Logic `toml:"logic"`
|
||||||
|
Patterns []string `toml:"patterns"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter applies regex-based filtering to log entries
|
||||||
|
type Filter struct {
|
||||||
|
config Config
|
||||||
|
patterns []*regexp.Regexp
|
||||||
|
mu sync.RWMutex
|
||||||
|
|
||||||
|
// Statistics
|
||||||
|
totalProcessed atomic.Uint64
|
||||||
|
totalMatched atomic.Uint64
|
||||||
|
totalDropped atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a new filter from configuration
|
||||||
|
func New(cfg Config) (*Filter, error) {
|
||||||
|
// Set defaults
|
||||||
|
if cfg.Type == "" {
|
||||||
|
cfg.Type = TypeInclude
|
||||||
|
}
|
||||||
|
if cfg.Logic == "" {
|
||||||
|
cfg.Logic = LogicOr
|
||||||
|
}
|
||||||
|
|
||||||
|
f := &Filter{
|
||||||
|
config: cfg,
|
||||||
|
patterns: make([]*regexp.Regexp, 0, len(cfg.Patterns)),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compile patterns
|
||||||
|
for i, pattern := range cfg.Patterns {
|
||||||
|
re, err := regexp.Compile(pattern)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
|
||||||
|
}
|
||||||
|
f.patterns = append(f.patterns, re)
|
||||||
|
}
|
||||||
|
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply checks if a log entry should be passed through
|
||||||
|
func (f *Filter) Apply(entry monitor.LogEntry) bool {
|
||||||
|
f.totalProcessed.Add(1)
|
||||||
|
|
||||||
|
// No patterns means pass everything
|
||||||
|
if len(f.patterns) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check against all fields that might contain the log content
|
||||||
|
text := entry.Message
|
||||||
|
if entry.Level != "" {
|
||||||
|
text = entry.Level + " " + text
|
||||||
|
}
|
||||||
|
if entry.Source != "" {
|
||||||
|
text = entry.Source + " " + text
|
||||||
|
}
|
||||||
|
|
||||||
|
matched := f.matches(text)
|
||||||
|
if matched {
|
||||||
|
f.totalMatched.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine if we should pass or drop
|
||||||
|
shouldPass := false
|
||||||
|
switch f.config.Type {
|
||||||
|
case TypeInclude:
|
||||||
|
shouldPass = matched
|
||||||
|
case TypeExclude:
|
||||||
|
shouldPass = !matched
|
||||||
|
}
|
||||||
|
|
||||||
|
if !shouldPass {
|
||||||
|
f.totalDropped.Add(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
return shouldPass
|
||||||
|
}
|
||||||
|
|
||||||
|
// matches checks if text matches the patterns according to the logic
|
||||||
|
func (f *Filter) matches(text string) bool {
|
||||||
|
switch f.config.Logic {
|
||||||
|
case LogicOr:
|
||||||
|
// Match any pattern
|
||||||
|
for _, re := range f.patterns {
|
||||||
|
if re.MatchString(text) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
|
||||||
|
case LogicAnd:
|
||||||
|
// Must match all patterns
|
||||||
|
for _, re := range f.patterns {
|
||||||
|
if !re.MatchString(text) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
|
||||||
|
default:
|
||||||
|
// Shouldn't happen after validation
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns filter statistics
|
||||||
|
func (f *Filter) GetStats() map[string]interface{} {
|
||||||
|
return map[string]interface{}{
|
||||||
|
"type": f.config.Type,
|
||||||
|
"logic": f.config.Logic,
|
||||||
|
"pattern_count": len(f.patterns),
|
||||||
|
"total_processed": f.totalProcessed.Load(),
|
||||||
|
"total_matched": f.totalMatched.Load(),
|
||||||
|
"total_dropped": f.totalDropped.Load(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdatePatterns allows dynamic pattern updates
|
||||||
|
func (f *Filter) UpdatePatterns(patterns []string) error {
|
||||||
|
compiled := make([]*regexp.Regexp, 0, len(patterns))
|
||||||
|
|
||||||
|
// Compile all patterns first
|
||||||
|
for i, pattern := range patterns {
|
||||||
|
re, err := regexp.Compile(pattern)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
|
||||||
|
}
|
||||||
|
compiled = append(compiled, re)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update atomically
|
||||||
|
f.mu.Lock()
|
||||||
|
f.patterns = compiled
|
||||||
|
f.config.Patterns = patterns
|
||||||
|
f.mu.Unlock()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@ -135,11 +135,11 @@ func (r *HTTPRouter) Shutdown() {
|
|||||||
fmt.Println("[ROUTER] Router shutdown complete")
|
fmt.Println("[ROUTER] Router shutdown complete")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *HTTPRouter) GetStats() map[string]interface{} {
|
func (r *HTTPRouter) GetStats() map[string]any {
|
||||||
r.mu.RLock()
|
r.mu.RLock()
|
||||||
defer r.mu.RUnlock()
|
defer r.mu.RUnlock()
|
||||||
|
|
||||||
serverStats := make(map[int]interface{})
|
serverStats := make(map[int]any)
|
||||||
totalRoutes := 0
|
totalRoutes := 0
|
||||||
|
|
||||||
for port, rs := range r.servers {
|
for port, rs := range r.servers {
|
||||||
@ -151,14 +151,14 @@ func (r *HTTPRouter) GetStats() map[string]interface{} {
|
|||||||
}
|
}
|
||||||
rs.routeMu.RUnlock()
|
rs.routeMu.RUnlock()
|
||||||
|
|
||||||
serverStats[port] = map[string]interface{}{
|
serverStats[port] = map[string]any{
|
||||||
"routes": routes,
|
"routes": routes,
|
||||||
"requests": rs.requests.Load(),
|
"requests": rs.requests.Load(),
|
||||||
"uptime": int(time.Since(rs.startTime).Seconds()),
|
"uptime": int(time.Since(rs.startTime).Seconds()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return map[string]interface{}{
|
return map[string]any{
|
||||||
"uptime_seconds": int(time.Since(r.startTime).Seconds()),
|
"uptime_seconds": int(time.Since(r.startTime).Seconds()),
|
||||||
"total_requests": r.totalRequests.Load(),
|
"total_requests": r.totalRequests.Load(),
|
||||||
"routed_requests": r.routedRequests.Load(),
|
"routed_requests": r.routedRequests.Load(),
|
||||||
|
|||||||
@ -40,22 +40,26 @@ func (ls *LogStream) Shutdown() {
|
|||||||
ls.Monitor.Stop()
|
ls.Monitor.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ls *LogStream) GetStats() map[string]interface{} {
|
func (ls *LogStream) GetStats() map[string]any {
|
||||||
monStats := ls.Monitor.GetStats()
|
monStats := ls.Monitor.GetStats()
|
||||||
|
|
||||||
stats := map[string]interface{}{
|
stats := map[string]any{
|
||||||
"name": ls.Name,
|
"name": ls.Name,
|
||||||
"uptime_seconds": int(time.Since(ls.Stats.StartTime).Seconds()),
|
"uptime_seconds": int(time.Since(ls.Stats.StartTime).Seconds()),
|
||||||
"monitor": monStats,
|
"monitor": monStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ls.FilterChain != nil {
|
||||||
|
stats["filters"] = ls.FilterChain.GetStats()
|
||||||
|
}
|
||||||
|
|
||||||
if ls.TCPServer != nil {
|
if ls.TCPServer != nil {
|
||||||
currentConnections := ls.TCPServer.GetActiveConnections()
|
currentConnections := ls.TCPServer.GetActiveConnections()
|
||||||
|
|
||||||
stats["tcp"] = map[string]interface{}{
|
stats["tcp"] = map[string]interface{}{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"port": ls.Config.TCPServer.Port,
|
"port": ls.Config.TCPServer.Port,
|
||||||
"connections": currentConnections, // Use current value
|
"connections": currentConnections,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -65,7 +69,7 @@ func (ls *LogStream) GetStats() map[string]interface{} {
|
|||||||
stats["http"] = map[string]interface{}{
|
stats["http"] = map[string]interface{}{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"port": ls.Config.HTTPServer.Port,
|
"port": ls.Config.HTTPServer.Port,
|
||||||
"connections": currentConnections, // Use current value
|
"connections": currentConnections,
|
||||||
"stream_path": ls.Config.HTTPServer.StreamPath,
|
"stream_path": ls.Config.HTTPServer.StreamPath,
|
||||||
"status_path": ls.Config.HTTPServer.StatusPath,
|
"status_path": ls.Config.HTTPServer.StatusPath,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -101,12 +101,12 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
|
|||||||
ctx.SetContentType("application/json")
|
ctx.SetContentType("application/json")
|
||||||
|
|
||||||
rs.routeMu.RLock()
|
rs.routeMu.RLock()
|
||||||
streams := make(map[string]interface{})
|
streams := make(map[string]any)
|
||||||
for prefix, stream := range rs.routes {
|
for prefix, stream := range rs.routes {
|
||||||
streamStats := stream.GetStats()
|
streamStats := stream.GetStats()
|
||||||
|
|
||||||
// Add routing information
|
// Add routing information
|
||||||
streamStats["routing"] = map[string]interface{}{
|
streamStats["routing"] = map[string]any{
|
||||||
"path_prefix": prefix,
|
"path_prefix": prefix,
|
||||||
"endpoints": map[string]string{
|
"endpoints": map[string]string{
|
||||||
"stream": prefix + stream.Config.HTTPServer.StreamPath,
|
"stream": prefix + stream.Config.HTTPServer.StreamPath,
|
||||||
@ -121,7 +121,7 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
|
|||||||
// Get router stats
|
// Get router stats
|
||||||
routerStats := rs.router.GetStats()
|
routerStats := rs.router.GetStats()
|
||||||
|
|
||||||
status := map[string]interface{}{
|
status := map[string]any{
|
||||||
"service": "LogWisp Router",
|
"service": "LogWisp Router",
|
||||||
"version": version.String(),
|
"version": version.String(),
|
||||||
"port": rs.port,
|
"port": rs.port,
|
||||||
@ -155,7 +155,7 @@ func (rs *routerServer) handleNotFound(ctx *fasthttp.RequestCtx) {
|
|||||||
}
|
}
|
||||||
rs.routeMu.RUnlock()
|
rs.routeMu.RUnlock()
|
||||||
|
|
||||||
response := map[string]interface{}{
|
response := map[string]any{
|
||||||
"error": "Not Found",
|
"error": "Not Found",
|
||||||
"requested_path": string(ctx.Path()),
|
"requested_path": string(ctx.Path()),
|
||||||
"available_routes": availableRoutes,
|
"available_routes": availableRoutes,
|
||||||
|
|||||||
@ -4,6 +4,7 @@ package logstream
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"logwisp/src/internal/filter"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -21,12 +22,13 @@ type Service struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type LogStream struct {
|
type LogStream struct {
|
||||||
Name string
|
Name string
|
||||||
Config config.StreamConfig
|
Config config.StreamConfig
|
||||||
Monitor monitor.Monitor
|
Monitor monitor.Monitor
|
||||||
TCPServer *stream.TCPStreamer
|
FilterChain *filter.Chain
|
||||||
HTTPServer *stream.HTTPStreamer
|
TCPServer *stream.TCPStreamer
|
||||||
Stats *StreamStats
|
HTTPServer *stream.HTTPStreamer
|
||||||
|
Stats *StreamStats
|
||||||
|
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
cancel context.CancelFunc
|
cancel context.CancelFunc
|
||||||
@ -39,6 +41,7 @@ type StreamStats struct {
|
|||||||
HTTPConnections int32
|
HTTPConnections int32
|
||||||
TotalBytesServed uint64
|
TotalBytesServed uint64
|
||||||
TotalEntriesServed uint64
|
TotalEntriesServed uint64
|
||||||
|
FilterStats map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
func New(ctx context.Context) *Service {
|
func New(ctx context.Context) *Service {
|
||||||
@ -79,11 +82,23 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
|
|||||||
return fmt.Errorf("failed to start monitor: %w", err)
|
return fmt.Errorf("failed to start monitor: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create filter chain
|
||||||
|
var filterChain *filter.Chain
|
||||||
|
if len(cfg.Filters) > 0 {
|
||||||
|
chain, err := filter.NewChain(cfg.Filters)
|
||||||
|
if err != nil {
|
||||||
|
streamCancel()
|
||||||
|
return fmt.Errorf("failed to create filter chain: %w", err)
|
||||||
|
}
|
||||||
|
filterChain = chain
|
||||||
|
}
|
||||||
|
|
||||||
// Create log stream
|
// Create log stream
|
||||||
ls := &LogStream{
|
ls := &LogStream{
|
||||||
Name: cfg.Name,
|
Name: cfg.Name,
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
Monitor: mon,
|
Monitor: mon,
|
||||||
|
FilterChain: filterChain,
|
||||||
Stats: &StreamStats{
|
Stats: &StreamStats{
|
||||||
StartTime: time.Now(),
|
StartTime: time.Now(),
|
||||||
},
|
},
|
||||||
@ -93,7 +108,18 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
|
|||||||
|
|
||||||
// Start TCP server if configured
|
// Start TCP server if configured
|
||||||
if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
|
if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
|
||||||
tcpChan := mon.Subscribe()
|
// Create filtered channel
|
||||||
|
rawChan := mon.Subscribe()
|
||||||
|
tcpChan := make(chan monitor.LogEntry, cfg.TCPServer.BufferSize)
|
||||||
|
|
||||||
|
// Start filter goroutine for TCP
|
||||||
|
s.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer s.wg.Done()
|
||||||
|
defer close(tcpChan)
|
||||||
|
s.filterLoop(streamCtx, rawChan, tcpChan, filterChain)
|
||||||
|
}()
|
||||||
|
|
||||||
ls.TCPServer = stream.NewTCPStreamer(tcpChan, *cfg.TCPServer)
|
ls.TCPServer = stream.NewTCPStreamer(tcpChan, *cfg.TCPServer)
|
||||||
|
|
||||||
if err := s.startTCPServer(ls); err != nil {
|
if err := s.startTCPServer(ls); err != nil {
|
||||||
@ -104,7 +130,18 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
|
|||||||
|
|
||||||
// Start HTTP server if configured
|
// Start HTTP server if configured
|
||||||
if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
|
if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
|
||||||
httpChan := mon.Subscribe()
|
// Create filtered channel
|
||||||
|
rawChan := mon.Subscribe()
|
||||||
|
httpChan := make(chan monitor.LogEntry, cfg.HTTPServer.BufferSize)
|
||||||
|
|
||||||
|
// Start filter goroutine for HTTP
|
||||||
|
s.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer s.wg.Done()
|
||||||
|
defer close(httpChan)
|
||||||
|
s.filterLoop(streamCtx, rawChan, httpChan, filterChain)
|
||||||
|
}()
|
||||||
|
|
||||||
ls.HTTPServer = stream.NewHTTPStreamer(httpChan, *cfg.HTTPServer)
|
ls.HTTPServer = stream.NewHTTPStreamer(httpChan, *cfg.HTTPServer)
|
||||||
|
|
||||||
if err := s.startHTTPServer(ls); err != nil {
|
if err := s.startHTTPServer(ls); err != nil {
|
||||||
@ -119,6 +156,31 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// filterLoop applies filters to log entries
|
||||||
|
func (s *Service) filterLoop(ctx context.Context, in <-chan monitor.LogEntry, out chan<- monitor.LogEntry, chain *filter.Chain) {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case entry, ok := <-in:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply filter chain if configured
|
||||||
|
if chain == nil || chain.Apply(entry) {
|
||||||
|
select {
|
||||||
|
case out <- entry:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
// Drop if output buffer is full
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) GetStream(name string) (*LogStream, error) {
|
func (s *Service) GetStream(name string) (*LogStream, error) {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
@ -178,17 +240,17 @@ func (s *Service) Shutdown() {
|
|||||||
s.wg.Wait()
|
s.wg.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) GetGlobalStats() map[string]interface{} {
|
func (s *Service) GetGlobalStats() map[string]any {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
stats := map[string]interface{}{
|
stats := map[string]any{
|
||||||
"streams": make(map[string]interface{}),
|
"streams": make(map[string]any),
|
||||||
"total_streams": len(s.streams),
|
"total_streams": len(s.streams),
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, stream := range s.streams {
|
for name, stream := range s.streams {
|
||||||
stats["streams"].(map[string]interface{})[name] = stream.GetStats()
|
stats["streams"].(map[string]any)[name] = stream.GetStats()
|
||||||
}
|
}
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|||||||
@ -192,9 +192,9 @@ func (l *Limiter) RemoveConnection(remoteAddr string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Returns rate limiter statistics
|
// Returns rate limiter statistics
|
||||||
func (l *Limiter) GetStats() map[string]interface{} {
|
func (l *Limiter) GetStats() map[string]any {
|
||||||
if l == nil {
|
if l == nil {
|
||||||
return map[string]interface{}{
|
return map[string]any{
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -210,13 +210,13 @@ func (l *Limiter) GetStats() map[string]interface{} {
|
|||||||
}
|
}
|
||||||
l.connMu.RUnlock()
|
l.connMu.RUnlock()
|
||||||
|
|
||||||
return map[string]interface{}{
|
return map[string]any{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"total_requests": l.totalRequests.Load(),
|
"total_requests": l.totalRequests.Load(),
|
||||||
"blocked_requests": l.blockedRequests.Load(),
|
"blocked_requests": l.blockedRequests.Load(),
|
||||||
"active_ips": activeIPs,
|
"active_ips": activeIPs,
|
||||||
"total_connections": totalConnections,
|
"total_connections": totalConnections,
|
||||||
"config": map[string]interface{}{
|
"config": map[string]any{
|
||||||
"requests_per_second": l.config.RequestsPerSecond,
|
"requests_per_second": l.config.RequestsPerSecond,
|
||||||
"burst_size": l.config.BurstSize,
|
"burst_size": l.config.BurstSize,
|
||||||
"limit_by": l.config.LimitBy,
|
"limit_by": l.config.LimitBy,
|
||||||
|
|||||||
@ -132,7 +132,7 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
|
|||||||
if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed {
|
if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed {
|
||||||
ctx.SetStatusCode(statusCode)
|
ctx.SetStatusCode(statusCode)
|
||||||
ctx.SetContentType("application/json")
|
ctx.SetContentType("application/json")
|
||||||
json.NewEncoder(ctx).Encode(map[string]interface{}{
|
json.NewEncoder(ctx).Encode(map[string]any{
|
||||||
"error": message,
|
"error": message,
|
||||||
"retry_after": "60", // seconds
|
"retry_after": "60", // seconds
|
||||||
})
|
})
|
||||||
@ -149,7 +149,7 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
|
|||||||
default:
|
default:
|
||||||
ctx.SetStatusCode(fasthttp.StatusNotFound)
|
ctx.SetStatusCode(fasthttp.StatusNotFound)
|
||||||
ctx.SetContentType("application/json")
|
ctx.SetContentType("application/json")
|
||||||
json.NewEncoder(ctx).Encode(map[string]interface{}{
|
json.NewEncoder(ctx).Encode(map[string]any{
|
||||||
"error": "Not Found",
|
"error": "Not Found",
|
||||||
"message": fmt.Sprintf("Available endpoints: %s (SSE stream), %s (status)",
|
"message": fmt.Sprintf("Available endpoints: %s (SSE stream), %s (status)",
|
||||||
h.streamPath, h.statusPath),
|
h.streamPath, h.statusPath),
|
||||||
@ -218,7 +218,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
|
|||||||
|
|
||||||
// Send initial connected event
|
// Send initial connected event
|
||||||
clientID := fmt.Sprintf("%d", time.Now().UnixNano())
|
clientID := fmt.Sprintf("%d", time.Now().UnixNano())
|
||||||
connectionInfo := map[string]interface{}{
|
connectionInfo := map[string]any{
|
||||||
"client_id": clientID,
|
"client_id": clientID,
|
||||||
"stream_path": h.streamPath,
|
"stream_path": h.streamPath,
|
||||||
"status_path": h.statusPath,
|
"status_path": h.statusPath,
|
||||||
@ -280,7 +280,7 @@ func (h *HTTPStreamer) formatHeartbeat() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if h.config.Heartbeat.Format == "json" {
|
if h.config.Heartbeat.Format == "json" {
|
||||||
data := make(map[string]interface{})
|
data := make(map[string]any)
|
||||||
data["type"] = "heartbeat"
|
data["type"] = "heartbeat"
|
||||||
|
|
||||||
if h.config.Heartbeat.IncludeTimestamp {
|
if h.config.Heartbeat.IncludeTimestamp {
|
||||||
@ -315,19 +315,19 @@ func (h *HTTPStreamer) formatHeartbeat() string {
|
|||||||
func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
|
func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
|
||||||
ctx.SetContentType("application/json")
|
ctx.SetContentType("application/json")
|
||||||
|
|
||||||
var rateLimitStats interface{}
|
var rateLimitStats any
|
||||||
if h.rateLimiter != nil {
|
if h.rateLimiter != nil {
|
||||||
rateLimitStats = h.rateLimiter.GetStats()
|
rateLimitStats = h.rateLimiter.GetStats()
|
||||||
} else {
|
} else {
|
||||||
rateLimitStats = map[string]interface{}{
|
rateLimitStats = map[string]any{
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
status := map[string]interface{}{
|
status := map[string]any{
|
||||||
"service": "LogWisp",
|
"service": "LogWisp",
|
||||||
"version": version.Short(),
|
"version": version.Short(),
|
||||||
"server": map[string]interface{}{
|
"server": map[string]any{
|
||||||
"type": "http",
|
"type": "http",
|
||||||
"port": h.config.Port,
|
"port": h.config.Port,
|
||||||
"active_clients": h.activeClients.Load(),
|
"active_clients": h.activeClients.Load(),
|
||||||
@ -339,8 +339,8 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
|
|||||||
"stream": h.streamPath,
|
"stream": h.streamPath,
|
||||||
"status": h.statusPath,
|
"status": h.statusPath,
|
||||||
},
|
},
|
||||||
"features": map[string]interface{}{
|
"features": map[string]any{
|
||||||
"heartbeat": map[string]interface{}{
|
"heartbeat": map[string]any{
|
||||||
"enabled": h.config.Heartbeat.Enabled,
|
"enabled": h.config.Heartbeat.Enabled,
|
||||||
"interval": h.config.Heartbeat.IntervalSeconds,
|
"interval": h.config.Heartbeat.IntervalSeconds,
|
||||||
"format": h.config.Heartbeat.Format,
|
"format": h.config.Heartbeat.Format,
|
||||||
|
|||||||
@ -116,7 +116,7 @@ func (t *TCPStreamer) broadcastLoop() {
|
|||||||
}
|
}
|
||||||
data = append(data, '\n')
|
data = append(data, '\n')
|
||||||
|
|
||||||
t.server.connections.Range(func(key, value interface{}) bool {
|
t.server.connections.Range(func(key, value any) bool {
|
||||||
conn := key.(gnet.Conn)
|
conn := key.(gnet.Conn)
|
||||||
conn.AsyncWrite(data, nil)
|
conn.AsyncWrite(data, nil)
|
||||||
return true
|
return true
|
||||||
@ -124,7 +124,7 @@ func (t *TCPStreamer) broadcastLoop() {
|
|||||||
|
|
||||||
case <-tickerChan:
|
case <-tickerChan:
|
||||||
if heartbeat := t.formatHeartbeat(); heartbeat != nil {
|
if heartbeat := t.formatHeartbeat(); heartbeat != nil {
|
||||||
t.server.connections.Range(func(key, value interface{}) bool {
|
t.server.connections.Range(func(key, value any) bool {
|
||||||
conn := key.(gnet.Conn)
|
conn := key.(gnet.Conn)
|
||||||
conn.AsyncWrite(heartbeat, nil)
|
conn.AsyncWrite(heartbeat, nil)
|
||||||
return true
|
return true
|
||||||
@ -142,7 +142,7 @@ func (t *TCPStreamer) formatHeartbeat() []byte {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
data := make(map[string]interface{})
|
data := make(map[string]any)
|
||||||
data["type"] = "heartbeat"
|
data["type"] = "heartbeat"
|
||||||
|
|
||||||
if t.config.Heartbeat.IncludeTimestamp {
|
if t.config.Heartbeat.IncludeTimestamp {
|
||||||
|
|||||||
Reference in New Issue
Block a user