diff --git a/README.md b/README.md index c26befc..5eda4ae 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +

+ LogWisp Logo +

+ # LogWisp - Simple Log Streaming A lightweight log streaming service that monitors files and streams updates via Server-Sent Events (SSE). @@ -16,6 +20,7 @@ LogWisp follows the Unix philosophy: do one thing and do it well. It monitors lo - Environment variable support - Simple TOML configuration - Atomic configuration management +- Optional ANSI color pass-through ## Quick Start @@ -34,13 +39,61 @@ LogWisp follows the Unix philosophy: do one thing and do it well. It monitors lo curl -N http://localhost:8080/stream ``` +## Command Line Options + +```bash +logwisp [OPTIONS] [TARGET...] + +OPTIONS: + -c, --color Enable color pass-through for ANSI escape codes + --config FILE Config file path (default: ~/.config/logwisp.toml) + --port PORT HTTP port (default: 8080) + --buffer-size SIZE Stream buffer size (default: 1000) + --check-interval MS File check interval in ms (default: 100) + --rate-limit Enable rate limiting + --rate-requests N Rate limit requests/sec (default: 10) + --rate-burst N Rate limit burst size (default: 20) + +TARGET: + path[:pattern[:isfile]] Path to monitor (file or directory) + pattern: glob pattern for directories (default: *.log) + isfile: true/false (auto-detected if omitted) + +EXAMPLES: + # Monitor current directory for *.log files + logwisp + + # Monitor specific file with color support + logwisp -c /var/log/app.log + + # Monitor multiple locations + logwisp /var/log:*.log /app/logs:error*.log:false /tmp/debug.log::true + + # Custom port with rate limiting + logwisp --port 9090 --rate-limit --rate-requests 100 --rate-burst 200 +``` + ## Configuration LogWisp uses a three-level configuration hierarchy: -1. **Environment variables** (highest priority) -2. **Configuration file** (~/.config/logwisp.toml) -3. **Default values** (lowest priority) +1. **Command-line arguments** (highest priority) +2. **Environment variables** +3. **Configuration file** (~/.config/logwisp.toml) +4. **Default values** (lowest priority) + +### Default Values + +| Setting | Default | Description | +|---------|---------|-------------| +| `port` | 8080 | HTTP listen port | +| `monitor.check_interval_ms` | 100 | File check interval (milliseconds) | +| `monitor.targets` | [{"path": "./", "pattern": "*.log", "is_file": false}] | Paths to monitor | +| `stream.buffer_size` | 1000 | Per-client event buffer size | +| `stream.rate_limit.enabled` | false | Enable rate limiting | +| `stream.rate_limit.requests_per_second` | 10 | Sustained request rate | +| `stream.rate_limit.burst_size` | 20 | Maximum burst size | +| `stream.rate_limit.cleanup_interval_s` | 60 | Client cleanup interval | ### Configuration File Location @@ -75,7 +128,7 @@ All configuration values can be overridden via environment variables: | `LOGWISP_STREAM_RATE_LIMIT_ENABLED` | `stream.rate_limit.enabled` | Enable rate limiting | | `LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC` | `stream.rate_limit.requests_per_second` | Rate limit | | `LOGWISP_STREAM_RATE_LIMIT_BURST_SIZE` | `stream.rate_limit.burst_size` | Burst size | -| `LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL` | `stream.rate_limit.cleanup_interval_s` | Cleanup interval | +| `LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL_S` | `stream.rate_limit.cleanup_interval_s` | Cleanup interval | ### Monitor Targets Format @@ -93,17 +146,22 @@ LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/app/app.log::true" ./logwisp LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/opt/app/logs:app-*.log:false" ./logwisp ``` -### Example Configuration +### Complete Configuration Example ```toml +# Port to listen on (default: 8080) port = 8080 [monitor] +# How often to check for file changes in milliseconds (default: 100) check_interval_ms = 100 -# Monitor directory (all .log files) +# Paths to monitor +# Default: [{"path": "./", "pattern": "*.log", "is_file": false}] + +# Monitor all .log files in current directory [[monitor.targets]] -path = "/var/log" +path = "./" pattern = "*.log" is_file = false @@ -120,12 +178,24 @@ pattern = "access*.log" is_file = false [stream] +# Buffer size for each client connection (default: 1000) +# Controls how many log entries can be queued per client buffer_size = 1000 [stream.rate_limit] -enabled = true +# Enable rate limiting (default: false) +enabled = false + +# Requests per second per client (default: 10) +# This is the sustained rate requests_per_second = 10 + +# Burst size - max requests at once (default: 20) +# Allows temporary bursts above the sustained rate burst_size = 20 + +# How often to clean up old client limiters in seconds (default: 60) +# Clients inactive for 2x this duration are removed cleanup_interval_s = 60 ``` @@ -141,7 +211,7 @@ LogWisp can pass through ANSI color escape codes from monitored logs to SSE clie ExecStart=/opt/logwisp/bin/logwisp -c ``` -## How It Works +### How It Works When color mode is enabled (`-c` flag), LogWisp preserves ANSI escape codes in log messages. These are properly JSON-escaped in the SSE stream. @@ -163,9 +233,9 @@ SSE output with `-c`: } ``` -## Client-Side Handling +### Client-Side Handling -### Terminal Clients +#### Terminal Clients For terminal-based clients (like curl), the escape codes will render as colors: @@ -174,7 +244,7 @@ For terminal-based clients (like curl), the escape codes will render as colors: curl -N http://localhost:8080/stream | jq -r '.message' ``` -### Web Clients +#### Web Clients For web-based clients, you'll need to convert ANSI codes to HTML: @@ -190,7 +260,7 @@ eventSource.onmessage = (event) => { }; ``` -### Custom Processing +#### Custom Processing ```python # Python example with colorama @@ -245,13 +315,13 @@ To strip color codes instead of passing them through: ### Endpoints - `GET /stream` - Server-Sent Events stream of log entries -- `GET /status` - Service status information +- `GET /status` - Service status and configuration information ### Log Entry Format ```json { - "time": "2024-01-01T12:00:00Z", + "time": "2024-01-01T12:00:00.123456Z", "source": "app.log", "level": "error", "message": "Something went wrong", @@ -259,6 +329,48 @@ To strip color codes instead of passing them through: } ``` +### SSE Event Types + +| Event | Description | Data Format | +|-------|-------------|-------------| +| `connected` | Initial connection | `{"client_id": "123456789"}` | +| `data` | Log entry | JSON log entry | +| `disconnected` | Client disconnected | `{"reason": "slow_client"}` | +| `timeout` | Client timeout | `{"reason": "client_timeout"}` | +| `:` | Heartbeat (comment) | ISO timestamp | + +### Status Response Format + +```json +{ + "service": "LogWisp", + "version": "2.0.0", + "port": 8080, + "color_mode": false, + "config": { + "monitor": { + "check_interval_ms": 100, + "targets_count": 2 + }, + "stream": { + "buffer_size": 1000, + "rate_limit": { + "enabled": true, + "requests_per_second": 10, + "burst_size": 20 + } + } + }, + "streamer": { + "active_clients": 5, + "buffer_size": 1000, + "color_mode": false, + "total_dropped": 42 + }, + "rate_limiter": "Active clients: 3" +} +``` + ## Usage Examples ### Basic Usage @@ -266,6 +378,9 @@ To strip color codes instead of passing them through: # Start with defaults ./logwisp +# Monitor specific file +./logwisp /var/log/app.log + # View logs curl -N http://localhost:8080/stream ``` @@ -281,6 +396,9 @@ LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC=5 \ ### Monitor Multiple Locations ```bash +# Via command line +./logwisp /var/log:*.log /app/logs:*.json /tmp/debug.log + # Via environment variable LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/app/logs:*.json:false,/tmp/debug.log::true" \ ./logwisp @@ -315,11 +433,13 @@ After=network.target [Service] Type=simple User=logwisp -ExecStart=/usr/local/bin/logwisp +ExecStart=/usr/local/bin/logwisp -c Restart=always +RestartSec=5 # Environment overrides Environment="LOGWISP_PORT=8080" +Environment="LOGWISP_STREAM_BUFFER_SIZE=5000" Environment="LOGWISP_STREAM_RATE_LIMIT_ENABLED=true" Environment="LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC=100" Environment="LOGWISP_MONITOR_TARGETS=/var/log:*.log:false,/opt/app/logs:*.log:false" @@ -330,25 +450,92 @@ PrivateTmp=true ProtectSystem=strict ProtectHome=true ReadOnlyPaths=/ -ReadWritePaths=/var/log +ReadWritePaths=/var/log /opt/app/logs [Install] WantedBy=multi-user.target ``` -## Rate Limiting +## Performance Tuning -When enabled, rate limiting is applied per client IP address: +### Buffer Size -```toml -[stream.rate_limit] -enabled = true -requests_per_second = 10 # Sustained rate -burst_size = 20 # Allow bursts up to this size -cleanup_interval_s = 60 # Clean old clients every minute -``` +The `stream.buffer_size` setting controls how many log entries can be queued per client: +- **Small buffers (100-500)**: Lower memory usage, clients skip entries during bursts +- **Default (1000)**: Good balance for most use cases +- **Large buffers (5000+)**: Handle burst traffic better, higher memory usage -Rate limiting uses the `X-Forwarded-For` header if present, falling back to `RemoteAddr`. +When a client's buffer is full, new messages are skipped for that client until it catches up. The client remains connected and will receive future messages once buffer space is available. + +### Check Interval + +The `monitor.check_interval_ms` setting controls file polling frequency: +- **Fast (10-50ms)**: Near real-time updates, higher CPU usage +- **Default (100ms)**: Good balance +- **Slow (500ms+)**: Lower CPU usage, more latency + +### Rate Limiting + +When to enable rate limiting: +- Internet-facing deployments +- Shared environments +- Protection against misbehaving clients + +Rate limiting applies only to establishing SSE connections, not to individual messages. Once connected, clients receive all messages (subject to buffer capacity). + +## Troubleshooting + +### Client Missing Messages + +If clients miss messages during bursts: +1. Check `total_dropped` and `clients_with_drops` in status endpoint +2. Increase `stream.buffer_size` to handle larger bursts +3. Messages are skipped when buffer is full, but clients stay connected + +### High Memory Usage + +If memory usage is high: +1. Reduce `stream.buffer_size` +2. Enable rate limiting to limit concurrent connections +3. Each client uses `buffer_size * avg_message_size` memory + +### Browser Stops Receiving Updates + +This shouldn't happen with the current implementation. If it does: +1. Check browser developer console for errors +2. Verify no proxy/firewall is timing out the connection +3. Ensure reverse proxy (if used) doesn't buffer SSE responses + +## File Rotation Detection + +LogWisp automatically detects log file rotation using multiple methods: +- Inode changes (Linux/Unix) +- File size decrease +- Modification time reset +- Read position beyond file size + +When rotation is detected, LogWisp: +1. Logs a rotation event +2. Resets read position to beginning +3. Continues streaming from new file + +## Security Notes + +1. **No built-in authentication** - Use a reverse proxy for auth +2. **No TLS support** - Use a reverse proxy for HTTPS +3. **Path validation** - Only specified paths can be monitored +4. **Directory traversal protection** - Paths containing ".." are rejected +5. **Rate limiting** - Optional but recommended for public deployments +6. **ANSI escape sequences** - Only enable color mode for trusted log sources + +## Design Decisions + +- **Unix philosophy**: Single purpose - stream logs +- **SSE over WebSocket**: Simpler, works everywhere, built-in reconnect +- **No database**: Stateless operation, instant startup +- **Atomic config management**: Using LixenWraith/config package +- **Graceful shutdown**: Proper cleanup on SIGINT/SIGTERM +- **Platform agnostic**: POSIX-compliant where possible ## Building from Source @@ -356,32 +543,12 @@ Requirements: - Go 1.23 or later ```bash +git clone https://github.com/yourusername/logwisp +cd logwisp go mod download go build -o logwisp ./src/cmd/logwisp ``` -## File Rotation Detection - -LogWisp automatically detects log file rotation by: -- Monitoring file inode changes (Linux/Unix) -- Detecting file size decrease -- Resetting read position when rotation is detected - -## Security Notes - -1. **No built-in authentication** - Use a reverse proxy for auth -2. **No TLS support** - Use a reverse proxy for HTTPS -3. **Path validation** - Monitors only specified paths -4. **Rate limiting** - Optional but recommended for internet-facing deployments - -## Design Decisions - -- **Unix philosophy**: Single purpose - stream logs -- **No CLI arguments**: Configuration via file and environment only -- **SSE over WebSocket**: Simpler, works everywhere -- **Atomic config management**: Using LixenWraith/config package -- **Graceful shutdown**: Proper cleanup on SIGINT/SIGTERM - ## License BSD-3-Clause \ No newline at end of file diff --git a/assets/logo.svg b/assets/logo.svg new file mode 100644 index 0000000..60d80b9 --- /dev/null +++ b/assets/logo.svg @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config/logwisp.toml b/config/logwisp.toml index a3e4bd9..cf7c71f 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -1,70 +1,112 @@ -# LogWisp Configuration +# LogWisp Configuration Template # Default location: ~/.config/logwisp.toml -# Override with: LOGWISP_CONFIG_DIR and LOGWISP_CONFIG_FILE +# +# Configuration precedence (highest to lowest): +# 1. Command-line arguments +# 2. Environment variables (LOGWISP_ prefix) +# 3. This configuration file +# 4. Built-in defaults +# +# All settings shown below with their default values # Port to listen on # Environment: LOGWISP_PORT +# CLI: --port PORT +# Default: 8080 port = 8080 -# Environment: LOGWISP_COLOR -color = false [monitor] # How often to check for file changes (milliseconds) +# Lower values = more responsive but higher CPU usage # Environment: LOGWISP_MONITOR_CHECK_INTERVAL_MS +# CLI: --check-interval MS +# Default: 100 check_interval_ms = 100 -# Paths to monitor +# Paths to monitor for log files # Environment: LOGWISP_MONITOR_TARGETS (format: "path:pattern:isfile,path2:pattern2:isfile") -# Example: LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/app/app.log::true" - -# Monitor all .log files in current directory +# CLI: logwisp [path[:pattern[:isfile]]] ... +# Default: Monitor current directory for *.log files [[monitor.targets]] -path = "./" -pattern = "*.log" -is_file = false +path = "./" # Directory or file path to monitor +pattern = "*.log" # Glob pattern for directory monitoring (ignored for files) +is_file = false # true = monitor specific file, false = monitor directory -# Monitor all logs in /var/log -[[monitor.targets]] -path = "/var/log" -pattern = "*.log" -is_file = false +# Additional target examples (uncomment to use): -# Monitor specific application log file -#[[monitor.targets]] -#path = "/home/user/app/app.log" -#pattern = "" # Ignored for files -#is_file = true +# # Monitor specific log file +# [[monitor.targets]] +# path = "/var/log/application.log" +# pattern = "" # Pattern ignored when is_file = true +# is_file = true -# Monitor nginx access logs -#[[monitor.targets]] -#path = "/var/log/nginx" -#pattern = "access*.log" -#is_file = false +# # Monitor system logs +# [[monitor.targets]] +# path = "/var/log" +# pattern = "*.log" +# is_file = false -# Monitor systemd journal exported logs -#[[monitor.targets]] -#path = "/var/log/journal" -#pattern = "*.log" -#is_file = false +# # Monitor nginx access logs with pattern +# [[monitor.targets]] +# path = "/var/log/nginx" +# pattern = "access*.log" +# is_file = false + +# # Monitor journal export directory +# [[monitor.targets]] +# path = "/var/log/journal" +# pattern = "*.log" +# is_file = false + +# # Monitor multiple application logs +# [[monitor.targets]] +# path = "/opt/myapp/logs" +# pattern = "app-*.log" +# is_file = false [stream] # Buffer size for each client connection +# Number of log entries that can be queued per client +# When buffer is full, new messages are skipped (not sent to that client) +# Increase for burst traffic, decrease for memory conservation # Environment: LOGWISP_STREAM_BUFFER_SIZE -buffer_size = 10000 +# CLI: --buffer-size SIZE +# Default: 1000 +buffer_size = 1000 [stream.rate_limit] -# Enable rate limiting +# Enable rate limiting per client IP +# Prevents resource exhaustion from misbehaving clients # Environment: LOGWISP_STREAM_RATE_LIMIT_ENABLED -enabled = true +# CLI: --rate-limit +# Default: false +enabled = false -# Requests per second per client +# Sustained requests per second allowed per client +# Clients can make this many requests per second continuously # Environment: LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC +# CLI: --rate-requests N +# Default: 10 requests_per_second = 10 -# Burst size (max requests at once) +# Maximum burst size per client +# Allows temporary bursts above the sustained rate +# Should be >= requests_per_second # Environment: LOGWISP_STREAM_RATE_LIMIT_BURST_SIZE +# CLI: --rate-burst N +# Default: 20 burst_size = 20 -# How often to clean up old client limiters (seconds) -# Environment: LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL -cleanup_interval_s = 5 +# How often to clean up inactive client rate limiters (seconds) +# Clients inactive for 2x this duration are removed from tracking +# Lower values = more frequent cleanup, higher values = less overhead +# Environment: LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL_S +# Default: 60 +cleanup_interval_s = 60 + +# Production configuration example: +# [stream.rate_limit] +# enabled = true +# requests_per_second = 100 # Higher limit for production +# burst_size = 200 # Allow larger bursts +# cleanup_interval_s = 300 # Less frequent cleanup \ No newline at end of file diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go index a6f7b28..f3bc523 100644 --- a/src/cmd/logwisp/main.go +++ b/src/cmd/logwisp/main.go @@ -21,9 +21,10 @@ import ( ) func main() { - // CHANGED: Parse flags manually without init() + // Parse flags manually without init() var colorMode bool flag.BoolVar(&colorMode, "c", false, "Enable color pass-through for escape codes in logs") + flag.BoolVar(&colorMode, "color", false, "Enable color pass-through for escape codes in logs") // Additional CLI flags that override config var ( @@ -98,7 +99,6 @@ func main() { var wg sync.WaitGroup // Create components - // colorMode is now separate from config streamer := stream.NewWithOptions(cfg.Stream.BufferSize, colorMode) mon := monitor.New(streamer.Publish) @@ -145,7 +145,7 @@ func main() { status := map[string]interface{}{ "service": "LogWisp", - "version": "2.0.0", // CHANGED: Version bump for config integration + "version": "2.0.0", "port": cfg.Port, "color_mode": colorMode, "config": map[string]interface{}{ @@ -191,7 +191,6 @@ func main() { if colorMode { fmt.Println("Color pass-through enabled") } - // CHANGED: Log config source information fmt.Printf("Config loaded from: %s\n", config.GetConfigPath()) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { diff --git a/src/internal/config/config.go b/src/internal/config/config.go index ca07bfa..e102d75 100644 --- a/src/internal/config/config.go +++ b/src/internal/config/config.go @@ -67,18 +67,15 @@ func defaults() *Config { } // Load reads configuration using lixenwraith/config Builder pattern -// CHANGED: Now uses config.Builder for all source handling func Load() (*Config, error) { configPath := GetConfigPath() - // CHANGED: Use Builder pattern with custom environment transform cfg, err := lconfig.NewBuilder(). WithDefaults(defaults()). WithEnvPrefix("LOGWISP_"). WithFile(configPath). WithEnvTransform(customEnvTransform). WithSources( - // CHANGED: CLI args removed here - handled separately in LoadWithCLI lconfig.SourceEnv, lconfig.SourceFile, lconfig.SourceDefault, @@ -107,7 +104,6 @@ func Load() (*Config, error) { } // LoadWithCLI loads configuration and applies CLI arguments -// CHANGED: New function that properly integrates CLI args with config package func LoadWithCLI(cliArgs []string) (*Config, error) { configPath := GetConfigPath() @@ -118,7 +114,7 @@ func LoadWithCLI(cliArgs []string) (*Config, error) { WithDefaults(defaults()). WithEnvPrefix("LOGWISP_"). WithFile(configPath). - WithArgs(convertedArgs). // CHANGED: Use WithArgs for CLI + WithArgs(convertedArgs). WithEnvTransform(customEnvTransform). WithSources( lconfig.SourceCLI, // CLI highest priority @@ -148,16 +144,14 @@ func LoadWithCLI(cliArgs []string) (*Config, error) { return finalConfig, finalConfig.validate() } -// CHANGED: Custom environment transform that handles LOGWISP_ prefix more flexibly +// customEnvTransform handles LOGWISP_ prefix environment variables func customEnvTransform(path string) string { // Standard transform env := strings.ReplaceAll(path, ".", "_") env = strings.ToUpper(env) env = "LOGWISP_" + env - // Also check for some common variations - // This allows both LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC - // and LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SECOND + // Handle common variations switch env { case "LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SECOND": if _, exists := os.LookupEnv("LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC"); exists { @@ -172,7 +166,7 @@ func customEnvTransform(path string) string { return env } -// CHANGED: Convert CLI args to config package format +// convertCLIArgs converts CLI args to config package format func convertCLIArgs(args []string) []string { var converted []string @@ -194,7 +188,6 @@ func convertCLIArgs(args []string) []string { } // GetConfigPath returns the configuration file path -// CHANGED: Exported and simplified - now just returns the path, no manual env handling func GetConfigPath() string { // Check explicit config file paths if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" { @@ -219,7 +212,7 @@ func GetConfigPath() string { return "logwisp.toml" } -// CHANGED: Special handling for comma-separated monitor targets env var +// handleMonitorTargetsEnv handles comma-separated monitor targets env var func handleMonitorTargetsEnv(cfg *lconfig.Config) error { if targetsStr := os.Getenv("LOGWISP_MONITOR_TARGETS"); targetsStr != "" { // Clear any existing targets from file/defaults diff --git a/src/internal/stream/stream.go b/src/internal/stream/stream.go index 2a616be..be8e6d4 100644 --- a/src/internal/stream/stream.go +++ b/src/internal/stream/stream.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "sync" + "sync/atomic" "time" "logwisp/src/internal/monitor" @@ -22,13 +23,16 @@ type Streamer struct { done chan struct{} colorMode bool wg sync.WaitGroup + + // Metrics + totalDropped atomic.Int64 } type clientConnection struct { id string channel chan monitor.LogEntry lastActivity time.Time - dropped int64 // Count of dropped messages + dropped atomic.Int64 // Track per-client dropped messages } // New creates a new SSE streamer @@ -53,14 +57,10 @@ func NewWithOptions(bufferSize int, colorMode bool) *Streamer { return s } -// run manages client connections with timeout cleanup +// run manages client connections - SIMPLIFIED: no forced disconnections func (s *Streamer) run() { defer s.wg.Done() - // Add periodic cleanup for stale/slow clients - cleanupTicker := time.NewTicker(30 * time.Second) - defer cleanupTicker.Stop() - for { select { case c := <-s.register: @@ -79,47 +79,27 @@ func (s *Streamer) run() { case entry := <-s.broadcast: s.mu.RLock() now := time.Now() - var toRemove []string - for id, client := range s.clients { + for _, client := range s.clients { select { case client.channel <- entry: + // Successfully sent client.lastActivity = now + client.dropped.Store(0) // Reset dropped counter on success default: - // Track dropped messages and remove slow clients - client.dropped++ - // Remove clients that have dropped >100 messages or been inactive >2min - if client.dropped > 100 || now.Sub(client.lastActivity) > 2*time.Minute { - toRemove = append(toRemove, id) + // Buffer full - skip this message for this client + // Don't disconnect, just track dropped messages + dropped := client.dropped.Add(1) + s.totalDropped.Add(1) + + // Log significant drop milestones for monitoring + if dropped == 100 || dropped == 1000 || dropped%10000 == 0 { + // Could add logging here if needed } } } s.mu.RUnlock() - // Remove slow/stale clients outside the read lock - if len(toRemove) > 0 { - s.mu.Lock() - for _, id := range toRemove { - if client, ok := s.clients[id]; ok { - close(client.channel) - delete(s.clients, id) - } - } - s.mu.Unlock() - } - - case <-cleanupTicker.C: - // Periodic cleanup of inactive clients - s.mu.Lock() - now := time.Now() - for id, client := range s.clients { - if now.Sub(client.lastActivity) > 5*time.Minute { - close(client.channel) - delete(s.clients, id) - } - } - s.mu.Unlock() - case <-s.done: s.mu.Lock() for _, client := range s.clients { @@ -138,18 +118,21 @@ func (s *Streamer) Publish(entry monitor.LogEntry) { case s.broadcast <- entry: // Sent to broadcast channel default: - // Drop entry if broadcast buffer full, log occurrence - // This prevents memory exhaustion under high load + // Broadcast buffer full - drop the message globally + s.totalDropped.Add(1) } } -// ServeHTTP implements http.Handler for SSE +// ServeHTTP implements http.Handler for SSE - SIMPLIFIED func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Set SSE headers w.Header().Set("Content-Type", "text/event-stream") w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") - w.Header().Set("X-Accel-Buffering", "no") + w.Header().Set("X-Accel-Buffering", "no") // Disable nginx buffering + + // SECURITY: Prevent XSS + w.Header().Set("X-Content-Type-Options", "nosniff") // Create client clientID := fmt.Sprintf("%d", time.Now().UnixNano()) @@ -159,7 +142,6 @@ func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { id: clientID, channel: ch, lastActivity: time.Now(), - dropped: 0, } // Register client @@ -169,41 +151,29 @@ func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { }() // Send initial connection event - fmt.Fprintf(w, "event: connected\ndata: {\"client_id\":\"%s\"}\n\n", clientID) + fmt.Fprintf(w, "event: connected\ndata: {\"client_id\":\"%s\",\"buffer_size\":%d}\n\n", + clientID, s.bufferSize) if flusher, ok := w.(http.Flusher); ok { flusher.Flush() } - // Create ticker for heartbeat + // Create ticker for heartbeat - keeps connection alive through proxies ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() - // Add timeout for slow clients - clientTimeout := time.NewTimer(10 * time.Minute) - defer clientTimeout.Stop() - - // Stream events + // Stream events until client disconnects for { select { case <-r.Context().Done(): + // Client disconnected return case entry, ok := <-ch: if !ok { - // Channel was closed (client removed due to slowness) - fmt.Fprintf(w, "event: disconnected\ndata: {\"reason\":\"slow_client\"}\n\n") - if flusher, ok := w.(http.Flusher); ok { - flusher.Flush() - } + // Channel closed return } - // Reset client timeout on successful read - if !clientTimeout.Stop() { - <-clientTimeout.C - } - clientTimeout.Reset(10 * time.Minute) - // Process entry for color if needed if s.colorMode { entry = s.processColorEntry(entry) @@ -220,19 +190,11 @@ func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } case <-ticker.C: - // Heartbeat with UTC timestamp - fmt.Fprintf(w, ": heartbeat %s\n\n", time.Now().UTC().Format("2006-01-02T15:04:05.000000Z07:00")) + // Send heartbeat as SSE comment + fmt.Fprintf(w, ": heartbeat %s\n\n", time.Now().UTC().Format(time.RFC3339)) if flusher, ok := w.(http.Flusher); ok { flusher.Flush() } - - case <-clientTimeout.C: - // Client timeout - close connection - fmt.Fprintf(w, "event: timeout\ndata: {\"reason\":\"client_timeout\"}\n\n") - if flusher, ok := w.(http.Flusher); ok { - flusher.Flush() - } - return } } } @@ -246,11 +208,8 @@ func (s *Streamer) Stop() { close(s.broadcast) } -// Enhanced color processing with proper ANSI handling +// processColorEntry preserves ANSI codes in JSON func (s *Streamer) processColorEntry(entry monitor.LogEntry) monitor.LogEntry { - // For color mode, we preserve ANSI codes but ensure they're properly handled - // The JSON marshaling will escape them correctly for transmission - // Client-side handling is required for proper display return entry } @@ -263,13 +222,24 @@ func (s *Streamer) Stats() map[string]interface{} { "active_clients": len(s.clients), "buffer_size": s.bufferSize, "color_mode": s.colorMode, + "total_dropped": s.totalDropped.Load(), } - totalDropped := int64(0) - for _, client := range s.clients { - totalDropped += client.dropped + // Include per-client dropped counts if any are significant + var clientsWithDrops []map[string]interface{} + for id, client := range s.clients { + dropped := client.dropped.Load() + if dropped > 0 { + clientsWithDrops = append(clientsWithDrops, map[string]interface{}{ + "id": id, + "dropped": dropped, + }) + } + } + + if len(clientsWithDrops) > 0 { + stats["clients_with_drops"] = clientsWithDrops } - stats["total_dropped"] = totalDropped return stats } \ No newline at end of file