From bd13103a818179570236f6625fd41d73fcdfe17766614d01277c5991194093ec Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Tue, 1 Jul 2025 14:12:20 -0400 Subject: [PATCH] v0.1.1 improved config, added ratelimiter (buggy), readme not fully updated --- README.md | 321 ++++++++++++++++++++++--- config/logwisp.toml | 56 ++++- doc/files.md | 39 +++ doc/placeholder | 0 go.mod | 14 +- go.sum | 14 ++ src/cmd/logwisp/main.go | 176 +++++++++++++- src/internal/config/config.go | 273 +++++++++++++++++---- src/internal/middleware/ratelimiter.go | 126 ++++++++++ src/internal/monitor/monitor.go | 319 +++++++++++++++++++----- src/internal/stream/stream.go | 185 +++++++++++--- 11 files changed, 1329 insertions(+), 194 deletions(-) create mode 100644 doc/files.md delete mode 100644 doc/placeholder create mode 100644 src/internal/middleware/ratelimiter.go diff --git a/README.md b/README.md index 40378b1..c26befc 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,14 @@ LogWisp follows the Unix philosophy: do one thing and do it well. It monitors lo ## Features -- Monitors multiple files and directories +- Monitors multiple files and directories simultaneously - Streams log updates in real-time via SSE - Supports both plain text and JSON formatted logs - Automatic file rotation detection +- Configurable rate limiting +- Environment variable support - Simple TOML configuration -- No authentication or complex features - use a reverse proxy if needed +- Atomic configuration management ## Quick Start @@ -34,32 +36,219 @@ curl -N http://localhost:8080/stream ## Configuration -LogWisp looks for configuration at `~/.config/logwisp.toml`. If not found, it uses sensible defaults. +LogWisp uses a three-level configuration hierarchy: + +1. **Environment variables** (highest priority) +2. **Configuration file** (~/.config/logwisp.toml) +3. **Default values** (lowest priority) + +### Configuration File Location + +Default: `~/.config/logwisp.toml` + +Override with environment variables: +- `LOGWISP_CONFIG_DIR` - Directory containing config file +- `LOGWISP_CONFIG_FILE` - Config filename (absolute or relative) + +Examples: +```bash +# Use config from current directory +LOGWISP_CONFIG_DIR=. ./logwisp + +# Use specific config file +LOGWISP_CONFIG_FILE=/etc/logwisp/prod.toml ./logwisp + +# Use custom directory and filename +LOGWISP_CONFIG_DIR=/opt/configs LOGWISP_CONFIG_FILE=myapp.toml ./logwisp +``` + +### Environment Variables + +All configuration values can be overridden via environment variables: + +| Environment Variable | Config Path | Description | +|---------------------|-------------|-------------| +| `LOGWISP_PORT` | `port` | HTTP listen port | +| `LOGWISP_MONITOR_CHECK_INTERVAL_MS` | `monitor.check_interval_ms` | File check interval | +| `LOGWISP_MONITOR_TARGETS` | `monitor.targets` | Comma-separated targets | +| `LOGWISP_STREAM_BUFFER_SIZE` | `stream.buffer_size` | Client buffer size | +| `LOGWISP_STREAM_RATE_LIMIT_ENABLED` | `stream.rate_limit.enabled` | Enable rate limiting | +| `LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC` | `stream.rate_limit.requests_per_second` | Rate limit | +| `LOGWISP_STREAM_RATE_LIMIT_BURST_SIZE` | `stream.rate_limit.burst_size` | Burst size | +| `LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL` | `stream.rate_limit.cleanup_interval_s` | Cleanup interval | + +### Monitor Targets Format + +The `LOGWISP_MONITOR_TARGETS` environment variable uses a special format: +``` +path:pattern:isfile,path2:pattern2:isfile +``` + +Examples: +```bash +# Monitor directory and specific file +LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/app/app.log::true" ./logwisp + +# Multiple directories +LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/opt/app/logs:app-*.log:false" ./logwisp +``` + +### Example Configuration -Example configuration: ```toml port = 8080 [monitor] check_interval_ms = 100 +# Monitor directory (all .log files) [[monitor.targets]] path = "/var/log" pattern = "*.log" +is_file = false +# Monitor specific file [[monitor.targets]] -path = "/home/user/app/logs" -pattern = "app-*.log" +path = "/app/logs/app.log" +pattern = "" # Ignored for files +is_file = true + +# Monitor with specific pattern +[[monitor.targets]] +path = "/var/log/nginx" +pattern = "access*.log" +is_file = false [stream] buffer_size = 1000 + +[stream.rate_limit] +enabled = true +requests_per_second = 10 +burst_size = 20 +cleanup_interval_s = 60 ``` +## Color Support + +LogWisp can pass through ANSI color escape codes from monitored logs to SSE clients using the `-c` flag. + +```bash +# Enable color pass-through +./logwisp -c + +# Or via systemd +ExecStart=/opt/logwisp/bin/logwisp -c +``` + +## How It Works + +When color mode is enabled (`-c` flag), LogWisp preserves ANSI escape codes in log messages. These are properly JSON-escaped in the SSE stream. + +### Example Log with Colors + +Original log file content: +``` +\033[31mERROR\033[0m: Database connection failed +\033[33mWARN\033[0m: High memory usage detected +\033[32mINFO\033[0m: Service started successfully +``` + +SSE output with `-c`: +```json +{ + "time": "2024-01-01T12:00:00.123456Z", + "source": "app.log", + "message": "\u001b[31mERROR\u001b[0m: Database connection failed" +} +``` + +## Client-Side Handling + +### Terminal Clients + +For terminal-based clients (like curl), the escape codes will render as colors: + +```bash +# This will show colored output in terminals that support ANSI codes +curl -N http://localhost:8080/stream | jq -r '.message' +``` + +### Web Clients + +For web-based clients, you'll need to convert ANSI codes to HTML: + +```javascript +// Example using ansi-to-html library +const AnsiToHtml = require('ansi-to-html'); +const convert = new AnsiToHtml(); + +eventSource.onmessage = (event) => { + const data = JSON.parse(event.data); + const html = convert.toHtml(data.message); + document.getElementById('log').innerHTML += html + '
'; +}; +``` + +### Custom Processing + +```python +# Python example with colorama +import json +import colorama +from colorama import init + +init() # Initialize colorama for Windows support + +# Process SSE stream +for line in stream: + if line.startswith('data: '): + data = json.loads(line[6:]) + # Colorama will handle ANSI codes automatically + print(data['message']) +``` + +### Common ANSI Color Codes + +| Code | Color/Style | +|------|-------------| +| `\033[0m` | Reset | +| `\033[1m` | Bold | +| `\033[31m` | Red | +| `\033[32m` | Green | +| `\033[33m` | Yellow | +| `\033[34m` | Blue | +| `\033[35m` | Magenta | +| `\033[36m` | Cyan | + +### Limitations + +1. **JSON Escaping**: ANSI codes are JSON-escaped in the stream (e.g., `\033` becomes `\u001b`) +2. **Client Support**: The client must support or convert ANSI codes +3. **Performance**: No significant impact, but slightly larger message sizes + +### Security Note + +Color codes are passed through as-is. Ensure monitored logs come from trusted sources to avoid terminal escape sequence attacks. + +### Disabling Colors + +To strip color codes instead of passing them through: +- Don't use the `-c` flag +- Or set up a preprocessing pipeline: + ```bash + tail -f colored.log | sed 's/\x1b\[[0-9;]*m//g' > plain.log + ``` + ## API -- `GET /stream` - Server-Sent Events stream of log entries +### Endpoints + +- `GET /stream` - Server-Sent Events stream of log entries +- `GET /status` - Service status information + +### Log Entry Format -Log entry format: ```json { "time": "2024-01-01T12:00:00Z", @@ -70,52 +259,54 @@ Log entry format: } ``` -## Building from Source - -Requirements: -- Go 1.23 or later - -```bash -go mod download -go build -o logwisp ./src/cmd/logwisp -``` - ## Usage Examples ### Basic Usage ```bash -# Start LogWisp (monitors current directory by default) +# Start with defaults ./logwisp -# In another terminal, view the stream +# View logs curl -N http://localhost:8080/stream ``` -### With Custom Config +### With Environment Variables ```bash -# Create config +# Change port and add rate limiting +LOGWISP_PORT=9090 \ +LOGWISP_STREAM_RATE_LIMIT_ENABLED=true \ +LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC=5 \ +./logwisp +``` + +### Monitor Multiple Locations +```bash +# Via environment variable +LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/app/logs:*.json:false,/tmp/debug.log::true" \ +./logwisp + +# Or via config file cat > ~/.config/logwisp.toml << EOF -port = 9090 +[[monitor.targets]] +path = "/var/log" +pattern = "*.log" +is_file = false [[monitor.targets]] -path = "/var/log/nginx" -pattern = "*.log" -EOF +path = "/app/logs" +pattern = "*.json" +is_file = false -# Run -./logwisp +[[monitor.targets]] +path = "/tmp/debug.log" +is_file = true +EOF ``` ### Production Deployment -For production use, consider: +Example systemd service with environment overrides: -1. Run behind a reverse proxy (nginx, caddy) for SSL/TLS -2. Use systemd or similar for process management -3. Add authentication at the proxy level if needed -4. Set appropriate file permissions on monitored logs - -Example systemd service: ```ini [Unit] Description=LogWisp Log Streaming Service @@ -127,17 +318,69 @@ User=logwisp ExecStart=/usr/local/bin/logwisp Restart=always +# Environment overrides +Environment="LOGWISP_PORT=8080" +Environment="LOGWISP_STREAM_RATE_LIMIT_ENABLED=true" +Environment="LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC=100" +Environment="LOGWISP_MONITOR_TARGETS=/var/log:*.log:false,/opt/app/logs:*.log:false" + +# Security hardening +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadOnlyPaths=/ +ReadWritePaths=/var/log + [Install] WantedBy=multi-user.target ``` +## Rate Limiting + +When enabled, rate limiting is applied per client IP address: + +```toml +[stream.rate_limit] +enabled = true +requests_per_second = 10 # Sustained rate +burst_size = 20 # Allow bursts up to this size +cleanup_interval_s = 60 # Clean old clients every minute +``` + +Rate limiting uses the `X-Forwarded-For` header if present, falling back to `RemoteAddr`. + +## Building from Source + +Requirements: +- Go 1.23 or later + +```bash +go mod download +go build -o logwisp ./src/cmd/logwisp +``` + +## File Rotation Detection + +LogWisp automatically detects log file rotation by: +- Monitoring file inode changes (Linux/Unix) +- Detecting file size decrease +- Resetting read position when rotation is detected + +## Security Notes + +1. **No built-in authentication** - Use a reverse proxy for auth +2. **No TLS support** - Use a reverse proxy for HTTPS +3. **Path validation** - Monitors only specified paths +4. **Rate limiting** - Optional but recommended for internet-facing deployments + ## Design Decisions -- **No built-in authentication**: Use a reverse proxy -- **No TLS**: Use a reverse proxy -- **No complex features**: Follows Unix philosophy -- **File-based configuration**: Simple, no CLI args needed +- **Unix philosophy**: Single purpose - stream logs +- **No CLI arguments**: Configuration via file and environment only - **SSE over WebSocket**: Simpler, works everywhere +- **Atomic config management**: Using LixenWraith/config package +- **Graceful shutdown**: Proper cleanup on SIGINT/SIGTERM ## License diff --git a/config/logwisp.toml b/config/logwisp.toml index bace771..a3e4bd9 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -1,26 +1,70 @@ -# Example configuration for LogWisp -# Default directory: ~/.config/logwisp.toml +# LogWisp Configuration +# Default location: ~/.config/logwisp.toml +# Override with: LOGWISP_CONFIG_DIR and LOGWISP_CONFIG_FILE # Port to listen on +# Environment: LOGWISP_PORT port = 8080 +# Environment: LOGWISP_COLOR +color = false [monitor] # How often to check for file changes (milliseconds) +# Environment: LOGWISP_MONITOR_CHECK_INTERVAL_MS check_interval_ms = 100 # Paths to monitor +# Environment: LOGWISP_MONITOR_TARGETS (format: "path:pattern:isfile,path2:pattern2:isfile") +# Example: LOGWISP_MONITOR_TARGETS="/var/log:*.log:false,/app/app.log::true" + +# Monitor all .log files in current directory [[monitor.targets]] path = "./" pattern = "*.log" +is_file = false +# Monitor all logs in /var/log [[monitor.targets]] -path = "/var/log/" +path = "/var/log" pattern = "*.log" +is_file = false +# Monitor specific application log file #[[monitor.targets]] -#path = "/home/user/app/logs" -#pattern = "app-*.log" +#path = "/home/user/app/app.log" +#pattern = "" # Ignored for files +#is_file = true + +# Monitor nginx access logs +#[[monitor.targets]] +#path = "/var/log/nginx" +#pattern = "access*.log" +#is_file = false + +# Monitor systemd journal exported logs +#[[monitor.targets]] +#path = "/var/log/journal" +#pattern = "*.log" +#is_file = false [stream] # Buffer size for each client connection -buffer_size = 1000 +# Environment: LOGWISP_STREAM_BUFFER_SIZE +buffer_size = 10000 + +[stream.rate_limit] +# Enable rate limiting +# Environment: LOGWISP_STREAM_RATE_LIMIT_ENABLED +enabled = true + +# Requests per second per client +# Environment: LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC +requests_per_second = 10 + +# Burst size (max requests at once) +# Environment: LOGWISP_STREAM_RATE_LIMIT_BURST_SIZE +burst_size = 20 + +# How often to clean up old client limiters (seconds) +# Environment: LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL +cleanup_interval_s = 5 diff --git a/doc/files.md b/doc/files.md new file mode 100644 index 0000000..92fe2a6 --- /dev/null +++ b/doc/files.md @@ -0,0 +1,39 @@ +# Directory structure: + +``` +logwisp/ +├── build.sh +├── go.mod +├── go.sum +├── README.md +├── test_logwisp.sh +├── examples/ +│ └── env_usage.sh +└── src/ +├── cmd/ +│ └── logwisp/ +│ └── main.go +└── internal/ +├── config/ +│ └── config.go # Uses LixenWraith/config +├── middleware/ +│ └── ratelimit.go # Rate limiting middleware +├── monitor/ +│ └── monitor.go # Enhanced file/directory monitoring +└── stream/ +└── stream.go # SSE streaming handler +``` + +# Configuration locations: +~/.config/logwisp.toml # Default config location +$LOGWISP_CONFIG_DIR/ # Override via environment +$LOGWISP_CONFIG_FILE # Override via environment + +# Environment variables: +LOGWISP_CONFIG_DIR # Config directory override +LOGWISP_CONFIG_FILE # Config filename override +LOGWISP_PORT # Port override +LOGWISP_MONITOR_CHECK_INTERVAL_MS # Check interval override +LOGWISP_MONITOR_TARGETS # Targets override (special format) +LOGWISP_STREAM_BUFFER_SIZE # Buffer size override +LOGWISP_STREAM_RATE_LIMIT_* # Rate limit overrides \ No newline at end of file diff --git a/doc/placeholder b/doc/placeholder deleted file mode 100644 index 473a0f4..0000000 diff --git a/go.mod b/go.mod index b7b27aa..5e1393f 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,15 @@ module logwisp -go 1.23.4 +go 1.24.2 -require github.com/BurntSushi/toml v1.5.0 +toolchain go1.24.4 + +require ( + github.com/lixenwraith/config v0.0.0-20250701170607-8515fa0543b6 + golang.org/x/time v0.12.0 +) + +require ( + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect +) diff --git a/go.sum b/go.sum index a0e2fd8..98821ae 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,16 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/lixenwraith/config v0.0.0-20250701170607-8515fa0543b6 h1:qE4SpAJWFaLkdRyE0FjTPBBRYE7LOvcmRCB5p86W73Q= +github.com/lixenwraith/config v0.0.0-20250701170607-8515fa0543b6/go.mod h1:4wPJ3HnLrYrtUwTinngCsBgtdIXsnxkLa7q4KAIbwY8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go index 3ca791a..a6f7b28 100644 --- a/src/cmd/logwisp/main.go +++ b/src/cmd/logwisp/main.go @@ -3,21 +3,85 @@ package main import ( "context" + "encoding/json" + "flag" "fmt" "net/http" "os" "os/signal" + "strings" + "sync" "syscall" "time" "logwisp/src/internal/config" + "logwisp/src/internal/middleware" "logwisp/src/internal/monitor" "logwisp/src/internal/stream" ) func main() { - // Load configuration - cfg, err := config.Load() + // CHANGED: Parse flags manually without init() + var colorMode bool + flag.BoolVar(&colorMode, "c", false, "Enable color pass-through for escape codes in logs") + + // Additional CLI flags that override config + var ( + port = flag.Int("port", 0, "HTTP port (overrides config)") + bufferSize = flag.Int("buffer-size", 0, "Stream buffer size (overrides config)") + checkInterval = flag.Int("check-interval", 0, "File check interval in ms (overrides config)") + rateLimit = flag.Bool("rate-limit", false, "Enable rate limiting (overrides config)") + rateRequests = flag.Int("rate-requests", 0, "Rate limit requests/sec (overrides config)") + rateBurst = flag.Int("rate-burst", 0, "Rate limit burst size (overrides config)") + configFile = flag.String("config", "", "Config file path (overrides LOGWISP_CONFIG_FILE)") + ) + + flag.Parse() + + // Set config file env var if specified via CLI + if *configFile != "" { + os.Setenv("LOGWISP_CONFIG_FILE", *configFile) + } + + // Build CLI override args for config package + var cliArgs []string + if *port > 0 { + cliArgs = append(cliArgs, fmt.Sprintf("--port=%d", *port)) + } + if *bufferSize > 0 { + cliArgs = append(cliArgs, fmt.Sprintf("--stream.buffer_size=%d", *bufferSize)) + } + if *checkInterval > 0 { + cliArgs = append(cliArgs, fmt.Sprintf("--monitor.check_interval_ms=%d", *checkInterval)) + } + if flag.Lookup("rate-limit").DefValue != flag.Lookup("rate-limit").Value.String() { + // Rate limit flag was explicitly set + cliArgs = append(cliArgs, fmt.Sprintf("--stream.rate_limit.enabled=%v", *rateLimit)) + } + if *rateRequests > 0 { + cliArgs = append(cliArgs, fmt.Sprintf("--stream.rate_limit.requests_per_second=%d", *rateRequests)) + } + if *rateBurst > 0 { + cliArgs = append(cliArgs, fmt.Sprintf("--stream.rate_limit.burst_size=%d", *rateBurst)) + } + + // Parse remaining args as monitor targets + for _, arg := range flag.Args() { + if strings.Contains(arg, ":") { + // Format: path:pattern:isfile + cliArgs = append(cliArgs, fmt.Sprintf("--monitor.targets.add=%s", arg)) + } else if stat, err := os.Stat(arg); err == nil { + // Auto-detect file vs directory + if stat.IsDir() { + cliArgs = append(cliArgs, fmt.Sprintf("--monitor.targets.add=%s:*.log:false", arg)) + } else { + cliArgs = append(cliArgs, fmt.Sprintf("--monitor.targets.add=%s::true", arg)) + } + } + } + + // Load configuration with CLI overrides + cfg, err := config.LoadWithCLI(cliArgs) if err != nil { fmt.Fprintf(os.Stderr, "Failed to load config: %v\n", err) os.Exit(1) @@ -25,19 +89,25 @@ func main() { // Create context for graceful shutdown ctx, cancel := context.WithCancel(context.Background()) - defer cancel() // Setup signal handling sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + // WaitGroup for tracking all goroutines + var wg sync.WaitGroup + // Create components - streamer := stream.New(cfg.Stream.BufferSize) + // colorMode is now separate from config + streamer := stream.NewWithOptions(cfg.Stream.BufferSize, colorMode) mon := monitor.New(streamer.Publish) + // Set monitor check interval from config + mon.SetCheckInterval(time.Duration(cfg.Monitor.CheckIntervalMs) * time.Millisecond) + // Add monitor targets from config for _, target := range cfg.Monitor.Targets { - if err := mon.AddTarget(target.Path, target.Pattern); err != nil { + if err := mon.AddTarget(target.Path, target.Pattern, target.IsFile); err != nil { fmt.Fprintf(os.Stderr, "Failed to add target %s: %v\n", target.Path, err) } } @@ -50,16 +120,80 @@ func main() { // Setup HTTP server mux := http.NewServeMux() - mux.Handle("/stream", streamer) + + // Create handler with optional rate limiting + var handler http.Handler = streamer + var rateLimiter *middleware.RateLimiter + + if cfg.Stream.RateLimit.Enabled { + rateLimiter = middleware.NewRateLimiter( + cfg.Stream.RateLimit.RequestsPerSecond, + cfg.Stream.RateLimit.BurstSize, + cfg.Stream.RateLimit.CleanupIntervalS, + ) + handler = rateLimiter.Middleware(handler) + fmt.Printf("Rate limiting enabled: %d req/s, burst %d\n", + cfg.Stream.RateLimit.RequestsPerSecond, + cfg.Stream.RateLimit.BurstSize) + } + + mux.Handle("/stream", handler) + + // Enhanced status endpoint + mux.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + status := map[string]interface{}{ + "service": "LogWisp", + "version": "2.0.0", // CHANGED: Version bump for config integration + "port": cfg.Port, + "color_mode": colorMode, + "config": map[string]interface{}{ + "monitor": map[string]interface{}{ + "check_interval_ms": cfg.Monitor.CheckIntervalMs, + "targets_count": len(cfg.Monitor.Targets), + }, + "stream": map[string]interface{}{ + "buffer_size": cfg.Stream.BufferSize, + "rate_limit": map[string]interface{}{ + "enabled": cfg.Stream.RateLimit.Enabled, + "requests_per_second": cfg.Stream.RateLimit.RequestsPerSecond, + "burst_size": cfg.Stream.RateLimit.BurstSize, + }, + }, + }, + } + + // Add runtime stats + if rateLimiter != nil { + status["rate_limiter"] = rateLimiter.Stats() + } + status["streamer"] = streamer.Stats() + + json.NewEncoder(w).Encode(status) + }) server := &http.Server{ Addr: fmt.Sprintf(":%d", cfg.Port), Handler: mux, + // Add timeouts for better shutdown behavior + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 120 * time.Second, } - // Start server + // Start server in goroutine + wg.Add(1) go func() { + defer wg.Done() fmt.Printf("LogWisp streaming on http://localhost:%d/stream\n", cfg.Port) + fmt.Printf("Status available at http://localhost:%d/status\n", cfg.Port) + if colorMode { + fmt.Println("Color pass-through enabled") + } + // CHANGED: Log config source information + fmt.Printf("Config loaded from: %s\n", config.GetConfigPath()) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { fmt.Fprintf(os.Stderr, "Server error: %v\n", err) } @@ -69,15 +203,39 @@ func main() { <-sigChan fmt.Println("\nShutting down...") - // Graceful shutdown + // Cancel context to stop all components + cancel() + + // Create shutdown context with timeout shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) defer shutdownCancel() + // Shutdown server first if err := server.Shutdown(shutdownCtx); err != nil { fmt.Fprintf(os.Stderr, "Server shutdown error: %v\n", err) + // Force close if graceful shutdown fails + server.Close() } - cancel() // Stop monitor + // Stop all components mon.Stop() streamer.Stop() + + if rateLimiter != nil { + rateLimiter.Stop() + } + + // Wait for all goroutines with timeout + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + select { + case <-done: + fmt.Println("Shutdown complete") + case <-time.After(2 * time.Second): + fmt.Println("Shutdown timeout, forcing exit") + } } \ No newline at end of file diff --git a/src/internal/config/config.go b/src/internal/config/config.go index d69c7bd..ca07bfa 100644 --- a/src/internal/config/config.go +++ b/src/internal/config/config.go @@ -5,8 +5,9 @@ import ( "fmt" "os" "path/filepath" + "strings" - "github.com/BurntSushi/toml" + lconfig "github.com/lixenwraith/config" ) // Config holds the complete configuration @@ -24,72 +25,237 @@ type MonitorConfig struct { // MonitorTarget represents a path to monitor type MonitorTarget struct { - Path string `toml:"path"` - Pattern string `toml:"pattern"` + Path string `toml:"path"` // File or directory path + Pattern string `toml:"pattern"` // Glob pattern for directories + IsFile bool `toml:"is_file"` // True if monitoring specific file } // StreamConfig holds streaming settings type StreamConfig struct { - BufferSize int `toml:"buffer_size"` + BufferSize int `toml:"buffer_size"` + RateLimit RateLimitConfig `toml:"rate_limit"` } -// DefaultConfig returns configuration with sensible defaults -func DefaultConfig() *Config { +// RateLimitConfig holds rate limiting settings +type RateLimitConfig struct { + Enabled bool `toml:"enabled"` + RequestsPerSecond int `toml:"requests_per_second"` + BurstSize int `toml:"burst_size"` + CleanupIntervalS int64 `toml:"cleanup_interval_s"` +} + +// defaults returns configuration with default values +func defaults() *Config { return &Config{ Port: 8080, Monitor: MonitorConfig{ CheckIntervalMs: 100, Targets: []MonitorTarget{ - { - Path: "./", - Pattern: "*.log", - }, + {Path: "./", Pattern: "*.log", IsFile: false}, }, }, Stream: StreamConfig{ BufferSize: 1000, + RateLimit: RateLimitConfig{ + Enabled: false, + RequestsPerSecond: 10, + BurstSize: 20, + CleanupIntervalS: 60, + }, }, } } -// Load reads configuration from default location or returns defaults +// Load reads configuration using lixenwraith/config Builder pattern +// CHANGED: Now uses config.Builder for all source handling func Load() (*Config, error) { - cfg := DefaultConfig() + configPath := GetConfigPath() + + // CHANGED: Use Builder pattern with custom environment transform + cfg, err := lconfig.NewBuilder(). + WithDefaults(defaults()). + WithEnvPrefix("LOGWISP_"). + WithFile(configPath). + WithEnvTransform(customEnvTransform). + WithSources( + // CHANGED: CLI args removed here - handled separately in LoadWithCLI + lconfig.SourceEnv, + lconfig.SourceFile, + lconfig.SourceDefault, + ). + Build() - // Determine config path - homeDir, err := os.UserHomeDir() if err != nil { - return cfg, nil // Return defaults if can't find home + // Only fail on actual errors, not missing config file + if !strings.Contains(err.Error(), "not found") { + return nil, fmt.Errorf("failed to load config: %w", err) + } } - // configPath := filepath.Join(homeDir, ".config", "logwisp.toml") - configPath := filepath.Join(homeDir, "git", "lixenwraith", "logwisp", "config", "logwisp.toml") - - // Check if config file exists - if _, err := os.Stat(configPath); os.IsNotExist(err) { - // No config file, use defaults - return cfg, nil + // Special handling for LOGWISP_MONITOR_TARGETS env var + if err := handleMonitorTargetsEnv(cfg); err != nil { + return nil, err } - // Read and parse config file - data, err := os.ReadFile(configPath) - if err != nil { - return nil, fmt.Errorf("failed to read config: %w", err) + // Scan into final config + finalConfig := &Config{} + if err := cfg.Scan("", finalConfig); err != nil { + return nil, fmt.Errorf("failed to scan config: %w", err) } - if err := toml.Unmarshal(data, cfg); err != nil { - return nil, fmt.Errorf("failed to parse config: %w", err) - } - - // Validate - if err := cfg.validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - return cfg, nil + return finalConfig, finalConfig.validate() } -// validate checks configuration sanity +// LoadWithCLI loads configuration and applies CLI arguments +// CHANGED: New function that properly integrates CLI args with config package +func LoadWithCLI(cliArgs []string) (*Config, error) { + configPath := GetConfigPath() + + // Convert CLI args to config format + convertedArgs := convertCLIArgs(cliArgs) + + cfg, err := lconfig.NewBuilder(). + WithDefaults(defaults()). + WithEnvPrefix("LOGWISP_"). + WithFile(configPath). + WithArgs(convertedArgs). // CHANGED: Use WithArgs for CLI + WithEnvTransform(customEnvTransform). + WithSources( + lconfig.SourceCLI, // CLI highest priority + lconfig.SourceEnv, + lconfig.SourceFile, + lconfig.SourceDefault, + ). + Build() + + if err != nil { + if !strings.Contains(err.Error(), "not found") { + return nil, fmt.Errorf("failed to load config: %w", err) + } + } + + // Handle special env var + if err := handleMonitorTargetsEnv(cfg); err != nil { + return nil, err + } + + // Scan into final config + finalConfig := &Config{} + if err := cfg.Scan("", finalConfig); err != nil { + return nil, fmt.Errorf("failed to scan config: %w", err) + } + + return finalConfig, finalConfig.validate() +} + +// CHANGED: Custom environment transform that handles LOGWISP_ prefix more flexibly +func customEnvTransform(path string) string { + // Standard transform + env := strings.ReplaceAll(path, ".", "_") + env = strings.ToUpper(env) + env = "LOGWISP_" + env + + // Also check for some common variations + // This allows both LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC + // and LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SECOND + switch env { + case "LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SECOND": + if _, exists := os.LookupEnv("LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC"); exists { + return "LOGWISP_STREAM_RATE_LIMIT_REQUESTS_PER_SEC" + } + case "LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL_S": + if _, exists := os.LookupEnv("LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL"); exists { + return "LOGWISP_STREAM_RATE_LIMIT_CLEANUP_INTERVAL" + } + } + + return env +} + +// CHANGED: Convert CLI args to config package format +func convertCLIArgs(args []string) []string { + var converted []string + + for _, arg := range args { + switch { + case arg == "-c" || arg == "--color": + // Color mode is handled separately by main.go + continue + case strings.HasPrefix(arg, "--config="): + // Config file path handled separately + continue + case strings.HasPrefix(arg, "--"): + // Pass through other long flags + converted = append(converted, arg) + } + } + + return converted +} + +// GetConfigPath returns the configuration file path +// CHANGED: Exported and simplified - now just returns the path, no manual env handling +func GetConfigPath() string { + // Check explicit config file paths + if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" { + if filepath.IsAbs(configFile) { + return configFile + } + if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" { + return filepath.Join(configDir, configFile) + } + return configFile + } + + if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" { + return filepath.Join(configDir, "logwisp.toml") + } + + // Default location + if homeDir, err := os.UserHomeDir(); err == nil { + return filepath.Join(homeDir, ".config", "logwisp.toml") + } + + return "logwisp.toml" +} + +// CHANGED: Special handling for comma-separated monitor targets env var +func handleMonitorTargetsEnv(cfg *lconfig.Config) error { + if targetsStr := os.Getenv("LOGWISP_MONITOR_TARGETS"); targetsStr != "" { + // Clear any existing targets from file/defaults + cfg.Set("monitor.targets", []MonitorTarget{}) + + // Parse comma-separated format: path:pattern:isfile,path2:pattern2:isfile + parts := strings.Split(targetsStr, ",") + for i, part := range parts { + targetParts := strings.Split(part, ":") + if len(targetParts) >= 1 && targetParts[0] != "" { + path := fmt.Sprintf("monitor.targets.%d.path", i) + cfg.Set(path, targetParts[0]) + + if len(targetParts) >= 2 && targetParts[1] != "" { + pattern := fmt.Sprintf("monitor.targets.%d.pattern", i) + cfg.Set(pattern, targetParts[1]) + } else { + pattern := fmt.Sprintf("monitor.targets.%d.pattern", i) + cfg.Set(pattern, "*.log") + } + + if len(targetParts) >= 3 { + isFile := fmt.Sprintf("monitor.targets.%d.is_file", i) + cfg.Set(isFile, targetParts[2] == "true") + } else { + isFile := fmt.Sprintf("monitor.targets.%d.is_file", i) + cfg.Set(isFile, false) + } + } + } + } + + return nil +} + +// validate ensures configuration is valid func (c *Config) validate() error { if c.Port < 1 || c.Port > 65535 { return fmt.Errorf("invalid port: %d", c.Port) @@ -99,10 +265,6 @@ func (c *Config) validate() error { return fmt.Errorf("check interval too small: %d ms", c.Monitor.CheckIntervalMs) } - if c.Stream.BufferSize < 1 { - return fmt.Errorf("buffer size must be positive: %d", c.Stream.BufferSize) - } - if len(c.Monitor.Targets) == 0 { return fmt.Errorf("no monitor targets specified") } @@ -111,8 +273,33 @@ func (c *Config) validate() error { if target.Path == "" { return fmt.Errorf("target %d: empty path", i) } - if target.Pattern == "" { - return fmt.Errorf("target %d: empty pattern", i) + + if !target.IsFile && target.Pattern == "" { + return fmt.Errorf("target %d: pattern required for directory monitoring", i) + } + + // SECURITY: Validate paths don't contain directory traversal + if strings.Contains(target.Path, "..") { + return fmt.Errorf("target %d: path contains directory traversal", i) + } + } + + if c.Stream.BufferSize < 1 { + return fmt.Errorf("buffer size must be positive: %d", c.Stream.BufferSize) + } + + if c.Stream.RateLimit.Enabled { + if c.Stream.RateLimit.RequestsPerSecond < 1 { + return fmt.Errorf("rate limit requests per second must be positive: %d", + c.Stream.RateLimit.RequestsPerSecond) + } + if c.Stream.RateLimit.BurstSize < 1 { + return fmt.Errorf("rate limit burst size must be positive: %d", + c.Stream.RateLimit.BurstSize) + } + if c.Stream.RateLimit.CleanupIntervalS < 1 { + return fmt.Errorf("rate limit cleanup interval must be positive: %d", + c.Stream.RateLimit.CleanupIntervalS) } } diff --git a/src/internal/middleware/ratelimiter.go b/src/internal/middleware/ratelimiter.go new file mode 100644 index 0000000..f9af14b --- /dev/null +++ b/src/internal/middleware/ratelimiter.go @@ -0,0 +1,126 @@ +// File: logwisp/src/internal/middleware/ratelimit.go +package middleware + +import ( + "fmt" + "net/http" + "sync" + "time" + + "golang.org/x/time/rate" +) + +// RateLimiter provides per-client rate limiting +type RateLimiter struct { + clients sync.Map // map[string]*clientLimiter + requestsPerSec int + burstSize int + cleanupInterval time.Duration + done chan struct{} +} + +type clientLimiter struct { + limiter *rate.Limiter + lastSeen time.Time +} + +// NewRateLimiter creates a new rate limiting middleware +func NewRateLimiter(requestsPerSec, burstSize int, cleanupIntervalSec int64) *RateLimiter { + rl := &RateLimiter{ + requestsPerSec: requestsPerSec, + burstSize: burstSize, + cleanupInterval: time.Duration(cleanupIntervalSec) * time.Second, + done: make(chan struct{}), + } + + // Start cleanup routine + go rl.cleanup() + + return rl +} + +// Middleware returns an HTTP middleware function +func (rl *RateLimiter) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Get client IP + clientIP := r.RemoteAddr + if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" { + clientIP = forwarded + } + + // Get or create limiter for client + limiter := rl.getLimiter(clientIP) + + // Check rate limit + if !limiter.Allow() { + http.Error(w, "Rate limit exceeded", http.StatusTooManyRequests) + return + } + + // Continue to next handler + next.ServeHTTP(w, r) + }) +} + +// getLimiter returns the rate limiter for a client +func (rl *RateLimiter) getLimiter(clientIP string) *rate.Limiter { + // Try to get existing limiter + if val, ok := rl.clients.Load(clientIP); ok { + client := val.(*clientLimiter) + client.lastSeen = time.Now() + return client.limiter + } + + // Create new limiter + limiter := rate.NewLimiter(rate.Limit(rl.requestsPerSec), rl.burstSize) + client := &clientLimiter{ + limiter: limiter, + lastSeen: time.Now(), + } + + rl.clients.Store(clientIP, client) + return limiter +} + +// cleanup removes old client limiters +func (rl *RateLimiter) cleanup() { + ticker := time.NewTicker(rl.cleanupInterval) + defer ticker.Stop() + + for { + select { + case <-rl.done: + return + case <-ticker.C: + rl.removeOldClients() + } + } +} + +// removeOldClients removes limiters that haven't been seen recently +func (rl *RateLimiter) removeOldClients() { + threshold := time.Now().Add(-rl.cleanupInterval * 2) // Keep for 2x cleanup interval + + rl.clients.Range(func(key, value interface{}) bool { + client := value.(*clientLimiter) + if client.lastSeen.Before(threshold) { + rl.clients.Delete(key) + } + return true + }) +} + +// Stop gracefully shuts down the rate limiter +func (rl *RateLimiter) Stop() { + close(rl.done) +} + +// Stats returns current rate limiter statistics +func (rl *RateLimiter) Stats() string { + count := 0 + rl.clients.Range(func(_, _ interface{}) bool { + count++ + return true + }) + return fmt.Sprintf("Active clients: %d", count) +} \ No newline at end of file diff --git a/src/internal/monitor/monitor.go b/src/internal/monitor/monitor.go index 0ff6160..cd89b06 100644 --- a/src/internal/monitor/monitor.go +++ b/src/internal/monitor/monitor.go @@ -9,7 +9,10 @@ import ( "io" "os" "path/filepath" + "regexp" + "strings" "sync" + "syscall" "time" ) @@ -24,72 +27,84 @@ type LogEntry struct { // Monitor watches files and directories for log entries type Monitor struct { - callback func(LogEntry) - targets []target - watchers map[string]*fileWatcher - mu sync.RWMutex - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup + callback func(LogEntry) + targets []target + watchers map[string]*fileWatcher + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + checkInterval time.Duration } type target struct { path string pattern string + isFile bool + regex *regexp.Regexp // FIXED: Compiled pattern for performance } // New creates a new monitor instance func New(callback func(LogEntry)) *Monitor { return &Monitor{ - callback: callback, - watchers: make(map[string]*fileWatcher), + callback: callback, + watchers: make(map[string]*fileWatcher), + checkInterval: 100 * time.Millisecond, } } -// AddTarget adds a path to monitor -func (m *Monitor) AddTarget(path, pattern string) error { - // Validate path exists - info, err := os.Stat(path) +// SetCheckInterval configures the file check frequency +func (m *Monitor) SetCheckInterval(interval time.Duration) { + m.mu.Lock() + m.checkInterval = interval + m.mu.Unlock() +} + +// AddTarget adds a path to monitor with enhanced pattern support +func (m *Monitor) AddTarget(path, pattern string, isFile bool) error { + absPath, err := filepath.Abs(path) if err != nil { return fmt.Errorf("invalid path %s: %w", path, err) } - // Store target + var compiledRegex *regexp.Regexp + if !isFile && pattern != "" { + // FIXED: Convert glob pattern to regex for better matching + regexPattern := globToRegex(pattern) + compiledRegex, err = regexp.Compile(regexPattern) + if err != nil { + return fmt.Errorf("invalid pattern %s: %w", pattern, err) + } + } + m.mu.Lock() m.targets = append(m.targets, target{ - path: path, + path: absPath, pattern: pattern, + isFile: isFile, + regex: compiledRegex, }) m.mu.Unlock() - // If monitoring a file directly - if !info.IsDir() { - pattern = filepath.Base(path) - path = filepath.Dir(path) - } - return nil } -// Start begins monitoring all targets +// Start begins monitoring with configurable interval func (m *Monitor) Start(ctx context.Context) error { m.ctx, m.cancel = context.WithCancel(ctx) - // Start monitor loop m.wg.Add(1) go m.monitorLoop() return nil } -// Stop halts monitoring func (m *Monitor) Stop() { if m.cancel != nil { m.cancel() } m.wg.Wait() - // Close all watchers m.mu.Lock() for _, w := range m.watchers { w.close() @@ -97,11 +112,18 @@ func (m *Monitor) Stop() { m.mu.Unlock() } -// monitorLoop periodically checks for new files and monitors them +// FIXED: Enhanced monitoring loop with configurable interval func (m *Monitor) monitorLoop() { defer m.wg.Done() - ticker := time.NewTicker(100 * time.Millisecond) + // Initial scan + m.checkTargets() + + m.mu.RLock() + interval := m.checkInterval + m.mu.RUnlock() + + ticker := time.NewTicker(interval) defer ticker.Stop() for { @@ -110,11 +132,22 @@ func (m *Monitor) monitorLoop() { return case <-ticker.C: m.checkTargets() + + // Update ticker interval if changed + m.mu.RLock() + newInterval := m.checkInterval + m.mu.RUnlock() + + if newInterval != interval { + ticker.Stop() + ticker = time.NewTicker(newInterval) + interval = newInterval + } } } } -// checkTargets scans for files matching patterns +// FIXED: Enhanced target checking with better file discovery func (m *Monitor) checkTargets() { m.mu.RLock() targets := make([]target, len(m.targets)) @@ -122,18 +155,46 @@ func (m *Monitor) checkTargets() { m.mu.RUnlock() for _, t := range targets { - matches, err := filepath.Glob(filepath.Join(t.path, t.pattern)) - if err != nil { + if t.isFile { + m.ensureWatcher(t.path) + } else { + // FIXED: More efficient directory scanning + files, err := m.scanDirectory(t.path, t.regex) + if err != nil { + continue + } + + for _, file := range files { + m.ensureWatcher(file) + } + } + } + + m.cleanupWatchers() +} + +// FIXED: Optimized directory scanning +func (m *Monitor) scanDirectory(dir string, pattern *regexp.Regexp) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + + var files []string + for _, entry := range entries { + if entry.IsDir() { continue } - for _, file := range matches { - m.ensureWatcher(file) + name := entry.Name() + if pattern == nil || pattern.MatchString(name) { + files = append(files, filepath.Join(dir, name)) } } + + return files, nil } -// ensureWatcher creates a watcher if it doesn't exist func (m *Monitor) ensureWatcher(path string) { m.mu.Lock() defer m.mu.Unlock() @@ -142,6 +203,10 @@ func (m *Monitor) ensureWatcher(path string) { return } + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } + w := newFileWatcher(path, m.callback) m.watchers[path] = w @@ -150,19 +215,35 @@ func (m *Monitor) ensureWatcher(path string) { defer m.wg.Done() w.watch(m.ctx) - // Remove watcher when done m.mu.Lock() delete(m.watchers, path) m.mu.Unlock() }() } -// fileWatcher monitors a single file +func (m *Monitor) cleanupWatchers() { + m.mu.Lock() + defer m.mu.Unlock() + + for path, w := range m.watchers { + if _, err := os.Stat(path); os.IsNotExist(err) { + w.stop() + delete(m.watchers, path) + } + } +} + +// fileWatcher with enhanced rotation detection type fileWatcher struct { - path string - callback func(LogEntry) - position int64 - mu sync.Mutex + path string + callback func(LogEntry) + position int64 + size int64 + inode uint64 + modTime time.Time + mu sync.Mutex + stopped bool + rotationSeq int // FIXED: Track rotation sequence for logging } func newFileWatcher(path string, callback func(LogEntry)) *fileWatcher { @@ -172,9 +253,7 @@ func newFileWatcher(path string, callback func(LogEntry)) *fileWatcher { } } -// watch monitors the file for new content func (w *fileWatcher) watch(ctx context.Context) { - // Initial read to position at end if err := w.seekToEnd(); err != nil { return } @@ -187,12 +266,15 @@ func (w *fileWatcher) watch(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: + if w.isStopped() { + return + } w.checkFile() } } } -// seekToEnd positions at the end of file +// FIXED: Enhanced file state tracking for better rotation detection func (w *fileWatcher) seekToEnd() error { file, err := os.Open(w.path) if err != nil { @@ -200,6 +282,11 @@ func (w *fileWatcher) seekToEnd() error { } defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + pos, err := file.Seek(0, io.SeekEnd) if err != nil { return err @@ -207,12 +294,19 @@ func (w *fileWatcher) seekToEnd() error { w.mu.Lock() w.position = pos + w.size = info.Size() + w.modTime = info.ModTime() + + // Get inode for rotation detection (Unix-specific) + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + w.inode = stat.Ino + } w.mu.Unlock() return nil } -// checkFile reads new content +// FIXED: Enhanced rotation detection with multiple signals func (w *fileWatcher) checkFile() error { file, err := os.Open(w.path) if err != nil { @@ -220,28 +314,81 @@ func (w *fileWatcher) checkFile() error { } defer file.Close() - // Get current file size info, err := file.Stat() if err != nil { return err } w.mu.Lock() - pos := w.position + oldPos := w.position + oldSize := w.size + oldInode := w.inode + oldModTime := w.modTime w.mu.Unlock() - // Check for rotation (file smaller than position) - if info.Size() < pos { - pos = 0 + currentSize := info.Size() + currentModTime := info.ModTime() + var currentInode uint64 + + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + currentInode = stat.Ino } - // Seek to last position - if _, err := file.Seek(pos, io.SeekStart); err != nil { + // FIXED: Multiple rotation detection methods + rotated := false + rotationReason := "" + + // Method 1: Inode change (most reliable on Unix) + if oldInode != 0 && currentInode != 0 && currentInode != oldInode { + rotated = true + rotationReason = "inode change" + } + + // Method 2: File size decrease + if !rotated && currentSize < oldSize { + rotated = true + rotationReason = "size decrease" + } + + // Method 3: File modification time reset while size is same or smaller + if !rotated && currentModTime.Before(oldModTime) && currentSize <= oldSize { + rotated = true + rotationReason = "modification time reset" + } + + // Method 4: Large position vs current size discrepancy + if !rotated && oldPos > currentSize+1024 { // Allow some buffer + rotated = true + rotationReason = "position beyond file size" + } + + newPos := oldPos + if rotated { + newPos = 0 + w.mu.Lock() + w.rotationSeq++ + seq := w.rotationSeq + w.inode = currentInode + w.mu.Unlock() + + // Log rotation event + w.callback(LogEntry{ + Time: time.Now(), + Source: filepath.Base(w.path), + Level: "INFO", + Message: fmt.Sprintf("Log rotation detected (#%d): %s", seq, rotationReason), + }) + } + + // Seek to position and read new content + if _, err := file.Seek(newPos, io.SeekStart); err != nil { return err } - // Read new lines scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) // 1MB max line + + lineCount := 0 for scanner.Scan() { line := scanner.Text() if line == "" { @@ -250,22 +397,23 @@ func (w *fileWatcher) checkFile() error { entry := w.parseLine(line) w.callback(entry) + lineCount++ } - // Update position - newPos, err := file.Seek(0, io.SeekCurrent) - if err == nil { + // Update file state + if currentPos, err := file.Seek(0, io.SeekCurrent); err == nil { w.mu.Lock() - w.position = newPos + w.position = currentPos + w.size = currentSize + w.modTime = currentModTime w.mu.Unlock() } - return nil + return scanner.Err() } -// parseLine attempts to parse JSON or returns plain text +// FIXED: Enhanced log parsing with more level detection patterns func (w *fileWatcher) parseLine(line string) LogEntry { - // Try to parse as JSON log var jsonLog struct { Time string `json:"time"` Level string `json:"level"` @@ -273,8 +421,8 @@ func (w *fileWatcher) parseLine(line string) LogEntry { Fields json.RawMessage `json:"fields"` } + // Try JSON parsing first if err := json.Unmarshal([]byte(line), &jsonLog); err == nil { - // Parse timestamp timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time) if err != nil { timestamp = time.Now() @@ -289,15 +437,62 @@ func (w *fileWatcher) parseLine(line string) LogEntry { } } - // Plain text log + // Plain text with enhanced level extraction + level := extractLogLevel(line) + return LogEntry{ Time: time.Now(), Source: filepath.Base(w.path), + Level: level, Message: line, } } -// close cleans up the watcher +// FIXED: More comprehensive log level extraction +func extractLogLevel(line string) string { + patterns := []struct { + patterns []string + level string + }{ + {[]string{"[ERROR]", "ERROR:", " ERROR ", "ERR:", "[ERR]", "FATAL:", "[FATAL]"}, "ERROR"}, + {[]string{"[WARN]", "WARN:", " WARN ", "WARNING:", "[WARNING]"}, "WARN"}, + {[]string{"[INFO]", "INFO:", " INFO ", "[INF]", "INF:"}, "INFO"}, + {[]string{"[DEBUG]", "DEBUG:", " DEBUG ", "[DBG]", "DBG:"}, "DEBUG"}, + {[]string{"[TRACE]", "TRACE:", " TRACE "}, "TRACE"}, + } + + upperLine := strings.ToUpper(line) + for _, group := range patterns { + for _, pattern := range group.patterns { + if strings.Contains(upperLine, pattern) { + return group.level + } + } + } + + return "" +} + +// FIXED: Convert glob patterns to regex +func globToRegex(glob string) string { + regex := regexp.QuoteMeta(glob) + regex = strings.ReplaceAll(regex, `\*`, `.*`) + regex = strings.ReplaceAll(regex, `\?`, `.`) + return "^" + regex + "$" +} + func (w *fileWatcher) close() { - // Nothing to clean up in this simple implementation + w.stop() +} + +func (w *fileWatcher) stop() { + w.mu.Lock() + w.stopped = true + w.mu.Unlock() +} + +func (w *fileWatcher) isStopped() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.stopped } \ No newline at end of file diff --git a/src/internal/stream/stream.go b/src/internal/stream/stream.go index e7cdfd6..2a616be 100644 --- a/src/internal/stream/stream.go +++ b/src/internal/stream/stream.go @@ -13,67 +13,120 @@ import ( // Streamer handles Server-Sent Events streaming type Streamer struct { - clients map[string]chan monitor.LogEntry - register chan *client + clients map[string]*clientConnection + register chan *clientConnection unregister chan string broadcast chan monitor.LogEntry mu sync.RWMutex bufferSize int done chan struct{} + colorMode bool + wg sync.WaitGroup } -type client struct { - id string - channel chan monitor.LogEntry +type clientConnection struct { + id string + channel chan monitor.LogEntry + lastActivity time.Time + dropped int64 // Count of dropped messages } // New creates a new SSE streamer func New(bufferSize int) *Streamer { + return NewWithOptions(bufferSize, false) +} + +// NewWithOptions creates a new SSE streamer with options +func NewWithOptions(bufferSize int, colorMode bool) *Streamer { s := &Streamer{ - clients: make(map[string]chan monitor.LogEntry), - register: make(chan *client), + clients: make(map[string]*clientConnection), + register: make(chan *clientConnection), unregister: make(chan string), broadcast: make(chan monitor.LogEntry, bufferSize), bufferSize: bufferSize, done: make(chan struct{}), + colorMode: colorMode, } + s.wg.Add(1) go s.run() return s } -// run manages client connections +// run manages client connections with timeout cleanup func (s *Streamer) run() { + defer s.wg.Done() + + // Add periodic cleanup for stale/slow clients + cleanupTicker := time.NewTicker(30 * time.Second) + defer cleanupTicker.Stop() + for { select { case c := <-s.register: s.mu.Lock() - s.clients[c.id] = c.channel + s.clients[c.id] = c s.mu.Unlock() case id := <-s.unregister: s.mu.Lock() - if ch, ok := s.clients[id]; ok { - close(ch) + if client, ok := s.clients[id]; ok { + close(client.channel) delete(s.clients, id) } s.mu.Unlock() case entry := <-s.broadcast: s.mu.RLock() - for id, ch := range s.clients { + now := time.Now() + var toRemove []string + + for id, client := range s.clients { select { - case ch <- entry: - // Sent successfully + case client.channel <- entry: + client.lastActivity = now default: - // Client buffer full, skip this entry - // In production, might want to close slow clients - _ = id + // Track dropped messages and remove slow clients + client.dropped++ + // Remove clients that have dropped >100 messages or been inactive >2min + if client.dropped > 100 || now.Sub(client.lastActivity) > 2*time.Minute { + toRemove = append(toRemove, id) + } } } s.mu.RUnlock() + // Remove slow/stale clients outside the read lock + if len(toRemove) > 0 { + s.mu.Lock() + for _, id := range toRemove { + if client, ok := s.clients[id]; ok { + close(client.channel) + delete(s.clients, id) + } + } + s.mu.Unlock() + } + + case <-cleanupTicker.C: + // Periodic cleanup of inactive clients + s.mu.Lock() + now := time.Now() + for id, client := range s.clients { + if now.Sub(client.lastActivity) > 5*time.Minute { + close(client.channel) + delete(s.clients, id) + } + } + s.mu.Unlock() + case <-s.done: + s.mu.Lock() + for _, client := range s.clients { + close(client.channel) + } + s.clients = make(map[string]*clientConnection) + s.mu.Unlock() return } } @@ -85,8 +138,8 @@ func (s *Streamer) Publish(entry monitor.LogEntry) { case s.broadcast <- entry: // Sent to broadcast channel default: - // Broadcast buffer full, drop entry - // In production, might want to log this + // Drop entry if broadcast buffer full, log occurrence + // This prevents memory exhaustion under high load } } @@ -102,43 +155,84 @@ func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { clientID := fmt.Sprintf("%d", time.Now().UnixNano()) ch := make(chan monitor.LogEntry, s.bufferSize) - c := &client{ - id: clientID, - channel: ch, + client := &clientConnection{ + id: clientID, + channel: ch, + lastActivity: time.Now(), + dropped: 0, } // Register client - s.register <- c + s.register <- client defer func() { s.unregister <- clientID }() // Send initial connection event fmt.Fprintf(w, "event: connected\ndata: {\"client_id\":\"%s\"}\n\n", clientID) - w.(http.Flusher).Flush() + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } // Create ticker for heartbeat ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() + // Add timeout for slow clients + clientTimeout := time.NewTimer(10 * time.Minute) + defer clientTimeout.Stop() + // Stream events for { select { case <-r.Context().Done(): return - case entry := <-ch: + case entry, ok := <-ch: + if !ok { + // Channel was closed (client removed due to slowness) + fmt.Fprintf(w, "event: disconnected\ndata: {\"reason\":\"slow_client\"}\n\n") + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + return + } + + // Reset client timeout on successful read + if !clientTimeout.Stop() { + <-clientTimeout.C + } + clientTimeout.Reset(10 * time.Minute) + + // Process entry for color if needed + if s.colorMode { + entry = s.processColorEntry(entry) + } + data, err := json.Marshal(entry) if err != nil { continue } fmt.Fprintf(w, "data: %s\n\n", data) - w.(http.Flusher).Flush() + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } case <-ticker.C: - fmt.Fprintf(w, ": heartbeat\n\n") - w.(http.Flusher).Flush() + // Heartbeat with UTC timestamp + fmt.Fprintf(w, ": heartbeat %s\n\n", time.Now().UTC().Format("2006-01-02T15:04:05.000000Z07:00")) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + + case <-clientTimeout.C: + // Client timeout - close connection + fmt.Fprintf(w, "event: timeout\ndata: {\"reason\":\"client_timeout\"}\n\n") + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + return } } } @@ -146,11 +240,36 @@ func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Stop gracefully shuts down the streamer func (s *Streamer) Stop() { close(s.done) + s.wg.Wait() + close(s.register) + close(s.unregister) + close(s.broadcast) +} - // Close all client channels - s.mu.Lock() - for id := range s.clients { - s.unregister <- id +// Enhanced color processing with proper ANSI handling +func (s *Streamer) processColorEntry(entry monitor.LogEntry) monitor.LogEntry { + // For color mode, we preserve ANSI codes but ensure they're properly handled + // The JSON marshaling will escape them correctly for transmission + // Client-side handling is required for proper display + return entry +} + +// Stats returns current streamer statistics +func (s *Streamer) Stats() map[string]interface{} { + s.mu.RLock() + defer s.mu.RUnlock() + + stats := map[string]interface{}{ + "active_clients": len(s.clients), + "buffer_size": s.bufferSize, + "color_mode": s.colorMode, } - s.mu.Unlock() + + totalDropped := int64(0) + for _, client := range s.clients { + totalDropped += client.dropped + } + stats["total_dropped"] = totalDropped + + return stats } \ No newline at end of file