From 80180f74a0108a728625afdf313c53d161ccc41fd8d12891f79fea114e5a946a Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Mon, 7 Jul 2025 18:20:46 -0400 Subject: [PATCH] v0.1.6 changed target check interval per stream, version info added, makefile added --- Makefile | 31 ++++++ README.md | 146 +++++++++++++++++++------ config/logwisp.toml | 80 ++++++++++++-- src/cmd/logwisp/main.go | 8 ++ src/internal/config/config.go | 3 - src/internal/config/loader.go | 7 +- src/internal/config/stream.go | 6 +- src/internal/config/validation.go | 12 +- src/internal/logstream/logstream.go | 3 +- src/internal/logstream/routerserver.go | 5 +- src/internal/stream/httpstreamer.go | 9 +- src/internal/stream/tcpserver.go | 3 +- src/internal/stream/tcpstreamer.go | 1 + src/internal/version/version.go | 24 ++++ 14 files changed, 272 insertions(+), 66 deletions(-) create mode 100644 Makefile create mode 100644 src/internal/version/version.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..220726e --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +# FILE: logwisp/Makefile +BINARY_NAME := logwisp +VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") +GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown") +BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S') + +LDFLAGS := -ldflags "-X 'logwisp/src/internal/version.Version=$(VERSION)' \ + -X 'logwisp/src/internal/version.GitCommit=$(GIT_COMMIT)' \ + -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'" + +.PHONY: build +build: + go build $(LDFLAGS) -o $(BINARY_NAME) ./src/cmd/logwisp + +.PHONY: install +install: build + install -m 755 $(BINARY_NAME) /usr/local/bin/ + +.PHONY: clean +clean: + rm -f $(BINARY_NAME) + +.PHONY: test +test: + go test -v ./... + +.PHONY: release +release: + @if [ -z "$(TAG)" ]; then echo "TAG is required: make release TAG=v1.0.0"; exit 1; fi + git tag -a $(TAG) -m "Release $(TAG)" + git push origin $(TAG) diff --git a/README.md b/README.md index 2a47d69..98ec688 100644 --- a/README.md +++ b/README.md @@ -10,21 +10,23 @@ A high-performance log streaming service with multi-stream architecture, support - **Multi-Stream Architecture**: Run multiple independent log streams, each with its own configuration - **Dual Protocol Support**: TCP (raw streaming) and HTTP/SSE (browser-friendly) -- **Real-time Monitoring**: Instant updates with configurable check intervals +- **Real-time Monitoring**: Instant updates with per-stream configurable check intervals - **File Rotation Detection**: Automatic detection and handling of log rotation - **Path-based Routing**: Optional HTTP router for consolidated access -- **Per-Stream Configuration**: Independent settings for each log stream +- **Per-Stream Configuration**: Independent settings including check intervals for each log stream - **Connection Statistics**: Real-time monitoring of active connections - **Flexible Targets**: Monitor individual files or entire directories -- **Zero Dependencies**: Only gnet and fasthttp beyond stdlib +- **Version Management**: Git tag-based versioning with build information +- **Configurable Heartbeats**: Keep connections alive with customizable formats +- **Minimal Direct Dependencies**: panjf2000/gnet/v2, valyala/fasthttp, lixenwraith/config, and stdlib ## Quick Start ```bash -# Build -go build -o logwisp ./src/cmd/logwisp +# Build with version information +make build -# Run with default configuration +# Run with default configuration if ~/.config/logwisp.toml doesn't exists ./logwisp # Run with custom config @@ -32,6 +34,9 @@ go build -o logwisp ./src/cmd/logwisp # Run with HTTP router (path-based routing) ./logwisp --router + +# Show version information +./logwisp --version ``` ## Architecture @@ -57,15 +62,13 @@ Configuration file location: `~/.config/logwisp.toml` ### Basic Multi-Stream Configuration ```toml -# Global defaults -[monitor] -check_interval_ms = 100 - # Application logs stream [[streams]] name = "app" [streams.monitor] +# Per-stream check interval in milliseconds +check_interval_ms = 100 targets = [ { path = "/var/log/myapp", pattern = "*.log", is_file = false }, { path = "/var/log/myapp/app.log", is_file = true } @@ -78,12 +81,21 @@ buffer_size = 2000 stream_path = "/stream" status_path = "/status" -# System logs stream +# Heartbeat configuration +[streams.httpserver.heartbeat] +enabled = true +interval_seconds = 30 +format = "comment" # or "json" for structured events +include_timestamp = true +include_stats = false + +# System logs stream with slower check interval [[streams]] name = "system" [streams.monitor] -check_interval_ms = 50 # Override global default +# Check every 60 seconds for slowly updating logs +check_interval_ms = 60000 targets = [ { path = "/var/log/syslog", is_file = true }, { path = "/var/log/auth.log", is_file = true } @@ -94,11 +106,12 @@ enabled = true port = 9090 buffer_size = 5000 -[streams.httpserver] +# TCP heartbeat (always JSON format) +[streams.tcpserver.heartbeat] enabled = true -port = 8443 -stream_path = "/logs" -status_path = "/health" +interval_seconds = 300 # 5 minutes +include_timestamp = true +include_stats = true ``` ### Target Configuration @@ -116,6 +129,42 @@ Monitor targets support both files and directories: { path = "./logs", pattern = "*.log", is_file = false } ``` +### Check Interval Configuration + +Each stream can have its own check interval based on log update frequency: + +- **High-frequency logs**: 50-100ms (e.g., application debug logs) +- **Normal logs**: 100-1000ms (e.g., application logs) +- **Low-frequency logs**: 10000-60000ms (e.g., system logs, archives) + +### Heartbeat Configuration + +Keep connections alive and detect stale clients with configurable heartbeats: + +```toml +[streams.httpserver.heartbeat] +enabled = true +interval_seconds = 30 +format = "comment" # "comment" for SSE comments, "json" for events +include_timestamp = true # Add timestamp to heartbeat +include_stats = true # Include connection count and uptime +``` + +**Heartbeat Formats**: + +Comment format (SSE): +``` +: heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s +``` + +JSON format (SSE): +``` +event: heartbeat +data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600} +``` + +TCP always uses JSON format with newline delimiter. + ## Usage Modes ### 1. Standalone Mode (Default) @@ -183,6 +232,11 @@ eventSource.addEventListener('message', (e) => { const logEntry = JSON.parse(e.data); console.log(`[${logEntry.time}] ${logEntry.level}: ${logEntry.message}`); }); + +eventSource.addEventListener('heartbeat', (e) => { + const heartbeat = JSON.parse(e.data); + console.log('Heartbeat:', heartbeat); +}); ``` ## Log Entry Format @@ -219,7 +273,7 @@ All log entries are streamed as JSON: ```json { "service": "LogWisp", - "version": "3.0.0", + "version": "v1.0.0", "server": { "type": "http", "port": 8080, @@ -274,24 +328,26 @@ When rotation is detected, a special log entry is generated: - **Per-client buffers**: Each client has independent buffer space - **Configurable sizes**: Adjust buffer sizes based on expected load -### Heartbeat Messages +### Per-Stream Check Intervals -Keep connections alive and detect stale clients: +Optimize resource usage by configuring check intervals based on log update frequency: ```toml -[streams.httpserver.heartbeat] -enabled = true -interval_seconds = 30 -include_timestamp = true -include_stats = true -format = "json" # or "comment" for SSE comments +# High-frequency application logs +[streams.monitor] +check_interval_ms = 50 # Check every 50ms + +# Low-frequency system logs +[streams.monitor] +check_interval_ms = 60000 # Check every minute ``` ## Performance Tuning ### Monitor Settings - `check_interval_ms`: Lower values = faster detection, higher CPU usage -- `buffer_size`: Larger buffers handle bursts better but use more memory +- Configure per-stream based on expected update frequency +- Use 10000ms+ for archival or slowly updating logs ### File Watcher Optimization - Use specific file paths when possible (more efficient than directory scanning) @@ -307,7 +363,7 @@ format = "json" # or "comment" for SSE comments ```bash # Clone repository -git clone https://github.com/yourusername/logwisp +git clone https://github.com/lixenwraith/logwisp cd logwisp # Install dependencies @@ -316,13 +372,24 @@ go get github.com/panjf2000/gnet/v2 go get github.com/valyala/fasthttp go get github.com/lixenwraith/config -# Build -go build -o logwisp ./src/cmd/logwisp +# Build with version information +make build # Run tests -go test ./... +make test + +# Create a release +make release TAG=v1.0.0 ``` +### Makefile Targets + +- `make build` - Build binary with version information +- `make install` - Install to /usr/local/bin +- `make clean` - Remove built binary +- `make test` - Run test suite +- `make release TAG=vX.Y.Z` - Create and push git tag + ## Deployment ### Systemd Service @@ -356,7 +423,7 @@ WantedBy=multi-user.target FROM golang:1.24 AS builder WORKDIR /app COPY . . -RUN go build -o logwisp ./src/cmd/logwisp +RUN make build FROM debian:bookworm-slim RUN useradd -r -s /bin/false logwisp @@ -411,6 +478,7 @@ services: 2. Verify file paths in configuration 3. Ensure files match the specified patterns 4. Check monitor statistics in status endpoint +5. Verify check_interval_ms is appropriate for log update frequency ### High Memory Usage 1. Reduce buffer sizes in configuration @@ -424,6 +492,12 @@ services: 3. Monitor client-side errors 4. Review dropped entry statistics +### Version Information +Use `./logwisp --version` to see: +- Version tag (from git tags) +- Git commit hash +- Build timestamp + ## License BSD-3-Clause @@ -438,9 +512,13 @@ Contributions are welcome! Please read our contributing guidelines and submit pu - [x] File and directory monitoring - [x] TCP and HTTP/SSE streaming - [x] Path-based HTTP routing +- [x] Per-stream check intervals +- [x] Version management +- [x] Configurable heartbeats +- [ ] Rate and connection limiting +- [ ] Log filtering and transformation +- [ ] Configurable logging support - [ ] Authentication (Basic, JWT, mTLS) - [ ] TLS/SSL support -- [ ] Rate limiting - [ ] Prometheus metrics export -- [ ] WebSocket support -- [ ] Log filtering and transformation \ No newline at end of file +- [ ] WebSocket support \ No newline at end of file diff --git a/config/logwisp.toml b/config/logwisp.toml index d8d493f..0e03e54 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -1,15 +1,15 @@ # LogWisp Multi-Stream Configuration # Location: ~/.config/logwisp.toml -# Global monitor defaults -[monitor] -check_interval_ms = 100 - # Stream 1: Application logs (public access) [[streams]] name = "app" [streams.monitor] +# Check interval in milliseconds (per-stream configuration) +check_interval_ms = 100 +# Array of folders and files to be monitored +# For file targets, pattern is ignored and can be omitted targets = [ { path = "/var/log/myapp", pattern = "*.log", is_file = false }, { path = "/var/log/myapp/app.log", pattern = "", is_file = true } @@ -22,17 +22,24 @@ buffer_size = 2000 stream_path = "/stream" status_path = "/status" +# HTTP SSE Heartbeat Configuration [streams.httpserver.heartbeat] enabled = true interval_seconds = 30 +# Format options: "comment" (SSE comments) or "json" (JSON events) format = "comment" +# Include timestamp in heartbeat +include_timestamp = true +# Include server statistics (client count, uptime) +include_stats = false # Stream 2: System logs (authenticated) [[streams]] name = "system" [streams.monitor] -check_interval_ms = 50 # More frequent checks +# More frequent checks for critical system logs +check_interval_ms = 50 targets = [ { path = "/var/log", pattern = "syslog*", is_file = false }, { path = "/var/log/auth.log", pattern = "", is_file = true } @@ -45,6 +52,14 @@ buffer_size = 5000 stream_path = "/logs" status_path = "/health" +# JSON format heartbeat with full stats +[streams.httpserver.heartbeat] +enabled = true +interval_seconds = 20 +format = "json" +include_timestamp = true +include_stats = true + # SSL placeholder [streams.httpserver.ssl] enabled = true @@ -69,14 +84,20 @@ enabled = true port = 9443 buffer_size = 5000 +# TCP heartbeat (always JSON format) [streams.tcpserver.heartbeat] -enabled = false +enabled = true +interval_seconds = 60 +include_timestamp = true +include_stats = true -# Stream 3: Debug logs (high-volume, no heartbeat) +# Stream 3: Debug logs (high-volume, less frequent checks) [[streams]] name = "debug" [streams.monitor] +# Check every 10 seconds for debug logs +check_interval_ms = 10000 targets = [ { path = "./debug", pattern = "*.debug", is_file = false } ] @@ -88,8 +109,9 @@ buffer_size = 10000 stream_path = "/stream" status_path = "/status" +# Disable heartbeat for high-volume stream [streams.httpserver.heartbeat] -enabled = false # Disable for high-volume +enabled = false # Rate limiting placeholder [streams.httpserver.rate_limit] @@ -98,6 +120,40 @@ requests_per_second = 100.0 burst_size = 1000 limit_by = "ip" +# Stream 4: Slow logs (infrequent updates) +[[streams]] +name = "archive" + +[streams.monitor] +# Check once per minute for archival logs +check_interval_ms = 60000 +targets = [ + { path = "/var/log/archive", pattern = "*.log.gz", is_file = false } +] + +[streams.tcpserver] +enabled = true +port = 9091 +buffer_size = 1000 + +# Minimal heartbeat for connection keep-alive +[streams.tcpserver.heartbeat] +enabled = true +interval_seconds = 300 # 5 minutes +include_timestamp = false +include_stats = false + +# Heartbeat Format Examples: +# +# Comment format (SSE): +# : heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s +# +# JSON format (SSE): +# event: heartbeat +# data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600} +# +# TCP always uses JSON format with newline delimiter + # Usage Examples: # # 1. Standard mode (each stream on its own port): @@ -105,6 +161,7 @@ limit_by = "ip" # - App logs: http://localhost:8080/stream # - System logs: https://localhost:8443/logs (with auth) # - Debug logs: http://localhost:8082/stream +# - Archive logs: tcp://localhost:9091 # # 2. Router mode (shared port with path routing): # ./logwisp --router @@ -117,5 +174,8 @@ limit_by = "ip" # ./logwisp --config /etc/logwisp/production.toml # # 4. Environment variables: -# LOGWISP_MONITOR_CHECK_INTERVAL_MS=50 -# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090 \ No newline at end of file +# LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50 +# LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090 +# +# 5. Show version: +# ./logwisp --version diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go index 7460015..0244563 100644 --- a/src/cmd/logwisp/main.go +++ b/src/cmd/logwisp/main.go @@ -12,6 +12,7 @@ import ( "logwisp/src/internal/config" "logwisp/src/internal/logstream" + "logwisp/src/internal/version" ) func main() { @@ -20,9 +21,15 @@ func main() { configFile = flag.String("config", "", "Config file path") useRouter = flag.Bool("router", false, "Use HTTP router for path-based routing") // routerPort = flag.Int("router-port", 0, "Override router port (default: first HTTP port)") + showVersion = flag.Bool("version", false, "Show version information") ) flag.Parse() + if *showVersion { + fmt.Println(version.String()) + os.Exit(0) + } + if *configFile != "" { os.Setenv("LOGWISP_CONFIG_FILE", *configFile) } @@ -101,6 +108,7 @@ func main() { os.Exit(1) } + fmt.Printf("LogWisp %s\n", version.Short()) fmt.Printf("\n%d stream(s) running. Press Ctrl+C to stop.\n", successCount) // Start periodic status display diff --git a/src/internal/config/config.go b/src/internal/config/config.go index 0ab5e19..7b66407 100644 --- a/src/internal/config/config.go +++ b/src/internal/config/config.go @@ -2,9 +2,6 @@ package config type Config struct { - // Global monitor settings - Monitor MonitorConfig `toml:"monitor"` - // Stream configurations Streams []StreamConfig `toml:"streams"` } diff --git a/src/internal/config/loader.go b/src/internal/config/loader.go index fee2248..38ed974 100644 --- a/src/internal/config/loader.go +++ b/src/internal/config/loader.go @@ -3,21 +3,20 @@ package config import ( "fmt" - lconfig "github.com/lixenwraith/config" "os" "path/filepath" "strings" + + lconfig "github.com/lixenwraith/config" ) func defaults() *Config { return &Config{ - Monitor: MonitorConfig{ - CheckIntervalMs: 100, - }, Streams: []StreamConfig{ { Name: "default", Monitor: &StreamMonitorConfig{ + CheckIntervalMs: 100, Targets: []MonitorTarget{ {Path: "./", Pattern: "*.log", IsFile: false}, }, diff --git a/src/internal/config/stream.go b/src/internal/config/stream.go index 2fc026e..9adb311 100644 --- a/src/internal/config/stream.go +++ b/src/internal/config/stream.go @@ -17,7 +17,7 @@ type StreamConfig struct { } type StreamMonitorConfig struct { - CheckIntervalMs *int `toml:"check_interval_ms"` + CheckIntervalMs int `toml:"check_interval_ms"` Targets []MonitorTarget `toml:"targets"` } @@ -35,8 +35,8 @@ func (s *StreamConfig) GetTargets(defaultTargets []MonitorTarget) []MonitorTarge } func (s *StreamConfig) GetCheckInterval(defaultInterval int) int { - if s.Monitor != nil && s.Monitor.CheckIntervalMs != nil { - return *s.Monitor.CheckIntervalMs + if s.Monitor != nil && s.Monitor.CheckIntervalMs > 0 { + return s.Monitor.CheckIntervalMs } return defaultInterval } \ No newline at end of file diff --git a/src/internal/config/validation.go b/src/internal/config/validation.go index 8fe8c26..5c07a4e 100644 --- a/src/internal/config/validation.go +++ b/src/internal/config/validation.go @@ -7,10 +7,6 @@ import ( ) func (c *Config) validate() error { - if c.Monitor.CheckIntervalMs < 10 { - return fmt.Errorf("check interval too small: %d ms", c.Monitor.CheckIntervalMs) - } - if len(c.Streams) == 0 { return fmt.Errorf("no streams configured") } @@ -29,11 +25,17 @@ func (c *Config) validate() error { } streamNames[stream.Name] = true - // Stream must have targets + // Stream must have monitor config with targets if stream.Monitor == nil || len(stream.Monitor.Targets) == 0 { return fmt.Errorf("stream '%s': no monitor targets specified", stream.Name) } + // Validate check interval + if stream.Monitor.CheckIntervalMs < 10 { + return fmt.Errorf("stream '%s': check interval too small: %d ms (min: 10ms)", + stream.Name, stream.Monitor.CheckIntervalMs) + } + for j, target := range stream.Monitor.Targets { if target.Path == "" { return fmt.Errorf("stream '%s' target %d: empty path", stream.Name, j) diff --git a/src/internal/logstream/logstream.go b/src/internal/logstream/logstream.go index 2ce666d..65473af 100644 --- a/src/internal/logstream/logstream.go +++ b/src/internal/logstream/logstream.go @@ -4,9 +4,10 @@ package logstream import ( "context" "fmt" - "logwisp/src/internal/config" "sync" "time" + + "logwisp/src/internal/config" ) func (ls *LogStream) Shutdown() { diff --git a/src/internal/logstream/routerserver.go b/src/internal/logstream/routerserver.go index c8f31c6..f2a4dc8 100644 --- a/src/internal/logstream/routerserver.go +++ b/src/internal/logstream/routerserver.go @@ -4,9 +4,11 @@ package logstream import ( "encoding/json" "fmt" - "github.com/valyala/fasthttp" "strings" "sync" + + "github.com/valyala/fasthttp" + "logwisp/src/internal/version" ) type routerServer struct { @@ -81,6 +83,7 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) { status := map[string]interface{}{ "service": "LogWisp Router", + "version": version.Short(), "port": rs.port, "streams": streams, "total_streams": len(streams), diff --git a/src/internal/stream/httpstreamer.go b/src/internal/stream/httpstreamer.go index 5e15bad..bd7ca91 100644 --- a/src/internal/stream/httpstreamer.go +++ b/src/internal/stream/httpstreamer.go @@ -14,6 +14,7 @@ import ( "github.com/valyala/fasthttp" "logwisp/src/internal/config" "logwisp/src/internal/monitor" + "logwisp/src/internal/version" ) type HTTPStreamer struct { @@ -286,7 +287,7 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) { status := map[string]interface{}{ "service": "LogWisp", - "version": "3.0.0", + "version": version.Short(), "server": map[string]interface{}{ "type": "http", "port": h.config.Port, @@ -318,17 +319,17 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) { ctx.SetBody(data) } -// GetActiveConnections returns the current number of active clients +// Returns the current number of active clients func (h *HTTPStreamer) GetActiveConnections() int32 { return h.activeClients.Load() } -// GetStreamPath returns the configured stream endpoint path +// Returns the configured stream endpoint path func (h *HTTPStreamer) GetStreamPath() string { return h.streamPath } -// GetStatusPath returns the configured status endpoint path +// Returns the configured status endpoint path func (h *HTTPStreamer) GetStatusPath() string { return h.statusPath } \ No newline at end of file diff --git a/src/internal/stream/tcpserver.go b/src/internal/stream/tcpserver.go index aa2b6e9..4e71dca 100644 --- a/src/internal/stream/tcpserver.go +++ b/src/internal/stream/tcpserver.go @@ -3,8 +3,9 @@ package stream import ( "fmt" - "github.com/panjf2000/gnet/v2" "sync" + + "github.com/panjf2000/gnet/v2" ) type tcpServer struct { diff --git a/src/internal/stream/tcpstreamer.go b/src/internal/stream/tcpstreamer.go index 0683f58..1fb0458 100644 --- a/src/internal/stream/tcpstreamer.go +++ b/src/internal/stream/tcpstreamer.go @@ -146,6 +146,7 @@ func (t *TCPStreamer) formatHeartbeat() []byte { data["uptime_seconds"] = int(time.Since(t.startTime).Seconds()) } + // For TCP, always use JSON format jsonData, _ := json.Marshal(data) return append(jsonData, '\n') } \ No newline at end of file diff --git a/src/internal/version/version.go b/src/internal/version/version.go new file mode 100644 index 0000000..4fb6156 --- /dev/null +++ b/src/internal/version/version.go @@ -0,0 +1,24 @@ +// FILE: src/internal/version/version.go +package version + +import "fmt" + +var ( + // Version is set at compile time via -ldflags + Version = "dev" + GitCommit = "unknown" + BuildTime = "unknown" +) + +// returns a formatted version string +func String() string { + if Version == "dev" { + return fmt.Sprintf("dev (commit: %s, built: %s)", GitCommit, BuildTime) + } + return fmt.Sprintf("%s (commit: %s, built: %s)", Version, GitCommit, BuildTime) +} + +// returns just the version tag +func Short() string { + return Version +} \ No newline at end of file