v0.1.6 changed target check interval per stream, version info added, makefile added

2025-07-07 18:20:46 -04:00
parent 069818bf3d
commit 80180f74a0
14 changed files with 272 additions and 66 deletions
--- a/31
+++ b/31
@ -0,0 +1,31 @@
+# FILE: logwisp/Makefile
+BINARY_NAME := logwisp
+VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo "dev")
+GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+BUILD_TIME := $(shell date -u '+%Y-%m-%d_%H:%M:%S')
+
+LDFLAGS := -ldflags "-X 'logwisp/src/internal/version.Version=$(VERSION)' \
+                     -X 'logwisp/src/internal/version.GitCommit=$(GIT_COMMIT)' \
+                     -X 'logwisp/src/internal/version.BuildTime=$(BUILD_TIME)'"
+
+.PHONY: build
+build:
+	go build $(LDFLAGS) -o $(BINARY_NAME) ./src/cmd/logwisp
+
+.PHONY: install
+install: build
+	install -m 755 $(BINARY_NAME) /usr/local/bin/
+
+.PHONY: clean
+clean:
+	rm -f $(BINARY_NAME)
+
+.PHONY: test
+test:
+	go test -v ./...
+
+.PHONY: release
+release:
+	@if [ -z "$(TAG)" ]; then echo "TAG is required: make release TAG=v1.0.0"; exit 1; fi
+	git tag -a $(TAG) -m "Release $(TAG)"
+	git push origin $(TAG)
--- a/README.md
+++ b/README.md
@ -10,21 +10,23 @@ A high-performance log streaming service with multi-stream architecture, support

 - **Multi-Stream Architecture**: Run multiple independent log streams, each with its own configuration
 - **Dual Protocol Support**: TCP (raw streaming) and HTTP/SSE (browser-friendly)
- **Real-time Monitoring**: Instant updates with configurable check intervals
+- **Real-time Monitoring**: Instant updates with per-stream configurable check intervals
 - **File Rotation Detection**: Automatic detection and handling of log rotation
 - **Path-based Routing**: Optional HTTP router for consolidated access
- **Per-Stream Configuration**: Independent settings for each log stream
+- **Per-Stream Configuration**: Independent settings including check intervals for each log stream
 - **Connection Statistics**: Real-time monitoring of active connections
 - **Flexible Targets**: Monitor individual files or entire directories
- **Zero Dependencies**: Only gnet and fasthttp beyond stdlib
+- **Version Management**: Git tag-based versioning with build information
+- **Configurable Heartbeats**: Keep connections alive with customizable formats
+- **Minimal Direct Dependencies**: panjf2000/gnet/v2, valyala/fasthttp, lixenwraith/config, and stdlib

 ## Quick Start

 ```bash
-# Build
-go build -o logwisp ./src/cmd/logwisp
+# Build with version information
+make build

-# Run with default configuration
+# Run with default configuration if ~/.config/logwisp.toml doesn't exists
 ./logwisp

 # Run with custom config
@ -32,6 +34,9 @@ go build -o logwisp ./src/cmd/logwisp

 # Run with HTTP router (path-based routing)
 ./logwisp --router
+
+# Show version information
+./logwisp --version
 ```

 ## Architecture
@ -57,15 +62,13 @@ Configuration file location: `~/.config/logwisp.toml`
 ### Basic Multi-Stream Configuration

 ```toml
-# Global defaults
-[monitor]
-check_interval_ms = 100
-
 # Application logs stream
 [[streams]]
 name = "app"

 [streams.monitor]
+# Per-stream check interval in milliseconds
+check_interval_ms = 100
 targets = [
    { path = "/var/log/myapp", pattern = "*.log", is_file = false },
    { path = "/var/log/myapp/app.log", is_file = true }
@ -78,12 +81,21 @@ buffer_size = 2000
 stream_path = "/stream"
 status_path = "/status"

-# System logs stream
+# Heartbeat configuration
+[streams.httpserver.heartbeat]
+enabled = true
+interval_seconds = 30
+format = "comment"  # or "json" for structured events
+include_timestamp = true
+include_stats = false
+
+# System logs stream with slower check interval
 [[streams]]
 name = "system"

 [streams.monitor]
-check_interval_ms = 50  # Override global default
+# Check every 60 seconds for slowly updating logs
+check_interval_ms = 60000
 targets = [
    { path = "/var/log/syslog", is_file = true },
    { path = "/var/log/auth.log", is_file = true }
@ -94,11 +106,12 @@ enabled = true
 port = 9090
 buffer_size = 5000

-[streams.httpserver]
+# TCP heartbeat (always JSON format)
+[streams.tcpserver.heartbeat]
 enabled = true
-port = 8443
-stream_path = "/logs"
-status_path = "/health"
+interval_seconds = 300  # 5 minutes
+include_timestamp = true
+include_stats = true
 ```

 ### Target Configuration
@ -116,6 +129,42 @@ Monitor targets support both files and directories:
 { path = "./logs", pattern = "*.log", is_file = false }
 ```

+### Check Interval Configuration
+
+Each stream can have its own check interval based on log update frequency:
+
+- **High-frequency logs**: 50-100ms (e.g., application debug logs)
+- **Normal logs**: 100-1000ms (e.g., application logs)
+- **Low-frequency logs**: 10000-60000ms (e.g., system logs, archives)
+
+### Heartbeat Configuration
+
+Keep connections alive and detect stale clients with configurable heartbeats:
+
+```toml
+[streams.httpserver.heartbeat]
+enabled = true
+interval_seconds = 30
+format = "comment"        # "comment" for SSE comments, "json" for events
+include_timestamp = true  # Add timestamp to heartbeat
+include_stats = true      # Include connection count and uptime
+```
+
+**Heartbeat Formats**:
+
+Comment format (SSE):
+```
+: heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s
+```
+
+JSON format (SSE):
+```
+event: heartbeat
+data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600}
+```
+
+TCP always uses JSON format with newline delimiter.
+
 ## Usage Modes

 ### 1. Standalone Mode (Default)
@ -183,6 +232,11 @@ eventSource.addEventListener('message', (e) => {
    const logEntry = JSON.parse(e.data);
    console.log(`[${logEntry.time}] ${logEntry.level}: ${logEntry.message}`);
 });
+
+eventSource.addEventListener('heartbeat', (e) => {
+    const heartbeat = JSON.parse(e.data);
+    console.log('Heartbeat:', heartbeat);
+});
 ```

 ## Log Entry Format
@ -219,7 +273,7 @@ All log entries are streamed as JSON:
 ```json
 {
  "service": "LogWisp",
-  "version": "3.0.0",
+  "version": "v1.0.0",
  "server": {
    "type": "http",
    "port": 8080,
@ -274,24 +328,26 @@ When rotation is detected, a special log entry is generated:
 - **Per-client buffers**: Each client has independent buffer space
 - **Configurable sizes**: Adjust buffer sizes based on expected load

-### Heartbeat Messages
+### Per-Stream Check Intervals

-Keep connections alive and detect stale clients:
+Optimize resource usage by configuring check intervals based on log update frequency:

 ```toml
-[streams.httpserver.heartbeat]
-enabled = true
-interval_seconds = 30
-include_timestamp = true
-include_stats = true
-format = "json"  # or "comment" for SSE comments
+# High-frequency application logs
+[streams.monitor]
+check_interval_ms = 50  # Check every 50ms
+
+# Low-frequency system logs
+[streams.monitor]
+check_interval_ms = 60000  # Check every minute
 ```

 ## Performance Tuning

 ### Monitor Settings
 - `check_interval_ms`: Lower values = faster detection, higher CPU usage
- `buffer_size`: Larger buffers handle bursts better but use more memory
+- Configure per-stream based on expected update frequency
+- Use 10000ms+ for archival or slowly updating logs

 ### File Watcher Optimization
 - Use specific file paths when possible (more efficient than directory scanning)
@ -307,7 +363,7 @@ format = "json"  # or "comment" for SSE comments

 ```bash
 # Clone repository
-git clone https://github.com/yourusername/logwisp
+git clone https://github.com/lixenwraith/logwisp
 cd logwisp

 # Install dependencies
@ -316,13 +372,24 @@ go get github.com/panjf2000/gnet/v2
 go get github.com/valyala/fasthttp
 go get github.com/lixenwraith/config

-# Build
-go build -o logwisp ./src/cmd/logwisp
+# Build with version information
+make build

 # Run tests
-go test ./...
+make test
+
+# Create a release
+make release TAG=v1.0.0
 ```

+### Makefile Targets
+
+- `make build` - Build binary with version information
+- `make install` - Install to /usr/local/bin
+- `make clean` - Remove built binary
+- `make test` - Run test suite
+- `make release TAG=vX.Y.Z` - Create and push git tag
+
 ## Deployment

 ### Systemd Service
@ -356,7 +423,7 @@ WantedBy=multi-user.target
 FROM golang:1.24 AS builder
 WORKDIR /app
 COPY . .
-RUN go build -o logwisp ./src/cmd/logwisp
+RUN make build

 FROM debian:bookworm-slim
 RUN useradd -r -s /bin/false logwisp
@ -411,6 +478,7 @@ services:
 2. Verify file paths in configuration
 3. Ensure files match the specified patterns
 4. Check monitor statistics in status endpoint
+5. Verify check_interval_ms is appropriate for log update frequency

 ### High Memory Usage
 1. Reduce buffer sizes in configuration
@ -424,6 +492,12 @@ services:
 3. Monitor client-side errors
 4. Review dropped entry statistics

+### Version Information
+Use `./logwisp --version` to see:
+- Version tag (from git tags)
+- Git commit hash
+- Build timestamp
+
 ## License

 BSD-3-Clause
@ -438,9 +512,13 @@ Contributions are welcome! Please read our contributing guidelines and submit pu
 - [x] File and directory monitoring
 - [x] TCP and HTTP/SSE streaming
 - [x] Path-based HTTP routing
+- [x] Per-stream check intervals
+- [x] Version management
+- [x] Configurable heartbeats
+- [ ] Rate and connection limiting
+- [ ] Log filtering and transformation
+- [ ] Configurable logging support
 - [ ] Authentication (Basic, JWT, mTLS)
 - [ ] TLS/SSL support
- [ ] Rate limiting
 - [ ] Prometheus metrics export
 - [ ] WebSocket support
- [ ] Log filtering and transformation
--- a/config/logwisp.toml
+++ b/config/logwisp.toml
@ -1,15 +1,15 @@
 # LogWisp Multi-Stream Configuration
 # Location: ~/.config/logwisp.toml

-# Global monitor defaults
-[monitor]
-check_interval_ms = 100
-
 # Stream 1: Application logs (public access)
 [[streams]]
 name = "app"

 [streams.monitor]
+# Check interval in milliseconds (per-stream configuration)
+check_interval_ms = 100
+# Array of folders and files to be monitored
+# For file targets, pattern is ignored and can be omitted
 targets = [
    { path = "/var/log/myapp", pattern = "*.log", is_file = false },
    { path = "/var/log/myapp/app.log", pattern = "", is_file = true }
@ -22,17 +22,24 @@ buffer_size = 2000
 stream_path = "/stream"
 status_path = "/status"

+# HTTP SSE Heartbeat Configuration
 [streams.httpserver.heartbeat]
 enabled = true
 interval_seconds = 30
+# Format options: "comment" (SSE comments) or "json" (JSON events)
 format = "comment"
+# Include timestamp in heartbeat
+include_timestamp = true
+# Include server statistics (client count, uptime)
+include_stats = false

 # Stream 2: System logs (authenticated)
 [[streams]]
 name = "system"

 [streams.monitor]
-check_interval_ms = 50  # More frequent checks
+# More frequent checks for critical system logs
+check_interval_ms = 50
 targets = [
    { path = "/var/log", pattern = "syslog*", is_file = false },
    { path = "/var/log/auth.log", pattern = "", is_file = true }
@ -45,6 +52,14 @@ buffer_size = 5000
 stream_path = "/logs"
 status_path = "/health"

+# JSON format heartbeat with full stats
+[streams.httpserver.heartbeat]
+enabled = true
+interval_seconds = 20
+format = "json"
+include_timestamp = true
+include_stats = true
+
 # SSL placeholder
 [streams.httpserver.ssl]
 enabled = true
@ -69,14 +84,20 @@ enabled = true
 port = 9443
 buffer_size = 5000

+# TCP heartbeat (always JSON format)
 [streams.tcpserver.heartbeat]
-enabled = false
+enabled = true
+interval_seconds = 60
+include_timestamp = true
+include_stats = true

-# Stream 3: Debug logs (high-volume, no heartbeat)
+# Stream 3: Debug logs (high-volume, less frequent checks)
 [[streams]]
 name = "debug"

 [streams.monitor]
+# Check every 10 seconds for debug logs
+check_interval_ms = 10000
 targets = [
    { path = "./debug", pattern = "*.debug", is_file = false }
 ]
@ -88,8 +109,9 @@ buffer_size = 10000
 stream_path = "/stream"
 status_path = "/status"

+# Disable heartbeat for high-volume stream
 [streams.httpserver.heartbeat]
-enabled = false  # Disable for high-volume
+enabled = false

 # Rate limiting placeholder
 [streams.httpserver.rate_limit]
@ -98,6 +120,40 @@ requests_per_second = 100.0
 burst_size = 1000
 limit_by = "ip"

+# Stream 4: Slow logs (infrequent updates)
+[[streams]]
+name = "archive"
+
+[streams.monitor]
+# Check once per minute for archival logs
+check_interval_ms = 60000
+targets = [
+    { path = "/var/log/archive", pattern = "*.log.gz", is_file = false }
+]
+
+[streams.tcpserver]
+enabled = true
+port = 9091
+buffer_size = 1000
+
+# Minimal heartbeat for connection keep-alive
+[streams.tcpserver.heartbeat]
+enabled = true
+interval_seconds = 300  # 5 minutes
+include_timestamp = false
+include_stats = false
+
+# Heartbeat Format Examples:
+#
+# Comment format (SSE):
+# : heartbeat 2025-01-07T10:30:00Z clients=5 uptime=3600s
+#
+# JSON format (SSE):
+# event: heartbeat
+# data: {"type":"heartbeat","timestamp":"2025-01-07T10:30:00Z","active_clients":5,"uptime_seconds":3600}
+#
+# TCP always uses JSON format with newline delimiter
+
 # Usage Examples:
 #
 # 1. Standard mode (each stream on its own port):
@ -105,6 +161,7 @@ limit_by = "ip"
 #    - App logs: http://localhost:8080/stream
 #    - System logs: https://localhost:8443/logs (with auth)
 #    - Debug logs: http://localhost:8082/stream
+#    - Archive logs: tcp://localhost:9091
 #
 # 2. Router mode (shared port with path routing):
 #    ./logwisp --router
@ -117,5 +174,8 @@ limit_by = "ip"
 #    ./logwisp --config /etc/logwisp/production.toml
 #
 # 4. Environment variables:
-#    LOGWISP_MONITOR_CHECK_INTERVAL_MS=50
+#    LOGWISP_STREAMS_0_MONITOR_CHECK_INTERVAL_MS=50
 #    LOGWISP_STREAMS_0_HTTPSERVER_PORT=8090
+#
+# 5. Show version:
+#    ./logwisp --version
--- a/src/cmd/logwisp/main.go
+++ b/src/cmd/logwisp/main.go
@ -12,6 +12,7 @@ import (

 	"logwisp/src/internal/config"
 	"logwisp/src/internal/logstream"
+	"logwisp/src/internal/version"
 )

 func main() {
@ -20,9 +21,15 @@ func main() {
 		configFile = flag.String("config", "", "Config file path")
 		useRouter  = flag.Bool("router", false, "Use HTTP router for path-based routing")
 		// routerPort = flag.Int("router-port", 0, "Override router port (default: first HTTP port)")
+		showVersion = flag.Bool("version", false, "Show version information")
 	)
 	flag.Parse()

+	if *showVersion {
+		fmt.Println(version.String())
+		os.Exit(0)
+	}
+
 	if *configFile != "" {
 		os.Setenv("LOGWISP_CONFIG_FILE", *configFile)
 	}
@ -101,6 +108,7 @@ func main() {
 		os.Exit(1)
 	}

+	fmt.Printf("LogWisp %s\n", version.Short())
 	fmt.Printf("\n%d stream(s) running. Press Ctrl+C to stop.\n", successCount)

 	// Start periodic status display
--- a/src/internal/config/config.go
+++ b/src/internal/config/config.go
@ -2,9 +2,6 @@
 package config

 type Config struct {
-	// Global monitor settings
-	Monitor MonitorConfig `toml:"monitor"`
-
 	// Stream configurations
 	Streams []StreamConfig `toml:"streams"`
 }
--- a/src/internal/config/loader.go
+++ b/src/internal/config/loader.go
@ -3,21 +3,20 @@ package config

 import (
 	"fmt"
-	lconfig "github.com/lixenwraith/config"
 	"os"
 	"path/filepath"
 	"strings"
+
+	lconfig "github.com/lixenwraith/config"
 )

 func defaults() *Config {
 	return &Config{
-		Monitor: MonitorConfig{
-			CheckIntervalMs: 100,
-		},
 		Streams: []StreamConfig{
 			{
 				Name: "default",
 				Monitor: &StreamMonitorConfig{
+					CheckIntervalMs: 100,
 					Targets: []MonitorTarget{
 						{Path: "./", Pattern: "*.log", IsFile: false},
 					},
--- a/src/internal/config/stream.go
+++ b/src/internal/config/stream.go
@ -17,7 +17,7 @@ type StreamConfig struct {
 }

 type StreamMonitorConfig struct {
-	CheckIntervalMs *int            `toml:"check_interval_ms"`
+	CheckIntervalMs int             `toml:"check_interval_ms"`
 	Targets         []MonitorTarget `toml:"targets"`
 }

@ -35,8 +35,8 @@ func (s *StreamConfig) GetTargets(defaultTargets []MonitorTarget) []MonitorTarge
 }

 func (s *StreamConfig) GetCheckInterval(defaultInterval int) int {
-	if s.Monitor != nil && s.Monitor.CheckIntervalMs != nil {
-		return *s.Monitor.CheckIntervalMs
+	if s.Monitor != nil && s.Monitor.CheckIntervalMs > 0 {
+		return s.Monitor.CheckIntervalMs
 	}
 	return defaultInterval
 }
--- a/src/internal/config/validation.go
+++ b/src/internal/config/validation.go
@ -7,10 +7,6 @@ import (
 )

 func (c *Config) validate() error {
-	if c.Monitor.CheckIntervalMs < 10 {
-		return fmt.Errorf("check interval too small: %d ms", c.Monitor.CheckIntervalMs)
-	}
-
 	if len(c.Streams) == 0 {
 		return fmt.Errorf("no streams configured")
 	}
@ -29,11 +25,17 @@ func (c *Config) validate() error {
 		}
 		streamNames[stream.Name] = true

-		// Stream must have targets
+		// Stream must have monitor config with targets
 		if stream.Monitor == nil || len(stream.Monitor.Targets) == 0 {
 			return fmt.Errorf("stream '%s': no monitor targets specified", stream.Name)
 		}

+		// Validate check interval
+		if stream.Monitor.CheckIntervalMs < 10 {
+			return fmt.Errorf("stream '%s': check interval too small: %d ms (min: 10ms)",
+				stream.Name, stream.Monitor.CheckIntervalMs)
+		}
+
 		for j, target := range stream.Monitor.Targets {
 			if target.Path == "" {
 				return fmt.Errorf("stream '%s' target %d: empty path", stream.Name, j)
--- a/src/internal/logstream/logstream.go
+++ b/src/internal/logstream/logstream.go
@ -4,9 +4,10 @@ package logstream
 import (
 	"context"
 	"fmt"
-	"logwisp/src/internal/config"
 	"sync"
 	"time"
+
+	"logwisp/src/internal/config"
 )

 func (ls *LogStream) Shutdown() {
--- a/src/internal/logstream/routerserver.go
+++ b/src/internal/logstream/routerserver.go
@ -4,9 +4,11 @@ package logstream
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/valyala/fasthttp"
 	"strings"
 	"sync"
+
+	"github.com/valyala/fasthttp"
+	"logwisp/src/internal/version"
 )

 type routerServer struct {
@ -81,6 +83,7 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {

 	status := map[string]interface{}{
 		"service":       "LogWisp Router",
+		"version":       version.Short(),
 		"port":          rs.port,
 		"streams":       streams,
 		"total_streams": len(streams),
--- a/src/internal/stream/httpstreamer.go
+++ b/src/internal/stream/httpstreamer.go
@ -14,6 +14,7 @@ import (
 	"github.com/valyala/fasthttp"
 	"logwisp/src/internal/config"
 	"logwisp/src/internal/monitor"
+	"logwisp/src/internal/version"
 )

 type HTTPStreamer struct {
@ -286,7 +287,7 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {

 	status := map[string]interface{}{
 		"service": "LogWisp",
-		"version": "3.0.0",
+		"version": version.Short(),
 		"server": map[string]interface{}{
 			"type":           "http",
 			"port":           h.config.Port,
@ -318,17 +319,17 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
 	ctx.SetBody(data)
 }

-// GetActiveConnections returns the current number of active clients
+// Returns the current number of active clients
 func (h *HTTPStreamer) GetActiveConnections() int32 {
 	return h.activeClients.Load()
 }

-// GetStreamPath returns the configured stream endpoint path
+// Returns the configured stream endpoint path
 func (h *HTTPStreamer) GetStreamPath() string {
 	return h.streamPath
 }

-// GetStatusPath returns the configured status endpoint path
+// Returns the configured status endpoint path
 func (h *HTTPStreamer) GetStatusPath() string {
 	return h.statusPath
 }
--- a/src/internal/stream/tcpserver.go
+++ b/src/internal/stream/tcpserver.go
@ -3,8 +3,9 @@ package stream

 import (
 	"fmt"
-	"github.com/panjf2000/gnet/v2"
 	"sync"
+
+	"github.com/panjf2000/gnet/v2"
 )

 type tcpServer struct {
--- a/src/internal/stream/tcpstreamer.go
+++ b/src/internal/stream/tcpstreamer.go
@ -146,6 +146,7 @@ func (t *TCPStreamer) formatHeartbeat() []byte {
 		data["uptime_seconds"] = int(time.Since(t.startTime).Seconds())
 	}

+	// For TCP, always use JSON format
 	jsonData, _ := json.Marshal(data)
 	return append(jsonData, '\n')
 }
--- a/src/internal/version/version.go
+++ b/src/internal/version/version.go
@ -0,0 +1,24 @@
+// FILE: src/internal/version/version.go
+package version
+
+import "fmt"
+
+var (
+	// Version is set at compile time via -ldflags
+	Version   = "dev"
+	GitCommit = "unknown"
+	BuildTime = "unknown"
+)
+
+// returns a formatted version string
+func String() string {
+	if Version == "dev" {
+		return fmt.Sprintf("dev (commit: %s, built: %s)", GitCommit, BuildTime)
+	}
+	return fmt.Sprintf("%s (commit: %s, built: %s)", Version, GitCommit, BuildTime)
+}
+
+// returns just the version tag
+func Short() string {
+	return Version
+}