From 294771653c0df4ac77e536b16e66eb7a110cce7487e9786abc131f7cd9b9adba Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Mon, 30 Jun 2025 20:55:31 -0400 Subject: [PATCH] v0.1.0 first commit --- .gitignore | 8 + LICENSE | 28 +++ README.md | 144 +++++++++++++++ config/logwisp.toml | 26 +++ doc/placeholder | 0 go.mod | 5 + go.sum | 2 + src/cmd/logwisp/main.go | 83 +++++++++ src/internal/config/config.go | 120 +++++++++++++ src/internal/monitor/monitor.go | 303 ++++++++++++++++++++++++++++++++ src/internal/stream/stream.go | 156 ++++++++++++++++ 11 files changed, 875 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 config/logwisp.toml create mode 100644 doc/placeholder create mode 100644 go.mod create mode 100644 go.sum create mode 100644 src/cmd/logwisp/main.go create mode 100644 src/internal/config/config.go create mode 100644 src/internal/monitor/monitor.go create mode 100644 src/internal/stream/stream.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0a164b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.idea +data +dev +log +cert +bin +script +build diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c71f04c --- /dev/null +++ b/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2025, Lixen Wraith + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..40378b1 --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +# LogWisp - Simple Log Streaming + +A lightweight log streaming service that monitors files and streams updates via Server-Sent Events (SSE). + +## Philosophy + +LogWisp follows the Unix philosophy: do one thing and do it well. It monitors log files and streams them over HTTP/SSE. That's it. + +## Features + +- Monitors multiple files and directories +- Streams log updates in real-time via SSE +- Supports both plain text and JSON formatted logs +- Automatic file rotation detection +- Simple TOML configuration +- No authentication or complex features - use a reverse proxy if needed + +## Quick Start + +1. Build: +```bash +./build.sh +``` + +2. Run with defaults (monitors current directory): +```bash +./logwisp +``` + +3. View logs: +```bash +curl -N http://localhost:8080/stream +``` + +## Configuration + +LogWisp looks for configuration at `~/.config/logwisp.toml`. If not found, it uses sensible defaults. + +Example configuration: +```toml +port = 8080 + +[monitor] +check_interval_ms = 100 + +[[monitor.targets]] +path = "/var/log" +pattern = "*.log" + +[[monitor.targets]] +path = "/home/user/app/logs" +pattern = "app-*.log" + +[stream] +buffer_size = 1000 +``` + +## API + +- `GET /stream` - Server-Sent Events stream of log entries + +Log entry format: +```json +{ + "time": "2024-01-01T12:00:00Z", + "source": "app.log", + "level": "error", + "message": "Something went wrong", + "fields": {"key": "value"} +} +``` + +## Building from Source + +Requirements: +- Go 1.23 or later + +```bash +go mod download +go build -o logwisp ./src/cmd/logwisp +``` + +## Usage Examples + +### Basic Usage +```bash +# Start LogWisp (monitors current directory by default) +./logwisp + +# In another terminal, view the stream +curl -N http://localhost:8080/stream +``` + +### With Custom Config +```bash +# Create config +cat > ~/.config/logwisp.toml << EOF +port = 9090 + +[[monitor.targets]] +path = "/var/log/nginx" +pattern = "*.log" +EOF + +# Run +./logwisp +``` + +### Production Deployment + +For production use, consider: + +1. Run behind a reverse proxy (nginx, caddy) for SSL/TLS +2. Use systemd or similar for process management +3. Add authentication at the proxy level if needed +4. Set appropriate file permissions on monitored logs + +Example systemd service: +```ini +[Unit] +Description=LogWisp Log Streaming Service +After=network.target + +[Service] +Type=simple +User=logwisp +ExecStart=/usr/local/bin/logwisp +Restart=always + +[Install] +WantedBy=multi-user.target +``` + +## Design Decisions + +- **No built-in authentication**: Use a reverse proxy +- **No TLS**: Use a reverse proxy +- **No complex features**: Follows Unix philosophy +- **File-based configuration**: Simple, no CLI args needed +- **SSE over WebSocket**: Simpler, works everywhere + +## License + +BSD-3-Clause \ No newline at end of file diff --git a/config/logwisp.toml b/config/logwisp.toml new file mode 100644 index 0000000..bace771 --- /dev/null +++ b/config/logwisp.toml @@ -0,0 +1,26 @@ +# Example configuration for LogWisp +# Default directory: ~/.config/logwisp.toml + +# Port to listen on +port = 8080 + +[monitor] +# How often to check for file changes (milliseconds) +check_interval_ms = 100 + +# Paths to monitor +[[monitor.targets]] +path = "./" +pattern = "*.log" + +[[monitor.targets]] +path = "/var/log/" +pattern = "*.log" + +#[[monitor.targets]] +#path = "/home/user/app/logs" +#pattern = "app-*.log" + +[stream] +# Buffer size for each client connection +buffer_size = 1000 diff --git a/doc/placeholder b/doc/placeholder new file mode 100644 index 0000000..473a0f4 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b7b27aa --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module logwisp + +go 1.23.4 + +require github.com/BurntSushi/toml v1.5.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a0e2fd8 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go new file mode 100644 index 0000000..3ca791a --- /dev/null +++ b/src/cmd/logwisp/main.go @@ -0,0 +1,83 @@ +// File: logwisp/src/cmd/logwisp/main.go +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "logwisp/src/internal/config" + "logwisp/src/internal/monitor" + "logwisp/src/internal/stream" +) + +func main() { + // Load configuration + cfg, err := config.Load() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to load config: %v\n", err) + os.Exit(1) + } + + // Create context for graceful shutdown + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Setup signal handling + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + + // Create components + streamer := stream.New(cfg.Stream.BufferSize) + mon := monitor.New(streamer.Publish) + + // Add monitor targets from config + for _, target := range cfg.Monitor.Targets { + if err := mon.AddTarget(target.Path, target.Pattern); err != nil { + fmt.Fprintf(os.Stderr, "Failed to add target %s: %v\n", target.Path, err) + } + } + + // Start monitoring + if err := mon.Start(ctx); err != nil { + fmt.Fprintf(os.Stderr, "Failed to start monitor: %v\n", err) + os.Exit(1) + } + + // Setup HTTP server + mux := http.NewServeMux() + mux.Handle("/stream", streamer) + + server := &http.Server{ + Addr: fmt.Sprintf(":%d", cfg.Port), + Handler: mux, + } + + // Start server + go func() { + fmt.Printf("LogWisp streaming on http://localhost:%d/stream\n", cfg.Port) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + fmt.Fprintf(os.Stderr, "Server error: %v\n", err) + } + }() + + // Wait for shutdown signal + <-sigChan + fmt.Println("\nShutting down...") + + // Graceful shutdown + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "Server shutdown error: %v\n", err) + } + + cancel() // Stop monitor + mon.Stop() + streamer.Stop() +} \ No newline at end of file diff --git a/src/internal/config/config.go b/src/internal/config/config.go new file mode 100644 index 0000000..d69c7bd --- /dev/null +++ b/src/internal/config/config.go @@ -0,0 +1,120 @@ +// File: logwisp/src/internal/config/config.go +package config + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/BurntSushi/toml" +) + +// Config holds the complete configuration +type Config struct { + Port int `toml:"port"` + Monitor MonitorConfig `toml:"monitor"` + Stream StreamConfig `toml:"stream"` +} + +// MonitorConfig holds monitoring settings +type MonitorConfig struct { + CheckIntervalMs int `toml:"check_interval_ms"` + Targets []MonitorTarget `toml:"targets"` +} + +// MonitorTarget represents a path to monitor +type MonitorTarget struct { + Path string `toml:"path"` + Pattern string `toml:"pattern"` +} + +// StreamConfig holds streaming settings +type StreamConfig struct { + BufferSize int `toml:"buffer_size"` +} + +// DefaultConfig returns configuration with sensible defaults +func DefaultConfig() *Config { + return &Config{ + Port: 8080, + Monitor: MonitorConfig{ + CheckIntervalMs: 100, + Targets: []MonitorTarget{ + { + Path: "./", + Pattern: "*.log", + }, + }, + }, + Stream: StreamConfig{ + BufferSize: 1000, + }, + } +} + +// Load reads configuration from default location or returns defaults +func Load() (*Config, error) { + cfg := DefaultConfig() + + // Determine config path + homeDir, err := os.UserHomeDir() + if err != nil { + return cfg, nil // Return defaults if can't find home + } + + // configPath := filepath.Join(homeDir, ".config", "logwisp.toml") + configPath := filepath.Join(homeDir, "git", "lixenwraith", "logwisp", "config", "logwisp.toml") + + // Check if config file exists + if _, err := os.Stat(configPath); os.IsNotExist(err) { + // No config file, use defaults + return cfg, nil + } + + // Read and parse config file + data, err := os.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to read config: %w", err) + } + + if err := toml.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + + // Validate + if err := cfg.validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + + return cfg, nil +} + +// validate checks configuration sanity +func (c *Config) validate() error { + if c.Port < 1 || c.Port > 65535 { + return fmt.Errorf("invalid port: %d", c.Port) + } + + if c.Monitor.CheckIntervalMs < 10 { + return fmt.Errorf("check interval too small: %d ms", c.Monitor.CheckIntervalMs) + } + + if c.Stream.BufferSize < 1 { + return fmt.Errorf("buffer size must be positive: %d", c.Stream.BufferSize) + } + + if len(c.Monitor.Targets) == 0 { + return fmt.Errorf("no monitor targets specified") + } + + for i, target := range c.Monitor.Targets { + if target.Path == "" { + return fmt.Errorf("target %d: empty path", i) + } + if target.Pattern == "" { + return fmt.Errorf("target %d: empty pattern", i) + } + } + + return nil +} \ No newline at end of file diff --git a/src/internal/monitor/monitor.go b/src/internal/monitor/monitor.go new file mode 100644 index 0000000..0ff6160 --- /dev/null +++ b/src/internal/monitor/monitor.go @@ -0,0 +1,303 @@ +// File: logwisp/src/internal/monitor/monitor.go +package monitor + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" +) + +// LogEntry represents a log line to be streamed +type LogEntry struct { + Time time.Time `json:"time"` + Source string `json:"source"` + Level string `json:"level,omitempty"` + Message string `json:"message"` + Fields json.RawMessage `json:"fields,omitempty"` +} + +// Monitor watches files and directories for log entries +type Monitor struct { + callback func(LogEntry) + targets []target + watchers map[string]*fileWatcher + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup +} + +type target struct { + path string + pattern string +} + +// New creates a new monitor instance +func New(callback func(LogEntry)) *Monitor { + return &Monitor{ + callback: callback, + watchers: make(map[string]*fileWatcher), + } +} + +// AddTarget adds a path to monitor +func (m *Monitor) AddTarget(path, pattern string) error { + // Validate path exists + info, err := os.Stat(path) + if err != nil { + return fmt.Errorf("invalid path %s: %w", path, err) + } + + // Store target + m.mu.Lock() + m.targets = append(m.targets, target{ + path: path, + pattern: pattern, + }) + m.mu.Unlock() + + // If monitoring a file directly + if !info.IsDir() { + pattern = filepath.Base(path) + path = filepath.Dir(path) + } + + return nil +} + +// Start begins monitoring all targets +func (m *Monitor) Start(ctx context.Context) error { + m.ctx, m.cancel = context.WithCancel(ctx) + + // Start monitor loop + m.wg.Add(1) + go m.monitorLoop() + + return nil +} + +// Stop halts monitoring +func (m *Monitor) Stop() { + if m.cancel != nil { + m.cancel() + } + m.wg.Wait() + + // Close all watchers + m.mu.Lock() + for _, w := range m.watchers { + w.close() + } + m.mu.Unlock() +} + +// monitorLoop periodically checks for new files and monitors them +func (m *Monitor) monitorLoop() { + defer m.wg.Done() + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.checkTargets() + } + } +} + +// checkTargets scans for files matching patterns +func (m *Monitor) checkTargets() { + m.mu.RLock() + targets := make([]target, len(m.targets)) + copy(targets, m.targets) + m.mu.RUnlock() + + for _, t := range targets { + matches, err := filepath.Glob(filepath.Join(t.path, t.pattern)) + if err != nil { + continue + } + + for _, file := range matches { + m.ensureWatcher(file) + } + } +} + +// ensureWatcher creates a watcher if it doesn't exist +func (m *Monitor) ensureWatcher(path string) { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.watchers[path]; exists { + return + } + + w := newFileWatcher(path, m.callback) + m.watchers[path] = w + + m.wg.Add(1) + go func() { + defer m.wg.Done() + w.watch(m.ctx) + + // Remove watcher when done + m.mu.Lock() + delete(m.watchers, path) + m.mu.Unlock() + }() +} + +// fileWatcher monitors a single file +type fileWatcher struct { + path string + callback func(LogEntry) + position int64 + mu sync.Mutex +} + +func newFileWatcher(path string, callback func(LogEntry)) *fileWatcher { + return &fileWatcher{ + path: path, + callback: callback, + } +} + +// watch monitors the file for new content +func (w *fileWatcher) watch(ctx context.Context) { + // Initial read to position at end + if err := w.seekToEnd(); err != nil { + return + } + + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + w.checkFile() + } + } +} + +// seekToEnd positions at the end of file +func (w *fileWatcher) seekToEnd() error { + file, err := os.Open(w.path) + if err != nil { + return err + } + defer file.Close() + + pos, err := file.Seek(0, io.SeekEnd) + if err != nil { + return err + } + + w.mu.Lock() + w.position = pos + w.mu.Unlock() + + return nil +} + +// checkFile reads new content +func (w *fileWatcher) checkFile() error { + file, err := os.Open(w.path) + if err != nil { + return err + } + defer file.Close() + + // Get current file size + info, err := file.Stat() + if err != nil { + return err + } + + w.mu.Lock() + pos := w.position + w.mu.Unlock() + + // Check for rotation (file smaller than position) + if info.Size() < pos { + pos = 0 + } + + // Seek to last position + if _, err := file.Seek(pos, io.SeekStart); err != nil { + return err + } + + // Read new lines + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + + entry := w.parseLine(line) + w.callback(entry) + } + + // Update position + newPos, err := file.Seek(0, io.SeekCurrent) + if err == nil { + w.mu.Lock() + w.position = newPos + w.mu.Unlock() + } + + return nil +} + +// parseLine attempts to parse JSON or returns plain text +func (w *fileWatcher) parseLine(line string) LogEntry { + // Try to parse as JSON log + var jsonLog struct { + Time string `json:"time"` + Level string `json:"level"` + Message string `json:"msg"` + Fields json.RawMessage `json:"fields"` + } + + if err := json.Unmarshal([]byte(line), &jsonLog); err == nil { + // Parse timestamp + timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time) + if err != nil { + timestamp = time.Now() + } + + return LogEntry{ + Time: timestamp, + Source: filepath.Base(w.path), + Level: jsonLog.Level, + Message: jsonLog.Message, + Fields: jsonLog.Fields, + } + } + + // Plain text log + return LogEntry{ + Time: time.Now(), + Source: filepath.Base(w.path), + Message: line, + } +} + +// close cleans up the watcher +func (w *fileWatcher) close() { + // Nothing to clean up in this simple implementation +} \ No newline at end of file diff --git a/src/internal/stream/stream.go b/src/internal/stream/stream.go new file mode 100644 index 0000000..e7cdfd6 --- /dev/null +++ b/src/internal/stream/stream.go @@ -0,0 +1,156 @@ +// File: logwisp/src/internal/stream/stream.go +package stream + +import ( + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "logwisp/src/internal/monitor" +) + +// Streamer handles Server-Sent Events streaming +type Streamer struct { + clients map[string]chan monitor.LogEntry + register chan *client + unregister chan string + broadcast chan monitor.LogEntry + mu sync.RWMutex + bufferSize int + done chan struct{} +} + +type client struct { + id string + channel chan monitor.LogEntry +} + +// New creates a new SSE streamer +func New(bufferSize int) *Streamer { + s := &Streamer{ + clients: make(map[string]chan monitor.LogEntry), + register: make(chan *client), + unregister: make(chan string), + broadcast: make(chan monitor.LogEntry, bufferSize), + bufferSize: bufferSize, + done: make(chan struct{}), + } + + go s.run() + return s +} + +// run manages client connections +func (s *Streamer) run() { + for { + select { + case c := <-s.register: + s.mu.Lock() + s.clients[c.id] = c.channel + s.mu.Unlock() + + case id := <-s.unregister: + s.mu.Lock() + if ch, ok := s.clients[id]; ok { + close(ch) + delete(s.clients, id) + } + s.mu.Unlock() + + case entry := <-s.broadcast: + s.mu.RLock() + for id, ch := range s.clients { + select { + case ch <- entry: + // Sent successfully + default: + // Client buffer full, skip this entry + // In production, might want to close slow clients + _ = id + } + } + s.mu.RUnlock() + + case <-s.done: + return + } + } +} + +// Publish sends a log entry to all connected clients +func (s *Streamer) Publish(entry monitor.LogEntry) { + select { + case s.broadcast <- entry: + // Sent to broadcast channel + default: + // Broadcast buffer full, drop entry + // In production, might want to log this + } +} + +// ServeHTTP implements http.Handler for SSE +func (s *Streamer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Set SSE headers + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("X-Accel-Buffering", "no") + + // Create client + clientID := fmt.Sprintf("%d", time.Now().UnixNano()) + ch := make(chan monitor.LogEntry, s.bufferSize) + + c := &client{ + id: clientID, + channel: ch, + } + + // Register client + s.register <- c + defer func() { + s.unregister <- clientID + }() + + // Send initial connection event + fmt.Fprintf(w, "event: connected\ndata: {\"client_id\":\"%s\"}\n\n", clientID) + w.(http.Flusher).Flush() + + // Create ticker for heartbeat + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + // Stream events + for { + select { + case <-r.Context().Done(): + return + + case entry := <-ch: + data, err := json.Marshal(entry) + if err != nil { + continue + } + + fmt.Fprintf(w, "data: %s\n\n", data) + w.(http.Flusher).Flush() + + case <-ticker.C: + fmt.Fprintf(w, ": heartbeat\n\n") + w.(http.Flusher).Flush() + } + } +} + +// Stop gracefully shuts down the streamer +func (s *Streamer) Stop() { + close(s.done) + + // Close all client channels + s.mu.Lock() + for id := range s.clients { + s.unregister <- id + } + s.mu.Unlock() +} \ No newline at end of file