v0.1.9 pre-stream log regex filtering added

src/internal/config/stream.go

@@ -1,6 +1,8 @@
 // FILE: src/internal/config/stream.go
 package config
 
+import "logwisp/src/internal/filter"
+
 type StreamConfig struct {
 	// Stream identifier (used in logs and metrics)
 	Name string `toml:"name"`
@@ -8,6 +10,9 @@ type StreamConfig struct {
 	// Monitor configuration for this stream
 	Monitor *StreamMonitorConfig `toml:"monitor"`
 
+	// Filter configuration
+	Filters []filter.Config `toml:"filters"`
+
 	// Server configurations
 	TCPServer  *TCPConfig  `toml:"tcpserver"`
 	HTTPServer *HTTPConfig `toml:"httpserver"`

src/internal/config/validation.go

@@ -3,6 +3,8 @@ package config
 
 import (
 	"fmt"
+	"logwisp/src/internal/filter"
+	"regexp"
 	"strings"
 )
 
@@ -36,6 +38,7 @@ func (c *Config) validate() error {
 				stream.Name, stream.Monitor.CheckIntervalMs)
 		}
 
+		// Validate targets
 		for j, target := range stream.Monitor.Targets {
 			if target.Path == "" {
 				return fmt.Errorf("stream '%s' target %d: empty path", stream.Name, j)
@@ -45,6 +48,13 @@ func (c *Config) validate() error {
 			}
 		}
 
+		// Validate filters
+		for j, filterCfg := range stream.Filters {
+			if err := validateFilter(stream.Name, j, &filterCfg); err != nil {
+				return err
+			}
+		}
+
 		// Validate TCP server
 		if stream.TCPServer != nil && stream.TCPServer.Enabled {
 			if stream.TCPServer.Port < 1 || stream.TCPServer.Port > 65535 {
@@ -222,5 +232,40 @@ func validateRateLimit(serverType, streamName string, rl *RateLimitConfig) error
 			streamName, serverType, rl.ResponseCode)
 	}
 
+	return nil
+}
+
+func validateFilter(streamName string, filterIndex int, cfg *filter.Config) error {
+	// Validate filter type
+	switch cfg.Type {
+	case filter.TypeInclude, filter.TypeExclude, "":
+		// Valid types
+	default:
+		return fmt.Errorf("stream '%s' filter[%d]: invalid type '%s' (must be 'include' or 'exclude')",
+			streamName, filterIndex, cfg.Type)
+	}
+
+	// Validate filter logic
+	switch cfg.Logic {
+	case filter.LogicOr, filter.LogicAnd, "":
+		// Valid logic
+	default:
+		return fmt.Errorf("stream '%s' filter[%d]: invalid logic '%s' (must be 'or' or 'and')",
+			streamName, filterIndex, cfg.Logic)
+	}
+
+	// Empty patterns is valid - passes everything
+	if len(cfg.Patterns) == 0 {
+		return nil
+	}
+
+	// Validate regex patterns
+	for i, pattern := range cfg.Patterns {
+		if _, err := regexp.Compile(pattern); err != nil {
+			return fmt.Errorf("stream '%s' filter[%d] pattern[%d] '%s': invalid regex: %w",
+				streamName, filterIndex, i, pattern, err)
+		}
+	}
+
 	return nil
 }

src/internal/filter/chain.go

@@ -0,0 +1,72 @@
+// FILE: src/internal/filter/chain.go
+package filter
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"logwisp/src/internal/monitor"
+)
+
+// Chain manages multiple filters in sequence
+type Chain struct {
+	filters []*Filter
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	totalPassed    atomic.Uint64
+}
+
+// NewChain creates a new filter chain from configurations
+func NewChain(configs []Config) (*Chain, error) {
+	chain := &Chain{
+		filters: make([]*Filter, 0, len(configs)),
+	}
+
+	for i, cfg := range configs {
+		filter, err := New(cfg)
+		if err != nil {
+			return nil, fmt.Errorf("filter[%d]: %w", i, err)
+		}
+		chain.filters = append(chain.filters, filter)
+	}
+
+	return chain, nil
+}
+
+// Apply runs all filters in sequence
+// Returns true if the entry passes all filters
+func (c *Chain) Apply(entry monitor.LogEntry) bool {
+	c.totalProcessed.Add(1)
+
+	// No filters means pass everything
+	if len(c.filters) == 0 {
+		c.totalPassed.Add(1)
+		return true
+	}
+
+	// All filters must pass
+	for _, filter := range c.filters {
+		if !filter.Apply(entry) {
+			return false
+		}
+	}
+
+	c.totalPassed.Add(1)
+	return true
+}
+
+// GetStats returns chain statistics
+func (c *Chain) GetStats() map[string]interface{} {
+	filterStats := make([]map[string]interface{}, len(c.filters))
+	for i, filter := range c.filters {
+		filterStats[i] = filter.GetStats()
+	}
+
+	return map[string]interface{}{
+		"filter_count":    len(c.filters),
+		"total_processed": c.totalProcessed.Load(),
+		"total_passed":    c.totalPassed.Load(),
+		"filters":         filterStats,
+	}
+}

src/internal/filter/filter.go

@@ -0,0 +1,173 @@
+// FILE: src/internal/filter/filter.go
+package filter
+
+import (
+	"fmt"
+	"regexp"
+	"sync"
+	"sync/atomic"
+
+	"logwisp/src/internal/monitor"
+)
+
+// Type represents the filter type
+type Type string
+
+const (
+	TypeInclude Type = "include" // Whitelist - only matching logs pass
+	TypeExclude Type = "exclude" // Blacklist - matching logs are dropped
+)
+
+// Logic represents how multiple patterns are combined
+type Logic string
+
+const (
+	LogicOr  Logic = "or"  // Match any pattern
+	LogicAnd Logic = "and" // Match all patterns
+)
+
+// Config represents filter configuration
+type Config struct {
+	Type     Type     `toml:"type"`
+	Logic    Logic    `toml:"logic"`
+	Patterns []string `toml:"patterns"`
+}
+
+// Filter applies regex-based filtering to log entries
+type Filter struct {
+	config   Config
+	patterns []*regexp.Regexp
+	mu       sync.RWMutex
+
+	// Statistics
+	totalProcessed atomic.Uint64
+	totalMatched   atomic.Uint64
+	totalDropped   atomic.Uint64
+}
+
+// New creates a new filter from configuration
+func New(cfg Config) (*Filter, error) {
+	// Set defaults
+	if cfg.Type == "" {
+		cfg.Type = TypeInclude
+	}
+	if cfg.Logic == "" {
+		cfg.Logic = LogicOr
+	}
+
+	f := &Filter{
+		config:   cfg,
+		patterns: make([]*regexp.Regexp, 0, len(cfg.Patterns)),
+	}
+
+	// Compile patterns
+	for i, pattern := range cfg.Patterns {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return nil, fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
+		}
+		f.patterns = append(f.patterns, re)
+	}
+
+	return f, nil
+}
+
+// Apply checks if a log entry should be passed through
+func (f *Filter) Apply(entry monitor.LogEntry) bool {
+	f.totalProcessed.Add(1)
+
+	// No patterns means pass everything
+	if len(f.patterns) == 0 {
+		return true
+	}
+
+	// Check against all fields that might contain the log content
+	text := entry.Message
+	if entry.Level != "" {
+		text = entry.Level + " " + text
+	}
+	if entry.Source != "" {
+		text = entry.Source + " " + text
+	}
+
+	matched := f.matches(text)
+	if matched {
+		f.totalMatched.Add(1)
+	}
+
+	// Determine if we should pass or drop
+	shouldPass := false
+	switch f.config.Type {
+	case TypeInclude:
+		shouldPass = matched
+	case TypeExclude:
+		shouldPass = !matched
+	}
+
+	if !shouldPass {
+		f.totalDropped.Add(1)
+	}
+
+	return shouldPass
+}
+
+// matches checks if text matches the patterns according to the logic
+func (f *Filter) matches(text string) bool {
+	switch f.config.Logic {
+	case LogicOr:
+		// Match any pattern
+		for _, re := range f.patterns {
+			if re.MatchString(text) {
+				return true
+			}
+		}
+		return false
+
+	case LogicAnd:
+		// Must match all patterns
+		for _, re := range f.patterns {
+			if !re.MatchString(text) {
+				return false
+			}
+		}
+		return true
+
+	default:
+		// Shouldn't happen after validation
+		return false
+	}
+}
+
+// GetStats returns filter statistics
+func (f *Filter) GetStats() map[string]interface{} {
+	return map[string]interface{}{
+		"type":            f.config.Type,
+		"logic":           f.config.Logic,
+		"pattern_count":   len(f.patterns),
+		"total_processed": f.totalProcessed.Load(),
+		"total_matched":   f.totalMatched.Load(),
+		"total_dropped":   f.totalDropped.Load(),
+	}
+}
+
+// UpdatePatterns allows dynamic pattern updates
+func (f *Filter) UpdatePatterns(patterns []string) error {
+	compiled := make([]*regexp.Regexp, 0, len(patterns))
+
+	// Compile all patterns first
+	for i, pattern := range patterns {
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			return fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
+		}
+		compiled = append(compiled, re)
+	}
+
+	// Update atomically
+	f.mu.Lock()
+	f.patterns = compiled
+	f.config.Patterns = patterns
+	f.mu.Unlock()
+
+	return nil
+}

src/internal/logstream/httprouter.go

@@ -135,11 +135,11 @@ func (r *HTTPRouter) Shutdown() {
 	fmt.Println("[ROUTER] Router shutdown complete")
 }
 
-func (r *HTTPRouter) GetStats() map[string]interface{} {
+func (r *HTTPRouter) GetStats() map[string]any {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 
-	serverStats := make(map[int]interface{})
+	serverStats := make(map[int]any)
 	totalRoutes := 0
 
 	for port, rs := range r.servers {
@@ -151,14 +151,14 @@ func (r *HTTPRouter) GetStats() map[string]interface{} {
 		}
 		rs.routeMu.RUnlock()
 
-		serverStats[port] = map[string]interface{}{
+		serverStats[port] = map[string]any{
 			"routes":   routes,
 			"requests": rs.requests.Load(),
 			"uptime":   int(time.Since(rs.startTime).Seconds()),
 		}
 	}
 
-	return map[string]interface{}{
+	return map[string]any{
 		"uptime_seconds":  int(time.Since(r.startTime).Seconds()),
 		"total_requests":  r.totalRequests.Load(),
 		"routed_requests": r.routedRequests.Load(),

src/internal/logstream/logstream.go

@@ -40,22 +40,26 @@ func (ls *LogStream) Shutdown() {
 	ls.Monitor.Stop()
 }
 
-func (ls *LogStream) GetStats() map[string]interface{} {
+func (ls *LogStream) GetStats() map[string]any {
 	monStats := ls.Monitor.GetStats()
 
-	stats := map[string]interface{}{
+	stats := map[string]any{
 		"name":           ls.Name,
 		"uptime_seconds": int(time.Since(ls.Stats.StartTime).Seconds()),
 		"monitor":        monStats,
 	}
 
+	if ls.FilterChain != nil {
+		stats["filters"] = ls.FilterChain.GetStats()
+	}
+
 	if ls.TCPServer != nil {
 		currentConnections := ls.TCPServer.GetActiveConnections()
 
 		stats["tcp"] = map[string]interface{}{
 			"enabled":     true,
 			"port":        ls.Config.TCPServer.Port,
-			"connections": currentConnections, // Use current value
+			"connections": currentConnections,
 		}
 	}
 
@@ -65,7 +69,7 @@ func (ls *LogStream) GetStats() map[string]interface{} {
 		stats["http"] = map[string]interface{}{
 			"enabled":     true,
 			"port":        ls.Config.HTTPServer.Port,
-			"connections": currentConnections, // Use current value
+			"connections": currentConnections,
 			"stream_path": ls.Config.HTTPServer.StreamPath,
 			"status_path": ls.Config.HTTPServer.StatusPath,
 		}

src/internal/logstream/routerserver.go

@@ -101,12 +101,12 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
 	ctx.SetContentType("application/json")
 
 	rs.routeMu.RLock()
-	streams := make(map[string]interface{})
+	streams := make(map[string]any)
 	for prefix, stream := range rs.routes {
 		streamStats := stream.GetStats()
 
 		// Add routing information
-		streamStats["routing"] = map[string]interface{}{
+		streamStats["routing"] = map[string]any{
 			"path_prefix": prefix,
 			"endpoints": map[string]string{
 				"stream": prefix + stream.Config.HTTPServer.StreamPath,
@@ -121,7 +121,7 @@ func (rs *routerServer) handleGlobalStatus(ctx *fasthttp.RequestCtx) {
 	// Get router stats
 	routerStats := rs.router.GetStats()
 
-	status := map[string]interface{}{
+	status := map[string]any{
 		"service":       "LogWisp Router",
 		"version":       version.String(),
 		"port":          rs.port,
@@ -155,7 +155,7 @@ func (rs *routerServer) handleNotFound(ctx *fasthttp.RequestCtx) {
 	}
 	rs.routeMu.RUnlock()
 
-	response := map[string]interface{}{
+	response := map[string]any{
 		"error":            "Not Found",
 		"requested_path":   string(ctx.Path()),
 		"available_routes": availableRoutes,

src/internal/logstream/service.go

@@ -4,6 +4,7 @@ package logstream
 import (
 	"context"
 	"fmt"
+	"logwisp/src/internal/filter"
 	"sync"
 	"time"
 
@@ -21,12 +22,13 @@ type Service struct {
 }
 
 type LogStream struct {
-	Name       string
-	Config     config.StreamConfig
-	Monitor    monitor.Monitor
-	TCPServer  *stream.TCPStreamer
-	HTTPServer *stream.HTTPStreamer
-	Stats      *StreamStats
+	Name        string
+	Config      config.StreamConfig
+	Monitor     monitor.Monitor
+	FilterChain *filter.Chain
+	TCPServer   *stream.TCPStreamer
+	HTTPServer  *stream.HTTPStreamer
+	Stats       *StreamStats
 
 	ctx    context.Context
 	cancel context.CancelFunc
@@ -39,6 +41,7 @@ type StreamStats struct {
 	HTTPConnections    int32
 	TotalBytesServed   uint64
 	TotalEntriesServed uint64
+	FilterStats        map[string]any
 }
 
 func New(ctx context.Context) *Service {
@@ -79,11 +82,23 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
 		return fmt.Errorf("failed to start monitor: %w", err)
 	}
 
+	// Create filter chain
+	var filterChain *filter.Chain
+	if len(cfg.Filters) > 0 {
+		chain, err := filter.NewChain(cfg.Filters)
+		if err != nil {
+			streamCancel()
+			return fmt.Errorf("failed to create filter chain: %w", err)
+		}
+		filterChain = chain
+	}
+
 	// Create log stream
 	ls := &LogStream{
-		Name:    cfg.Name,
-		Config:  cfg,
-		Monitor: mon,
+		Name:        cfg.Name,
+		Config:      cfg,
+		Monitor:     mon,
+		FilterChain: filterChain,
 		Stats: &StreamStats{
 			StartTime: time.Now(),
 		},
@@ -93,7 +108,18 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
 
 	// Start TCP server if configured
 	if cfg.TCPServer != nil && cfg.TCPServer.Enabled {
-		tcpChan := mon.Subscribe()
+		// Create filtered channel
+		rawChan := mon.Subscribe()
+		tcpChan := make(chan monitor.LogEntry, cfg.TCPServer.BufferSize)
+
+		// Start filter goroutine for TCP
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			defer close(tcpChan)
+			s.filterLoop(streamCtx, rawChan, tcpChan, filterChain)
+		}()
+
 		ls.TCPServer = stream.NewTCPStreamer(tcpChan, *cfg.TCPServer)
 
 		if err := s.startTCPServer(ls); err != nil {
@@ -104,7 +130,18 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
 
 	// Start HTTP server if configured
 	if cfg.HTTPServer != nil && cfg.HTTPServer.Enabled {
-		httpChan := mon.Subscribe()
+		// Create filtered channel
+		rawChan := mon.Subscribe()
+		httpChan := make(chan monitor.LogEntry, cfg.HTTPServer.BufferSize)
+
+		// Start filter goroutine for HTTP
+		s.wg.Add(1)
+		go func() {
+			defer s.wg.Done()
+			defer close(httpChan)
+			s.filterLoop(streamCtx, rawChan, httpChan, filterChain)
+		}()
+
 		ls.HTTPServer = stream.NewHTTPStreamer(httpChan, *cfg.HTTPServer)
 
 		if err := s.startHTTPServer(ls); err != nil {
@@ -119,6 +156,31 @@ func (s *Service) CreateStream(cfg config.StreamConfig) error {
 	return nil
 }
 
+// filterLoop applies filters to log entries
+func (s *Service) filterLoop(ctx context.Context, in <-chan monitor.LogEntry, out chan<- monitor.LogEntry, chain *filter.Chain) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case entry, ok := <-in:
+			if !ok {
+				return
+			}
+
+			// Apply filter chain if configured
+			if chain == nil || chain.Apply(entry) {
+				select {
+				case out <- entry:
+				case <-ctx.Done():
+					return
+				default:
+					// Drop if output buffer is full
+				}
+			}
+		}
+	}
+}
+
 func (s *Service) GetStream(name string) (*LogStream, error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
@@ -178,17 +240,17 @@ func (s *Service) Shutdown() {
 	s.wg.Wait()
 }
 
-func (s *Service) GetGlobalStats() map[string]interface{} {
+func (s *Service) GetGlobalStats() map[string]any {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
-	stats := map[string]interface{}{
-		"streams":       make(map[string]interface{}),
+	stats := map[string]any{
+		"streams":       make(map[string]any),
 		"total_streams": len(s.streams),
 	}
 
 	for name, stream := range s.streams {
-		stats["streams"].(map[string]interface{})[name] = stream.GetStats()
+		stats["streams"].(map[string]any)[name] = stream.GetStats()
 	}
 
 	return stats

src/internal/ratelimit/limiter.go

@@ -192,9 +192,9 @@ func (l *Limiter) RemoveConnection(remoteAddr string) {
 }
 
 // Returns rate limiter statistics
-func (l *Limiter) GetStats() map[string]interface{} {
+func (l *Limiter) GetStats() map[string]any {
 	if l == nil {
-		return map[string]interface{}{
+		return map[string]any{
 			"enabled": false,
 		}
 	}
@@ -210,13 +210,13 @@ func (l *Limiter) GetStats() map[string]interface{} {
 	}
 	l.connMu.RUnlock()
 
-	return map[string]interface{}{
+	return map[string]any{
 		"enabled":           true,
 		"total_requests":    l.totalRequests.Load(),
 		"blocked_requests":  l.blockedRequests.Load(),
 		"active_ips":        activeIPs,
 		"total_connections": totalConnections,
-		"config": map[string]interface{}{
+		"config": map[string]any{
 			"requests_per_second": l.config.RequestsPerSecond,
 			"burst_size":          l.config.BurstSize,
 			"limit_by":            l.config.LimitBy,

src/internal/stream/httpstreamer.go

@@ -132,7 +132,7 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
 	if allowed, statusCode, message := h.rateLimiter.CheckHTTP(remoteAddr); !allowed {
 		ctx.SetStatusCode(statusCode)
 		ctx.SetContentType("application/json")
-		json.NewEncoder(ctx).Encode(map[string]interface{}{
+		json.NewEncoder(ctx).Encode(map[string]any{
 			"error":       message,
 			"retry_after": "60", // seconds
 		})
@@ -149,7 +149,7 @@ func (h *HTTPStreamer) requestHandler(ctx *fasthttp.RequestCtx) {
 	default:
 		ctx.SetStatusCode(fasthttp.StatusNotFound)
 		ctx.SetContentType("application/json")
-		json.NewEncoder(ctx).Encode(map[string]interface{}{
+		json.NewEncoder(ctx).Encode(map[string]any{
 			"error": "Not Found",
 			"message": fmt.Sprintf("Available endpoints: %s (SSE stream), %s (status)",
 				h.streamPath, h.statusPath),
@@ -218,7 +218,7 @@ func (h *HTTPStreamer) handleStream(ctx *fasthttp.RequestCtx) {
 
 		// Send initial connected event
 		clientID := fmt.Sprintf("%d", time.Now().UnixNano())
-		connectionInfo := map[string]interface{}{
+		connectionInfo := map[string]any{
 			"client_id":   clientID,
 			"stream_path": h.streamPath,
 			"status_path": h.statusPath,
@@ -280,7 +280,7 @@ func (h *HTTPStreamer) formatHeartbeat() string {
 	}
 
 	if h.config.Heartbeat.Format == "json" {
-		data := make(map[string]interface{})
+		data := make(map[string]any)
 		data["type"] = "heartbeat"
 
 		if h.config.Heartbeat.IncludeTimestamp {
@@ -315,19 +315,19 @@ func (h *HTTPStreamer) formatHeartbeat() string {
 func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
 	ctx.SetContentType("application/json")
 
-	var rateLimitStats interface{}
+	var rateLimitStats any
 	if h.rateLimiter != nil {
 		rateLimitStats = h.rateLimiter.GetStats()
 	} else {
-		rateLimitStats = map[string]interface{}{
+		rateLimitStats = map[string]any{
 			"enabled": false,
 		}
 	}
 
-	status := map[string]interface{}{
+	status := map[string]any{
 		"service": "LogWisp",
 		"version": version.Short(),
-		"server": map[string]interface{}{
+		"server": map[string]any{
 			"type":           "http",
 			"port":           h.config.Port,
 			"active_clients": h.activeClients.Load(),
@@ -339,8 +339,8 @@ func (h *HTTPStreamer) handleStatus(ctx *fasthttp.RequestCtx) {
 			"stream": h.streamPath,
 			"status": h.statusPath,
 		},
-		"features": map[string]interface{}{
-			"heartbeat": map[string]interface{}{
+		"features": map[string]any{
+			"heartbeat": map[string]any{
 				"enabled":  h.config.Heartbeat.Enabled,
 				"interval": h.config.Heartbeat.IntervalSeconds,
 				"format":   h.config.Heartbeat.Format,

src/internal/stream/tcpstreamer.go

@@ -116,7 +116,7 @@ func (t *TCPStreamer) broadcastLoop() {
 			}
 			data = append(data, '\n')
 
-			t.server.connections.Range(func(key, value interface{}) bool {
+			t.server.connections.Range(func(key, value any) bool {
 				conn := key.(gnet.Conn)
 				conn.AsyncWrite(data, nil)
 				return true
@@ -124,7 +124,7 @@ func (t *TCPStreamer) broadcastLoop() {
 
 		case <-tickerChan:
 			if heartbeat := t.formatHeartbeat(); heartbeat != nil {
-				t.server.connections.Range(func(key, value interface{}) bool {
+				t.server.connections.Range(func(key, value any) bool {
 					conn := key.(gnet.Conn)
 					conn.AsyncWrite(heartbeat, nil)
 					return true
@@ -142,7 +142,7 @@ func (t *TCPStreamer) formatHeartbeat() []byte {
 		return nil
 	}
 
-	data := make(map[string]interface{})
+	data := make(map[string]any)
 	data["type"] = "heartbeat"
 
 	if t.config.Heartbeat.IncludeTimestamp {