v0.13.1 folder restructure, test script added, format adapter async fix
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Chain manages a sequence of filters, applying them in order
|
||||
type Chain struct {
|
||||
filters []*Filter
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
totalPassed atomic.Uint64
|
||||
}
|
||||
|
||||
// NewChain creates a new filter chain from a slice of filter configurations
|
||||
func NewChain(configs []config.FilterConfig, logger *log.Logger) (*Chain, error) {
|
||||
chain := &Chain{
|
||||
filters: make([]*Filter, 0, len(configs)),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
for i, cfg := range configs {
|
||||
filter, err := NewFilter(cfg, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("filter[%d]: %w", i, err)
|
||||
}
|
||||
chain.filters = append(chain.filters, filter)
|
||||
}
|
||||
|
||||
logger.Info("msg", "Filter chain created",
|
||||
"component", "filter_chain",
|
||||
"filter_count", len(configs))
|
||||
return chain, nil
|
||||
}
|
||||
|
||||
// Apply runs a log entry through all filters in the chain
|
||||
func (c *Chain) Apply(entry core.LogEntry) bool {
|
||||
c.totalProcessed.Add(1)
|
||||
|
||||
// No filters means pass everything
|
||||
if len(c.filters) == 0 {
|
||||
c.totalPassed.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
// All filters must pass
|
||||
for i, filter := range c.filters {
|
||||
if !filter.Apply(entry) {
|
||||
c.logger.Debug("msg", "Entry filtered out",
|
||||
"component", "filter_chain",
|
||||
"filter_index", i,
|
||||
"filter_type", filter.config.Type)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
c.totalPassed.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
// GetStats returns aggregated statistics for the entire chain
|
||||
func (c *Chain) GetStats() map[string]any {
|
||||
filterStats := make([]map[string]any, len(c.filters))
|
||||
for i, filter := range c.filters {
|
||||
filterStats[i] = filter.GetStats()
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"filter_count": len(c.filters),
|
||||
"total_processed": c.totalProcessed.Load(),
|
||||
"total_passed": c.totalPassed.Load(),
|
||||
"filters": filterStats,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Filter applies regex-based filtering to log entries
|
||||
type Filter struct {
|
||||
config config.FilterConfig
|
||||
patterns []*regexp.Regexp
|
||||
mu sync.RWMutex
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
totalMatched atomic.Uint64
|
||||
totalDropped atomic.Uint64
|
||||
}
|
||||
|
||||
// NewFilter creates a new filter from a configuration
|
||||
func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) {
|
||||
// Validate enums before setting defaults
|
||||
if cfg.Type != "" {
|
||||
validateType := lconfig.OneOf(config.FilterTypeInclude, config.FilterTypeExclude)
|
||||
if err := validateType(cfg.Type); err != nil {
|
||||
return nil, fmt.Errorf("type: %w", err)
|
||||
}
|
||||
}
|
||||
if cfg.Logic != "" {
|
||||
validateLogic := lconfig.OneOf(config.FilterLogicOr, config.FilterLogicAnd)
|
||||
if err := validateLogic(cfg.Logic); err != nil {
|
||||
return nil, fmt.Errorf("logic: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
if cfg.Type == "" {
|
||||
cfg.Type = config.FilterTypeInclude
|
||||
}
|
||||
if cfg.Logic == "" {
|
||||
cfg.Logic = config.FilterLogicOr
|
||||
}
|
||||
|
||||
f := &Filter{
|
||||
config: cfg,
|
||||
patterns: make([]*regexp.Regexp, 0, len(cfg.Patterns)),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
// Compile patterns
|
||||
for i, pattern := range cfg.Patterns {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pattern[%d] '%s': %w", i, pattern, err)
|
||||
}
|
||||
f.patterns = append(f.patterns, re)
|
||||
}
|
||||
|
||||
logger.Debug("msg", "Filter created",
|
||||
"component", "filter",
|
||||
"type", cfg.Type,
|
||||
"logic", cfg.Logic,
|
||||
"pattern_count", len(cfg.Patterns))
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Apply determines if a log entry should be passed through based on the filter's rules
|
||||
func (f *Filter) Apply(entry core.LogEntry) bool {
|
||||
f.totalProcessed.Add(1)
|
||||
|
||||
// No patterns means pass everything
|
||||
if len(f.patterns) == 0 {
|
||||
f.logger.Debug("msg", "No patterns configured, passing entry",
|
||||
"component", "filter",
|
||||
"type", f.config.Type)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check against all fields that might contain the log content
|
||||
text := entry.Message
|
||||
if entry.Level != "" {
|
||||
text = entry.Level + " " + text
|
||||
}
|
||||
if entry.Source != "" {
|
||||
text = entry.Source + " " + text
|
||||
}
|
||||
|
||||
f.logger.Debug("msg", "Filter checking entry",
|
||||
"component", "filter",
|
||||
"type", f.config.Type,
|
||||
"logic", f.config.Logic,
|
||||
"entry_level", entry.Level,
|
||||
"entry_source", entry.Source,
|
||||
"entry_message", entry.Message[:min(100, len(entry.Message))], // First 100 chars
|
||||
"text_to_match", text[:min(150, len(text))], // First 150 chars
|
||||
"patterns", f.config.Patterns)
|
||||
|
||||
for i, pattern := range f.config.Patterns {
|
||||
isMatch := f.patterns[i].MatchString(text)
|
||||
f.logger.Debug("msg", "Pattern match result",
|
||||
"component", "filter",
|
||||
"pattern_index", i,
|
||||
"pattern", pattern,
|
||||
"matched", isMatch)
|
||||
}
|
||||
|
||||
matched := f.matches(text)
|
||||
if matched {
|
||||
f.totalMatched.Add(1)
|
||||
}
|
||||
f.logger.Debug("msg", "Filter final match result",
|
||||
"component", "filter",
|
||||
"matched", matched)
|
||||
|
||||
// Determine if we should pass or drop
|
||||
shouldPass := false
|
||||
switch f.config.Type {
|
||||
case config.FilterTypeInclude:
|
||||
shouldPass = matched
|
||||
case config.FilterTypeExclude:
|
||||
shouldPass = !matched
|
||||
}
|
||||
|
||||
f.logger.Debug("msg", "Filter decision",
|
||||
"component", "filter",
|
||||
"type", f.config.Type,
|
||||
"matched", matched,
|
||||
"should_pass", shouldPass)
|
||||
|
||||
if !shouldPass {
|
||||
f.totalDropped.Add(1)
|
||||
}
|
||||
|
||||
return shouldPass
|
||||
}
|
||||
|
||||
// GetStats returns the filter's current statistics
|
||||
func (f *Filter) GetStats() map[string]any {
|
||||
return map[string]any{
|
||||
"type": f.config.Type,
|
||||
"logic": f.config.Logic,
|
||||
"pattern_count": len(f.patterns),
|
||||
"total_processed": f.totalProcessed.Load(),
|
||||
"total_matched": f.totalMatched.Load(),
|
||||
"total_dropped": f.totalDropped.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
// UpdatePatterns allows for dynamic, thread-safe updates to the filter's regex patterns
|
||||
func (f *Filter) UpdatePatterns(patterns []string) error {
|
||||
compiled := make([]*regexp.Regexp, 0, len(patterns))
|
||||
|
||||
// Compile all patterns first
|
||||
for i, pattern := range patterns {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
|
||||
}
|
||||
compiled = append(compiled, re)
|
||||
}
|
||||
|
||||
// Update atomically
|
||||
f.mu.Lock()
|
||||
f.patterns = compiled
|
||||
f.config.Patterns = patterns
|
||||
f.mu.Unlock()
|
||||
|
||||
f.logger.Info("msg", "Filter patterns updated",
|
||||
"component", "filter",
|
||||
"pattern_count", len(patterns))
|
||||
return nil
|
||||
}
|
||||
|
||||
// matches checks if the given text matches the filter's patterns according to its logic
|
||||
func (f *Filter) matches(text string) bool {
|
||||
switch f.config.Logic {
|
||||
case config.FilterLogicOr:
|
||||
// Match any pattern
|
||||
for _, re := range f.patterns {
|
||||
if re.MatchString(text) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
case config.FilterLogicAnd:
|
||||
// Must match all patterns
|
||||
for _, re := range f.patterns {
|
||||
if !re.MatchString(text) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
default:
|
||||
// Shouldn't happen after validation
|
||||
f.logger.Warn("msg", "Unknown filter logic",
|
||||
"component", "filter",
|
||||
"logic", f.config.Logic)
|
||||
return false
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user