142 lines
3.6 KiB
Go
142 lines
3.6 KiB
Go
package flow
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync/atomic"
|
|
|
|
"github.com/lixenwraith/logwisp/internal/config"
|
|
"github.com/lixenwraith/logwisp/internal/core"
|
|
"github.com/lixenwraith/logwisp/internal/filter"
|
|
"github.com/lixenwraith/logwisp/internal/format"
|
|
|
|
"github.com/lixenwraith/log"
|
|
)
|
|
|
|
// Flow takes a pipeline's entries through rate limit, filters and format to
|
|
// the TransportEvent its sinks get
|
|
type Flow struct {
|
|
rateLimiter *RateLimiter
|
|
filterChain *filter.Chain
|
|
formatter format.Formatter
|
|
heartbeat *HeartbeatGenerator
|
|
logger *log.Logger
|
|
|
|
totalProcessed atomic.Uint64
|
|
totalDropped atomic.Uint64
|
|
totalFormatted atomic.Uint64
|
|
}
|
|
|
|
func NewFlow(cfg *config.FlowConfig, logger *log.Logger) (*Flow, error) {
|
|
if cfg == nil {
|
|
cfg = &config.FlowConfig{}
|
|
}
|
|
|
|
f := &Flow{
|
|
logger: logger,
|
|
}
|
|
|
|
if cfg.RateLimit != nil {
|
|
limiter, err := NewRateLimiter(*cfg.RateLimit, logger)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create rate limiter: %w", err)
|
|
}
|
|
f.rateLimiter = limiter
|
|
}
|
|
|
|
if len(cfg.Filters) > 0 {
|
|
chain, err := filter.NewChain(cfg.Filters, logger)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create filter chain: %w", err)
|
|
}
|
|
f.filterChain = chain
|
|
}
|
|
|
|
formatter, err := format.NewFormatter(cfg.Format)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create formatter: %w", err)
|
|
}
|
|
f.formatter = formatter
|
|
|
|
// Heartbeats are written like the entries
|
|
if cfg.Heartbeat != nil {
|
|
hb, err := NewHeartbeatGenerator(cfg.Heartbeat, formatter, logger)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("heartbeat: %w", err)
|
|
}
|
|
f.heartbeat = hb
|
|
}
|
|
|
|
logger.Info("msg", "Flow processor created",
|
|
"component", "flow",
|
|
"rate_limiter", f.rateLimiter != nil,
|
|
"filter_chain", f.filterChain != nil,
|
|
"formatter", formatter.Name(),
|
|
"heartbeat", f.heartbeat != nil)
|
|
|
|
return f, nil
|
|
}
|
|
|
|
// Process applies the flow's stages to an entry and returns the event for the
|
|
// sinks, or the stage that dropped it and, for filters, which one
|
|
func (f *Flow) Process(entry core.LogEntry) (event core.TransportEvent, stage string, index int) {
|
|
f.totalProcessed.Add(1)
|
|
if f.rateLimiter != nil && !f.rateLimiter.Allow(entry) {
|
|
f.totalDropped.Add(1)
|
|
return core.TransportEvent{Entry: entry}, "rate_limit", 0
|
|
}
|
|
if f.filterChain != nil {
|
|
if i := f.filterChain.Apply(entry); i >= 0 {
|
|
f.totalDropped.Add(1)
|
|
return core.TransportEvent{Entry: entry}, "filters", i
|
|
}
|
|
}
|
|
formatted, err := f.formatter.Format(entry)
|
|
if err != nil {
|
|
f.logger.Error("msg", "Failed to format log entry",
|
|
"component", "flow",
|
|
"error", err)
|
|
f.totalDropped.Add(1)
|
|
return core.TransportEvent{Entry: entry}, "format", 0
|
|
}
|
|
f.totalFormatted.Add(1)
|
|
// The structured entry rides along so chain sinks are format-independent
|
|
return core.TransportEvent{Time: entry.Time, Payload: formatted, Entry: entry}, "", 0
|
|
}
|
|
|
|
// StartHeartbeat returns the heartbeat events until ctx ends, or nil without
|
|
// a heartbeat
|
|
func (f *Flow) StartHeartbeat(ctx context.Context) <-chan core.TransportEvent {
|
|
if f.heartbeat == nil {
|
|
return nil
|
|
}
|
|
return f.heartbeat.Start(ctx)
|
|
}
|
|
|
|
func (f *Flow) GetStats() map[string]any {
|
|
stats := map[string]any{
|
|
"total_processed": f.totalProcessed.Load(),
|
|
"total_dropped": f.totalDropped.Load(),
|
|
"total_formatted": f.totalFormatted.Load(),
|
|
}
|
|
|
|
if f.rateLimiter != nil {
|
|
stats["rate_limiter"] = f.rateLimiter.GetStats()
|
|
}
|
|
|
|
if f.filterChain != nil {
|
|
stats["filters"] = f.filterChain.GetStats()
|
|
}
|
|
|
|
if f.formatter != nil {
|
|
stats["formatter"] = f.formatter.Name()
|
|
}
|
|
|
|
if f.heartbeat != nil {
|
|
stats["heartbeat_enabled"] = true
|
|
stats["heartbeat_interval_ms"] = f.heartbeat.IntervalMS()
|
|
}
|
|
|
|
return stats
|
|
}
|