v0.33.0 tui tap for live view and console enhancements

internal/pipeline/pipeline.go

@@ -21,13 +21,11 @@ import (
 type Pipeline struct {
 	Config *config.PipelineConfig
 
-	// Components
 	Registry *Registry
-	Sources  map[string]source.Source // Track instances by ID
+	Sources  map[string]source.Source
 	Sinks    map[string]sink.Sink
 	Sessions *session.Manager
 
-	// Pipeline flow
 	Flow   *flow.Flow
 	Stats  *PipelineStats
 	logger *log.Logger
@@ -35,7 +33,8 @@ type Pipeline struct {
 	// Sinks that stall dispatch rather than drop (core.CapBackpressure)
 	backpressure map[string]bool
 
-	// Runtime
+	tap atomic.Pointer[core.Tap] // a viewer's copy of what the flow decides
+
 	ctx          context.Context
 	cancel       context.CancelFunc
 	wg           sync.WaitGroup
@@ -45,7 +44,6 @@ type Pipeline struct {
 	finishedOnce sync.Once
 }
 
-// PipelineStats contains runtime statistics for a pipeline
 type PipelineStats struct {
 	StartTime                 time.Time
 	TotalEntriesDroppedBySink atomic.Uint64
@@ -54,18 +52,14 @@ type PipelineStats struct {
 	FlowStats                 map[string]any
 }
 
-// NewPipeline creates a new pipeline with registry support
 func NewPipeline(
 	cfg *config.PipelineConfig,
 	logger *log.Logger,
 ) (*Pipeline, error) {
-	// Create pipeline context
 	pipelineCtx, pipelineCancel := context.WithCancel(context.Background())
 
-	// Create session manager with default timeout
 	sessionManager := session.NewManager(core.SessionDefaultMaxIdleTime)
 
-	// Create pipeline instance with registry
 	pipeline := &Pipeline{
 		Config:   cfg,
 		Registry: NewRegistry(cfg.Name, logger),
@@ -81,17 +75,14 @@ func NewPipeline(
 		finished:     make(chan struct{}),
 	}
 
-	// Create flow processor
 	flowProcessor, err := flow.NewFlow(cfg.Flow, logger)
 	if err != nil {
-		// If flow fails, stop session manager
 		pipelineCancel()
 		sessionManager.Stop()
 		return nil, fmt.Errorf("failed to create flow processor: %w", err)
 	}
 	pipeline.Flow = flowProcessor
 
-	// Initialize sources and sinks
 	if err := pipeline.initializeComponents(); err != nil {
 		pipelineCancel()
 		sessionManager.Stop()
@@ -102,10 +93,8 @@ func NewPipeline(
 }
 
 func (p *Pipeline) initializeComponents() error {
-	// Create sources based on plugin config if available
 	if len(p.Config.PluginSources) > 0 {
 		for _, srcCfg := range p.Config.PluginSources {
-			// Create session proxy for this source instance
 			sessionProxy := session.NewProxy(p.Sessions, srcCfg.ID)
 
 			src, err := p.Registry.CreateSource(
@@ -116,12 +105,11 @@ func (p *Pipeline) initializeComponents() error {
 				sessionProxy,
 			)
 			if err != nil {
-				return err // the registry names the plugin
+				return p.part("source", srcCfg.ID, err) // the registry names the plugin
 			}
 
-			// Check and inject capabilities using core interfaces
 			if err := p.initSourceCapabilities(src, srcCfg); err != nil {
-				return fmt.Errorf("failed to initiate capabilities for source %s: %w", srcCfg.ID, err)
+				return p.part("source", srcCfg.ID, fmt.Errorf("failed to initiate capabilities for source %s: %w", srcCfg.ID, err))
 			}
 
 			p.Sources[srcCfg.ID] = src
@@ -130,10 +118,8 @@ func (p *Pipeline) initializeComponents() error {
 		return fmt.Errorf("no plugin sources defined")
 	}
 
-	// Create sinks based on plugin config if available
 	if len(p.Config.PluginSinks) > 0 {
 		for _, sinkCfg := range p.Config.PluginSinks {
-			// Create session proxy for this sink instance
 			sessionProxy := session.NewProxy(p.Sessions, sinkCfg.ID)
 
 			snk, err := p.Registry.CreateSink(
@@ -144,12 +130,11 @@ func (p *Pipeline) initializeComponents() error {
 				sessionProxy,
 			)
 			if err != nil {
-				return err // the registry names the plugin
+				return p.part("sink", sinkCfg.ID, err) // the registry names the plugin
 			}
 
-			// Check and inject capabilities using core interfaces
 			if err := p.initSinkCapabilities(snk, sinkCfg); err != nil {
-				return fmt.Errorf("failed to initiate capabilities for sink %s: %w", sinkCfg.ID, err)
+				return p.part("sink", sinkCfg.ID, fmt.Errorf("failed to initiate capabilities for sink %s: %w", sinkCfg.ID, err))
 			}
 
 			p.Sinks[sinkCfg.ID] = snk
@@ -161,24 +146,34 @@ func (p *Pipeline) initializeComponents() error {
 	return nil
 }
 
-// initSourceCapabilities checks and injects optional capabilities
+// PartError is a source's or sink's error, which names its part for a screen
+// to place; its text is the error's own
+type PartError struct {
+	Pipeline, Role, ID string
+	Err                error
+}
+
+func (e *PartError) Error() string { return e.Err.Error() }
+func (e *PartError) Unwrap() error { return e.Err }
+
+func (p *Pipeline) part(role, id string, err error) error {
+	return &PartError{p.Config.Name, role, id, err}
+}
+
 func (p *Pipeline) initSourceCapabilities(s source.Source, cfg config.PluginSourceConfig) error {
-	// Initiate and activate source capabilities
 	var hasTLS, hasAuth bool
 	for _, c := range s.Capabilities() {
 		switch c {
-		// Network capabilities
 		case core.CapNetLimit:
-			continue // No-op for now, placeholder
+			continue
 		case core.CapTLS, core.CapProxyTLS:
 			hasTLS = true
 		case core.CapAuth:
 			hasAuth = true
 
-		// Session capabilities
 		case core.CapSessionAware:
 		case core.CapMultiSession:
-			continue // TODO
+			continue
 
 		default:
 			return fmt.Errorf("unknown capability type: %s", c)
@@ -202,27 +197,23 @@ func checkAuthCapability(hasTLS, hasAuth bool) error {
 	return nil
 }
 
-// initSinkCapabilities checks and injects optional capabilities
 func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig) error {
-	// Initiate and activate sink capabilities
 	var hasTLS, hasAuth bool
 	for _, c := range s.Capabilities() {
 		switch c {
 		case core.CapBackpressure:
 			p.backpressure[cfg.ID] = true
 
-		// Network capabilities
 		case core.CapNetLimit:
-			continue // No-op for now, placeholder
+			continue
 		case core.CapTLS, core.CapProxyTLS:
 			hasTLS = true
 		case core.CapAuth:
 			hasAuth = true
 
-		// Session capabilities
 		case core.CapSessionAware:
 		case core.CapMultiSession:
-			continue // TODO
+			continue
 
 		default:
 			return fmt.Errorf("unknown capability type: %s", c)
@@ -238,19 +229,23 @@ func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig
 
 // run connects the subscribed sources through the flow to the sinks. It
 // returns once every source channel has closed, at Stop or at end of input.
-func (p *Pipeline) run(subscriptions []<-chan core.LogEntry) {
+func (p *Pipeline) run(ids []string, subscriptions []<-chan core.LogEntry) {
 	defer p.wg.Done()
 	defer p.finish()
 	defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name)
 
 	var componentWg sync.WaitGroup
-	for _, ch := range subscriptions {
+	for i, ch := range subscriptions {
 		componentWg.Add(1)
 		go func() {
 			defer componentWg.Done()
 			// Range drains in-flight entries once the source closes the channel
 			for entry := range ch {
-				if event, passed := p.Flow.Process(entry); passed {
+				event, stage, index := p.Flow.Process(entry)
+				if t := p.tap.Load(); t != nil {
+					t.Offer(core.Tapped{Pipeline: p.Config.Name, Source: ids[i], Stage: stage, Index: index, Event: event})
+				}
+				if stage == "" {
 					p.dispatch(event)
 				}
 			}
@@ -270,6 +265,9 @@ func (p *Pipeline) run(subscriptions []<-chan core.LogEntry) {
 					if !ok {
 						return
 					}
+					if t := p.tap.Load(); t != nil {
+						t.Offer(core.Tapped{Pipeline: p.Config.Name, Event: event})
+					}
 					p.dispatch(event)
 				case <-hbCtx.Done():
 					return
@@ -303,13 +301,16 @@ func (p *Pipeline) dispatch(event core.TransportEvent) {
 	}
 }
 
+// SetTap offers what the flow decides to t from the next entry on; nil stops
+func (p *Pipeline) SetTap(t *core.Tap) { p.tap.Store(t) }
+
 func (p *Pipeline) finish() { p.finishedOnce.Do(func() { close(p.finished) }) }
 
 // Finished is closed when every source has ended, or the pipeline shut down.
 // Only a finite source (console at end of input) ends without Stop.
 func (p *Pipeline) Finished() <-chan struct{} { return p.finished }
 
-// Start starts the pipeline operation and all its components including flow, sources, and sinks
+// Start starts the sinks before the sources, so the first entry has somewhere to go
 func (p *Pipeline) Start() error {
 	if !p.running.CompareAndSwap(false, true) {
 		return fmt.Errorf("pipeline %s is already running", p.Config.Name)
@@ -319,33 +320,34 @@ func (p *Pipeline) Start() error {
 	p.ctx, p.cancel = context.WithCancel(context.Background())
 	p.halt = make(chan struct{})
 
-	// Start all sinks
 	for id, s := range p.Sinks {
 		if err := s.Start(p.ctx); err != nil {
-			return fmt.Errorf("failed to start sink %s: %w", id, err)
+			return p.part("sink", id, fmt.Errorf("failed to start sink %s: %w", id, err))
 		}
 	}
 
-	// Subscribed before Start: a source publishes from its first entry on
+	// Subscribed before Start: a source publishes from its first entry on. One
+	// loop keeps ids beside their channels; map order differs between loops.
+	ids := make([]string, 0, len(p.Sources))
 	subscriptions := make([]<-chan core.LogEntry, 0, len(p.Sources))
-	for _, src := range p.Sources {
-		subscriptions = append(subscriptions, src.Subscribe())
+	for id, src := range p.Sources {
+		ids, subscriptions = append(ids, id), append(subscriptions, src.Subscribe())
 	}
 	for id, src := range p.Sources {
 		if err := src.Start(); err != nil {
-			return fmt.Errorf("failed to start source %s: %w", id, err)
+			return p.part("source", id, fmt.Errorf("failed to start source %s: %w", id, err))
 		}
 	}
 
-	// Start the central processing loop
 	p.Stats.StartTime = time.Now()
 	p.wg.Add(1)
-	go p.run(subscriptions)
+	go p.run(ids, subscriptions)
 
 	return nil
 }
 
-// Stop stops the pipeline operation and all its components including flow, sources, and sinks
+// Stop stops the sources, lets the run loop drain, then stops the sinks, so
+// entries in flight reach them
 func (p *Pipeline) Stop() error {
 	if !p.running.CompareAndSwap(true, false) {
 		return fmt.Errorf("pipeline %s is not running", p.Config.Name)
@@ -354,7 +356,6 @@ func (p *Pipeline) Stop() error {
 	p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name)
 	close(p.halt)
 
-	// 1. Stop all sources concurrently to halt new data ingress and close their channels
 	var sourceWg sync.WaitGroup
 	for _, src := range p.Sources {
 		sourceWg.Add(1)
@@ -365,10 +366,8 @@ func (p *Pipeline) Stop() error {
 	}
 	sourceWg.Wait()
 
-	// 2. Wait for the run loop to finish processing and sending all in-flight data
 	p.wg.Wait()
 
-	// 3. Stop all sinks concurrently now that no new data will be sent
 	var sinkWg sync.WaitGroup
 	for _, s := range p.Sinks {
 		sinkWg.Add(1)
@@ -384,20 +383,18 @@ func (p *Pipeline) Stop() error {
 	return nil
 }
 
-// Shutdown gracefully stops the pipeline and all its components, deinitializing them for app shutdown or complete pipeline removal by service
+// Shutdown stops the pipeline for good, with its session manager
 func (p *Pipeline) Shutdown() {
 	p.logger.Info("msg", "Shutting down pipeline",
 		"component", "pipeline",
 		"pipeline", p.Config.Name)
 
-	// Ensure the pipeline is stopped before shutting down
 	if p.running.Load() {
 		if err := p.Stop(); err != nil {
 			p.logger.Error("msg", "Error stopping pipeline during shutdown", "error", err)
 		}
 	}
 
-	// Stop long-running components
 	if p.Sessions != nil {
 		p.Sessions.Stop()
 	}
@@ -408,10 +405,8 @@ func (p *Pipeline) Shutdown() {
 		"pipeline", p.Config.Name)
 }
 
-// GetStats returns a map of pipeline statistics
 func (p *Pipeline) GetStats() map[string]any {
-	// Recovery to handle concurrent access during shutdown
-	// When service is shutting down, sources/sinks might be nil or partially stopped
+	// Sources and sinks may be partly stopped while the service shuts down
 	defer func() {
 		if r := recover(); r != nil {
 			p.logger.Error("msg", "Panic getting pipeline stats",
@@ -420,7 +415,6 @@ func (p *Pipeline) GetStats() map[string]any {
 		}
 	}()
 
-	// 1. Live collect source stats
 	sources := make([]map[string]any, 0, len(p.Sources))
 	for _, src := range p.Sources {
 		if src == nil {
@@ -438,7 +432,6 @@ func (p *Pipeline) GetStats() map[string]any {
 		})
 	}
 
-	// 2. Live collect sink stats
 	sinks := make([]map[string]any, 0, len(p.Sinks))
 	for _, snk := range p.Sinks {
 		if snk == nil {
@@ -456,7 +449,6 @@ func (p *Pipeline) GetStats() map[string]any {
 		})
 	}
 
-	// 3. Collect flow stats and calculate filtered total
 	var flowStats map[string]any
 	var totalFiltered uint64
 	var totalProcessed uint64
@@ -464,12 +456,10 @@ func (p *Pipeline) GetStats() map[string]any {
 	if p.Flow != nil {
 		flowStats = p.Flow.GetStats()
 
-		// Map the top-level processed counter directly from Flow's source of truth
 		if tp, ok := flowStats["total_processed"].(uint64); ok {
 			totalProcessed = tp
 		}
 
-		// Calculate total dropped specifically by the filter chain
 		if filters, ok := flowStats["filters"].(map[string]any); ok {
 			if totalPassed, ok := filters["total_passed"].(uint64); ok {
 				if tProc, ok := filters["total_processed"].(uint64); ok {
@@ -479,7 +469,6 @@ func (p *Pipeline) GetStats() map[string]any {
 		}
 	}
 
-	// 4. Calculate Uptime
 	var uptime int
 	if p.running.Load() && !p.Stats.StartTime.IsZero() {
 		uptime = int(time.Since(p.Stats.StartTime).Seconds())

internal/pipeline/pipeline_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -17,9 +18,12 @@ import (
 	"github.com/lixenwraith/log"
 )
 
-// finiteSource publishes n entries as soon as it starts, then ends
+// finiteSource publishes n entries as soon as it starts, then ends, once hold
+// closes when there is one. Entries carry its id as their Source.
 type finiteSource struct {
+	id   string
 	n    int
+	hold chan struct{}
 	subs []chan core.LogEntry
 }
 
@@ -33,9 +37,13 @@ func (s *finiteSource) Start() error {
 	go func() {
 		for i := range s.n {
 			for _, ch := range s.subs {
-				ch <- core.LogEntry{Time: time.Now(), Message: strconv.Itoa(i)}
+				msg := strconv.Itoa(i)
+				ch <- core.LogEntry{Time: time.Now(), Source: s.id, Message: msg, RawSize: int64(len(msg))}
 			}
 		}
+		if s.hold != nil {
+			<-s.hold
+		}
 		for _, ch := range s.subs {
 			close(ch)
 		}
@@ -74,7 +82,8 @@ var testSinks = map[string]*slowSink{}
 func init() {
 	plugin.RegisterSource("random", func(id string, cfg map[string]any, _ *log.Logger, _ *session.Proxy) (source.Source, error) {
 		n, _ := strconv.Atoi(fmt.Sprint(cfg["n"]))
-		return &finiteSource{n: n}, nil
+		hold, _ := cfg["hold"].(chan struct{})
+		return &finiteSource{id: id, n: n, hold: hold}, nil
 	})
 	plugin.RegisterSink("null", func(id string, _ map[string]any, _ *log.Logger, _ *session.Proxy) (sink.Sink, error) {
 		return testSinks[id], nil
@@ -132,3 +141,127 @@ func TestStopReleasesStalledBackpressureSink(t *testing.T) {
 		t.Fatal("Stop blocked behind a stalled backpressure sink")
 	}
 }
+
+// The tap gets each entry's verdict under its source's id: the stage that
+// dropped it, or what the sinks got. Drops come only when All is set, and
+// heartbeats come as passed, from no source.
+func TestTapSeesEachVerdict(t *testing.T) {
+	const n = 1000
+	verdict := func(msg string) (string, int) {
+		switch {
+		case len(msg) > 2:
+			return "rate_limit", 0
+		case strings.HasPrefix(msg, "1"):
+			return "filters", 0
+		case !strings.HasSuffix(msg, "0"):
+			return "filters", 1
+		}
+		return "", 0
+	}
+	for _, all := range []bool{false, true} {
+		hold := make(chan struct{})
+		sink := &slowSink{in: make(chan core.TransportEvent, 4*n)}
+		testSinks[t.Name()] = sink
+		p, err := NewPipeline(&config.PipelineConfig{
+			Name: t.Name(),
+			Flow: &config.FlowConfig{
+				RateLimit: &config.RateLimitConfig{EntriesPerSecond: 1e6, Policy: "drop", MaxEntrySizeBytes: 2},
+				Filters:   []config.FilterConfig{{Type: "exclude", Patterns: []string{" 1"}}, {Type: "include", Patterns: []string{"0$"}}},
+				Heartbeat: &config.HeartbeatConfig{Enabled: true, IntervalMS: 100},
+			},
+			PluginSources: []config.PluginSourceConfig{
+				{ID: "a", Type: "random", Config: map[string]any{"n": n, "hold": hold}},
+				{ID: "b", Type: "random", Config: map[string]any{"n": n, "hold": hold}},
+			},
+			PluginSinks: []config.PluginSinkConfig{{ID: t.Name(), Type: "null"}},
+		}, log.NewLogger())
+		if err != nil {
+			t.Fatal(err)
+		}
+		tap := core.NewTap(4 * n)
+		tap.All.Store(all)
+		p.SetTap(tap)
+		if err := p.Start(); err != nil {
+			t.Fatal(err)
+		}
+		var got []core.Tapped
+		for beat := false; !beat; {
+			select {
+			case r := <-tap.Records():
+				if beat = r.Source == ""; beat && (r.Stage != "" || r.Event.Entry.Source != "heartbeat" || len(r.Event.Payload) == 0) {
+					t.Errorf("heartbeat %+v", r)
+				}
+				got = append(got, r)
+			case <-time.After(5 * time.Second):
+				t.Fatal("no heartbeat reached the tap")
+			}
+		}
+		close(hold)
+		<-p.Finished()
+		p.Shutdown()
+		for len(tap.Records()) > 0 {
+			got = append(got, <-tap.Records())
+		}
+
+		seen, passed := map[string]int{}, 0
+		for _, r := range got {
+			e := r.Event.Entry
+			if r.Stage == "" {
+				passed++
+			}
+			if r.Source == "" {
+				continue
+			}
+			stage, index := verdict(e.Message)
+			if r.Pipeline != t.Name() || r.Source != e.Source || r.Stage != stage || r.Index != index ||
+				stage == "" && string(r.Event.Payload) != e.Message+"\n" {
+				t.Errorf("all=%v: %s %q: got %s %s[%d] %q, want %s[%d]", all, r.Source, e.Message, r.Pipeline, r.Stage, r.Index, r.Event.Payload, stage, index)
+			}
+			seen[r.Source+e.Message]++
+		}
+		for i := range n {
+			msg := strconv.Itoa(i)
+			want := 1
+			if stage, _ := verdict(msg); stage != "" && !all {
+				want = 0
+			}
+			for _, id := range []string{"a", "b"} {
+				if seen[id+msg] != want {
+					t.Errorf("all=%v: %s %q tapped %d times, want %d", all, id, msg, seen[id+msg], want)
+				}
+			}
+		}
+		if tap.Dropped() != 0 || len(sink.in) != passed {
+			t.Errorf("all=%v: a tap with room dropped %d; the sink got %d, %d passed", all, tap.Dropped(), len(sink.in), passed)
+		}
+	}
+}
+
+// A tap nobody reads costs the pipeline nothing: every entry still reaches
+// the sink, and the tap counts what it could not take.
+func TestFullTapNeverStallsAPipeline(t *testing.T) {
+	const n = 2000
+	s := &slowSink{in: make(chan core.TransportEvent, 1), out: make(chan string, n)}
+	p := newTestPipeline(t, n, s)
+	tap := core.NewTap(1)
+	p.SetTap(tap)
+	if err := p.Start(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-p.Finished():
+	case <-time.After(10 * time.Second):
+		t.Fatal("a full tap stalled the pipeline")
+	}
+	for i := range n {
+		select {
+		case <-s.out:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("the sink got %d of %d", i, n)
+		}
+	}
+	p.Shutdown()
+	if tap.Dropped() != n-1 {
+		t.Fatalf("tap dropped %d, want %d", tap.Dropped(), n-1)
+	}
+}