v0.33.0 tui tap for live view and console enhancements

internal/pipeline/pipeline_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
@@ -17,9 +18,12 @@ import (
 	"github.com/lixenwraith/log"
 )
 
-// finiteSource publishes n entries as soon as it starts, then ends
+// finiteSource publishes n entries as soon as it starts, then ends, once hold
+// closes when there is one. Entries carry its id as their Source.
 type finiteSource struct {
+	id   string
 	n    int
+	hold chan struct{}
 	subs []chan core.LogEntry
 }
 
@@ -33,9 +37,13 @@ func (s *finiteSource) Start() error {
 	go func() {
 		for i := range s.n {
 			for _, ch := range s.subs {
-				ch <- core.LogEntry{Time: time.Now(), Message: strconv.Itoa(i)}
+				msg := strconv.Itoa(i)
+				ch <- core.LogEntry{Time: time.Now(), Source: s.id, Message: msg, RawSize: int64(len(msg))}
 			}
 		}
+		if s.hold != nil {
+			<-s.hold
+		}
 		for _, ch := range s.subs {
 			close(ch)
 		}
@@ -74,7 +82,8 @@ var testSinks = map[string]*slowSink{}
 func init() {
 	plugin.RegisterSource("random", func(id string, cfg map[string]any, _ *log.Logger, _ *session.Proxy) (source.Source, error) {
 		n, _ := strconv.Atoi(fmt.Sprint(cfg["n"]))
-		return &finiteSource{n: n}, nil
+		hold, _ := cfg["hold"].(chan struct{})
+		return &finiteSource{id: id, n: n, hold: hold}, nil
 	})
 	plugin.RegisterSink("null", func(id string, _ map[string]any, _ *log.Logger, _ *session.Proxy) (sink.Sink, error) {
 		return testSinks[id], nil
@@ -132,3 +141,127 @@ func TestStopReleasesStalledBackpressureSink(t *testing.T) {
 		t.Fatal("Stop blocked behind a stalled backpressure sink")
 	}
 }
+
+// The tap gets each entry's verdict under its source's id: the stage that
+// dropped it, or what the sinks got. Drops come only when All is set, and
+// heartbeats come as passed, from no source.
+func TestTapSeesEachVerdict(t *testing.T) {
+	const n = 1000
+	verdict := func(msg string) (string, int) {
+		switch {
+		case len(msg) > 2:
+			return "rate_limit", 0
+		case strings.HasPrefix(msg, "1"):
+			return "filters", 0
+		case !strings.HasSuffix(msg, "0"):
+			return "filters", 1
+		}
+		return "", 0
+	}
+	for _, all := range []bool{false, true} {
+		hold := make(chan struct{})
+		sink := &slowSink{in: make(chan core.TransportEvent, 4*n)}
+		testSinks[t.Name()] = sink
+		p, err := NewPipeline(&config.PipelineConfig{
+			Name: t.Name(),
+			Flow: &config.FlowConfig{
+				RateLimit: &config.RateLimitConfig{EntriesPerSecond: 1e6, Policy: "drop", MaxEntrySizeBytes: 2},
+				Filters:   []config.FilterConfig{{Type: "exclude", Patterns: []string{" 1"}}, {Type: "include", Patterns: []string{"0$"}}},
+				Heartbeat: &config.HeartbeatConfig{Enabled: true, IntervalMS: 100},
+			},
+			PluginSources: []config.PluginSourceConfig{
+				{ID: "a", Type: "random", Config: map[string]any{"n": n, "hold": hold}},
+				{ID: "b", Type: "random", Config: map[string]any{"n": n, "hold": hold}},
+			},
+			PluginSinks: []config.PluginSinkConfig{{ID: t.Name(), Type: "null"}},
+		}, log.NewLogger())
+		if err != nil {
+			t.Fatal(err)
+		}
+		tap := core.NewTap(4 * n)
+		tap.All.Store(all)
+		p.SetTap(tap)
+		if err := p.Start(); err != nil {
+			t.Fatal(err)
+		}
+		var got []core.Tapped
+		for beat := false; !beat; {
+			select {
+			case r := <-tap.Records():
+				if beat = r.Source == ""; beat && (r.Stage != "" || r.Event.Entry.Source != "heartbeat" || len(r.Event.Payload) == 0) {
+					t.Errorf("heartbeat %+v", r)
+				}
+				got = append(got, r)
+			case <-time.After(5 * time.Second):
+				t.Fatal("no heartbeat reached the tap")
+			}
+		}
+		close(hold)
+		<-p.Finished()
+		p.Shutdown()
+		for len(tap.Records()) > 0 {
+			got = append(got, <-tap.Records())
+		}
+
+		seen, passed := map[string]int{}, 0
+		for _, r := range got {
+			e := r.Event.Entry
+			if r.Stage == "" {
+				passed++
+			}
+			if r.Source == "" {
+				continue
+			}
+			stage, index := verdict(e.Message)
+			if r.Pipeline != t.Name() || r.Source != e.Source || r.Stage != stage || r.Index != index ||
+				stage == "" && string(r.Event.Payload) != e.Message+"\n" {
+				t.Errorf("all=%v: %s %q: got %s %s[%d] %q, want %s[%d]", all, r.Source, e.Message, r.Pipeline, r.Stage, r.Index, r.Event.Payload, stage, index)
+			}
+			seen[r.Source+e.Message]++
+		}
+		for i := range n {
+			msg := strconv.Itoa(i)
+			want := 1
+			if stage, _ := verdict(msg); stage != "" && !all {
+				want = 0
+			}
+			for _, id := range []string{"a", "b"} {
+				if seen[id+msg] != want {
+					t.Errorf("all=%v: %s %q tapped %d times, want %d", all, id, msg, seen[id+msg], want)
+				}
+			}
+		}
+		if tap.Dropped() != 0 || len(sink.in) != passed {
+			t.Errorf("all=%v: a tap with room dropped %d; the sink got %d, %d passed", all, tap.Dropped(), len(sink.in), passed)
+		}
+	}
+}
+
+// A tap nobody reads costs the pipeline nothing: every entry still reaches
+// the sink, and the tap counts what it could not take.
+func TestFullTapNeverStallsAPipeline(t *testing.T) {
+	const n = 2000
+	s := &slowSink{in: make(chan core.TransportEvent, 1), out: make(chan string, n)}
+	p := newTestPipeline(t, n, s)
+	tap := core.NewTap(1)
+	p.SetTap(tap)
+	if err := p.Start(); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-p.Finished():
+	case <-time.After(10 * time.Second):
+		t.Fatal("a full tap stalled the pipeline")
+	}
+	for i := range n {
+		select {
+		case <-s.out:
+		case <-time.After(5 * time.Second):
+			t.Fatalf("the sink got %d of %d", i, n)
+		}
+	}
+	p.Shutdown()
+	if tap.Dropped() != n-1 {
+		t.Fatalf("tap dropped %d, want %d", tap.Dropped(), n-1)
+	}
+}