v0.33.0 tui tap for live view and console enhancements

internal/core/const.go

@@ -29,6 +29,12 @@ func Network(host string) (string, error) {
 const (
 	MaxLogEntryBytes = 1024 * 1024
 
+	// A tapped entry's message, fields and payload are cut to this, and its
+	// level, source and node to TapNameBytes, so a viewer's scrollback holds
+	// no source's whole line
+	TapEntryBytes = 16 * 1024
+	TapNameBytes  = 256
+
 	FileWatcherPollInterval = 100 * time.Millisecond
 
 	SessionDefaultMaxIdleTime = 30 * time.Minute

internal/core/flow.go

@@ -3,6 +3,7 @@ package core
 import (
 	"encoding/json"
 	"strings"
+	"sync/atomic"
 	"time"
 )
 
@@ -26,17 +27,81 @@ type TransportEvent struct {
 	Entry LogEntry
 }
 
-// Levels are the entry levels by rising severity, each with the words naming
-// it in a line: the sources, the console sink's painter and the viewer share them
-var Levels = []struct {
+// Tapped is what a pipeline's flow decided about one entry: Stage names the
+// stage that dropped Event.Entry (rate_limit, filters at Index, format), or is
+// "" when it passed and Event.Payload is what the sinks got.
+type Tapped struct {
+	Pipeline string
+	Source   string // the source's id; "" for a heartbeat
+	Stage    string
+	Index    int
+	Event    TransportEvent
+}
+
+// Tap hands Tapped records to a viewer without ever waiting: a full tap drops
+// and counts. Unless All is set it takes only what passed, so a strict filter
+// cannot crowd those out.
+type Tap struct {
+	ch    chan Tapped
+	All   atomic.Bool
+	drops atomic.Uint64
+}
+
+func NewTap(size int) *Tap { return &Tap{ch: make(chan Tapped, size)} }
+
+// Offer clips only what it can keep, so a full tap costs the pipeline nothing;
+// a reader racing it costs a drop, never a wait
+func (t *Tap) Offer(r Tapped) {
+	switch {
+	case r.Stage != "" && !t.All.Load():
+		return
+	case len(t.ch) == cap(t.ch):
+		t.drops.Add(1)
+		return
+	}
+	e := &r.Event.Entry
+	r.Event.Payload = clip(r.Event.Payload, TapEntryBytes)
+	e.Message, e.Fields = clip(e.Message, TapEntryBytes), clip(e.Fields, TapEntryBytes)
+	e.Level, e.Source, e.Node = clip(e.Level, TapNameBytes), clip(e.Source, TapNameBytes), clip(e.Node, TapNameBytes)
+	select {
+	case t.ch <- r:
+	default:
+		t.drops.Add(1)
+	}
+}
+
+func (t *Tap) Records() <-chan Tapped { return t.ch }
+
+// Dropped counts the records a full tap could not take
+func (t *Tap) Dropped() uint64 { return t.drops.Load() }
+
+// clip copies the first n bytes of a longer s, so a kept record does not pin
+// its line
+func clip[T ~string | ~[]byte](s T, n int) T {
+	if len(s) <= n {
+		return s
+	}
+	head := make([]byte, n)
+	copy(head, s)
+	return T(head)
+}
+
+// Level is a level: the words naming it in a line, and the further values a
+// level key may hold, which prose would misread ("critical path")
+type Level struct {
 	Level string   `json:"level"`
 	Names []string `json:"names"`
-}{
-	{"TRACE", []string{"TRACE"}},
-	{"DEBUG", []string{"DEBUG", "DBG"}},
-	{"INFO", []string{"INFO", "INF"}},
-	{"WARN", []string{"WARN", "WARNING"}},
-	{"ERROR", []string{"ERROR", "ERR", "FATAL"}},
+	Keyed []string `json:"-"`
+}
+
+// Levels are the entry levels by rising severity: the sources, the console
+// sink's painter and the viewer share them
+var Levels = []Level{
+	{"TRACE", []string{"TRACE"}, []string{"VERBOSE"}},
+	{"DEBUG", []string{"DEBUG", "DBG"}, nil},
+	{"INFO", []string{"INFO", "INF"}, []string{"I", "NOTICE", "INFORMATION"}},
+	{"WARN", []string{"WARN", "WARNING"}, []string{"W"}},
+	{"ERROR", []string{"ERROR", "ERR", "FATAL"}, []string{"E", "CRIT", "CRITICAL", "ALERT", "EMERG", "EMERGENCY", "PANIC", "DPANIC"}},
 }
 
 // LevelWord finds the first whole word of text, in any case, that names a

internal/core/flow_test.go

@@ -1,6 +1,11 @@
 package core
 
-import "testing"
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"unsafe"
+)
 
 // A line's level is its first whole word, in any case, naming a level, not the
 // most severe one named; a wanted level skips the names of the others.
@@ -22,3 +27,41 @@ func TestLevelIsTheFirstWordNamingOne(t *testing.T) {
 		}
 	}
 }
+
+// A tapped record keeps at most TapEntryBytes of an entry's message, fields
+// and payload and TapNameBytes of its level, source and node, copied, so a
+// viewer's scrollback holds none of the source's line
+func TestTapClipsLongEntriesByCopying(t *testing.T) {
+	line := strings.Repeat("x", MaxLogEntryBytes)
+	payload, fields := []byte(line), json.RawMessage(line)
+	tap := NewTap(1)
+	tap.Offer(Tapped{Event: TransportEvent{Payload: payload, Entry: LogEntry{Message: line, Fields: fields, Level: line, Source: line, Node: line}}})
+	r := <-tap.Records()
+	e := r.Event.Entry
+	if len(e.Message) != TapEntryBytes || len(e.Fields) != TapEntryBytes || len(r.Event.Payload) != TapEntryBytes {
+		t.Fatalf("kept %d, %d and %d bytes, want %d", len(e.Message), len(e.Fields), len(r.Event.Payload), TapEntryBytes)
+	}
+	if len(e.Level) != TapNameBytes || len(e.Source) != TapNameBytes || len(e.Node) != TapNameBytes {
+		t.Fatalf("kept %d, %d and %d bytes of the names, want %d", len(e.Level), len(e.Source), len(e.Node), TapNameBytes)
+	}
+	for _, s := range []string{e.Message, e.Level, e.Source, e.Node} {
+		if unsafe.StringData(s) == unsafe.StringData(line) {
+			t.Fatal("a clipped part shares the source's line")
+		}
+	}
+	if &e.Fields[0] == &fields[0] || &r.Event.Payload[0] == &payload[0] {
+		t.Fatal("a clipped part shares the source's line")
+	}
+}
+
+// A full tap drops and counts without copying anything, so a tap nobody
+// reads costs a pipeline nothing
+func TestAFullTapCostsNothing(t *testing.T) {
+	line := strings.Repeat("x", MaxLogEntryBytes)
+	tap := NewTap(1)
+	rec := Tapped{Event: TransportEvent{Payload: []byte(line), Entry: LogEntry{Message: line, Fields: json.RawMessage(line), Level: line}}}
+	tap.Offer(rec)
+	if allocs := testing.AllocsPerRun(100, func() { tap.Offer(rec) }); allocs != 0 || tap.Dropped() != 101 {
+		t.Fatalf("%v allocations an offer, %d dropped", allocs, tap.Dropped())
+	}
+}