v0.34.0 doc sweep and update

internal/config/config.go

@@ -15,7 +15,6 @@ type Config struct {
 	// Selected file path is runtime metadata, never a configurable override.
 	ConfigFile string `toml:"-"`
 
-	// Existing fields
 	Logging   *LogConfig       `toml:"logging"`
 	Pipelines []PipelineConfig `toml:"pipelines"`
 }
@@ -79,15 +78,15 @@ type HeartbeatConfig struct {
 	Enabled          bool   `toml:"enabled" help:"send heartbeats"`
 	IntervalMS       int64  `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"`
 	IncludeTimestamp bool   `toml:"include_timestamp" help:"stamp each heartbeat with its time"`
-	IncludeStats     bool   `toml:"include_stats" help:"add the pipeline's counters"`
-	Format           string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"heartbeat format"`
+	IncludeStats     bool   `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"`
+	Format           string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"no effect: heartbeats use the flow's format"`
 }
 
 // FormatConfig is how a pipeline writes its entries
 type FormatConfig struct {
 	Type            string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"entry format"`
 	Flags           int64  `toml:"flags,omitempty" lw:"zero=by type" help:"formatter flags"`
-	TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339" help:"Go time layout of timestamps"`
+	TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"Go time layout of timestamps"`
 	SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"`
 }
 
@@ -143,7 +142,7 @@ type PluginSourceConfig struct {
 	ID         string         `toml:"id"`
 	Type       string         `toml:"type"`
 	Config     map[string]any `toml:"config"`
-	ConfigFile string         `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
+	ConfigFile string         `toml:"config_file,omitempty"` // accepted, never read
 }
 
 // NullSourceOptions defines settings for a null source (no configuration needed)
@@ -152,10 +151,10 @@ type NullSourceOptions struct{}
 // RandomSourceOptions defines settings for a random log generator source
 type RandomSourceOptions struct {
 	IntervalMS int64  `toml:"interval_ms" default:"500" lw:"min=1" help:"milliseconds between entries"`
-	JitterMS   int64  `toml:"jitter_ms" lw:"min=0,zero=none" help:"random milliseconds added to the interval, at most the interval"`
+	JitterMS   int64  `toml:"jitter_ms" lw:"min=0,zero=none" help:"milliseconds the interval varies by, half each way; at most the interval"`
 	Format     string `toml:"format" default:"txt" lw:"enum=raw|txt|json" help:"entry format"`
-	Length     int64  `toml:"length" default:"20" lw:"min=1" help:"message characters"`
-	Special    bool   `toml:"special" help:"mix in control characters"`
+	Length     int64  `toml:"length" default:"20" lw:"min=1" help:"bytes of random text"`
+	Special    bool   `toml:"special" help:"mix in control and non-ASCII bytes"`
 }
 
 // FileSourceOptions defines settings for a file-based source
@@ -209,7 +208,7 @@ type PluginSinkConfig struct {
 	ID         string         `toml:"id"`
 	Type       string         `toml:"type"`
 	Config     map[string]any `toml:"config"`
-	ConfigFile string         `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
+	ConfigFile string         `toml:"config_file,omitempty"` // accepted, never read
 }
 
 // NullSinkOptions defines settings for a null sink (no configuration needed)
@@ -230,7 +229,7 @@ type FileSinkOptions struct {
 	MaxSizeMB       int64   `toml:"max_size_mb" default:"100" lw:"min=1" help:"size at which a file rotates"`
 	MaxTotalSizeMB  int64   `toml:"max_total_size_mb" default:"1000" lw:"min=1" help:"size of all files, beyond which the oldest go"`
 	MinDiskFreeMB   int64   `toml:"min_disk_free_mb" lw:"zero=none" help:"free space kept on the disk; negative means 100"`
-	RetentionHours  float64 `toml:"retention_hours" default:"168" lw:"min=0" help:"hours a file is kept"`
+	RetentionHours  float64 `toml:"retention_hours" default:"168" lw:"min=0,zero=kept" help:"hours a .log file is kept; 0 keeps them"`
 	BufferSize      int64   `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"`
 	FlushIntervalMs int64   `toml:"flush_interval_ms" default:"100" lw:"min=1" help:"milliseconds between flushes"`
 }

internal/config/schema.go

@@ -42,13 +42,13 @@ type Plugin struct {
 var plugins = []Plugin{
 	{"source", "console", Local, false, true, "Standard input, line by line", reflect.TypeFor[ConsoleSourceOptions]()},
 	{"source", "file", Local, false, false, "Follow the files of a directory that match a pattern", reflect.TypeFor[FileSourceOptions]()},
-	{"source", "random", Local, false, false, "Random entries, for tests", reflect.TypeFor[RandomSourceOptions]()},
-	{"source", "null", Local, false, false, "Nothing, ever", reflect.TypeFor[NullSourceOptions]()},
+	{"source", "random", Local, false, false, "Generated entries, to check a deployment before real logs", reflect.TypeFor[RandomSourceOptions]()},
+	{"source", "null", Local, false, false, "Nothing; holds a source's place until one is chosen", reflect.TypeFor[NullSourceOptions]()},
 	{"source", "tcp_chain", ChainListener, false, false, "Entries from tcp_chain sinks of other instances", reflect.TypeFor[TCPChainSourceOptions]()},
 	{"source", "http_chain", ChainListener, true, false, "Entry batches from http_chain sinks of other instances", reflect.TypeFor[HTTPChainSourceOptions]()},
 	{"sink", "console", Local, false, false, "Standard output or standard error", reflect.TypeFor[ConsoleSinkOptions]()},
 	{"sink", "file", Local, false, false, "Rotating files", reflect.TypeFor[FileSinkOptions]()},
-	{"sink", "null", Local, false, false, "Discard", reflect.TypeFor[NullSinkOptions]()},
+	{"sink", "null", Local, false, false, "Discards; holds a sink's place until one is chosen", reflect.TypeFor[NullSinkOptions]()},
 	{"sink", "tcp", Listener, false, false, "Serve entries to TCP clients, one per line", reflect.TypeFor[TCPSinkOptions]()},
 	{"sink", "http", Listener, true, false, "Serve entries as server-sent events, with a browser viewer", reflect.TypeFor[HTTPSinkOptions]()},
 	{"sink", "tcp_chain", Dialer, false, false, "Forward entries to a tcp_chain source", reflect.TypeFor[TCPChainSinkOptions]()},

internal/config/schema_test.go

@@ -32,10 +32,11 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) {
 	}
 }
 
-// A zero number takes the default unless zero means something there, a switch
-// that defaults on stays off when set off, a negative number is refused, and
-// what moved from the constructors keeps its meaning: min_disk_free_mb below
-// zero is the default, a backoff maximum below the minimum the default maximum
+// A zero number takes the default unless zero means something there (a file
+// sink's retention_hours 0 keeps files), a switch that defaults on stays off
+// when set off, a negative number is refused, and what moved from the
+// constructors keeps its meaning: min_disk_free_mb below zero is the default,
+// a backoff maximum below the minimum the default maximum
 func TestZeroDefaultsAndNegatives(t *testing.T) {
 	tcp, err := Decode[TCPSinkOptions]("sink", "tcp", map[string]any{"port": int64(1), "buffer_size": int64(0), "keep_alive": false})
 	if err != nil || tcp.BufferSize != 1000 || tcp.KeepAlive || tcp.Host != "0.0.0.0" {
@@ -55,6 +56,16 @@ func TestZeroDefaultsAndNegatives(t *testing.T) {
 			t.Errorf("min_disk_free_mb %v: %+v %v, want %d", in, file, err, want)
 		}
 	}
+	for in, want := range map[any]float64{nil: 168, int64(0): 0, 0.0: 0, 2.5: 2.5} {
+		m := map[string]any{"directory": "d", "name": "n", "retention_hours": in}
+		if in == nil {
+			delete(m, "retention_hours")
+		}
+		file, err := Decode[FileSinkOptions]("sink", "file", m)
+		if err != nil || file.RetentionHours != want {
+			t.Errorf("retention_hours %v: %+v %v, want %v", in, file, err, want)
+		}
+	}
 	chain, err := Decode[TCPChainSinkOptions]("sink", "tcp_chain", map[string]any{"host": "h", "port": int64(1), "backoff_min_ms": int64(800), "backoff_max_ms": int64(700)})
 	if err != nil || chain.BackoffMaxMS != 30000 {
 		t.Fatalf("backoff maximum below the minimum: %+v %v", chain, err)

internal/sink/http/web/scram.js

@@ -16,9 +16,10 @@ const encoder = new TextEncoder();
 
 /**
  * Logs in to the logwisp http sink mounted at base (a URL ending in "/"), per
- * logwisp's doc/scram-auth-plan.md; it resolves only once the server has proved
- * it holds this user's verifier. The session is an HttpOnly cookie, or with
- * session "token" the returned token. onProgress(fraction) follows Argon2.
+ * logwisp's doc/security.md (Password authentication) and doc/chaining.md
+ * (Wire protocol); it resolves only once the server has proved it holds this
+ * user's verifier. The session is an HttpOnly cookie, or with session "token"
+ * the returned token. onProgress(fraction) follows Argon2.
  */
 export async function login(base, username, password, { onProgress, session = "cookie" } = {}) {
   if (session !== "cookie" && session !== "token") {

internal/sink/null/null.go

@@ -15,35 +15,30 @@ import (
 	"github.com/lixenwraith/log"
 )
 
-// init registers the component in plugin factory
 func init() {
 	if err := plugin.RegisterSink("null", NewNullSinkPlugin); err != nil {
 		panic(fmt.Sprintf("failed to register null sink: %v", err))
 	}
 }
 
-// NullSink discards all received transport events, used for testing
+// NullSink discards every event: it holds a sink's place until one is chosen,
+// or isolates a source or flow while it is measured
 type NullSink struct {
-	// Plugin identity and session management
 	id      string
 	proxy   *session.Proxy
 	session *session.Session
 
-	// Application
 	input  chan core.TransportEvent
 	logger *log.Logger
 
-	// Runtime
 	done      chan struct{}
 	startTime time.Time
 
-	// Statistics
 	totalReceived atomic.Uint64
 	totalBytes    atomic.Uint64
 	lastReceived  atomic.Value // time.Time
 }
 
-// NewNullSinkPlugin creates a null sink through plugin factory
 func NewNullSinkPlugin(
 	id string,
 	configMap map[string]any,
@@ -62,7 +57,6 @@ func NewNullSinkPlugin(
 	}
 	ns.lastReceived.Store(time.Time{})
 
-	// Create session for null sink
 	ns.session = proxy.CreateSession(
 		"null://devnull",
 		map[string]any{
@@ -78,7 +72,6 @@ func NewNullSinkPlugin(
 	return ns, nil
 }
 
-// Capabilities returns supported capabilities
 func (ns *NullSink) Capabilities() []core.Capability {
 	return []core.Capability{
 		core.CapSessionAware,
@@ -86,14 +79,11 @@ func (ns *NullSink) Capabilities() []core.Capability {
 	}
 }
 
-// Input returns the channel for sending transport events
 func (ns *NullSink) Input() chan<- core.TransportEvent {
 	return ns.input
 }
 
-// Start begins the processing loop
 func (ns *NullSink) Start(ctx context.Context) error {
-
 	ns.startTime = time.Now()
 	go ns.processLoop(ctx)
 	ns.logger.Debug("msg", "Null sink started",
@@ -102,7 +92,6 @@ func (ns *NullSink) Start(ctx context.Context) error {
 	return nil
 }
 
-// Stop gracefully shuts down the sink
 func (ns *NullSink) Stop() {
 	if ns.session != nil {
 		ns.proxy.RemoveSession(ns.session.ID)
@@ -113,7 +102,6 @@ func (ns *NullSink) Stop() {
 		"total_received", ns.totalReceived.Load())
 }
 
-// GetStats returns sink statistics
 func (ns *NullSink) GetStats() sink.SinkStats {
 	lastRcv, _ := ns.lastReceived.Load().(time.Time)
 
@@ -129,7 +117,6 @@ func (ns *NullSink) GetStats() sink.SinkStats {
 	}
 }
 
-// processLoop reads transport events and discards them
 func (ns *NullSink) processLoop(ctx context.Context) {
 	for {
 		select {
@@ -137,7 +124,6 @@ func (ns *NullSink) processLoop(ctx context.Context) {
 			if !ok {
 				return
 			}
-			// Discard the event, only update stats
 			ns.totalReceived.Add(1)
 			ns.totalBytes.Add(uint64(len(event.Payload)))
 			ns.lastReceived.Store(time.Now())

internal/source/file/file_watcher.go

@@ -310,8 +310,6 @@ func (w *fileWatcher) checkFile() error {
 	return nil
 }
 
-// initPosition records the file's metadata and, unless the watcher was created
-// to read from the start, sets the initial read position to the end
 // scanLinesCapped is bufio.ScanLines, except that a line longer than
 // core.MaxLogEntryBytes continues in the next one, as on the console source:
 // the scanner would otherwise stop at it and every poll re-read the lines before.
@@ -323,6 +321,8 @@ func scanLinesCapped(data []byte, atEOF bool) (int, []byte, error) {
 	return advance, token, err
 }
 
+// initPosition records the file's metadata and, unless the watcher was created
+// to read from the start, sets the initial read position to the end.
 func (w *fileWatcher) initPosition() error {
 	file, err := os.Open(w.directory)
 	if err != nil {

internal/source/null/null.go

@@ -14,34 +14,28 @@ import (
 	"github.com/lixenwraith/log"
 )
 
-// init registers the component in plugin factory
 func init() {
 	if err := plugin.RegisterSource("null", NewNullSourcePlugin); err != nil {
 		panic(fmt.Sprintf("failed to register null source: %v", err))
 	}
 }
 
-// NullSource generates no log entries, used for testing
+// NullSource generates nothing: it holds a source's place until one is chosen
 type NullSource struct {
-	// Plugin identity and session management
 	id      string
 	proxy   *session.Proxy
 	session *session.Session
 
-	// Application
 	subscribers []chan core.LogEntry
 	logger      *log.Logger
 
-	// Runtime
 	done chan struct{}
 
-	// Statistics
 	totalEntries  atomic.Uint64
 	startTime     time.Time
 	lastEntryTime atomic.Value // time.Time
 }
 
-// NewNullSourcePlugin creates a null source through plugin factory
 func NewNullSourcePlugin(
 	id string,
 	configMap map[string]any,
@@ -60,7 +54,6 @@ func NewNullSourcePlugin(
 	}
 	ns.lastEntryTime.Store(time.Time{})
 
-	// Create session for null source
 	ns.session = proxy.CreateSession(
 		"null://void",
 		map[string]any{
@@ -76,21 +69,18 @@ func NewNullSourcePlugin(
 	return ns, nil
 }
 
-// Capabilities returns supported capabilities
 func (ns *NullSource) Capabilities() []core.Capability {
 	return []core.Capability{
 		core.CapSessionAware,
 	}
 }
 
-// Subscribe returns a channel for receiving log entries
 func (ns *NullSource) Subscribe() <-chan core.LogEntry {
 	ch := make(chan core.LogEntry, 1000)
 	ns.subscribers = append(ns.subscribers, ch)
 	return ch
 }
 
-// Start begins the source operation (no-op for null source)
 func (ns *NullSource) Start() error {
 	ns.startTime = time.Now()
 	ns.proxy.UpdateActivity(ns.session.ID)
@@ -100,7 +90,6 @@ func (ns *NullSource) Start() error {
 	return nil
 }
 
-// Stop signals the source to stop
 func (ns *NullSource) Stop() {
 	close(ns.done)
 	if ns.session != nil {
@@ -114,7 +103,6 @@ func (ns *NullSource) Stop() {
 		"instance_id", ns.id)
 }
 
-// GetStats returns the source's statistics
 func (ns *NullSource) GetStats() source.SourceStats {
 	lastEntry, _ := ns.lastEntryTime.Load().(time.Time)
 

internal/source/random/random.go

@@ -17,49 +17,41 @@ import (
 	"github.com/lixenwraith/log"
 )
 
-// init registers the component in plugin factory
 func init() {
 	if err := plugin.RegisterSource("random", NewRandomSourcePlugin); err != nil {
 		panic(fmt.Sprintf("failed to register random source: %v", err))
 	}
 }
 
-// RandomSource generates random log entries for testing
+// RandomSource generates entries, to check a deployment before real logs are wired
 type RandomSource struct {
-	// Plugin identity and session management
 	id      string
 	proxy   *session.Proxy
 	session *session.Session
 
-	// Configuration
 	config *config.RandomSourceOptions
 
-	// Application
 	subscribers []chan core.LogEntry
 	logger      *log.Logger
 	rng         *rand.Rand
 	mu          sync.RWMutex
 
-	// Runtime
 	done   chan struct{}
 	wg     sync.WaitGroup
 	cancel chan struct{}
 
-	// Statistics
 	totalEntries   atomic.Uint64
 	droppedEntries atomic.Uint64
 	startTime      time.Time
 	lastEntryTime  atomic.Value // time.Time
 }
 
-// NewRandomSourcePlugin creates a random source through plugin factory
 func NewRandomSourcePlugin(
 	id string,
 	configMap map[string]any,
 	logger *log.Logger,
 	proxy *session.Proxy,
 ) (source.Source, error) {
-	// Step 1: Create empty config struct with defaults
 	opts, err := config.Decode[config.RandomSourceOptions]("source", "random", configMap)
 	if err != nil {
 		return nil, err
@@ -77,7 +69,6 @@ func NewRandomSourcePlugin(
 	}
 	rs.lastEntryTime.Store(time.Time{})
 
-	// Create session for random source
 	rs.session = proxy.CreateSession(
 		fmt.Sprintf("random://%s", id),
 		map[string]any{
@@ -98,14 +89,12 @@ func NewRandomSourcePlugin(
 	return rs, nil
 }
 
-// Capabilities returns supported capabilities
 func (rs *RandomSource) Capabilities() []core.Capability {
 	return []core.Capability{
 		core.CapSessionAware,
 	}
 }
 
-// Subscribe returns a channel for receiving log entries
 func (rs *RandomSource) Subscribe() <-chan core.LogEntry {
 	rs.mu.Lock()
 	defer rs.mu.Unlock()
@@ -114,7 +103,6 @@ func (rs *RandomSource) Subscribe() <-chan core.LogEntry {
 	return ch
 }
 
-// Start begins generating random log entries
 func (rs *RandomSource) Start() error {
 	rs.startTime = time.Now()
 	rs.wg.Add(1)
@@ -127,7 +115,6 @@ func (rs *RandomSource) Start() error {
 	return nil
 }
 
-// Stop signals the source to stop generating
 func (rs *RandomSource) Stop() {
 	close(rs.cancel)
 	rs.wg.Wait()
@@ -148,7 +135,6 @@ func (rs *RandomSource) Stop() {
 		"total_entries", rs.totalEntries.Load())
 }
 
-// GetStats returns the source's statistics
 func (rs *RandomSource) GetStats() source.SourceStats {
 	lastEntry, _ := rs.lastEntryTime.Load().(time.Time)
 
@@ -169,12 +155,10 @@ func (rs *RandomSource) GetStats() source.SourceStats {
 	}
 }
 
-// generateLoop continuously generates random log entries at configured intervals
 func (rs *RandomSource) generateLoop() {
 	defer rs.wg.Done()
 
 	for {
-		// Calculate next interval with jitter
 		interval := time.Duration(rs.config.IntervalMS) * time.Millisecond
 		if rs.config.JitterMS > 0 {
 			jitter := time.Duration(rs.rng.Intn(int(rs.config.JitterMS))) * time.Millisecond
@@ -194,7 +178,6 @@ func (rs *RandomSource) generateLoop() {
 	}
 }
 
-// generateEntry creates a random log entry based on configured format
 func (rs *RandomSource) generateEntry() core.LogEntry {
 	now := time.Now()
 
@@ -247,7 +230,6 @@ func (rs *RandomSource) generateEntry() core.LogEntry {
 	}
 }
 
-// generateRandomString creates a random string of specified length
 func (rs *RandomSource) generateRandomString(length int) string {
 	const normalChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "
 	const specialChars = "\t\n\r\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F"
@@ -259,17 +241,14 @@ func (rs *RandomSource) generateRandomString(length int) string {
 		// Reserve space for at least one special and one unicode char
 		normalLength := length - 2
 
-		// Generate normal characters
 		for i := 0; i < normalLength; i++ {
 			result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
 		}
 
-		// Insert special character at random position
 		specialPos := rs.rng.Intn(len(result) + 1)
 		specialChar := specialChars[rs.rng.Intn(len(specialChars))]
 		result = append(result[:specialPos], append([]byte{specialChar}, result[specialPos:]...)...)
 
-		// Insert unicode character at random position
 		unicodePos := rs.rng.Intn(len(result) + 1)
 		unicodeChar := unicodeChars[rs.rng.Intn(len(unicodeChars)/3)*3:]
 		if len(unicodeChar) >= 3 {
@@ -281,12 +260,10 @@ func (rs *RandomSource) generateRandomString(length int) string {
 			}
 		}
 
-		// Trim to exact length if needed
 		if len(result) > length {
 			result = result[:length]
 		}
 	} else {
-		// Normal generation without special characters
 		for i := 0; i < length; i++ {
 			result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
 		}
@@ -295,13 +272,11 @@ func (rs *RandomSource) generateRandomString(length int) string {
 	return string(result)
 }
 
-// randomLogLevel returns a random log level
 func (rs *RandomSource) randomLogLevel() string {
 	levels := []string{"DEBUG", "INFO", "WARN", "ERROR"}
 	return levels[rs.rng.Intn(len(levels))]
 }
 
-// publish sends a log entry to all subscribers
 func (rs *RandomSource) publish(entry core.LogEntry) {
 	rs.mu.RLock()
 	defer rs.mu.RUnlock()

internal/tui/inspector.go

@@ -109,7 +109,7 @@ func (a *app) items(n compose.Node, path string) []string {
 // placeholder says what an unset key means: its default, what zero means, or
 // the kind of value it takes
 func placeholder(k config.Key) string {
-	if k.Zero != "" {
+	if k.Zero != "" && k.Default == "" {
 		return "(" + k.Zero + ")"
 	}
 	return cmp.Or(k.Default, k.Hint)

internal/tui/tui_test.go

@@ -249,6 +249,16 @@ func TestInspectorHelpSitsUnderItsKeys(t *testing.T) {
 	}
 }
 
+// An unset key shows its default, what zero means when it has none, or the
+// kind of value it takes
+func TestAnUnsetKeyShowsItsDefaultFirst(t *testing.T) {
+	for want, k := range map[string]config.Key{"168": {Default: "168", Zero: "kept"}, "(off)": {Zero: "off"}, "dir": {Hint: "dir"}} {
+		if got := placeholder(k); got != want {
+			t.Errorf("%+v shows %q, want %q", k, got, want)
+		}
+	}
+}
+
 // Hints follow their text, three columns on, not the screen's far edge: the
 // keys after the status, the bar's verbs after the pipelines
 func TestHintsFollowTheirText(t *testing.T) {