v0.38.0 lwconf updates for deployment

lwconf: sparse stages, every problem of a part, pastes read by form

cmd/lwconf/lwconf.test.mjs

@@ -123,9 +123,9 @@ test("a preset's pipeline, a single file answered by dirs, reads back from every
   const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs);
   assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" });
   assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]);
-  assert.deepEqual(engine.parse("lw -p serve:path=/var/log/app.log,allow=192.0.2.0/24,allow=198.51.100.7", "command", dirs).value, pipelines);
+  assert.deepEqual(engine.parse("lw -p serve:path=/var/log/app.log,allow=192.0.2.0/24,allow=198.51.100.7", "command", dirs).value.pipelines, pipelines);
   for (const { name } of schema.forms) {
-    assert.deepEqual(engine.parse(engine.emit(pipelines, name).value, name).value, pipelines, name);
+    assert.deepEqual(engine.parse(engine.emit(pipelines, name).value, name).value, { pipelines, left_out: [] }, name);
   }
   assert.deepEqual(engine.validate(pipelines).value, { pipelines, problems: [] });
 });

cmd/lwconf/main.go

@@ -47,7 +47,12 @@ var calls = map[string]struct {
 		if err != nil {
 			return nil, err
 		}
-		return pipelines(f.Read(a[0], isDir))
+		c, err := f.Parse(a[0], isDir)
+		value, err := pipelines(c, err)
+		if err != nil {
+			return nil, err
+		}
+		return map[string]any{"pipelines": value, "left_out": append([]string{}, c.LeftOut...)}, nil
 	}},
 	"validate": {[]string{"pipelines"}, func(a []string) (any, error) {
 		c, err := compose.FromJSON([]byte(a[0]))

doc/architecture.md

@@ -278,10 +278,14 @@ answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key",
 - `schema()`: what `lw --schema` prints.
 - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to
   a string, a list key to strings.
-- `parse(text, form, dirs)`: the pipelines of a form's pasted text.
+- `parse(text, form, dirs)`: `{pipelines, left_out}`, the pipelines of a
+  form's pasted text and the names of what it held besides them: a file's
+  settings, an environment's `LOGWISP_` variables no pipeline flag has. Text
+  in another form's shape is refused naming that form.
 - `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled
-  (defaults, kinds, ids) and the first problem of each part (a value of the
+  (kinds, ids) and every problem of each part, each key's and the first rule
-  wrong kind too) as `{path, pipeline, node, key, text}`.
+  between a table's keys (a value of the wrong kind too), as `{path,
+  pipeline, node, key, text}`.
 - `emit(pipelines, form)`: the text of `command`, `environment` or `file`.
 - `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is
   `{op, pipeline, ...}`:
@@ -298,7 +302,10 @@ answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key",
   `{role, id, index}`: `source` or `sink` by its id, `filters` by its index,
   or the flow stage `rate_limit`, `format` or `heartbeat`, which `set` turns
   on and `remove` off; `add` takes `source`, `sink`, or `filters` with the
-  type `include` or `exclude`.
+  type `include`, `exclude` or `""` for the default.
+- Stages and options hold the keys set, so the forms carry no default an
+  edit did not set: a stage key's zero is unset, and an option's zero is
+  unset where it takes the key's default.
 
 ## Resource Management
 

doc/cli.md

@@ -80,7 +80,7 @@ pipelines have [their own flags](#pipelines):
     speaks HTTP: `auth.node_binding` only on the chain sources, and
     `auth.trusted_proxies` on both HTTP listeners, though only the `http`
     sink admits it
-  - a flow stage: its keys, and `on`, the values that turn it on
+  - a flow stage: its keys, a `summary`, and `on`, the values that turn it on
   - a preset key: its default and help, and the choices (`enum`) and `hint`
     of the option it sets
   - `forms`: `command`, `environment` and `file`, each with its hint
@@ -419,10 +419,11 @@ lw -c dev.toml --logging.level=debug
 `lw --check` validates without running: it builds every pipeline and plugin as
 a start would (options, TLS files, credentials files, startup warnings), but
 binds no port, reads no stdin, and opens, reads or creates no log file; it
-prints `configuration ok: N pipeline(s)` and exits 0. Otherwise it prints the
+prints `configuration ok: N pipeline(s)` and exits 0. Otherwise it prints
-first problem of each setting, flow stage, source, sink and pipeline, a line
+every key's problem of each setting, flow stage, source, sink and pipeline,
-each named by its key's path, or else the first file that fails to open, and
+and the first broken rule between a table's keys, a line each named by its
-exits 1 (2 when a named file is missing, see [Exit Codes](#exit-codes)):
+key's path, or else the first file that fails to open, and exits 1 (2 when a
+named file is missing, see [Exit Codes](#exit-codes)):
 
 ```bash
 lw --check -c /etc/logwisp/logwisp.toml

doc/configuration.md

@@ -192,8 +192,10 @@ and variables replace the file's whole, and an array-indexed path such as
 ## Validation
 
 The configuration is checked offline when it loads, at every reload and by
-`lw --check`. Each check reports the first problem of each table (the top
+`lw --check`. Each check reports every key's problem and the first broken
-level, `logging`, each flow stage, each source and sink) by its key's path:
+rule between a table's keys, in every table (the top level, `logging`, each
+flow stage, each source and sink), by its key's path, a list entry's by its
+index (`acl.allow[1]`):
 
 ```
 Error: Failed to load config: invalid configuration:

doc/lw.1

@@ -145,10 +145,10 @@ and credentials files and printing startup warnings, without binding a port
 or opening, reading or creating a log file; print
 .Dq configuration ok
 and exit 0.
-Otherwise print the first problem of each setting, flow stage, source, sink
+Otherwise print every key's problem of each setting, flow stage, source,
-and pipeline, a line each named by its key's path, or else the first file
+sink and pipeline, and the first broken rule between a table's keys, a line
-that fails to open, and exit 1, or 2 when a named configuration file is
+each named by its key's path, or else the first file that fails to open, and
-missing
+exit 1, or 2 when a named configuration file is missing
 .Pq see Sx EXIT STATUS .
 .It Fl \-dump
 Print the effective configuration, with the options, environment and

internal/compose/compose.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"regexp"
 	"slices"
 	"strconv"
 	"strings"
@@ -22,11 +23,13 @@ import (
 	"github.com/lixenwraith/toml"
 )
 
-// Composition is the pipelines being composed. Its flow stages hold their
+// Composition is the pipelines being composed. Its flow stages and plugin
-// defaults and its plugin options the kinds they declare, so each form it
+// options hold the keys set, the options in the kinds they declare, so each
-// writes loads back to the same pipelines.
+// form it writes loads back to the same pipelines. LeftOut is what a pasted
+// form held besides pipelines: a file's settings, other LOGWISP_ variables.
 type Composition struct {
 	Pipelines []config.PipelineConfig
+	LeftOut   []string
 }
 
 // Node is a part of a pipeline: a source or sink by its id, or a flow stage
@@ -112,9 +115,10 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
 	return c, c.settle()
 }
 
-// settle gives every flow stage its defaults, every plugin's options their
+// settle gives every stage the values its spec writes out (FillOn), every
-// kinds and every source and sink an id; the rules are Validate's. Its error
+// plugin's options their kinds, unsetting a zero that takes its key's
-// joins a plugin's each holding a value of the wrong kind, kept as given.
+// default, and every source and sink an id; the rules are Validate's. Its
+// error joins a plugin's each holding a value of the wrong kind, kept as given.
 func (c *Composition) settle() error {
 	var errs []error
 	for i := range c.Pipelines {
@@ -126,23 +130,20 @@ func (c *Composition) settle() error {
 			func(j int) string { return p.PluginSources[j].Type })
 		name(len(p.PluginSinks), func(j int) *string { return &p.PluginSinks[j].ID },
 			func(j int) string { return p.PluginSinks[j].Type })
-		for _, stage := range stages(p.Flow) {
+		for key, stage := range stages(p.Flow) {
 			if !stage.IsNil() {
-				config.Fill(stage.Interface())
+				config.FillOn(key, stage.Interface())
 			}
 		}
-		for j := range p.Flow.Filters {
-			config.Fill(&p.Flow.Filters[j])
-		}
 		for j, s := range p.PluginSources {
 			p.PluginSources[j].Config = orEmpty(s.Config)
-			if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
+			if err := coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
 				errs = append(errs, config.At(Node{Role: "source", ID: s.ID}.Path(i), err))
 			}
 		}
 		for j, s := range p.PluginSinks {
 			p.PluginSinks[j].Config = orEmpty(s.Config)
-			if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
+			if err := coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
 				errs = append(errs, config.At(Node{Role: "sink", ID: s.ID}.Path(i), err))
 			}
 		}
@@ -150,6 +151,32 @@ func (c *Composition) settle() error {
 	return errors.Join(errs...)
 }
 
+// coerce gives a plugin's options their kinds, then unsets a number at zero
+// where zero takes the key's default: lw reads it so, and the options then
+// show the default rather than a 0 that means it
+func coerce(role, typ string, m map[string]any) error {
+	err := config.Coerce(role, typ, m)
+	if p, ok := config.LookupPlugin(role, typ); ok {
+		unsetDefaults(p.Keys(), m)
+	}
+	return err
+}
+
+func unsetDefaults(keys []config.Key, m map[string]any) {
+	for _, k := range keys {
+		switch v := m[k.Name]; v {
+		case int64(0), 0.0:
+			if k.Default != "" && k.Zero == "" {
+				delete(m, k.Name)
+			}
+		default:
+			if sub, ok := v.(map[string]any); ok {
+				unsetDefaults(k.Keys, sub)
+			}
+		}
+	}
+}
+
 // name gives the parts a file left unnamed the ids their specs would get: the
 // shell forms write no empty id
 func name(n int, id func(int) *string, typ func(int) string) {
@@ -236,7 +263,8 @@ func (c *Composition) RemovePipeline(i int) error {
 }
 
 // Add appends a source or sink of a catalogue type under config.FreeID, or a
-// filter of type include or exclude, and returns its node
+// filter of type include or exclude, or of the default for "", and returns
+// its node
 func (c *Composition) Add(pi int, role, typ string) (Node, error) {
 	p, err := c.pipeline(pi)
 	if err != nil {
@@ -253,9 +281,7 @@ func (c *Composition) Add(pi int, role, typ string) (Node, error) {
 			ids = append(ids, s.ID)
 		}
 	case "filters":
-		f := config.FilterConfig{Type: config.FilterType(typ)}
+		p.Flow.Filters = append(p.Flow.Filters, config.FilterConfig{Type: config.FilterType(typ)})
-		config.Fill(&f)
-		p.Flow.Filters = append(p.Flow.Filters, f)
 		return Node{Role: role, Index: len(p.Flow.Filters) - 1}, nil
 	default:
 		return Node{}, fmt.Errorf("cannot add to %s", role)
@@ -392,7 +418,8 @@ func (c *Composition) MoveFilter(pi, from, to int) error {
 
 // Set gives a key at a dotted path its values: one, or several for a list. A
 // flow stage that is off turns on. A value of the wrong kind leaves the key as
-// it was; any other rule is Validate's.
+// it was, and a zero that takes the key's default unsets it; any other rule
+// is Validate's.
 func (c *Composition) Set(pi int, n Node, key string, values ...string) error {
 	return c.edit(pi, n, func(m map[string]any) error {
 		unset(m, strings.Split(key, "."))
@@ -435,7 +462,7 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er
 		if err := change(next); err != nil {
 			return err
 		}
-		if err := config.Coerce(n.Role, typ, next); err != nil {
+		if err := coerce(n.Role, typ, next); err != nil {
 			return err
 		}
 		*m = next
@@ -462,7 +489,8 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er
 }
 
 // editStage edits a stage's keys as a spec sets them; the table they scan into
-// replaces the stage, which turns on as a spec turns it on
+// replaces the stage, which turns on as a spec turns it on and holds the keys
+// set, a key at its zero unset
 func editStage(stage reflect.Value, key string, change func(map[string]any) error) error {
 	m := config.StageOn(key, nil)
 	if !stage.IsNil() {
@@ -475,7 +503,7 @@ func editStage(stage reflect.Value, key string, change func(map[string]any) erro
 	if err := config.Scan(m, next.Interface()); err != nil {
 		return err
 	}
-	config.Fill(next.Interface())
+	config.FillOn(key, next.Interface())
 	stage.Set(next)
 	return nil
 }
@@ -639,21 +667,40 @@ func (c *Composition) File() ([]byte, error) {
 }
 
 // Form is a way to write a composition, named for what it is, and to read it
-// back to the same pipelines
+// back to the same pipelines; looks matches text in its shape
 type Form struct {
 	Name  string                                           `json:"name"`
 	Hint  string                                           `json:"hint"`
 	Write func(*Composition) (string, error)               `json:"-"`
 	Read  func(string, config.IsDir) (*Composition, error) `json:"-"`
+	looks *regexp.Regexp
 }
 
 var forms = []Form{
-	{"command", "an lw command line", (*Composition).CommandLine, FromCommandLine},
+	{"command", "an lw command line", (*Composition).CommandLine, FromCommandLine,
-	{"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment},
+		regexp.MustCompile(`^\s*(\S*/)?lw(\s|$)|(^|\s)(--[a-z]|-p\s)`)},
+	{"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment,
+		regexp.MustCompile(`(?m)^\s*(export\s+)?LOGWISP_[A-Z0-9_]*=`)},
 	{"file", "the pipelines of a configuration file", func(c *Composition) (string, error) {
 		data, err := c.File()
 		return string(data), err
-	}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }},
+	}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) },
+		regexp.MustCompile(`(?m)^\s*(\[\[?pipelines\b|pipelines\s*=)`)},
+}
+
+// Parse reads text pasted as the form. Refused, text in another form's shape
+// and not in this one's is refused as that form, which reads it.
+func (f Form) Parse(text string, isDir config.IsDir) (*Composition, error) {
+	c, err := f.Read(text, isDir)
+	if err == nil || f.looks.MatchString(text) {
+		return c, err
+	}
+	for _, g := range forms {
+		if g.looks.MatchString(text) {
+			return nil, fmt.Errorf("this is %s, not %s: read it as the %s form", g.Hint, f.Hint, g.Name)
+		}
+	}
+	return nil, err
 }
 
 // Forms are what a composition writes and reads: lw --tui prints one, the
@@ -662,7 +709,8 @@ func Forms() []Form { return slices.Clone(forms) }
 
 // FromEnvironment starts from NAME='value' lines as an environment file holds
 // them, quoted as a POSIX shell reads them: '#' starts a comment line, a word
-// without '=', such as export, assigns nothing, and a later assignment wins
+// without '=', such as export, assigns nothing, and a later assignment wins.
+// LeftOut names the LOGWISP_ variables no pipeline flag has.
 func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) {
 	vars := map[string]string{}
 	for line := range strings.Lines(text) {
@@ -683,26 +731,28 @@ func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) {
 	for name, value := range vars {
 		environ = append(environ, name+"="+value)
 	}
-	pipelines, err := config.EnvPipelines(environ, isDir)
+	pipelines, unread, err := config.EnvPipelines(environ, isDir)
 	if err != nil {
 		return nil, err
 	}
-	c := &Composition{Pipelines: pipelines}
+	c := &Composition{Pipelines: pipelines, LeftOut: unread}
 	return c, c.settle()
 }
 
 // FromFile starts from a configuration file's pipelines, refusing a misspelled
-// key as lw does; its settings are the host's and its console sinks keep the
+// key as lw does; its settings are the host's, which LeftOut names, and its
-// color they name
+// console sinks keep the color they name
 func FromFile(data []byte) (*Composition, error) {
-	cfg, err := config.ParseFile(data)
+	cfg, settings, err := config.ParseFile(data)
 	if err != nil {
 		return nil, err
 	}
 	if len(cfg.Pipelines) == 0 {
 		return nil, errors.New("no [[pipelines]]")
 	}
-	return FromConfig(&config.Config{Pipelines: cfg.Pipelines})
+	c, err := FromConfig(&config.Config{Pipelines: cfg.Pipelines})
+	c.LeftOut = settings
+	return c, err
 }
 
 // FromJSON starts from pipelines as the website keeps them: a JSON list of

internal/compose/compose_test.go

@@ -206,8 +206,8 @@ func TestShellFormsRefuseControlCharacters(t *testing.T) {
 }
 
 // Set refuses a value of the wrong kind, a plugin's and a stage's alike, and
-// leaves the key as it was; Unset returns a key to its default, dropping a
+// leaves the key as it was; Unset leaves a key unset, dropping a table it
-// table it leaves empty
+// leaves empty
 func TestSetKeepsAKeyItCannotTake(t *testing.T) {
 	c := &Composition{}
 	if err := c.AddPipeline("p"); err != nil {
@@ -228,7 +228,7 @@ func TestSetKeepsAKeyItCannotTake(t *testing.T) {
 		}
 	}
 	if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) ||
-		beat.IntervalMS != 1000 {
+		beat.IntervalMS != 0 {
 		t.Fatalf("options %v, heartbeat %+v", m, beat)
 	}
 }
@@ -325,18 +325,67 @@ func TestPipelineNamesAreUnique(t *testing.T) {
 }
 
 // A stage an edit turns on does what its spec would: a rate limit drops what
-// passes its rate, a heartbeat beats
+// passes its rate, a heartbeat beats. A stage and a filter hold only the keys
+// set, so the forms carry no default the edit did not set.
 func TestAStageTurnsOnAsItsSpecDoes(t *testing.T) {
 	c := &Composition{}
 	if err := c.AddPipeline("p"); err != nil {
 		t.Fatal(err)
 	}
-	for _, err := range []error{c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500")} {
+	_, err := c.Add(0, "filters", "")
+	for _, err := range []error{err, c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500"),
+		c.Set(0, Node{Role: "filters"}, "patterns", "x")} {
 		if err != nil {
 			t.Fatal(err)
 		}
 	}
-	if f := c.Pipelines[0].Flow; f.RateLimit.Policy != "drop" || !f.Heartbeat.Enabled {
+	f := c.Pipelines[0].Flow
-		t.Fatalf("%+v %+v", f.RateLimit, f.Heartbeat)
+	if *f.RateLimit != (config.RateLimitConfig{EntriesPerSecond: 5, Policy: "drop"}) || *f.Heartbeat != (config.HeartbeatConfig{Enabled: true, IntervalMS: 500}) ||
+		!reflect.DeepEqual(f.Filters, []config.FilterConfig{{Patterns: []string{"x"}}}) {
+		t.Fatalf("%+v %+v %+v", f.RateLimit, f.Heartbeat, f.Filters)
+	}
+}
+
+// A zero where zero takes the key's default is unset, as lw reads it; where
+// zero means something of its own (retention_hours 0 keeps files), it stays
+func TestAZeroTakingTheDefaultIsUnset(t *testing.T) {
+	c, err := FromJSON([]byte(`[{"name":"p","plugin_sources":[{"type":"file","config":{"directory":"/x","check_interval_ms":0}}],
+		"plugin_sinks":[{"type":"file","config":{"directory":"/y","name":"n","retention_hours":0}}]}]`))
+	if err == nil {
+		err = c.Set(0, Node{Role: "sink", ID: "file"}, "buffer_size", "0")
+	}
+	if src, sink := c.Pipelines[0].PluginSources[0].Config, c.Pipelines[0].PluginSinks[0].Config; err != nil ||
+		!reflect.DeepEqual(src, map[string]any{"directory": "/x"}) || !reflect.DeepEqual(sink, map[string]any{"directory": "/y", "name": "n", "retention_hours": 0.0}) {
+		t.Fatalf("%v %v %v", src, sink, err)
+	}
+}
+
+// Text pasted as one form that has another's shape is refused as that form;
+// text in no form's shape, or in this one's, gets this form's own refusal.
+// The settings a file holds, and an environment's LOGWISP_ variables no
+// pipeline flag has, are left out by name.
+func TestAPasteIsReadAsItsForm(t *testing.T) {
+	form := map[string]Form{}
+	for _, f := range Forms() {
+		form[f.Name] = f
+	}
+	for text, want := range map[[2]string]string{
+		{"file", "lw --source null --sink null"}:         "this is an lw command line, not the pipelines of a configuration file: read it as the command form",
+		{"command", "LOGWISP_SOURCE='null'\n"}:           "this is LOGWISP_ variables for one pipeline, not an lw command line: read it as the environment form",
+		{"environment", "[[pipelines]]\nname = \"p\"\n"}: "this is the pipelines of a configuration file, not LOGWISP_ variables for one pipeline: read it as the file form",
+		{"command", "lw -c lw.toml"}:                     "-c: not a pipeline flag",
+		{"file", "hello world"}:                          "expected '=' after key",
+	} {
+		if _, err := form[text[0]].Parse(text[1], nil); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("%s %q: %v", text[0], text[1], err)
+		}
+	}
+	file, err := form["file"].Parse("quiet = true\n[logging]\nlevel = \"info\"\n[[pipelines]]\nname = \"p\"\n", nil)
+	if err != nil || !slices.Equal(file.LeftOut, []string{"logging.level", "quiet"}) {
+		t.Fatalf("file: %+v %v", file, err)
+	}
+	env, err := form["environment"].Parse("LOGWISP_SORCE_1='null'\nLOGWISP_QUIET=true\nLOGWISP_SINK=''\nLOGWISP_SOURCE_1=null\nHOME=/root\n", nil)
+	if err != nil || !slices.Equal(env.LeftOut, []string{"LOGWISP_QUIET", "LOGWISP_SORCE_1"}) {
+		t.Fatalf("environment: %+v %v", env, err)
 	}
 }

internal/config/config.go

@@ -69,28 +69,31 @@ type PipelineConfig struct {
 // --- Flow ---
 
 // FlowConfig holds the stages between sources and sinks, in the order entries
-// pass them; the heartbeat runs beside them
+// pass them; the heartbeat runs beside them. Help is each stage's summary.
 type FlowConfig struct {
-	RateLimit *RateLimitConfig `toml:"rate_limit"`
+	RateLimit *RateLimitConfig `toml:"rate_limit" help:"Drop entries beyond a rate, and those over a size"`
-	Filters   []FilterConfig   `toml:"filters,omitempty"`
+	Filters   []FilterConfig   `toml:"filters,omitempty" help:"Pass or drop entries by RE2 patterns, in order"`
-	Format    *FormatConfig    `toml:"format"`
+	Format    *FormatConfig    `toml:"format" help:"How entries are written: raw, txt or json"`
-	Heartbeat *HeartbeatConfig `toml:"heartbeat"`
+	Heartbeat *HeartbeatConfig `toml:"heartbeat" help:"A keep-alive entry on a timer, beside the entries"`
 }
 
 // --- Flow stages ---
+//
+// A stage key's zero is its unset, the default or what zero means there, so
+// a stage is written with the keys set alone.
 
 // HeartbeatConfig defines settings for periodic keep-alive or status messages
 type HeartbeatConfig struct {
-	Enabled          bool   `toml:"enabled" help:"send heartbeats; false keeps these settings and sends none"`
+	Enabled          bool   `toml:"enabled,omitempty" help:"send heartbeats; false keeps these settings and sends none"`
-	IntervalMS       int64  `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"`
+	IntervalMS       int64  `toml:"interval_ms,omitempty" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"`
-	IncludeTimestamp bool   `toml:"include_timestamp" help:"stamp each heartbeat with its time"`
+	IncludeTimestamp bool   `toml:"include_timestamp,omitempty" help:"stamp each heartbeat with its time"`
-	IncludeStats     bool   `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"`
+	IncludeStats     bool   `toml:"include_stats,omitempty" help:"add type, beat_count and, from the second beat, the measured interval_ms"`
-	Format           string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"kept for older files; heartbeats are written in the flow's format"`
+	Format           string `toml:"format,omitempty" default:"txt" lw:"enum=txt|json|raw" help:"kept for older files; heartbeats are written in the flow's format"`
 }
 
 // FormatConfig is how a pipeline writes its entries
 type FormatConfig struct {
-	Type            string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"raw writes each message as it came, txt a line with time and level, json one object an entry"`
+	Type            string `toml:"type,omitempty" default:"raw" lw:"enum=raw|txt|json" help:"raw writes each message as it came, txt a line with time and level, json one object an entry"`
 	Flags           int64  `toml:"flags,omitempty" lw:"zero=by type" help:"for experts, a sum of bits over the type's choice: 1 writes messages verbatim, 16 drops the time, 32 the level"`
 	TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"timestamp layout: the reference time 2006-01-02 15:04:05 written as timestamps should look, such as 02 Jan 15:04"`
 	SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"`
@@ -108,10 +111,10 @@ const (
 
 // RateLimitConfig defines the configuration for pipeline-level rate limiting
 type RateLimitConfig struct {
-	EntriesPerSecond  float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second let through; unset, the stage limits nothing, sizes included"`
+	EntriesPerSecond  float64 `toml:"entries_per_second,omitempty" lw:"min=0,zero=off" help:"entries a second let through; unset, the stage limits nothing, sizes included"`
-	BurstEntries      float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries that may pass in a burst before entries_per_second holds them back"`
+	BurstEntries      float64 `toml:"burst_entries,omitempty" lw:"min=0,zero=one second's worth" help:"entries that may pass in a burst before entries_per_second holds them back"`
-	Policy            string  `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop enforces entries_per_second and max_entry_size_bytes; pass lets every entry through"`
+	Policy            string  `toml:"policy,omitempty" default:"pass" lw:"enum=pass|drop" help:"drop enforces entries_per_second and max_entry_size_bytes; pass lets every entry through"`
-	MaxEntrySizeBytes int64   `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries; needs entries_per_second and policy drop"`
+	MaxEntrySizeBytes int64   `toml:"max_entry_size_bytes,omitempty" lw:"min=0,zero=unlimited" help:"drop larger entries; needs entries_per_second and policy drop"`
 }
 
 // FilterType represents the filter's behavior (include or exclude)
@@ -136,8 +139,8 @@ const (
 
 // FilterConfig represents the configuration for a single filter
 type FilterConfig struct {
-	Type     FilterType  `toml:"type" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"`
+	Type     FilterType  `toml:"type,omitempty" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"`
-	Logic    FilterLogic `toml:"logic" default:"or" lw:"enum=or|and" help:"match any pattern, or all"`
+	Logic    FilterLogic `toml:"logic,omitempty" default:"or" lw:"enum=or|and" help:"match any pattern, or all"`
 	Patterns []string    `toml:"patterns,omitempty" lw:"required,hint=regex" help:"RE2 patterns"`
 }
 

internal/config/schema.go

@@ -172,11 +172,12 @@ type Key struct {
 var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer",
 	reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"}
 
-// Stage is a flow stage: its key in the flow table, whether it is a list (the
+// Stage is a flow stage: its key in the flow table, what it does, whether it
-// filters), its options, and the values that turn it on, which a file's
+// is a list (the filters), its options, and the values that turn it on, which
-// defaults leave off
+// a file's defaults leave off
 type Stage struct {
 	Key     string         `json:"key"`
+	Summary string         `json:"summary"`
 	List    bool           `json:"list,omitempty"`
 	Keys    []Key          `json:"keys"`
 	On      map[string]any `json:"on,omitempty"`
@@ -188,7 +189,7 @@ func FlowStages() []Stage {
 	t := reflect.TypeFor[FlowConfig]()
 	for _, o := range options(t) {
 		ft := t.Field(o.index).Type
-		out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])})
+		out = append(out, Stage{o.Name, o.Help, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])})
 	}
 	return out
 }
@@ -232,8 +233,8 @@ func settingKeys(v reflect.Value, prefix string) []Key {
 
 // Decode reads a plugin's config map into its options: unknown keys, kinds,
 // the defaults, the tag rules and the options' own Check, then the host, tls,
-// acl and auth rules of its side; a leading ~ in a path is the home directory.
+// acl and auth rules of its side, its error joining every problem found; a
-// It reads no file.
+// leading ~ in a path is the home directory. It reads no file.
 func Decode[T any](role, typ string, m map[string]any) (*T, error) {
 	v, err := decode(role, typ, m)
 	if err != nil {
@@ -259,30 +260,27 @@ func decode(role, typ string, m map[string]any) (any, error) {
 	if err := lconfig.ScanMap(m, v.Interface()); err != nil {
 		return nil, err
 	}
-	if err := Settle(v.Interface()); err != nil {
+	errs := []error{Settle(v.Interface())}
-		return nil, err
-	}
 	expandPaths(v.Elem())
 	if n, ok := v.Interface().(networked); ok {
-		if err := p.checkNetwork(n); err != nil {
+		errs = append(errs, p.checkNetwork(n))
-			return nil, err
 	}
+	if err := errors.Join(errs...); err != nil {
+		return nil, err
 	}
 	return v.Interface(), nil
 }
 
-// Settle gives a decoded table its defaults, applies the tag rules, then its
+// Settle gives a decoded table its defaults, applies the tag rules and its own
-// own Check, as Decode does before its side's rules, for the flow stages a
+// Check, as Decode does before its side's rules, for the flow stages a file
-// file decodes typed
+// decodes typed; its error joins every problem
 func Settle(v any) error {
 	Fill(v)
-	if err := checkTags(reflect.ValueOf(v).Elem(), ""); err != nil {
+	err := checkTags(reflect.ValueOf(v).Elem(), "")
-		return err
-	}
 	if c, ok := v.(interface{ Check() error }); ok {
-		return c.Check()
+		err = errors.Join(err, c.Check())
 	}
-	return nil
+	return err
 }
 
 // Fill gives a table its defaults and checks nothing: Settle's first step
@@ -369,11 +367,12 @@ func setDefault(f reflect.Value, o option) {
 	}
 }
 
-// checkTags applies the tag rules to a table's own keys, naming each by
+// checkTags applies the tag rules to a table's own keys, joining a problem a
-// prefix and key. A required list needs an entry. A zero number is the
+// key, each named by prefix and key. A required list needs an entry. A zero
-// default or its zero meaning, and meets the range unless required; an empty
+// number is the default or its zero meaning, and meets the range unless
-// enum is one only by its zero or default.
+// required; an empty enum is one only by its zero or default.
 func checkTags(v reflect.Value, prefix string) error {
+	var errs []error
 	for _, o := range options(v.Type()) {
 		f := v.Field(o.index)
 		var err error
@@ -404,10 +403,10 @@ func checkTags(v reflect.Value, prefix string) error {
 			}
 		}
 		if err != nil {
-			return &KeyError{prefix + o.Name, err}
+			errs = append(errs, &KeyError{prefix + o.Name, err})
 		}
 	}
-	return nil
+	return errors.Join(errs...)
 }
 
 func num(n float64) string { return strconv.FormatFloat(n, 'f', -1, 64) }
@@ -425,7 +424,8 @@ func sideKeys(t reflect.Type, side string) []string {
 
 // Coerce gives command-line and environment values, which are strings, and
 // JSON's floats the kinds a plugin's options declare, so a dump prints them
-// typed. Keys and types it does not know are Decode's to refuse.
+// typed; its error joins a key each it cannot. Keys and types it does not
+// know are Decode's to refuse.
 func Coerce(role, typ string, m map[string]any) error {
 	p, ok := LookupPlugin(role, typ)
 	if !ok {
@@ -435,6 +435,7 @@ func Coerce(role, typ string, m map[string]any) error {
 }
 
 func coerce(t reflect.Type, m map[string]any, prefix string) error {
+	var errs []error
 	for _, o := range options(t) {
 		value, ok := m[o.Name]
 		if !ok {
@@ -445,18 +446,17 @@ func coerce(t reflect.Type, m map[string]any, prefix string) error {
 			ft = ft.Elem()
 		}
 		if sub, ok := value.(map[string]any); ok && ft.Kind() == reflect.Struct {
-			if err := coerce(ft, sub, prefix+o.Name+"."); err != nil {
+			errs = append(errs, coerce(ft, sub, prefix+o.Name+"."))
-				return err
-			}
 			continue
 		}
 		v, err := coerceValue(ft.Kind(), value)
 		if err != nil {
-			return &KeyError{prefix + o.Name, err}
+			errs = append(errs, &KeyError{prefix + o.Name, err})
+			continue
 		}
 		m[o.Name] = v
 	}
-	return nil
+	return errors.Join(errs...)
 }
 
 // coerceValue converts text, a JSON float for an integer and a TOML integer

internal/config/schema_test.go

@@ -93,7 +93,7 @@ func TestCoerceGivesTheDeclaredKinds(t *testing.T) {
 
 // ValidateConfig decodes every plugin and flow stage without opening a file,
 // and names the path of what it refuses: a rule between a table's keys by
-// the table's
+// the table's, a list entry by its index
 func TestValidateConfigNamesThePath(t *testing.T) {
 	valid := func() *Config {
 		return &Config{Color: "auto", Pipelines: []PipelineConfig{{
@@ -114,6 +114,9 @@ func TestValidateConfigNamesThePath(t *testing.T) {
 		"pipelines[0].plugin_sinks[out].config.tls: client_auth requires client_ca_file": func(c *Config) {
 			c.Pipelines[0].PluginSinks[0].Config["tls"].(map[string]any)["client_auth"] = true
 		},
+		`pipelines[0].plugin_sinks[out].config.acl.allow[1]: "10.0.0.0.1": not an IP address or CIDR; host names and IPv6 zones are not accepted`: func(c *Config) {
+			c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"allow": []any{"10.0.0.0/8", "10.0.0.0.1"}}
+		},
 		"pipelines[0].plugin_sinks[out].config.acl: requests_per_second_per_client applies only to HTTP listeners": func(c *Config) {
 			c.Pipelines[0].PluginSinks[0].Type = "tcp"
 			c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"requests_per_second_per_client": 2.5}
@@ -152,19 +155,25 @@ func TestValidateConfigNamesThePath(t *testing.T) {
 	}
 }
 
-// A configuration that does not load names every problem, a line each, each
+// A configuration that does not load names every problem, a line each: the
-// part's first: the settings, each flow stage, each plugin, each pipeline
+// settings, each flow stage, each plugin and each pipeline, every key's of a
+// part, and the first rule between a table's keys
 func TestLoadReportsEveryProblemALine(t *testing.T) {
 	isolateConfig(t)
-	_, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file",
+	_, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file:from=middle",
-		"sink", "tcp:port=70000", "rate-limit", "policy=bogus", "filter", "include:patterns=(",
+		"sink", "tcp:port=70000,buffer_size=-1,tls.enabled=true,acl.allow=x", "rate-limit", "policy=bogus", "filter", "include:patterns=(,logic=nor",
 		"pipeline", "a", "sink", "null")})
 	want := strings.Join([]string{"invalid configuration:",
 		`color: must be one of auto, always, never, got "bogus"`,
 		`pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "bogus"`,
+		`pipelines[0].flow.filters[0].logic: must be one of or, and, got "nor"`,
 		"pipelines[0].flow.filters[0].patterns[0]: \"(\": error parsing regexp: missing closing ): `(`",
 		"pipelines[0].plugin_sources[file].config.directory: must be set",
+		`pipelines[0].plugin_sources[file].config.from: must be one of start, end, got "middle"`,
 		"pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000",
+		"pipelines[0].plugin_sinks[tcp].config.buffer_size: must be at least 1, got -1",
+		`pipelines[0].plugin_sinks[tcp].config.acl.allow[0]: "x": not an IP address or CIDR; host names and IPv6 zones are not accepted`,
+		"pipelines[0].plugin_sinks[tcp].config.tls: listeners need cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file",
 		`pipelines[1].name: duplicate name "a"`,
 	}, "\n")
 	if err == nil || err.Error() != want {

internal/config/spec.go

@@ -48,8 +48,9 @@ var specKinds = []specKind{
 		p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
 		return cmp.Or(err, Coerce("source", typ, opts))
 	}, func(p *PipelineConfig) (specs []string) {
+		var ids []string
 		for _, s := range p.PluginSources {
-			specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
+			specs, ids = append(specs, formatSpec(s.Type, givenID(s.ID, s.Type, ids), s.Config)), append(ids, s.ID)
 		}
 		return specs
 	}},
@@ -62,8 +63,9 @@ var specKinds = []specKind{
 		p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
 		return cmp.Or(err, Coerce("sink", typ, opts))
 	}, func(p *PipelineConfig) (specs []string) {
+		var ids []string
 		for _, s := range p.PluginSinks {
-			specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
+			specs, ids = append(specs, formatSpec(s.Type, givenID(s.ID, s.Type, ids), s.Config)), append(ids, s.ID)
 		}
 		return specs
 	}},
@@ -75,7 +77,6 @@ var specKinds = []specKind{
 		return err
 	}, func(p *PipelineConfig) (specs []string) {
 		for _, f := range p.Flow.Filters {
-			Fill(&f)
 			specs = append(specs, typedSpec(&f))
 		}
 		return specs
@@ -89,7 +90,6 @@ var specKinds = []specKind{
 			return nil
 		}
 		format := *p.Flow.Format
-		Fill(&format)
 		return []string{typedSpec(&format)}
 	}},
 	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
@@ -100,8 +100,10 @@ var specKinds = []specKind{
 			return nil
 		}
 		rate := *p.Flow.RateLimit
-		Fill(&rate) // the policy, whose default here is drop
+		m := Values(&rate)
-		return []string{formatSpec("", Values(&rate))}
+		Fill(&rate)
+		m["policy"] = rate.Policy // a spec's default is drop, a file's pass
+		return []string{formatSpec("", m)}
 	}},
 	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
 		p.Flow.Heartbeat = &HeartbeatConfig{}
@@ -111,7 +113,6 @@ var specKinds = []specKind{
 			return nil
 		}
 		beat := *p.Flow.Heartbeat
-		Fill(&beat)
 		m := Values(&beat)
 		m["enabled"] = beat.Enabled
 		return []string{formatSpec("", m)}
@@ -137,6 +138,18 @@ func StageOn(stage string, opts map[string]any) map[string]any {
 	return on
 }
 
+// FillOn gives the keys a spec of the stage turns on their file defaults
+// where unset, as that spec writes them: a stage from a file and from its
+// spec are then one
+func FillOn(stage string, v any) {
+	rv := reflect.ValueOf(v).Elem()
+	for _, o := range options(rv.Type()) {
+		if _, on := stageOn[stage][o.Name]; on && o.Default != "" && rv.Field(o.index).IsZero() {
+			setDefault(rv.Field(o.index), o)
+		}
+	}
+}
+
 // Spec is one pipeline flag: its name without dashes, and its value.
 type Spec struct{ Flag, Value string }
 
@@ -219,13 +232,22 @@ func envPipelineSpecs(environ []string) []pipelineSpec {
 }
 
 // EnvPipelines builds the pipeline of LOGWISP_ variables, NAME=value each, as
-// Load does without pipeline flags
+// Load does without pipeline flags; unread are the other LOGWISP_ variables
-func EnvPipelines(environ []string, isDir IsDir) ([]PipelineConfig, error) {
+// set, sorted: settings, or names no flag has
+func EnvPipelines(environ []string, isDir IsDir) (pipelines []PipelineConfig, unread []string, err error) {
 	specs := envPipelineSpecs(environ)
-	if len(specs) == 0 {
+	for _, kv := range environ {
-		return nil, errors.New("no LOGWISP_ pipeline variables")
+		name, value, _ := strings.Cut(kv, "=")
+		if strings.HasPrefix(name, "LOGWISP_") && value != "" && !slices.ContainsFunc(specs, func(s pipelineSpec) bool { return s.name == name }) {
+			unread = append(unread, name)
 		}
-	return buildPipelines(specs, isDir)
+	}
+	slices.Sort(unread)
+	if len(specs) == 0 {
+		return nil, unread, errors.New("no LOGWISP_ pipeline variables")
+	}
+	pipelines, err = buildPipelines(specs, isDir)
+	return pipelines, unread, err
 }
 
 // SpecPipelines builds the pipelines of command-line specs, as Load does
@@ -280,8 +302,9 @@ func buildPipelines(specs []pipelineSpec, isDir IsDir) ([]PipelineConfig, error)
 }
 
 // PipelineSpecs writes pipelines as the flags that build them: --pipeline,
-// ids, flow stage defaults and heartbeat's enabled explicit, as some flags'
+// the ids and keys set, and a typed stage's type, the rate limit's policy and
-// defaults differ from a file's
+// the heartbeat's enabled explicit, as those flags' defaults differ from a
+// file's
 func PipelineSpecs(pipelines []PipelineConfig) []Spec {
 	var specs []Spec
 	for _, p := range pipelines {
@@ -298,14 +321,30 @@ func PipelineSpecs(pipelines []PipelineConfig) []Spec {
 	return specs
 }
 
-// typedSpec writes a stage whose type key is its spec's TYPE
+// typedSpec writes a stage whose type key is its spec's TYPE: the type set,
+// or its default
 func typedSpec(stage any) string {
 	m := Values(stage)
+	if _, set := m["type"]; !set {
+		filled := reflect.New(reflect.TypeOf(stage).Elem())
+		filled.Elem().Set(reflect.ValueOf(stage).Elem())
+		Fill(filled.Interface())
+		m["type"] = Values(filled.Interface())["type"]
+	}
 	typ, _ := m["type"].(string)
 	delete(m, "type")
 	return formatSpec(typ, m)
 }
 
+// givenID is an instance's id as its spec gives it: none for the one
+// instanceID would take, after the instances before
+func givenID(id, typ string, before []string) map[string]any {
+	if id == FreeID(typ, before) {
+		return nil
+	}
+	return map[string]any{"id": id}
+}
+
 // formatSpec writes TYPE[:key=value,...] or key=value,...: tables as dotted
 // keys, lists as a repeated key
 func formatSpec(typ string, tables ...map[string]any) string {

internal/config/spec_test.go

@@ -216,16 +216,21 @@ func TestReloadKeepsSpecPipelines(t *testing.T) {
 	}
 }
 
-// Written specs build the pipelines a file would: each flow stage with its
+// Written specs build the pipelines a file would, with the keys it set: a
-// defaults, so the rate limit keeps its file policy and the heartbeat its off
+// typed stage names its type, the rate limit keeps its file policy and the
+// heartbeat its off, and an id is written only where a spec would not take it
 func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) {
 	p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5}, Heartbeat: &HeartbeatConfig{}},
 		PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}},
-		PluginSinks:   []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}}
+		PluginSinks:   []PluginSinkConfig{{ID: "null_2", Type: "null", Config: map[string]any{}}, {ID: "null", Type: "null", Config: map[string]any{}}}}
-	got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil)
+	specs := PipelineSpecs([]PipelineConfig{p})
-	p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"},
+	got, err := SpecPipelines(specs, nil)
-		Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}}
+	p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"}, Heartbeat: &HeartbeatConfig{}}
 	if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) {
 		t.Fatalf("got %+v %v\nwant %+v", got, err, p)
 	}
+	if want := []Spec{{"pipeline", "p"}, {"source", "null:id=in"}, {"sink", "null:id=null_2"}, {"sink", "null"}, {"format", "raw"},
+		{"rate-limit", "entries_per_second=5,policy=pass"}, {"heartbeat", "enabled=false"}}; !reflect.DeepEqual(specs, want) {
+		t.Fatalf("specs %v, want %v", specs, want)
+	}
 }

internal/config/validate.go

@@ -24,9 +24,9 @@ import (
 
 // ValidateConfig checks a configuration without reading a file it names:
 // the settings, then each pipeline's flow stages, and its sources and sinks
-// decoded in full. Its error joins every problem, one a part, each a
+// decoded in full. Its error joins every problem, each a *KeyError naming the
-// *KeyError naming the refused key's path, save "no pipelines configured".
+// refused key's path, save "no pipelines configured". lw --check opens the
-// lw --check opens the files.
+// files.
 func ValidateConfig(cfg *Config) error {
 	if cfg == nil {
 		return fmt.Errorf("config is nil")
@@ -99,8 +99,8 @@ func ValidatePipelines(pipelines []PipelineConfig) error {
 	return errors.Join(errs...)
 }
 
-// checkFlow settles copies of a pipeline's flow stages, which keep their
+// checkFlow settles copies of a pipeline's flow stages, which keep the keys
-// defaults for the dump: a problem a stage
+// set for the dump
 func checkFlow(f *FlowConfig, path string) error {
 	if f == nil {
 		return nil
@@ -126,8 +126,16 @@ func checkFlow(f *FlowConfig, path string) error {
 	return errors.Join(errs...)
 }
 
-// At addresses err to the path of the table it was found in
+// At addresses err to the path of the table it was found in, each problem of
+// a joined one
 func At(path string, err error) error {
+	if joined, ok := err.(interface{ Unwrap() []error }); ok {
+		var errs []error
+		for _, e := range joined.Unwrap() {
+			errs = append(errs, At(path, e))
+		}
+		return errors.Join(errs...)
+	}
 	if ke, ok := errors.AsType[*KeyError](err); ok {
 		return &KeyError{path + "." + ke.Key, ke.Err}
 	}
@@ -167,19 +175,17 @@ func (o *HTTPChainSinkOptions) network() (string, *TLSOptions, *AuthOptions, *AC
 	return o.Host, o.TLS, o.Auth, nil
 }
 
-// checkNetwork applies the host, tls, acl and auth rules of the plugin's side
+// checkNetwork applies the host, tls, acl and auth rules of the plugin's
+// side, joining their problems; acl entries are read by the host's family
 func (p Plugin) checkNetwork(n networked) error {
 	host, t, a, acl := n.network()
+	var errs []error
 	if _, err := core.Network(host); err != nil {
-		return &KeyError{"host", err}
+		errs = append(errs, &KeyError{"host", err})
+	} else {
+		errs = append(errs, table("acl", acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0)))
 	}
-	if err := t.Check(p.Side != Dialer); err != nil {
+	return errors.Join(append(errs, table("tls", t.Check(p.Side != Dialer)), table("auth", a.Check(p.Side, p.HTTP, t.State())))...)
-		return table("tls", err)
-	}
-	if err := acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0); err != nil {
-		return table("acl", err)
-	}
-	return table("auth", a.Check(p.Side, p.HTTP, t.State()))
 }
 
 // Check holds the rules between tls keys on a listener or a dialer; a table
@@ -221,8 +227,9 @@ func (o *TLSOptions) Check(listener bool) error {
 		return fmt.Errorf("tls: cert_file and key_file must be set together")
 	}
 	if o.PinSHA256 != "" {
-		_, err := ParsePins(o.PinSHA256)
+		if _, err := ParsePins(o.PinSHA256); err != nil {
-		return err
+			return &KeyError{"tls.pin_sha256", err}
+		}
 	}
 	return nil
 }
@@ -237,7 +244,7 @@ func ParsePins(s string) ([][]byte, error) {
 		b64, ok := strings.CutPrefix(strings.TrimSpace(p), PinPrefix)
 		h, err := base64.StdEncoding.DecodeString(b64)
 		if !ok || err != nil || len(h) != sha256.Size {
-			return nil, fmt.Errorf("tls: pin_sha256 %q: want %sBASE64 of a SHA-256, ';' between several", p, PinPrefix)
+			return nil, fmt.Errorf("pin %q: want %sBASE64 of a SHA-256, ';' between several", p, PinPrefix)
 		}
 		pins = append(pins, h)
 	}
@@ -290,7 +297,7 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
 	case o.Type == "mtls":
 		for i, pattern := range o.AllowPatterns {
 			if _, err := regexp.Compile(pattern); err != nil {
-				return fmt.Errorf("auth: allow_patterns[%d] %q: %w", i, pattern, err)
+				return &KeyError{fmt.Sprintf("auth.allow_patterns[%d]", i), fmt.Errorf("%q: %w", pattern, err)}
 			}
 		}
 		return nil
@@ -320,9 +327,9 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
 		// Binds the certificate to the user: a peer needs its own of both
 		return fmt.Errorf("auth: identity under type %q binds the client certificate to the user and requires tls.client_auth", "scram")
 	}
-	for _, e := range o.TrustedProxies {
+	for i, e := range o.TrustedProxies {
 		if _, err := ParsePrefix(e, "tcp"); err != nil {
-			return fmt.Errorf("auth: trusted_proxies entry %q: %w", e, err)
+			return &KeyError{fmt.Sprintf("auth.trusted_proxies[%d]", i), fmt.Errorf("%q: %w", e, err)}
 		}
 	}
 	return nil
@@ -357,7 +364,8 @@ func (o *AuthOptions) scramKey() string {
 
 // Check holds the acl rules of a listener on host, http for one serving
 // requests, proxied for one behind auth.trusted_proxies, whose rules take
-// clients of either family, as does a PROXY header
+// clients of either family, as does a PROXY header; an entry's problem is
+// at its key
 func (o *ACLOptions) Check(host string, http, proxied bool) error {
 	if o == nil {
 		return nil
@@ -384,17 +392,18 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error {
 	if header || proxied {
 		clients = "tcp"
 	}
+	var errs []error
 	for _, list := range []struct {
 		key, network string
 		entries      []string
 	}{{"proxy_from", network, o.ProxyFrom}, {"allow", clients, o.Allow}, {"deny", clients, o.Deny}} {
-		for _, e := range list.entries {
+		for i, e := range list.entries {
 			if _, err := ParsePrefix(e, list.network); err != nil {
-				return fmt.Errorf("acl: %s entry %q: %w", list.key, e, err)
+				errs = append(errs, &KeyError{fmt.Sprintf("acl.%s[%d]", list.key, i), fmt.Errorf("%q: %w", e, err)})
 			}
 		}
 	}
-	return nil
+	return errors.Join(errs...)
 }
 
 // httpKey names the first key of table v tagged http that is set
@@ -537,8 +546,8 @@ func Scan(configMap map[string]any, target any) error {
 	return lconfig.ScanMap(configMap, target)
 }
 
-// checkKeys walks nested tables against the toml tags of t, refusing unknown
+// checkKeys walks nested tables against the toml tags of t, refusing every
-// keys. Map-typed fields hold free-form keys and are not descended into.
+// unknown key. Map-typed fields hold free-form keys and are not descended into.
 func checkKeys(m map[string]any, t reflect.Type, prefix string) error {
 	for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice {
 		t = t.Elem()
@@ -552,17 +561,17 @@ func checkKeys(m map[string]any, t reflect.Type, prefix string) error {
 			fields[name] = f.Type
 		}
 	}
+	var errs []error
 	for _, key := range slices.Sorted(maps.Keys(m)) {
 		ft, ok := fields[key]
 		if !ok {
-			return fmt.Errorf("unknown key %q", prefix+key)
+			errs = append(errs, fmt.Errorf("unknown key %q", prefix+key))
+			continue
 		}
 		var tables []map[string]any
 		switch v := m[key].(type) {
 		case map[string]any:
-			if err := checkKeys(v, ft, prefix+key+"."); err != nil {
+			errs = append(errs, checkKeys(v, ft, prefix+key+"."))
-				return err
-			}
 		case []map[string]any:
 			tables = v
 		case []any:
@@ -573,12 +582,10 @@ func checkKeys(m map[string]any, t reflect.Type, prefix string) error {
 			}
 		}
 		for i, table := range tables {
-			if err := checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil {
+			errs = append(errs, checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)))
-				return err
 		}
 	}
-	}
+	return errors.Join(errs...)
-	return nil
 }
 
 // checkFileKeys rejects keys the configuration file declares that Config does
@@ -600,14 +607,27 @@ func checkFileKeys(path string) error {
 }
 
 // ParseFile reads a configuration file's text, refusing the keys lw refuses
-// there, and opens nothing it names
+// there, and opens nothing it names; settings are the paths it sets outside
-func ParseFile(data []byte) (*Config, error) {
+// the pipelines, sorted
+func ParseFile(data []byte) (cfg *Config, settings []string, err error) {
 	root, err := fileTable(data)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	cfg := &Config{}
+	var walk func(m map[string]any, prefix string)
-	return cfg, Scan(root, cfg)
+	walk = func(m map[string]any, prefix string) {
+		for k, v := range m {
+			if sub, ok := v.(map[string]any); ok {
+				walk(sub, prefix+k+".")
+			} else if prefix != "" || k != "pipelines" {
+				settings = append(settings, prefix+k)
+			}
+		}
+	}
+	walk(root, "")
+	slices.Sort(settings)
+	cfg = &Config{}
+	return cfg, settings, Scan(root, cfg)
 }
 
 // fileTable is a file's root table without the keys lw ignores there

internal/tui/canvas.go

@@ -338,7 +338,7 @@ func (a *app) summary(n compose.Node) string {
 	if n.Role == "filters" {
 		var out []string
 		for _, f := range a.pipeline().Flow.Filters {
-			sign := map[config.FilterType]string{"include": "+", "exclude": "-"}[f.Type]
+			sign := map[config.FilterType]string{"include": "+", "exclude": "-"}[filled(f).Type]
 			out = append(out, sign+strings.Join(f.Patterns, "|"))
 		}
 		return strings.Join(out, " ")

internal/tui/inspector.go

@@ -52,7 +52,7 @@ func (a *app) lines() []line {
 	filters := a.pipeline().Flow.Filters
 	for i, f := range filters {
 		fn := compose.Node{Role: "filters", Index: i}
-		out = append(out, line{node: fn, group: fmt.Sprintf("%d %s", i+1, f.Type), item: -1})
+		out = append(out, line{node: fn, group: fmt.Sprintf("%d %s", i+1, filled(f).Type), item: -1})
 		if a.insp.open[a.foldKey(fn, "")] {
 			out = a.keyLines(out, fn, a.keys(n), "", 1)
 		}
@@ -60,6 +60,12 @@ func (a *app) lines() []line {
 	return append(out, line{node: compose.Node{Role: "filters", Index: len(filters)}, item: -1, add: true})
 }
 
+// filled is a filter as it runs: an unset type or logic its default
+func filled(f config.FilterConfig) config.FilterConfig {
+	config.Fill(&f)
+	return f
+}
+
 func (a *app) keyLines(out []line, n compose.Node, keys []config.Key, prefix string, depth int) []line {
 	for _, k := range keys {
 		name := k.Name
@@ -285,7 +291,7 @@ func choice(k config.Key, cur any) int {
 // keys, a filter's logic and patterns
 func (a *app) groupSummary(l line) string {
 	if l.key.Name == "" {
-		f := a.pipeline().Flow.Filters[l.node.Index]
+		f := filled(a.pipeline().Flow.Filters[l.node.Index])
 		return string(f.Logic) + "  " + strings.Join(f.Patterns, " ")
 	}
 	switch v := a.value(l.node, l.key.Name).(type) {

internal/tui/view.go

@@ -761,7 +761,7 @@ func (a *app) verdict(w *row) string {
 		text := fmt.Sprintf("dropped by filter %d", w.index+1)
 		if pi := a.pipeIndex(w.pipe); pi >= 0 {
 			if fs := a.comp.Pipelines[pi].Flow.Filters; w.index < len(fs) {
-				text += " (" + string(fs[w.index].Type) + " " + strings.Join(fs[w.index].Patterns, " | ") + ")"
+				text += " (" + string(filled(fs[w.index]).Type) + " " + strings.Join(fs[w.index].Patterns, " | ") + ")"
 			}
 		}
 		return text