From 6a168212de4c901bff13fac99d28990302edf5494aa6171577e2df4a7a03099b Mon Sep 17 00:00:00 2001 From: Lixen <41302669+lixenwraith@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:34:20 -0400 Subject: [PATCH] v0.38.0 lwconf updates for deployment lwconf: sparse stages, every problem of a part, pastes read by form --- cmd/lwconf/lwconf.test.mjs | 4 +- cmd/lwconf/main.go | 7 +- doc/architecture.md | 15 ++-- doc/cli.md | 11 +-- doc/configuration.md | 6 +- doc/lw.1 | 8 +-- internal/compose/compose.go | 114 ++++++++++++++++++++++--------- internal/compose/compose_test.go | 63 +++++++++++++++-- internal/config/config.go | 37 +++++----- internal/config/schema.go | 72 +++++++++---------- internal/config/schema_test.go | 19 ++++-- internal/config/spec.go | 69 +++++++++++++++---- internal/config/spec_test.go | 17 +++-- internal/config/validate.go | 100 ++++++++++++++++----------- internal/tui/canvas.go | 2 +- internal/tui/inspector.go | 10 ++- internal/tui/view.go | 2 +- 17 files changed, 376 insertions(+), 180 deletions(-) diff --git a/cmd/lwconf/lwconf.test.mjs b/cmd/lwconf/lwconf.test.mjs index 460b0c2..4e176d5 100644 --- a/cmd/lwconf/lwconf.test.mjs +++ b/cmd/lwconf/lwconf.test.mjs @@ -123,9 +123,9 @@ test("a preset's pipeline, a single file answered by dirs, reads back from every const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs); assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" }); assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]); - assert.deepEqual(engine.parse("lw -p serve:path=/var/log/app.log,allow=192.0.2.0/24,allow=198.51.100.7", "command", dirs).value, pipelines); + assert.deepEqual(engine.parse("lw -p serve:path=/var/log/app.log,allow=192.0.2.0/24,allow=198.51.100.7", "command", dirs).value.pipelines, pipelines); for (const { name } of schema.forms) { - assert.deepEqual(engine.parse(engine.emit(pipelines, name).value, name).value, pipelines, name); + assert.deepEqual(engine.parse(engine.emit(pipelines, name).value, name).value, { pipelines, left_out: [] }, name); } assert.deepEqual(engine.validate(pipelines).value, { pipelines, problems: [] }); }); diff --git a/cmd/lwconf/main.go b/cmd/lwconf/main.go index 12c140d..5329f42 100644 --- a/cmd/lwconf/main.go +++ b/cmd/lwconf/main.go @@ -47,7 +47,12 @@ var calls = map[string]struct { if err != nil { return nil, err } - return pipelines(f.Read(a[0], isDir)) + c, err := f.Parse(a[0], isDir) + value, err := pipelines(c, err) + if err != nil { + return nil, err + } + return map[string]any{"pipelines": value, "left_out": append([]string{}, c.LeftOut...)}, nil }}, "validate": {[]string{"pipelines"}, func(a []string) (any, error) { c, err := compose.FromJSON([]byte(a[0])) diff --git a/doc/architecture.md b/doc/architecture.md index 7b45644..8adba91 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -278,10 +278,14 @@ answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key", - `schema()`: what `lw --schema` prints. - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to a string, a list key to strings. -- `parse(text, form, dirs)`: the pipelines of a form's pasted text. +- `parse(text, form, dirs)`: `{pipelines, left_out}`, the pipelines of a + form's pasted text and the names of what it held besides them: a file's + settings, an environment's `LOGWISP_` variables no pipeline flag has. Text + in another form's shape is refused naming that form. - `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled - (defaults, kinds, ids) and the first problem of each part (a value of the - wrong kind too) as `{path, pipeline, node, key, text}`. + (kinds, ids) and every problem of each part, each key's and the first rule + between a table's keys (a value of the wrong kind too), as `{path, + pipeline, node, key, text}`. - `emit(pipelines, form)`: the text of `command`, `environment` or `file`. - `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is `{op, pipeline, ...}`: @@ -298,7 +302,10 @@ answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key", `{role, id, index}`: `source` or `sink` by its id, `filters` by its index, or the flow stage `rate_limit`, `format` or `heartbeat`, which `set` turns on and `remove` off; `add` takes `source`, `sink`, or `filters` with the - type `include` or `exclude`. + type `include`, `exclude` or `""` for the default. +- Stages and options hold the keys set, so the forms carry no default an + edit did not set: a stage key's zero is unset, and an option's zero is + unset where it takes the key's default. ## Resource Management diff --git a/doc/cli.md b/doc/cli.md index 76e5cbe..e784ae6 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -80,7 +80,7 @@ pipelines have [their own flags](#pipelines): speaks HTTP: `auth.node_binding` only on the chain sources, and `auth.trusted_proxies` on both HTTP listeners, though only the `http` sink admits it - - a flow stage: its keys, and `on`, the values that turn it on + - a flow stage: its keys, a `summary`, and `on`, the values that turn it on - a preset key: its default and help, and the choices (`enum`) and `hint` of the option it sets - `forms`: `command`, `environment` and `file`, each with its hint @@ -419,10 +419,11 @@ lw -c dev.toml --logging.level=debug `lw --check` validates without running: it builds every pipeline and plugin as a start would (options, TLS files, credentials files, startup warnings), but binds no port, reads no stdin, and opens, reads or creates no log file; it -prints `configuration ok: N pipeline(s)` and exits 0. Otherwise it prints the -first problem of each setting, flow stage, source, sink and pipeline, a line -each named by its key's path, or else the first file that fails to open, and -exits 1 (2 when a named file is missing, see [Exit Codes](#exit-codes)): +prints `configuration ok: N pipeline(s)` and exits 0. Otherwise it prints +every key's problem of each setting, flow stage, source, sink and pipeline, +and the first broken rule between a table's keys, a line each named by its +key's path, or else the first file that fails to open, and exits 1 (2 when a +named file is missing, see [Exit Codes](#exit-codes)): ```bash lw --check -c /etc/logwisp/logwisp.toml diff --git a/doc/configuration.md b/doc/configuration.md index ce35d4d..1dca6a0 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -192,8 +192,10 @@ and variables replace the file's whole, and an array-indexed path such as ## Validation The configuration is checked offline when it loads, at every reload and by -`lw --check`. Each check reports the first problem of each table (the top -level, `logging`, each flow stage, each source and sink) by its key's path: +`lw --check`. Each check reports every key's problem and the first broken +rule between a table's keys, in every table (the top level, `logging`, each +flow stage, each source and sink), by its key's path, a list entry's by its +index (`acl.allow[1]`): ``` Error: Failed to load config: invalid configuration: diff --git a/doc/lw.1 b/doc/lw.1 index c8be856..ffd791a 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -145,10 +145,10 @@ and credentials files and printing startup warnings, without binding a port or opening, reading or creating a log file; print .Dq configuration ok and exit 0. -Otherwise print the first problem of each setting, flow stage, source, sink -and pipeline, a line each named by its key's path, or else the first file -that fails to open, and exit 1, or 2 when a named configuration file is -missing +Otherwise print every key's problem of each setting, flow stage, source, +sink and pipeline, and the first broken rule between a table's keys, a line +each named by its key's path, or else the first file that fails to open, and +exit 1, or 2 when a named configuration file is missing .Pq see Sx EXIT STATUS . .It Fl \-dump Print the effective configuration, with the options, environment and diff --git a/internal/compose/compose.go b/internal/compose/compose.go index 12e8401..a3ac7d8 100644 --- a/internal/compose/compose.go +++ b/internal/compose/compose.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "reflect" + "regexp" "slices" "strconv" "strings" @@ -22,11 +23,13 @@ import ( "github.com/lixenwraith/toml" ) -// Composition is the pipelines being composed. Its flow stages hold their -// defaults and its plugin options the kinds they declare, so each form it -// writes loads back to the same pipelines. +// Composition is the pipelines being composed. Its flow stages and plugin +// options hold the keys set, the options in the kinds they declare, so each +// form it writes loads back to the same pipelines. LeftOut is what a pasted +// form held besides pipelines: a file's settings, other LOGWISP_ variables. type Composition struct { Pipelines []config.PipelineConfig + LeftOut []string } // Node is a part of a pipeline: a source or sink by its id, or a flow stage @@ -112,9 +115,10 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) { return c, c.settle() } -// settle gives every flow stage its defaults, every plugin's options their -// kinds and every source and sink an id; the rules are Validate's. Its error -// joins a plugin's each holding a value of the wrong kind, kept as given. +// settle gives every stage the values its spec writes out (FillOn), every +// plugin's options their kinds, unsetting a zero that takes its key's +// default, and every source and sink an id; the rules are Validate's. Its +// error joins a plugin's each holding a value of the wrong kind, kept as given. func (c *Composition) settle() error { var errs []error for i := range c.Pipelines { @@ -126,23 +130,20 @@ func (c *Composition) settle() error { func(j int) string { return p.PluginSources[j].Type }) name(len(p.PluginSinks), func(j int) *string { return &p.PluginSinks[j].ID }, func(j int) string { return p.PluginSinks[j].Type }) - for _, stage := range stages(p.Flow) { + for key, stage := range stages(p.Flow) { if !stage.IsNil() { - config.Fill(stage.Interface()) + config.FillOn(key, stage.Interface()) } } - for j := range p.Flow.Filters { - config.Fill(&p.Flow.Filters[j]) - } for j, s := range p.PluginSources { p.PluginSources[j].Config = orEmpty(s.Config) - if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil { + if err := coerce("source", s.Type, p.PluginSources[j].Config); err != nil { errs = append(errs, config.At(Node{Role: "source", ID: s.ID}.Path(i), err)) } } for j, s := range p.PluginSinks { p.PluginSinks[j].Config = orEmpty(s.Config) - if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil { + if err := coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil { errs = append(errs, config.At(Node{Role: "sink", ID: s.ID}.Path(i), err)) } } @@ -150,6 +151,32 @@ func (c *Composition) settle() error { return errors.Join(errs...) } +// coerce gives a plugin's options their kinds, then unsets a number at zero +// where zero takes the key's default: lw reads it so, and the options then +// show the default rather than a 0 that means it +func coerce(role, typ string, m map[string]any) error { + err := config.Coerce(role, typ, m) + if p, ok := config.LookupPlugin(role, typ); ok { + unsetDefaults(p.Keys(), m) + } + return err +} + +func unsetDefaults(keys []config.Key, m map[string]any) { + for _, k := range keys { + switch v := m[k.Name]; v { + case int64(0), 0.0: + if k.Default != "" && k.Zero == "" { + delete(m, k.Name) + } + default: + if sub, ok := v.(map[string]any); ok { + unsetDefaults(k.Keys, sub) + } + } + } +} + // name gives the parts a file left unnamed the ids their specs would get: the // shell forms write no empty id func name(n int, id func(int) *string, typ func(int) string) { @@ -236,7 +263,8 @@ func (c *Composition) RemovePipeline(i int) error { } // Add appends a source or sink of a catalogue type under config.FreeID, or a -// filter of type include or exclude, and returns its node +// filter of type include or exclude, or of the default for "", and returns +// its node func (c *Composition) Add(pi int, role, typ string) (Node, error) { p, err := c.pipeline(pi) if err != nil { @@ -253,9 +281,7 @@ func (c *Composition) Add(pi int, role, typ string) (Node, error) { ids = append(ids, s.ID) } case "filters": - f := config.FilterConfig{Type: config.FilterType(typ)} - config.Fill(&f) - p.Flow.Filters = append(p.Flow.Filters, f) + p.Flow.Filters = append(p.Flow.Filters, config.FilterConfig{Type: config.FilterType(typ)}) return Node{Role: role, Index: len(p.Flow.Filters) - 1}, nil default: return Node{}, fmt.Errorf("cannot add to %s", role) @@ -392,7 +418,8 @@ func (c *Composition) MoveFilter(pi, from, to int) error { // Set gives a key at a dotted path its values: one, or several for a list. A // flow stage that is off turns on. A value of the wrong kind leaves the key as -// it was; any other rule is Validate's. +// it was, and a zero that takes the key's default unsets it; any other rule +// is Validate's. func (c *Composition) Set(pi int, n Node, key string, values ...string) error { return c.edit(pi, n, func(m map[string]any) error { unset(m, strings.Split(key, ".")) @@ -435,7 +462,7 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er if err := change(next); err != nil { return err } - if err := config.Coerce(n.Role, typ, next); err != nil { + if err := coerce(n.Role, typ, next); err != nil { return err } *m = next @@ -462,7 +489,8 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er } // editStage edits a stage's keys as a spec sets them; the table they scan into -// replaces the stage, which turns on as a spec turns it on +// replaces the stage, which turns on as a spec turns it on and holds the keys +// set, a key at its zero unset func editStage(stage reflect.Value, key string, change func(map[string]any) error) error { m := config.StageOn(key, nil) if !stage.IsNil() { @@ -475,7 +503,7 @@ func editStage(stage reflect.Value, key string, change func(map[string]any) erro if err := config.Scan(m, next.Interface()); err != nil { return err } - config.Fill(next.Interface()) + config.FillOn(key, next.Interface()) stage.Set(next) return nil } @@ -639,21 +667,40 @@ func (c *Composition) File() ([]byte, error) { } // Form is a way to write a composition, named for what it is, and to read it -// back to the same pipelines +// back to the same pipelines; looks matches text in its shape type Form struct { Name string `json:"name"` Hint string `json:"hint"` Write func(*Composition) (string, error) `json:"-"` Read func(string, config.IsDir) (*Composition, error) `json:"-"` + looks *regexp.Regexp } var forms = []Form{ - {"command", "an lw command line", (*Composition).CommandLine, FromCommandLine}, - {"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment}, + {"command", "an lw command line", (*Composition).CommandLine, FromCommandLine, + regexp.MustCompile(`^\s*(\S*/)?lw(\s|$)|(^|\s)(--[a-z]|-p\s)`)}, + {"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment, + regexp.MustCompile(`(?m)^\s*(export\s+)?LOGWISP_[A-Z0-9_]*=`)}, {"file", "the pipelines of a configuration file", func(c *Composition) (string, error) { data, err := c.File() return string(data), err - }, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }}, + }, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }, + regexp.MustCompile(`(?m)^\s*(\[\[?pipelines\b|pipelines\s*=)`)}, +} + +// Parse reads text pasted as the form. Refused, text in another form's shape +// and not in this one's is refused as that form, which reads it. +func (f Form) Parse(text string, isDir config.IsDir) (*Composition, error) { + c, err := f.Read(text, isDir) + if err == nil || f.looks.MatchString(text) { + return c, err + } + for _, g := range forms { + if g.looks.MatchString(text) { + return nil, fmt.Errorf("this is %s, not %s: read it as the %s form", g.Hint, f.Hint, g.Name) + } + } + return nil, err } // Forms are what a composition writes and reads: lw --tui prints one, the @@ -662,7 +709,8 @@ func Forms() []Form { return slices.Clone(forms) } // FromEnvironment starts from NAME='value' lines as an environment file holds // them, quoted as a POSIX shell reads them: '#' starts a comment line, a word -// without '=', such as export, assigns nothing, and a later assignment wins +// without '=', such as export, assigns nothing, and a later assignment wins. +// LeftOut names the LOGWISP_ variables no pipeline flag has. func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) { vars := map[string]string{} for line := range strings.Lines(text) { @@ -683,26 +731,28 @@ func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) { for name, value := range vars { environ = append(environ, name+"="+value) } - pipelines, err := config.EnvPipelines(environ, isDir) + pipelines, unread, err := config.EnvPipelines(environ, isDir) if err != nil { return nil, err } - c := &Composition{Pipelines: pipelines} + c := &Composition{Pipelines: pipelines, LeftOut: unread} return c, c.settle() } // FromFile starts from a configuration file's pipelines, refusing a misspelled -// key as lw does; its settings are the host's and its console sinks keep the -// color they name +// key as lw does; its settings are the host's, which LeftOut names, and its +// console sinks keep the color they name func FromFile(data []byte) (*Composition, error) { - cfg, err := config.ParseFile(data) + cfg, settings, err := config.ParseFile(data) if err != nil { return nil, err } if len(cfg.Pipelines) == 0 { return nil, errors.New("no [[pipelines]]") } - return FromConfig(&config.Config{Pipelines: cfg.Pipelines}) + c, err := FromConfig(&config.Config{Pipelines: cfg.Pipelines}) + c.LeftOut = settings + return c, err } // FromJSON starts from pipelines as the website keeps them: a JSON list of diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go index f5e5d9e..282285c 100644 --- a/internal/compose/compose_test.go +++ b/internal/compose/compose_test.go @@ -206,8 +206,8 @@ func TestShellFormsRefuseControlCharacters(t *testing.T) { } // Set refuses a value of the wrong kind, a plugin's and a stage's alike, and -// leaves the key as it was; Unset returns a key to its default, dropping a -// table it leaves empty +// leaves the key as it was; Unset leaves a key unset, dropping a table it +// leaves empty func TestSetKeepsAKeyItCannotTake(t *testing.T) { c := &Composition{} if err := c.AddPipeline("p"); err != nil { @@ -228,7 +228,7 @@ func TestSetKeepsAKeyItCannotTake(t *testing.T) { } } if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) || - beat.IntervalMS != 1000 { + beat.IntervalMS != 0 { t.Fatalf("options %v, heartbeat %+v", m, beat) } } @@ -325,18 +325,67 @@ func TestPipelineNamesAreUnique(t *testing.T) { } // A stage an edit turns on does what its spec would: a rate limit drops what -// passes its rate, a heartbeat beats +// passes its rate, a heartbeat beats. A stage and a filter hold only the keys +// set, so the forms carry no default the edit did not set. func TestAStageTurnsOnAsItsSpecDoes(t *testing.T) { c := &Composition{} if err := c.AddPipeline("p"); err != nil { t.Fatal(err) } - for _, err := range []error{c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500")} { + _, err := c.Add(0, "filters", "") + for _, err := range []error{err, c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500"), + c.Set(0, Node{Role: "filters"}, "patterns", "x")} { if err != nil { t.Fatal(err) } } - if f := c.Pipelines[0].Flow; f.RateLimit.Policy != "drop" || !f.Heartbeat.Enabled { - t.Fatalf("%+v %+v", f.RateLimit, f.Heartbeat) + f := c.Pipelines[0].Flow + if *f.RateLimit != (config.RateLimitConfig{EntriesPerSecond: 5, Policy: "drop"}) || *f.Heartbeat != (config.HeartbeatConfig{Enabled: true, IntervalMS: 500}) || + !reflect.DeepEqual(f.Filters, []config.FilterConfig{{Patterns: []string{"x"}}}) { + t.Fatalf("%+v %+v %+v", f.RateLimit, f.Heartbeat, f.Filters) + } +} + +// A zero where zero takes the key's default is unset, as lw reads it; where +// zero means something of its own (retention_hours 0 keeps files), it stays +func TestAZeroTakingTheDefaultIsUnset(t *testing.T) { + c, err := FromJSON([]byte(`[{"name":"p","plugin_sources":[{"type":"file","config":{"directory":"/x","check_interval_ms":0}}], + "plugin_sinks":[{"type":"file","config":{"directory":"/y","name":"n","retention_hours":0}}]}]`)) + if err == nil { + err = c.Set(0, Node{Role: "sink", ID: "file"}, "buffer_size", "0") + } + if src, sink := c.Pipelines[0].PluginSources[0].Config, c.Pipelines[0].PluginSinks[0].Config; err != nil || + !reflect.DeepEqual(src, map[string]any{"directory": "/x"}) || !reflect.DeepEqual(sink, map[string]any{"directory": "/y", "name": "n", "retention_hours": 0.0}) { + t.Fatalf("%v %v %v", src, sink, err) + } +} + +// Text pasted as one form that has another's shape is refused as that form; +// text in no form's shape, or in this one's, gets this form's own refusal. +// The settings a file holds, and an environment's LOGWISP_ variables no +// pipeline flag has, are left out by name. +func TestAPasteIsReadAsItsForm(t *testing.T) { + form := map[string]Form{} + for _, f := range Forms() { + form[f.Name] = f + } + for text, want := range map[[2]string]string{ + {"file", "lw --source null --sink null"}: "this is an lw command line, not the pipelines of a configuration file: read it as the command form", + {"command", "LOGWISP_SOURCE='null'\n"}: "this is LOGWISP_ variables for one pipeline, not an lw command line: read it as the environment form", + {"environment", "[[pipelines]]\nname = \"p\"\n"}: "this is the pipelines of a configuration file, not LOGWISP_ variables for one pipeline: read it as the file form", + {"command", "lw -c lw.toml"}: "-c: not a pipeline flag", + {"file", "hello world"}: "expected '=' after key", + } { + if _, err := form[text[0]].Parse(text[1], nil); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s %q: %v", text[0], text[1], err) + } + } + file, err := form["file"].Parse("quiet = true\n[logging]\nlevel = \"info\"\n[[pipelines]]\nname = \"p\"\n", nil) + if err != nil || !slices.Equal(file.LeftOut, []string{"logging.level", "quiet"}) { + t.Fatalf("file: %+v %v", file, err) + } + env, err := form["environment"].Parse("LOGWISP_SORCE_1='null'\nLOGWISP_QUIET=true\nLOGWISP_SINK=''\nLOGWISP_SOURCE_1=null\nHOME=/root\n", nil) + if err != nil || !slices.Equal(env.LeftOut, []string{"LOGWISP_QUIET", "LOGWISP_SORCE_1"}) { + t.Fatalf("environment: %+v %v", env, err) } } diff --git a/internal/config/config.go b/internal/config/config.go index 76b8676..cd8b39e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -69,28 +69,31 @@ type PipelineConfig struct { // --- Flow --- // FlowConfig holds the stages between sources and sinks, in the order entries -// pass them; the heartbeat runs beside them +// pass them; the heartbeat runs beside them. Help is each stage's summary. type FlowConfig struct { - RateLimit *RateLimitConfig `toml:"rate_limit"` - Filters []FilterConfig `toml:"filters,omitempty"` - Format *FormatConfig `toml:"format"` - Heartbeat *HeartbeatConfig `toml:"heartbeat"` + RateLimit *RateLimitConfig `toml:"rate_limit" help:"Drop entries beyond a rate, and those over a size"` + Filters []FilterConfig `toml:"filters,omitempty" help:"Pass or drop entries by RE2 patterns, in order"` + Format *FormatConfig `toml:"format" help:"How entries are written: raw, txt or json"` + Heartbeat *HeartbeatConfig `toml:"heartbeat" help:"A keep-alive entry on a timer, beside the entries"` } // --- Flow stages --- +// +// A stage key's zero is its unset, the default or what zero means there, so +// a stage is written with the keys set alone. // HeartbeatConfig defines settings for periodic keep-alive or status messages type HeartbeatConfig struct { - Enabled bool `toml:"enabled" help:"send heartbeats; false keeps these settings and sends none"` - IntervalMS int64 `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"` - IncludeTimestamp bool `toml:"include_timestamp" help:"stamp each heartbeat with its time"` - IncludeStats bool `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"` - Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"kept for older files; heartbeats are written in the flow's format"` + Enabled bool `toml:"enabled,omitempty" help:"send heartbeats; false keeps these settings and sends none"` + IntervalMS int64 `toml:"interval_ms,omitempty" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"` + IncludeTimestamp bool `toml:"include_timestamp,omitempty" help:"stamp each heartbeat with its time"` + IncludeStats bool `toml:"include_stats,omitempty" help:"add type, beat_count and, from the second beat, the measured interval_ms"` + Format string `toml:"format,omitempty" default:"txt" lw:"enum=txt|json|raw" help:"kept for older files; heartbeats are written in the flow's format"` } // FormatConfig is how a pipeline writes its entries type FormatConfig struct { - Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"raw writes each message as it came, txt a line with time and level, json one object an entry"` + Type string `toml:"type,omitempty" default:"raw" lw:"enum=raw|txt|json" help:"raw writes each message as it came, txt a line with time and level, json one object an entry"` Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"for experts, a sum of bits over the type's choice: 1 writes messages verbatim, 16 drops the time, 32 the level"` TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"timestamp layout: the reference time 2006-01-02 15:04:05 written as timestamps should look, such as 02 Jan 15:04"` SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"` @@ -108,10 +111,10 @@ const ( // RateLimitConfig defines the configuration for pipeline-level rate limiting type RateLimitConfig struct { - EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second let through; unset, the stage limits nothing, sizes included"` - BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries that may pass in a burst before entries_per_second holds them back"` - Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop enforces entries_per_second and max_entry_size_bytes; pass lets every entry through"` - MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries; needs entries_per_second and policy drop"` + EntriesPerSecond float64 `toml:"entries_per_second,omitempty" lw:"min=0,zero=off" help:"entries a second let through; unset, the stage limits nothing, sizes included"` + BurstEntries float64 `toml:"burst_entries,omitempty" lw:"min=0,zero=one second's worth" help:"entries that may pass in a burst before entries_per_second holds them back"` + Policy string `toml:"policy,omitempty" default:"pass" lw:"enum=pass|drop" help:"drop enforces entries_per_second and max_entry_size_bytes; pass lets every entry through"` + MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes,omitempty" lw:"min=0,zero=unlimited" help:"drop larger entries; needs entries_per_second and policy drop"` } // FilterType represents the filter's behavior (include or exclude) @@ -136,8 +139,8 @@ const ( // FilterConfig represents the configuration for a single filter type FilterConfig struct { - Type FilterType `toml:"type" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"` - Logic FilterLogic `toml:"logic" default:"or" lw:"enum=or|and" help:"match any pattern, or all"` + Type FilterType `toml:"type,omitempty" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"` + Logic FilterLogic `toml:"logic,omitempty" default:"or" lw:"enum=or|and" help:"match any pattern, or all"` Patterns []string `toml:"patterns,omitempty" lw:"required,hint=regex" help:"RE2 patterns"` } diff --git a/internal/config/schema.go b/internal/config/schema.go index 63e0c64..84ef1b2 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -172,14 +172,15 @@ type Key struct { var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer", reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"} -// Stage is a flow stage: its key in the flow table, whether it is a list (the -// filters), its options, and the values that turn it on, which a file's -// defaults leave off +// Stage is a flow stage: its key in the flow table, what it does, whether it +// is a list (the filters), its options, and the values that turn it on, which +// a file's defaults leave off type Stage struct { - Key string `json:"key"` - List bool `json:"list,omitempty"` - Keys []Key `json:"keys"` - On map[string]any `json:"on,omitempty"` + Key string `json:"key"` + Summary string `json:"summary"` + List bool `json:"list,omitempty"` + Keys []Key `json:"keys"` + On map[string]any `json:"on,omitempty"` } // FlowStages lists the flow table's stages, in the order entries pass them @@ -188,7 +189,7 @@ func FlowStages() []Stage { t := reflect.TypeFor[FlowConfig]() for _, o := range options(t) { ft := t.Field(o.index).Type - out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])}) + out = append(out, Stage{o.Name, o.Help, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])}) } return out } @@ -232,8 +233,8 @@ func settingKeys(v reflect.Value, prefix string) []Key { // Decode reads a plugin's config map into its options: unknown keys, kinds, // the defaults, the tag rules and the options' own Check, then the host, tls, -// acl and auth rules of its side; a leading ~ in a path is the home directory. -// It reads no file. +// acl and auth rules of its side, its error joining every problem found; a +// leading ~ in a path is the home directory. It reads no file. func Decode[T any](role, typ string, m map[string]any) (*T, error) { v, err := decode(role, typ, m) if err != nil { @@ -259,30 +260,27 @@ func decode(role, typ string, m map[string]any) (any, error) { if err := lconfig.ScanMap(m, v.Interface()); err != nil { return nil, err } - if err := Settle(v.Interface()); err != nil { - return nil, err - } + errs := []error{Settle(v.Interface())} expandPaths(v.Elem()) if n, ok := v.Interface().(networked); ok { - if err := p.checkNetwork(n); err != nil { - return nil, err - } + errs = append(errs, p.checkNetwork(n)) + } + if err := errors.Join(errs...); err != nil { + return nil, err } return v.Interface(), nil } -// Settle gives a decoded table its defaults, applies the tag rules, then its -// own Check, as Decode does before its side's rules, for the flow stages a -// file decodes typed +// Settle gives a decoded table its defaults, applies the tag rules and its own +// Check, as Decode does before its side's rules, for the flow stages a file +// decodes typed; its error joins every problem func Settle(v any) error { Fill(v) - if err := checkTags(reflect.ValueOf(v).Elem(), ""); err != nil { - return err - } + err := checkTags(reflect.ValueOf(v).Elem(), "") if c, ok := v.(interface{ Check() error }); ok { - return c.Check() + err = errors.Join(err, c.Check()) } - return nil + return err } // Fill gives a table its defaults and checks nothing: Settle's first step @@ -369,11 +367,12 @@ func setDefault(f reflect.Value, o option) { } } -// checkTags applies the tag rules to a table's own keys, naming each by -// prefix and key. A required list needs an entry. A zero number is the -// default or its zero meaning, and meets the range unless required; an empty -// enum is one only by its zero or default. +// checkTags applies the tag rules to a table's own keys, joining a problem a +// key, each named by prefix and key. A required list needs an entry. A zero +// number is the default or its zero meaning, and meets the range unless +// required; an empty enum is one only by its zero or default. func checkTags(v reflect.Value, prefix string) error { + var errs []error for _, o := range options(v.Type()) { f := v.Field(o.index) var err error @@ -404,10 +403,10 @@ func checkTags(v reflect.Value, prefix string) error { } } if err != nil { - return &KeyError{prefix + o.Name, err} + errs = append(errs, &KeyError{prefix + o.Name, err}) } } - return nil + return errors.Join(errs...) } func num(n float64) string { return strconv.FormatFloat(n, 'f', -1, 64) } @@ -425,7 +424,8 @@ func sideKeys(t reflect.Type, side string) []string { // Coerce gives command-line and environment values, which are strings, and // JSON's floats the kinds a plugin's options declare, so a dump prints them -// typed. Keys and types it does not know are Decode's to refuse. +// typed; its error joins a key each it cannot. Keys and types it does not +// know are Decode's to refuse. func Coerce(role, typ string, m map[string]any) error { p, ok := LookupPlugin(role, typ) if !ok { @@ -435,6 +435,7 @@ func Coerce(role, typ string, m map[string]any) error { } func coerce(t reflect.Type, m map[string]any, prefix string) error { + var errs []error for _, o := range options(t) { value, ok := m[o.Name] if !ok { @@ -445,18 +446,17 @@ func coerce(t reflect.Type, m map[string]any, prefix string) error { ft = ft.Elem() } if sub, ok := value.(map[string]any); ok && ft.Kind() == reflect.Struct { - if err := coerce(ft, sub, prefix+o.Name+"."); err != nil { - return err - } + errs = append(errs, coerce(ft, sub, prefix+o.Name+".")) continue } v, err := coerceValue(ft.Kind(), value) if err != nil { - return &KeyError{prefix + o.Name, err} + errs = append(errs, &KeyError{prefix + o.Name, err}) + continue } m[o.Name] = v } - return nil + return errors.Join(errs...) } // coerceValue converts text, a JSON float for an integer and a TOML integer diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index 442e12e..0b57e9f 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -93,7 +93,7 @@ func TestCoerceGivesTheDeclaredKinds(t *testing.T) { // ValidateConfig decodes every plugin and flow stage without opening a file, // and names the path of what it refuses: a rule between a table's keys by -// the table's +// the table's, a list entry by its index func TestValidateConfigNamesThePath(t *testing.T) { valid := func() *Config { return &Config{Color: "auto", Pipelines: []PipelineConfig{{ @@ -114,6 +114,9 @@ func TestValidateConfigNamesThePath(t *testing.T) { "pipelines[0].plugin_sinks[out].config.tls: client_auth requires client_ca_file": func(c *Config) { c.Pipelines[0].PluginSinks[0].Config["tls"].(map[string]any)["client_auth"] = true }, + `pipelines[0].plugin_sinks[out].config.acl.allow[1]: "10.0.0.0.1": not an IP address or CIDR; host names and IPv6 zones are not accepted`: func(c *Config) { + c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"allow": []any{"10.0.0.0/8", "10.0.0.0.1"}} + }, "pipelines[0].plugin_sinks[out].config.acl: requests_per_second_per_client applies only to HTTP listeners": func(c *Config) { c.Pipelines[0].PluginSinks[0].Type = "tcp" c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"requests_per_second_per_client": 2.5} @@ -152,19 +155,25 @@ func TestValidateConfigNamesThePath(t *testing.T) { } } -// A configuration that does not load names every problem, a line each, each -// part's first: the settings, each flow stage, each plugin, each pipeline +// A configuration that does not load names every problem, a line each: the +// settings, each flow stage, each plugin and each pipeline, every key's of a +// part, and the first rule between a table's keys func TestLoadReportsEveryProblemALine(t *testing.T) { isolateConfig(t) - _, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file", - "sink", "tcp:port=70000", "rate-limit", "policy=bogus", "filter", "include:patterns=(", + _, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file:from=middle", + "sink", "tcp:port=70000,buffer_size=-1,tls.enabled=true,acl.allow=x", "rate-limit", "policy=bogus", "filter", "include:patterns=(,logic=nor", "pipeline", "a", "sink", "null")}) want := strings.Join([]string{"invalid configuration:", `color: must be one of auto, always, never, got "bogus"`, `pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "bogus"`, + `pipelines[0].flow.filters[0].logic: must be one of or, and, got "nor"`, "pipelines[0].flow.filters[0].patterns[0]: \"(\": error parsing regexp: missing closing ): `(`", "pipelines[0].plugin_sources[file].config.directory: must be set", + `pipelines[0].plugin_sources[file].config.from: must be one of start, end, got "middle"`, "pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000", + "pipelines[0].plugin_sinks[tcp].config.buffer_size: must be at least 1, got -1", + `pipelines[0].plugin_sinks[tcp].config.acl.allow[0]: "x": not an IP address or CIDR; host names and IPv6 zones are not accepted`, + "pipelines[0].plugin_sinks[tcp].config.tls: listeners need cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file", `pipelines[1].name: duplicate name "a"`, }, "\n") if err == nil || err.Error() != want { diff --git a/internal/config/spec.go b/internal/config/spec.go index 1c02142..9f60dbf 100644 --- a/internal/config/spec.go +++ b/internal/config/spec.go @@ -48,8 +48,9 @@ var specKinds = []specKind{ p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts}) return cmp.Or(err, Coerce("source", typ, opts)) }, func(p *PipelineConfig) (specs []string) { + var ids []string for _, s := range p.PluginSources { - specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config)) + specs, ids = append(specs, formatSpec(s.Type, givenID(s.ID, s.Type, ids), s.Config)), append(ids, s.ID) } return specs }}, @@ -62,8 +63,9 @@ var specKinds = []specKind{ p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts}) return cmp.Or(err, Coerce("sink", typ, opts)) }, func(p *PipelineConfig) (specs []string) { + var ids []string for _, s := range p.PluginSinks { - specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config)) + specs, ids = append(specs, formatSpec(s.Type, givenID(s.ID, s.Type, ids), s.Config)), append(ids, s.ID) } return specs }}, @@ -75,7 +77,6 @@ var specKinds = []specKind{ return err }, func(p *PipelineConfig) (specs []string) { for _, f := range p.Flow.Filters { - Fill(&f) specs = append(specs, typedSpec(&f)) } return specs @@ -89,7 +90,6 @@ var specKinds = []specKind{ return nil } format := *p.Flow.Format - Fill(&format) return []string{typedSpec(&format)} }}, {"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error { @@ -100,8 +100,10 @@ var specKinds = []specKind{ return nil } rate := *p.Flow.RateLimit - Fill(&rate) // the policy, whose default here is drop - return []string{formatSpec("", Values(&rate))} + m := Values(&rate) + Fill(&rate) + m["policy"] = rate.Policy // a spec's default is drop, a file's pass + return []string{formatSpec("", m)} }}, {"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error { p.Flow.Heartbeat = &HeartbeatConfig{} @@ -111,7 +113,6 @@ var specKinds = []specKind{ return nil } beat := *p.Flow.Heartbeat - Fill(&beat) m := Values(&beat) m["enabled"] = beat.Enabled return []string{formatSpec("", m)} @@ -137,6 +138,18 @@ func StageOn(stage string, opts map[string]any) map[string]any { return on } +// FillOn gives the keys a spec of the stage turns on their file defaults +// where unset, as that spec writes them: a stage from a file and from its +// spec are then one +func FillOn(stage string, v any) { + rv := reflect.ValueOf(v).Elem() + for _, o := range options(rv.Type()) { + if _, on := stageOn[stage][o.Name]; on && o.Default != "" && rv.Field(o.index).IsZero() { + setDefault(rv.Field(o.index), o) + } + } +} + // Spec is one pipeline flag: its name without dashes, and its value. type Spec struct{ Flag, Value string } @@ -219,13 +232,22 @@ func envPipelineSpecs(environ []string) []pipelineSpec { } // EnvPipelines builds the pipeline of LOGWISP_ variables, NAME=value each, as -// Load does without pipeline flags -func EnvPipelines(environ []string, isDir IsDir) ([]PipelineConfig, error) { +// Load does without pipeline flags; unread are the other LOGWISP_ variables +// set, sorted: settings, or names no flag has +func EnvPipelines(environ []string, isDir IsDir) (pipelines []PipelineConfig, unread []string, err error) { specs := envPipelineSpecs(environ) - if len(specs) == 0 { - return nil, errors.New("no LOGWISP_ pipeline variables") + for _, kv := range environ { + name, value, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "LOGWISP_") && value != "" && !slices.ContainsFunc(specs, func(s pipelineSpec) bool { return s.name == name }) { + unread = append(unread, name) + } } - return buildPipelines(specs, isDir) + slices.Sort(unread) + if len(specs) == 0 { + return nil, unread, errors.New("no LOGWISP_ pipeline variables") + } + pipelines, err = buildPipelines(specs, isDir) + return pipelines, unread, err } // SpecPipelines builds the pipelines of command-line specs, as Load does @@ -280,8 +302,9 @@ func buildPipelines(specs []pipelineSpec, isDir IsDir) ([]PipelineConfig, error) } // PipelineSpecs writes pipelines as the flags that build them: --pipeline, -// ids, flow stage defaults and heartbeat's enabled explicit, as some flags' -// defaults differ from a file's +// the ids and keys set, and a typed stage's type, the rate limit's policy and +// the heartbeat's enabled explicit, as those flags' defaults differ from a +// file's func PipelineSpecs(pipelines []PipelineConfig) []Spec { var specs []Spec for _, p := range pipelines { @@ -298,14 +321,30 @@ func PipelineSpecs(pipelines []PipelineConfig) []Spec { return specs } -// typedSpec writes a stage whose type key is its spec's TYPE +// typedSpec writes a stage whose type key is its spec's TYPE: the type set, +// or its default func typedSpec(stage any) string { m := Values(stage) + if _, set := m["type"]; !set { + filled := reflect.New(reflect.TypeOf(stage).Elem()) + filled.Elem().Set(reflect.ValueOf(stage).Elem()) + Fill(filled.Interface()) + m["type"] = Values(filled.Interface())["type"] + } typ, _ := m["type"].(string) delete(m, "type") return formatSpec(typ, m) } +// givenID is an instance's id as its spec gives it: none for the one +// instanceID would take, after the instances before +func givenID(id, typ string, before []string) map[string]any { + if id == FreeID(typ, before) { + return nil + } + return map[string]any{"id": id} +} + // formatSpec writes TYPE[:key=value,...] or key=value,...: tables as dotted // keys, lists as a repeated key func formatSpec(typ string, tables ...map[string]any) string { diff --git a/internal/config/spec_test.go b/internal/config/spec_test.go index 5ae55cb..e602d94 100644 --- a/internal/config/spec_test.go +++ b/internal/config/spec_test.go @@ -216,16 +216,21 @@ func TestReloadKeepsSpecPipelines(t *testing.T) { } } -// Written specs build the pipelines a file would: each flow stage with its -// defaults, so the rate limit keeps its file policy and the heartbeat its off +// Written specs build the pipelines a file would, with the keys it set: a +// typed stage names its type, the rate limit keeps its file policy and the +// heartbeat its off, and an id is written only where a spec would not take it func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) { p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5}, Heartbeat: &HeartbeatConfig{}}, PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}}, - PluginSinks: []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}} - got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil) - p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"}, - Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}} + PluginSinks: []PluginSinkConfig{{ID: "null_2", Type: "null", Config: map[string]any{}}, {ID: "null", Type: "null", Config: map[string]any{}}}} + specs := PipelineSpecs([]PipelineConfig{p}) + got, err := SpecPipelines(specs, nil) + p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"}, Heartbeat: &HeartbeatConfig{}} if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) { t.Fatalf("got %+v %v\nwant %+v", got, err, p) } + if want := []Spec{{"pipeline", "p"}, {"source", "null:id=in"}, {"sink", "null:id=null_2"}, {"sink", "null"}, {"format", "raw"}, + {"rate-limit", "entries_per_second=5,policy=pass"}, {"heartbeat", "enabled=false"}}; !reflect.DeepEqual(specs, want) { + t.Fatalf("specs %v, want %v", specs, want) + } } diff --git a/internal/config/validate.go b/internal/config/validate.go index 2234244..ff08289 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -24,9 +24,9 @@ import ( // ValidateConfig checks a configuration without reading a file it names: // the settings, then each pipeline's flow stages, and its sources and sinks -// decoded in full. Its error joins every problem, one a part, each a -// *KeyError naming the refused key's path, save "no pipelines configured". -// lw --check opens the files. +// decoded in full. Its error joins every problem, each a *KeyError naming the +// refused key's path, save "no pipelines configured". lw --check opens the +// files. func ValidateConfig(cfg *Config) error { if cfg == nil { return fmt.Errorf("config is nil") @@ -99,8 +99,8 @@ func ValidatePipelines(pipelines []PipelineConfig) error { return errors.Join(errs...) } -// checkFlow settles copies of a pipeline's flow stages, which keep their -// defaults for the dump: a problem a stage +// checkFlow settles copies of a pipeline's flow stages, which keep the keys +// set for the dump func checkFlow(f *FlowConfig, path string) error { if f == nil { return nil @@ -126,8 +126,16 @@ func checkFlow(f *FlowConfig, path string) error { return errors.Join(errs...) } -// At addresses err to the path of the table it was found in +// At addresses err to the path of the table it was found in, each problem of +// a joined one func At(path string, err error) error { + if joined, ok := err.(interface{ Unwrap() []error }); ok { + var errs []error + for _, e := range joined.Unwrap() { + errs = append(errs, At(path, e)) + } + return errors.Join(errs...) + } if ke, ok := errors.AsType[*KeyError](err); ok { return &KeyError{path + "." + ke.Key, ke.Err} } @@ -167,19 +175,17 @@ func (o *HTTPChainSinkOptions) network() (string, *TLSOptions, *AuthOptions, *AC return o.Host, o.TLS, o.Auth, nil } -// checkNetwork applies the host, tls, acl and auth rules of the plugin's side +// checkNetwork applies the host, tls, acl and auth rules of the plugin's +// side, joining their problems; acl entries are read by the host's family func (p Plugin) checkNetwork(n networked) error { host, t, a, acl := n.network() + var errs []error if _, err := core.Network(host); err != nil { - return &KeyError{"host", err} + errs = append(errs, &KeyError{"host", err}) + } else { + errs = append(errs, table("acl", acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0))) } - if err := t.Check(p.Side != Dialer); err != nil { - return table("tls", err) - } - if err := acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0); err != nil { - return table("acl", err) - } - return table("auth", a.Check(p.Side, p.HTTP, t.State())) + return errors.Join(append(errs, table("tls", t.Check(p.Side != Dialer)), table("auth", a.Check(p.Side, p.HTTP, t.State())))...) } // Check holds the rules between tls keys on a listener or a dialer; a table @@ -221,8 +227,9 @@ func (o *TLSOptions) Check(listener bool) error { return fmt.Errorf("tls: cert_file and key_file must be set together") } if o.PinSHA256 != "" { - _, err := ParsePins(o.PinSHA256) - return err + if _, err := ParsePins(o.PinSHA256); err != nil { + return &KeyError{"tls.pin_sha256", err} + } } return nil } @@ -237,7 +244,7 @@ func ParsePins(s string) ([][]byte, error) { b64, ok := strings.CutPrefix(strings.TrimSpace(p), PinPrefix) h, err := base64.StdEncoding.DecodeString(b64) if !ok || err != nil || len(h) != sha256.Size { - return nil, fmt.Errorf("tls: pin_sha256 %q: want %sBASE64 of a SHA-256, ';' between several", p, PinPrefix) + return nil, fmt.Errorf("pin %q: want %sBASE64 of a SHA-256, ';' between several", p, PinPrefix) } pins = append(pins, h) } @@ -290,7 +297,7 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error { case o.Type == "mtls": for i, pattern := range o.AllowPatterns { if _, err := regexp.Compile(pattern); err != nil { - return fmt.Errorf("auth: allow_patterns[%d] %q: %w", i, pattern, err) + return &KeyError{fmt.Sprintf("auth.allow_patterns[%d]", i), fmt.Errorf("%q: %w", pattern, err)} } } return nil @@ -320,9 +327,9 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error { // Binds the certificate to the user: a peer needs its own of both return fmt.Errorf("auth: identity under type %q binds the client certificate to the user and requires tls.client_auth", "scram") } - for _, e := range o.TrustedProxies { + for i, e := range o.TrustedProxies { if _, err := ParsePrefix(e, "tcp"); err != nil { - return fmt.Errorf("auth: trusted_proxies entry %q: %w", e, err) + return &KeyError{fmt.Sprintf("auth.trusted_proxies[%d]", i), fmt.Errorf("%q: %w", e, err)} } } return nil @@ -357,7 +364,8 @@ func (o *AuthOptions) scramKey() string { // Check holds the acl rules of a listener on host, http for one serving // requests, proxied for one behind auth.trusted_proxies, whose rules take -// clients of either family, as does a PROXY header +// clients of either family, as does a PROXY header; an entry's problem is +// at its key func (o *ACLOptions) Check(host string, http, proxied bool) error { if o == nil { return nil @@ -384,17 +392,18 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error { if header || proxied { clients = "tcp" } + var errs []error for _, list := range []struct { key, network string entries []string }{{"proxy_from", network, o.ProxyFrom}, {"allow", clients, o.Allow}, {"deny", clients, o.Deny}} { - for _, e := range list.entries { + for i, e := range list.entries { if _, err := ParsePrefix(e, list.network); err != nil { - return fmt.Errorf("acl: %s entry %q: %w", list.key, e, err) + errs = append(errs, &KeyError{fmt.Sprintf("acl.%s[%d]", list.key, i), fmt.Errorf("%q: %w", e, err)}) } } } - return nil + return errors.Join(errs...) } // httpKey names the first key of table v tagged http that is set @@ -537,8 +546,8 @@ func Scan(configMap map[string]any, target any) error { return lconfig.ScanMap(configMap, target) } -// checkKeys walks nested tables against the toml tags of t, refusing unknown -// keys. Map-typed fields hold free-form keys and are not descended into. +// checkKeys walks nested tables against the toml tags of t, refusing every +// unknown key. Map-typed fields hold free-form keys and are not descended into. func checkKeys(m map[string]any, t reflect.Type, prefix string) error { for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice { t = t.Elem() @@ -552,17 +561,17 @@ func checkKeys(m map[string]any, t reflect.Type, prefix string) error { fields[name] = f.Type } } + var errs []error for _, key := range slices.Sorted(maps.Keys(m)) { ft, ok := fields[key] if !ok { - return fmt.Errorf("unknown key %q", prefix+key) + errs = append(errs, fmt.Errorf("unknown key %q", prefix+key)) + continue } var tables []map[string]any switch v := m[key].(type) { case map[string]any: - if err := checkKeys(v, ft, prefix+key+"."); err != nil { - return err - } + errs = append(errs, checkKeys(v, ft, prefix+key+".")) case []map[string]any: tables = v case []any: @@ -573,12 +582,10 @@ func checkKeys(m map[string]any, t reflect.Type, prefix string) error { } } for i, table := range tables { - if err := checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil { - return err - } + errs = append(errs, checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i))) } } - return nil + return errors.Join(errs...) } // checkFileKeys rejects keys the configuration file declares that Config does @@ -600,14 +607,27 @@ func checkFileKeys(path string) error { } // ParseFile reads a configuration file's text, refusing the keys lw refuses -// there, and opens nothing it names -func ParseFile(data []byte) (*Config, error) { +// there, and opens nothing it names; settings are the paths it sets outside +// the pipelines, sorted +func ParseFile(data []byte) (cfg *Config, settings []string, err error) { root, err := fileTable(data) if err != nil { - return nil, err + return nil, nil, err } - cfg := &Config{} - return cfg, Scan(root, cfg) + var walk func(m map[string]any, prefix string) + walk = func(m map[string]any, prefix string) { + for k, v := range m { + if sub, ok := v.(map[string]any); ok { + walk(sub, prefix+k+".") + } else if prefix != "" || k != "pipelines" { + settings = append(settings, prefix+k) + } + } + } + walk(root, "") + slices.Sort(settings) + cfg = &Config{} + return cfg, settings, Scan(root, cfg) } // fileTable is a file's root table without the keys lw ignores there diff --git a/internal/tui/canvas.go b/internal/tui/canvas.go index a4233d0..ab1c723 100644 --- a/internal/tui/canvas.go +++ b/internal/tui/canvas.go @@ -338,7 +338,7 @@ func (a *app) summary(n compose.Node) string { if n.Role == "filters" { var out []string for _, f := range a.pipeline().Flow.Filters { - sign := map[config.FilterType]string{"include": "+", "exclude": "-"}[f.Type] + sign := map[config.FilterType]string{"include": "+", "exclude": "-"}[filled(f).Type] out = append(out, sign+strings.Join(f.Patterns, "|")) } return strings.Join(out, " ") diff --git a/internal/tui/inspector.go b/internal/tui/inspector.go index b1fade4..527dacb 100644 --- a/internal/tui/inspector.go +++ b/internal/tui/inspector.go @@ -52,7 +52,7 @@ func (a *app) lines() []line { filters := a.pipeline().Flow.Filters for i, f := range filters { fn := compose.Node{Role: "filters", Index: i} - out = append(out, line{node: fn, group: fmt.Sprintf("%d %s", i+1, f.Type), item: -1}) + out = append(out, line{node: fn, group: fmt.Sprintf("%d %s", i+1, filled(f).Type), item: -1}) if a.insp.open[a.foldKey(fn, "")] { out = a.keyLines(out, fn, a.keys(n), "", 1) } @@ -60,6 +60,12 @@ func (a *app) lines() []line { return append(out, line{node: compose.Node{Role: "filters", Index: len(filters)}, item: -1, add: true}) } +// filled is a filter as it runs: an unset type or logic its default +func filled(f config.FilterConfig) config.FilterConfig { + config.Fill(&f) + return f +} + func (a *app) keyLines(out []line, n compose.Node, keys []config.Key, prefix string, depth int) []line { for _, k := range keys { name := k.Name @@ -285,7 +291,7 @@ func choice(k config.Key, cur any) int { // keys, a filter's logic and patterns func (a *app) groupSummary(l line) string { if l.key.Name == "" { - f := a.pipeline().Flow.Filters[l.node.Index] + f := filled(a.pipeline().Flow.Filters[l.node.Index]) return string(f.Logic) + " " + strings.Join(f.Patterns, " ") } switch v := a.value(l.node, l.key.Name).(type) { diff --git a/internal/tui/view.go b/internal/tui/view.go index e5c6ad3..ef23ad9 100644 --- a/internal/tui/view.go +++ b/internal/tui/view.go @@ -761,7 +761,7 @@ func (a *app) verdict(w *row) string { text := fmt.Sprintf("dropped by filter %d", w.index+1) if pi := a.pipeIndex(w.pipe); pi >= 0 { if fs := a.comp.Pipelines[pi].Flow.Filters; w.index < len(fs) { - text += " (" + string(fs[w.index].Type) + " " + strings.Join(fs[w.index].Patterns, " | ") + ")" + text += " (" + string(filled(fs[w.index]).Type) + " " + strings.Join(fs[w.index].Patterns, " | ") + ")" } } return text