From 2d727729439423825a74675b982eea5d27aad31df142d33f8749688ff70eddc3 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Fri, 9 Oct 2026 18:33:50 -0400 Subject: [PATCH] v0.29.0 the configuration engine, and lw --schema --- AGENTS.md | 4 + cmd/lw/bootstrap.go | 9 +- cmd/lw/cli.go | 10 +- cmd/lw/cli_test.go | 120 +++++++ cmd/lw/completion.go | 6 +- cmd/lw/help.go | 4 +- cmd/lw/main.go | 10 + cmd/lw/test_helpers_test.go | 5 + deploy/package/completion/_lw | 1 + deploy/package/completion/lw.bash | 4 +- deploy/package/completion/lw.fish | 1 + doc/cli.md | 5 +- doc/configuration.md | 4 +- doc/lw.1 | 6 +- doc/todo.md | 46 +-- go.mod | 8 +- go.sum | 16 +- internal/compose/compose.go | 571 ++++++++++++++++++++++++++++++ internal/compose/compose_test.go | 119 +++++++ internal/config/config.go | 46 +-- internal/config/loader.go | 26 +- internal/config/preset.go | 97 +++-- internal/config/preset_test.go | 27 ++ internal/config/schema.go | 210 ++++++++--- internal/config/schema_test.go | 4 +- internal/config/spec.go | 219 ++++++++++-- internal/config/spec_test.go | 14 + internal/config/validate.go | 34 +- 28 files changed, 1386 insertions(+), 240 deletions(-) create mode 100644 internal/compose/compose.go create mode 100644 internal/compose/compose_test.go diff --git a/AGENTS.md b/AGENTS.md index ce52891..18e5b50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,6 +79,10 @@ change touches. `config.Decode`, which applies the defaults, rules and pure checks and refuses unknown keys at any depth; they keep only I/O. Rules that read nothing belong in a `Check` method, so `ValidateConfig` sees them offline. +- `internal/compose` is the engine `lw --tui` and the website's WebAssembly + share: it edits pipelines and writes the command line, environment and file + forms, each loading back to the same pipelines; it reads no file and links + no TLS, HTTP or terminal code (its test runs `go list -deps`). - `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands table whose groups (`auth`, `tls`, `preset`, `config`) share one flag mechanism; `config.Load` takes the parsed `config.Args`. Presets are rows of diff --git a/cmd/lw/bootstrap.go b/cmd/lw/bootstrap.go index 3750daa..f0e20cd 100644 --- a/cmd/lw/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -135,14 +135,7 @@ func checkConfig(cfg *config.Config) int { // dumpConfig prints the effective configuration, flags and environment // resolved into it, as a file lw -c reads back. func dumpConfig(cfg *config.Config, w io.Writer) int { - // Snapshot gave each console sink the top-level color: keep it inherited - for _, p := range cfg.Pipelines { - for _, s := range p.PluginSinks { - if s.Type == "console" && s.Config["color"] == cfg.Color { - delete(s.Config, "color") - } - } - } + config.Uninherit(cfg.Pipelines, cfg.Color) data, err := toml.Marshal(cfg) if err == nil { _, err = w.Write(data) diff --git a/cmd/lw/cli.go b/cmd/lw/cli.go index b6766fe..f978f53 100644 --- a/cmd/lw/cli.go +++ b/cmd/lw/cli.go @@ -58,7 +58,7 @@ var shorts = map[string]struct { // switches are lw's own options that take no value and are no setting, so no // file or environment variable turns one on -var switches = []string{"check", "dump", "version"} +var switches = []string{"check", "dump", "version", "schema"} // usageError is a command-line mistake: exit status 2 rather than 1 type usageError string @@ -241,7 +241,7 @@ func parseCommandLine(argv []string) (inv invocation, err error) { case arg == "--color": // Bare, it means always: config takes no bare flag for a string key inv.load.Overrides = append(inv.load.Overrides, "--color="+cmp.Or(next(&i), "always")) - case strings.HasPrefix(name, "--") && slices.Contains(specFlags, name[2:]): + case strings.HasPrefix(name, "--") && slices.ContainsFunc(specFlags, isFlag(name[2:])): if !inline { value = next(&i) } @@ -271,6 +271,10 @@ func parseCommandLine(argv []string) (inv invocation, err error) { return inv, err } +func isFlag(name string) func(config.SpecFlag) bool { + return func(f config.SpecFlag) bool { return f.Flag == name } +} + // presetCommands makes each preset a subcommand whose flags are its keys func presetCommands() []subcommand { var subs []subcommand @@ -294,7 +298,7 @@ func presetCommands() []subcommand { for name, v := range values { set[name] = *v } - pipeline, err := config.ExpandPreset(p.Name, set) + pipeline, err := config.ExpandPreset(p.Name, set, config.HostIsDir) if err != nil { return usageError(err.Error()) } diff --git a/cmd/lw/cli_test.go b/cmd/lw/cli_test.go index b552889..a6e5556 100644 --- a/cmd/lw/cli_test.go +++ b/cmd/lw/cli_test.go @@ -4,6 +4,7 @@ import ( "bytes" "flag" "os" + "os/exec" "path/filepath" "reflect" "regexp" @@ -12,7 +13,9 @@ import ( "testing" shipped "github.com/lixenwraith/logwisp/config" + "github.com/lixenwraith/logwisp/internal/compose" "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/plugin" "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/logwisp/internal/tlsx" ) @@ -187,6 +190,123 @@ func TestDumpReadsBack(t *testing.T) { } } +// The command line, file and environment a composition writes, read by a +// POSIX shell and loaded as lw loads them, give its pipelines back: every +// preset, a second sink, a file's stage without its defaults, an integer for +// a number, an edge and aggregator pair, and values to escape and quote. The +// environment holds one pipeline. +func TestEveryFormLoadsTheSamePipelines(t *testing.T) { + dir := t.TempDir() + empty := filepath.Join(dir, "empty.toml") + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatal(err) + } + users, pass := filepath.Join(dir, "users.toml"), filepath.Join(dir, "it's.pass") + values := map[string]map[string]string{ + "pipe": {"format": "txt"}, + "tail": {"path": dir}, + "serve": {"path": filepath.Join(dir, "*.log"), "listen": "127.0.0.1:15841", "tls": "self", "users": users, "allow": "127.0.0.0/8", "deny": "127.0.0.2"}, + "edge": {"path": dir + "/", "to": "127.0.0.1:15842", "transport": "http", "pin": "sha256//" + strings.Repeat("A", 43) + "=", + "user": `edge,01=a\b`, "password_file": pass}, + "aggregator": {"listen": "127.0.0.1:15843", "users": users, "out": filepath.Join(dir, "out dir"), "allow": "10.0.0.0/8,127.0.0.1"}, + } + compositions := map[string]*compose.Composition{} + for _, p := range config.Presets() { + c, err := compose.FromPreset(p.Name, values[p.Name], config.HostIsDir) + if err != nil { + t.Fatalf("%s: %v", p.Name, err) + } + compositions[p.Name] = c + } + if _, err := compositions["tail"].Add(0, "sink", "null"); err != nil { + t.Fatal(err) + } + written := filepath.Join(dir, "written.toml") + if err := os.WriteFile(written, []byte(`[[pipelines]] +name = "written" +flow.rate_limit.rate = 5 +plugin_sources = [{id = "in", type = "null"}] +plugin_sinks = [{id = "out", type = "http", config = {port = 15844, acl = {requests_per_second_per_client = 2}}}] +`), 0o600); err != nil { + t.Fatal(err) + } + c, err := compose.FromConfig(loadTestConfig(t, written)) + if err != nil { + t.Fatal(err) + } + compositions["written"] = c + pair := &compose.Composition{Pipelines: append(compositions["edge"].Pipelines, compositions["aggregator"].Pipelines...)} + filter, err := pair.Add(0, "filters", "exclude") + if err == nil { + err = pair.Set(0, filter, "patterns", `password=\S{8,64}`, `it's "quoted"`) + } + if err != nil { + t.Fatal(err) + } + compositions["pair"] = pair + shell := func(script string) []string { + out, err := exec.Command("sh", "-c", script).Output() + if err != nil { + t.Fatalf("sh: %v\n%s", err, script) + } + return strings.Split(strings.TrimSuffix(string(out), "\x00"), "\x00") + } + for name, c := range compositions { + line, err := c.CommandLine() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + words := shell("set --" + strings.TrimPrefix(line, "lw") + `printf '%s\0' "$@"`) + loaded := []*config.Config{loadTestConfig(t, empty, words...)} + file := filepath.Join(dir, name+".toml") + data, err := c.File() + if err == nil { + err = os.WriteFile(file, data, 0o600) + } + if err != nil { + t.Fatal(err) + } + loaded = append(loaded, loadTestConfig(t, file)) + env, err := c.Environment() + if len(c.Pipelines) > 1 { + if err == nil { + t.Errorf("%s: the environment took %d pipelines", name, len(c.Pipelines)) + } + } else { + var names []string + for _, line := range strings.Split(strings.TrimSpace(env), "\n") { + names = append(names, `"$`+line[:strings.IndexByte(line, '=')]+`"`) + } + testutil.ClearEnvPrefix(t, "LOGWISP_") + for i, value := range shell(env + "printf '%s\\0' " + strings.Join(names, " ")) { + t.Setenv(strings.Trim(names[i], `"$`), value) + } + loaded = append(loaded, loadWithEnvironment(t, empty)) + } + for _, cfg := range loaded { + back, err := compose.FromConfig(cfg) + if err != nil || !reflect.DeepEqual(back, c) { + t.Errorf("%s: loads back as\n%+v %v\nwant %+v\nfrom %s\n%s\n%s", name, back, err, c, line, data, env) + } + } + } +} + +// lw --schema lists the sources and sinks lw builds +func TestSchemaListsWhatLwBuilds(t *testing.T) { + s := compose.NewSchema() + for _, p := range s.Sources { + if _, ok := plugin.GetSource(p.Type); !ok { + t.Errorf("source %s has no factory", p.Type) + } + } + for _, p := range s.Sinks { + if _, ok := plugin.GetSink(p.Type); !ok { + t.Errorf("sink %s has no factory", p.Type) + } + } +} + // Every preset expands to plugins that build, as lw --check builds them func TestEveryPresetBuilds(t *testing.T) { testLogger(t) diff --git a/cmd/lw/completion.go b/cmd/lw/completion.go index ca536eb..fb6a175 100644 --- a/cmd/lw/completion.go +++ b/cmd/lw/completion.go @@ -77,8 +77,8 @@ func (s shell) script() []byte { // options maps each long option lw itself takes to its short letter, if any func options() map[string]string { options := map[string]string{} - for _, long := range config.SpecFlags() { - options[long] = "" + for _, f := range config.SpecFlags() { + options[f.Flag] = "" } for long := range config.Settings() { options[long] = "" @@ -99,7 +99,7 @@ func options() map[string]string { func completionModel() (top []compFlag, cmds []compCommand, presets string) { settings, specFlags, shortOf := config.Settings(), config.SpecFlags(), options() for _, long := range slices.Sorted(maps.Keys(shortOf)) { - f := compFlag{long: long, short: shortOf[long], value: "text", repeat: slices.Contains(specFlags, long)} + f := compFlag{long: long, short: shortOf[long], value: "text", repeat: slices.ContainsFunc(specFlags, isFlag(long))} if v, ok := valueOf[long]; ok { f.value = v } else if settings[long] || slices.Contains(switches, long) { diff --git a/cmd/lw/help.go b/cmd/lw/help.go index 4959fe1..d168d6c 100644 --- a/cmd/lw/help.go +++ b/cmd/lw/help.go @@ -19,7 +19,7 @@ Usage: lw -t|--check [options] Build every pipeline and plugin, report, exit lw --dump [options] Print the effective configuration as TOML, exit lw help | -h | --help - lw -V | --version + lw -V | --version | --schema Commands (lw COMMAND -h lists its subcommands): %s @@ -37,6 +37,8 @@ after '='; short options do not combine. lw takes options only. (on a terminal; default), always (a bare --color) or never -V, --version Print the version and exit + --schema Print every setting, flag, plugin option and + preset as JSON, exit -h, --help Print this help and exit -- End the options diff --git a/cmd/lw/main.go b/cmd/lw/main.go index 84bff39..4d08b5d 100644 --- a/cmd/lw/main.go +++ b/cmd/lw/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "os" @@ -9,6 +10,7 @@ import ( "reflect" "syscall" + "github.com/lixenwraith/logwisp/internal/compose" "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/logwisp/internal/core" "github.com/lixenwraith/logwisp/internal/version" @@ -33,6 +35,14 @@ func main() { case inv.on["version"]: fmt.Println(version.String()) os.Exit(0) + case inv.on["schema"]: + out := json.NewEncoder(os.Stdout) + out.SetIndent("", " ") + if err := out.Encode(compose.NewSchema()); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } + os.Exit(0) } // --- 1. Initial setup --- diff --git a/cmd/lw/test_helpers_test.go b/cmd/lw/test_helpers_test.go index 15885ef..0905271 100644 --- a/cmd/lw/test_helpers_test.go +++ b/cmd/lw/test_helpers_test.go @@ -20,6 +20,11 @@ func testLogger(t *testing.T) { func loadTestConfig(t *testing.T, path string, args ...string) *config.Config { t.Helper() testutil.ClearEnvPrefix(t, "LOGWISP_") + return loadWithEnvironment(t, path, args...) +} + +func loadWithEnvironment(t *testing.T, path string, args ...string) *config.Config { + t.Helper() inv, err := parseCommandLine(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...)) if err != nil { t.Fatal(err) diff --git a/deploy/package/completion/_lw b/deploy/package/completion/_lw index b6962e1..9c3ef14 100644 --- a/deploy/package/completion/_lw +++ b/deploy/package/completion/_lw @@ -202,6 +202,7 @@ _arguments -S \ '*'{-p=,--preset=}':preset:(pipe tail serve edge aggregator)' \ '(-q --quiet)'{-q,--quiet} \ '*--rate-limit=:value:_default' \ + '--schema' \ '*--sink=:value:_default' \ '*--source=:value:_default' \ '--status_reporter' \ diff --git a/deploy/package/completion/lw.bash b/deploy/package/completion/lw.bash index fa5e60e..cb73301 100644 --- a/deploy/package/completion/lw.bash +++ b/deploy/package/completion/lw.bash @@ -10,7 +10,7 @@ _lw() { elif [[ $prev == = ]]; then prev=${COMP_WORDS[i-2]} fi case ${COMP_WORDS[1]}:$i in - *:1) words='--auto_reload --check -t --color --config -c --dump --filter --format --heartbeat --help -h --logging.console.target --logging.file.directory --logging.file.max_size_mb --logging.file.max_total_size_mb --logging.file.name --logging.file.retention_hours --logging.format --logging.level --logging.output --logging.sanitization --pipeline --preset -p --quiet -q --rate-limit --sink --source --status_reporter --version -V auth tls preset config help' ;; + *:1) words='--auto_reload --check -t --color --config -c --dump --filter --format --heartbeat --help -h --logging.console.target --logging.file.directory --logging.file.max_size_mb --logging.file.max_total_size_mb --logging.file.name --logging.file.retention_hours --logging.format --logging.level --logging.output --logging.sanitization --pipeline --preset -p --quiet -q --rate-limit --schema --sink --source --status_reporter --version -V auth tls preset config help' ;; auth:2) words='add-user remove-user token stream' ;; auth:*) case ${COMP_WORDS[2]} in @@ -95,7 +95,7 @@ _lw() { --preset|-p) mapfile -t COMPREPLY < <(compgen -W 'pipe tail serve edge aggregator' -- "$cur"); return ;; --config|-c|--filter|--format|--heartbeat|--logging.console.target|--logging.file.max_size_mb|--logging.file.max_total_size_mb|--logging.file.name|--logging.file.retention_hours|--logging.format|--logging.level|--logging.output|--logging.sanitization|--pipeline|--rate-limit|--sink|--source) COMPREPLY=(); return ;; esac - words='--auto_reload --check -t --color --config -c --dump --filter --format --heartbeat --help -h --logging.console.target --logging.file.directory --logging.file.max_size_mb --logging.file.max_total_size_mb --logging.file.name --logging.file.retention_hours --logging.format --logging.level --logging.output --logging.sanitization --pipeline --preset -p --quiet -q --rate-limit --sink --source --status_reporter --version -V' ;; + words='--auto_reload --check -t --color --config -c --dump --filter --format --heartbeat --help -h --logging.console.target --logging.file.directory --logging.file.max_size_mb --logging.file.max_total_size_mb --logging.file.name --logging.file.retention_hours --logging.format --logging.level --logging.output --logging.sanitization --pipeline --preset -p --quiet -q --rate-limit --schema --sink --source --status_reporter --version -V' ;; esac mapfile -t COMPREPLY < <(compgen -W "$words" -- "$cur") } diff --git a/deploy/package/completion/lw.fish b/deploy/package/completion/lw.fish index ef0150a..4e2e5ee 100644 --- a/deploy/package/completion/lw.fish +++ b/deploy/package/completion/lw.fish @@ -37,6 +37,7 @@ complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l preset -o p -x -a 'pipe tail serve edge aggregator' complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l quiet -o q complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l rate-limit -r -F +complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l schema complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l sink -r -F complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l source -r -F complete -c lw -n 'not __lw_in auth; and not __lw_in tls; and not __lw_in preset; and not __lw_in config' -l status_reporter diff --git a/doc/cli.md b/doc/cli.md index aa07772..5efae76 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -11,7 +11,7 @@ lw tls COMMAND [flags] lw preset NAME [flags] lw config init [--out FILE] lw help | -h | --help -lw -V | --version +lw -V | --version | --schema ``` Without a configuration file or pipeline options, lw is a filter: stdin to @@ -66,6 +66,9 @@ pipelines have [their own flags](#pipelines): exit; see [Usage Patterns](#usage-patterns) - `--dump`: print the effective configuration, flags, environment and presets resolved into it, as TOML that `-c` reads back, and exit +- `--schema`: print every setting, pipeline flag, flow stage, plugin option + and preset as one JSON document (kinds, defaults, rules, help), and exit; + tools build their option lists from it - `--color [WHEN]`, WHEN `auto`, `always` or `never`: level names in color on console sinks, default `auto` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`); bare, `always`. A console sink's own `color` wins diff --git a/doc/configuration.md b/doc/configuration.md index 1bdfb2c..45fb69f 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -50,8 +50,8 @@ Top-level keys, each as type and default: set their own: `"auto"` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`), `"always"` or `"never"`; see the [console sink](sinks.md#console). -`--version`, `--check` and `--dump` print and exit; they are not persistent -settings. +`--version`, `--check`, `--dump` and `--schema` print and exit; they are not +persistent settings. Note that `status_reporter` writes at DEBUG level, so it produces nothing unless `logging.level = "debug"`. diff --git a/doc/lw.1 b/doc/lw.1 index 652f7b8..496a4c4 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -24,7 +24,7 @@ .Ar command .Op Ar flags .Nm -.Fl V | Fl \-version +.Fl V | Fl \-version | Fl \-schema .Nm .Cm help | Fl h | Fl \-help .Sh DESCRIPTION @@ -146,6 +146,10 @@ Print the effective configuration, with the options, environment and presets resolved into it, as a file .Fl c reads back, and exit. +.It Fl \-schema +Print every setting, pipeline flag, flow stage, plugin option and preset as +one JSON document, with kinds, defaults, rules and help, and exit; tools build +their option lists from it. .It Fl q , Fl \-quiet Turn off the log of .Nm diff --git a/doc/todo.md b/doc/todo.md index 5784059..fda8afb 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -92,42 +92,23 @@ What exists: offline and names the refused key's path. - `internal/config` already compiles for `GOOS=js`; it links no net/http, crypto/tls or terminal code. +- The engine, `internal/compose`: starts empty, from a preset (whose path + `config.IsDir` resolves, nil off the host), a loaded configuration or a + pasted command line; adds, removes and moves sources, sinks and filters, + sets and unsets keys, validates; writes a command line, environment + variables or a file, each loading back to the same pipelines. Its test + fails when it links TLS, HTTP, process, signal or terminal code. +- `lw --schema`: settings, pipeline flags, flow stages in order, the tls, + auth and acl tables, sources, sinks and presets, as one JSON document. - `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions, layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal, ConfirmDialog, scrolling, mouse. `github.com/lixenwraith/color`: RGB, blending, RGBTo256, RGBTo16. -What is missing: -- A host-independent preset: `fileSource` (`preset.go`) decides directory or - pattern with `os.Stat`, so off the target host `path=/var/log` expands to - directory `/var`, pattern `log`. -- Emitters: only parsers exist for the three forms. - Steps: 1. Schema, in `internal/config`: done. -2. Engine, `internal/compose`. - - Imports internal/config, internal/version and lixenwraith/toml only; a - test fails when `go list -deps` shows net, crypto/tls, os/exec, os/signal, - x/term or the auth, log and terminal modules. - - Starts: empty; a preset, with a `stat` function that is nil off the host - (the UI then asks "directory or file pattern"); a loaded configuration; a - pasted command line (wrapping `cliSpecs` and `buildPipelines`). - - Operations: add or remove a source, sink or filter; set or unset an - option; move a filter; validate. - - Emitters, one form per call: - - Command line: `--pipeline NAME`, `id=`, the rate limit policy and - heartbeat `enabled` always explicit (their command-line defaults differ - from the file's); `--key=value` when a value starts with `-`; `,` `=` - `\` escaped; POSIX-quoted. - - Environment: `LOGWISP_SOURCE_1` and the rest, single-quoted - `KEY='value'` lines; an error for more than one pipeline, which the - environment form cannot hold. - - File: the TOML `lw --dump` prints for those pipelines. -3. `lw --schema`, a parser switch like `--help` (not a Config field, so no - file or environment can trigger it). It prints one JSON document: version; - settings with kind, default, enum, flag and variable; the pipeline flags; - flow stage order; the tls, auth and acl tables; sources, sinks, presets. - The site generates its catalogue from it. +2. Engine, `internal/compose`: done. +3. `lw --schema`: done. The site generates its catalogue from it. 4. In `lixenwraith/terminal`, released and pinned before the TUI: - Full-screen 8- and 16-color output (SGR 30-37, 90-97, 40-47) and the default colors 39/49; it emits only 256-color and truecolor today. @@ -191,13 +172,6 @@ Steps: eval, so the page needs only `'wasm-unsafe-eval'` in its `script-src`. Verify: -- A Go test in `cmd/lw`: for every preset, an edge and aggregator pair, - escaped values and two pipelines, the pipelines loaded from the emitted - command line, file and environment (one pipeline) are equal. It runs in an - empty directory with the `LOGWISP_` environment cleared, since a discovered - file would add settings. -- A schema test: every option has a kind, every enum values, every catalogue - row a factory; `TestEveryPluginRejectsUnknownKeys` still passes. - CI builds `cmd/lwconf` for `GOOS=js` and runs a node test of its calls. - The diagram rendered into a cell buffer at 80x24 and 64x24, per tier; a scripted session on a pseudo-terminal for Run and each output. diff --git a/go.mod b/go.mod index 35868be..cd43cb8 100644 --- a/go.mod +++ b/go.mod @@ -7,13 +7,13 @@ require ( github.com/lixenwraith/color v0.1.3 github.com/lixenwraith/config v0.2.2 github.com/lixenwraith/log v0.1.11 - github.com/lixenwraith/terminal v0.1.9 + github.com/lixenwraith/terminal v0.2.0 github.com/lixenwraith/toml v0.1.3 - golang.org/x/term v0.46.0 + golang.org/x/term v0.47.0 ) require ( github.com/golang-jwt/jwt/v5 v5.3.1 // indirect - golang.org/x/crypto v0.57.0 // indirect - golang.org/x/sys v0.48.0 // indirect + golang.org/x/crypto v0.58.0 // indirect + golang.org/x/sys v0.49.0 // indirect ) diff --git a/go.sum b/go.sum index cfeeb49..109d5e1 100644 --- a/go.sum +++ b/go.sum @@ -8,13 +8,13 @@ github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZ github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg= github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKhU= github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg= -github.com/lixenwraith/terminal v0.1.9 h1:WPxLdqdzZoqtPgTb86oEwE66lF1+iPsJ/gpfo6h/rik= -github.com/lixenwraith/terminal v0.1.9/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44= +github.com/lixenwraith/terminal v0.2.0 h1:Xwsg8LoAeV7E//JLIFmEBTTV9e03jpze3EUAOdqS6P8= +github.com/lixenwraith/terminal v0.2.0/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44= github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI= github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0= -golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= -golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= -golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= diff --git a/internal/compose/compose.go b/internal/compose/compose.go new file mode 100644 index 0000000..47ecb87 --- /dev/null +++ b/internal/compose/compose.go @@ -0,0 +1,571 @@ +// Package compose is the configuration engine lw --tui and the website share: +// pipelines started empty, from a preset, a configuration or a command line, +// edited, validated, and written as a command line, environment or file. It +// reads no file and links no network or terminal code, so it builds as WebAssembly. +package compose + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/version" + + "github.com/lixenwraith/toml" +) + +// Composition is the pipelines being composed. Its flow stages hold their +// defaults and its plugin options the kinds they declare, so each form it +// writes loads back to the same pipelines. +type Composition struct { + Pipelines []config.PipelineConfig +} + +// Node is a part of a pipeline: a source or sink by its id, or a flow stage +// by its key in the flow table: filters with an index, format, rate_limit or +// heartbeat +type Node struct { + Role string + ID string + Index int +} + +// FromConfig starts from a loaded configuration's pipelines, its console sinks +// leaving the top-level color inherited +func FromConfig(cfg *config.Config) (*Composition, error) { + c := &Composition{} + for _, p := range cfg.Pipelines { + c.Pipelines = append(c.Pipelines, copyPipeline(p)) + } + config.Uninherit(c.Pipelines, cfg.Color) + return c, c.settle() +} + +// FromPreset starts from a preset's pipeline; isDir answers whether its path +// is a directory, nil off the target host (see config.IsDir) +func FromPreset(name string, values map[string]string, isDir config.IsDir) (*Composition, error) { + p, err := config.ExpandPreset(name, values, isDir) + if err != nil { + return nil, err + } + c := &Composition{Pipelines: []config.PipelineConfig{p}} + return c, c.settle() +} + +// FromCommandLine starts from a pasted lw command line, quoted as a POSIX +// shell reads it; it takes pipeline flags only, as a composition holds no settings +func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) { + words, err := split(strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(line)) // pasted line ends + if err != nil { + return nil, err + } + if len(words) > 0 && (words[0] == "lw" || strings.HasSuffix(words[0], "/lw")) { + words = words[1:] + } + var specs []config.Spec + for i := 0; i < len(words); i++ { + name, value, inline := strings.Cut(words[i], "=") + if name == "-p" { + name = "--preset" + } + flag, long := strings.CutPrefix(name, "--") + if !long || !slices.ContainsFunc(config.SpecFlags(), func(f config.SpecFlag) bool { return f.Flag == flag }) { + return nil, fmt.Errorf("%s: not a pipeline flag", words[i]) + } + if !inline { + if i++; i == len(words) || strings.HasPrefix(words[i], "-") { + return nil, fmt.Errorf("%s requires a value; one that starts with '-' follows '='", name) + } + value = words[i] + } + specs = append(specs, config.Spec{Flag: flag, Value: value}) + } + pipelines, err := config.SpecPipelines(specs, isDir) + if err != nil { + return nil, err + } + c := &Composition{Pipelines: pipelines} + return c, c.settle() +} + +// settle gives every flow stage its defaults and every plugin's options their +// kinds; the rules are Validate's +func (c *Composition) settle() error { + for i := range c.Pipelines { + p := &c.Pipelines[i] + if p.Flow == nil { + p.Flow = &config.FlowConfig{} + } + for _, stage := range stages(p.Flow) { + if !stage.IsNil() { + config.Fill(stage.Interface()) + } + } + for j := range p.Flow.Filters { + config.Fill(&p.Flow.Filters[j]) + } + for j, s := range p.PluginSources { + p.PluginSources[j].Config = orEmpty(s.Config) + if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil { + return fmt.Errorf("source %s: %w", s.ID, err) + } + } + for j, s := range p.PluginSinks { + p.PluginSinks[j].Config = orEmpty(s.Config) + if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil { + return fmt.Errorf("sink %s: %w", s.ID, err) + } + } + } + return nil +} + +// stages are the flow table's fields that hold one stage each, by key: a +// pointer, nil while the stage is off. The filters are a list. +func stages(f *config.FlowConfig) map[string]reflect.Value { + v, out := reflect.ValueOf(f).Elem(), map[string]reflect.Value{} + for i := range v.NumField() { + if v.Field(i).Kind() == reflect.Pointer { + key, _, _ := strings.Cut(v.Type().Field(i).Tag.Get("toml"), ",") + out[key] = v.Field(i) + } + } + return out +} + +func orEmpty(m map[string]any) map[string]any { + if m == nil { + return map[string]any{} + } + return m +} + +func (c *Composition) pipeline(i int) (*config.PipelineConfig, error) { + if i < 0 || i >= len(c.Pipelines) { + return nil, fmt.Errorf("no pipeline %d", i) + } + return &c.Pipelines[i], nil +} + +// AddPipeline appends an empty pipeline under a name no other has +func (c *Composition) AddPipeline(name string) error { + if name == "" || slices.ContainsFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }) { + return fmt.Errorf("pipeline %q: a name no other pipeline has", name) + } + c.Pipelines = append(c.Pipelines, config.PipelineConfig{Name: name, Flow: &config.FlowConfig{}}) + return nil +} + +func (c *Composition) RemovePipeline(i int) error { + if _, err := c.pipeline(i); err != nil { + return err + } + c.Pipelines = slices.Delete(c.Pipelines, i, i+1) + return nil +} + +// Add appends a source or sink of a catalogue type under config.FreeID, or a +// filter of type include or exclude, and returns its node +func (c *Composition) Add(pi int, role, typ string) (Node, error) { + p, err := c.pipeline(pi) + if err != nil { + return Node{}, err + } + var ids []string + switch role { + case "source": + for _, s := range p.PluginSources { + ids = append(ids, s.ID) + } + case "sink": + for _, s := range p.PluginSinks { + ids = append(ids, s.ID) + } + case "filters": + f := config.FilterConfig{Type: config.FilterType(typ)} + config.Fill(&f) + p.Flow.Filters = append(p.Flow.Filters, f) + return Node{Role: role, Index: len(p.Flow.Filters) - 1}, nil + default: + return Node{}, fmt.Errorf("cannot add to %s", role) + } + if _, ok := config.LookupPlugin(role, typ); !ok { + return Node{}, fmt.Errorf("unknown %s type %q", role, typ) + } + n := Node{Role: role, ID: config.FreeID(typ, ids)} + if role == "source" { + p.PluginSources = append(p.PluginSources, config.PluginSourceConfig{ID: n.ID, Type: typ, Config: map[string]any{}}) + } else { + p.PluginSinks = append(p.PluginSinks, config.PluginSinkConfig{ID: n.ID, Type: typ, Config: map[string]any{}}) + } + return n, nil +} + +// Remove deletes a source, sink or filter, or turns a flow stage off +func (c *Composition) Remove(pi int, n Node) error { + p, err := c.pipeline(pi) + if err != nil { + return err + } + switch i := index(p, n); { + case i < 0: + case n.Role == "source": + p.PluginSources = slices.Delete(p.PluginSources, i, i+1) + return nil + case n.Role == "sink": + p.PluginSinks = slices.Delete(p.PluginSinks, i, i+1) + return nil + case n.Role == "filters": + p.Flow.Filters = slices.Delete(p.Flow.Filters, i, i+1) + return nil + } + if stage, ok := stages(p.Flow)[n.Role]; ok { + stage.SetZero() + return nil + } + return n.missing() +} + +// index finds a node in its list, -1 when it is in none +func index(p *config.PipelineConfig, n Node) int { + switch n.Role { + case "source": + return slices.IndexFunc(p.PluginSources, func(s config.PluginSourceConfig) bool { return s.ID == n.ID }) + case "sink": + return slices.IndexFunc(p.PluginSinks, func(s config.PluginSinkConfig) bool { return s.ID == n.ID }) + case "filters": + if n.Index >= 0 && n.Index < len(p.Flow.Filters) { + return n.Index + } + } + return -1 +} + +func (n Node) missing() error { + if n.Role == "filters" { + return fmt.Errorf("no filter %d", n.Index) + } + return fmt.Errorf("no %s %q", n.Role, n.ID) +} + +// MoveFilter moves a filter to index to; entries pass filters in order +func (c *Composition) MoveFilter(pi, from, to int) error { + p, err := c.pipeline(pi) + if err != nil { + return err + } + if n := len(p.Flow.Filters); from < 0 || from >= n || to < 0 || to >= n { + return fmt.Errorf("no filter %d or %d", from, to) + } + f := p.Flow.Filters[from] + p.Flow.Filters = slices.Insert(slices.Delete(p.Flow.Filters, from, from+1), to, f) + return nil +} + +// Set gives a key at a dotted path its values: one, or several for a list. A +// flow stage that is off turns on. A value of the wrong kind leaves the key as +// it was; any other rule is Validate's. +func (c *Composition) Set(pi int, n Node, key string, values ...string) error { + return c.edit(pi, n, func(m map[string]any) error { + unset(m, strings.Split(key, ".")) + for _, v := range values { + if err := config.SetOption(m, key, v); err != nil { + return err + } + } + return nil + }) +} + +// Unset returns a key to its default +func (c *Composition) Unset(pi int, n Node, key string) error { + return c.edit(pi, n, func(m map[string]any) error { + unset(m, strings.Split(key, ".")) + return nil + }) +} + +// unset deletes a dotted key, and the tables it leaves empty +func unset(m map[string]any, path []string) { + if sub, ok := m[path[0]].(map[string]any); ok && len(path) > 1 { + if unset(sub, path[1:]); len(sub) > 0 { + return + } + } + delete(m, path[0]) +} + +// edit changes a copy of a node's options, which replaces them when they +// decode to the kinds they declare +func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) error { + p, err := c.pipeline(pi) + if err != nil { + return err + } + plugin := func(typ string, m *map[string]any) error { + next := config.Clone(*m) + if err := change(next); err != nil { + return err + } + if err := config.Coerce(n.Role, typ, next); err != nil { + return err + } + *m = next + return nil + } + switch i := index(p, n); { + case i < 0: + case n.Role == "source": + return plugin(p.PluginSources[i].Type, &p.PluginSources[i].Config) + case n.Role == "sink": + return plugin(p.PluginSinks[i].Type, &p.PluginSinks[i].Config) + case n.Role == "filters": + filter := &p.Flow.Filters[i] + if err := editStage(reflect.ValueOf(&filter).Elem(), change); err != nil { + return err + } + p.Flow.Filters[i] = *filter + return nil + } + if stage, ok := stages(p.Flow)[n.Role]; ok { + return editStage(stage, change) + } + return n.missing() +} + +// editStage edits a stage's keys as a spec sets them; the table they scan into +// replaces the stage, which turns on +func editStage(stage reflect.Value, change func(map[string]any) error) error { + m := map[string]any{} + if !stage.IsNil() { + m = config.Values(stage.Interface()) + } + if err := change(m); err != nil { + return err + } + next := reflect.New(stage.Type().Elem()) + if err := config.Scan(m, next.Interface()); err != nil { + return err + } + config.Fill(next.Interface()) + stage.Set(next) + return nil +} + +// Validate applies every rule lw applies at load, naming the refused key's path +func (c *Composition) Validate() error { return config.ValidatePipelines(c.Pipelines) } + +// CommandLine writes the composition as an lw command line, a flag a line, +// quoted for a POSIX shell +func (c *Composition) CommandLine() (string, error) { + specs, err := c.shellSpecs() + if err != nil { + return "", err + } + lines := []string{"lw"} + for _, s := range specs { + if strings.HasPrefix(s.Value, "-") { + lines = append(lines, quote("--"+s.Flag+"="+s.Value)) + } else { + lines = append(lines, "--"+s.Flag+" "+quote(s.Value)) + } + } + return strings.Join(lines, " \\\n ") + "\n", nil +} + +// Environment writes the composition as LOGWISP_ variables, KEY='value' a +// line as an environment file holds them; the environment holds one pipeline +func (c *Composition) Environment() (string, error) { + if len(c.Pipelines) != 1 { + return "", fmt.Errorf("the environment holds one pipeline, not %d", len(c.Pipelines)) + } + specs, err := c.shellSpecs() + if err != nil { + return "", err + } + flags, count := config.SpecFlags(), map[string]int{} + var b strings.Builder + for _, s := range specs { + f := flags[slices.IndexFunc(flags, func(f config.SpecFlag) bool { return f.Flag == s.Flag })] + name := f.Variable + if f.Many { + count[f.Flag]++ + name += "_" + strconv.Itoa(count[f.Flag]) + } + fmt.Fprintf(&b, "%s='%s'\n", name, strings.ReplaceAll(s.Value, "'", `'\''`)) + } + return b.String(), nil +} + +// shellSpecs are the specs of a valid composition, for a form pasted into a +// shell, which carries no control character: a terminal acts on ^U or ESC +// before the shell reads the line +func (c *Composition) shellSpecs() ([]config.Spec, error) { + if err := c.Validate(); err != nil { + return nil, err + } + specs, pipeline := config.PipelineSpecs(c.Pipelines), "" + for _, s := range specs { + if s.Flag == "pipeline" { + pipeline = s.Value + } + if strings.ContainsFunc(s.Value, func(r rune) bool { return unicode.IsControl(r) || r == utf8.RuneError }) { + return nil, fmt.Errorf("pipeline %q: a --%s value holds a control character, which only the file form carries", pipeline, s.Flag) + } + } + return specs, nil +} + +// File writes the composition as a configuration file's pipelines, as lw +// --dump prints them +func (c *Composition) File() ([]byte, error) { + if err := c.Validate(); err != nil { + return nil, err + } + return toml.Marshal(map[string]any{"pipelines": c.Pipelines}) +} + +// quote leaves a word of these characters bare, and single-quotes any other, +// or one starting with '=', which zsh expands +func quote(word string) string { + const bare = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_@%+=:,./-" + if word != "" && word[0] != '=' && strings.Trim(word, bare) == "" { + return word + } + return "'" + strings.ReplaceAll(word, "'", `'\''`) + "'" +} + +// split reads words as a POSIX shell does, without expansions: quotes, '\' +// escapes, and '\' before a newline continuing the line +func split(line string) ([]string, error) { + var words []string + var word strings.Builder + in := false + for i := 0; i < len(line); i++ { + switch c := line[i]; { + case c == '\\' && i+1 < len(line): + if i++; line[i] != '\n' { + word.WriteByte(line[i]) + in = true + } + case c == '\'': + end := strings.IndexByte(line[i+1:], '\'') + if end < 0 { + return nil, errors.New("unterminated '") + } + word.WriteString(line[i+1 : i+1+end]) + i, in = i+1+end, true + case c == '"': + for i, in = i+1, true; ; i++ { + if i == len(line) { + return nil, errors.New(`unterminated "`) + } + if line[i] == '"' { + break + } + if line[i] == '\\' && i+1 < len(line) && strings.IndexByte("$`\"\\\n", line[i+1]) >= 0 { + if i++; line[i] == '\n' { + continue + } + } + word.WriteByte(line[i]) + } + case c == ' ' || c == '\t' || c == '\n': + if in { + words, in = append(words, word.String()), false + word.Reset() + } + default: + word.WriteByte(c) + in = true + } + } + if in { + words = append(words, word.String()) + } + return words, nil +} + +func copyPipeline(p config.PipelineConfig) config.PipelineConfig { + f := config.FlowConfig{} + if p.Flow != nil { + f = *p.Flow + } + for _, stage := range stages(&f) { + if !stage.IsNil() { + next := reflect.New(stage.Type().Elem()) + next.Elem().Set(stage.Elem()) + stage.Set(next) + } + } + f.Filters = slices.Clone(f.Filters) + for i := range f.Filters { + f.Filters[i].Patterns = slices.Clone(f.Filters[i].Patterns) + } + p.Flow = &f + p.PluginSources = slices.Clone(p.PluginSources) + for i, s := range p.PluginSources { + p.PluginSources[i].Config = config.Clone(orEmpty(s.Config)) + } + p.PluginSinks = slices.Clone(p.PluginSinks) + for i, s := range p.PluginSinks { + p.PluginSinks[i].Config = config.Clone(orEmpty(s.Config)) + } + return p +} + +// Schema is what lw --schema prints, and the website builds its catalogue +// from. A setting's flag is --KEY, a pipeline flag's --FLAG. +type Schema struct { + Version string `json:"version"` + Settings []Setting `json:"settings"` + Flags []config.SpecFlag `json:"pipeline_flags"` + Flow []config.Stage `json:"flow"` + Tables map[string][]config.Key `json:"tables"` + Sources []Plugin `json:"sources"` + Sinks []Plugin `json:"sinks"` + Presets []config.Preset `json:"presets"` +} + +// Setting is a settings key with its environment variable +type Setting struct { + config.Key + Variable string `json:"variable"` +} + +// Plugin is a catalogue row with its options +type Plugin struct { + config.Plugin + Keys []config.Key `json:"keys"` +} + +func NewSchema() Schema { + s := Schema{ + Version: version.Short(), + Flags: config.SpecFlags(), + Flow: config.FlowStages(), + Tables: map[string][]config.Key{ + "tls": config.KeysOf[config.TLSOptions](), + "auth": config.KeysOf[config.AuthOptions](), + "acl": config.KeysOf[config.ACLOptions](), + }, + Presets: config.Presets(), + } + for _, k := range config.SettingKeys() { + s.Settings = append(s.Settings, Setting{k, "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(k.Name, ".", "_"))}) + } + for _, p := range config.Plugins() { + if p.Role == "source" { + s.Sources = append(s.Sources, Plugin{p, p.Keys()}) + } else { + s.Sinks = append(s.Sinks, Plugin{p, p.Keys()}) + } + } + return s +} diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go new file mode 100644 index 0000000..a4c65c8 --- /dev/null +++ b/internal/compose/compose_test.go @@ -0,0 +1,119 @@ +package compose + +import ( + "os" + "os/exec" + "reflect" + "slices" + "strings" + "testing" +) + +// The engine builds as WebAssembly for the website: it links no TLS, HTTP, +// process, signal or terminal code, nor lw's log, auth or terminal modules +func TestEngineLinksNoHostCode(t *testing.T) { + // A packager's -buildmode (makepkg's pie) is the host's; wasm has only the + // default + flags := slices.DeleteFunc(strings.Fields(os.Getenv("GOFLAGS")), func(f string) bool { + return strings.HasPrefix(f, "-buildmode") + }) + cmd := exec.Command("go", "list", "-deps", ".") + cmd.Env = append(os.Environ(), "GOOS=js", "GOARCH=wasm", "GOFLAGS="+strings.Join(flags, " ")) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("go list: %v\n%s", err, out) + } + for _, dep := range strings.Fields(string(out)) { + module, mine := strings.CutPrefix(dep, "github.com/lixenwraith/") + if slices.Contains([]string{"crypto/tls", "net/http", "os/exec", "os/signal", "golang.org/x/term"}, dep) || + mine && slices.Contains([]string{"log", "auth", "terminal"}, strings.Split(module, "/")[0]) { + t.Errorf("links %s", dep) + } + } +} + +// A pasted command line, its line ends LF or CRLF, starts the composition that +// wrote it: escaped and quoted values, nested keys, lists, moved filters, +// stages whose flags' defaults differ from a file's, a value starting with '-' +func TestPastedCommandLineStartsTheSameComposition(t *testing.T) { + c := &Composition{} + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(c.AddPipeline("edge")) + src, err := c.Add(0, "source", "file") + must(err) + must(c.Set(0, src, "directory", "/var/log/my app")) + must(c.Set(0, src, "pattern", "it's *.log")) + sink, err := c.Add(0, "sink", "tcp_chain") + must(err) + for key, value := range map[string]string{"host": "agg.example", "port": "9000", "tls.enabled": "true", + "tls.pin_sha256": "sha256//" + strings.Repeat("A", 43) + "=", "auth.type": "scram", + "auth.username": `edge,01=a\b\`, "auth.password_file": "/etc/lw/edge.pass"} { + must(c.Set(0, sink, key, value)) + } + exclude, err := c.Add(0, "filters", "exclude") + must(err) + must(c.Set(0, exclude, "patterns", `password=\S{8,64}`, `"quoted"`)) + _, err = c.Add(0, "filters", "include") + must(err) + must(c.MoveFilter(0, 1, 0)) + must(c.Set(0, Node{Role: "rate_limit"}, "rate", "100")) + must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000")) + must(c.Set(0, Node{Role: "heartbeat"}, "enabled", "true")) + must(c.AddPipeline("-b")) + for _, role := range []string{"source", "sink"} { + _, err = c.Add(1, role, "null") + must(err) + } + must(c.Set(1, Node{Role: "heartbeat"}, "interval_ms", "2000")) + line, err := c.CommandLine() + must(err) + for _, pasted := range []string{line, strings.ReplaceAll(line, "\n", "\r\n")} { + got, err := FromCommandLine(pasted, nil) + if err != nil || !reflect.DeepEqual(got, c) { + t.Fatalf("%q\n got %+v %v\nwant %+v", pasted, got, err, c) + } + } +} + +// The command line and environment carry no control character, which a +// terminal acts on before the shell reads the paste; the file carries it +func TestShellFormsRefuseControlCharacters(t *testing.T) { + c, err := FromPreset("tail", map[string]string{"path": "/srv/a\x15b/"}, nil) + if err != nil { + t.Fatal(err) + } + _, line := c.CommandLine() + _, env := c.Environment() + if _, err := c.File(); err != nil || line == nil || env == nil { + t.Fatalf("command line %v, environment %v, file %v", line, env, err) + } +} + +// Set refuses a value of the wrong kind and leaves the key as it was; Unset +// returns a key to its default, dropping a table it leaves empty +func TestSetKeepsAKeyItCannotTake(t *testing.T) { + c := &Composition{} + if err := c.AddPipeline("p"); err != nil { + t.Fatal(err) + } + n, _ := c.Add(0, "sink", "tcp") + beat := Node{Role: "heartbeat"} + for _, err := range []error{c.Set(0, n, "port", "8080"), c.Set(0, n, "tls.enabled", "true"), + c.Unset(0, n, "tls.enabled"), c.Set(0, beat, "interval_ms", "5000"), c.Unset(0, beat, "interval_ms")} { + if err != nil { + t.Fatal(err) + } + } + if err := c.Set(0, n, "port", "http"); err == nil || !strings.HasPrefix(err.Error(), "port: ") { + t.Fatalf("port http: %v", err) + } + if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) || + beat.IntervalMS != 1000 { + t.Fatalf("options %v, heartbeat %+v", m, beat) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index b8d3aa0..a5244bc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,15 +4,13 @@ package config // Config is the top-level configuration structure for the LogWisp application type Config struct { - Quiet bool `toml:"quiet"` + Quiet bool `toml:"quiet" help:"silence lw's own log and notices; pipeline output still flows"` - // Console sinks without their own color: "auto" is a terminal with - // NO_COLOR unset and TERM not dumb - Color string `toml:"color" lw:"enum=auto|always|never"` + // "auto" is a terminal with NO_COLOR unset and TERM not dumb + Color string `toml:"color" lw:"enum=auto|always|never" help:"level names in color on console sinks without their own"` - // Runtime behavior flags - StatusReporter bool `toml:"status_reporter"` - ConfigAutoReload bool `toml:"auto_reload"` + StatusReporter bool `toml:"status_reporter" help:"log pipeline statistics every 30 s at debug; without a file, off"` + ConfigAutoReload bool `toml:"auto_reload" help:"reload when the file changes"` // Selected file path is runtime metadata, never a configurable override. ConfigFile string `toml:"-"` @@ -26,10 +24,10 @@ type Config struct { // LogConfig represents the logging configuration for the LogWisp application itself type LogConfig struct { - Output string `toml:"output" lw:"enum=file|stdout|stderr|split|all|none"` - Level string `toml:"level" lw:"enum=debug|info|warn|error"` - Format string `toml:"format" lw:"enum=raw|txt|json,zero=the log default"` - Sanitization string `toml:"sanitization" lw:"enum=raw|json|txt|shell,zero=the log default"` // console output + Output string `toml:"output" lw:"enum=file|stdout|stderr|split|all|none" help:"where lw's own log goes; split: warn and error to stderr"` + Level string `toml:"level" lw:"enum=debug|info|warn|error" help:"lowest level logged; without a file, warn"` + Format string `toml:"format" lw:"enum=raw|txt|json,zero=the log default" help:"format of lw's own log"` + Sanitization string `toml:"sanitization" lw:"enum=raw|json|txt|shell,zero=the log default" help:"how the console log escapes control characters"` // File output settings (when Output includes "file" or "all") File *LogFileConfig `toml:"file"` @@ -40,25 +38,16 @@ type LogConfig struct { // LogFileConfig defines settings for file-based application logging type LogFileConfig struct { - // Directory for log files - Directory string `toml:"directory"` - - // Base name for log files - Name string `toml:"name"` - - // Maximum size per log file in MB - MaxSizeMB int64 `toml:"max_size_mb"` - - // Maximum total size of all logs in MB - MaxTotalSizeMB int64 `toml:"max_total_size_mb"` - - // Log retention in hours (0 = disabled) - RetentionHours float64 `toml:"retention_hours"` + Directory string `toml:"directory" lw:"hint=dir" help:"where file and all write; the limits below delete old .log files in it"` + Name string `toml:"name" help:"base name of the files"` + MaxSizeMB int64 `toml:"max_size_mb" help:"size at which a file rotates"` + MaxTotalSizeMB int64 `toml:"max_total_size_mb" help:"size of all .log files there, beyond which the oldest go"` + RetentionHours float64 `toml:"retention_hours" help:"hours a .log file is kept; 0 keeps them"` } // LogConsoleConfig defines settings for console-based application logging type LogConsoleConfig struct { - Target string `toml:"target" lw:"enum=stdout|stderr|split"` + Target string `toml:"target" lw:"enum=stdout|stderr|split" help:"accepted without effect: logging.output chooses the console"` } // --- Pipeline --- @@ -74,12 +63,13 @@ type PipelineConfig struct { // --- Flow --- -// FlowConfig consolidates all processing stages between sources and sinks +// FlowConfig holds the stages between sources and sinks, in the order entries +// pass them; the heartbeat runs beside them type FlowConfig struct { - Heartbeat *HeartbeatConfig `toml:"heartbeat"` RateLimit *RateLimitConfig `toml:"rate_limit"` Filters []FilterConfig `toml:"filters,omitempty"` Format *FormatConfig `toml:"format"` + Heartbeat *HeartbeatConfig `toml:"heartbeat"` } // --- Flow stages --- diff --git a/internal/config/loader.go b/internal/config/loader.go index fb92ca1..4449aa5 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -35,15 +35,9 @@ type Args struct { // Settings maps each --path setting Load takes to whether it is a switch, which // takes no separate value; the parser must not leave config to guess. func Settings() map[string]bool { - c := lconfig.New() - if err := c.RegisterStructWithTags("", defaults(), lconfig.FormatTOML); err != nil { - panic(err) // Config is static: every run fails or none does - } settings := map[string]bool{} - for path, value := range c.GetRegisteredPathsWithDefaults() { - if _, tables := value.([]PipelineConfig); !tables { // the pipeline flags set those - _, settings[path] = value.(bool) - } + for _, k := range SettingKeys() { + settings[k.Name] = k.Kind == "bool" } return settings } @@ -59,7 +53,7 @@ func Load(args Args) (*Manager, error) { if len(specs) == 0 { specs = envPipelineSpecs() } - if _, err := buildPipelines(specs); err != nil { + if _, err := buildPipelines(specs, HostIsDir); err != nil { return nil, err } m := &Manager{path: configPath, explicit: isExplicit, specs: specs} @@ -132,12 +126,24 @@ func (m *Manager) Snapshot() (*Config, error) { return cfg, nil } +// Uninherit removes the color Snapshot gave each console sink, so a dump or a +// composition leaves it inherited +func Uninherit(pipelines []PipelineConfig, color string) { + for _, p := range pipelines { + for _, s := range p.PluginSinks { + if s.Type == "console" && s.Config["color"] == color { + delete(s.Config, "color") + } + } + } +} + // usePipelines replaces the file's or default pipelines with the spec ones. func (m *Manager) usePipelines(cfg *Config) error { if len(m.specs) == 0 { return nil } - pipelines, err := buildPipelines(m.specs) + pipelines, err := buildPipelines(m.specs, HostIsDir) cfg.Pipelines = pipelines return err } diff --git a/internal/config/preset.go b/internal/config/preset.go index 92123ad..b704d68 100644 --- a/internal/config/preset.go +++ b/internal/config/preset.go @@ -2,6 +2,7 @@ package config import ( "cmp" + "errors" "fmt" "maps" "net" @@ -16,16 +17,34 @@ import ( // a pipeline with it. Later flags add sources, sinks and filters, or replace // its format, rate limit and heartbeat. type Preset struct { - Name, Summary string - Params []PresetParam - build func(p *PipelineConfig, v map[string]string) error + Name string `json:"name"` + Summary string `json:"summary"` + Params []PresetParam `json:"keys"` + build func(p *PipelineConfig, v map[string]string, isDir IsDir) error } // PresetParam is one key. A list takes several values, ',' between them or // the key repeated; any other key takes one. type PresetParam struct { - Name, Default, Help string - Required, List bool + Name string `json:"name"` + Default string `json:"default,omitempty"` + Help string `json:"help"` + Required bool `json:"required,omitempty"` + List bool `json:"list,omitempty"` +} + +// IsDir tells whether a preset's path is a directory, whose files it follows, +// or a file pattern. Off the target host it is the user's answer, or nil: a +// path ending in '/' is then a directory, one with a glob a pattern, and any +// other fails with ErrPathKind. +type IsDir func(path string) (bool, error) + +var ErrPathKind = errors.New("path: a directory or a file pattern?") + +// HostIsDir asks this host: a path that does not exist is a pattern +func HostIsDir(path string) (bool, error) { + fi, err := os.Stat(path) + return err == nil && fi.IsDir(), nil } // Keys several presets share @@ -50,7 +69,7 @@ var ( var presets = []Preset{ {"pipe", "Standard input to standard output, line for line: lw without a file", []PresetParam{formatParam("raw")}, - func(p *PipelineConfig, v map[string]string) error { + func(p *PipelineConfig, v map[string]string, isDir IsDir) error { stdio := pipeDefault("") p.PluginSources, p.PluginSinks = stdio.PluginSources, stdio.PluginSinks p.Flow.Format = &FormatConfig{Type: v["format"]} @@ -58,10 +77,11 @@ var presets = []Preset{ }}, {"tail", "Follow files to standard output, like tail -F", []PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")}, - func(p *PipelineConfig, v map[string]string) error { - p.PluginSources, p.PluginSinks = pathOrStdin(v), pipeDefault("").PluginSinks + func(p *PipelineConfig, v map[string]string, isDir IsDir) error { + sources, err := pathOrStdin(v, isDir) + p.PluginSources, p.PluginSinks = sources, pipeDefault("").PluginSinks p.Flow.Format = &FormatConfig{Type: v["format"]} - return nil + return err }}, {"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream", slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"), @@ -71,8 +91,12 @@ var presets = []Preset{ {Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, {Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"}, }, aclParams, tlsParams), - func(p *PipelineConfig, v map[string]string) error { - p.PluginSources = pathOrStdin(v) + func(p *PipelineConfig, v map[string]string, isDir IsDir) error { + sources, err := pathOrStdin(v, isDir) + if err != nil { + return err + } + p.PluginSources = sources sink, err := listener(v, "off") if err != nil { return err @@ -115,8 +139,12 @@ var presets = []Preset{ {Name: "key", Help: "client key file"}, {Name: "node", Help: "origin label; default: this host's name"}, }, - func(p *PipelineConfig, v map[string]string) error { - p.PluginSources = pathOrStdin(v) + func(p *PipelineConfig, v map[string]string, isDir IsDir) error { + sources, err := pathOrStdin(v, isDir) + if err != nil { + return err + } + p.PluginSources = sources typ, err := chainType(v["transport"]) if err != nil { return err @@ -159,7 +187,7 @@ var presets = []Preset{ {Name: "out", Help: "directory for the received entries; default: standard output"}, formatParam("json"), }, aclParams, tlsParams), - func(p *PipelineConfig, v map[string]string) error { + func(p *PipelineConfig, v map[string]string, isDir IsDir) error { typ, err := chainType(v["transport"]) if err != nil { return err @@ -197,18 +225,18 @@ var presets = []Preset{ func Presets() []Preset { return slices.Clone(presets) } // ExpandPreset builds the pipeline a preset makes of values, as --preset does -func ExpandPreset(name string, values map[string]string) (PipelineConfig, error) { +func ExpandPreset(name string, values map[string]string, isDir IsDir) (PipelineConfig, error) { p := PipelineConfig{Flow: &FlowConfig{}} each := map[string][]string{} for k, v := range values { each[k] = []string{v} } - err := applyPreset(&p, name, each) + err := applyPreset(&p, name, each, isDir) return p, err } // applyPreset fills defaults and checks keys before the preset's build -func applyPreset(p *PipelineConfig, name string, values map[string][]string) error { +func applyPreset(p *PipelineConfig, name string, values map[string][]string, isDir IsDir) error { i := slices.IndexFunc(presets, func(r Preset) bool { return r.Name == name }) if i < 0 { var names []string @@ -239,24 +267,31 @@ func applyPreset(p *PipelineConfig, name string, values map[string][]string) err } } p.Name = cmp.Or(p.Name, name) - return r.build(p, v) + return r.build(p, v, isDir) } -func pathOrStdin(v map[string]string) []PluginSourceConfig { - if v["path"] == "" { - return pipeDefault("").PluginSources +// pathOrStdin follows path: a directory's files, or the files a glob or a +// file name matches in its directory; without one, standard input +func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error) { + path := v["path"] + if path == "" { + return pipeDefault("").PluginSources, nil } - return []PluginSourceConfig{fileSource(v["path"], v["from"])} -} - -// fileSource follows path: a directory's files, or the files a glob or a -// file name matches in its directory. -func fileSource(path, from string) PluginSourceConfig { - dir, pattern := path, "*" - if fi, err := os.Stat(path); err != nil || !fi.IsDir() { - dir, pattern = filepath.Dir(path), filepath.Base(path) + dir, err := strings.HasSuffix(path, "/"), error(nil) + switch { + case isDir != nil: + dir, err = isDir(path) + case !dir && !strings.ContainsAny(path, `*?[\`): + err = ErrPathKind } - return PluginSourceConfig{ID: "file", Type: "file", Config: map[string]any{"directory": dir, "pattern": pattern, "from": from}} + if err != nil { + return nil, err + } + directory, pattern := path, "*" + if !dir { + directory, pattern = filepath.Dir(path), filepath.Base(path) + } + return []PluginSourceConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": directory, "pattern": pattern, "from": v["from"]}}}, nil } // tlsModes is what each preset tls mode sets, and the preset keys it needs diff --git a/internal/config/preset_test.go b/internal/config/preset_test.go index d15c104..8e04a9d 100644 --- a/internal/config/preset_test.go +++ b/internal/config/preset_test.go @@ -1,6 +1,7 @@ package config import ( + "errors" "reflect" "strings" "testing" @@ -62,3 +63,29 @@ func TestPresetKeysAreChecked(t *testing.T) { } } } + +// A preset path is what isDir answers, a directory named with brackets too; +// without isDir, off the target host, it is a directory when it ends in '/', +// a pattern when it holds a glob, and otherwise ErrPathKind asks +func TestPresetPathKind(t *testing.T) { + yes := func(string) (bool, error) { return true, nil } + for _, c := range []struct { + path string + isDir IsDir + want string + }{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""}} { + p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir) + if c.want == "" { + if !errors.Is(err, ErrPathKind) { + t.Errorf("%s: %v, want ErrPathKind", c.path, err) + } + continue + } + if err != nil { + t.Fatal(err) + } + if m := p.PluginSources[0].Config; m["directory"].(string)+" "+m["pattern"].(string) != c.want { + t.Errorf("%s: %v, want %s", c.path, m, c.want) + } + } +} diff --git a/internal/config/schema.go b/internal/config/schema.go index 2319db5..6e5f0c1 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -16,6 +16,10 @@ import ( // keys it takes type Side int +var sideNames = []string{"local", "listener", "chain_listener", "dialer"} + +func (s Side) MarshalText() ([]byte, error) { return []byte(sideNames[s]), nil } + const ( Local Side = iota // no network Listener // the tcp and http sinks @@ -25,12 +29,13 @@ const ( // Plugin is a catalogue row: a source or sink type and its options table type Plugin struct { - Role, Type string // "source" or "sink", and the type its config names - Side Side - HTTP bool // speaks HTTP: takes bearer tokens and request limits - Single bool // one instance per process - Summary string - options reflect.Type + Role string `json:"-"` // "source" or "sink" + Type string `json:"type"` + Side Side `json:"side"` + HTTP bool `json:"http"` // speaks HTTP: takes bearer tokens and request limits + Single bool `json:"single"` // one instance per process + Summary string `json:"summary"` + options reflect.Type } var plugins = []Plugin{ @@ -49,6 +54,12 @@ var plugins = []Plugin{ {"sink", "http_chain", Dialer, true, false, "Post entry batches to an http_chain source", reflect.TypeFor[HTTPChainSinkOptions]()}, } +// Plugins lists the catalogue +func Plugins() []Plugin { return slices.Clone(plugins) } + +// Keys describes the plugin's options +func (p Plugin) Keys() []Key { return keys(p.options) } + // LookupPlugin finds the catalogue row of a source or sink type func LookupPlugin(role, typ string) (Plugin, bool) { i := slices.IndexFunc(plugins, func(p Plugin) bool { return p.Role == role && p.Type == typ }) @@ -73,13 +84,8 @@ func (e *KeyError) Unwrap() error { return e.Err } // it does, and lw: lists its rules: required, enum=a|b, min=N, max=N, // zero=MEANING (zero is a value, not the default), hint=KIND, listener or dialer. type option struct { - key, def, help string - required bool - enum []string - min, max *float64 - zero string - hint, side string - index int + Key + index int } // options reads the keys of an options table; a malformed tag panics, as the @@ -92,31 +98,31 @@ func options(t reflect.Type) []option { if key == "" || key == "-" { continue } - o := option{key: key, def: f.Tag.Get("default"), help: f.Tag.Get("help"), index: i} + o := option{Key{Name: key, Kind: kinds[f.Type.Kind()], Default: f.Tag.Get("default"), Help: f.Tag.Get("help")}, i} for rule := range strings.SplitSeq(f.Tag.Get("lw"), ",") { name, value, _ := strings.Cut(rule, "=") switch name { case "": case "required": - o.required = true + o.Required = true case "listener", "dialer": - o.side = name + o.Side = name case "enum": - o.enum = strings.Split(value, "|") + o.Enum = strings.Split(value, "|") case "min", "max": n, err := strconv.ParseFloat(value, 64) if err != nil { panic(fmt.Sprintf("%s.%s: lw %s=%q", t.Name(), key, name, value)) } if name == "min" { - o.min = &n + o.Min = &n } else { - o.max = &n + o.Max = &n } case "zero": - o.zero = value + o.Zero = value case "hint": - o.hint = value + o.Hint = value default: panic(fmt.Sprintf("%s.%s: unknown lw rule %q", t.Name(), key, rule)) } @@ -126,6 +132,81 @@ func options(t reflect.Type) []option { return opts } +// Key describes one option, for lw --schema and the composer: its tags, and +// its kind, a table's being "table" (tls, auth or acl) +type Key struct { + Name string `json:"key"` + Kind string `json:"kind"` + Default string `json:"default,omitempty"` + Help string `json:"help,omitempty"` + Required bool `json:"required,omitempty"` + Enum []string `json:"enum,omitempty"` + Min *float64 `json:"min,omitempty"` + Max *float64 `json:"max,omitempty"` + Zero string `json:"zero,omitempty"` + Hint string `json:"hint,omitempty"` + Side string `json:"side,omitempty"` +} + +var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer", + reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"} + +// Stage is a flow stage: its key in the flow table, whether it is a list (the +// filters), and its options +type Stage struct { + Key string `json:"key"` + List bool `json:"list,omitempty"` + Keys []Key `json:"keys"` +} + +// FlowStages lists the flow table's stages, in the order entries pass them +func FlowStages() []Stage { + var out []Stage + t := reflect.TypeFor[FlowConfig]() + for _, o := range options(t) { + ft := t.Field(o.index).Type + out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem())}) + } + return out +} + +// KeysOf describes the options of a table: tls, auth or acl +func KeysOf[T any]() []Key { return keys(reflect.TypeFor[T]()) } + +func keys(t reflect.Type) []Key { + var out []Key + for _, o := range options(t) { + if o.Kind == "" { + panic(fmt.Sprintf("%s.%s: no kind", t.Name(), o.Name)) + } + out = append(out, o.Key) + } + return out +} + +// SettingKeys describes the settings by their dotted paths, with the defaults +// lw has when it reads a file +func SettingKeys() []Key { return settingKeys(reflect.ValueOf(defaults()).Elem(), "") } + +func settingKeys(v reflect.Value, prefix string) []Key { + var out []Key + for _, o := range options(v.Type()) { + switch f := v.Field(o.index); f.Kind() { + case reflect.Pointer: + out = append(out, settingKeys(f.Elem(), prefix+o.Name+".")...) + case reflect.Slice: // the pipelines, which their flags set + default: + k := o.Key + k.Name = prefix + k.Name + if !f.IsZero() { + k.Default = fmt.Sprint(f.Interface()) + } + out = append(out, k) + } + } + return out +} + // Decode reads a plugin's config map into its options: unknown keys, kinds, // the defaults, the tag rules, the tls, auth and acl rules of its side, then the // options' own Check. It reads no file. @@ -145,7 +226,7 @@ func decode(role, typ string, m map[string]any) (any, error) { if err := checkKeys(m, p.options, ""); err != nil { return nil, err } - m = clone(m) + m = Clone(m) if err := coerce(p.options, m, ""); err != nil { return nil, err // a file's wrong kind, named by its key } @@ -168,9 +249,8 @@ func decode(role, typ string, m map[string]any) (any, error) { // Settle gives a decoded table its defaults, applies the tag rules, then its // own Check: Decode's last steps, for the flow stages a file decodes typed func Settle(v any) error { - rv := reflect.ValueOf(v).Elem() - settle(rv) - if err := checkTags(rv, ""); err != nil { + Fill(v) + if err := checkTags(reflect.ValueOf(v).Elem(), ""); err != nil { return err } if c, ok := v.(interface{ Check() error }); ok { @@ -179,15 +259,18 @@ func Settle(v any) error { return nil } +// Fill gives a table its defaults and checks nothing: Settle's first step +func Fill(v any) { settle(reflect.ValueOf(v).Elem()) } + // preset sets every default before decoding, so a switch that defaults on // stays on unless set off; it does so in the tables m sets func preset(v reflect.Value, m map[string]any) { for _, o := range options(v.Type()) { f := v.Field(o.index) - if sub, ok := m[o.key].(map[string]any); ok && f.Kind() == reflect.Pointer && f.Type().Elem().Kind() == reflect.Struct { + if sub, ok := m[o.Name].(map[string]any); ok && f.Kind() == reflect.Pointer && f.Type().Elem().Kind() == reflect.Struct { f.Set(reflect.New(f.Type().Elem())) preset(f.Elem(), sub) - } else if o.def != "" { + } else if o.Default != "" { setDefault(f, o) } } @@ -203,7 +286,7 @@ func settle(v reflect.Value) { if !f.IsNil() { settle(f.Elem()) } - case o.def != "" && o.zero == "" && f.Kind() != reflect.Bool && f.IsZero(): + case o.Default != "" && o.Zero == "" && f.Kind() != reflect.Bool && f.IsZero(): setDefault(f, o) } } @@ -213,24 +296,24 @@ func setDefault(f reflect.Value, o option) { var err error switch f.Kind() { case reflect.String: - f.SetString(o.def) + f.SetString(o.Default) case reflect.Int64: var n int64 - n, err = strconv.ParseInt(o.def, 10, 64) + n, err = strconv.ParseInt(o.Default, 10, 64) f.SetInt(n) case reflect.Float64: var n float64 - n, err = strconv.ParseFloat(o.def, 64) + n, err = strconv.ParseFloat(o.Default, 64) f.SetFloat(n) case reflect.Bool: var b bool - b, err = strconv.ParseBool(o.def) + b, err = strconv.ParseBool(o.Default) f.SetBool(b) default: err = errors.New("no default for this kind") } if err != nil { - panic(fmt.Sprintf("%s: default %q: %v", o.key, o.def, err)) + panic(fmt.Sprintf("%s: default %q: %v", o.Name, o.Default, err)) } } @@ -243,12 +326,12 @@ func checkTags(v reflect.Value, prefix string) error { f := v.Field(o.index) var err error switch { - case o.required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == ""): + case o.Required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == ""): err = errors.New("must be set") - case f.Kind() == reflect.String && len(o.enum) > 0 && (f.String() != "" || o.zero == "" && o.def == "") && !slices.Contains(o.enum, f.String()): - err = fmt.Errorf("must be one of %s, got %q", strings.Join(o.enum, ", "), f.String()) + case f.Kind() == reflect.String && len(o.Enum) > 0 && (f.String() != "" || o.Zero == "" && o.Default == "") && !slices.Contains(o.Enum, f.String()): + err = fmt.Errorf("must be one of %s, got %q", strings.Join(o.Enum, ", "), f.String()) case f.CanFloat() || f.CanInt(): - if f.IsZero() && !o.required { + if f.IsZero() && !o.Required { break } n := float64(0) @@ -260,16 +343,16 @@ func checkTags(v reflect.Value, prefix string) error { switch { case math.IsNaN(n) || math.IsInf(n, 0): err = errors.New("must be a finite number") - case o.min != nil && o.max != nil && !(n >= *o.min && n <= *o.max): - err = fmt.Errorf("must be from %s to %s, got %s", num(*o.min), num(*o.max), num(n)) - case o.min != nil && !(n >= *o.min): - err = fmt.Errorf("must be at least %s, got %s", num(*o.min), num(n)) - case o.max != nil && !(n <= *o.max): - err = fmt.Errorf("must be at most %s, got %s", num(*o.max), num(n)) + case o.Min != nil && o.Max != nil && !(n >= *o.Min && n <= *o.Max): + err = fmt.Errorf("must be from %s to %s, got %s", num(*o.Min), num(*o.Max), num(n)) + case o.Min != nil && !(n >= *o.Min): + err = fmt.Errorf("must be at least %s, got %s", num(*o.Min), num(n)) + case o.Max != nil && !(n <= *o.Max): + err = fmt.Errorf("must be at most %s, got %s", num(*o.Max), num(n)) } } if err != nil { - return &KeyError{prefix + o.key, err} + return &KeyError{prefix + o.Name, err} } } return nil @@ -281,8 +364,8 @@ func num(n float64) string { return strconv.FormatFloat(n, 'f', -1, 64) } func sideKeys(t reflect.Type, side string) []string { var keys []string for _, o := range options(t) { - if o.side == side { - keys = append(keys, o.key) + if o.Side == side { + keys = append(keys, o.Name) } } return keys @@ -301,7 +384,7 @@ func Coerce(role, typ string, m map[string]any) error { func coerce(t reflect.Type, m map[string]any, prefix string) error { for _, o := range options(t) { - value, ok := m[o.key] + value, ok := m[o.Name] if !ok { continue } @@ -310,25 +393,28 @@ func coerce(t reflect.Type, m map[string]any, prefix string) error { ft = ft.Elem() } if sub, ok := value.(map[string]any); ok && ft.Kind() == reflect.Struct { - if err := coerce(ft, sub, prefix+o.key+"."); err != nil { + if err := coerce(ft, sub, prefix+o.Name+"."); err != nil { return err } continue } v, err := coerceValue(ft.Kind(), value) if err != nil { - return &KeyError{prefix + o.key, err} + return &KeyError{prefix + o.Name, err} } - m[o.key] = v + m[o.Name] = v } return nil } -// coerceValue converts text, and a JSON float for an integer; its errors -// never hold the value, which may be a misplaced secret +// coerceValue converts text, a JSON float for an integer and a TOML integer +// for a number; its errors never hold the value, which may be a misplaced secret func coerceValue(kind reflect.Kind, value any) (any, error) { s, text := value.(string) + n, integer := value.(int64) switch { + case kind == reflect.Float64 && integer: + return float64(n), nil case kind == reflect.Slice && text: return []any{s}, nil case kind == reflect.Int64 && text: @@ -357,14 +443,22 @@ func coerceValue(kind reflect.Kind, value any) (any, error) { return value, nil } -// clone copies m and the tables in it, so decoding leaves the caller's map be -func clone(m map[string]any) map[string]any { +// Clone copies an options map with its tables and lists, so decoding or an +// edit leaves the caller's map be +func Clone(m map[string]any) map[string]any { c := make(map[string]any, len(m)) for k, v := range m { - if sub, ok := v.(map[string]any); ok { - v = clone(sub) - } - c[k] = v + c[k] = cloneValue(v) } return c } + +func cloneValue(v any) any { + switch v := v.(type) { + case map[string]any: + return Clone(v) + case []any: + return slices.Clone(v) + } + return v +} diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index 9a2be59..3ac0479 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -13,8 +13,8 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) { for _, p := range plugins { m := map[string]any{} for _, o := range options(p.options) { - if o.required { - m[o.key] = "1" + if o.Required { + m[o.Name] = "1" } } if err := Coerce(p.Role, p.Type, m); err != nil { diff --git a/internal/config/spec.go b/internal/config/spec.go index 545a971..0dd9cd8 100644 --- a/internal/config/spec.go +++ b/internal/config/spec.go @@ -4,7 +4,9 @@ import ( "cmp" "errors" "fmt" + "maps" "os" + "reflect" "slices" "strconv" "strings" @@ -17,11 +19,12 @@ type specKind struct { typed bool // SPEC starts with TYPE many bool // repeatable within a pipeline; the environment adds _1.._N first bool // starts its pipeline, and names it by TYPE unless --pipeline did - apply func(p *PipelineConfig, typ string, opts map[string]any) error + apply func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error + write func(p *PipelineConfig) []string // its specs that build p's part; nil for preset } var specKinds = []specKind{ - {"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error { values := map[string][]string{} for k, v := range opts { switch v := v.(type) { @@ -35,9 +38,9 @@ var specKinds = []specKind{ return fmt.Errorf("preset keys do not nest: %q", k) } } - return applyPreset(p, typ, values) - }}, - {"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { + return applyPreset(p, typ, values, isDir) + }, nil}, + {"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error { ids := make([]string, len(p.PluginSources)) for i, s := range p.PluginSources { ids[i] = s.ID @@ -45,8 +48,13 @@ var specKinds = []specKind{ id, err := instanceID(typ, opts, ids) p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts}) return cmp.Or(err, Coerce("source", typ, opts)) + }, func(p *PipelineConfig) (specs []string) { + for _, s := range p.PluginSources { + specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config)) + } + return specs }}, - {"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error { ids := make([]string, len(p.PluginSinks)) for i, s := range p.PluginSinks { ids[i] = s.ID @@ -54,27 +62,61 @@ var specKinds = []specKind{ id, err := instanceID(typ, opts, ids) p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts}) return cmp.Or(err, Coerce("sink", typ, opts)) + }, func(p *PipelineConfig) (specs []string) { + for _, s := range p.PluginSinks { + specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config)) + } + return specs }}, - {"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error { opts["type"] = typ f := FilterConfig{} err := Scan(opts, &f) p.Flow.Filters = append(p.Flow.Filters, f) return err + }, func(p *PipelineConfig) (specs []string) { + for _, f := range p.Flow.Filters { + Fill(&f) + specs = append(specs, typedSpec(&f)) + } + return specs }}, - {"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error { opts["type"] = typ p.Flow.Format = &FormatConfig{} return Scan(opts, p.Flow.Format) + }, func(p *PipelineConfig) []string { + if p.Flow.Format == nil { + return nil + } + format := *p.Flow.Format + Fill(&format) + return []string{typedSpec(&format)} }}, // Naming a stage turns it on: the file defaults (pass, disabled) would not. - {"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { + {"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error { p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"} return Scan(opts, p.Flow.RateLimit) + }, func(p *PipelineConfig) []string { + if p.Flow.RateLimit == nil { + return nil + } + rate := *p.Flow.RateLimit + Fill(&rate) // the policy, whose default here is drop + return []string{formatSpec("", Values(&rate))} }}, - {"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { + {"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error { p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true} return Scan(opts, p.Flow.Heartbeat) + }, func(p *PipelineConfig) []string { + if p.Flow.Heartbeat == nil { + return nil + } + beat := *p.Flow.Heartbeat + Fill(&beat) + m := Values(&beat) + m["enabled"] = beat.Enabled + return []string{formatSpec("", m)} }}, } @@ -88,15 +130,28 @@ type pipelineSpec struct { // Spec is one pipeline flag: its name without dashes, and its value. type Spec struct{ Flag, Value string } -// SpecFlags names the pipeline flags, for the command-line parser. -func SpecFlags() []string { - flags := []string{"pipeline"} +// SpecFlag is a pipeline flag, for the parser and lw --schema: its name +// without dashes, its environment variable, which takes _1.._N when Many, and +// whether SPEC starts with TYPE +type SpecFlag struct { + Flag string `json:"flag"` + Variable string `json:"variable"` + Typed bool `json:"typed"` + Many bool `json:"many"` +} + +func SpecFlags() []SpecFlag { + flags := []SpecFlag{{"pipeline", specVariable("pipeline"), false, false}} for _, k := range specKinds { - flags = append(flags, k.flag) + flags = append(flags, SpecFlag{k.flag, specVariable(k.flag), k.typed, k.many}) } return flags } +func specVariable(flag string) string { + return "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(flag, "-", "_")) +} + // cliSpecs looks up the kind of each command-line spec. func cliSpecs(in []Spec) ([]pipelineSpec, error) { var specs []pipelineSpec @@ -121,12 +176,12 @@ func cliSpecs(in []Spec) ([]pipelineSpec, error) { // counts as unset, so container templates may leave optional ones blank. func envPipelineSpecs() []pipelineSpec { var specs []pipelineSpec - if name := os.Getenv("LOGWISP_PIPELINE"); name != "" { - specs = append(specs, pipelineSpec{"LOGWISP_PIPELINE", nil, name}) + if name := os.Getenv(specVariable("pipeline")); name != "" { + specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name}) } for i := range specKinds { kind := &specKinds[i] - base := "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(kind.flag, "-", "_")) + base := specVariable(kind.flag) type numbered struct { n uint64 name, spec string @@ -150,9 +205,18 @@ func envPipelineSpecs() []pipelineSpec { return specs } +// SpecPipelines builds the pipelines of command-line specs, as Load does +func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) { + s, err := cliSpecs(specs) + if err != nil { + return nil, err + } + return buildPipelines(s, isDir) +} + // buildPipelines turns specs, in order, into pipelines; specs before the first // --pipeline go to one named "cli". Every call returns fresh maps. -func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { +func buildPipelines(specs []pipelineSpec, isDir IsDir) ([]PipelineConfig, error) { var pipelines []PipelineConfig var seen map[*specKind]bool for _, s := range specs { @@ -181,7 +245,7 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value) } seen[s.kind] = true - if err := s.kind.apply(p, typ, opts); err != nil { + if err := s.kind.apply(p, typ, opts, isDir); err != nil { return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err) } } @@ -191,6 +255,104 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { return pipelines, nil } +// PipelineSpecs writes pipelines as the flags that build them: --pipeline, +// ids, flow stage defaults and heartbeat's enabled explicit, as some flags' +// defaults differ from a file's +func PipelineSpecs(pipelines []PipelineConfig) []Spec { + var specs []Spec + for _, p := range pipelines { + specs = append(specs, Spec{"pipeline", p.Name}) + p.Flow = cmp.Or(p.Flow, &FlowConfig{}) + for _, k := range specKinds { + if k.write != nil { + for _, value := range k.write(&p) { + specs = append(specs, Spec{k.flag, value}) + } + } + } + } + return specs +} + +// typedSpec writes a stage whose type key is its spec's TYPE +func typedSpec(stage any) string { + m := Values(stage) + typ, _ := m["type"].(string) + delete(m, "type") + return formatSpec(typ, m) +} + +// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a +// repeated key +func formatSpec(typ string, tables ...map[string]any) string { + var parts []string + if typ != "" { + parts = append(parts, escape(typ)) + } + var add func(prefix string, m map[string]any) + add = func(prefix string, m map[string]any) { + for _, k := range slices.Sorted(maps.Keys(m)) { + switch v := m[k].(type) { + case map[string]any: + add(prefix+k+".", v) + case []any: + for _, e := range v { + parts = append(parts, escape(prefix+k)+"="+escape(text(e))) + } + default: + parts = append(parts, escape(prefix+k)+"="+escape(text(v))) + } + } + } + for _, m := range tables { + add("", m) + } + return strings.Join(parts, ",") +} + +func text(v any) string { + if f, ok := v.(float64); ok { + return strconv.FormatFloat(f, 'g', -1, 64) + } + return fmt.Sprint(v) +} + +// escape is unescape's inverse: ',' and '=' escaped, and '\' where an escape +// would otherwise start, so a regex's \d stays as typed +func escape(s string) string { + var b strings.Builder + followed := s + "," // as a value is, unless it ends the spec + for i := range len(s) { + if s[i] == ',' || s[i] == '=' || escapes(followed, i) { + b.WriteByte('\\') + } + b.WriteByte(s[i]) + } + return b.String() +} + +// Values gives a table's keys that are not zero, as a spec sets them, for the +// composer to edit and Scan back +func Values(v any) map[string]any { + rv, m := reflect.ValueOf(v).Elem(), map[string]any{} + for _, o := range options(rv.Type()) { + switch f := rv.Field(o.index); { + case f.IsZero(): + case f.Kind() == reflect.String: + m[o.Name] = f.String() + case f.Kind() == reflect.Slice: + var list []any + for i := range f.Len() { + list = append(list, f.Index(i).Interface()) + } + m[o.Name] = list + default: + m[o.Name] = f.Interface() + } + } + return m +} + // useStdio makes a pipeline without a source read stdin, and one without a // sink write stdout, as a filter does. func useStdio(p *PipelineConfig) { @@ -225,7 +387,7 @@ func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err er case typed && key == "type": err = errors.New(`TYPE already sets "type"`) default: - err = setOption(opts, key, unescape(value)) + err = SetOption(opts, key, unescape(value)) } if err != nil { return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err) @@ -262,8 +424,8 @@ func escapes(s string, i int) bool { return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0 } -// setOption stores value at a dotted key; a second value turns it into a list. -func setOption(opts map[string]any, key, value string) error { +// SetOption stores value at a dotted key; a second value turns it into a list. +func SetOption(opts map[string]any, key, value string) error { segments := strings.Split(key, ".") if slices.Contains(segments, "") { return fmt.Errorf("empty key in %q", key) @@ -293,8 +455,7 @@ func setOption(opts map[string]any, key, value string) error { return nil } -// instanceID takes the id option; an unnamed instance gets the first of TYPE, -// TYPE_2, TYPE_3... that no earlier instance of its role holds. +// instanceID takes the id option, or FreeID for an unnamed instance func instanceID(typ string, opts map[string]any, taken []string) (string, error) { id, _ := opts["id"].(string) if _, set := opts["id"]; set && id == "" { @@ -304,9 +465,15 @@ func instanceID(typ string, opts map[string]any, taken []string) (string, error) if id != "" { return id, nil } - id = typ + return FreeID(typ, taken), nil +} + +// FreeID is the first of TYPE, TYPE_2, TYPE_3... that no earlier instance of +// its role holds +func FreeID(typ string, taken []string) string { + id := typ for n := 2; slices.Contains(taken, id); n++ { id = typ + "_" + strconv.Itoa(n) } - return id, nil + return id } diff --git a/internal/config/spec_test.go b/internal/config/spec_test.go index effa77e..a0b1082 100644 --- a/internal/config/spec_test.go +++ b/internal/config/spec_test.go @@ -213,3 +213,17 @@ func TestReloadKeepsSpecPipelines(t *testing.T) { t.Fatalf("reload after the file was removed: %v", err) } } + +// Written specs build the pipelines a file would: each flow stage with its +// defaults, so the rate limit keeps its file policy and the heartbeat its off +func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) { + p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{Rate: 5}, Heartbeat: &HeartbeatConfig{}}, + PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}}, + PluginSinks: []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}} + got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil) + p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{Rate: 5, Policy: "pass"}, + Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}} + if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) { + t.Fatalf("got %+v %v\nwant %+v", got, err, p) + } +} diff --git a/internal/config/validate.go b/internal/config/validate.go index 58ce005..7b4ee84 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -29,20 +29,6 @@ func ValidateConfig(cfg *Config) error { if cfg == nil { return fmt.Errorf("config is nil") } - - if len(cfg.Pipelines) == 0 { - return fmt.Errorf("no pipelines configured") - } - - // Reject duplicate pipeline names (service map is keyed by name) - names := make(map[string]struct{}, len(cfg.Pipelines)) - for i, p := range cfg.Pipelines { - if _, dup := names[p.Name]; dup { - return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name) - } - names[p.Name] = struct{}{} - } - if err := checkTags(reflect.ValueOf(cfg).Elem(), ""); err != nil { return err } @@ -57,8 +43,24 @@ func ValidateConfig(cfg *Config) error { } } + return ValidatePipelines(cfg.Pipelines) +} + +// ValidatePipelines is ValidateConfig's check of the pipelines +func ValidatePipelines(pipelines []PipelineConfig) error { + if len(pipelines) == 0 { + return fmt.Errorf("no pipelines configured") + } + // Reject duplicate pipeline names (service map is keyed by name) + names := make(map[string]struct{}, len(pipelines)) + for i, p := range pipelines { + if _, dup := names[p.Name]; dup { + return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name) + } + names[p.Name] = struct{}{} + } single := map[string]string{} // role and type of a single-instance plugin: its pipeline - for i, p := range cfg.Pipelines { + for i, p := range pipelines { if err := lconfig.NonEmpty(p.Name); err != nil { return fmt.Errorf("pipeline[%d].name: %w", i, err) } @@ -196,7 +198,7 @@ func (o *TLSOptions) Check(listener bool) error { other = "listener" } for _, opt := range options(v.Type()) { - if f := v.Field(opt.index); opt.side == other && !f.IsZero() && (f.Kind() != reflect.Slice || f.Len() > 0) { + if f := v.Field(opt.index); opt.Side == other && !f.IsZero() && (f.Kind() != reflect.Slice || f.Len() > 0) { return fmt.Errorf("tls: %s apply to %ss", and(sideKeys(v.Type(), other)), other) } }