From de87b9a008b2b37afc787707762241f3e01eb4c29684e45e67033a07fe28e1e8 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Tue, 13 Oct 2026 14:23:37 -0400 Subject: [PATCH] v0.35.2 lwconf, doc and deploy update --- cmd/lwconf/lwconf.js | 5 +- cmd/lwconf/lwconf.test.mjs | 16 ++++++- cmd/lwconf/main.go | 9 +++- config/logwisp.toml | 10 ++-- deploy/package/completion/_lw | 32 ++++++------- deploy/package/completion/lw.fish | 32 ++++++------- doc/architecture.md | 7 +-- doc/cli.md | 5 +- doc/configuration.md | 4 +- doc/networking.md | 4 +- internal/authz/authz_test.go | 2 +- internal/compose/compose.go | 35 ++++++++++---- internal/compose/compose_test.go | 24 ++++++++-- internal/config/config.go | 78 +++++++++++++++---------------- internal/config/preset.go | 75 ++++++++++++++++------------- internal/config/preset_test.go | 45 ++++++++++-------- internal/config/schema.go | 27 +++++------ internal/config/schema_test.go | 5 +- internal/config/validate.go | 20 +++++--- internal/tui/tui_test.go | 3 +- test/acl-test.sh | 2 +- 21 files changed, 255 insertions(+), 185 deletions(-) diff --git a/cmd/lwconf/lwconf.js b/cmd/lwconf/lwconf.js index 93b68a8..cfdda5d 100644 --- a/cmd/lwconf/lwconf.js +++ b/cmd/lwconf/lwconf.js @@ -31,8 +31,9 @@ function script(src, integrity) { } // Engine's calls take objects or JSON text and return the engine's answer, -// {value} or {error, path?, pipeline?, node?, key?}; dirs answers whether a -// preset path is a directory, {"/var/log/app.log": false} +// {value} or {error, path?, pipeline?, node?, key?, ask?}; ask is the path a +// preset names whose kind dirs must answer: pass dirs {[ask]: true for a +// directory, false for a file} and call again export class Engine { #api; constructor(api) { this.#api = api; } diff --git a/cmd/lwconf/lwconf.test.mjs b/cmd/lwconf/lwconf.test.mjs index 6a2d1b5..21ae60a 100644 --- a/cmd/lwconf/lwconf.test.mjs +++ b/cmd/lwconf/lwconf.test.mjs @@ -72,10 +72,22 @@ test("load takes an engine a server sends as plain bytes", async () => { } }); +// A path neither ending in '/' nor holding a glob is a question, in a preset +// or a pasted one, whose ask dirs answers +test("a preset path of unknown kind is asked, and dirs answers it", () => { + const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false }; + const asked = engine.preset("tail", path); + assert.deepEqual([asked.path, asked.ask, asked.value], ["path", "/var/log/app.log", undefined]); + assert.match(asked.error, /^path: "\/var\/log\/app\.log": a directory or a file\? end a directory with \//); + const pasted = engine.parse("lw -p tail:path=/var/log/app.log", "command"); + assert.deepEqual([pasted.path, pasted.ask], ["path", "/var/log/app.log"]); + const { value: pipelines } = engine.preset("tail", path, dirs); + assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" }); +}); + // The environment holds one pipeline, which a preset makes test("a preset's pipeline, a single file answered by dirs, reads back from every form", () => { const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false }; - assert.deepEqual(engine.preset("tail", path), { error: "path: a directory or a file pattern?", path: "path" }); const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs); assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" }); assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]); @@ -166,7 +178,7 @@ test("a refused call answers why and where, and the engine keeps answering", () [["edit", JSON.stringify([{ name: "p", plugin_sinks: [{ type: "tcp" }] }]), JSON.stringify({ op: "set", pipeline: 0, node: { role: "sink", id: "tcp" }, key: "port", values: ["x"] })], { error: /port: not an integer$/, path: "pipelines[0].plugin_sinks[tcp].config.port", pipeline: 0, key: "port" }], - [["emit", JSON.stringify([{ name: "p" }]), "command"], { error: /no sources defined/, path: "pipelines[0].plugin_sources" }], + [["emit", JSON.stringify([{ name: "p" }]), "command"], { error: /plugin_sources: add a source; a pipeline needs at least one\n/, path: "pipelines[0].plugin_sources" }], [["emit", "[]", "yaml"], { error: /no form "yaml"/ }], [["edit", "[]", JSON.stringify({ op: "add_pipeline", nmae: "p" })], { error: /unknown field "nmae"/ }], [["edit", "[]", JSON.stringify({ op: "rename" })], { error: /^no op "rename" \(valid: add, add_pipeline/ }], diff --git a/cmd/lwconf/main.go b/cmd/lwconf/main.go index e752159..12c140d 100644 --- a/cmd/lwconf/main.go +++ b/cmd/lwconf/main.go @@ -4,7 +4,7 @@ // globalThis.lwconf for lwconf.js, calls taking strings and returning JSON, // {"value"} or {"error"} with its path, one in a pipeline placed on its node // and key, any other a preset's or flag's key. Pipelines are compose's JSON -// form; dirs answers config.IsDir by path, {} leaving '/' and globs to tell. +// form; dirs answers config.IsDir by path; an error's ask is a path to answer. package main import ( @@ -146,7 +146,8 @@ var edits = map[string]func(*compose.Composition, op) (compose.Node, error){ } // problem is where a problem, or a call's refusal, lies as far as the engine -// knows: its path, and the pipeline, node and key Locate reads from it +// knows: its path, and the pipeline, node and key Locate reads from it; ask +// is a preset's path whose kind dirs must answer type problem struct { Error string `json:"error,omitempty"` Path string `json:"path,omitempty"` @@ -154,6 +155,7 @@ type problem struct { Node *compose.Node `json:"node,omitempty"` Key string `json:"key,omitempty"` Text string `json:"text,omitempty"` + Ask string `json:"ask,omitempty"` } func place(err error) problem { @@ -161,6 +163,9 @@ func place(err error) problem { if ke, ok := errors.AsType[*config.KeyError](err); ok { p.Path = ke.Key } + if pk, ok := errors.AsType[*config.PathKindError](err); ok { + p.Ask = pk.Path + } if pi, n, key, text, ok := compose.Locate(err); ok { p.Pipeline, p.Key, p.Text = &pi, key, text if n.Role != "" { diff --git a/config/logwisp.toml b/config/logwisp.toml index b74be60..6fc2e7a 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -93,7 +93,7 @@ # interval_ms = 1000 # At least 100 # include_timestamp = false # false: level only; the raw format shows neither # include_stats = false # Adds type, beat_count and, from the second beat, interval_ms -# format = "txt" # txt|json|raw; no effect: heartbeats use the flow's format +# format = "txt" # txt|json|raw; kept for older files: heartbeats use the flow's format ###============================================================================ ### TLS (a [...tls] table on every network source and sink) @@ -235,7 +235,7 @@ # id = "chain_in" # type = "tcp_chain" # [pipelines.plugin_sources.config] -# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only # port = 9440 # Required (example) # buffer_size = 1000 # max_connections = 0 # 0 = unlimited @@ -274,7 +274,7 @@ # id = "hchain_in" # type = "http_chain" # [pipelines.plugin_sources.config] -# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only # port = 9441 # Required (example) # ingest_path = "/ingest" # Must start with "/"; not "/auth" # buffer_size = 1000 @@ -353,7 +353,7 @@ # id = "http_out" # type = "http" # [pipelines.plugin_sinks.config] -# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only # port = 8081 # Required (example) # stream_path = "/stream" # Exact; starts with "/", clean (one trailing "/" allowed), none of { } % ? #; not /auth or under it # status_path = "/status" # Same rules; differs from stream_path @@ -399,7 +399,7 @@ # id = "tcp_out" # type = "tcp" # [pipelines.plugin_sinks.config] -# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only # port = 9090 # Required (example) # buffer_size = 1000 # client_buffer_size = 256 diff --git a/deploy/package/completion/_lw b/deploy/package/completion/_lw index ad0bc51..ca98c8d 100644 --- a/deploy/package/completion/_lw +++ b/deploy/package/completion/_lw @@ -94,31 +94,31 @@ preset:*) (( CURRENT -= 2 )) case ${words[1]} in pipe) _arguments -S \ - '--format=[raw|txt|json: entry format]:value:_default' \ + '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; tail) _arguments -S \ - '--format=[raw|txt|json: entry format]:value:_default' \ + '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \ '--from=[start|end: where reading a file starts]:value:_default' \ - '--path=[file, directory (its files) or glob to follow]:value:_default' \ + '--path=[a file, a directory (its files) or a glob such as /var/log/*.log, to follow]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; serve) _arguments -S \ '--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \ '--cert=[certificate file, for tls=files]:value:_default' \ '--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \ - '--format=[raw|txt|json: entry format]:value:_default' \ + '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \ '--from=[start|end: where reading a file starts]:value:_default' \ '--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \ '--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \ '--issuer_key=[CA key file, for tls=issuer]:value:_default' \ '--key=[key file, for tls=files]:value:_default' \ - '--listen=[HOST:PORT; [IPV6\]:PORT]:value:_default' \ + '--listen=[HOST:PORT to listen on: 127.0.0.1:8080 this machine only, 0.0.0.0:8080 every IPv4 interface, [::\]:8080 every IPv6 interface]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ - '--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ + '--path=[a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input]:value:_default' \ '--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \ '--tls=[off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \ - '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ + '(-u --user)'{-u=,--user=}'[the one SCRAM user readers log in as, in place of users]:value:_default' \ '--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \ - '--viewer=[true|false: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \ + '--viewer=[true|false: a login page, then the viewer, for browsers behind proxy; without user or users the viewer is on anyway]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; edge) _arguments -S \ '--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \ @@ -127,29 +127,29 @@ preset:*) '--key=[client key file]:value:_default' \ '--node=[origin label; default: this host'\''s name]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ - '--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ + '--path=[a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input]:value:_default' \ '--pin=[sha256//BASE64 of the aggregator'\''s key, which it logs with tls=self]:value:_default' \ '--server_name=[name the aggregator'\''s certificate carries; default: to'\''s host]:value:_default' \ - '--to=[the aggregator'\''s HOST:PORT; [IPV6\]:PORT]:value:_default' \ - '--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ - '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ + '--to=[the aggregator'\''s HOST:PORT, an IPv6 address in brackets: logs.example.net:9000, [2001:db8::1\]:9000]:value:_default' \ + '--transport=[tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)]:value:_default' \ + '(-u --user)'{-u=,--user=}'[SCRAM user this edge logs in to the aggregator as]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; aggregator) _arguments -S \ '--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \ '--cert=[certificate file, for tls=files]:value:_default' \ '--client_ca=[CA file that verifies edge certificates (mTLS)]:value:_default' \ '--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \ - '--format=[raw|txt|json: entry format]:value:_default' \ + '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \ '--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \ '--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \ '--issuer_key=[CA key file, for tls=issuer]:value:_default' \ '--key=[key file, for tls=files]:value:_default' \ - '--listen=[HOST:PORT; [IPV6\]:PORT; [::\] for IPv6 only]:value:_default' \ + '--listen=[HOST:PORT to listen on: 0.0.0.0:9000 every IPv4 interface, [::\]:9000 every IPv6 interface, 127.0.0.1:9000 this machine only]:value:_default' \ '--out=[directory for the received entries; default: standard output]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--tls=[self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \ - '--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ - '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ + '--transport=[tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)]:value:_default' \ + '(-u --user)'{-u=,--user=}'[the one SCRAM user edges log in as, in place of users]:value:_default' \ '--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; esac diff --git a/deploy/package/completion/lw.fish b/deploy/package/completion/lw.fish index b50baa7..64bf8c2 100644 --- a/deploy/package/completion/lw.fish +++ b/deploy/package/completion/lw.fish @@ -96,29 +96,29 @@ complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a serve -d 'Serve files or standard input live: a browser viewer at /, SSE at /stream' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a edge -d 'Forward files or standard input to an aggregator over TLS, authenticated' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a aggregator -d 'Receive from edges over TLS, authenticated, to files or standard output' -complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json: entry format' +complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry' complete -c lw -n '__lw_in preset pipe' -l help -o h -d 'print the flags' -complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json: entry format' +complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry' complete -c lw -n '__lw_in preset tail' -l from -r -F -d 'start|end: where reading a file starts' -complete -c lw -n '__lw_in preset tail' -l path -r -F -d 'file, directory (its files) or glob to follow' +complete -c lw -n '__lw_in preset tail' -l path -r -F -d 'a file, a directory (its files) or a glob such as /var/log/*.log, to follow' complete -c lw -n '__lw_in preset tail' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset serve' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them' complete -c lw -n '__lw_in preset serve' -l cert -r -F -d 'certificate file, for tls=files' complete -c lw -n '__lw_in preset serve' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them' -complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json: entry format' +complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry' complete -c lw -n '__lw_in preset serve' -l from -r -F -d 'start|end: where reading a file starts' complete -c lw -n '__lw_in preset serve' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them' complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer' complete -c lw -n '__lw_in preset serve' -l issuer_key -r -F -d 'CA key file, for tls=issuer' complete -c lw -n '__lw_in preset serve' -l key -r -F -d 'key file, for tls=files' -complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT' +complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT to listen on: 127.0.0.1:8080 this machine only, 0.0.0.0:8080 every IPv4 interface, [::]:8080 every IPv6 interface' complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' -complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' +complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input' complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them' complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files' -complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' +complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'the one SCRAM user readers log in as, in place of users' complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM' -complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true|false: the login page and viewer, behind proxy; without user or users the viewer is always on' +complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true|false: a login page, then the viewer, for browsers behind proxy; without user or users the viewer is on anyway' complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots' complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca' @@ -126,28 +126,28 @@ complete -c lw -n '__lw_in preset edge' -l from -r -F -d 'start|end: where readi complete -c lw -n '__lw_in preset edge' -l key -r -F -d 'client key file' complete -c lw -n '__lw_in preset edge' -l node -r -F -d 'origin label; default: this host\'s name' complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' -complete -c lw -n '__lw_in preset edge' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' +complete -c lw -n '__lw_in preset edge' -l path -r -F -d 'a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input' complete -c lw -n '__lw_in preset edge' -l pin -r -F -d 'sha256//BASE64 of the aggregator\'s key, which it logs with tls=self' complete -c lw -n '__lw_in preset edge' -l server_name -r -F -d 'name the aggregator\'s certificate carries; default: to\'s host' -complete -c lw -n '__lw_in preset edge' -l to -r -F -d 'the aggregator\'s HOST:PORT; [IPV6]:PORT' -complete -c lw -n '__lw_in preset edge' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' -complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' +complete -c lw -n '__lw_in preset edge' -l to -r -F -d 'the aggregator\'s HOST:PORT, an IPv6 address in brackets: logs.example.net:9000, [2001:db8::1]:9000' +complete -c lw -n '__lw_in preset edge' -l transport -r -F -d 'tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)' +complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user this edge logs in to the aggregator as' complete -c lw -n '__lw_in preset edge' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files' complete -c lw -n '__lw_in preset aggregator' -l client_ca -r -F -d 'CA file that verifies edge certificates (mTLS)' complete -c lw -n '__lw_in preset aggregator' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them' -complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json: entry format' +complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry' complete -c lw -n '__lw_in preset aggregator' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer' complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key file, for tls=issuer' complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls=files' -complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT; [::] for IPv6 only' +complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT to listen on: 0.0.0.0:9000 every IPv4 interface, [::]:9000 every IPv6 interface, 127.0.0.1:9000 this machine only' complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output' complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files' -complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' -complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' +complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)' +complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'the one SCRAM user edges log in as, in place of users' complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM' complete -c lw -n '__lw_in preset aggregator' -l help -o h -d 'print the flags' diff --git a/doc/architecture.md b/doc/architecture.md index 2f6c4f6..0f9a1e4 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -254,8 +254,8 @@ never a key: - `value`, `at` and `group` read a node's value and sort problems by node. The engine's calls take strings (`lwconf.js` writes objects as JSON) and -answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`, as -far as the engine places the refusal: +answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key", +"ask"}`, as far as the engine places the refusal: - `schema()`: what `lw --schema` prints. - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to @@ -273,7 +273,8 @@ far as the engine places the refusal: - `set` (`node`, `key`, `values`), `unset` (`node`, `key`) - `dirs` answers whether a path a preset names is a directory, `{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, one - with a glob a pattern, and any other is refused at `path`. + with a glob a pattern, and any other is refused at `path` with `ask` naming + it: answer it in `dirs` and call again. - Pipelines are a JSON list in the file's shape (`name`, `flow`, `plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node is `{role, id, index}`: a source's or sink's id, a filter's index. diff --git a/doc/cli.md b/doc/cli.md index f29faa8..9fd22c3 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -457,8 +457,9 @@ WebAssembly; `t` runs them under the screen and shows what flows. - Start: the pipelines `lw` would run with the same options (`-c FILE`, pipeline flags, `LOGWISP_SOURCE`, a discovered file); with none but the - built-in pipe, the preset menu (or `empty`). A pasted `lw` command line - replaces the pipelines, in the preset menu too. + built-in pipe, the preset menu (or `empty`). A pasted `lw` command line of + pipeline flags, or the `lw` part of `... | lw ...`, replaces the + pipelines, in the preset menu too. - Screen: the pipeline bar on top; SOURCES, FLOW and SINKS side by side from 64 columns, stacked below; each source and sink a tinted bar with its type, whether it listens or dials, and a summary; the four flow stages in diff --git a/doc/configuration.md b/doc/configuration.md index b2f4720..ae00ef2 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -172,8 +172,8 @@ Options, each as type and default: timestamp; the `raw` format shows neither. - `include_stats` (bool, `false`): add `type = "heartbeat"`, `beat_count` and, from the second beat, the measured `interval_ms` as fields. -- `format` (string, `txt`): `txt`, `json` or `raw`; no effect: heartbeats go - through the flow's `format`. +- `format` (string, `txt`): `txt`, `json` or `raw`, kept for older files: + heartbeats go through the flow's `format`. Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They are generated after the flow's filter and rate-limit stages, so filters do not diff --git a/doc/networking.md b/doc/networking.md index 3ff73d4..e69799b 100644 --- a/doc/networking.md +++ b/doc/networking.md @@ -209,8 +209,8 @@ preamble, headers, and entry encoding. - `PROXY header: …` — malformed, too long, or incomplete after 10 s. - `PROXY header from a peer outside proxy_from` — at the first read, after the connection opened: list the proxy in `proxy_from`. -- `acl: allow entry "...": not of the listener's family` at startup — an IPv4 - entry on an IPv6 `host`, or the reverse: each family's listener takes its own. +- `acl: allow entry "...": IPv4, and the listener's host is IPv6` at startup, + or the reverse: each family's listener takes its own. Behind `proxy_from` or `trusted_proxies`, `allow` and `deny` take both; `proxy_from` its own. diff --git a/internal/authz/authz_test.go b/internal/authz/authz_test.go index 6f0fbd5..d94a410 100644 --- a/internal/authz/authz_test.go +++ b/internal/authz/authz_test.go @@ -156,7 +156,7 @@ func TestNewValidation(t *testing.T) { {"trusted proxies on a chain source", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.ChainListener, HTTP, "only to the http sink"}, {"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to HTTP listeners"}, {"trusted proxies with certificate binding", listener(config.AuthOptions{TrustedProxies: proxies, Identity: "cn"}), f.serverTLS, config.Listener, HTTP, "TLS-terminating proxy"}, - {"malformed trusted proxy", listener(config.AuthOptions{TrustedProxies: []string{"localhost"}}), nil, config.Listener, HTTP, "neither an address"}, + {"malformed trusted proxy", listener(config.AuthOptions{TrustedProxies: []string{"localhost"}}), nil, config.Listener, HTTP, "host names and IPv6 zones are not accepted"}, {"trusted proxies on a dialer", dialer(config.AuthOptions{TrustedProxies: proxies}), &tls.Config{}, config.Dialer, HTTP, "only to listeners"}, } for _, tc := range tests { diff --git a/internal/compose/compose.go b/internal/compose/compose.go index b4a6ffe..d2707a0 100644 --- a/internal/compose/compose.go +++ b/internal/compose/compose.go @@ -61,13 +61,21 @@ func FromPreset(name string, values map[string]string, isDir config.IsDir) (*Com // FromCommandLine starts from a pasted lw command line, quoted as a POSIX // shell reads it; it takes pipeline flags only, at least one, as a composition -// holds no settings +// holds no settings. Of a line piped into lw it reads lw's part, whose +// pipelines read standard input. func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) { words, err := split(strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(line)) // pasted line ends if err != nil { return nil, err } - if len(words) > 0 && (words[0] == "lw" || strings.HasSuffix(words[0], "/lw")) { + lw := func(word string) bool { return word == "lw" || strings.HasSuffix(word, "/lw") } + for i := len(words) - 2; i >= 0; i-- { + if words[i] == "|" && lw(words[i+1]) { + words = words[i+1:] + break + } + } + if len(words) > 0 && lw(words[0]) { words = words[1:] } var specs []config.Spec @@ -77,8 +85,12 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) { name = "--preset" } flag, long := strings.CutPrefix(name, "--") - if !long || !slices.ContainsFunc(config.SpecFlags(), func(f config.SpecFlag) bool { return f.Flag == flag }) { - return nil, fmt.Errorf("%s: not a pipeline flag", words[i]) + if flags := config.SpecFlags(); !long || !slices.ContainsFunc(flags, func(f config.SpecFlag) bool { return f.Flag == flag }) { + var names []string + for _, f := range flags { + names = append(names, "--"+f.Flag) + } + return nil, fmt.Errorf("%s: not a pipeline flag; paste lw's %s only", words[i], strings.Join(names, ", ")) } if !inline { if i++; i == len(words) || strings.HasPrefix(words[i], "-") { @@ -205,8 +217,11 @@ func (c *Composition) RenamePipeline(i int, name string) error { // free refuses a name that is empty or held by a pipeline other than i func (c *Composition) free(name string, i int) error { - if j := slices.IndexFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }); name == "" || j >= 0 && j != i { - return fmt.Errorf("pipeline %q: a name no other pipeline has", name) + switch j := slices.IndexFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }); { + case name == "": + return errors.New("a pipeline needs a name; give it one") + case j >= 0 && j != i: + return fmt.Errorf("another pipeline is named %q; choose another name", name) } return nil } @@ -574,7 +589,7 @@ func (c *Composition) CommandLine() (string, error) { // line as an environment file holds them; the environment holds one pipeline func (c *Composition) Environment() (string, error) { if len(c.Pipelines) != 1 { - return "", fmt.Errorf("the environment holds one pipeline, not %d", len(c.Pipelines)) + return "", fmt.Errorf("the environment holds one pipeline, not %d; choose the command or file form, or remove pipelines down to one", len(c.Pipelines)) } specs, err := c.shellSpecs() if err != nil { @@ -632,9 +647,9 @@ type Form struct { } var forms = []Form{ - {"command", "lw's flags", (*Composition).CommandLine, FromCommandLine}, - {"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment, FromEnvironment}, - {"file", "a configuration file's pipelines", func(c *Composition) (string, error) { + {"command", "an lw command line", (*Composition).CommandLine, FromCommandLine}, + {"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment}, + {"file", "the pipelines of a configuration file", func(c *Composition) (string, error) { data, err := c.File() return string(data), err }, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }}, diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go index a07787f..f1ce454 100644 --- a/internal/compose/compose_test.go +++ b/internal/compose/compose_test.go @@ -115,6 +115,18 @@ func TestPastedEnvironmentReadsAsAShellRunsIt(t *testing.T) { } } +// Of a line piped into lw, lw's part is read, its pipeline reading standard +// input; a word that is no pipeline flag is refused, naming those to paste +func TestPastedCommandLineIsLwsPart(t *testing.T) { + c, err := FromCommandLine("tail -F app.log | lw --filter include:patterns=ERROR", nil) + if err != nil || c.Pipelines[0].PluginSources[0].Type != "console" || len(c.Pipelines[0].Flow.Filters) != 1 { + t.Fatalf("%v %v", c, err) + } + if _, err := FromCommandLine("lw -c lw.toml --sink null", nil); err == nil || !strings.HasPrefix(err.Error(), "-c: not a pipeline flag; paste lw's --pipeline, --preset, ") { + t.Fatalf("-c: %v", err) + } +} + // Locate places a problem by its path: a plugin's key, a key of its table, a // rule between its table's keys, a list entry, the plugin itself under an id // holding ']', a filter's list entry, a stage and the pipeline; and nothing @@ -192,8 +204,9 @@ func TestShellFormsRefuseControlCharacters(t *testing.T) { } } -// Set refuses a value of the wrong kind and leaves the key as it was; Unset -// returns a key to its default, dropping a table it leaves empty +// Set refuses a value of the wrong kind, a plugin's and a stage's alike, and +// leaves the key as it was; Unset returns a key to its default, dropping a +// table it leaves empty func TestSetKeepsAKeyItCannotTake(t *testing.T) { c := &Composition{} if err := c.AddPipeline("p"); err != nil { @@ -207,8 +220,11 @@ func TestSetKeepsAKeyItCannotTake(t *testing.T) { t.Fatal(err) } } - if err := c.Set(0, n, "port", "http"); err == nil || !strings.HasPrefix(err.Error(), "port: ") { - t.Fatalf("port http: %v", err) + for want, err := range map[string]error{"port: not an integer": c.Set(0, n, "port", "http"), + "interval_ms: not an integer": c.Set(0, beat, "interval_ms", "soon")} { + if err == nil || err.Error() != want { + t.Errorf("%v, want %s", err, want) + } } if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) || beat.IntervalMS != 1000 { diff --git a/internal/config/config.go b/internal/config/config.go index 9791b22..56e31e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,18 +75,18 @@ type FlowConfig struct { // HeartbeatConfig defines settings for periodic keep-alive or status messages type HeartbeatConfig struct { - Enabled bool `toml:"enabled" help:"send heartbeats"` + Enabled bool `toml:"enabled" help:"send heartbeats; false keeps these settings and sends none"` IntervalMS int64 `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"` IncludeTimestamp bool `toml:"include_timestamp" help:"stamp each heartbeat with its time"` IncludeStats bool `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"` - Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"no effect: heartbeats use the flow's format"` + Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"kept for older files; heartbeats are written in the flow's format"` } // FormatConfig is how a pipeline writes its entries type FormatConfig struct { - Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"entry format"` - Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"formatter flags"` - TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"Go time layout of timestamps"` + Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"raw writes each message as it came, txt a line with time and level, json one object an entry"` + Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"for experts, a sum of bits over the type's choice: 1 writes messages verbatim, 16 drops the time, 32 the level"` + TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"timestamp layout: the reference time 2006-01-02 15:04:05 written as timestamps should look, such as 02 Jan 15:04"` SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"` } @@ -102,10 +102,10 @@ const ( // RateLimitConfig defines the configuration for pipeline-level rate limiting type RateLimitConfig struct { - EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second the limit lets through"` - BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries let through at once before the rate applies"` - Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop entries over the rate, or pass them"` - MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries"` + EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second let through; unset, the stage limits nothing, sizes included"` + BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries that may pass in a burst before entries_per_second holds them back"` + Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop enforces entries_per_second and max_entry_size_bytes; pass lets every entry through"` + MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries; needs entries_per_second and policy drop"` } // FilterType represents the filter's behavior (include or exclude) @@ -174,31 +174,31 @@ type ConsoleSourceOptions struct { // TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting // NDJSON entries from upstream logwisp tcp_chain sinks type TCPChainSourceOptions struct { - TLS *TLSOptions `toml:"tls"` - Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` + TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"` + Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"connections at once"` ReadTimeoutMS int64 `toml:"read_timeout_ms" lw:"min=0,zero=none" help:"milliseconds a connection may idle"` HelloTimeoutMS int64 `toml:"hello_timeout_ms" default:"10000" lw:"min=1" help:"milliseconds for a peer's hello"` TrustNode bool `toml:"trust_node" default:"true" help:"keep the node label a peer declares; off, the label is its address"` - Auth *AuthOptions `toml:"auth"` - ACL *ACLOptions `toml:"acl"` + Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"` + ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"` } // HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting // NDJSON batches from upstream logwisp http_chain sinks type HTTPChainSourceOptions struct { - TLS *TLSOptions `toml:"tls"` - Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` + TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"` + Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` IngestPath string `toml:"ingest_path" default:"/ingest" lw:"hint=path" help:"path batches are posted to"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` MaxBodyBytes int64 `toml:"max_body_bytes" default:"8388608" lw:"min=1" help:"largest batch accepted"` ReadTimeoutMS int64 `toml:"read_timeout_ms" default:"30000" lw:"min=1" help:"milliseconds to read a request"` TrustNode bool `toml:"trust_node" default:"true" help:"keep the node label a peer declares; off, the label is its address"` - Auth *AuthOptions `toml:"auth"` - ACL *ACLOptions `toml:"acl"` + Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"` + ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"` } // --- Sinks --- @@ -228,7 +228,7 @@ type FileSinkOptions struct { Name string `toml:"name" lw:"required" help:"base name of the files"` MaxSizeMB int64 `toml:"max_size_mb" default:"100" lw:"min=1" help:"size at which a file rotates"` MaxTotalSizeMB int64 `toml:"max_total_size_mb" default:"1000" lw:"min=1" help:"size of all files, beyond which the oldest go"` - MinDiskFreeMB int64 `toml:"min_disk_free_mb" lw:"zero=none" help:"free space kept on the disk; negative means 100"` + MinDiskFreeMB int64 `toml:"min_disk_free_mb" lw:"zero=none" help:"megabytes kept free on the disk, the oldest .log files deleted for it; a negative value means 100"` RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0,zero=kept" help:"hours a .log file is kept; 0 keeps them"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` FlushIntervalMs int64 `toml:"flush_interval_ms" default:"100" lw:"min=1" help:"milliseconds between flushes"` @@ -236,8 +236,8 @@ type FileSinkOptions struct { // TCPSinkOptions defines settings for a TCP server sink type TCPSinkOptions struct { - TLS *TLSOptions `toml:"tls"` - Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` + TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"` + Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued for all clients"` ClientBufferSize int64 `toml:"client_buffer_size" default:"256" lw:"min=1" help:"entries queued per client"` @@ -245,14 +245,14 @@ type TCPSinkOptions struct { KeepAlive bool `toml:"keep_alive" default:"true" help:"TCP keep-alive"` KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms" default:"30000" lw:"min=1" help:"milliseconds idle before a keep-alive probe"` MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"clients at once"` - Auth *AuthOptions `toml:"auth"` - ACL *ACLOptions `toml:"acl"` + Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"` + ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"` } // HTTPSinkOptions defines settings for an HTTP SSE server sink type HTTPSinkOptions struct { - TLS *TLSOptions `toml:"tls"` - Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` + TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"` + Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` StreamPath string `toml:"stream_path" default:"/stream" lw:"hint=path" help:"path of the event stream"` StatusPath string `toml:"status_path" default:"/status" lw:"hint=path" help:"path of the status document"` @@ -263,14 +263,14 @@ type HTTPSinkOptions struct { ReplayLines int64 `toml:"replay_lines" lw:"min=0,zero=none" help:"recent entries a new stream gets first"` LoginPage bool `toml:"login_page" help:"scram behind auth.trusted_proxies only: a login page at /auth/login"` ViewerPage bool `toml:"viewer_page" help:"scram behind auth.trusted_proxies only: the viewer after a login; without scram it is always at /"` - Auth *AuthOptions `toml:"auth"` - ACL *ACLOptions `toml:"acl"` + Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"` + ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"` } // TCPChainSinkOptions defines settings for a stdlib TCP client forwarding // entries to a downstream logwisp tcp_chain source type TCPChainSinkOptions struct { - TLS *TLSOptions `toml:"tls"` + TLS *TLSOptions `toml:"tls" help:"TLS to the source it connects to"` Node string `toml:"node" lw:"zero=the host name" help:"origin label of the entries"` Host string `toml:"host" lw:"required,hint=host" help:"host of the tcp_chain source"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port of the tcp_chain source"` @@ -281,13 +281,13 @@ type TCPChainSinkOptions struct { BackoffMaxMS int64 `toml:"backoff_max_ms" default:"30000" lw:"min=1" help:"longest wait before reconnecting; below the first, the default"` KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms" default:"30000" lw:"min=1" help:"milliseconds idle before a keep-alive probe"` KeepAlive bool `toml:"keep_alive" default:"true" help:"TCP keep-alive"` - Auth *AuthOptions `toml:"auth"` + Auth *AuthOptions `toml:"auth" help:"how it authenticates with the source: scram logs in, mtls checks the source's identity"` } // HTTPChainSinkOptions defines settings for a stdlib HTTP client posting // NDJSON batches to a downstream logwisp http_chain source type HTTPChainSinkOptions struct { - TLS *TLSOptions `toml:"tls"` + TLS *TLSOptions `toml:"tls" help:"TLS to the source it connects to"` Node string `toml:"node" lw:"zero=the host name" help:"origin label of the entries"` Host string `toml:"host" lw:"required,hint=host" help:"host of the http_chain source"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port of the http_chain source"` @@ -299,7 +299,7 @@ type HTTPChainSinkOptions struct { RequestTimeoutMS int64 `toml:"request_timeout_ms" default:"10000" lw:"min=1" help:"milliseconds for a request: dial, write and response"` BackoffMinMS int64 `toml:"backoff_min_ms" default:"500" lw:"min=1" help:"first wait before retrying"` BackoffMaxMS int64 `toml:"backoff_max_ms" default:"30000" lw:"min=1" help:"longest wait before retrying; below the first, the default"` - Auth *AuthOptions `toml:"auth"` + Auth *AuthOptions `toml:"auth" help:"how it authenticates with the source: scram logs in, mtls checks the source's identity"` } // --- Auth Options --- @@ -312,14 +312,14 @@ type HTTPChainSinkOptions struct { type AuthOptions struct { Type string `toml:"type" default:"none" lw:"enum=none|mtls|scram" help:"how peers authenticate"` Identity string `toml:"identity" lw:"enum=cn|san_dns|san_uri|san_email,zero=cn under mtls; no binding under scram" help:"certificate field naming the peer; under scram it must equal the user"` - Allow []string `toml:"allow" help:"mtls: identities admitted; with allow_patterns empty, any the CA issued"` + Allow []string `toml:"allow" help:"mtls: identities admitted; with neither this nor allow_patterns set, any the CA issued"` AllowPatterns []string `toml:"allow_patterns" lw:"hint=regex" help:"mtls: RE2 patterns of identities admitted; anchor them"` - NodeBinding string `toml:"node_binding" lw:"listener,enum=none|assert|force,zero=force" help:"chain sources: how a peer's identity binds its node label; overrides trust_node"` - CredentialsFile string `toml:"credentials_file" lw:"listener,hint=file" help:"scram: verifiers, from lw auth add-user"` + NodeBinding string `toml:"node_binding" lw:"chain_listener,enum=none|assert|force,zero=force" help:"chain sources: force labels every entry with the peer's identity, assert refuses a peer declaring another node, none leaves it to trust_node"` + CredentialsFile string `toml:"credentials_file" lw:"listener,hint=file" help:"scram: file of the users who may log in, made by lw auth add-user"` // exp has whole seconds, a sub-second expires_in reads as unknown, and 10 s // leaves room for renewal ahead of expiry TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,http,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"` - TrustedProxies []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"` + TrustedProxies []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: addresses or CIDRs of the proxies that end browsers' TLS, through which browsers then log in"` Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"` PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"` } @@ -329,8 +329,8 @@ type AuthOptions struct { // the L4 proxies whose PROXY header names the client the rules then see; the // per-client limits count that client. ACLOptions.Check holds the rules. type ACLOptions struct { - Allow []string `toml:"allow" lw:"hint=cidr" help:"addresses admitted; empty, all deny does not list"` - Deny []string `toml:"deny" lw:"hint=cidr" help:"addresses refused"` + Allow []string `toml:"allow" lw:"hint=cidr" help:"addresses or CIDRs that may connect; unset, any that deny does not refuse"` + Deny []string `toml:"deny" lw:"hint=cidr" help:"addresses or CIDRs refused, even when allow lists them"` ProxyProtocol string `toml:"proxy_protocol" default:"off" lw:"enum=off|optional|required" help:"read a PROXY header from proxy_from"` ProxyFrom []string `toml:"proxy_from" lw:"hint=cidr" help:"proxies that may send a PROXY header"` MaxConnectionsPerClient int64 `toml:"max_connections_per_client" lw:"min=0,zero=unlimited" help:"connections a client may hold"` @@ -346,12 +346,12 @@ type ACLOptions struct { // may present cert_file and key_file. type TLSOptions struct { Enabled bool `toml:"enabled" help:"use TLS"` - CertFile string `toml:"cert_file" lw:"hint=file" help:"certificate presented; a dialer's for mTLS"` + CertFile string `toml:"cert_file" lw:"hint=file" help:"certificate file this end presents; a dialer's, for a listener with client_auth"` KeyFile string `toml:"key_file" lw:"hint=file" help:"key of cert_file"` SelfSigned bool `toml:"self_signed" lw:"listener" help:"a self-signed certificate on a key made at startup"` - IssuerCertFile string `toml:"issuer_cert_file" lw:"listener,hint=file" help:"CA certificate (lw tls ca) that issues the listener's at startup"` + IssuerCertFile string `toml:"issuer_cert_file" lw:"listener,hint=file" help:"CA certificate file (lw tls ca) that signs a certificate made at startup"` IssuerKeyFile string `toml:"issuer_key_file" lw:"listener,hint=file" help:"key of issuer_cert_file"` - Hosts []string `toml:"hosts" lw:"listener" help:"names and addresses a made certificate carries beyond host, this machine's and loopback"` + Hosts []string `toml:"hosts" lw:"listener" help:"more names and addresses for a certificate made at startup, which always carries host, this machine's name and localhost"` ClientAuth bool `toml:"client_auth" lw:"listener" help:"require and verify client certificates"` ClientCAFile string `toml:"client_ca_file" lw:"listener,hint=file" help:"CA bundle verifying client certificates"` CAFile string `toml:"ca_file" lw:"dialer,hint=file,zero=the system roots" help:"CA bundle verifying the server"` diff --git a/internal/config/preset.go b/internal/config/preset.go index b113064..4f70e56 100644 --- a/internal/config/preset.go +++ b/internal/config/preset.go @@ -63,10 +63,19 @@ func mirror[T any](p PresetParam, key string) PresetParam { // IsDir tells whether a preset's path is a directory, whose files it follows, // or a file pattern; a path ending in '/' is a directory without asking. Off // the target host it is the user's Answers, or nil: a path with a glob is then -// a pattern, and any other fails with ErrPathKind. +// a pattern, and any other fails at key path with a PathKindError. type IsDir func(path string) (bool, error) -var ErrPathKind error = &KeyError{"path", errors.New("a directory or a file pattern?")} +// ErrPathKind is what every PathKindError unwraps to +var ErrPathKind = errors.New("a directory or a file?") + +// PathKindError asks whether Path, which a preset names, is a directory +type PathKindError struct{ Path string } + +func (e *PathKindError) Error() string { + return fmt.Sprintf("%q: %v end a directory with / to follow its files", e.Path, ErrPathKind) +} +func (e *PathKindError) Unwrap() error { return ErrPathKind } // HostIsDir asks this host, a leading ~ being the home directory: a path that // does not exist is a pattern @@ -85,16 +94,16 @@ func Answers(dirs map[string]bool) IsDir { if strings.ContainsAny(path, `*?[\`) { return false, nil } - return false, ErrPathKind + return false, &KeyError{"path", &PathKindError{path}} } } // Keys several presets share var ( - pathParam = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"} + pathParam = PresetParam{Name: "path", Help: "a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input"} fromParam = mirror[FileSourceOptions](PresetParam{Name: "from", Default: "end", Help: "where reading a file starts"}, "from") formatParam = func(def string) PresetParam { - return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "entry format"}, "type") + return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "raw writes each message as it came, txt a line with time and level, json one object an entry"}, "type") } tlsParams = []PresetParam{ mirror[TLSOptions](PresetParam{Name: "cert", Help: "certificate file, for tls=files"}, "cert_file"), @@ -107,13 +116,13 @@ var ( mirror[ACLOptions](PresetParam{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, "allow"), mirror[ACLOptions](PresetParam{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, "deny"), } - transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp_chain or http_chain", Enum: transports()} + transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)", Enum: transports()} usersParam = func(who string) PresetParam { return mirror[AuthOptions](PresetParam{Name: "users", Help: "credentials file (lw auth add-user): " + who + " log in with SCRAM"}, "credentials_file") } - userParams = []PresetParam{ - {Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"}, - mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file"), + userParams = func(help string) []PresetParam { + return []PresetParam{{Name: "user", Help: help}, + mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file")} } ) @@ -146,7 +155,7 @@ var presets = []Preset{ return nil }}, {"tail", "Follow files to standard output, like tail -F", - []PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")}, + []PresetParam{{Name: "path", Required: true, Help: "a file, a directory (its files) or a glob such as /var/log/*.log, to follow"}, fromParam, formatParam("raw")}, func(p *PipelineConfig, v map[string]string, isDir IsDir) error { sources, err := pathOrStdin(v, isDir) p.PluginSources, p.PluginSinks = sources, pipeDefault("").PluginSinks @@ -155,12 +164,12 @@ var presets = []Preset{ }}, {"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream", slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"), - {Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"}, + {Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT to listen on: 127.0.0.1:8080 this machine only, 0.0.0.0:8080 every IPv4 interface, [::]:8080 every IPv6 interface"}, serveTLS, usersParam("readers"), - }, userParams, []PresetParam{ + }, userParams("the one SCRAM user readers log in as, in place of users"), []PresetParam{ mirror[AuthOptions](PresetParam{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, "trusted_proxies"), - mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "the login page and viewer, behind proxy; without user or users the viewer is always on"}, "viewer_page"), + mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "a login page, then the viewer, for browsers behind proxy; without user or users the viewer is on anyway"}, "viewer_page"), }, aclParams, tlsParams), func(p *PipelineConfig, v map[string]string, isDir IsDir) error { sources, err := pathOrStdin(v, isDir) @@ -177,11 +186,11 @@ var presets = []Preset{ case err != nil: return err case auth != nil && v["tls"] == "off" && v["proxy"] == "": - return fmt.Errorf("user and users need tls (self, issuer or files), or proxy") + return &KeyError{"tls", errors.New("user and users need self, issuer or files, or proxy")} case auth != nil: sink["auth"] = auth case v["proxy"] != "": - return fmt.Errorf("proxy needs user or users: browsers log in with SCRAM") + return &KeyError{"proxy", errors.New("needs user or users: browsers log in with SCRAM")} } if v["proxy"] != "" { auth["trusted_proxies"] = list(v["proxy"]) @@ -189,12 +198,12 @@ var presets = []Preset{ switch v["viewer"] { case "true": if v["proxy"] == "" { - return fmt.Errorf("viewer=true needs proxy, behind which browsers log in; without users the viewer is always on") + return &KeyError{"viewer", errors.New("true needs proxy, behind which browsers log in; without users the viewer is always on")} } sink["login_page"], sink["viewer_page"] = true, true case "false": default: - return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"]) + return &KeyError{"viewer", fmt.Errorf("must be true or false, got %q", v["viewer"])} } sink["replay_lines"] = int64(1000) // a browser opened late sees the recent entries p.Flow.Format = &FormatConfig{Type: v["format"]} @@ -203,12 +212,12 @@ var presets = []Preset{ }}, {"edge", "Forward files or standard input to an aggregator over TLS, authenticated", slices.Concat([]PresetParam{pathParam, fromParam, - {Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"}, + {Name: "to", Required: true, Help: "the aggregator's HOST:PORT, an IPv6 address in brackets: logs.example.net:9000, [2001:db8::1]:9000"}, transportParam, mirror[TLSOptions](PresetParam{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, "ca_file"), {Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"}, {Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"}, - }, userParams, []PresetParam{ + }, userParams("SCRAM user this edge logs in to the aggregator as"), []PresetParam{ mirror[TLSOptions](PresetParam{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, "cert_file"), mirror[TLSOptions](PresetParam{Name: "key", Help: "client key file"}, "key_file"), {Name: "node", Help: "origin label; default: this host's name"}, @@ -241,7 +250,7 @@ var presets = []Preset{ case v["cert"] != "" && v["key"] != "": auth = map[string]any{"type": "mtls"} default: - return fmt.Errorf("edge needs user, or cert and key: it never sends unauthenticated") + return &KeyError{"user", errors.New("required unless cert and key are set; an edge never sends unauthenticated")} } sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth} if v["node"] != "" { @@ -252,11 +261,11 @@ var presets = []Preset{ }}, {"aggregator", "Receive from edges over TLS, authenticated, to files or standard output", slices.Concat([]PresetParam{ - {Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"}, + {Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT to listen on: 0.0.0.0:9000 every IPv4 interface, [::]:9000 every IPv6 interface, 127.0.0.1:9000 this machine only"}, transportParam, aggregatorTLS, usersParam("edges"), - }, userParams, []PresetParam{ + }, userParams("the one SCRAM user edges log in as, in place of users"), []PresetParam{ mirror[TLSOptions](PresetParam{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, "client_ca_file"), mirror[FileSinkOptions](PresetParam{Name: "out", Help: "directory for the received entries; default: standard output"}, "directory"), formatParam("json"), @@ -283,7 +292,7 @@ var presets = []Preset{ case v["client_ca"] != "": src["auth"] = map[string]any{"type": "mtls"} default: - return fmt.Errorf("aggregator needs user, users or client_ca: it never receives unauthenticated") + return &KeyError{"users", errors.New("required unless user or client_ca is set; an aggregator never receives unauthenticated")} } p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src}) p.PluginSinks = pipeDefault("").PluginSinks @@ -331,13 +340,13 @@ func applyPreset(p *PipelineConfig, name string, values map[string][]string, isD case j < 0: return fmt.Errorf("preset %s has no key %q (valid: %s)", name, k, strings.Join(keys, ", ")) case len(each) > 1 && !r.Params[j].List: - return fmt.Errorf("preset %s key %q takes one value", name, k) + return &KeyError{k, errors.New("takes one value")} } v[k] = cmp.Or(strings.Join(each, ","), v[k]) } for _, k := range r.Params { if k.Required && v[k.Name] == "" { - return fmt.Errorf("preset %s needs %s", name, k.Name) + return &KeyError{k.Name, errors.New("must be set")} } } p.Name = cmp.Or(p.Name, name) @@ -404,20 +413,20 @@ func listener(v map[string]string, key PresetParam) (map[string]any, error) { mode := cmp.Or(v["tls"], key.Default) i := slices.IndexFunc(tlsModes, func(m tlsMode) bool { return m.name == mode }) if i < 0 || !slices.Contains(key.Enum, mode) { - return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(key.Enum, ", ")) + return nil, &KeyError{"tls", fmt.Errorf("must be one of %s, got %q", strings.Join(key.Enum, ", "), mode)} } m := tlsModes[i] for _, other := range tlsModes { for k := range other.needs { if _, needed := m.needs[k]; needed && v[k] == "" { - return nil, fmt.Errorf("tls=%s needs %s", mode, k) + return nil, &KeyError{k, fmt.Errorf("must be set for tls=%s", mode)} } else if !needed && v[k] != "" { - return nil, fmt.Errorf("%s does not apply to tls=%s", k, mode) + return nil, &KeyError{k, fmt.Errorf("does not apply to tls=%s", mode)} } } } if v["hosts"] != "" && !m.made { - return nil, fmt.Errorf("hosts does not apply to tls=%s", mode) + return nil, &KeyError{"hosts", fmt.Errorf("does not apply to tls=%s", mode)} } if mode == "off" { return opts, nil @@ -439,11 +448,11 @@ func listener(v map[string]string, key PresetParam) (map[string]any, error) { func scramAuth(v map[string]string) (map[string]any, error) { switch { case v["user"] != "" && v["users"] != "": - return nil, fmt.Errorf("user or users, not both") + return nil, &KeyError{"user", errors.New("set user or users, not both")} case v["user"] != "": return map[string]any{"type": "scram", "username": v["user"], "password_file": cmp.Or(v["password_file"], "/dev/tty")}, nil case v["password_file"] != "": - return nil, fmt.Errorf("password_file needs user") + return nil, &KeyError{"password_file", errors.New("needs user")} case v["users"] != "": return map[string]any{"type": "scram", "credentials_file": v["users"]}, nil } @@ -454,7 +463,7 @@ func splitAddr(key, addr string) (string, int64, error) { host, p, err := net.SplitHostPort(addr) port, perr := strconv.ParseUint(p, 10, 16) if err != nil || perr != nil || port == 0 { - return "", 0, fmt.Errorf("%s %q: want HOST:PORT, an IPv6 address in brackets", key, addr) + return "", 0, &KeyError{key, fmt.Errorf("must be HOST:PORT, an IPv6 address in brackets as in [::1]:9000, got %q", addr)} } return host, int64(port), nil } @@ -462,7 +471,7 @@ func splitAddr(key, addr string) (string, int64, error) { // chainType is a transport's chain plugin type, both a source and a sink func chainType(transport string) (string, error) { if !slices.Contains(transports(), transport) { - return "", fmt.Errorf("transport %q (valid: %s)", transport, strings.Join(transports(), ", ")) + return "", &KeyError{"transport", fmt.Errorf("must be one of %s, got %q", strings.Join(transports(), ", "), transport)} } return transport + "_chain", nil } diff --git a/internal/config/preset_test.go b/internal/config/preset_test.go index d7926ac..bebb35b 100644 --- a/internal/config/preset_test.go +++ b/internal/config/preset_test.go @@ -39,34 +39,39 @@ func TestPresetStartsAndNamesItsPipeline(t *testing.T) { } // Keys are checked against the preset's own, required ones enforced, and -// the edge and aggregator refuse to run without authentication. +// the edge and aggregator refuse to run without authentication; a refusal of +// one key is a KeyError at it, the others name the preset. func TestPresetKeysAreChecked(t *testing.T) { isolateConfig(t) for spec, want := range map[string]string{ "nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`, "tail:path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`, - "tail": "preset tail needs path", + "tail": "path: must be set", "tail:path=x,format.type=json": "preset keys do not nest", - "tail:path=x,path=y": `key "path" takes one value`, - "serve:listen=8080": `listen "8080": want HOST:PORT`, - "serve:tls=self,cert=c": "cert does not apply to tls=self", - "serve:tls=files,cert=c": "tls=files needs key", - "serve:viewer=yes": `viewer "yes"`, - "serve:proxy=127.0.0.1": "proxy needs user or users", - "serve:viewer=true": "viewer=true needs proxy", - "serve:user=u": "user and users need tls (self, issuer or files), or proxy", - "serve:tls=self,user=u,users=f": "user or users, not both", - "edge:to=agg:9000": "it never sends unauthenticated", - "edge:to=agg:9000,password_file=p": "password_file needs user", - "edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`, - "aggregator:users=u,tls=off": `tls "off" (valid: `, - "aggregator:user=u,users=f": "user or users, not both", - "aggregator": "it never receives unauthenticated", + "tail:path=x,path=y": "path: takes one value", + "serve:listen=8080": `listen: must be HOST:PORT, an IPv6 address in brackets as in [::1]:9000, got "8080"`, + "serve:tls=self,cert=c": "cert: does not apply to tls=self", + "serve:tls=files,cert=c": "key: must be set for tls=files", + "serve:viewer=yes": `viewer: must be true or false, got "yes"`, + "serve:proxy=127.0.0.1": "proxy: needs user or users", + "serve:viewer=true": "viewer: true needs proxy", + "serve:user=u": "tls: user and users need self, issuer or files, or proxy", + "serve:tls=self,user=u,users=f": "user: set user or users, not both", + "edge:to=agg:9000": "user: required unless cert and key are set; an edge never sends unauthenticated", + "edge:to=agg:9000,password_file=p": "password_file: needs user", + "edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport: must be one of tcp, http, got "udp"`, + "aggregator:users=u,tls=off": `tls: must be one of self, issuer, files, got "off"`, + "aggregator:user=u,users=f": "user: set user or users, not both", + "aggregator": "users: required unless user or client_ca is set; an aggregator never receives unauthenticated", } { _, err := Load(Args{Specs: specs("preset", spec)}) if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("%s: err = %v, want %s", spec, err, want) } + ke, keyed := errors.AsType[*KeyError](err) + if keyed == strings.Contains(want, "preset") || keyed && !strings.HasPrefix(want, ke.Key+": ") { + t.Errorf("%s: %v placed at %v", spec, err, ke) + } } } @@ -127,7 +132,7 @@ func TestPresetUserWithoutPasswordFileAsks(t *testing.T) { // A preset path ending in '/' is a directory, there yet or not; any other is // what isDir answers, a directory named with brackets too: this host's, // reading ~ as home, or off it the user's answer, a single file too; without -// one it is a pattern when it holds a glob, and otherwise ErrPathKind asks +// one it is a pattern when it holds a glob, and otherwise PathKindError asks func TestPresetPathKind(t *testing.T) { home := t.TempDir() t.Setenv("HOME", home) @@ -145,8 +150,8 @@ func TestPresetPathKind(t *testing.T) { {"~/logs", HostIsDir, "~/logs *"}, {home + "/later/", HostIsDir, home + "/later/ *"}} { p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir) if c.want == "" { - if !errors.Is(err, ErrPathKind) { - t.Errorf("%s: %v, want ErrPathKind", c.path, err) + if pk, ok := errors.AsType[*PathKindError](err); !errors.Is(err, ErrPathKind) || !ok || pk.Path != c.path { + t.Errorf("%s: %v, want ErrPathKind asking about it", c.path, err) } continue } diff --git a/internal/config/schema.go b/internal/config/schema.go index f334126..daf62e0 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -35,7 +35,7 @@ type Plugin struct { Type string `json:"type"` Side Side `json:"side"` HTTP bool `json:"http"` // speaks HTTP: takes bearer tokens and request limits - Single bool `json:"single"` // one instance per process + Single bool `json:"single"` // one instance per process, as standard input is one Summary string `json:"summary"` options reflect.Type } @@ -45,15 +45,15 @@ var plugins = []Plugin{ {"source", "file", Local, false, false, "Follow the files of a directory that match a pattern", reflect.TypeFor[FileSourceOptions]()}, {"source", "random", Local, false, false, "Generated entries, to check a deployment before real logs", reflect.TypeFor[RandomSourceOptions]()}, {"source", "null", Local, false, false, "Nothing; holds a source's place until one is chosen", reflect.TypeFor[NullSourceOptions]()}, - {"source", "tcp_chain", ChainListener, false, false, "Entries from tcp_chain sinks of other instances", reflect.TypeFor[TCPChainSourceOptions]()}, - {"source", "http_chain", ChainListener, true, false, "Entry batches from http_chain sinks of other instances", reflect.TypeFor[HTTPChainSourceOptions]()}, + {"source", "tcp_chain", ChainListener, false, false, "Receive entries from other nodes' tcp_chain sinks, each over one lasting TCP connection", reflect.TypeFor[TCPChainSourceOptions]()}, + {"source", "http_chain", ChainListener, true, false, "Receive entries from other nodes' http_chain sinks, as batches each posts over HTTP", reflect.TypeFor[HTTPChainSourceOptions]()}, {"sink", "console", Local, false, false, "Standard output or standard error", reflect.TypeFor[ConsoleSinkOptions]()}, {"sink", "file", Local, false, false, "Rotating files", reflect.TypeFor[FileSinkOptions]()}, {"sink", "null", Local, false, false, "Discards; holds a sink's place until one is chosen", reflect.TypeFor[NullSinkOptions]()}, {"sink", "tcp", Listener, false, false, "Serve entries to TCP clients, one per line", reflect.TypeFor[TCPSinkOptions]()}, {"sink", "http", Listener, true, false, "Serve entries as server-sent events, with a browser viewer", reflect.TypeFor[HTTPSinkOptions]()}, - {"sink", "tcp_chain", Dialer, false, false, "Forward entries to a tcp_chain source", reflect.TypeFor[TCPChainSinkOptions]()}, - {"sink", "http_chain", Dialer, true, false, "Post entry batches to an http_chain source", reflect.TypeFor[HTTPChainSinkOptions]()}, + {"sink", "tcp_chain", Dialer, false, false, "Forward entries to another node's tcp_chain source over one lasting TCP connection", reflect.TypeFor[TCPChainSinkOptions]()}, + {"sink", "http_chain", Dialer, true, false, "Forward entries to another node's http_chain source in batches, an HTTP POST each", reflect.TypeFor[HTTPChainSinkOptions]()}, } // Plugins lists the catalogue @@ -64,12 +64,9 @@ func Plugins() []Plugin { return slices.Clone(plugins) } func (p Plugin) Keys() []Key { return p.applying(keys(p.options)) } func (p Plugin) applying(all []Key) []Key { - side, out := "listener", []Key{} - if p.Side == Dialer { - side = "dialer" - } + out := []Key{} for _, k := range all { - if (k.Side == "" || k.Side == side) && (!k.HTTP || p.HTTP) { + if (k.Side == "" || k.Side == sideNames[p.Side] || k.Side == "listener" && p.Side == ChainListener) && (!k.HTTP || p.HTTP) { if k.Keys != nil { k.Keys = p.applying(k.Keys) } @@ -99,10 +96,10 @@ func (e *KeyError) Error() string { return e.Key + ": " + e.Err.Error() } func (e *KeyError) Unwrap() error { return e.Err } // option is one key of an options table, from its field's tags: toml names -// it, default: holds the default as typed on a command line, help: says what -// it does, and lw: lists its rules: required, enum=a|b, min=N, max=N, -// zero=MEANING (zero is a value, not the default), hint=KIND, listener or -// dialer, and http (only plugins that speak HTTP take it). +// it, default: holds the default as a command line types it, help: says what +// it does, lw: lists its rules: required, enum=a|b, min=N, max=N, zero=MEANING +// (a value, not the default), hint=KIND, a side (listener, a chain source's +// too; chain_listener; dialer), and http (only plugins that speak HTTP take it). type option struct { Key index int @@ -125,7 +122,7 @@ func options(t reflect.Type) []option { case "": case "required": o.Required = true - case "listener", "dialer": + case "listener", "chain_listener", "dialer": o.Side = name case "http": o.HTTP = true diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index 0536119..615f7a0 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -173,7 +173,8 @@ func TestLoadReportsEveryProblemALine(t *testing.T) { } // A plugin lists the table keys that apply to it: those of its side, a chain -// source's being a listener's, and the HTTP ones only when it speaks HTTP +// source's being a listener's and its own, and the HTTP ones only when it +// speaks HTTP func TestTableKeysFollowThePlugin(t *testing.T) { has := func(role, typ, path string) bool { p, _ := LookupPlugin(role, typ) @@ -190,7 +191,7 @@ func TestTableKeysFollowThePlugin(t *testing.T) { for path, want := range map[string]bool{ "sink tcp_chain tls.pin_sha256": true, "sink tcp_chain tls.self_signed": false, "source tcp_chain tls.self_signed": true, "source tcp_chain tls.pin_sha256": false, - "source tcp_chain auth.node_binding": true, "sink tcp_chain auth.credentials_file": false, + "source tcp_chain auth.node_binding": true, "sink http auth.node_binding": false, "sink tcp_chain auth.credentials_file": false, "sink tcp acl.requests_per_second_per_client": false, "sink tcp auth.token_lifetime_ms": false, "sink http acl.requests_per_second_per_client": true, "sink http auth.trusted_proxies": true, } { diff --git a/internal/config/validate.go b/internal/config/validate.go index 6875e66..99598b0 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -59,10 +59,10 @@ func ValidatePipelines(pipelines []PipelineConfig) error { } names[p.Name] = true if len(p.PluginSources) == 0 { - at(path+".plugin_sources", errors.New("no sources defined")) + at(path+".plugin_sources", errors.New("add a source; a pipeline needs at least one")) } if len(p.PluginSinks) == 0 { - at(path+".plugin_sinks", errors.New("no sinks defined")) + at(path+".plugin_sinks", errors.New("add a sink; a pipeline needs at least one")) } if err := checkFlow(p.Flow, path+".flow"); err != nil { errs = append(errs, err) @@ -77,7 +77,7 @@ func ValidatePipelines(pipelines []PipelineConfig) error { case !ok: at(path+".type", fmt.Errorf("unknown %s type %q", role, typ)) case plugin.Single && single[role+typ] != "": - at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it", typ, role, single[role+typ])) + at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it, as lw has one standard input; give this pipeline another %s", typ, role, single[role+typ], role)) default: ids[list+id] = true if plugin.Single { @@ -414,15 +414,17 @@ func ParsePrefix(entry, network string) (netip.Prefix, error) { if err != nil { addr, aerr := netip.ParseAddr(entry) if aerr != nil || addr.Zone() != "" { - return netip.Prefix{}, fmt.Errorf("neither an address nor a CIDR (and no zone)") + return netip.Prefix{}, errors.New("not an IP address or CIDR; host names and IPv6 zones are not accepted") } prefix = netip.PrefixFrom(addr, addr.BitLen()) } switch a := prefix.Addr(); { case a.Is4In6(): return netip.Prefix{}, fmt.Errorf("IPv4-mapped; write it as IPv4") - case network == "tcp4" && !a.Is4(), network == "tcp6" && a.Is4(): - return netip.Prefix{}, fmt.Errorf("not of the listener's family (%s)", network) + case network == "tcp4" && !a.Is4(): + return netip.Prefix{}, errors.New("IPv6, and the listener's host is IPv4; write an IPv4 address or CIDR") + case network == "tcp6" && a.Is4(): + return netip.Prefix{}, errors.New("IPv4, and the listener's host is IPv6; write an IPv6 address or CIDR") } return prefix, nil } @@ -520,11 +522,15 @@ func (c *FilterConfig) Check() error { } // Scan decodes a flow stage's command-line map into target, rejecting keys -// target does not declare: a misspelled key must fail, not be dropped. +// target does not declare: a misspelled key must fail, not be dropped. Its +// values take their kinds as a plugin's do, a wrong one refused at its key. func Scan(configMap map[string]any, target any) error { if err := checkKeys(configMap, reflect.TypeOf(target), ""); err != nil { return err } + if err := coerce(reflect.TypeOf(target).Elem(), configMap, ""); err != nil { + return err + } return lconfig.ScanMap(configMap, target) } diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index e84c25d..eddc23b 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -811,7 +811,8 @@ func TestPresetFormShowsTheChoices(t *testing.T) { ps := config.Presets() tail := ps[slices.IndexFunc(ps, func(p config.Preset) bool { return p.Name == "tail" })].Params format := tail[slices.IndexFunc(tail, func(p config.PresetParam) bool { return p.Name == "format" })] - if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.Contains(screen, format.Usage()) || len(format.Enum) == 0 { + note := strings.Join(format.Enum, "|") + ": " + strings.Fields(format.Help)[0] // the help wraps + if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.Contains(screen, note) || len(format.Enum) == 0 { t.Fatalf("format's choices not shown:\n%s", screen) } } diff --git a/test/acl-test.sh b/test/acl-test.sh index 1afb91a..713deec 100755 --- a/test/acl-test.sh +++ b/test/acl-test.sh @@ -96,7 +96,7 @@ section "Rules" rejects() { ! lw --check --source null --sink "$1" >/dev/null 2>"$LOG/check.err" && grep -q -- "$2" "$LOG/check.err" && echo 1 || echo 0; } # warns SINK PATTERN: lw --check builds SINK and WARNs PATTERN warns() { lw --check --source null --sink "$1" 2>&1 >/dev/null | grep ' WARN ' | grep -q -- "$2" && echo 1 || echo 0; } -check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp:host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")" +check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp:host=::1,port=15881,acl.allow=10.0.0.0/8' "IPv4, and the listener's host is IPv6")" check "an acl on a dialer is refused" "$(rejects 'tcp_chain:host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')" check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp:host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')" check "a deny list alone on a wildcard listener warns" "$(warns 'tcp:port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')"