v0.35.2 lwconf, doc and deploy update
Package / arch (push) Canceled after 0s
Package / user (push) Canceled after 0s
Package / deb (push) Canceled after 0s
Package / freebsd (push) Canceled after 0s
Package / image (push) Canceled after 0s
Test / test (push) Canceled after 0s
Test / e2e (push) Canceled after 0s

This commit is contained in:
lixen committed 2026-10-13 14:23:37 -04:00
1 parent 0b8701ec13
commit de87b9a008
21 files changed
+255 -185

No files matched your search

+3 -2
View File
@@ -31,8 +31,9 @@ function script(src, integrity) {
} }
// Engine's calls take objects or JSON text and return the engine's answer, // Engine's calls take objects or JSON text and return the engine's answer,
// {value} or {error, path?, pipeline?, node?, key?}; dirs answers whether a // {value} or {error, path?, pipeline?, node?, key?, ask?}; ask is the path a
// preset path is a directory, {"/var/log/app.log": false} // preset names whose kind dirs must answer: pass dirs {[ask]: true for a
// directory, false for a file} and call again
export class Engine { export class Engine {
#api; #api;
constructor(api) { this.#api = api; } constructor(api) { this.#api = api; }
+14 -2
View File
@@ -72,10 +72,22 @@ test("load takes an engine a server sends as plain bytes", async () => {
} }
}); });
// A path neither ending in '/' nor holding a glob is a question, in a preset
// or a pasted one, whose ask dirs answers
test("a preset path of unknown kind is asked, and dirs answers it", () => {
const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false };
const asked = engine.preset("tail", path);
assert.deepEqual([asked.path, asked.ask, asked.value], ["path", "/var/log/app.log", undefined]);
assert.match(asked.error, /^path: "\/var\/log\/app\.log": a directory or a file\? end a directory with \//);
const pasted = engine.parse("lw -p tail:path=/var/log/app.log", "command");
assert.deepEqual([pasted.path, pasted.ask], ["path", "/var/log/app.log"]);
const { value: pipelines } = engine.preset("tail", path, dirs);
assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" });
});
// The environment holds one pipeline, which a preset makes // The environment holds one pipeline, which a preset makes
test("a preset's pipeline, a single file answered by dirs, reads back from every form", () => { test("a preset's pipeline, a single file answered by dirs, reads back from every form", () => {
const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false }; const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false };
assert.deepEqual(engine.preset("tail", path), { error: "path: a directory or a file pattern?", path: "path" });
const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs); const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs);
assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" }); assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" });
assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]); assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]);
@@ -166,7 +178,7 @@ test("a refused call answers why and where, and the engine keeps answering", ()
[["edit", JSON.stringify([{ name: "p", plugin_sinks: [{ type: "tcp" }] }]), [["edit", JSON.stringify([{ name: "p", plugin_sinks: [{ type: "tcp" }] }]),
JSON.stringify({ op: "set", pipeline: 0, node: { role: "sink", id: "tcp" }, key: "port", values: ["x"] })], JSON.stringify({ op: "set", pipeline: 0, node: { role: "sink", id: "tcp" }, key: "port", values: ["x"] })],
{ error: /port: not an integer$/, path: "pipelines[0].plugin_sinks[tcp].config.port", pipeline: 0, key: "port" }], { error: /port: not an integer$/, path: "pipelines[0].plugin_sinks[tcp].config.port", pipeline: 0, key: "port" }],
[["emit", JSON.stringify([{ name: "p" }]), "command"], { error: /no sources defined/, path: "pipelines[0].plugin_sources" }], [["emit", JSON.stringify([{ name: "p" }]), "command"], { error: /plugin_sources: add a source; a pipeline needs at least one\n/, path: "pipelines[0].plugin_sources" }],
[["emit", "[]", "yaml"], { error: /no form "yaml"/ }], [["emit", "[]", "yaml"], { error: /no form "yaml"/ }],
[["edit", "[]", JSON.stringify({ op: "add_pipeline", nmae: "p" })], { error: /unknown field "nmae"/ }], [["edit", "[]", JSON.stringify({ op: "add_pipeline", nmae: "p" })], { error: /unknown field "nmae"/ }],
[["edit", "[]", JSON.stringify({ op: "rename" })], { error: /^no op "rename" \(valid: add, add_pipeline/ }], [["edit", "[]", JSON.stringify({ op: "rename" })], { error: /^no op "rename" \(valid: add, add_pipeline/ }],
+7 -2
View File
@@ -4,7 +4,7 @@
// globalThis.lwconf for lwconf.js, calls taking strings and returning JSON, // globalThis.lwconf for lwconf.js, calls taking strings and returning JSON,
// {"value"} or {"error"} with its path, one in a pipeline placed on its node // {"value"} or {"error"} with its path, one in a pipeline placed on its node
// and key, any other a preset's or flag's key. Pipelines are compose's JSON // and key, any other a preset's or flag's key. Pipelines are compose's JSON
// form; dirs answers config.IsDir by path, {} leaving '/' and globs to tell. // form; dirs answers config.IsDir by path; an error's ask is a path to answer.
package main package main
import ( import (
@@ -146,7 +146,8 @@ var edits = map[string]func(*compose.Composition, op) (compose.Node, error){
} }
// problem is where a problem, or a call's refusal, lies as far as the engine // problem is where a problem, or a call's refusal, lies as far as the engine
// knows: its path, and the pipeline, node and key Locate reads from it // knows: its path, and the pipeline, node and key Locate reads from it; ask
// is a preset's path whose kind dirs must answer
type problem struct { type problem struct {
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
Path string `json:"path,omitempty"` Path string `json:"path,omitempty"`
@@ -154,6 +155,7 @@ type problem struct {
Node *compose.Node `json:"node,omitempty"` Node *compose.Node `json:"node,omitempty"`
Key string `json:"key,omitempty"` Key string `json:"key,omitempty"`
Text string `json:"text,omitempty"` Text string `json:"text,omitempty"`
Ask string `json:"ask,omitempty"`
} }
func place(err error) problem { func place(err error) problem {
@@ -161,6 +163,9 @@ func place(err error) problem {
if ke, ok := errors.AsType[*config.KeyError](err); ok { if ke, ok := errors.AsType[*config.KeyError](err); ok {
p.Path = ke.Key p.Path = ke.Key
} }
if pk, ok := errors.AsType[*config.PathKindError](err); ok {
p.Ask = pk.Path
}
if pi, n, key, text, ok := compose.Locate(err); ok { if pi, n, key, text, ok := compose.Locate(err); ok {
p.Pipeline, p.Key, p.Text = &pi, key, text p.Pipeline, p.Key, p.Text = &pi, key, text
if n.Role != "" { if n.Role != "" {
+5 -5
View File
@@ -93,7 +93,7 @@
# interval_ms = 1000 # At least 100 # interval_ms = 1000 # At least 100
# include_timestamp = false # false: level only; the raw format shows neither # include_timestamp = false # false: level only; the raw format shows neither
# include_stats = false # Adds type, beat_count and, from the second beat, interval_ms # include_stats = false # Adds type, beat_count and, from the second beat, interval_ms
# format = "txt" # txt|json|raw; no effect: heartbeats use the flow's format # format = "txt" # txt|json|raw; kept for older files: heartbeats use the flow's format
###============================================================================ ###============================================================================
### TLS (a [...tls] table on every network source and sink) ### TLS (a [...tls] table on every network source and sink)
@@ -235,7 +235,7 @@
# id = "chain_in" # id = "chain_in"
# type = "tcp_chain" # type = "tcp_chain"
# [pipelines.plugin_sources.config] # [pipelines.plugin_sources.config]
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only # host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only
# port = 9440 # Required (example) # port = 9440 # Required (example)
# buffer_size = 1000 # buffer_size = 1000
# max_connections = 0 # 0 = unlimited # max_connections = 0 # 0 = unlimited
@@ -274,7 +274,7 @@
# id = "hchain_in" # id = "hchain_in"
# type = "http_chain" # type = "http_chain"
# [pipelines.plugin_sources.config] # [pipelines.plugin_sources.config]
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only # host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only
# port = 9441 # Required (example) # port = 9441 # Required (example)
# ingest_path = "/ingest" # Must start with "/"; not "/auth" # ingest_path = "/ingest" # Must start with "/"; not "/auth"
# buffer_size = 1000 # buffer_size = 1000
@@ -353,7 +353,7 @@
# id = "http_out" # id = "http_out"
# type = "http" # type = "http"
# [pipelines.plugin_sinks.config] # [pipelines.plugin_sinks.config]
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only # host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only
# port = 8081 # Required (example) # port = 8081 # Required (example)
# stream_path = "/stream" # Exact; starts with "/", clean (one trailing "/" allowed), none of { } % ? #; not /auth or under it # stream_path = "/stream" # Exact; starts with "/", clean (one trailing "/" allowed), none of { } % ? #; not /auth or under it
# status_path = "/status" # Same rules; differs from stream_path # status_path = "/status" # Same rules; differs from stream_path
@@ -399,7 +399,7 @@
# id = "tcp_out" # id = "tcp_out"
# type = "tcp" # type = "tcp"
# [pipelines.plugin_sinks.config] # [pipelines.plugin_sinks.config]
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only # host = "0.0.0.0" # IPv4 only; "::" IPv6 only; 127.0.0.1 or ::1 this machine only
# port = 9090 # Required (example) # port = 9090 # Required (example)
# buffer_size = 1000 # buffer_size = 1000
# client_buffer_size = 256 # client_buffer_size = 256
+16 -16
View File
@@ -94,31 +94,31 @@ preset:*)
(( CURRENT -= 2 )) (( CURRENT -= 2 ))
case ${words[1]} in case ${words[1]} in
pipe) _arguments -S \ pipe) _arguments -S \
'--format=[raw|txt|json: entry format]:value:_default' \ '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;; '(-h --help)'{-h,--help}'[print the flags]' ;;
tail) _arguments -S \ tail) _arguments -S \
'--format=[raw|txt|json: entry format]:value:_default' \ '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \
'--from=[start|end: where reading a file starts]:value:_default' \ '--from=[start|end: where reading a file starts]:value:_default' \
'--path=[file, directory (its files) or glob to follow]:value:_default' \ '--path=[a file, a directory (its files) or a glob such as /var/log/*.log, to follow]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;; '(-h --help)'{-h,--help}'[print the flags]' ;;
serve) _arguments -S \ serve) _arguments -S \
'--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \ '--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \
'--cert=[certificate file, for tls=files]:value:_default' \ '--cert=[certificate file, for tls=files]:value:_default' \
'--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \ '--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \
'--format=[raw|txt|json: entry format]:value:_default' \ '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \
'--from=[start|end: where reading a file starts]:value:_default' \ '--from=[start|end: where reading a file starts]:value:_default' \
'--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \ '--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \
'--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \ '--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \
'--issuer_key=[CA key file, for tls=issuer]:value:_default' \ '--issuer_key=[CA key file, for tls=issuer]:value:_default' \
'--key=[key file, for tls=files]:value:_default' \ '--key=[key file, for tls=files]:value:_default' \
'--listen=[HOST:PORT; [IPV6\]:PORT]:value:_default' \ '--listen=[HOST:PORT to listen on: 127.0.0.1:8080 this machine only, 0.0.0.0:8080 every IPv4 interface, [::\]:8080 every IPv6 interface]:value:_default' \
'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
'--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ '--path=[a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input]:value:_default' \
'--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \ '--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \
'--tls=[off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \ '--tls=[off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '(-u --user)'{-u=,--user=}'[the one SCRAM user readers log in as, in place of users]:value:_default' \
'--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \ '--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \
'--viewer=[true|false: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \ '--viewer=[true|false: a login page, then the viewer, for browsers behind proxy; without user or users the viewer is on anyway]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;; '(-h --help)'{-h,--help}'[print the flags]' ;;
edge) _arguments -S \ edge) _arguments -S \
'--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \ '--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \
@@ -127,29 +127,29 @@ preset:*)
'--key=[client key file]:value:_default' \ '--key=[client key file]:value:_default' \
'--node=[origin label; default: this host'\''s name]:value:_default' \ '--node=[origin label; default: this host'\''s name]:value:_default' \
'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
'--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ '--path=[a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input]:value:_default' \
'--pin=[sha256//BASE64 of the aggregator'\''s key, which it logs with tls=self]:value:_default' \ '--pin=[sha256//BASE64 of the aggregator'\''s key, which it logs with tls=self]:value:_default' \
'--server_name=[name the aggregator'\''s certificate carries; default: to'\''s host]:value:_default' \ '--server_name=[name the aggregator'\''s certificate carries; default: to'\''s host]:value:_default' \
'--to=[the aggregator'\''s HOST:PORT; [IPV6\]:PORT]:value:_default' \ '--to=[the aggregator'\''s HOST:PORT, an IPv6 address in brackets: logs.example.net:9000, [2001:db8::1\]:9000]:value:_default' \
'--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ '--transport=[tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '(-u --user)'{-u=,--user=}'[SCRAM user this edge logs in to the aggregator as]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;; '(-h --help)'{-h,--help}'[print the flags]' ;;
aggregator) _arguments -S \ aggregator) _arguments -S \
'--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \ '--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \
'--cert=[certificate file, for tls=files]:value:_default' \ '--cert=[certificate file, for tls=files]:value:_default' \
'--client_ca=[CA file that verifies edge certificates (mTLS)]:value:_default' \ '--client_ca=[CA file that verifies edge certificates (mTLS)]:value:_default' \
'--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \ '--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \
'--format=[raw|txt|json: entry format]:value:_default' \ '--format=[raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry]:value:_default' \
'--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \ '--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \
'--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \ '--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \
'--issuer_key=[CA key file, for tls=issuer]:value:_default' \ '--issuer_key=[CA key file, for tls=issuer]:value:_default' \
'--key=[key file, for tls=files]:value:_default' \ '--key=[key file, for tls=files]:value:_default' \
'--listen=[HOST:PORT; [IPV6\]:PORT; [::\] for IPv6 only]:value:_default' \ '--listen=[HOST:PORT to listen on: 0.0.0.0:9000 every IPv4 interface, [::\]:9000 every IPv6 interface, 127.0.0.1:9000 this machine only]:value:_default' \
'--out=[directory for the received entries; default: standard output]:value:_default' \ '--out=[directory for the received entries; default: standard output]:value:_default' \
'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
'--tls=[self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \ '--tls=[self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \
'--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ '--transport=[tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '(-u --user)'{-u=,--user=}'[the one SCRAM user edges log in as, in place of users]:value:_default' \
'--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \ '--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;; '(-h --help)'{-h,--help}'[print the flags]' ;;
esac esac
+16 -16
View File
@@ -96,29 +96,29 @@ complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a
complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a serve -d 'Serve files or standard input live: a browser viewer at /, SSE at /stream' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a serve -d 'Serve files or standard input live: a browser viewer at /, SSE at /stream'
complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a edge -d 'Forward files or standard input to an aggregator over TLS, authenticated' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a edge -d 'Forward files or standard input to an aggregator over TLS, authenticated'
complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a aggregator -d 'Receive from edges over TLS, authenticated, to files or standard output' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a aggregator -d 'Receive from edges over TLS, authenticated, to files or standard output'
complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry'
complete -c lw -n '__lw_in preset pipe' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset pipe' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry'
complete -c lw -n '__lw_in preset tail' -l from -r -F -d 'start|end: where reading a file starts' complete -c lw -n '__lw_in preset tail' -l from -r -F -d 'start|end: where reading a file starts'
complete -c lw -n '__lw_in preset tail' -l path -r -F -d 'file, directory (its files) or glob to follow' complete -c lw -n '__lw_in preset tail' -l path -r -F -d 'a file, a directory (its files) or a glob such as /var/log/*.log, to follow'
complete -c lw -n '__lw_in preset tail' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset tail' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in preset serve' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them' complete -c lw -n '__lw_in preset serve' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them'
complete -c lw -n '__lw_in preset serve' -l cert -r -F -d 'certificate file, for tls=files' complete -c lw -n '__lw_in preset serve' -l cert -r -F -d 'certificate file, for tls=files'
complete -c lw -n '__lw_in preset serve' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them' complete -c lw -n '__lw_in preset serve' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them'
complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry'
complete -c lw -n '__lw_in preset serve' -l from -r -F -d 'start|end: where reading a file starts' complete -c lw -n '__lw_in preset serve' -l from -r -F -d 'start|end: where reading a file starts'
complete -c lw -n '__lw_in preset serve' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them' complete -c lw -n '__lw_in preset serve' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them'
complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer' complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer'
complete -c lw -n '__lw_in preset serve' -l issuer_key -r -F -d 'CA key file, for tls=issuer' complete -c lw -n '__lw_in preset serve' -l issuer_key -r -F -d 'CA key file, for tls=issuer'
complete -c lw -n '__lw_in preset serve' -l key -r -F -d 'key file, for tls=files' complete -c lw -n '__lw_in preset serve' -l key -r -F -d 'key file, for tls=files'
complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT' complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT to listen on: 127.0.0.1:8080 this machine only, 0.0.0.0:8080 every IPv4 interface, [::]:8080 every IPv6 interface'
complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input'
complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them' complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them'
complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files' complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files'
complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'the one SCRAM user readers log in as, in place of users'
complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM' complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM'
complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true|false: the login page and viewer, behind proxy; without user or users the viewer is always on' complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true|false: a login page, then the viewer, for browsers behind proxy; without user or users the viewer is on anyway'
complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots' complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots'
complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca' complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca'
@@ -126,28 +126,28 @@ complete -c lw -n '__lw_in preset edge' -l from -r -F -d 'start|end: where readi
complete -c lw -n '__lw_in preset edge' -l key -r -F -d 'client key file' complete -c lw -n '__lw_in preset edge' -l key -r -F -d 'client key file'
complete -c lw -n '__lw_in preset edge' -l node -r -F -d 'origin label; default: this host\'s name' complete -c lw -n '__lw_in preset edge' -l node -r -F -d 'origin label; default: this host\'s name'
complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
complete -c lw -n '__lw_in preset edge' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' complete -c lw -n '__lw_in preset edge' -l path -r -F -d 'a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input'
complete -c lw -n '__lw_in preset edge' -l pin -r -F -d 'sha256//BASE64 of the aggregator\'s key, which it logs with tls=self' complete -c lw -n '__lw_in preset edge' -l pin -r -F -d 'sha256//BASE64 of the aggregator\'s key, which it logs with tls=self'
complete -c lw -n '__lw_in preset edge' -l server_name -r -F -d 'name the aggregator\'s certificate carries; default: to\'s host' complete -c lw -n '__lw_in preset edge' -l server_name -r -F -d 'name the aggregator\'s certificate carries; default: to\'s host'
complete -c lw -n '__lw_in preset edge' -l to -r -F -d 'the aggregator\'s HOST:PORT; [IPV6]:PORT' complete -c lw -n '__lw_in preset edge' -l to -r -F -d 'the aggregator\'s HOST:PORT, an IPv6 address in brackets: logs.example.net:9000, [2001:db8::1]:9000'
complete -c lw -n '__lw_in preset edge' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' complete -c lw -n '__lw_in preset edge' -l transport -r -F -d 'tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)'
complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user this edge logs in to the aggregator as'
complete -c lw -n '__lw_in preset edge' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset edge' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them'
complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files' complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files'
complete -c lw -n '__lw_in preset aggregator' -l client_ca -r -F -d 'CA file that verifies edge certificates (mTLS)' complete -c lw -n '__lw_in preset aggregator' -l client_ca -r -F -d 'CA file that verifies edge certificates (mTLS)'
complete -c lw -n '__lw_in preset aggregator' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them'
complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json: raw writes each message as it came, txt a line with time and level, json one object an entry'
complete -c lw -n '__lw_in preset aggregator' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them'
complete -c lw -n '__lw_in preset aggregator' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer' complete -c lw -n '__lw_in preset aggregator' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer'
complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key file, for tls=issuer' complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key file, for tls=issuer'
complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls=files' complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls=files'
complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT; [::] for IPv6 only' complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT to listen on: 0.0.0.0:9000 every IPv4 interface, [::]:9000 every IPv6 interface, 127.0.0.1:9000 this machine only'
complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output' complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output'
complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files' complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files'
complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)'
complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'the one SCRAM user edges log in as, in place of users'
complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM' complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM'
complete -c lw -n '__lw_in preset aggregator' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset aggregator' -l help -o h -d 'print the flags'
+4 -3
View File
@@ -254,8 +254,8 @@ never a key:
- `value`, `at` and `group` read a node's value and sort problems by node. - `value`, `at` and `group` read a node's value and sort problems by node.
The engine's calls take strings (`lwconf.js` writes objects as JSON) and The engine's calls take strings (`lwconf.js` writes objects as JSON) and
answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`, as answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key",
far as the engine places the refusal: "ask"}`, as far as the engine places the refusal:
- `schema()`: what `lw --schema` prints. - `schema()`: what `lw --schema` prints.
- `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to
@@ -273,7 +273,8 @@ far as the engine places the refusal:
- `set` (`node`, `key`, `values`), `unset` (`node`, `key`) - `set` (`node`, `key`, `values`), `unset` (`node`, `key`)
- `dirs` answers whether a path a preset names is a directory, - `dirs` answers whether a path a preset names is a directory,
`{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, one `{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, one
with a glob a pattern, and any other is refused at `path`. with a glob a pattern, and any other is refused at `path` with `ask` naming
it: answer it in `dirs` and call again.
- Pipelines are a JSON list in the file's shape (`name`, `flow`, - Pipelines are a JSON list in the file's shape (`name`, `flow`,
`plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node is `plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node is
`{role, id, index}`: a source's or sink's id, a filter's index. `{role, id, index}`: a source's or sink's id, a filter's index.
+3 -2
View File
@@ -457,8 +457,9 @@ WebAssembly; `t` runs them under the screen and shows what flows.
- Start: the pipelines `lw` would run with the same options (`-c FILE`, - Start: the pipelines `lw` would run with the same options (`-c FILE`,
pipeline flags, `LOGWISP_SOURCE`, a discovered file); with none but the pipeline flags, `LOGWISP_SOURCE`, a discovered file); with none but the
built-in pipe, the preset menu (or `empty`). A pasted `lw` command line built-in pipe, the preset menu (or `empty`). A pasted `lw` command line of
replaces the pipelines, in the preset menu too. pipeline flags, or the `lw` part of `... | lw ...`, replaces the
pipelines, in the preset menu too.
- Screen: the pipeline bar on top; SOURCES, FLOW and SINKS side by side - Screen: the pipeline bar on top; SOURCES, FLOW and SINKS side by side
from 64 columns, stacked below; each source and sink a tinted bar with its from 64 columns, stacked below; each source and sink a tinted bar with its
type, whether it listens or dials, and a summary; the four flow stages in type, whether it listens or dials, and a summary; the four flow stages in
+2 -2
View File
@@ -172,8 +172,8 @@ Options, each as type and default:
timestamp; the `raw` format shows neither. timestamp; the `raw` format shows neither.
- `include_stats` (bool, `false`): add `type = "heartbeat"`, `beat_count` and, - `include_stats` (bool, `false`): add `type = "heartbeat"`, `beat_count` and,
from the second beat, the measured `interval_ms` as fields. from the second beat, the measured `interval_ms` as fields.
- `format` (string, `txt`): `txt`, `json` or `raw`; no effect: heartbeats go - `format` (string, `txt`): `txt`, `json` or `raw`, kept for older files:
through the flow's `format`. heartbeats go through the flow's `format`.
Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They
are generated after the flow's filter and rate-limit stages, so filters do not are generated after the flow's filter and rate-limit stages, so filters do not
+2 -2
View File
@@ -209,8 +209,8 @@ preamble, headers, and entry encoding.
- `PROXY header: …` — malformed, too long, or incomplete after 10 s. - `PROXY header: …` — malformed, too long, or incomplete after 10 s.
- `PROXY header from a peer outside proxy_from` — at the first read, after - `PROXY header from a peer outside proxy_from` — at the first read, after
the connection opened: list the proxy in `proxy_from`. the connection opened: list the proxy in `proxy_from`.
- `acl: allow entry "...": not of the listener's family` at startup — an IPv4 - `acl: allow entry "...": IPv4, and the listener's host is IPv6` at startup,
entry on an IPv6 `host`, or the reverse: each family's listener takes its own. or the reverse: each family's listener takes its own.
Behind `proxy_from` or `trusted_proxies`, `allow` and `deny` take both; Behind `proxy_from` or `trusted_proxies`, `allow` and `deny` take both;
`proxy_from` its own. `proxy_from` its own.
+1 -1
View File
@@ -156,7 +156,7 @@ func TestNewValidation(t *testing.T) {
{"trusted proxies on a chain source", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.ChainListener, HTTP, "only to the http sink"}, {"trusted proxies on a chain source", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.ChainListener, HTTP, "only to the http sink"},
{"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to HTTP listeners"}, {"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to HTTP listeners"},
{"trusted proxies with certificate binding", listener(config.AuthOptions{TrustedProxies: proxies, Identity: "cn"}), f.serverTLS, config.Listener, HTTP, "TLS-terminating proxy"}, {"trusted proxies with certificate binding", listener(config.AuthOptions{TrustedProxies: proxies, Identity: "cn"}), f.serverTLS, config.Listener, HTTP, "TLS-terminating proxy"},
{"malformed trusted proxy", listener(config.AuthOptions{TrustedProxies: []string{"localhost"}}), nil, config.Listener, HTTP, "neither an address"}, {"malformed trusted proxy", listener(config.AuthOptions{TrustedProxies: []string{"localhost"}}), nil, config.Listener, HTTP, "host names and IPv6 zones are not accepted"},
{"trusted proxies on a dialer", dialer(config.AuthOptions{TrustedProxies: proxies}), &tls.Config{}, config.Dialer, HTTP, "only to listeners"}, {"trusted proxies on a dialer", dialer(config.AuthOptions{TrustedProxies: proxies}), &tls.Config{}, config.Dialer, HTTP, "only to listeners"},
} }
for _, tc := range tests { for _, tc := range tests {
+25 -10
View File
@@ -61,13 +61,21 @@ func FromPreset(name string, values map[string]string, isDir config.IsDir) (*Com
// FromCommandLine starts from a pasted lw command line, quoted as a POSIX // FromCommandLine starts from a pasted lw command line, quoted as a POSIX
// shell reads it; it takes pipeline flags only, at least one, as a composition // shell reads it; it takes pipeline flags only, at least one, as a composition
// holds no settings // holds no settings. Of a line piped into lw it reads lw's part, whose
// pipelines read standard input.
func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) { func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
words, err := split(strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(line)) // pasted line ends words, err := split(strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(line)) // pasted line ends
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(words) > 0 && (words[0] == "lw" || strings.HasSuffix(words[0], "/lw")) { lw := func(word string) bool { return word == "lw" || strings.HasSuffix(word, "/lw") }
for i := len(words) - 2; i >= 0; i-- {
if words[i] == "|" && lw(words[i+1]) {
words = words[i+1:]
break
}
}
if len(words) > 0 && lw(words[0]) {
words = words[1:] words = words[1:]
} }
var specs []config.Spec var specs []config.Spec
@@ -77,8 +85,12 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
name = "--preset" name = "--preset"
} }
flag, long := strings.CutPrefix(name, "--") flag, long := strings.CutPrefix(name, "--")
if !long || !slices.ContainsFunc(config.SpecFlags(), func(f config.SpecFlag) bool { return f.Flag == flag }) { if flags := config.SpecFlags(); !long || !slices.ContainsFunc(flags, func(f config.SpecFlag) bool { return f.Flag == flag }) {
return nil, fmt.Errorf("%s: not a pipeline flag", words[i]) var names []string
for _, f := range flags {
names = append(names, "--"+f.Flag)
}
return nil, fmt.Errorf("%s: not a pipeline flag; paste lw's %s only", words[i], strings.Join(names, ", "))
} }
if !inline { if !inline {
if i++; i == len(words) || strings.HasPrefix(words[i], "-") { if i++; i == len(words) || strings.HasPrefix(words[i], "-") {
@@ -205,8 +217,11 @@ func (c *Composition) RenamePipeline(i int, name string) error {
// free refuses a name that is empty or held by a pipeline other than i // free refuses a name that is empty or held by a pipeline other than i
func (c *Composition) free(name string, i int) error { func (c *Composition) free(name string, i int) error {
if j := slices.IndexFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }); name == "" || j >= 0 && j != i { switch j := slices.IndexFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }); {
return fmt.Errorf("pipeline %q: a name no other pipeline has", name) case name == "":
return errors.New("a pipeline needs a name; give it one")
case j >= 0 && j != i:
return fmt.Errorf("another pipeline is named %q; choose another name", name)
} }
return nil return nil
} }
@@ -574,7 +589,7 @@ func (c *Composition) CommandLine() (string, error) {
// line as an environment file holds them; the environment holds one pipeline // line as an environment file holds them; the environment holds one pipeline
func (c *Composition) Environment() (string, error) { func (c *Composition) Environment() (string, error) {
if len(c.Pipelines) != 1 { if len(c.Pipelines) != 1 {
return "", fmt.Errorf("the environment holds one pipeline, not %d", len(c.Pipelines)) return "", fmt.Errorf("the environment holds one pipeline, not %d; choose the command or file form, or remove pipelines down to one", len(c.Pipelines))
} }
specs, err := c.shellSpecs() specs, err := c.shellSpecs()
if err != nil { if err != nil {
@@ -632,9 +647,9 @@ type Form struct {
} }
var forms = []Form{ var forms = []Form{
{"command", "lw's flags", (*Composition).CommandLine, FromCommandLine}, {"command", "an lw command line", (*Composition).CommandLine, FromCommandLine},
{"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment, FromEnvironment}, {"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment},
{"file", "a configuration file's pipelines", func(c *Composition) (string, error) { {"file", "the pipelines of a configuration file", func(c *Composition) (string, error) {
data, err := c.File() data, err := c.File()
return string(data), err return string(data), err
}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }}, }, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }},
+20 -4
View File
@@ -115,6 +115,18 @@ func TestPastedEnvironmentReadsAsAShellRunsIt(t *testing.T) {
} }
} }
// Of a line piped into lw, lw's part is read, its pipeline reading standard
// input; a word that is no pipeline flag is refused, naming those to paste
func TestPastedCommandLineIsLwsPart(t *testing.T) {
c, err := FromCommandLine("tail -F app.log | lw --filter include:patterns=ERROR", nil)
if err != nil || c.Pipelines[0].PluginSources[0].Type != "console" || len(c.Pipelines[0].Flow.Filters) != 1 {
t.Fatalf("%v %v", c, err)
}
if _, err := FromCommandLine("lw -c lw.toml --sink null", nil); err == nil || !strings.HasPrefix(err.Error(), "-c: not a pipeline flag; paste lw's --pipeline, --preset, ") {
t.Fatalf("-c: %v", err)
}
}
// Locate places a problem by its path: a plugin's key, a key of its table, a // Locate places a problem by its path: a plugin's key, a key of its table, a
// rule between its table's keys, a list entry, the plugin itself under an id // rule between its table's keys, a list entry, the plugin itself under an id
// holding ']', a filter's list entry, a stage and the pipeline; and nothing // holding ']', a filter's list entry, a stage and the pipeline; and nothing
@@ -192,8 +204,9 @@ func TestShellFormsRefuseControlCharacters(t *testing.T) {
} }
} }
// Set refuses a value of the wrong kind and leaves the key as it was; Unset // Set refuses a value of the wrong kind, a plugin's and a stage's alike, and
// returns a key to its default, dropping a table it leaves empty // leaves the key as it was; Unset returns a key to its default, dropping a
// table it leaves empty
func TestSetKeepsAKeyItCannotTake(t *testing.T) { func TestSetKeepsAKeyItCannotTake(t *testing.T) {
c := &Composition{} c := &Composition{}
if err := c.AddPipeline("p"); err != nil { if err := c.AddPipeline("p"); err != nil {
@@ -207,8 +220,11 @@ func TestSetKeepsAKeyItCannotTake(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
if err := c.Set(0, n, "port", "http"); err == nil || !strings.HasPrefix(err.Error(), "port: ") { for want, err := range map[string]error{"port: not an integer": c.Set(0, n, "port", "http"),
t.Fatalf("port http: %v", err) "interval_ms: not an integer": c.Set(0, beat, "interval_ms", "soon")} {
if err == nil || err.Error() != want {
t.Errorf("%v, want %s", err, want)
}
} }
if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) || if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) ||
beat.IntervalMS != 1000 { beat.IntervalMS != 1000 {
+39 -39
View File
@@ -75,18 +75,18 @@ type FlowConfig struct {
// HeartbeatConfig defines settings for periodic keep-alive or status messages // HeartbeatConfig defines settings for periodic keep-alive or status messages
type HeartbeatConfig struct { type HeartbeatConfig struct {
Enabled bool `toml:"enabled" help:"send heartbeats"` Enabled bool `toml:"enabled" help:"send heartbeats; false keeps these settings and sends none"`
IntervalMS int64 `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"` IntervalMS int64 `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"`
IncludeTimestamp bool `toml:"include_timestamp" help:"stamp each heartbeat with its time"` IncludeTimestamp bool `toml:"include_timestamp" help:"stamp each heartbeat with its time"`
IncludeStats bool `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"` IncludeStats bool `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"`
Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"no effect: heartbeats use the flow's format"` Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"kept for older files; heartbeats are written in the flow's format"`
} }
// FormatConfig is how a pipeline writes its entries // FormatConfig is how a pipeline writes its entries
type FormatConfig struct { type FormatConfig struct {
Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"entry format"` Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"raw writes each message as it came, txt a line with time and level, json one object an entry"`
Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"formatter flags"` Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"for experts, a sum of bits over the type's choice: 1 writes messages verbatim, 16 drops the time, 32 the level"`
TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"Go time layout of timestamps"` TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"timestamp layout: the reference time 2006-01-02 15:04:05 written as timestamps should look, such as 02 Jan 15:04"`
SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"` SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"`
} }
@@ -102,10 +102,10 @@ const (
// RateLimitConfig defines the configuration for pipeline-level rate limiting // RateLimitConfig defines the configuration for pipeline-level rate limiting
type RateLimitConfig struct { type RateLimitConfig struct {
EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second the limit lets through"` EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second let through; unset, the stage limits nothing, sizes included"`
BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries let through at once before the rate applies"` BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries that may pass in a burst before entries_per_second holds them back"`
Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop entries over the rate, or pass them"` Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop enforces entries_per_second and max_entry_size_bytes; pass lets every entry through"`
MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries"` MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries; needs entries_per_second and policy drop"`
} }
// FilterType represents the filter's behavior (include or exclude) // FilterType represents the filter's behavior (include or exclude)
@@ -174,31 +174,31 @@ type ConsoleSourceOptions struct {
// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting // TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting
// NDJSON entries from upstream logwisp tcp_chain sinks // NDJSON entries from upstream logwisp tcp_chain sinks
type TCPChainSourceOptions struct { type TCPChainSourceOptions struct {
TLS *TLSOptions `toml:"tls"` TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"`
Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"`
Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"`
BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"`
MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"connections at once"` MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"connections at once"`
ReadTimeoutMS int64 `toml:"read_timeout_ms" lw:"min=0,zero=none" help:"milliseconds a connection may idle"` ReadTimeoutMS int64 `toml:"read_timeout_ms" lw:"min=0,zero=none" help:"milliseconds a connection may idle"`
HelloTimeoutMS int64 `toml:"hello_timeout_ms" default:"10000" lw:"min=1" help:"milliseconds for a peer's hello"` HelloTimeoutMS int64 `toml:"hello_timeout_ms" default:"10000" lw:"min=1" help:"milliseconds for a peer's hello"`
TrustNode bool `toml:"trust_node" default:"true" help:"keep the node label a peer declares; off, the label is its address"` TrustNode bool `toml:"trust_node" default:"true" help:"keep the node label a peer declares; off, the label is its address"`
Auth *AuthOptions `toml:"auth"` Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"`
ACL *ACLOptions `toml:"acl"` ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"`
} }
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting // HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
// NDJSON batches from upstream logwisp http_chain sinks // NDJSON batches from upstream logwisp http_chain sinks
type HTTPChainSourceOptions struct { type HTTPChainSourceOptions struct {
TLS *TLSOptions `toml:"tls"` TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"`
Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"`
Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"`
IngestPath string `toml:"ingest_path" default:"/ingest" lw:"hint=path" help:"path batches are posted to"` IngestPath string `toml:"ingest_path" default:"/ingest" lw:"hint=path" help:"path batches are posted to"`
BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"`
MaxBodyBytes int64 `toml:"max_body_bytes" default:"8388608" lw:"min=1" help:"largest batch accepted"` MaxBodyBytes int64 `toml:"max_body_bytes" default:"8388608" lw:"min=1" help:"largest batch accepted"`
ReadTimeoutMS int64 `toml:"read_timeout_ms" default:"30000" lw:"min=1" help:"milliseconds to read a request"` ReadTimeoutMS int64 `toml:"read_timeout_ms" default:"30000" lw:"min=1" help:"milliseconds to read a request"`
TrustNode bool `toml:"trust_node" default:"true" help:"keep the node label a peer declares; off, the label is its address"` TrustNode bool `toml:"trust_node" default:"true" help:"keep the node label a peer declares; off, the label is its address"`
Auth *AuthOptions `toml:"auth"` Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"`
ACL *ACLOptions `toml:"acl"` ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"`
} }
// --- Sinks --- // --- Sinks ---
@@ -228,7 +228,7 @@ type FileSinkOptions struct {
Name string `toml:"name" lw:"required" help:"base name of the files"` Name string `toml:"name" lw:"required" help:"base name of the files"`
MaxSizeMB int64 `toml:"max_size_mb" default:"100" lw:"min=1" help:"size at which a file rotates"` MaxSizeMB int64 `toml:"max_size_mb" default:"100" lw:"min=1" help:"size at which a file rotates"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb" default:"1000" lw:"min=1" help:"size of all files, beyond which the oldest go"` MaxTotalSizeMB int64 `toml:"max_total_size_mb" default:"1000" lw:"min=1" help:"size of all files, beyond which the oldest go"`
MinDiskFreeMB int64 `toml:"min_disk_free_mb" lw:"zero=none" help:"free space kept on the disk; negative means 100"` MinDiskFreeMB int64 `toml:"min_disk_free_mb" lw:"zero=none" help:"megabytes kept free on the disk, the oldest .log files deleted for it; a negative value means 100"`
RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0,zero=kept" help:"hours a .log file is kept; 0 keeps them"` RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0,zero=kept" help:"hours a .log file is kept; 0 keeps them"`
BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"`
FlushIntervalMs int64 `toml:"flush_interval_ms" default:"100" lw:"min=1" help:"milliseconds between flushes"` FlushIntervalMs int64 `toml:"flush_interval_ms" default:"100" lw:"min=1" help:"milliseconds between flushes"`
@@ -236,8 +236,8 @@ type FileSinkOptions struct {
// TCPSinkOptions defines settings for a TCP server sink // TCPSinkOptions defines settings for a TCP server sink
type TCPSinkOptions struct { type TCPSinkOptions struct {
TLS *TLSOptions `toml:"tls"` TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"`
Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"`
Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"`
BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued for all clients"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued for all clients"`
ClientBufferSize int64 `toml:"client_buffer_size" default:"256" lw:"min=1" help:"entries queued per client"` ClientBufferSize int64 `toml:"client_buffer_size" default:"256" lw:"min=1" help:"entries queued per client"`
@@ -245,14 +245,14 @@ type TCPSinkOptions struct {
KeepAlive bool `toml:"keep_alive" default:"true" help:"TCP keep-alive"` KeepAlive bool `toml:"keep_alive" default:"true" help:"TCP keep-alive"`
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms" default:"30000" lw:"min=1" help:"milliseconds idle before a keep-alive probe"` KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms" default:"30000" lw:"min=1" help:"milliseconds idle before a keep-alive probe"`
MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"clients at once"` MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"clients at once"`
Auth *AuthOptions `toml:"auth"` Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"`
ACL *ACLOptions `toml:"acl"` ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"`
} }
// HTTPSinkOptions defines settings for an HTTP SSE server sink // HTTPSinkOptions defines settings for an HTTP SSE server sink
type HTTPSinkOptions struct { type HTTPSinkOptions struct {
TLS *TLSOptions `toml:"tls"` TLS *TLSOptions `toml:"tls" help:"TLS for the connections it accepts"`
Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on; :: for IPv6"` Host string `toml:"host" default:"0.0.0.0" lw:"hint=host" help:"address to listen on, one family per listener: 0.0.0.0 every IPv4 interface, :: every IPv6 interface, 127.0.0.1 or ::1 this machine only"`
Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port to listen on"`
StreamPath string `toml:"stream_path" default:"/stream" lw:"hint=path" help:"path of the event stream"` StreamPath string `toml:"stream_path" default:"/stream" lw:"hint=path" help:"path of the event stream"`
StatusPath string `toml:"status_path" default:"/status" lw:"hint=path" help:"path of the status document"` StatusPath string `toml:"status_path" default:"/status" lw:"hint=path" help:"path of the status document"`
@@ -263,14 +263,14 @@ type HTTPSinkOptions struct {
ReplayLines int64 `toml:"replay_lines" lw:"min=0,zero=none" help:"recent entries a new stream gets first"` ReplayLines int64 `toml:"replay_lines" lw:"min=0,zero=none" help:"recent entries a new stream gets first"`
LoginPage bool `toml:"login_page" help:"scram behind auth.trusted_proxies only: a login page at /auth/login"` LoginPage bool `toml:"login_page" help:"scram behind auth.trusted_proxies only: a login page at /auth/login"`
ViewerPage bool `toml:"viewer_page" help:"scram behind auth.trusted_proxies only: the viewer after a login; without scram it is always at /"` ViewerPage bool `toml:"viewer_page" help:"scram behind auth.trusted_proxies only: the viewer after a login; without scram it is always at /"`
Auth *AuthOptions `toml:"auth"` Auth *AuthOptions `toml:"auth" help:"how peers authenticate: mtls by client certificate, scram by user and password"`
ACL *ACLOptions `toml:"acl"` ACL *ACLOptions `toml:"acl" help:"client addresses admitted or refused, and per-client limits"`
} }
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding // TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
// entries to a downstream logwisp tcp_chain source // entries to a downstream logwisp tcp_chain source
type TCPChainSinkOptions struct { type TCPChainSinkOptions struct {
TLS *TLSOptions `toml:"tls"` TLS *TLSOptions `toml:"tls" help:"TLS to the source it connects to"`
Node string `toml:"node" lw:"zero=the host name" help:"origin label of the entries"` Node string `toml:"node" lw:"zero=the host name" help:"origin label of the entries"`
Host string `toml:"host" lw:"required,hint=host" help:"host of the tcp_chain source"` Host string `toml:"host" lw:"required,hint=host" help:"host of the tcp_chain source"`
Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port of the tcp_chain source"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port of the tcp_chain source"`
@@ -281,13 +281,13 @@ type TCPChainSinkOptions struct {
BackoffMaxMS int64 `toml:"backoff_max_ms" default:"30000" lw:"min=1" help:"longest wait before reconnecting; below the first, the default"` BackoffMaxMS int64 `toml:"backoff_max_ms" default:"30000" lw:"min=1" help:"longest wait before reconnecting; below the first, the default"`
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms" default:"30000" lw:"min=1" help:"milliseconds idle before a keep-alive probe"` KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms" default:"30000" lw:"min=1" help:"milliseconds idle before a keep-alive probe"`
KeepAlive bool `toml:"keep_alive" default:"true" help:"TCP keep-alive"` KeepAlive bool `toml:"keep_alive" default:"true" help:"TCP keep-alive"`
Auth *AuthOptions `toml:"auth"` Auth *AuthOptions `toml:"auth" help:"how it authenticates with the source: scram logs in, mtls checks the source's identity"`
} }
// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting // HTTPChainSinkOptions defines settings for a stdlib HTTP client posting
// NDJSON batches to a downstream logwisp http_chain source // NDJSON batches to a downstream logwisp http_chain source
type HTTPChainSinkOptions struct { type HTTPChainSinkOptions struct {
TLS *TLSOptions `toml:"tls"` TLS *TLSOptions `toml:"tls" help:"TLS to the source it connects to"`
Node string `toml:"node" lw:"zero=the host name" help:"origin label of the entries"` Node string `toml:"node" lw:"zero=the host name" help:"origin label of the entries"`
Host string `toml:"host" lw:"required,hint=host" help:"host of the http_chain source"` Host string `toml:"host" lw:"required,hint=host" help:"host of the http_chain source"`
Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port of the http_chain source"` Port int64 `toml:"port" lw:"required,min=1,max=65535" help:"port of the http_chain source"`
@@ -299,7 +299,7 @@ type HTTPChainSinkOptions struct {
RequestTimeoutMS int64 `toml:"request_timeout_ms" default:"10000" lw:"min=1" help:"milliseconds for a request: dial, write and response"` RequestTimeoutMS int64 `toml:"request_timeout_ms" default:"10000" lw:"min=1" help:"milliseconds for a request: dial, write and response"`
BackoffMinMS int64 `toml:"backoff_min_ms" default:"500" lw:"min=1" help:"first wait before retrying"` BackoffMinMS int64 `toml:"backoff_min_ms" default:"500" lw:"min=1" help:"first wait before retrying"`
BackoffMaxMS int64 `toml:"backoff_max_ms" default:"30000" lw:"min=1" help:"longest wait before retrying; below the first, the default"` BackoffMaxMS int64 `toml:"backoff_max_ms" default:"30000" lw:"min=1" help:"longest wait before retrying; below the first, the default"`
Auth *AuthOptions `toml:"auth"` Auth *AuthOptions `toml:"auth" help:"how it authenticates with the source: scram logs in, mtls checks the source's identity"`
} }
// --- Auth Options --- // --- Auth Options ---
@@ -312,14 +312,14 @@ type HTTPChainSinkOptions struct {
type AuthOptions struct { type AuthOptions struct {
Type string `toml:"type" default:"none" lw:"enum=none|mtls|scram" help:"how peers authenticate"` Type string `toml:"type" default:"none" lw:"enum=none|mtls|scram" help:"how peers authenticate"`
Identity string `toml:"identity" lw:"enum=cn|san_dns|san_uri|san_email,zero=cn under mtls; no binding under scram" help:"certificate field naming the peer; under scram it must equal the user"` Identity string `toml:"identity" lw:"enum=cn|san_dns|san_uri|san_email,zero=cn under mtls; no binding under scram" help:"certificate field naming the peer; under scram it must equal the user"`
Allow []string `toml:"allow" help:"mtls: identities admitted; with allow_patterns empty, any the CA issued"` Allow []string `toml:"allow" help:"mtls: identities admitted; with neither this nor allow_patterns set, any the CA issued"`
AllowPatterns []string `toml:"allow_patterns" lw:"hint=regex" help:"mtls: RE2 patterns of identities admitted; anchor them"` AllowPatterns []string `toml:"allow_patterns" lw:"hint=regex" help:"mtls: RE2 patterns of identities admitted; anchor them"`
NodeBinding string `toml:"node_binding" lw:"listener,enum=none|assert|force,zero=force" help:"chain sources: how a peer's identity binds its node label; overrides trust_node"` NodeBinding string `toml:"node_binding" lw:"chain_listener,enum=none|assert|force,zero=force" help:"chain sources: force labels every entry with the peer's identity, assert refuses a peer declaring another node, none leaves it to trust_node"`
CredentialsFile string `toml:"credentials_file" lw:"listener,hint=file" help:"scram: verifiers, from lw auth add-user"` CredentialsFile string `toml:"credentials_file" lw:"listener,hint=file" help:"scram: file of the users who may log in, made by lw auth add-user"`
// exp has whole seconds, a sub-second expires_in reads as unknown, and 10 s // exp has whole seconds, a sub-second expires_in reads as unknown, and 10 s
// leaves room for renewal ahead of expiry // leaves room for renewal ahead of expiry
TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,http,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"` TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,http,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
TrustedProxies []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"` TrustedProxies []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: addresses or CIDRs of the proxies that end browsers' TLS, through which browsers then log in"`
Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"` Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"`
PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"` PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"`
} }
@@ -329,8 +329,8 @@ type AuthOptions struct {
// the L4 proxies whose PROXY header names the client the rules then see; the // the L4 proxies whose PROXY header names the client the rules then see; the
// per-client limits count that client. ACLOptions.Check holds the rules. // per-client limits count that client. ACLOptions.Check holds the rules.
type ACLOptions struct { type ACLOptions struct {
Allow []string `toml:"allow" lw:"hint=cidr" help:"addresses admitted; empty, all deny does not list"` Allow []string `toml:"allow" lw:"hint=cidr" help:"addresses or CIDRs that may connect; unset, any that deny does not refuse"`
Deny []string `toml:"deny" lw:"hint=cidr" help:"addresses refused"` Deny []string `toml:"deny" lw:"hint=cidr" help:"addresses or CIDRs refused, even when allow lists them"`
ProxyProtocol string `toml:"proxy_protocol" default:"off" lw:"enum=off|optional|required" help:"read a PROXY header from proxy_from"` ProxyProtocol string `toml:"proxy_protocol" default:"off" lw:"enum=off|optional|required" help:"read a PROXY header from proxy_from"`
ProxyFrom []string `toml:"proxy_from" lw:"hint=cidr" help:"proxies that may send a PROXY header"` ProxyFrom []string `toml:"proxy_from" lw:"hint=cidr" help:"proxies that may send a PROXY header"`
MaxConnectionsPerClient int64 `toml:"max_connections_per_client" lw:"min=0,zero=unlimited" help:"connections a client may hold"` MaxConnectionsPerClient int64 `toml:"max_connections_per_client" lw:"min=0,zero=unlimited" help:"connections a client may hold"`
@@ -346,12 +346,12 @@ type ACLOptions struct {
// may present cert_file and key_file. // may present cert_file and key_file.
type TLSOptions struct { type TLSOptions struct {
Enabled bool `toml:"enabled" help:"use TLS"` Enabled bool `toml:"enabled" help:"use TLS"`
CertFile string `toml:"cert_file" lw:"hint=file" help:"certificate presented; a dialer's for mTLS"` CertFile string `toml:"cert_file" lw:"hint=file" help:"certificate file this end presents; a dialer's, for a listener with client_auth"`
KeyFile string `toml:"key_file" lw:"hint=file" help:"key of cert_file"` KeyFile string `toml:"key_file" lw:"hint=file" help:"key of cert_file"`
SelfSigned bool `toml:"self_signed" lw:"listener" help:"a self-signed certificate on a key made at startup"` SelfSigned bool `toml:"self_signed" lw:"listener" help:"a self-signed certificate on a key made at startup"`
IssuerCertFile string `toml:"issuer_cert_file" lw:"listener,hint=file" help:"CA certificate (lw tls ca) that issues the listener's at startup"` IssuerCertFile string `toml:"issuer_cert_file" lw:"listener,hint=file" help:"CA certificate file (lw tls ca) that signs a certificate made at startup"`
IssuerKeyFile string `toml:"issuer_key_file" lw:"listener,hint=file" help:"key of issuer_cert_file"` IssuerKeyFile string `toml:"issuer_key_file" lw:"listener,hint=file" help:"key of issuer_cert_file"`
Hosts []string `toml:"hosts" lw:"listener" help:"names and addresses a made certificate carries beyond host, this machine's and loopback"` Hosts []string `toml:"hosts" lw:"listener" help:"more names and addresses for a certificate made at startup, which always carries host, this machine's name and localhost"`
ClientAuth bool `toml:"client_auth" lw:"listener" help:"require and verify client certificates"` ClientAuth bool `toml:"client_auth" lw:"listener" help:"require and verify client certificates"`
ClientCAFile string `toml:"client_ca_file" lw:"listener,hint=file" help:"CA bundle verifying client certificates"` ClientCAFile string `toml:"client_ca_file" lw:"listener,hint=file" help:"CA bundle verifying client certificates"`
CAFile string `toml:"ca_file" lw:"dialer,hint=file,zero=the system roots" help:"CA bundle verifying the server"` CAFile string `toml:"ca_file" lw:"dialer,hint=file,zero=the system roots" help:"CA bundle verifying the server"`
+42 -33
View File
@@ -63,10 +63,19 @@ func mirror[T any](p PresetParam, key string) PresetParam {
// IsDir tells whether a preset's path is a directory, whose files it follows, // IsDir tells whether a preset's path is a directory, whose files it follows,
// or a file pattern; a path ending in '/' is a directory without asking. Off // or a file pattern; a path ending in '/' is a directory without asking. Off
// the target host it is the user's Answers, or nil: a path with a glob is then // the target host it is the user's Answers, or nil: a path with a glob is then
// a pattern, and any other fails with ErrPathKind. // a pattern, and any other fails at key path with a PathKindError.
type IsDir func(path string) (bool, error) type IsDir func(path string) (bool, error)
var ErrPathKind error = &KeyError{"path", errors.New("a directory or a file pattern?")} // ErrPathKind is what every PathKindError unwraps to
var ErrPathKind = errors.New("a directory or a file?")
// PathKindError asks whether Path, which a preset names, is a directory
type PathKindError struct{ Path string }
func (e *PathKindError) Error() string {
return fmt.Sprintf("%q: %v end a directory with / to follow its files", e.Path, ErrPathKind)
}
func (e *PathKindError) Unwrap() error { return ErrPathKind }
// HostIsDir asks this host, a leading ~ being the home directory: a path that // HostIsDir asks this host, a leading ~ being the home directory: a path that
// does not exist is a pattern // does not exist is a pattern
@@ -85,16 +94,16 @@ func Answers(dirs map[string]bool) IsDir {
if strings.ContainsAny(path, `*?[\`) { if strings.ContainsAny(path, `*?[\`) {
return false, nil return false, nil
} }
return false, ErrPathKind return false, &KeyError{"path", &PathKindError{path}}
} }
} }
// Keys several presets share // Keys several presets share
var ( var (
pathParam = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"} pathParam = PresetParam{Name: "path", Help: "a file, a directory (its files) or a glob such as /var/log/*.log, to follow; default: standard input"}
fromParam = mirror[FileSourceOptions](PresetParam{Name: "from", Default: "end", Help: "where reading a file starts"}, "from") fromParam = mirror[FileSourceOptions](PresetParam{Name: "from", Default: "end", Help: "where reading a file starts"}, "from")
formatParam = func(def string) PresetParam { formatParam = func(def string) PresetParam {
return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "entry format"}, "type") return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "raw writes each message as it came, txt a line with time and level, json one object an entry"}, "type")
} }
tlsParams = []PresetParam{ tlsParams = []PresetParam{
mirror[TLSOptions](PresetParam{Name: "cert", Help: "certificate file, for tls=files"}, "cert_file"), mirror[TLSOptions](PresetParam{Name: "cert", Help: "certificate file, for tls=files"}, "cert_file"),
@@ -107,13 +116,13 @@ var (
mirror[ACLOptions](PresetParam{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, "allow"), mirror[ACLOptions](PresetParam{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, "allow"),
mirror[ACLOptions](PresetParam{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, "deny"), mirror[ACLOptions](PresetParam{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, "deny"),
} }
transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp_chain or http_chain", Enum: transports()} transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp sends over one lasting connection (tcp_chain), http posts batches (http_chain)", Enum: transports()}
usersParam = func(who string) PresetParam { usersParam = func(who string) PresetParam {
return mirror[AuthOptions](PresetParam{Name: "users", Help: "credentials file (lw auth add-user): " + who + " log in with SCRAM"}, "credentials_file") return mirror[AuthOptions](PresetParam{Name: "users", Help: "credentials file (lw auth add-user): " + who + " log in with SCRAM"}, "credentials_file")
} }
userParams = []PresetParam{ userParams = func(help string) []PresetParam {
{Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"}, return []PresetParam{{Name: "user", Help: help},
mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file"), mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file")}
} }
) )
@@ -146,7 +155,7 @@ var presets = []Preset{
return nil return nil
}}, }},
{"tail", "Follow files to standard output, like tail -F", {"tail", "Follow files to standard output, like tail -F",
[]PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")}, []PresetParam{{Name: "path", Required: true, Help: "a file, a directory (its files) or a glob such as /var/log/*.log, to follow"}, fromParam, formatParam("raw")},
func(p *PipelineConfig, v map[string]string, isDir IsDir) error { func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir) sources, err := pathOrStdin(v, isDir)
p.PluginSources, p.PluginSinks = sources, pipeDefault("").PluginSinks p.PluginSources, p.PluginSinks = sources, pipeDefault("").PluginSinks
@@ -155,12 +164,12 @@ var presets = []Preset{
}}, }},
{"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream", {"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream",
slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"), slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"),
{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"}, {Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT to listen on: 127.0.0.1:8080 this machine only, 0.0.0.0:8080 every IPv4 interface, [::]:8080 every IPv6 interface"},
serveTLS, serveTLS,
usersParam("readers"), usersParam("readers"),
}, userParams, []PresetParam{ }, userParams("the one SCRAM user readers log in as, in place of users"), []PresetParam{
mirror[AuthOptions](PresetParam{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, "trusted_proxies"), mirror[AuthOptions](PresetParam{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, "trusted_proxies"),
mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "the login page and viewer, behind proxy; without user or users the viewer is always on"}, "viewer_page"), mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "a login page, then the viewer, for browsers behind proxy; without user or users the viewer is on anyway"}, "viewer_page"),
}, aclParams, tlsParams), }, aclParams, tlsParams),
func(p *PipelineConfig, v map[string]string, isDir IsDir) error { func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir) sources, err := pathOrStdin(v, isDir)
@@ -177,11 +186,11 @@ var presets = []Preset{
case err != nil: case err != nil:
return err return err
case auth != nil && v["tls"] == "off" && v["proxy"] == "": case auth != nil && v["tls"] == "off" && v["proxy"] == "":
return fmt.Errorf("user and users need tls (self, issuer or files), or proxy") return &KeyError{"tls", errors.New("user and users need self, issuer or files, or proxy")}
case auth != nil: case auth != nil:
sink["auth"] = auth sink["auth"] = auth
case v["proxy"] != "": case v["proxy"] != "":
return fmt.Errorf("proxy needs user or users: browsers log in with SCRAM") return &KeyError{"proxy", errors.New("needs user or users: browsers log in with SCRAM")}
} }
if v["proxy"] != "" { if v["proxy"] != "" {
auth["trusted_proxies"] = list(v["proxy"]) auth["trusted_proxies"] = list(v["proxy"])
@@ -189,12 +198,12 @@ var presets = []Preset{
switch v["viewer"] { switch v["viewer"] {
case "true": case "true":
if v["proxy"] == "" { if v["proxy"] == "" {
return fmt.Errorf("viewer=true needs proxy, behind which browsers log in; without users the viewer is always on") return &KeyError{"viewer", errors.New("true needs proxy, behind which browsers log in; without users the viewer is always on")}
} }
sink["login_page"], sink["viewer_page"] = true, true sink["login_page"], sink["viewer_page"] = true, true
case "false": case "false":
default: default:
return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"]) return &KeyError{"viewer", fmt.Errorf("must be true or false, got %q", v["viewer"])}
} }
sink["replay_lines"] = int64(1000) // a browser opened late sees the recent entries sink["replay_lines"] = int64(1000) // a browser opened late sees the recent entries
p.Flow.Format = &FormatConfig{Type: v["format"]} p.Flow.Format = &FormatConfig{Type: v["format"]}
@@ -203,12 +212,12 @@ var presets = []Preset{
}}, }},
{"edge", "Forward files or standard input to an aggregator over TLS, authenticated", {"edge", "Forward files or standard input to an aggregator over TLS, authenticated",
slices.Concat([]PresetParam{pathParam, fromParam, slices.Concat([]PresetParam{pathParam, fromParam,
{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"}, {Name: "to", Required: true, Help: "the aggregator's HOST:PORT, an IPv6 address in brackets: logs.example.net:9000, [2001:db8::1]:9000"},
transportParam, transportParam,
mirror[TLSOptions](PresetParam{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, "ca_file"), mirror[TLSOptions](PresetParam{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, "ca_file"),
{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"}, {Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"}, {Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
}, userParams, []PresetParam{ }, userParams("SCRAM user this edge logs in to the aggregator as"), []PresetParam{
mirror[TLSOptions](PresetParam{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, "cert_file"), mirror[TLSOptions](PresetParam{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, "cert_file"),
mirror[TLSOptions](PresetParam{Name: "key", Help: "client key file"}, "key_file"), mirror[TLSOptions](PresetParam{Name: "key", Help: "client key file"}, "key_file"),
{Name: "node", Help: "origin label; default: this host's name"}, {Name: "node", Help: "origin label; default: this host's name"},
@@ -241,7 +250,7 @@ var presets = []Preset{
case v["cert"] != "" && v["key"] != "": case v["cert"] != "" && v["key"] != "":
auth = map[string]any{"type": "mtls"} auth = map[string]any{"type": "mtls"}
default: default:
return fmt.Errorf("edge needs user, or cert and key: it never sends unauthenticated") return &KeyError{"user", errors.New("required unless cert and key are set; an edge never sends unauthenticated")}
} }
sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth} sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth}
if v["node"] != "" { if v["node"] != "" {
@@ -252,11 +261,11 @@ var presets = []Preset{
}}, }},
{"aggregator", "Receive from edges over TLS, authenticated, to files or standard output", {"aggregator", "Receive from edges over TLS, authenticated, to files or standard output",
slices.Concat([]PresetParam{ slices.Concat([]PresetParam{
{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"}, {Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT to listen on: 0.0.0.0:9000 every IPv4 interface, [::]:9000 every IPv6 interface, 127.0.0.1:9000 this machine only"},
transportParam, transportParam,
aggregatorTLS, aggregatorTLS,
usersParam("edges"), usersParam("edges"),
}, userParams, []PresetParam{ }, userParams("the one SCRAM user edges log in as, in place of users"), []PresetParam{
mirror[TLSOptions](PresetParam{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, "client_ca_file"), mirror[TLSOptions](PresetParam{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, "client_ca_file"),
mirror[FileSinkOptions](PresetParam{Name: "out", Help: "directory for the received entries; default: standard output"}, "directory"), mirror[FileSinkOptions](PresetParam{Name: "out", Help: "directory for the received entries; default: standard output"}, "directory"),
formatParam("json"), formatParam("json"),
@@ -283,7 +292,7 @@ var presets = []Preset{
case v["client_ca"] != "": case v["client_ca"] != "":
src["auth"] = map[string]any{"type": "mtls"} src["auth"] = map[string]any{"type": "mtls"}
default: default:
return fmt.Errorf("aggregator needs user, users or client_ca: it never receives unauthenticated") return &KeyError{"users", errors.New("required unless user or client_ca is set; an aggregator never receives unauthenticated")}
} }
p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src}) p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src})
p.PluginSinks = pipeDefault("").PluginSinks p.PluginSinks = pipeDefault("").PluginSinks
@@ -331,13 +340,13 @@ func applyPreset(p *PipelineConfig, name string, values map[string][]string, isD
case j < 0: case j < 0:
return fmt.Errorf("preset %s has no key %q (valid: %s)", name, k, strings.Join(keys, ", ")) return fmt.Errorf("preset %s has no key %q (valid: %s)", name, k, strings.Join(keys, ", "))
case len(each) > 1 && !r.Params[j].List: case len(each) > 1 && !r.Params[j].List:
return fmt.Errorf("preset %s key %q takes one value", name, k) return &KeyError{k, errors.New("takes one value")}
} }
v[k] = cmp.Or(strings.Join(each, ","), v[k]) v[k] = cmp.Or(strings.Join(each, ","), v[k])
} }
for _, k := range r.Params { for _, k := range r.Params {
if k.Required && v[k.Name] == "" { if k.Required && v[k.Name] == "" {
return fmt.Errorf("preset %s needs %s", name, k.Name) return &KeyError{k.Name, errors.New("must be set")}
} }
} }
p.Name = cmp.Or(p.Name, name) p.Name = cmp.Or(p.Name, name)
@@ -404,20 +413,20 @@ func listener(v map[string]string, key PresetParam) (map[string]any, error) {
mode := cmp.Or(v["tls"], key.Default) mode := cmp.Or(v["tls"], key.Default)
i := slices.IndexFunc(tlsModes, func(m tlsMode) bool { return m.name == mode }) i := slices.IndexFunc(tlsModes, func(m tlsMode) bool { return m.name == mode })
if i < 0 || !slices.Contains(key.Enum, mode) { if i < 0 || !slices.Contains(key.Enum, mode) {
return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(key.Enum, ", ")) return nil, &KeyError{"tls", fmt.Errorf("must be one of %s, got %q", strings.Join(key.Enum, ", "), mode)}
} }
m := tlsModes[i] m := tlsModes[i]
for _, other := range tlsModes { for _, other := range tlsModes {
for k := range other.needs { for k := range other.needs {
if _, needed := m.needs[k]; needed && v[k] == "" { if _, needed := m.needs[k]; needed && v[k] == "" {
return nil, fmt.Errorf("tls=%s needs %s", mode, k) return nil, &KeyError{k, fmt.Errorf("must be set for tls=%s", mode)}
} else if !needed && v[k] != "" { } else if !needed && v[k] != "" {
return nil, fmt.Errorf("%s does not apply to tls=%s", k, mode) return nil, &KeyError{k, fmt.Errorf("does not apply to tls=%s", mode)}
} }
} }
} }
if v["hosts"] != "" && !m.made { if v["hosts"] != "" && !m.made {
return nil, fmt.Errorf("hosts does not apply to tls=%s", mode) return nil, &KeyError{"hosts", fmt.Errorf("does not apply to tls=%s", mode)}
} }
if mode == "off" { if mode == "off" {
return opts, nil return opts, nil
@@ -439,11 +448,11 @@ func listener(v map[string]string, key PresetParam) (map[string]any, error) {
func scramAuth(v map[string]string) (map[string]any, error) { func scramAuth(v map[string]string) (map[string]any, error) {
switch { switch {
case v["user"] != "" && v["users"] != "": case v["user"] != "" && v["users"] != "":
return nil, fmt.Errorf("user or users, not both") return nil, &KeyError{"user", errors.New("set user or users, not both")}
case v["user"] != "": case v["user"] != "":
return map[string]any{"type": "scram", "username": v["user"], "password_file": cmp.Or(v["password_file"], "/dev/tty")}, nil return map[string]any{"type": "scram", "username": v["user"], "password_file": cmp.Or(v["password_file"], "/dev/tty")}, nil
case v["password_file"] != "": case v["password_file"] != "":
return nil, fmt.Errorf("password_file needs user") return nil, &KeyError{"password_file", errors.New("needs user")}
case v["users"] != "": case v["users"] != "":
return map[string]any{"type": "scram", "credentials_file": v["users"]}, nil return map[string]any{"type": "scram", "credentials_file": v["users"]}, nil
} }
@@ -454,7 +463,7 @@ func splitAddr(key, addr string) (string, int64, error) {
host, p, err := net.SplitHostPort(addr) host, p, err := net.SplitHostPort(addr)
port, perr := strconv.ParseUint(p, 10, 16) port, perr := strconv.ParseUint(p, 10, 16)
if err != nil || perr != nil || port == 0 { if err != nil || perr != nil || port == 0 {
return "", 0, fmt.Errorf("%s %q: want HOST:PORT, an IPv6 address in brackets", key, addr) return "", 0, &KeyError{key, fmt.Errorf("must be HOST:PORT, an IPv6 address in brackets as in [::1]:9000, got %q", addr)}
} }
return host, int64(port), nil return host, int64(port), nil
} }
@@ -462,7 +471,7 @@ func splitAddr(key, addr string) (string, int64, error) {
// chainType is a transport's chain plugin type, both a source and a sink // chainType is a transport's chain plugin type, both a source and a sink
func chainType(transport string) (string, error) { func chainType(transport string) (string, error) {
if !slices.Contains(transports(), transport) { if !slices.Contains(transports(), transport) {
return "", fmt.Errorf("transport %q (valid: %s)", transport, strings.Join(transports(), ", ")) return "", &KeyError{"transport", fmt.Errorf("must be one of %s, got %q", strings.Join(transports(), ", "), transport)}
} }
return transport + "_chain", nil return transport + "_chain", nil
} }
+25 -20
View File
@@ -39,34 +39,39 @@ func TestPresetStartsAndNamesItsPipeline(t *testing.T) {
} }
// Keys are checked against the preset's own, required ones enforced, and // Keys are checked against the preset's own, required ones enforced, and
// the edge and aggregator refuse to run without authentication. // the edge and aggregator refuse to run without authentication; a refusal of
// one key is a KeyError at it, the others name the preset.
func TestPresetKeysAreChecked(t *testing.T) { func TestPresetKeysAreChecked(t *testing.T) {
isolateConfig(t) isolateConfig(t)
for spec, want := range map[string]string{ for spec, want := range map[string]string{
"nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`, "nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`,
"tail:path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`, "tail:path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`,
"tail": "preset tail needs path", "tail": "path: must be set",
"tail:path=x,format.type=json": "preset keys do not nest", "tail:path=x,format.type=json": "preset keys do not nest",
"tail:path=x,path=y": `key "path" takes one value`, "tail:path=x,path=y": "path: takes one value",
"serve:listen=8080": `listen "8080": want HOST:PORT`, "serve:listen=8080": `listen: must be HOST:PORT, an IPv6 address in brackets as in [::1]:9000, got "8080"`,
"serve:tls=self,cert=c": "cert does not apply to tls=self", "serve:tls=self,cert=c": "cert: does not apply to tls=self",
"serve:tls=files,cert=c": "tls=files needs key", "serve:tls=files,cert=c": "key: must be set for tls=files",
"serve:viewer=yes": `viewer "yes"`, "serve:viewer=yes": `viewer: must be true or false, got "yes"`,
"serve:proxy=127.0.0.1": "proxy needs user or users", "serve:proxy=127.0.0.1": "proxy: needs user or users",
"serve:viewer=true": "viewer=true needs proxy", "serve:viewer=true": "viewer: true needs proxy",
"serve:user=u": "user and users need tls (self, issuer or files), or proxy", "serve:user=u": "tls: user and users need self, issuer or files, or proxy",
"serve:tls=self,user=u,users=f": "user or users, not both", "serve:tls=self,user=u,users=f": "user: set user or users, not both",
"edge:to=agg:9000": "it never sends unauthenticated", "edge:to=agg:9000": "user: required unless cert and key are set; an edge never sends unauthenticated",
"edge:to=agg:9000,password_file=p": "password_file needs user", "edge:to=agg:9000,password_file=p": "password_file: needs user",
"edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`, "edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport: must be one of tcp, http, got "udp"`,
"aggregator:users=u,tls=off": `tls "off" (valid: `, "aggregator:users=u,tls=off": `tls: must be one of self, issuer, files, got "off"`,
"aggregator:user=u,users=f": "user or users, not both", "aggregator:user=u,users=f": "user: set user or users, not both",
"aggregator": "it never receives unauthenticated", "aggregator": "users: required unless user or client_ca is set; an aggregator never receives unauthenticated",
} { } {
_, err := Load(Args{Specs: specs("preset", spec)}) _, err := Load(Args{Specs: specs("preset", spec)})
if err == nil || !strings.Contains(err.Error(), want) { if err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%s: err = %v, want %s", spec, err, want) t.Errorf("%s: err = %v, want %s", spec, err, want)
} }
ke, keyed := errors.AsType[*KeyError](err)
if keyed == strings.Contains(want, "preset") || keyed && !strings.HasPrefix(want, ke.Key+": ") {
t.Errorf("%s: %v placed at %v", spec, err, ke)
}
} }
} }
@@ -127,7 +132,7 @@ func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
// A preset path ending in '/' is a directory, there yet or not; any other is // A preset path ending in '/' is a directory, there yet or not; any other is
// what isDir answers, a directory named with brackets too: this host's, // what isDir answers, a directory named with brackets too: this host's,
// reading ~ as home, or off it the user's answer, a single file too; without // reading ~ as home, or off it the user's answer, a single file too; without
// one it is a pattern when it holds a glob, and otherwise ErrPathKind asks // one it is a pattern when it holds a glob, and otherwise PathKindError asks
func TestPresetPathKind(t *testing.T) { func TestPresetPathKind(t *testing.T) {
home := t.TempDir() home := t.TempDir()
t.Setenv("HOME", home) t.Setenv("HOME", home)
@@ -145,8 +150,8 @@ func TestPresetPathKind(t *testing.T) {
{"~/logs", HostIsDir, "~/logs *"}, {home + "/later/", HostIsDir, home + "/later/ *"}} { {"~/logs", HostIsDir, "~/logs *"}, {home + "/later/", HostIsDir, home + "/later/ *"}} {
p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir) p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir)
if c.want == "" { if c.want == "" {
if !errors.Is(err, ErrPathKind) { if pk, ok := errors.AsType[*PathKindError](err); !errors.Is(err, ErrPathKind) || !ok || pk.Path != c.path {
t.Errorf("%s: %v, want ErrPathKind", c.path, err) t.Errorf("%s: %v, want ErrPathKind asking about it", c.path, err)
} }
continue continue
} }
+12 -15
View File
@@ -35,7 +35,7 @@ type Plugin struct {
Type string `json:"type"` Type string `json:"type"`
Side Side `json:"side"` Side Side `json:"side"`
HTTP bool `json:"http"` // speaks HTTP: takes bearer tokens and request limits HTTP bool `json:"http"` // speaks HTTP: takes bearer tokens and request limits
Single bool `json:"single"` // one instance per process Single bool `json:"single"` // one instance per process, as standard input is one
Summary string `json:"summary"` Summary string `json:"summary"`
options reflect.Type options reflect.Type
} }
@@ -45,15 +45,15 @@ var plugins = []Plugin{
{"source", "file", Local, false, false, "Follow the files of a directory that match a pattern", reflect.TypeFor[FileSourceOptions]()}, {"source", "file", Local, false, false, "Follow the files of a directory that match a pattern", reflect.TypeFor[FileSourceOptions]()},
{"source", "random", Local, false, false, "Generated entries, to check a deployment before real logs", reflect.TypeFor[RandomSourceOptions]()}, {"source", "random", Local, false, false, "Generated entries, to check a deployment before real logs", reflect.TypeFor[RandomSourceOptions]()},
{"source", "null", Local, false, false, "Nothing; holds a source's place until one is chosen", reflect.TypeFor[NullSourceOptions]()}, {"source", "null", Local, false, false, "Nothing; holds a source's place until one is chosen", reflect.TypeFor[NullSourceOptions]()},
{"source", "tcp_chain", ChainListener, false, false, "Entries from tcp_chain sinks of other instances", reflect.TypeFor[TCPChainSourceOptions]()}, {"source", "tcp_chain", ChainListener, false, false, "Receive entries from other nodes' tcp_chain sinks, each over one lasting TCP connection", reflect.TypeFor[TCPChainSourceOptions]()},
{"source", "http_chain", ChainListener, true, false, "Entry batches from http_chain sinks of other instances", reflect.TypeFor[HTTPChainSourceOptions]()}, {"source", "http_chain", ChainListener, true, false, "Receive entries from other nodes' http_chain sinks, as batches each posts over HTTP", reflect.TypeFor[HTTPChainSourceOptions]()},
{"sink", "console", Local, false, false, "Standard output or standard error", reflect.TypeFor[ConsoleSinkOptions]()}, {"sink", "console", Local, false, false, "Standard output or standard error", reflect.TypeFor[ConsoleSinkOptions]()},
{"sink", "file", Local, false, false, "Rotating files", reflect.TypeFor[FileSinkOptions]()}, {"sink", "file", Local, false, false, "Rotating files", reflect.TypeFor[FileSinkOptions]()},
{"sink", "null", Local, false, false, "Discards; holds a sink's place until one is chosen", reflect.TypeFor[NullSinkOptions]()}, {"sink", "null", Local, false, false, "Discards; holds a sink's place until one is chosen", reflect.TypeFor[NullSinkOptions]()},
{"sink", "tcp", Listener, false, false, "Serve entries to TCP clients, one per line", reflect.TypeFor[TCPSinkOptions]()}, {"sink", "tcp", Listener, false, false, "Serve entries to TCP clients, one per line", reflect.TypeFor[TCPSinkOptions]()},
{"sink", "http", Listener, true, false, "Serve entries as server-sent events, with a browser viewer", reflect.TypeFor[HTTPSinkOptions]()}, {"sink", "http", Listener, true, false, "Serve entries as server-sent events, with a browser viewer", reflect.TypeFor[HTTPSinkOptions]()},
{"sink", "tcp_chain", Dialer, false, false, "Forward entries to a tcp_chain source", reflect.TypeFor[TCPChainSinkOptions]()}, {"sink", "tcp_chain", Dialer, false, false, "Forward entries to another node's tcp_chain source over one lasting TCP connection", reflect.TypeFor[TCPChainSinkOptions]()},
{"sink", "http_chain", Dialer, true, false, "Post entry batches to an http_chain source", reflect.TypeFor[HTTPChainSinkOptions]()}, {"sink", "http_chain", Dialer, true, false, "Forward entries to another node's http_chain source in batches, an HTTP POST each", reflect.TypeFor[HTTPChainSinkOptions]()},
} }
// Plugins lists the catalogue // Plugins lists the catalogue
@@ -64,12 +64,9 @@ func Plugins() []Plugin { return slices.Clone(plugins) }
func (p Plugin) Keys() []Key { return p.applying(keys(p.options)) } func (p Plugin) Keys() []Key { return p.applying(keys(p.options)) }
func (p Plugin) applying(all []Key) []Key { func (p Plugin) applying(all []Key) []Key {
side, out := "listener", []Key{} out := []Key{}
if p.Side == Dialer {
side = "dialer"
}
for _, k := range all { for _, k := range all {
if (k.Side == "" || k.Side == side) && (!k.HTTP || p.HTTP) { if (k.Side == "" || k.Side == sideNames[p.Side] || k.Side == "listener" && p.Side == ChainListener) && (!k.HTTP || p.HTTP) {
if k.Keys != nil { if k.Keys != nil {
k.Keys = p.applying(k.Keys) k.Keys = p.applying(k.Keys)
} }
@@ -99,10 +96,10 @@ func (e *KeyError) Error() string { return e.Key + ": " + e.Err.Error() }
func (e *KeyError) Unwrap() error { return e.Err } func (e *KeyError) Unwrap() error { return e.Err }
// option is one key of an options table, from its field's tags: toml names // option is one key of an options table, from its field's tags: toml names
// it, default: holds the default as typed on a command line, help: says what // it, default: holds the default as a command line types it, help: says what
// it does, and lw: lists its rules: required, enum=a|b, min=N, max=N, // it does, lw: lists its rules: required, enum=a|b, min=N, max=N, zero=MEANING
// zero=MEANING (zero is a value, not the default), hint=KIND, listener or // (a value, not the default), hint=KIND, a side (listener, a chain source's
// dialer, and http (only plugins that speak HTTP take it). // too; chain_listener; dialer), and http (only plugins that speak HTTP take it).
type option struct { type option struct {
Key Key
index int index int
@@ -125,7 +122,7 @@ func options(t reflect.Type) []option {
case "": case "":
case "required": case "required":
o.Required = true o.Required = true
case "listener", "dialer": case "listener", "chain_listener", "dialer":
o.Side = name o.Side = name
case "http": case "http":
o.HTTP = true o.HTTP = true
+3 -2
View File
@@ -173,7 +173,8 @@ func TestLoadReportsEveryProblemALine(t *testing.T) {
} }
// A plugin lists the table keys that apply to it: those of its side, a chain // A plugin lists the table keys that apply to it: those of its side, a chain
// source's being a listener's, and the HTTP ones only when it speaks HTTP // source's being a listener's and its own, and the HTTP ones only when it
// speaks HTTP
func TestTableKeysFollowThePlugin(t *testing.T) { func TestTableKeysFollowThePlugin(t *testing.T) {
has := func(role, typ, path string) bool { has := func(role, typ, path string) bool {
p, _ := LookupPlugin(role, typ) p, _ := LookupPlugin(role, typ)
@@ -190,7 +191,7 @@ func TestTableKeysFollowThePlugin(t *testing.T) {
for path, want := range map[string]bool{ for path, want := range map[string]bool{
"sink tcp_chain tls.pin_sha256": true, "sink tcp_chain tls.self_signed": false, "sink tcp_chain tls.pin_sha256": true, "sink tcp_chain tls.self_signed": false,
"source tcp_chain tls.self_signed": true, "source tcp_chain tls.pin_sha256": false, "source tcp_chain tls.self_signed": true, "source tcp_chain tls.pin_sha256": false,
"source tcp_chain auth.node_binding": true, "sink tcp_chain auth.credentials_file": false, "source tcp_chain auth.node_binding": true, "sink http auth.node_binding": false, "sink tcp_chain auth.credentials_file": false,
"sink tcp acl.requests_per_second_per_client": false, "sink tcp auth.token_lifetime_ms": false, "sink tcp acl.requests_per_second_per_client": false, "sink tcp auth.token_lifetime_ms": false,
"sink http acl.requests_per_second_per_client": true, "sink http auth.trusted_proxies": true, "sink http acl.requests_per_second_per_client": true, "sink http auth.trusted_proxies": true,
} { } {
+13 -7
View File
@@ -59,10 +59,10 @@ func ValidatePipelines(pipelines []PipelineConfig) error {
} }
names[p.Name] = true names[p.Name] = true
if len(p.PluginSources) == 0 { if len(p.PluginSources) == 0 {
at(path+".plugin_sources", errors.New("no sources defined")) at(path+".plugin_sources", errors.New("add a source; a pipeline needs at least one"))
} }
if len(p.PluginSinks) == 0 { if len(p.PluginSinks) == 0 {
at(path+".plugin_sinks", errors.New("no sinks defined")) at(path+".plugin_sinks", errors.New("add a sink; a pipeline needs at least one"))
} }
if err := checkFlow(p.Flow, path+".flow"); err != nil { if err := checkFlow(p.Flow, path+".flow"); err != nil {
errs = append(errs, err) errs = append(errs, err)
@@ -77,7 +77,7 @@ func ValidatePipelines(pipelines []PipelineConfig) error {
case !ok: case !ok:
at(path+".type", fmt.Errorf("unknown %s type %q", role, typ)) at(path+".type", fmt.Errorf("unknown %s type %q", role, typ))
case plugin.Single && single[role+typ] != "": case plugin.Single && single[role+typ] != "":
at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it", typ, role, single[role+typ])) at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it, as lw has one standard input; give this pipeline another %s", typ, role, single[role+typ], role))
default: default:
ids[list+id] = true ids[list+id] = true
if plugin.Single { if plugin.Single {
@@ -414,15 +414,17 @@ func ParsePrefix(entry, network string) (netip.Prefix, error) {
if err != nil { if err != nil {
addr, aerr := netip.ParseAddr(entry) addr, aerr := netip.ParseAddr(entry)
if aerr != nil || addr.Zone() != "" { if aerr != nil || addr.Zone() != "" {
return netip.Prefix{}, fmt.Errorf("neither an address nor a CIDR (and no zone)") return netip.Prefix{}, errors.New("not an IP address or CIDR; host names and IPv6 zones are not accepted")
} }
prefix = netip.PrefixFrom(addr, addr.BitLen()) prefix = netip.PrefixFrom(addr, addr.BitLen())
} }
switch a := prefix.Addr(); { switch a := prefix.Addr(); {
case a.Is4In6(): case a.Is4In6():
return netip.Prefix{}, fmt.Errorf("IPv4-mapped; write it as IPv4") return netip.Prefix{}, fmt.Errorf("IPv4-mapped; write it as IPv4")
case network == "tcp4" && !a.Is4(), network == "tcp6" && a.Is4(): case network == "tcp4" && !a.Is4():
return netip.Prefix{}, fmt.Errorf("not of the listener's family (%s)", network) return netip.Prefix{}, errors.New("IPv6, and the listener's host is IPv4; write an IPv4 address or CIDR")
case network == "tcp6" && a.Is4():
return netip.Prefix{}, errors.New("IPv4, and the listener's host is IPv6; write an IPv6 address or CIDR")
} }
return prefix, nil return prefix, nil
} }
@@ -520,11 +522,15 @@ func (c *FilterConfig) Check() error {
} }
// Scan decodes a flow stage's command-line map into target, rejecting keys // Scan decodes a flow stage's command-line map into target, rejecting keys
// target does not declare: a misspelled key must fail, not be dropped. // target does not declare: a misspelled key must fail, not be dropped. Its
// values take their kinds as a plugin's do, a wrong one refused at its key.
func Scan(configMap map[string]any, target any) error { func Scan(configMap map[string]any, target any) error {
if err := checkKeys(configMap, reflect.TypeOf(target), ""); err != nil { if err := checkKeys(configMap, reflect.TypeOf(target), ""); err != nil {
return err return err
} }
if err := coerce(reflect.TypeOf(target).Elem(), configMap, ""); err != nil {
return err
}
return lconfig.ScanMap(configMap, target) return lconfig.ScanMap(configMap, target)
} }
+2 -1
View File
@@ -811,7 +811,8 @@ func TestPresetFormShowsTheChoices(t *testing.T) {
ps := config.Presets() ps := config.Presets()
tail := ps[slices.IndexFunc(ps, func(p config.Preset) bool { return p.Name == "tail" })].Params tail := ps[slices.IndexFunc(ps, func(p config.Preset) bool { return p.Name == "tail" })].Params
format := tail[slices.IndexFunc(tail, func(p config.PresetParam) bool { return p.Name == "format" })] format := tail[slices.IndexFunc(tail, func(p config.PresetParam) bool { return p.Name == "format" })]
if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.Contains(screen, format.Usage()) || len(format.Enum) == 0 { note := strings.Join(format.Enum, "|") + ": " + strings.Fields(format.Help)[0] // the help wraps
if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.Contains(screen, note) || len(format.Enum) == 0 {
t.Fatalf("format's choices not shown:\n%s", screen) t.Fatalf("format's choices not shown:\n%s", screen)
} }
} }
+1 -1
View File
@@ -96,7 +96,7 @@ section "Rules"
rejects() { ! lw --check --source null --sink "$1" >/dev/null 2>"$LOG/check.err" && grep -q -- "$2" "$LOG/check.err" && echo 1 || echo 0; } rejects() { ! lw --check --source null --sink "$1" >/dev/null 2>"$LOG/check.err" && grep -q -- "$2" "$LOG/check.err" && echo 1 || echo 0; }
# warns SINK PATTERN: lw --check builds SINK and WARNs PATTERN # warns SINK PATTERN: lw --check builds SINK and WARNs PATTERN
warns() { lw --check --source null --sink "$1" 2>&1 >/dev/null | grep ' WARN ' | grep -q -- "$2" && echo 1 || echo 0; } warns() { lw --check --source null --sink "$1" 2>&1 >/dev/null | grep ' WARN ' | grep -q -- "$2" && echo 1 || echo 0; }
check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp:host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")" check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp:host=::1,port=15881,acl.allow=10.0.0.0/8' "IPv4, and the listener's host is IPv6")"
check "an acl on a dialer is refused" "$(rejects 'tcp_chain:host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')" check "an acl on a dialer is refused" "$(rejects 'tcp_chain:host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')"
check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp:host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')" check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp:host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')"
check "a deny list alone on a wildcard listener warns" "$(warns 'tcp:port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')" check "a deny list alone on a wildcard listener warns" "$(warns 'tcp:port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')"