diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index c412543..3a69863 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -66,14 +66,15 @@ jobs: test ! -e /etc/logwisp/logwisp.toml lw config init lw --check - lw --dump | grep -q 'id = "default_source"' + lw --dump | grep -q '^status_reporter = true' if lw config init; then exit 1; fi make uninstall PREFIX="$HOME/.local" test ! -e "$HOME/.local/bin/lw" test -f "$HOME/.config/logwisp/logwisp.toml" - # make deb on a host with systemd: the service runs from the package's - # configuration and survives a reinstall that keeps an edited one; remove - # stops and disables it and leaves the conffile, purge removes that too + # make deb on a host with systemd: the service runs once the package's + # configuration, which ships no pipeline, gets one, and survives a reinstall + # that keeps it; remove stops and disables it and leaves the conffile, purge + # removes that too deb: runs-on: ubuntu-latest timeout-minutes: 10 @@ -87,6 +88,7 @@ jobs: sudo apt-get install -y ./bin/logwisp_*.deb lw --version test -f /usr/share/zsh/vendor-completions/_lw + printf '[[pipelines]]\nname = "ci"\n[[pipelines.plugin_sources]]\nid = "in"\ntype = "null"\n[[pipelines.plugin_sinks]]\nid = "out"\ntype = "null"\n' | sudo tee -a /etc/logwisp/logwisp.toml sudo systemctl enable --now logwisp sleep 3 systemctl is-active logwisp || { sudo journalctl -u logwisp --no-pager; exit 1; } @@ -104,7 +106,7 @@ jobs: sudo apt-get purge -y logwisp test ! -e /etc/logwisp/logwisp.toml # BSD make on FreeBSD: build, install twice keeping an edited configuration, - # run the rc.d service, uninstall + # run the rc.d service once the configuration has a pipeline, uninstall freebsd: runs-on: ubuntu-latest timeout-minutes: 20 @@ -125,6 +127,7 @@ jobs: lw --version pw groupadd logwisp pw useradd logwisp -g logwisp -d /nonexistent -s /usr/sbin/nologin + printf '[[pipelines]]\nname = "ci"\n[[pipelines.plugin_sources]]\nid = "in"\ntype = "null"\n[[pipelines.plugin_sinks]]\nid = "out"\ntype = "null"\n' >> /usr/local/etc/logwisp/logwisp.toml service logwisp onestart sleep 3 service logwisp onestatus diff --git a/Makefile b/Makefile index 5a0703c..25bbe55 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ IMAGE_RUN = $(CONTAINER_ENGINE) run --read-only --user 65532:65532 \ --network none --cap-drop ALL --security-opt no-new-privileges DESTDIR ?= -# SERVICE=no installs lw alone: no service files and no system configuration, +# SERVICE=no leaves out the service files and the system configuration, # which lw config init then places where the user wants it. SERVICE ?= yes INSTALL_OS_DEFAULT != uname -s @@ -234,7 +234,7 @@ install: put 0644 $(PKG_DIR)/completion/_lw "$$root$(ZSHCOMPDIR)/_lw"; \ put 0644 $(PKG_DIR)/completion/lw.fish "$$root$(FISHCOMPDIR)/lw.fish"; \ put 0644 LICENSE "$$root$(LICENSEDIR)/LICENSE"; \ - for src in README.md doc/*.md; do put 0644 "$$src" "$$root$(DOCDIR)/$$src"; done; \ + for src in README.md logwisp-logo.svg doc/*.md doc/lw.1 config/logwisp.toml; do put 0644 "$$src" "$$root$(DOCDIR)/$$src"; done; \ if [ '$(SERVICE)' = no ]; then echo "skip the service and $$etc (SERVICE=no): lw config init writes a configuration"; exit 0; fi; \ if [ -e "$$etc/logwisp.toml" ]; then echo "keep $$etc/logwisp.toml"; \ else put 0644 config/logwisp.toml "$$etc/logwisp.toml"; fi; \ diff --git a/README.md b/README.md index dc7a0c1..2f5ebde 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,18 @@ streams, or downstream LogWisp nodes. ### Inputs `file` (directory tail with rotation detection and JSON line parsing), -`console` (stdin, one reader per process), `random` (synthetic generator), -`null`, and the chain ingest listeners `tcp_chain` and `http_chain`. +`console` (stdin, one reader per process), `random` (generated entries, to +check a deployment before real logs), `null` (nothing; holds a source's place +until one is chosen), and the chain ingest listeners `tcp_chain` and +`http_chain`. ### Outputs `console` (level names in color and control characters escaped on a terminal), `file` (rotating with retention), `http` (Server-Sent Events, a JSON status endpoint and a browser -viewer), `tcp` (broadcast server), `null`, and the chain -forwarders `tcp_chain` and `http_chain`. +viewer), `tcp` (broadcast server), `null` (discards; holds a sink's place +until one is chosen), and the chain forwarders `tcp_chain` and `http_chain`. ### Processing @@ -87,13 +89,14 @@ exponential backoff and jitter. - Node binding: a chain source can label entries from the sender's certificate identity or username instead of from what the sender claims, so origin attribution is not forgeable -- Fail-closed configuration: unknown keys are rejected, and startup warns about - expiring certificates, disabled verification, unanchored allow patterns and - world-readable secrets +- Address rules: an `acl` block admits or refuses peers by address, or by the + client an L4 proxy's PROXY header names, and limits each client +- Fail-closed configuration: every option is validated when the file loads, + unknown keys included, and startup warns about expiring certificates, + disabled verification, unanchored allow patterns and world-readable secrets See [Security](doc/security.md) for configuration and the exact boundary, and -the [mTLS](doc/mtls-auth-plan.md) and [SCRAM](doc/scram-auth-plan.md) -authentication designs for the rationale and what is deliberately left out. +[Current State](doc/security.md#current-state) for what is left out. ## Documentation @@ -102,7 +105,7 @@ authentication designs for the rationale and what is deliberately left out. - [Deployment](doc/deployment.md): `deploy/lw-deploy.sh` for edges, aggregators, containers, services, jails - [Architecture](doc/architecture.md): component model, data flow, - concurrency, back-pressure + concurrency, back-pressure, the configuration engine's WebAssembly API - [Configuration](doc/configuration.md): TOML structure, precedence, environment and CLI overrides - [Sources](doc/sources.md) and [Sinks](doc/sinks.md): every plugin and its @@ -111,15 +114,16 @@ authentication designs for the rationale and what is deliberately left out. matching, output shaping and sanitization - [Chaining](doc/chaining.md): multi-node topologies and the wire protocol - [Networking](doc/networking.md): listeners, dialers, timeouts, limits -- [Security](doc/security.md): TLS, mTLS, peer authorization, threat model -- Design notes: [mTLS](doc/mtls-auth-plan.md) and - [password](doc/scram-auth-plan.md) authentication -- [CLI](doc/cli.md): filter use, flags, pipeline specifications, signals, - exit codes, `lw --tui`, `lw auth`; also the `lw(1)` manual, +- [Security](doc/security.md): TLS, mTLS, password (SCRAM) authentication, + address ACLs, certificates made at startup; threat model and current limits +- [CLI](doc/cli.md): filter use, flags, pipeline specifications, presets, + environment, signals, exit codes, `lw --tui`, `--schema`, `lw auth`, + `lw tls`, `lw preset`, `lw config`; also the `lw(1)` manual, [`doc/lw.1`](doc/lw.1) - [Operations](doc/operations.md): running, monitoring, tuning, troubleshooting -- [To Do](doc/todo.md): planned work: packaging, `lw --tui` and its shared configuration engine +- [To Do](doc/todo.md): the maintainer's release and packaging steps, and + what `lw --tui` and the schema leave for later A fully annotated configuration covering every option ships as [`config/logwisp.toml`](config/logwisp.toml). @@ -191,10 +195,10 @@ docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \ `sudo make install` installs the binary, the manual, shell completion, a systemd unit or FreeBSD rc.d script and its configuration; `make deb` and -`make arch` package them for Debian, Ubuntu and Arch. Without a service, -`make install SERVICE=no PREFIX="$HOME/.local"` installs `lw` alone and -`lw config init` writes the annotated configuration it reads. See -[Installation](doc/installation.md). +`make arch` package them for Debian, Ubuntu and Arch. +`make install SERVICE=no PREFIX="$HOME/.local"` installs `lw`, its manual, +completion and documentation, and no service; `lw config init` writes the +annotated configuration it reads. See [Installation](doc/installation.md). ## System Requirements diff --git a/cmd/lw/cli.go b/cmd/lw/cli.go index 95b416b..50e5922 100644 --- a/cmd/lw/cli.go +++ b/cmd/lw/cli.go @@ -198,7 +198,7 @@ func parseCommandLine(argv []string) (inv invocation, err error) { } specFlags, settings := config.SpecFlags(), config.Settings() unexpected := func(word string) error { - return fmt.Errorf("unexpected argument %q: lw takes options only (a configuration file is -c FILE, a switch's value follows '=')", word) + return fmt.Errorf("unexpected argument %q: lw takes options only (a configuration file is -c FILE, a boolean key's value follows '=')", word) } next := func(i *int) string { if *i+1 < len(argv) && !strings.HasPrefix(argv[*i+1], "-") { diff --git a/cmd/lw/help.go b/cmd/lw/help.go index a506cd0..23d6f9c 100644 --- a/cmd/lw/help.go +++ b/cmd/lw/help.go @@ -26,8 +26,9 @@ Usage: Commands (lw COMMAND -h lists its subcommands): %s Options: a short option is its long one; a value follows as the next argument -or after '=' (-c FILE, --config=FILE), one that starts with '-' or a switch's -after '='; short options do not combine. lw takes options only. +or after '=' (-c FILE, --config=FILE), one starting with '-' after '='; a +BOOL key's only after '=', and -t, --dump, -T, -V, --schema take none. +Short options do not combine. lw takes options only. -c, --config FILE Configuration file (default: ~/.config/logwisp/logwisp.toml if it exists, else ./logwisp.toml) @@ -82,9 +83,10 @@ key alone means true: Pipelines (they replace the configuration file's pipelines): -p, --preset NAME[:KEY=VALUE,...] - Start a pipeline with a preset: pipe, tail, - serve, edge, aggregator; lw preset NAME -h - lists its keys, lw preset NAME prints it + Begin a pipeline with a preset (a second + needs its own --pipeline): pipe, tail, serve, + edge, aggregator; lw preset NAME -h lists + its keys, lw preset NAME prints it --pipeline NAME Start a pipeline named NAME, which the pipeline options after it configure; those before any --pipeline make one named after diff --git a/cmd/lw/reload_test.go b/cmd/lw/reload_test.go index 28ac59a..c4ba4f3 100644 --- a/cmd/lw/reload_test.go +++ b/cmd/lw/reload_test.go @@ -39,26 +39,15 @@ func TestWatchErrorsDoNotTriggerReloadAndChangesAreCoalesced(t *testing.T) { } } -func TestShippedConfigurationBuildsWithCheckedPluginDecoder(t *testing.T) { - testLogger(t) +// The shipped file defines no pipeline, so lw keeps its built-in pipe +func TestShippedConfigurationKeepsTheBuiltInPipeline(t *testing.T) { path, err := filepath.Abs(filepath.Join("..", "..", "config", "logwisp.toml")) if err != nil { t.Fatal(err) } - cfg := loadTestConfig(t, path) - if len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "default" { - t.Fatalf("sample array-of-tables decoded incorrectly: %+v", cfg.Pipelines) + if cfg := loadTestConfig(t, path); len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "pipe" { + t.Fatalf("the sample defines pipelines: %+v", cfg.Pipelines) } - svc, err := bootstrapService(context.Background(), cfg, nil) - if err != nil { - t.Fatal(err) - } - defer svc.Shutdown() - pl, err := svc.GetPipeline("default") - if err != nil || len(pl.Sources) != 1 || len(pl.Sinks) != 1 { - t.Fatalf("sample plugins failed to build: %v", err) - } - svc.Shutdown() // A later reload failure or process exit can close it again. } // Once the first service is built a reload reads no descriptor, pipe or diff --git a/cmd/lwconf/lwconf.js b/cmd/lwconf/lwconf.js index 32f8574..93b68a8 100644 --- a/cmd/lwconf/lwconf.js +++ b/cmd/lwconf/lwconf.js @@ -55,7 +55,7 @@ export function fields(schema, role, type) { const rows = (keys, prefix) => keys.map(key => { const path = prefix + key.key; if (Array.isArray(key.keys)) return { path, key, rows: rows(key.keys, path + ".") }; - return { path, key, placeholder: key.zero ? `(${key.zero})` : (key.default ?? key.hint ?? "") }; + return { path, key, placeholder: key.default ?? (key.zero ? `(${key.zero})` : (key.hint ?? "")) }; }); return rows(plugin?.keys ?? schema.flow.find(s => s.key === role)?.keys ?? [], ""); } diff --git a/cmd/lwconf/lwconf.test.mjs b/cmd/lwconf/lwconf.test.mjs index a6c64da..6a2d1b5 100644 --- a/cmd/lwconf/lwconf.test.mjs +++ b/cmd/lwconf/lwconf.test.mjs @@ -152,6 +152,7 @@ test("fields are a node's keys, its tables' of the plugin's side as groups", () assert.deepEqual(rows.find(r => r.path === "port").key.max, 65535); assert.equal(fields(schema, "rate_limit").find(r => r.path === "entries_per_second").placeholder, "(off)"); assert.equal(fields(schema, "heartbeat").find(r => r.path === "interval_ms").placeholder, "1000"); + assert.equal(fields(schema, "sink", "file").find(r => r.path === "retention_hours").placeholder, "168"); }); test("a refused call answers why and where, and the engine keeps answering", () => { diff --git a/config/logwisp.toml b/config/logwisp.toml index 58a9202..b74be60 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -1,310 +1,333 @@ ############################################################################### -### LogWisp Configuration -### Default location: ~/.config/logwisp/logwisp.toml -### Precedence: CLI flags > Environment > File > Defaults +### LogWisp configuration +### lw reads -c FILE, else LOGWISP_CONFIG_FILE / LOGWISP_CONFIG_DIR, else +### ~/.config/logwisp/logwisp.toml if it exists, else ./logwisp.toml +### Precedence: flags > environment > file > built-in defaults; pipeline flags +### or variables replace the file's pipelines whole ### -### Commented values are the built-in defaults unless marked "example". -### Uncommenting a default is a no-op. +### Every key is listed with its built-in default, commented: uncommenting a +### default is a no-op. Values marked "example" are not defaults, and keys +### marked "Required" have none. As shipped the file defines no pipeline, so +### lw runs its built-in one, stdin to stdout. ### -### NOTE: http/tcp ingest sources and http_client/tcp_client sinks are not -### available in this version. TLS is available on every network source and -### sink (the [...tls] blocks below), the [...auth] blocks authenticate peers by -### certificate (mtls) or password (scram; credentials via `lw auth`), and the -### listeners' [...acl] blocks admit peers by address, or by the client an L4 -### proxy's PROXY header names, and limit each client. Unknown keys anywhere in -### this file or in any plugin config are rejected at startup. +### Every setting is also a flag (--logging.level=debug) and a variable +### (LOGWISP_LOGGING_LEVEL=debug). A pipeline is set whole by pipeline flags +### (--source, --sink, --preset, ...) or variables (LOGWISP_SOURCE, ...), never +### key by key. ### -### Environment overrides use the LOGWISP_ prefix -### (LOGWISP_QUIET, LOGWISP_LOGGING_LEVEL, ...). Bare names are ignored. -### LOGWISP_CONFIG_FILE and LOGWISP_CONFIG_DIR select the file. Paths such as -### pipelines.0.name cannot be set from the CLI or environment at all. +### TLS is available on every network source and sink (the [...tls] tables), +### [...auth] authenticates peers by certificate (mtls) or password (scram; +### credentials via `lw auth`), and the listeners' [...acl] admits peers by +### address, or by the client an L4 proxy's PROXY header names, and limits +### each client. Unknown keys are refused, except a top-level config_file, +### version, check or dump, which is ignored. ############################################################################### ############################################################################### -### Global Settings +### Settings ############################################################################### -quiet = false # No log of lw's own; pipeline output still flows -status_reporter = true # Periodic status logging (30s, DEBUG level); false without a file -auto_reload = false # Watch file; read at startup (signals always reread it) -color = "auto" # auto|always|never: level names on console sinks; auto on a terminal, NO_COLOR unset +# quiet = false # No log of lw's own; pipeline output still flows +# status_reporter = true # Periodic status logging (30s, DEBUG level); false without a file +# auto_reload = false # Watch file; read at startup (signals always reread it) +# color = "auto" # auto|always|never: level names on console sinks; auto on a terminal, NO_COLOR unset ############################################################################### -### Logging (LogWisp's internal operational logging) +### Logging (lw's own log) ############################################################################### -[logging] -output = "stderr" # file|stdout|stderr|split|all|none; stdout is the console sink's -level = "info" # debug|info|warn|error; warn without a file +# [logging] +# output = "stderr" # file|stdout|stderr|split|all|none; stdout is the console sink's +# level = "info" # debug|info|warn|error; warn without a file # format = "txt" # raw|txt|json -# sanitization = "" # raw|json|txt|shell (empty = logger default) +# sanitization = "" # raw|json|txt|shell; empty = the log default # [logging.file] # Used when output is "file" or "all" -# directory = "./log" +# directory = "./log" # lw's alone: the limits below delete any .log file in it # name = "logwisp" # max_size_mb = 100 -# max_total_size_mb = 1000 -# retention_hours = 168.0 # 7 days +# max_total_size_mb = 1000 # past it the oldest .log files there are deleted +# retention_hours = 168.0 # .log files there older than this are deleted; 0 keeps them ## Validated but NOT applied: the console destination comes from `output` above. # [logging.console] -# target = "stdout" # stdout|stderr|split +# target = "stderr" # stdout|stderr|split ############################################################################### -### Pipelines -### Each pipeline: plugin_sources -> flow (rate_limit|filters|format) -> plugin_sinks -### Names must be unique. 1+ source and 1+ sink required. +### Pipelines: plugin_sources -> flow -> plugin_sinks +### A pipeline has a unique name, one source and one sink at least. The blocks +### below form one pipeline holding every source and sink: uncomment +### [[pipelines]], its name and the blocks it needs. ############################################################################### -[[pipelines]] -name = "default" +# [[pipelines]] +# name = "default" # example ###============================================================================ -### Flow (processing between sources and sinks) +### Flow: the stages between sources and sinks, validated whether on or off ###============================================================================ -## policy="pass" short-circuits the size check too: enforcing a size cap needs -## entries_per_second > 0 AND policy = "drop". +## policy = "pass" lets every entry through, the size check included: a limit +## needs entries_per_second > 0 and policy = "drop". # [pipelines.flow.rate_limit] -# entries_per_second = 0.0 # 0 = limiter disabled -# burst_entries = 0.0 # Bucket capacity (0 = entries_per_second) +# entries_per_second = 0.0 # 0 = off +# burst_entries = 0.0 # Bucket capacity; 0 = entries_per_second # policy = "pass" # pass|drop # max_entry_size_bytes = 0 # 0 = unlimited -## Filters: sequential include/exclude chain, matched against " " +## Filters run in order, each matching " " # [[pipelines.flow.filters]] # type = "include" # include|exclude -# logic = "or" # or|and -# patterns = [".*ERROR.*", ".*WARN.*"] # example; RE2 syntax +# logic = "or" # or|and: any pattern, or all +# patterns = [] # RE2, e.g. [".*ERROR.*", ".*WARN.*"]; none passes every entry # [pipelines.flow.format] -# type = "raw" # raw|json|txt -# flags = 0 # Formatter flags (0 = defaults per type) -# timestamp_format = "" # Go time layout (formatter default if empty) -# sanitizer_policy = "" # raw|json|txt|shell (defaults per type) +# type = "raw" # raw|txt|json +# flags = 0 # Formatter flags; 0 = by type +# timestamp_format = "" # Go time layout; empty = RFC 3339 with nanoseconds +# sanitizer_policy = "" # raw|json|txt|shell; empty = by type -## Flow-level heartbeat (fan-out to all sinks, traverses chain links) +## Heartbeat: sent to every sink, through chain links # [pipelines.flow.heartbeat] # enabled = false -# interval_ms = 1000 # Minimum 100 -# include_timestamp = false -# include_stats = false -# format = "txt" # accepted, no effect: heartbeats use flow.format +# interval_ms = 1000 # At least 100 +# include_timestamp = false # false: level only; the raw format shows neither +# include_stats = false # Adds type, beat_count and, from the second beat, interval_ms +# format = "txt" # txt|json|raw; no effect: heartbeats use the flow's format ###============================================================================ -### TLS (shared shape; applies to every network source and sink) +### TLS (a [...tls] table on every network source and sink) +### Listeners (tcp/http sinks, tcp_chain/http_chain sources) take one of +### cert_file + key_file, self_signed, or issuer_cert_file + issuer_key_file; +### client_auth + client_ca_file make it mTLS. Dialers (tcp_chain/http_chain +### sinks) verify the server by ca_file + server_name, or by pin_sha256, and +### present cert_file + key_file for mTLS. `lw tls ca` / `lw tls cert` make a +### CA and certificates. A table not enabled has no effect; only its keys +### and kinds are checked. ### -### Listeners (tcp/http sinks, tcp_chain/http_chain sources): -### cert_file + key_file, self_signed, or issuer_cert_file + issuer_key_file -### (exactly one); client_auth + client_ca_file enable mTLS. -### Dialers (tcp_chain/http_chain sinks): -### ca_file + server_name, or pin_sha256, verify the server; cert_file + -### key_file present a client identity for mTLS. -### `lw tls ca` / `lw tls cert` make a CA and certificates. -### -### enabled = false Master switch -### cert_file = "" Local certificate -### key_file = "" Private key for cert_file (set together) -### client_auth = false Listener: require and verify a client certificate -### client_ca_file = "" Listener: CA bundle verifying client certs -### ca_file = "" Dialer: CA bundle verifying the server (empty = system) -### server_name = "" Dialer: SNI / name to verify (empty = configured host) -### self_signed = false Listener: make a self-signed certificate at startup; -### it logs the pin_sha256 dialers set (changes on restart) -### issuer_cert_file = "" Listener: make one at startup signed by this CA ... -### issuer_key_file = "" ... and its key (lw tls ca writes both) -### hosts = [] Listener: names/addresses of a made certificate, beyond -### host, os.Hostname(), localhost, 127.0.0.1, ::1 -### insecure_skip_verify = false Dialer: disable verification (never in prod) -### pin_sha256 = "" Dialer: sha256//BASE64 of the server key, in place of -### ca_file; ';' between several -### min_version = "1.3" "1.2" or "1.3". No max_version, no cipher_suites. +### enabled master switch +### cert_file, key_file the certificate presented and its key, set together +### self_signed listener: make a self-signed certificate at startup; +### it logs the pin_sha256 dialers set (new each start) +### issuer_cert_file, issuer_key_file +### listener: make one at startup, signed by this CA +### (lw tls ca writes both) +### hosts listener: names/addresses of a made certificate, +### beyond host, os.Hostname(), localhost, 127.0.0.1, ::1 +### client_auth listener: require and verify a client certificate +### client_ca_file listener: CA bundle verifying client certificates +### ca_file dialer: server CA bundle; empty = the system roots +### server_name dialer: name to verify; empty = the configured host +### insecure_skip_verify dialer: verify nothing (never in production) +### pin_sha256 dialer: sha256//BASE64 of the server key, in place of +### ca_file; ';' between several +### min_version "1.2" or "1.3"; no max_version, no cipher_suites ### ### TLS alone is a CA membership check: ANY certificate the CA signed is -### accepted. Add an [...auth] block to decide WHICH of them may connect. +### accepted. An [...auth] table decides WHICH of them may connect. ###============================================================================ ###============================================================================ -### AUTH (shared shape; sits beside [...tls] on every network source and sink) -### -### Both methods REQUIRE tls.enabled; dialers forbid tls.insecure_skip_verify. -### On the http sink the block gates stream_path AND status_path. -### Listeners (tcp/http sinks, tcp_chain/http_chain sources): +### AUTH (an [...auth] table beside [...tls] on every network source and sink) +### Both methods REQUIRE tls.enabled, except scram behind trusted_proxies on +### the http sink; dialers forbid tls.insecure_skip_verify. A table naming +### peers or credentials with type = "none" is refused. +### Listeners (tcp/http sinks, tcp_chain/http_chain sources): ### mtls authorizes the client certificate; REQUIRES tls.client_auth. ### scram admits the users in credentials_file (`lw auth add-user`), or ### the one username whose password is in password_file. -### Dialers (tcp_chain/http_chain sinks): +### Dialers (tcp_chain/http_chain sinks): ### mtls pins the server identity. ### scram logs in as username with the password in password_file. -### Chain sources additionally bind the node label to the identity or username. -### A block naming peers or credentials with type = "none" is refused. ### -### type = "none" none | mtls | scram -### identity = "cn" cn | san_dns | san_uri | san_email. Unset under scram; -### set on a scram listener (needs tls.client_auth), the -### certificate's field must equal the username. -### allow = [] mtls: exact identities. Empty allow AND allow_patterns -### admits any identity the CA vouches for (logged WARN). -### allow_patterns = [] mtls: RE2 patterns; anchor them yourself (^...$) -### node_binding = "" Chain sources only, default "force" with mtls or scram: -### none - trust_node governs, as before -### assert - declared label must equal the identity; -### per-entry node labels still follow trust_node -### force - label AND every entry take the identity -### Overrides trust_node. See doc/security.md. -### credentials_file = "" scram listeners: verifiers; read at (re)load, not watched -### token_lifetime_ms = 0 scram, http sink and http_chain source: 10 s to 24 h, 0 = 15 minutes -### username = "" scram dialers, or a listener's one user -### password_file = "" scram, with username: one trailing line break is -### trimmed; /dev/fd/N or a pipe is read once and kept -### for reloads, /dev/tty asks once -### trusted_proxies = [] scram http sink: TLS-terminating reverse proxies -### (addresses or CIDRs); browsers log in through them +### type none | mtls | scram +### identity cn | san_dns | san_uri | san_email; unset = cn under +### mtls. Set on a scram listener, it binds the client +### certificate to the user (needs tls.client_auth) +### allow mtls: exact identities. Empty allow AND +### allow_patterns admits any identity the CA vouches +### for (logged WARN) +### allow_patterns mtls: RE2 patterns; anchor them yourself (^...$) +### node_binding chain sources: binds the node label to the identity +### or username, overriding trust_node; unset = force +### none - trust_node governs +### assert - declared label must equal the identity; +### per-entry node labels follow trust_node +### force - label AND every entry take the identity +### credentials_file scram listener: verifiers, read at (re)load only +### token_lifetime_ms scram, HTTP listeners: 10 s to 24 h; 0 = 15 minutes +### username scram: a dialer's user, or a listener's one user +### password_file scram, with username: one trailing line break is +### trimmed; /dev/fd/N or a pipe is read once and kept +### for reloads, /dev/tty asks once +### trusted_proxies scram, http sink: TLS-terminating reverse proxies +### (addresses or CIDRs); browsers log in through them ###============================================================================ ###============================================================================ -### ACL (beside [...tls] and [...auth] on the listeners; dialers have none) -### +### ACL (an [...acl] table beside [...tls] and [...auth] on the listeners) ### Applied as a connection is accepted, before TLS: a refused peer is closed ### at once and counted in acl_denied (acl_limited past a limit), with a WARN ### at most once a minute. -### allow = [] addresses or CIDRs of the listener's family (either behind -### proxy_from or trusted_proxies); empty = all -### deny = [] refused, listed in allow or not -### proxy_protocol = "off" off | optional | required: PROXY v1/v2 headers, -### read before TLS; the rules then see the client. -### optional lets a headerless stream pass as the -### proxy: use it only while rolling headers out -### proxy_from = [] the proxies that may send one; a header from any -### other peer is refused -### max_connections_per_client = 0 per client (an IPv6 one by its /64); -### refused behind trusted_proxies -### requests_per_second_per_client = 0 HTTP listeners, /auth aside: past it, -### 429; behind trusted_proxies, per -### forwarded client +### +### allow addresses or CIDRs of the listener's family (either +### behind proxy_from or trusted_proxies); empty = all +### deny refused, listed in allow or not +### proxy_protocol off | optional | required: PROXY v1/v2 headers, read +### before TLS; the rules then see the client. optional +### lets a headerless stream pass as the proxy: use it +### only while rolling headers out +### proxy_from the proxies that may send one; a header from any +### other peer is refused +### max_connections_per_client +### per client (an IPv6 one by its /64); 0 = unlimited; +### refused behind trusted_proxies +### requests_per_second_per_client +### HTTP listeners, /auth aside: past it, 429; behind +### trusted_proxies, per forwarded client; 0 = unlimited ###============================================================================ ###============================================================================ -### Sources (1+ required) +### Sources (one at least; ids are examples, unique among a pipeline's sources) ###============================================================================ -## Null source (testing) +## Null source: nothing; holds a source's place until one is chosen # [[pipelines.plugin_sources]] # id = "null_in" # type = "null" -[[pipelines.plugin_sources]] -id = "default_source" -type = "file" -[pipelines.plugin_sources.config] -directory = "./" # Directory to monitor (required, not recursive) -pattern = "*.log" # Glob pattern (* and ? only) -## Tailing an already-open file polls at a fixed 100ms, regardless of this value. -check_interval_ms = 100 # Directory rescan interval (min 10) -## raw = true never parses a line; with format type "raw" the file is relayed byte for byte. -raw = false # Keep the whole line as the message -from = "end" # "end" or "start" of a newly discovered file +## File source: follows the files of a directory that match a pattern +# [[pipelines.plugin_sources]] +# id = "file_in" +# type = "file" +# [pipelines.plugin_sources.config] +# directory = "/var/log/app" # Required (example); not recursive +# pattern = "*" # Glob of the whole file name (* and ? only) +# check_interval_ms = 100 # Directory rescan, at least 10; open files are polled every 100 ms +## raw = true never parses a line; with format type "raw" each line is relayed +## as written, except that empty lines are skipped and CRLF becomes LF. +# raw = false +# from = "end" # end|start: where a newly found file is read from -## Console source (stdin; one in the whole configuration, ends lw's pipeline at end of input) +## Console source (stdin; one in the whole configuration). lw exits at end of +## input only when it is the configuration's only source. # [[pipelines.plugin_sources]] # id = "console_in" # type = "console" # [pipelines.plugin_sources.config] # buffer_size = 1000 -## Random source (testing; special=true exercises sanitizer policies) +## Random source: generated entries, to check a deployment before real logs +## (paths, permissions, sinks, chains; special = true for sanitizer policies) # [[pipelines.plugin_sources]] # id = "random_in" # type = "random" # [pipelines.plugin_sources.config] # interval_ms = 500 -# jitter_ms = 0 # Clamped to interval_ms +# jitter_ms = 0 # The interval varies by it, half each way; at most interval_ms # format = "txt" # raw|txt|json -# length = 20 -# special = false +# length = 20 # Bytes of random text +# special = false # Mix in control and non-ASCII bytes -## TCP chain source (stdlib listener; receives NDJSON from upstream tcp_chain sinks) -## Topology: jail [file source -> tcp_chain sink] -> host [tcp_chain source -> aggregate sink] +## TCP chain source (receives NDJSON from upstream tcp_chain sinks) +## Topology: edge [file source -> tcp_chain sink] -> aggregator [tcp_chain source -> sinks] # [[pipelines.plugin_sources]] # id = "chain_in" # type = "tcp_chain" # [pipelines.plugin_sources.config] -# host = "0.0.0.0" # IPv4 only; "::" or "::1": IPv6 only -# port = 9440 # Required +# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# port = 9440 # Required (example) # buffer_size = 1000 # max_connections = 0 # 0 = unlimited -# read_timeout_ms = 0 # idle deadline, 0 = none +# read_timeout_ms = 0 # Idle deadline; 0 = none # hello_timeout_ms = 10000 # Protocol preamble deadline # trust_node = true # false: label entries by remote address # [pipelines.plugin_sources.config.tls] -# enabled = true # example: mTLS listener -# cert_file = "/etc/logwisp/tls/server.crt" -# key_file = "/etc/logwisp/tls/server.key" -# client_auth = true -# client_ca_file = "/etc/logwisp/tls/client-ca.crt" +# enabled = false +# cert_file = "" +# key_file = "" +# self_signed = false +# issuer_cert_file = "" +# issuer_key_file = "" +# hosts = [] +# client_auth = false +# client_ca_file = "" # min_version = "1.3" # [pipelines.plugin_sources.config.auth] # authenticate senders -# type = "none" # none | mtls | scram; mtls requires client_auth -# identity = "cn" # mtls default; under scram unset, set binds the cert to the user -# allow = [] # mtls: exact identities; empty = any the CA issued -# allow_patterns = [] # mtls: RE2, anchor them yourself -# node_binding = "force" # none | assert | force; overrides trust_node +# type = "none" +# identity = "" +# allow = [] +# allow_patterns = [] +# node_binding = "" # credentials_file = "" # scram: written by `lw auth add-user` -# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS -# allow = [] # empty = every address deny does not list +# username = "" +# password_file = "" +# [pipelines.plugin_sources.config.acl] +# allow = [] # deny = [] -# proxy_protocol = "off" # off | optional | required -# proxy_from = [] # proxies that may send a PROXY header -# max_connections_per_client = 0 # per client at once; 0 = unlimited +# proxy_protocol = "off" +# proxy_from = [] +# max_connections_per_client = 0 -## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks) +## HTTP chain source (receives NDJSON batches from upstream http_chain sinks) # [[pipelines.plugin_sources]] # id = "hchain_in" # type = "http_chain" # [pipelines.plugin_sources.config] -# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only) -# port = 9441 # Required -# ingest_path = "/ingest" # Must start with "/" +# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# port = 9441 # Required (example) +# ingest_path = "/ingest" # Must start with "/"; not "/auth" # buffer_size = 1000 -# max_body_bytes = 8388608 # per-request cap (8 MiB) +# max_body_bytes = 8388608 # Per-request cap (8 MiB) # read_timeout_ms = 30000 # trust_node = true # false: label entries by remote address # [pipelines.plugin_sources.config.tls] -# enabled = true # example: mTLS listener -# cert_file = "/etc/logwisp/tls/server.crt" -# key_file = "/etc/logwisp/tls/server.key" -# client_auth = true -# client_ca_file = "/etc/logwisp/tls/client-ca.crt" +# enabled = false +# cert_file = "" +# key_file = "" +# self_signed = false +# issuer_cert_file = "" +# issuer_key_file = "" +# hosts = [] +# client_auth = false +# client_ca_file = "" +# min_version = "1.3" # [pipelines.plugin_sources.config.auth] # authenticate senders -# type = "none" # none | mtls | scram; mtls requires client_auth -# identity = "cn" # mtls default; under scram unset, set binds the cert to the user -# allow = [] # mtls: exact identities; empty = any the CA issued -# allow_patterns = [] # mtls: RE2, anchor them yourself -# node_binding = "force" # none | assert | force; overrides trust_node +# type = "none" +# identity = "" +# allow = [] +# allow_patterns = [] +# node_binding = "" # credentials_file = "" # scram: written by `lw auth add-user` -# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes -# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS -# allow = [] # empty = every address deny does not list +# token_lifetime_ms = 0 +# trusted_proxies = [] # the http sink only +# username = "" +# password_file = "" +# [pipelines.plugin_sources.config.acl] +# allow = [] # deny = [] -# proxy_protocol = "off" # off | optional | required -# proxy_from = [] # proxies that may send a PROXY header -# max_connections_per_client = 0 # per client at once; 0 = unlimited -# requests_per_second_per_client = 0 # per client, a second's worth at once; 0 = unlimited +# proxy_protocol = "off" +# proxy_from = [] +# max_connections_per_client = 0 +# requests_per_second_per_client = 0 ###============================================================================ -### Sinks (1+ required, fan-out) +### Sinks (one at least, each getting every entry; ids are examples, unique +### among a pipeline's sinks) ###============================================================================ -## Null sink (testing) +## Null sink: discards; holds a sink's place until one is chosen, or isolates a +## source or flow while it is measured # [[pipelines.plugin_sinks]] # id = "null_out" # type = "null" -[[pipelines.plugin_sinks]] -id = "default_sink" -type = "console" -[pipelines.plugin_sinks.config] -target = "stdout" # stdout|stderr ("split" NOT supported) +## Console sink +# [[pipelines.plugin_sinks]] +# id = "console_out" +# type = "console" +# [pipelines.plugin_sinks.config] +# target = "stdout" # stdout|stderr ("split" NOT supported) # buffer_size = 1000 # escape = "auto" # auto|always|never: control characters as , auto on a terminal # color = "auto" # auto|always|never: level names in color (default: top-level color) @@ -314,146 +337,166 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # id = "file_out" # type = "file" # [pipelines.plugin_sinks.config] -# directory = "./logs" # Required -# name = "output" # Required +# directory = "/var/log/logwisp" # Required (example) +# name = "output" # Required (example) # max_size_mb = 100 # max_total_size_mb = 1000 -# min_disk_free_mb = 0 # 0 = no floor (only negatives become 100) -# retention_hours = 168.0 +# min_disk_free_mb = 0 # 0 = no floor; a negative value means 100 +# retention_hours = 168.0 # .log files there older than this are deleted; 0 keeps them # buffer_size = 1000 # flush_interval_ms = 100 -## HTTP sink (SSE streaming server + JSON status endpoint) -## Without an auth block both endpoints are UNAUTHENTICATED and the stream sends -## Access-Control-Allow-Origin: *. Bind to a trusted interface. +## HTTP sink (server-sent events, a JSON status document and a browser viewer) +## Without an auth table both endpoints are UNAUTHENTICATED and the stream +## sends Access-Control-Allow-Origin: *. Bind to a trusted interface. # [[pipelines.plugin_sinks]] # id = "http_out" # type = "http" # [pipelines.plugin_sinks.config] -# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only) -# port = 8081 # Required -# stream_path = "/stream" # Exact; starts with "/", clean, none of { } % ? # +# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# port = 8081 # Required (example) +# stream_path = "/stream" # Exact; starts with "/", clean (one trailing "/" allowed), none of { } % ? #; not /auth or under it # status_path = "/status" # Same rules; differs from stream_path # buffer_size = 1000 # Sink input queue # client_buffer_size = 256 # Per-client send queue -# write_timeout_ms = 0 # Per-event deadline, 0 = none +# write_timeout_ms = 0 # Per-event deadline; 0 = none # max_connections = 0 # 0 = unlimited -# replay_lines = 0 # Last entries a new stream gets first, 0 = none -# login_page = false # GET /auth/login; scram behind auth.trusted_proxies -# viewer_page = false # GET /auth/view, needs login_page; scram + trusted_proxies only - # (without auth or under mtls the viewer is always on) +# replay_lines = 0 # Last entries a new stream gets first; 0 = none +# login_page = false # GET /auth/login; scram behind auth.trusted_proxies only +# viewer_page = false # GET /auth/view, needs login_page; without auth or under mtls the viewer is always at / # [pipelines.plugin_sinks.config.tls] -# enabled = true # example -# cert_file = "/etc/logwisp/tls/server.crt" -# key_file = "/etc/logwisp/tls/server.key" +# enabled = false +# cert_file = "" +# key_file = "" +# self_signed = false +# issuer_cert_file = "" +# issuer_key_file = "" +# hosts = [] # client_auth = false # client_ca_file = "" +# min_version = "1.3" # [pipelines.plugin_sinks.config.auth] # gates BOTH stream_path and status_path -# type = "none" # none | mtls | scram; browsers: mtls, or scram behind a proxy -# identity = "cn" # mtls default; under scram unset, set binds the cert to the user -# allow = [] # mtls: exact identities; empty = any the CA issued -# allow_patterns = [] # mtls: RE2, anchor them yourself +# type = "none" # browsers: mtls, or scram behind a proxy +# identity = "" +# allow = [] +# allow_patterns = [] +# node_binding = "" # chain sources only # credentials_file = "" # scram: log in with `lw auth token` -# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes +# token_lifetime_ms = 0 # trusted_proxies = [] # scram: proxies ending the browsers' TLS; tls may then be off -# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS -# allow = [] # empty = every address deny does not list +# username = "" +# password_file = "" +# [pipelines.plugin_sinks.config.acl] +# allow = [] # deny = [] -# proxy_protocol = "off" # off | optional | required -# proxy_from = [] # proxies that may send a PROXY header -# max_connections_per_client = 0 # per client at once; 0 = unlimited -# requests_per_second_per_client = 0 # per client, a second's worth at once; 0 = unlimited +# proxy_protocol = "off" +# proxy_from = [] +# max_connections_per_client = 0 +# requests_per_second_per_client = 0 -## TCP sink (streaming server) +## TCP sink (serves entries to TCP clients, one per line) # [[pipelines.plugin_sinks]] # id = "tcp_out" # type = "tcp" # [pipelines.plugin_sinks.config] -# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only) -# port = 9090 # Required +# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only +# port = 9090 # Required (example) # buffer_size = 1000 # client_buffer_size = 256 -# write_timeout_ms = 5000 # Missed deadline disconnects the client -# keep_alive = true +# write_timeout_ms = 5000 # A missed deadline disconnects the client +# keep_alive = true # false: no TCP keep-alive # keep_alive_period_ms = 30000 -# max_connections = 0 +# max_connections = 0 # 0 = unlimited # [pipelines.plugin_sinks.config.tls] -# enabled = true # example -# cert_file = "/etc/logwisp/tls/server.crt" -# key_file = "/etc/logwisp/tls/server.key" -# client_auth = true -# client_ca_file = "/etc/logwisp/tls/client-ca.crt" +# enabled = false +# cert_file = "" +# key_file = "" +# self_signed = false +# issuer_cert_file = "" +# issuer_key_file = "" +# hosts = [] +# client_auth = false +# client_ca_file = "" +# min_version = "1.3" # [pipelines.plugin_sinks.config.auth] # authenticate stream readers -# type = "none" # none | mtls | scram; mtls requires client_auth -# identity = "cn" # mtls default; under scram unset, set binds the cert to the user -# allow = [] # mtls: exact identities; empty = any the CA issued -# allow_patterns = [] # mtls: RE2, anchor them yourself +# type = "none" +# identity = "" +# allow = [] +# allow_patterns = [] +# node_binding = "" # chain sources only # credentials_file = "" # scram: readers use `lw auth stream` -# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS -# allow = [] # empty = every address deny does not list +# username = "" +# password_file = "" +# [pipelines.plugin_sinks.config.acl] +# allow = [] # deny = [] -# proxy_protocol = "off" # off | optional | required -# proxy_from = [] # proxies that may send a PROXY header -# max_connections_per_client = 0 # per client at once; 0 = unlimited +# proxy_protocol = "off" +# proxy_from = [] +# max_connections_per_client = 0 -## TCP chain sink (stdlib client; forwards to downstream tcp_chain source) +## TCP chain sink (forwards to a downstream tcp_chain source) ## Do NOT point a chain sink at a chain source in the SAME pipeline: entries ## loop back in and amplify without bound. # [[pipelines.plugin_sinks]] # id = "chain_out" # type = "tcp_chain" # [pipelines.plugin_sinks.config] -# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1" -# port = 9440 # Required -# node = "" # origin label, default hostname; preserved across hops +# host = "aggregator.example.org" # Required (example); an IPv6 literal bare, e.g. "2001:db8::1" +# port = 9440 # Required (example) +# node = "" # Origin label, kept across hops; empty = the host name # buffer_size = 1000 # dial_timeout_ms = 5000 # write_timeout_ms = 5000 # backoff_min_ms = 500 -# backoff_max_ms = 30000 -# keep_alive = true +# backoff_max_ms = 30000 # Below backoff_min_ms: 30000 +# keep_alive = true # false: no TCP keep-alive # keep_alive_period_ms = 30000 # [pipelines.plugin_sinks.config.tls] -# enabled = true # example: mTLS dialer -# ca_file = "/etc/logwisp/tls/ca.crt" +# enabled = false +# cert_file = "" +# key_file = "" +# ca_file = "" # server_name = "" # insecure_skip_verify = false -# cert_file = "/etc/logwisp/tls/client.crt" -# key_file = "/etc/logwisp/tls/client.key" +# pin_sha256 = "" # min_version = "1.3" # [pipelines.plugin_sinks.config.auth] # pin the server (mtls) or log in (scram) -# type = "none" # none | mtls | scram; both require tls.enabled -# identity = "cn" # mtls: cn | san_dns | san_uri | san_email -# allow = [] # mtls: exact identities; empty = any the CA issued -# allow_patterns = [] # mtls: RE2, anchor them yourself -# username = "" # scram -# password_file = "" # scram +# type = "none" +# identity = "" # mtls only; unset = cn +# allow = [] +# allow_patterns = [] +# username = "" +# password_file = "" -## HTTP chain sink (stdlib client; batched NDJSON POST to downstream http_chain source) +## HTTP chain sink (posts NDJSON batches to a downstream http_chain source) # [[pipelines.plugin_sinks]] # id = "hchain_out" # type = "http_chain" # [pipelines.plugin_sinks.config] -# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1" -# port = 9441 # Required -# ingest_path = "/ingest" -# node = "" # origin label, default hostname; preserved across hops +# host = "aggregator.example.org" # Required (example); an IPv6 literal bare, e.g. "2001:db8::1" +# port = 9441 # Required (example) +# ingest_path = "/ingest" # Must start with "/"; not "/auth" +# node = "" # Origin label, kept across hops; empty = the host name # buffer_size = 1000 # max_batch_count = 100 # max_batch_bytes = 1048576 # flush_interval_ms = 1000 -# request_timeout_ms = 10000 # Covers dial + write + response +# request_timeout_ms = 10000 # Covers dial, write and response # backoff_min_ms = 500 -# backoff_max_ms = 30000 +# backoff_max_ms = 30000 # Below backoff_min_ms: 30000 # [pipelines.plugin_sinks.config.tls] -# enabled = true # example: mTLS dialer -# ca_file = "/etc/logwisp/tls/ca.crt" -# cert_file = "/etc/logwisp/tls/client.crt" -# key_file = "/etc/logwisp/tls/client.key" +# enabled = false +# cert_file = "" +# key_file = "" +# ca_file = "" +# server_name = "" +# insecure_skip_verify = false +# pin_sha256 = "" +# min_version = "1.3" # [pipelines.plugin_sinks.config.auth] # pin the server (mtls) or log in (scram) -# type = "none" # none | mtls | scram; both require tls.enabled -# identity = "cn" # mtls: cn | san_dns | san_uri | san_email -# allow = [] # mtls: exact identities; empty = any the CA issued -# allow_patterns = [] # mtls: RE2, anchor them yourself -# username = "" # scram -# password_file = "" # scram +# type = "none" +# identity = "" # mtls only; unset = cn +# allow = [] +# allow_patterns = [] +# username = "" +# password_file = "" diff --git a/config/sample_test.go b/config/sample_test.go new file mode 100644 index 0000000..2d22f75 --- /dev/null +++ b/config/sample_test.go @@ -0,0 +1,108 @@ +package config + +import ( + "cmp" + "fmt" + "strings" + "testing" + + schema "github.com/lixenwraith/logwisp/internal/config" + + "github.com/lixenwraith/toml" +) + +// The sample lists every key of the settings, the flow stages and each +// catalogued plugin, tls, auth and acl included, with the schema's default; +// a line marked example shows another value +func TestSampleListsEveryKeyWithItsDefault(t *testing.T) { + all, shown := uncommented(t, true), uncommented(t, false) + for _, k := range schema.SettingKeys() { + i := strings.LastIndex(k.Name, ".") + 1 + at := k.Name[:i] + k.Name = k.Name[i:] + checkKeys(t, at, []schema.Key{k}, table(all, at), table(shown, at)) + } + pipeline, pipelineShown := first(all["pipelines"]), first(shown["pipelines"]) + for _, stage := range schema.FlowStages() { + checkKeys(t, "flow."+stage.Key+".", stage.Keys, first(table(pipeline, "flow")[stage.Key]), first(table(pipelineShown, "flow")[stage.Key])) + } + for _, p := range schema.Plugins() { + list := "plugin_" + p.Role + "s" + block := ofType(pipeline[list], p.Type) + if block == nil { + t.Errorf("%s %s: no block in the sample", p.Type, p.Role) + continue + } + checkKeys(t, p.Type+" "+p.Role+": ", p.Keys(), table(block, "config"), table(ofType(pipelineShown[list], p.Type), "config")) + } +} + +// uncommented is the sample with its commented keys and tables made active +// (prose stays under "##"); without examples it drops the lines marked so +func uncommented(t *testing.T, examples bool) map[string]any { + t.Helper() + var b strings.Builder + for line := range strings.Lines(string(Sample)) { + line = strings.TrimPrefix(line, "# ") + if _, comment, _ := strings.Cut(line, " # "); examples || !strings.Contains(comment, "example") { + b.WriteString(line) + } + } + root, err := toml.NewParser([]byte(b.String())).Parse() + if err != nil { + t.Fatalf("the sample uncommented: %v", err) + } + return root +} + +// zeros is how the sample writes a kind's zero, the default of a key without one +var zeros = map[string]string{"integer": "0", "number": "0", "bool": "false", "list": "[]"} + +// checkKeys finds each key in all, the sample, and compares the value shown, +// without examples, with the default +func checkKeys(t *testing.T, path string, keys []schema.Key, all, shown map[string]any) { + t.Helper() + for _, k := range keys { + _, ok := all[k.Name] + switch want := cmp.Or(k.Default, zeros[k.Kind]); { + case !ok: + t.Errorf("%s%s: not in the sample", path, k.Name) + case k.Kind == "table": + checkKeys(t, path+k.Name+".", k.Keys, table(all, k.Name), table(shown, k.Name)) + default: + if v, ok := shown[k.Name]; ok && fmt.Sprint(v) != want { + t.Errorf("%s%s = %v, the default is %q", path, k.Name, v, want) + } + } + } +} + +// table descends a dotted path of tables +func table(m map[string]any, path string) map[string]any { + for key := range strings.SplitSeq(path, ".") { + if key != "" { + m, _ = m[key].(map[string]any) + } + } + return m +} + +// first is a table, or the first of an array of tables +func first(v any) map[string]any { + if list, ok := v.([]map[string]any); ok && len(list) > 0 { + return list[0] + } + m, _ := v.(map[string]any) + return m +} + +// ofType is the block of an array of plugin tables with the type given +func ofType(v any, typ string) map[string]any { + list, _ := v.([]map[string]any) + for _, m := range list { + if m["type"] == typ { + return m + } + } + return nil +} diff --git a/deploy/README.md b/deploy/README.md index 28ea759..11865e1 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -12,20 +12,4 @@ deploy/lw-deploy.sh --dry-run ... # print the configuration and commands only deploy/lw-deploy.sh --yes ... # no questions: defaults, or fail on a missing value ``` -- Roles - - `edge`: tails log directories, forwards the entries to an aggregator - - `aggregator`: receives from edges; writes files, serves HTTP and TCP streams - - `standalone`: tails log directories; writes files, serves streams -- Runtimes - - `docker`, `podman`: a hardened container of the `logwisp` image (Linux) - - `native`: systemd on Linux, rc.d on FreeBSD, `--jail NAME` for a jail - - `manual`: the configuration only, for `lw -c` -- Every question has a flag. Before changing anything the script prints its - plan and the command line that repeats it without questions, ready for the - next node. It starts nothing that `lw --check` rejects. -- The native runtimes install the service files of `deploy/package/`. -- Not covered, and printed by the script: filters, rate limits, heartbeat, - several pipelines, proxy mode, ACLs, certificate creation. Edit the generated - `logwisp.toml` for those. - -Scenarios and what each runtime writes: [doc/deployment.md](../doc/deployment.md). +Roles, runtimes, scenarios and what is not covered: [doc/deployment.md](../doc/deployment.md). diff --git a/deploy/lw-deploy.sh b/deploy/lw-deploy.sh index d556141..03b9c08 100755 --- a/deploy/lw-deploy.sh +++ b/deploy/lw-deploy.sh @@ -70,7 +70,7 @@ Outputs (aggregator, standalone) --tcp-port PORT serve a plain TCP stream --format raw|txt|json entry format (default txt) -TLS (this script creates no certificates: doc/security.md#enabling-mtls) +TLS (this script creates no certificates: lw tls makes them, doc/cli.md#lw-tls) --tls, --no-tls TLS on every network link of the node --cert-file FILE this node's certificate: listeners, and edges under mtls --key-file FILE its private key @@ -122,7 +122,7 @@ Not covered by this script; edit the generated configuration instead - proxy mode: browser logins through a TLS-terminating proxy (trusted_proxies) - acl on the listeners: address rules, PROXY headers, per-client limits - mtls identities other than the certificate CN; node_binding other than force -- creating certificates: openssl or your PKI (doc/security.md#enabling-mtls) +- creating certificates: lw tls, openssl or your PKI (doc/cli.md#lw-tls) - several native instances on one host: use one per host or jail EOF } @@ -612,7 +612,7 @@ resolve() { [ -z "$SINK_ALLOW" ] || _isa=mtls [ -z "$ADD_VIEWERS" ] || _isa=scram if [ "$TLS" = yes ]; then - [ "$INTERACTIVE" = 0 ] || note "Certificates come from openssl or your PKI: doc/security.md#enabling-mtls" >&2 + [ "$INTERACTIVE" = 0 ] || note "Certificates come from lw tls, openssl or your PKI: doc/cli.md#lw-tls" >&2 if [ "$ROLE" = edge ]; then ask_opt CA_FILE --ca-file "CA bundle verifying the aggregator (empty: the system roots)" v_file else diff --git a/deploy/package/freebsd/Makefile b/deploy/package/freebsd/Makefile index c848032..e0e7954 100644 --- a/deploy/package/freebsd/Makefile +++ b/deploy/package/freebsd/Makefile @@ -46,8 +46,9 @@ post-install: ${INSTALL_DATA} ${WRKSRC}/config/logwisp.toml ${STAGEDIR}${ETCDIR}/logwisp.toml.sample post-install-DOCS-on: - ${MKDIR} ${STAGEDIR}${DOCSDIR}/doc - ${INSTALL_DATA} ${WRKSRC}/README.md ${STAGEDIR}${DOCSDIR} - ${INSTALL_DATA} ${WRKSRC}/doc/*.md ${STAGEDIR}${DOCSDIR}/doc + ${MKDIR} ${STAGEDIR}${DOCSDIR}/doc ${STAGEDIR}${DOCSDIR}/config + ${INSTALL_DATA} ${WRKSRC}/README.md ${WRKSRC}/logwisp-logo.svg ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/doc/*.md ${WRKSRC}/doc/lw.1 ${STAGEDIR}${DOCSDIR}/doc + ${INSTALL_DATA} ${WRKSRC}/config/logwisp.toml ${STAGEDIR}${DOCSDIR}/config .include diff --git a/doc/README.md b/doc/README.md index 0fd0e2e..72ea60c 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,135 +1,32 @@ # LogWisp Documentation -LogWisp is a pipeline-based log transport and processing system written in Go. -It collects log entries from files, stdin, or other LogWisp nodes; rate-limits, -filters, and formats them; and distributes them to files, consoles, live network -streams, or downstream LogWisp nodes. +The [README](../README.md) has LogWisp's features, a quick start and the +system requirements. -## Documentation Map +- [Installation](installation.md): building, installing, services, + the container image, packaging +- [Deployment](deployment.md): `deploy/lw-deploy.sh` for edges, + aggregators, containers, services, jails +- [Architecture](architecture.md): component model, data flow, + concurrency, back-pressure, the configuration engine's WebAssembly API +- [Configuration](configuration.md): TOML structure, precedence, + environment and CLI overrides +- [Sources](sources.md) and [Sinks](sinks.md): every plugin and its + options +- [Filters](filters.md) and [Formatters](formatters.md): pattern + matching, output shaping and sanitization +- [Chaining](chaining.md): multi-node topologies and the wire protocol +- [Networking](networking.md): listeners, dialers, timeouts, limits +- [Security](security.md): TLS, mTLS, password (SCRAM) authentication, + address ACLs, certificates made at startup; threat model and current limits +- [CLI](cli.md): filter use, flags, pipeline specifications, presets, + environment, signals, exit codes, `lw --tui`, `--schema`, `lw auth`, + `lw tls`, `lw preset`, `lw config`; also the `lw(1)` manual, + [`lw.1`](lw.1) +- [Operations](operations.md): running, monitoring, tuning, + troubleshooting +- [To Do](todo.md): the maintainer's release and packaging steps, and + what `lw --tui` and the schema leave for later -- [Installation](installation.md): building, installing, and running as a - service -- [Deployment](deployment.md): `deploy/lw-deploy.sh` for edges, aggregators, - containers, services, jails -- [Architecture](architecture.md): component model, data flow, concurrency, - back-pressure -- [Configuration](configuration.md): TOML structure, precedence, environment - and CLI overrides -- [Sources](sources.md): every input plugin and its options -- [Sinks](sinks.md): every output plugin and its options -- [Filters](filters.md): pattern-based inclusion and exclusion -- [Formatters](formatters.md): output shaping and sanitization -- [Chaining](chaining.md): multi-node topologies and the chain wire protocol -- [Networking](networking.md): listeners, dialers, timeouts, connection limits -- [Security](security.md): TLS, mTLS, and peer authorization; threat model and - current limits -- [mTLS Authentication](mtls-auth-plan.md): design and rationale for - certificate-based authorization -- [Password Authentication](scram-auth-plan.md): design and rationale for - Argon2id-SCRAM authentication; mTLS hardening -- [CLI](cli.md): flags, signals, exit codes, `lw --tui`, `lw auth`; also - the `lw(1)` manual, [`lw.1`](lw.1) -- [Operations](operations.md): running, monitoring, tuning, troubleshooting -- [To Do](todo.md): planned work: packaging, `lw --tui` and its shared configuration engine - -A fully annotated configuration covering every option lives at +A fully annotated configuration covering every option ships as [`config/logwisp.toml`](../config/logwisp.toml). - -## Capabilities - -### Pipeline - -- A Unix filter without configuration: stdin to stdout, line for line -- Independent named pipelines, each `sources → flow → sinks` -- Fan-in (many sources per pipeline) and fan-out (many sinks per pipeline) -- Isolated sinks: a stalled sink drops its own events rather than stall the - pipeline; only the console sink waits, so a filter loses no line -- Hot reload of pipeline configuration via `SIGHUP`/`SIGUSR1` or a file watch - -### Inputs - -`file` (directory tail with rotation detection), `console` (stdin, one -reader per process), -`random` (synthetic generator), `null`, and the chain ingest listeners -`tcp_chain` and `http_chain`. - -### Outputs - -`console` (control characters escaped on a terminal), `file` (rotating), -`http` (Server-Sent Events plus a JSON status -endpoint), `tcp` (broadcast server), `null`, and the chain forwarders -`tcp_chain` and `http_chain`. - -### Processing - -- Token-bucket rate limiting with an optional per-entry size cap -- Chainable include/exclude regex filters with `or`/`and` logic -- `raw`, `txt`, and `json` formatting with selectable sanitizer policies -- Optional flow-level heartbeat entries - -### Transport security and authentication - -- TLS 1.2/1.3 on every network source and sink, listener and dialer alike -- Mutual TLS: listeners can require and verify client certificates; dialers can - present a client identity -- Authorization by certificate identity, per listener: named peers rather than - everything the CA issued, with the `http` sink's endpoints gated too -- Password authentication (Argon2id-SCRAM) bound to the listener's certificate, - with bearer tokens on HTTP listeners and a `lw auth` CLI, and browser - logins behind a TLS-terminating proxy -- Node binding, so a chain source labels entries from the sender's certificate - identity or username rather than from what the sender claims - -See [Security](security.md) for what each layer does and does not give you. - -## Quick Start - -Without a configuration file `lw` is a filter, exiting 0 at the end of input; -a pipeline without `--source` reads stdin and one without `--sink` writes -stdout: - -```bash -lw < app.log > copy.log -tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN -``` - -As a service, a file holds the pipelines: - -```toml -[[pipelines]] -name = "app" - -[pipelines.flow.format] -type = "json" -sanitizer_policy = "json" - -[[pipelines.plugin_sources]] -id = "app_logs" -type = "file" -[pipelines.plugin_sources.config] -directory = "/var/log/myapp" -pattern = "*.log" - -[[pipelines.plugin_sinks]] -id = "stdout" -type = "console" -[pipelines.plugin_sinks.config] -target = "stdout" -``` - -```bash -lw -c config.toml -``` - -## System Requirements - -- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+) -- **Architecture**: amd64 -- **Go**: 1.27.2+ to build from source - -Network sources and sinks bind and dial IPv4 or IPv6, each keeping strictly to -the family of its host ([Networking](networking.md#address-family)). - -## License - -BSD 3-Clause. diff --git a/doc/architecture.md b/doc/architecture.md index 8f5df1b..2f6c4f6 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -7,27 +7,36 @@ plugins, sessions, TLS, statistics — hangs off that spine. ``` main -└── Service - ├── Pipeline "app" - │ ├── Registry instance tracking, single-instance enforcement - │ ├── Session Manager per-pipeline connection/session bookkeeping - │ ├── Sources[] plugin instances, keyed by id - │ ├── Flow - │ │ ├── Rate Limiter optional, token bucket - │ │ ├── Filter Chain optional, ordered - │ │ ├── Formatter raw | txt | json, with sanitizer - │ │ └── Heartbeat optional generator - │ └── Sinks[] plugin instances, keyed by id - ├── Pipeline "audit" - │ └── ... - └── Status Reporter optional, 30s interval +├── Service +│ ├── Pipeline "app" +│ │ ├── Registry creates each plugin from its factory +│ │ ├── Session Manager per-pipeline connection/session bookkeeping +│ │ ├── Sources[] plugin instances, keyed by id +│ │ ├── Flow +│ │ │ ├── Rate Limiter optional, token bucket +│ │ │ ├── Filter Chain optional, ordered +│ │ │ ├── Formatter raw | txt | json, with sanitizer +│ │ │ └── Heartbeat optional generator +│ │ └── Sinks[] plugin instances, keyed by id +│ └── Pipeline "audit" +│ └── ... +└── Status Reporter cmd/lw, optional, 30 s: Service.GetGlobalStats ``` Package map: -- `cmd/lw`: entry point, help, logger bootstrap, signal loop, status - reporter. -- `internal/config`: typed config schema, loading, top-level validation. +- `cmd/lw`: entry point: the command line (`cli.go`: lw's flags and the + `auth`, `tls`, `preset` and `config` commands), help, completion, logger + bootstrap, signal and reload loop, status reporter. +- `cmd/lwconf`: `internal/compose` as WebAssembly, and `lwconf.js`, which + loads it: see [Configuration Engine](#configuration-engine). +- `internal/config`: option schema (tags, catalogue) and `Decode`; loading + from file, environment and flags; presets; offline validation of every + setting and plugin option. +- `internal/compose`: the configuration engine `lw --tui` and `cmd/lwconf` + share. It edits pipelines and writes and reads back the command-line, + environment and file forms; it reads no file and links no TLS, HTTP or + terminal code. - `internal/service`: owns the pipeline set; start, stop, shutdown, global stats. - `internal/pipeline`: pipeline runtime and per-pipeline plugin registry. @@ -41,7 +50,8 @@ Package map: - `internal/chain`: chain wire protocol: hello preamble, entry codec, backoff. - `internal/tlsx`: the single seam between `TLSOptions` and `crypto/tls`. - `internal/authz`: the single seam between `AuthOptions` and the network - plugins: `Admit`, `AuthorizeRequest`, `Greet`, `Prepare`. + plugins: `Admit`, `AuthorizeRequest`, `Greet`, `Prepare`; `Authorize` + refuses under `scram`. - `internal/netacl`: a listener's `acl` block: address rules, PROXY headers and per-client limits; its `Table` is SCRAM throttling's limiter too. - `internal/session`: session manager and per-instance proxy. @@ -53,15 +63,16 @@ Package map: - `internal/logline`: reads a line of any common shape into time, level, tag, message and fields for the live view; holds the one rule for runes a terminal acts on, which the console sink shares. +- `internal/version`: version, commit and build time, stamped at build. +- `internal/testutil`: fixtures for tests only. ## Plugin Registration Every plugin registers itself in an `init()` function, and -`cmd/lw/bootstrap.go` blank-imports each package to trigger those -`init()`s. Adding a plugin therefore means writing the package, calling -`plugin.RegisterSource` / `plugin.RegisterSink`, adding one blank import, and -adding the type's row to the catalogue in `internal/config/schema.go`, which -registration requires. +`cmd/lw/bootstrap.go` blank-imports each package to trigger it. Adding a +plugin means writing the package, calling `plugin.RegisterSource` / +`plugin.RegisterSink`, adding one blank import, and adding the type's row to +the catalogue in `internal/config/schema.go`, which registration requires. The row holds the type's options struct, whose tags declare its defaults and rules, and whether the type is single. The `console` source is single, @@ -77,10 +88,11 @@ instance of a single type fails, naming both pipelines. starts. Publication is non-blocking: a full subscriber channel increments the source's `dropped_entries` counter. The console source alone waits, since stdin can simply be read later. -2. **Flow** applies, in order: rate limit → filter chain → formatter. A drop at - any stage ends the entry's life and increments `flow.total_dropped`. The - verdict, passed or the stage that dropped it (a filter by its index), goes - to the pipeline's tap when a viewer set one, as `lw --tui`'s live view does. +2. **Flow** applies, in order: rate limit → filter chain → formatter. A rate + limit or filter drop ends the entry's life and increments + `flow.total_dropped`. The verdict, passed or the stage that dropped it (a + filter by its index), goes to the pipeline's tap when a viewer set one, as + `lw --tui`'s live view does. 3. The formatter output becomes a `core.TransportEvent`, which carries both the formatted `Payload` and the original structured `Entry`. 4. **Dispatch** sends the event to every sink's input channel with a @@ -109,8 +121,7 @@ loses nothing. Each stage drops and counts instead of waiting: - Source → subscriber: a full channel drops the entry; source `dropped_entries`. The console source waits instead and reads stdin later. -- Flow: the rate limiter, a filter rejection or a format error drops it; - `flow.total_dropped`. +- Flow: the rate limiter or a filter rejection drops it; `flow.total_dropped`. - Pipeline → sink: a full sink input drops it for that sink only; pipeline `total_dropped_by_sink`. A sink declaring `core.CapBackpressure`, the console and null sinks, makes dispatch wait instead, so a filter loses no line to a @@ -124,11 +135,12 @@ loses nothing. Each stage drops and counts instead of waiting: the view shows them. The `tcp_chain` sink is the one deliberate exception. It holds a line across -reconnects until it is written or the process shuts down, so a downstream -outage propagates backwards as a full input buffer and surfaces as -`total_dropped_by_sink` on the pipeline rather than as silent data loss inside -the sink. The `http_chain` sink retries a batch with backoff, and drops it only -on a non-retryable response or on shutdown (`dropped_batches`). +reconnects until it is written or the sink stops (shutdown or reload, after +`sink.FlushBound`), so a downstream outage propagates backwards as a full +input buffer and surfaces as `total_dropped_by_sink` on the pipeline rather +than as silent data loss inside the sink. The `http_chain` sink retries a +batch with backoff, and drops it only on a non-retryable response or on +shutdown (`dropped_batches`). ## Concurrency Model @@ -138,20 +150,23 @@ on a non-retryable response or on shutdown (`dropped_batches`). - Each network sink runs one broadcast/broker goroutine plus, per connection, a writer goroutine (and for TCP, a reader goroutine that exists only to detect disconnects and refresh session activity). -- Chain sinks run a single run-loop goroutine that exclusively owns the - connection or the pending batch, so no locking is needed around either. +- Chain sinks run one run-loop goroutine that owns the pending line or batch; + the `tcp_chain` sink adds a reader goroutine per link that closes the link + when the source writes anything or closes it. - Statistics are atomics; configuration and registries use RW mutexes; shutdown is context cancellation plus wait groups. ### Shutdown ordering -`Pipeline.Stop` is deliberately ordered so in-flight data drains: +`Pipeline.Stop` is ordered so in-flight data drains: 1. Release dispatch waits on backpressure sinks; stop all sources concurrently, each closing its subscriber channels. 2. Wait for the run loop, which ends when every subscription channel closes. -3. Stop all sinks concurrently; console and file sinks write their queue - first. Then cancel the pipeline context. +3. Stop all sinks concurrently: console and file sinks write their queue; + network sinks deliver theirs within `sink.FlushBound`, their write or + request timeout when set below 2 s, else 2 s. Then cancel the pipeline + context. A finite source ends on its own: the console source at the end of stdin. The run loop then returns and the pipeline's `Finished` channel closes; once every @@ -176,7 +191,7 @@ The network plugins, by role: - `http_chain` sink: chain protocol, batched POST with retry. TLS is built in exactly one place, `internal/tlsx`, which exposes -`Server(opts)` for listeners and `Client(opts, host)` for dialers. See +`Server(opts, host)` for listeners and `Client(opts, host)` for dialers. See [Security](security.md). ## Sessions @@ -194,18 +209,17 @@ the corresponding SSE client. Authorization decisions do not read session metadata — `internal/authz` makes them from the handshake, the SCRAM exchange or a bearer token, at the point of -connection or request, and their outcome is *recorded* in the session. That -ordering matters: a session exists only for a peer that was already admitted. -See the [mTLS](mtls-auth-plan.md) and [SCRAM](scram-auth-plan.md) -authentication designs. +connection or request, and their outcome is *recorded* in the session, so a +session exists only for a peer that was already admitted. See +[Security](security.md#the-auth-block) and +[Chaining](chaining.md#wire-protocol). ## Configuration Reload Reload (signal or file watch) rebuilds the entire service: -1. Signals reread the selected file; watches consume a newly published snapshot. - The application owns its manager and detached snapshots; no global mutable - target is shared with the running service. Validate every candidate again. +1. Signals (`SIGHUP`, `SIGUSR1`) reread the file; a watch takes the published + snapshot. Every candidate is validated again. 2. Build a **new** service from it. If construction fails, the old service keeps running untouched. 3. Shut the old service down, start the new one, and restart the status @@ -222,15 +236,58 @@ certificate rotation. Failed candidates release their session workers. Once the old service has stopped, a bind/start failure requires another corrected reload or process restart. +## Configuration Engine + +`internal/compose` is what `lw --tui` edits with and what the website runs as +WebAssembly, so no part of LogWisp is rewritten in JavaScript. `make wasm` +writes the files a site drops in +([Installation](installation.md#building-from-source)): `lwconf.wasm` is +7.1 MB, 1.9 MB at gzip -9, and the toolchain's `wasm_exec.js` uses no eval, +so the page needs only `'wasm-unsafe-eval'` in its `script-src`. + +`lwconf.js` is one dependency-free ES module that knows the calls and shapes, +never a key: + +- `load({execURL, execSRI, wasmURL, wasmSRI})` fetches both by their + integrity and resolves to the engine. +- `fields(schema, role, type)` is a node's form rows, a table's a group. +- `value`, `at` and `group` read a node's value and sort problems by node. + +The engine's calls take strings (`lwconf.js` writes objects as JSON) and +answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`, as +far as the engine places the refusal: + +- `schema()`: what `lw --schema` prints. +- `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to + a string, a list key to strings. +- `parse(text, form, dirs)`: the pipelines of a form's pasted text. +- `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled + (defaults, kinds, ids) and the first problem of each part (a value of the + wrong kind too) as `{path, pipeline, node, key, text}`. +- `emit(pipelines, form)`: the text of `command`, `environment` or `file`. +- `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is + `{op, pipeline, ...}`: + - `add_pipeline` and `rename_pipeline` (`name`), `remove_pipeline` + - `add` (`role`, `type`), `retype` (`node`, `type`), `remove` (`node`) + - `move_filter` (`from`, `to`) + - `set` (`node`, `key`, `values`), `unset` (`node`, `key`) +- `dirs` answers whether a path a preset names is a directory, + `{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, one + with a glob a pattern, and any other is refused at `path`. +- Pipelines are a JSON list in the file's shape (`name`, `flow`, + `plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node is + `{role, id, index}`: a source's or sink's id, a filter's index. + ## Resource Management - Every buffer is bounded; the drop-not-block policy keeps memory flat under load. -- The `tcp` and `http` sinks and the `tcp_chain` source accept a - `max_connections` cap. Admission is a load-then-check, so a burst can - over-admit by roughly one connection. +- `max_connections` caps connections on the `tcp` sink and the `tcp_chain` + source, and streams on the `http` sink. Admission is a load-then-check, so a + burst can over-admit by roughly one. - Chain listeners bound a single line at `core.MaxLogEntryBytes` (1 MiB); an - oversized line is a protocol violation and terminates the connection. + oversized line terminates a `tcp_chain` link and fails an `http_chain` + batch with `400`. - The `http_chain` source caps each request body at `max_body_bytes`. - File sinks rotate on size, cap total rotated size, and honour a retention window. diff --git a/doc/chaining.md b/doc/chaining.md index 4de85bc..1cef5e4 100644 --- a/doc/chaining.md +++ b/doc/chaining.md @@ -43,13 +43,14 @@ Chained entries carry a `node` label identifying where they originated. - `false`: always overwrite with the sender's remote address. Relays preserve `node`, so a label survives any number of hops and identifies -the original producer rather than the last relay. +the original producer, unless a hop's source overwrites it: `trust_node = +false`, or `node_binding = "force"` (the default with an `auth` block). +Formatters write the label as a prefix on the source +([Formatters](formatters.md#node-identity-in-output)). With an `auth` block the source can instead bind the label to the sender's -authenticated identity — its certificate identity under `mtls`, its username -under `scram`. `assert` and `force` take the connection label over from -`trust_node`; only `force` also overrides the per-entry labels. -`auth.node_binding`: +authenticated identity: its certificate identity under `mtls`, its username +under `scram`. `auth.node_binding`: - `none`: `trust_node` governs the connection label and the per-entry `node` field. @@ -64,10 +65,6 @@ per-entry `node` field. Pick `assert` on a relay-to-relay hop, where the relay should prove its own identity but the origin labels it forwards must survive. See [Security](security.md#node-binding). -Formatters render node identity as a syslog-style prefix on the source field: -`edge-01/app.log`. In JSON output the node therefore appears inside the source -field, not as a separate top-level key. - > `trust_node = true` with no `auth` block means any peer that can connect can > claim **any** node label, including one belonging to another host. On an > untrusted network set an `auth` block (`mtls` or `scram`) and keep @@ -96,6 +93,11 @@ A persistent connection carrying newline-delimited JSON. Line size is bounded at 1 MiB. An oversized line is a protocol violation and terminates the connection, because the scanner cannot resynchronize afterwards. +The cap applies to the encoded line, envelope and JSON escapes included, so a +message near 1 MiB exceeds it (the console and file sources cut lines at +1 MiB). On `tcp_chain` such an entry ends the link and is lost, together with +any line the sink writes before it notices the close. On `http_chain` it fails +its whole batch with `400`, which the sink drops. Under `auth.type = "scram"` the hello also carries the SCRAM client-first message, and three more lines precede the entries, each one JSON object of at @@ -127,15 +129,19 @@ Batches of NDJSON delivered by `POST`, with the preamble expressed as headers. - Response header - `X-Logwisp-Accepted`: the number of entries ingested. -Responses: `204` on success, `400` for a bad protocol version or a malformed -body, `413` when the body cap is exceeded, `405` for a non-`POST` method, `401` -for a missing or expired token, `403` for a refused certificate or node label. +Responses: `204` on success; `400` for a bad protocol version or a malformed +body, a line over 1 MiB included; `413` past `max_body_bytes`; `404` for +another path; `405` for a non-`POST` method; `429` (with `Retry-After`) past +`acl.requests_per_second_per_client`; `401` for a missing, invalid or expired +token; `403` for a refused certificate or node label. -Under `scram` the sink first logs in with two `POST /auth` requests carrying the -same JSON as the TCP lines: the hello (`200` and the challenge), then the proof -(`200` with the final message, `token` and `expires_in` in seconds). A refusal is -`{"error":"…"}`: `401` for a failed proof, `429` when throttled, `503` when the -listener is busy, `400` or `413` for a malformed or oversized body. +Under `scram` the sink first logs in with two `POST /auth` requests, each +`Content-Type: application/json` and carrying the same JSON as the TCP lines: +the hello (`200` and the challenge), then the proof (`200` with the final +message, `token` and `expires_in` in seconds). A refusal is `{"error":"…"}`: +`401` for a failed proof, `429` when throttled, `503` when the listener is +busy, `415` without the JSON content type, `400` or `413` for a malformed or +oversized body. ### Entry encoding @@ -155,19 +161,16 @@ in at ingest. ## Delivery Semantics -- `tcp_chain`: per line, each held across reconnects. - - On failure it retries with exponential backoff plus ±20 % jitter until - the line is written or the sink shuts down. - - Back-pressure appears upstream as `total_dropped_by_sink`. -- `http_chain`: at least once per batch. - - It retries transport errors, `408`, `429`, `5xx` and, under `scram`, failed - logins and `401`. - - It drops the batch on any other non-2xx, counted in `dropped_batches`. - -`http_chain` batches can be delivered twice when a successful request's response -is lost. There is no de-duplication downstream; design your consumers to -tolerate it, or use `tcp_chain` where each line is written once per successful -write. +- `tcp_chain`: per line, held across reconnects until the socket accepts it, + with exponential backoff plus ±20 % jitter. There is no acknowledgement: + lines in flight when a link breaks are lost (at most once). +- `http_chain`: at least once per batch. It retries transport errors, `408`, + `429`, `5xx` and, under `scram`, failed logins and `401`. It drops the batch + on any other non-2xx, counted in `dropped_batches`. A batch can arrive twice + when a successful request's response is lost; there is no de-duplication + downstream. +- Both: while a link is down or a batch retries, the sink's input queue fills + and the pipeline counts drops in `total_dropped_by_sink`. Neither transport persists anything to disk. Entries buffered in memory during an outage are lost if the process exits. @@ -191,7 +194,7 @@ pattern = "*.log" id = "forward" type = "tcp_chain" [pipelines.plugin_sinks.config] -host = "relay.internal" +host = "relay.example.org" port = 15801 node = "edge-01" [pipelines.plugin_sinks.config.tls] @@ -201,7 +204,7 @@ cert_file = "/etc/logwisp/tls/edge-01.crt" key_file = "/etc/logwisp/tls/edge-01.key" [pipelines.plugin_sinks.config.auth] type = "mtls" -allow = ["relay.internal"] # pin the relay, not just its hostname +allow = ["relay.example.org"] # pin the relay, not just its hostname ``` **Relay** — ingest, keep errors only, archive and stream: @@ -252,27 +255,23 @@ port = 8080 Entries arriving on this relay are labelled `edge-01` or `edge-02` because that is what their certificates say, regardless of the `node` each edge configured. -`test/mtls-chain-test.sh` builds exactly this shape against a throwaway PKI. +`test/mtls-chain-test.sh` runs a relay of this kind (mTLS ingest with +`node_binding = "force"`) against a throwaway PKI. To use passwords instead of client certificates, replace both `auth` blocks: `type = "scram"` with `username` and `password_file` on the edge, and on the relay a `credentials_file`, or one `username` with its `password_file`. -`client_auth` and the edge's certificate become -optional, and entries are labelled with the username. See +`client_auth` and the edge's certificate become optional, and entries are +labelled with the username. See [Password Authentication](security.md#password-authentication-scram). ## Operational Notes -- **Formatting is a relay decision.** Because chain links carry structured - entries, the edge node's `flow.format` affects only its own local sinks. Set - the output shape on the node that owns the human-facing sink. - **Filtering early saves bandwidth.** A filter on the edge drops entries before they cross the network; a filter on the relay is easier to change centrally. - **Rate limits are per pipeline.** An edge limit protects the link; a relay limit protects the relay from a noisy edge. - **Heartbeats traverse chain links** as ordinary structured entries and keep otherwise-idle links and their sessions warm. -- **Ports** used by the bundled test scripts: `15801` tcp_chain ingest, `15802` - http_chain ingest, `15803` tcp sink, `15804` http sink. - **Dial the listener's own literal** when testing locally: a listener on `127.0.0.1` takes IPv4 only, and `localhost` may resolve to `::1`. diff --git a/doc/cli.md b/doc/cli.md index 067520a..5ad2885 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -33,11 +33,13 @@ first argument. one-letter short form that stands for it in every command that has it: - `-c` `--config`, `-p` `--preset`, `-q` `--quiet`, `-t` `--check`, `-T` `--tui`, `-V` `--version`, `-h` `--help` - - `-u` `--user` in `lw auth` and `lw preset edge` + - `-u` `--user` in `lw auth` and in `lw preset serve`, `edge` and + `aggregator` - A value follows as the next argument or after `=` (`-c FILE`, `-c=FILE`, `--config FILE`, `--config=FILE`); a value that starts with `-` follows `=`, and `--` ends the options. -- A switch (`--check`, `--quiet`, any boolean key) takes no separate value: +- `--check`, `--dump`, `--tui`, `--version` and `--schema` take no value + (`--check=false` is a usage error); a boolean key takes one only after `=`: `--quiet=false`, not `--quiet false`. - Short options do not combine (`-q -t`, not `-qt`) or take an attached value (`-cFILE`). @@ -75,8 +77,8 @@ pipelines have [their own flags](#pipelines): - a key: its kind, default, rules and help; a table (`tls`, `auth`, `acl`) holds its own keys, those of the plugin's side (listener or dialer), HTTP's only on a plugin that speaks HTTP. Every listener lists - `auth.node_binding` and `auth.trusted_proxies`, which validation admits - only on chain sources and on the http sink + `auth.node_binding`, which only chain sources use, and every HTTP + listener `auth.trusted_proxies`, which only the `http` sink admits - a flow stage: its keys, and `on`, the values that turn it on - a preset key: its default and help, and the choices (`enum`) and `hint` of the option it sets @@ -116,19 +118,22 @@ destination is derived from `--logging.output`. ### Pipelines Pipeline flags define whole pipelines. They replace the configuration file's -pipelines and the built-in default; every other key keeps its precedence. +pipelines and the built-in default, and a reload keeps them; every other key +keeps its [precedence](configuration.md#configuration-precedence). -- `--pipeline NAME` starts a pipeline; specs before the first one go to a - pipeline named `cli`, or after the preset that starts them -- `-p SPEC`, `--preset SPEC` starts a pipeline with a [preset](#presets); - later flags add sources, sinks and filters to it, or replace its format, - rate limit and heartbeat +- `--pipeline NAME` starts a pipeline named NAME; specs before the first one + go to one pipeline, named after its preset, else `cli` +- `-p SPEC`, `--preset SPEC` begins the pipeline with a [preset](#presets), + as its first spec; a second preset needs its own `--pipeline NAME`. Later + flags add sources, sinks and filters, set a rate limit and heartbeat, or + replace its format - `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable - `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline - a pipeline without `--source` reads stdin (console source `stdin`), one without `--sink` writes stdout (console sink `stdout`) - stdin has one reader, so the whole configuration holds at most one console - source: `pipeline "b": a console source already reads stdin in pipeline "a"` + source: `pipelines[1].plugin_sources[stdin]: one console source may run, + and pipeline "a" has it` A SPEC is `TYPE:key=value,...`, or `key=value,...` without a TYPE: @@ -161,8 +166,8 @@ Errors name the flag and the offending part, and nothing starts: --source file,directory=/x: TYPE ends at ':', as in file:directory=/x ``` -A misspelled plugin key fails when the plugin is built, before any listener -opens: `failed to create sink http: ... unknown key "tls.enabeld"`. +A misspelled plugin key fails validation, before anything is built or opened: +`pipelines[0].plugin_sinks[http].config: unknown key "tls.enabeld"`. ```bash # tail a directory, serve it over SSE on every interface; a browser at @@ -189,17 +194,23 @@ Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns). `--preset NAME:key=value,...` takes the [SPEC](#pipelines) syntax. A list key (`hosts`, `proxy`, `allow`, `deny`) takes several values: repeated (`hosts=a,hosts=b`) or `\,`-separated in a SPEC, `,`-separated in `lw preset` -flags and the TUI; any other key takes one. -`lw preset NAME [--key value ...]` prints the pipeline a preset expands to, -ready for a configuration file; its flags are the keys, spelled as in the SPEC -(`--password_file`), and `-u` is `--user`; -`lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An -unknown key fails and lists the valid ones. A `path` is a file, a directory -(its files) or a glob; a preset that takes one reads stdin without it, and -`from` (`end`, or `start`) is where reading a file starts. `user` names one -SCRAM user, an edge's login or a listener's only user in place of `users`; -`password_file` holds its password and defaults to `/dev/tty`, which asks -([password sources](security.md#one-user-without-a-credentials-file)). +flags (a repeated flag keeps only its last value) and the TUI; any other key +takes one. `lw preset NAME [--key value ...]` prints the pipeline a preset +expands to, ready for a configuration file; its flags are the keys, spelled as +in the SPEC (`--password_file`). `lw preset NAME -h` lists its keys and +defaults, `lw preset -h` the presets. An unknown key fails and lists the valid +ones. + +- `path`: a file, a directory (its files) or a glob, and this host decides + which: an existing directory is followed whole, any other path is a file + name or glob in its directory; for a directory not here, as in a `--dump` + for another host, write `DIR/*`. Without one, `serve` and `edge` read stdin +- `from` (`end`, or `start`): where reading a file starts +- `user`: one SCRAM user, an edge's login or a listener's only user in place + of `users`; `password_file` holds its password and defaults to `/dev/tty`, + which asks ([password sources](security.md#one-user-without-a-credentials-file)) + +The presets, each with its keys: - `pipe`: stdin to stdout, the built-in default - `format` (`raw`) @@ -242,7 +253,7 @@ lw --preset tail:path=/var/log/app lw --preset serve:path=/var/log/app # serve it over HTTPS, self-signed, to SCRAM users; keep the result as a file -lw --preset serve:path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml +lw --preset 'serve:path=/var/log/app/*,tls=self,users=/etc/logwisp/users.toml' --dump > serve.toml # an aggregator with a certificate made at startup, and an edge pinning it lw --preset aggregator:users=/etc/logwisp/users.toml,out=/var/log/edges @@ -264,8 +275,10 @@ changes when lw restarts; a fleet signs with an issuer instead, and edges set ## Environment Variables -Configuration paths map to environment variables by replacing `.` with `_`, -uppercasing and adding `LOGWISP_`: +Every setting is also a variable: its path with `.` as `_`, uppercased, after +`LOGWISP_`. An empty variable, a setting's or a pipeline's, counts as unset. +They rank below flags and above the file +([precedence](configuration.md#configuration-precedence)). ```bash export LOGWISP_QUIET=true @@ -273,10 +286,7 @@ export LOGWISP_LOGGING_LEVEL=debug export LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp ``` -Bare names used by older versions are now ignored. Rename them to the prefixed -forms when upgrading; see [Configuration](configuration.md#environment-variables). - -The path resolver reads these variables directly: +The path resolver reads two more: - `LOGWISP_CONFIG_FILE`: the configuration file path, joined onto `LOGWISP_CONFIG_DIR` when both are set @@ -290,14 +300,16 @@ Any pipeline flag on the command line makes LogWisp ignore all of them. - `LOGWISP_PIPELINE`: the pipeline's name, default the `LOGWISP_PRESET` name, else `cli` -- `LOGWISP_PRESET`: the [preset](#presets) that starts it +- `LOGWISP_PRESET`: the [preset](#presets) that begins it - `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each - `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered one in numeric order; likewise `LOGWISP_SINK_N` and `LOGWISP_FILTER_N` - `LOGWISP_FORMAT`, `LOGWISP_RATE_LIMIT`, `LOGWISP_HEARTBEAT` -- an empty variable counts as unset -A container needs no configuration file: +A container needs no configuration file. Without one lw logs less +([Built-in Defaults](#built-in-defaults)), so pass `LOGWISP_LOGGING_LEVEL=info` +and `LOGWISP_STATUS_REPORTER=true`; the image sets neither, as a variable would +override a mounted file's value: ```bash docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \ @@ -306,45 +318,21 @@ docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \ -e LOGWISP_FILTER='exclude:patterns=DEBUG' \ -e LOGWISP_SINK='http:host=0.0.0.0,port=8080' \ -e LOGWISP_SINK_1=console \ - -e LOGWISP_LOGGING_LEVEL=info \ - logwisp + -e LOGWISP_LOGGING_LEVEL=info -e LOGWISP_STATUS_REPORTER=true \ + logwisp:dev ``` -Without a file, lw logs at `warn` with no status reporter, so a container -configured through the environment passes `LOGWISP_LOGGING_LEVEL=info` (and -`LOGWISP_STATUS_REPORTER=true`) to keep the service defaults. The image does -not set them: environment variables would override a mounted file's values. - -## Precedence - -- Pipelines - - pipeline flags, if any are given - - else pipeline variables, if any are set - - else the configuration file's `[[pipelines]]` - - else the built-in `pipe` ([Built-in Defaults](#built-in-defaults)) -- Every other key - 1. command-line flags - 2. environment variables - 3. configuration file - 4. built-in defaults; `logging.level` and `status_reporter` depend on - whether a file was found - -A reload rereads the file and keeps the command-line or environment pipelines. - ## Signals - `SIGINT`, `SIGTERM`: graceful shutdown -- `SIGHUP`, `SIGUSR1`: reload the configuration +- `SIGHUP`, `SIGUSR1`: reload: the file is reread and the whole service + rebuilt; a configuration error leaves the running service untouched + ([Hot Reload](configuration.md#hot-reload)) +- `SIGHUP` before the signal handler is installed is ignored, so LogWisp + survives a terminal hang-up like `nohup` - On the `lw --tui` screen, `SIGTERM` and `SIGHUP` close it, stopping what `t` runs, and lw exits 0 -`SIGHUP` is ignored during startup, before the signal handler is installed, so -LogWisp survives a terminal hang-up like `nohup`. Once running, it triggers a -reload rather than terminating. - -Reload rebuilds the whole service. A configuration error leaves the running -service untouched; see [Configuration](configuration.md#hot-reload). - ## Exit Codes - `0`: the end of input, a clean shutdown (`SIGINT`, `SIGTERM`), `--version` / @@ -353,15 +341,12 @@ service untouched; see [Configuration](configuration.md#hot-reload). included, and a bad value on the command line), a logger init failure, a service bootstrap failure - `2`: a usage error (an unknown option, a word no option takes, a missing - value or `-c` path), or an explicitly requested configuration file that is - not found + value or `-c` path), or a configuration file named by `-c`, `--config=` or + a `LOGWISP_CONFIG_*` variable that does not exist + ([File Location](configuration.md#file-location)) - killed by `SIGPIPE` (shell status `141`): the reader of stdout went away, as in `lw | head`; `cat` ends the same way -A missing file is exit code 2 only when it was named explicitly (`-c`, -`--config=`, or the `LOGWISP_CONFIG_*` variables). A missing discovered default -is not an error, and LogWisp starts on built-in defaults. - ## Built-in Defaults Without a configuration file and without pipeline flags or variables, lw runs @@ -417,9 +402,6 @@ journalctl -f | lw --sink http:host=127.0.0.1,port=8080 lw --source file:directory=/var/log/app,pattern='*.log' > all.log ``` -A console sink never drops: when its reader is slow, the pipeline waits, and -so does the reading of stdin. - **Development** ```bash @@ -439,7 +421,7 @@ exits 1 (2 when a named file is missing, see [Exit Codes](#exit-codes)): ```bash lw --check -c /etc/logwisp/logwisp.toml -lw --check --source file:directory=/var/log/app --sink http:port=8080 +lw --check --source file --sink tcp:port=70000 ``` ```text @@ -490,11 +472,13 @@ WebAssembly; `t` runs them under the screen and shows what flows. required key, then edits a value (empty returns it to its default), opens a list's choices or adds on an add row; in a value, Enter sets it, and Tab or Up/Down set it and move. Left/Right step a choice or set a switch; - Space flips it; Esc goes back. `a` adds a source, sink or filter; `c` - changes a source's or sink's type in its place; `d` deletes, or returns a - value to its default; Space turns a flow stage on or off; `J`/`K` move a - filter; `[` `]` switch pipelines; `p` presets; `!` shows every problem, - moving to the first; `r` runs; `t` taps; `o` output; `?` keys; `q` quits. + Space flips it; Esc goes back; Tab moves to the next column. `a` adds a + source, sink or filter; `c` changes a source's or sink's type in its place, + and moves to a filter's (in the inspector) or the format's `type` row; `d` + deletes, or returns a value to its default; Space turns a flow stage on or + off; `J`/`K` move a filter; `[` `]` switch pipelines; `p` presets; `!` shows + every problem, moving to the first; `r` runs; `t` taps; `o` output; `?` + keys; `q` or Ctrl-C quits. - Pipelines: Up from a column's top row reaches the bar, where `[` `]` or Left/Right switch, `a` adds one (empty or from a preset), Enter renames and `d` deletes, asking first when it has parts. @@ -520,7 +504,8 @@ WebAssembly; `t` runs them under the screen and shows what flows. dropped by sinks, and missed by the screen: the view never slows a pipeline. - Live view: from 40 rows it sits under the diagram; shorter, it takes the - screen while it has the keys. `t` or Esc gives them back to the diagram. + screen while it has the keys. `t` or Esc gives them back to the diagram + (Esc first closes an open detail). - The canvas cursor picks its point, so the view opens on the first source: a source shows its entries, a flow stage (or the filter the inspector is on) what reaches it, the heartbeat its own, the sinks (or @@ -533,13 +518,13 @@ WebAssembly; `t` runs them under the screen and shows what flows. screen drops columns; a dropped entry is marked; a rune a terminal would act on or draw wider than a cell shows as a placeholder. - Keys: arrows or `jk` move (up pauses, down onto the newest follows), - PgUp/PgDn; `g` the oldest, `G` or `f` the newest; Enter the row's - detail, `J`/`K` scroll it; `/` only rows matching a pattern, ignoring - case when it is all lower case (clearing it, or a level, keeps the - focused row in place, paused there unless it is the newest); `1`-`5` - trace to error and up, `0` all; `s` chooses the point; - `i` inspects the stage that dropped the row; `c` clears; `q`, `?`, `r`, - `o`, `[` and `]` act as on the diagram. + PgUp/PgDn; `g` or Home the oldest, `G`, End or `f` the newest; Enter the + row's detail, `J`/`K` scroll it; `/` only rows matching a pattern, + ignoring case when it is all lower case (clearing it, or a level, keeps + the focused row in place, paused there unless it is the newest); `1`-`5` + trace to error and up, `0` all; `s` chooses the point; `i` inspects the + stage that dropped the row; `c` clears; `q`, Ctrl-C, `?`, `r`, `o`, `[` + and `]` act as on the diagram. - The detail, beside the rows from 160 columns and under them otherwise: time and the stamp as written, level, origin, the verdict with a filter's patterns, the layout read, message, fields, and the payload as @@ -574,8 +559,6 @@ lw auth token --url https://HOST:PORT[/PATH] -u NAME [--password-file FILE lw auth stream --addr HOST:PORT -u NAME [--password-file FILE] [TLS flags] ``` -`-u` is short for `--user`. - - `add-user`: adds a user, or replaces its password - `remove-user`: removes a user; refuses the last one - `token`: logs in to an `http` sink or `http_chain` source and prints a @@ -611,8 +594,9 @@ echo; a file, `/dev/fd/N` or a pipe is read as by a `--ca-file` (default: system roots) or `--pin-sha256` (the `tls.pin_sha256` a self-signed listener logs), `--server-name` (default: the host), and `--cert-file` / `--key-file` for a listener with `tls.client_auth`. There is no -flag to skip verification: an unverified server could relay the login. Redirects are not followed. `--unbound` logs in to an `http` sink behind -a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's +flag to skip verification: an unverified server could relay the login. +Redirects are not followed. `--unbound` logs in to an `http` sink behind a +TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's certificate across the two requests; only then may `--url` carry the path the proxy mounts LogWisp at. `stream` exits `0` on `SIGINT` or `SIGTERM`, and `1` when the server ends the stream (a reload or shutdown). @@ -665,8 +649,8 @@ lw tls cert --ca-dir DIR --name NAME [--hosts NAME,...] [--server] [--client] `tls.issuer_cert_file` and `tls.issuer_key_file`. - `cert`: `NAME.crt` and `NAME.key`, in `--out` (default: `--ca-dir`). - `--server` for a listener: `--hosts` lists the DNS names and IP addresses - it carries, default `NAME` (`--host`, its earlier name, still works); - `--client` for a dialer presenting a certificate; both may be given. + it carries, default `NAME`; `--host` is an alias. `--client` for a dialer + presenting a certificate; one of the two is required, both may be given. - `NAME` is the subject CN, the identity `auth.type = "mtls"` matches. - Valid 397 days (the longest browsers accept), never past the CA. It prints the certificate's `pin_sha256`. @@ -687,10 +671,12 @@ lw config init [--out FILE] `config/logwisp.toml`, every option with its default, to edit and run. It never replaces a file (remove one to replace it). Without `--out` it writes `~/.config/logwisp/logwisp.toml`, the file lw reads without `-c` unless a -`LOGWISP_CONFIG_*` variable names another; with that file in place, lw -without pipeline options runs its pipelines rather than filtering stdin. -Any other path is run with `-c FILE`; lw prints which applies. `lw --dump` prints the -effective configuration instead, flags and environment included. +`LOGWISP_CONFIG_*` variable names another; any other path is run with +`-c FILE`, and lw prints which applies. As shipped the file defines no +pipeline, so lw still filters stdin, though its own log on stderr is at the +file's default `info` rather than `warn`; once it defines `[[pipelines]]`, lw +without pipeline options runs them instead. `lw --dump` prints the effective +configuration, flags and environment included. ```bash lw config init diff --git a/doc/configuration.md b/doc/configuration.md index c87335e..b2f4720 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -6,12 +6,12 @@ and its default ships as [`config/logwisp.toml`](../config/logwisp.toml). ## Configuration Precedence - Pipelines come from the first of these that defines any, replacing the rest - wholesale (the default pipeline's rate limit and formatter included): + wholesale: 1. pipeline flags: `--source`, `--sink`, ... ([CLI](cli.md#pipelines)) 2. pipeline variables: `LOGWISP_SOURCE`, ... ([CLI](cli.md#pipeline-variables)) 3. the file's `[[pipelines]]` - 4. the built-in default pipeline + 4. the built-in `pipe` pipeline ([CLI](cli.md#built-in-defaults)) - Every other key merges, highest priority first: 1. command-line flags 2. environment variables @@ -28,33 +28,28 @@ The path is resolved before any other configuration is read: 4. `~/.config/logwisp/logwisp.toml`, if it exists 5. `./logwisp.toml` -Missing file behaviour differs by how it was chosen. An explicitly requested -file that does not exist is a fatal error (exit code 2); a missing discovered -default is not an error, and LogWisp starts on built-in defaults. - -File-selection flags are consumed before schema overrides are parsed. A missing -or empty path is a usage error; `--` stops option parsing. The selected path is -runtime metadata and cannot be redirected by a `config_file` key inside the file. +A file named by `-c` or a variable must exist (exit code 2); a missing +discovered one is not an error, and lw runs on built-in defaults. A relative +path that climbs with `..` is refused; give it absolute. A file may hold at +most 10 MiB. ## Global Settings Top-level keys, each as type and default: - `quiet` (bool, `false`): disable all application logging and console - diagnostics; pipeline output still flows. + diagnostics; pipeline output still flows. It overrides every logging + setting. - `status_reporter` (bool, `true` with a configuration file, `false` without): - emit a status report every 30 s at DEBUG level. + a status report every 30 s, shown only at `logging.level = "debug"`. - `auto_reload` (bool, `false`): watch the config file and reload pipelines on change. - `color` (string, `"auto"`): level names in color on console sinks, which may set their own: `"auto"` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`), `"always"` or `"never"`; see the [console sink](sinks.md#console). -`--version`, `--check`, `--dump` and `--schema` print and exit; they are not -persistent settings. - -Note that `status_reporter` writes at DEBUG level, so it produces nothing unless -`logging.level = "debug"`. +`--version`, `--check`, `--dump`, `--schema` and `--tui` are switches, not +settings; the matching variables are ignored. ## Application Logging @@ -76,7 +71,7 @@ max_total_size_mb = 1000 retention_hours = 168.0 ``` -### Output modes +`output` modes: - `file`: files only. - `stdout`: standard output only. @@ -85,23 +80,12 @@ retention_hours = 168.0 - `all`: files plus split console. - `none`: no application logging. -`[logging.file]` applies only to the `file` and `all` modes. The `stdout`, -`split` and `all` modes mix the log into a console sink's stdout. - -Without a configuration file (none named, none found) lw is a command-line -tool: `level` defaults to `warn` and `status_reporter` to `false`. Explicit -values from the environment or flags still win. A container configured only -through variables or flags passes `LOGWISP_LOGGING_LEVEL=info` to log as a -service; the image does not set it, since a variable would override a mounted -file's value. - -> `[logging.console].target` is accepted and validated (`stdout`, `stderr`, -> `split`) but **not applied**. The console destination is derived from -> `logging.output`. The key is retained for compatibility; setting it has no -> effect. - -`quiet = true` overrides every logging setting and disables both file and -console output. +`[logging.file]` applies only to the `file` and `all` modes. Its limits delete +any `.log` file in `directory`, lw's or not: give lw a directory of its own. +`retention_hours = 0` keeps files. The `stdout`, `split` and `all` modes mix +the log into a console sink's stdout. `[logging.console].target` is validated +(`stdout`, `stderr`, `split`) but not applied: `logging.output` chooses the +console. ## Pipeline Configuration @@ -131,7 +115,7 @@ interval_ms = 30000 # --- sources: one or more --- [[pipelines.plugin_sources]] -id = "app_logs" # unique within the pipeline +id = "app_logs" # unique among the pipeline's sources type = "file" [pipelines.plugin_sources.config] directory = "/var/log/myapp" @@ -146,14 +130,13 @@ port = 8080 Every source and sink is a plugin instance with three keys: -- `id`: instance identifier, unique within the pipeline; appears in logs and - stats. +- `id`: instance identifier, unique among the pipeline's sources, and among + its sinks; appears in logs and stats. - `type`: registered plugin type. - `config`: plugin-specific table; see [Sources](sources.md) and [Sinks](sinks.md). -`config_file` is reserved on both structures for a future include mechanism and -is not implemented. +`config_file` on a source or sink is accepted and ignored. ### Flow stages @@ -168,19 +151,16 @@ Every stage is optional: Options, each as type and default: -- `entries_per_second` (float, `0`): `<= 0` disables the limiter entirely. +- `entries_per_second` (float, `0`): `0` disables the limiter; a negative value + is refused. - `burst_entries` (float, `entries_per_second`): token bucket capacity. -- `policy` (string, `pass`): `pass` allows everything through, `drop` discards - over-limit entries. +- `policy` (string, `pass`): `pass` or `drop`, in lower case like every enum. + `drop` discards entries over the rate or the size cap; `pass` lets + everything through. - `max_entry_size_bytes` (int, `0`): per-entry byte cap; `0` = unlimited. -Two behaviours are easy to trip over: - -- The limiter is constructed only when `entries_per_second > 0`. With `0`, - `max_entry_size_bytes` is never enforced. -- `policy = "pass"` short-circuits the whole check, including the size cap. - To enforce a size cap you need `entries_per_second > 0` **and** - `policy = "drop"`. +The limiter, its size cap included, acts only with `entries_per_second > 0` +and `policy = "drop"`. #### Heartbeat @@ -188,138 +168,98 @@ Options, each as type and default: - `enabled` (bool, `false`): enable heartbeat generation. - `interval_ms` (int, `1000`): interval; minimum `100`. -- `include_timestamp` (bool, `false`): `false` formats with level only, no - timestamp. -- `include_stats` (bool, `false`): attach `beat_count` and measured - `interval_ms` as fields. -- `format` (string, `txt`): `txt`, `json` or `raw` are accepted but have no - effect: heartbeats go through the flow's `format`, like every entry. +- `include_timestamp` (bool, `false`): `false` writes the level only, no + timestamp; the `raw` format shows neither. +- `include_stats` (bool, `false`): add `type = "heartbeat"`, `beat_count` and, + from the second beat, the measured `interval_ms` as fields. +- `format` (string, `txt`): `txt`, `json` or `raw`; no effect: heartbeats go + through the flow's `format`. Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They are generated after the flow's filter and rate-limit stages, so filters do not suppress them, and they reach every sink in the pipeline. -> `format = "comment"` (SSE comment framing) appears in older documentation and -> in a code path in the generator, but the validator rejects it and the pipeline -> fails to start. Use `txt`, `json`, or `raw`. +## Environment Variables and Flags -## Environment Variables - -Environment overrides are derived from the TOML path: `.` becomes `_`, the -result is uppercased, and `LOGWISP_` is prepended: - -``` -quiet LOGWISP_QUIET -status_reporter LOGWISP_STATUS_REPORTER -logging.level LOGWISP_LOGGING_LEVEL -logging.file.directory LOGWISP_LOGGING_FILE_DIRECTORY -``` - -Migration: the old custom transform accidentally read bare names such as `QUIET` -and `LOGGING_LEVEL`. Rename those variables to their prefixed forms; bare names -are now ignored. `LOGWISP_CONFIG_FILE` and `LOGWISP_CONFIG_DIR` still select the -file directly. - -Only scalar paths that exist in the configuration schema can be set this way. -Array elements cannot: `LOGWISP_PIPELINES_0_NAME` has no effect. A whole -pipeline can: see [pipeline variables](cli.md#pipeline-variables). - -## Command-Line Overrides - -Any scalar configuration path is settable as a flag using its TOML path: - -```bash -lw --logging.level=debug --status_reporter=false -lw --logging.level debug # space form also works -lw --quiet # bare flag means true -``` - -A bare flag is accepted only for a boolean key, which takes its value only -after `=`: `--logging.file.directory --quiet` fails with -`--logging.file.directory needs a value`, and `--quiet false` with `unexpected -argument "false"`. An unknown key is an error too (`unknown option ---nosuch.key`); each is a usage error, exit code 2. Single-dash options are -lw's own ([CLI](cli.md#options)). - -An empty `LOGWISP_` variable counts as unset, as the pipeline variables -do. - -An array-indexed path such as `--pipelines.0.name=x` is an error (exit 1); -whole pipelines have [their own flags](cli.md#pipelines). +Every setting is also a variable and a flag by its path: `logging.level` is +`LOGWISP_LOGGING_LEVEL` and `--logging.level`. Pipelines are not: their flags +and variables replace the file's whole, and an array-indexed path such as +`--pipelines.0.name=x` is an unknown option (exit code 2). See +[CLI](cli.md#options) and [Environment Variables](cli.md#environment-variables). ## Validation -Startup validation is intentionally split. +The configuration is checked offline when it loads, at every reload and by +`lw --check`. Each check reports the first problem of each table (the top +level, `logging`, each flow stage, each source and sink) by its key's path: -`internal/config` validates only global structure: +``` +Error: Failed to load config: invalid configuration: +pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "Drop" +pipelines[0].plugin_sinks[http].config: unknown key "tls.enabeld" +``` -- no key the file schema does not declare, at any depth outside a plugin's - `config` table: `config file "…": unknown key - "pipelines[0].plugin_sinks[0].confg"`. A misspelled table path would otherwise - drop the whole table it heads. A top-level `config_file` is ignored -- pipeline [specs](cli.md#pipelines): syntax, and unknown flow-stage keys such - as `--rate-limit entries_per_second=1,polcy=drop` -- at least one pipeline -- unique, non-empty pipeline names -- at least one source and one sink per pipeline -- `logging.output`, `logging.level`, `logging.format`, `logging.sanitization`, - and `logging.console.target` enum membership +- Pipeline flags and variables come first, alone: their syntax and keys + (`--rate-limit entries_per_second=1,polcy=drop: unknown key "polcy"`). +- Settings: the `color` enum and the `logging` ones (`output`, `level`, + `format`, `sanitization`, `console.target`). +- Pipelines: at least one; names set and unique; a source and a sink each; + ids unique among a pipeline's sources and among its sinks; one `console` + source in the whole configuration. +- Plugin options are decoded in full: unknown keys at any depth, kinds, + required keys, enums, ranges, paths, and the `tls`, `auth` and `acl` rules + of the plugin's side; a `tls` table's rules only when it is `enabled`. +- Enums are case-sensitive: `policy = "Drop"` and `target = "STDERR"` are + refused. +- In plugin options and flow stages, `0`, or `""` for a string, takes the + key's default, or the meaning its help gives zero (off, none, unlimited). A + number below its key's minimum is refused, a negative one included where + zero means the default (`buffer_size = -1`). +- Flow stages are checked whether enabled or not: kinds, enums and ranges, + and filter patterns must compile ([Filters](filters.md)). +- `auth` enums (`identity`, `node_binding`) are checked whatever `type` is; a + block naming peers or credentials with `type` `none` or unset is refused + ([Security](security.md#validation-1)). +- Once the rest is valid, the file itself: the first key the schema does not + declare outside a plugin's `config` is refused (`config file "…": unknown + key "pipelines[0].plugin_sinks[0].confg"`), since a misspelled table path + would drop the whole table it heads. A top-level `config_file`, `version`, + `check` or `dump` is ignored. +- Files a plugin names (certificates, keys, CA files, credentials, password + files) are read when it is built: at start, at reload and by `lw --check`. + Those failures name the pipeline and plugin. -Everything else is validated by the plugin constructor that owns it — port -range, required paths, path prefixes, enum values, regex compilation, TLS and -credentials file loading. A failure there aborts pipeline construction with a -message naming the pipeline, plugin id, and offending key. A key the plugin does -not declare, at any depth, is such a failure (`unknown key "tls.enabeld"`): a -misspelled option must not silently fall back to its default. For the same -reason an `auth` block that names peers or credentials (`allow`, -`allow_patterns`, `credentials_file`, `token_lifetime_ms`, `username`, -`password_file`, `trusted_proxies`) while its `type` is `none` or unset is refused: auth was -intended and the type forgotten. See [Security](security.md#the-auth-block). - -There is **no** cross-pipeline port-conflict detection. Two sinks bound to the -same port fail at listener bind time, when the pipeline starts. +Two listeners on one port pass validation; the second fails at bind and lw +exits 1 ([Networking](networking.md#network-plugins)). ## Hot Reload -```toml -auto_reload = true -``` +`auto_reload = true` watches the configuration file. `SIGHUP` or `SIGUSR1` +rereads it even when watching is off and always rebuilds, so certificates and +credentials rotate without a TOML edit. CLI and environment overrides, +pipelines included, are captured at startup and keep their precedence. A +discovered default file that does not exist is not an error, so an instance +configured by flags or variables alone still rebuilds; once read, though, its +removal fails the reload and the running service stays as it is; restart to +run without the file. -or send `SIGHUP` / `SIGUSR1`. Signals reread the selected file even when watching -is disabled and always rebuild, allowing certificate and credentials rotation -without TOML edits; `auto_reload` watches only the configuration file. -CLI and environment overrides, pipelines included, are captured at startup and -keep their precedence. As at startup, a discovered default file that does not -exist is not an error, so an instance configured by flags or variables alone -still rebuilds. Once read, though, its removal fails the reload and the running -service stays as it is; restart to run without the file. +Reload builds a new service from a detached snapshot of the configuration, +validated again, and shuts the old one down only if that succeeds: parse, +validation and construction errors leave the running service intact. Listener +bind failures occur after the old service has stopped and can leave lw without +working pipelines; correct the file and signal again. -Reload rebuilds the whole service: a new service is constructed from the new -configuration first, and only if that succeeds is the old one shut down. Each -candidate is a detached snapshot and runs top-level validation again. Parse, -conversion, validation and construction errors leave the running service intact. -Listener bind/start failures occur after shutdown of the old service and can leave -the application without working pipelines; correct the file and signal again. +Watch errors (deletion, permission changes, timeout) are logged without +rebuilding. Queued changes are combined, and unchanged pipelines and status +settings do not trigger another rebuild. Removed file keys fall back to the +remaining sources on the next successful load. -Watch errors (including deletion, permission changes and timeout) are logged -without rebuilding. Queued path changes are combined, and unchanged pipelines -and status settings do not trigger another rebuild. Removed file keys fall back -to the remaining sources on the next successful load. +- Reloaded: pipelines, sources, sinks, flow stages, `status_reporter`, `color` +- Not reloaded: `logging.*` (applied once at startup), `quiet`, `auto_reload` + (the watcher is not restarted) -What a reload applies: - -- Reloaded: - - pipelines, sources, sinks - - filters, formatters, rate limits, heartbeats - - `status_reporter` - - `color` -- Not reloaded: - - `logging.*`: applied once at startup - - `quiet` - - `auto_reload`: the watcher is not restarted - -Because the rebuild is total, listeners close and reopen and every connected -client is disconnected. Chain sinks reconnect on their own backoff schedule. +The rebuild is total: listeners close and reopen, every connected client is +disconnected, and chain sinks reconnect on their own backoff schedule. ## Type Reference @@ -332,7 +272,5 @@ Each TOML type, its Go type, and its command-line / environment form: - Array (`[]T`): only in [pipeline specs](cli.md#pipelines), as a repeated key. - Table (struct): nested path with `.` (flags) or `_` (environment). -Integer fields reject fractions, overflow and negative unsigned values. Non-finite -floats and integer-to-float precision loss are rejected too. This applies to plugin -maps when their constructors decode them. Only TOML configuration files are accepted; -JSON log formatting and network payloads are unaffected. +Integer keys refuse fractions and out-of-range values; number keys refuse inf +and NaN. Configuration files are TOML only. diff --git a/doc/deployment.md b/doc/deployment.md index 7c0747f..5085bda 100644 --- a/doc/deployment.md +++ b/doc/deployment.md @@ -5,7 +5,7 @@ for what its flags leave open, prints a plan with the equivalent non-interactive command, then writes the files, runs `lw --check` on them (the node's binary or image, with its mounts) and starts the node. `--dry-run` stops at printing; `--yes` never asks and fails on a missing required value. It creates no -certificates: see [Security](security.md#enabling-mtls). +certificates: `lw tls` makes them ([CLI](cli.md#lw-tls)). ## Roles @@ -123,14 +123,15 @@ needs `--force`. Application jails run an edge each; a `logs` jail aggregates. Every jail runs already, and the `logs` jail has an address the others reach (`AGG_ADDR`). -1. Certificates: a CA, and a server certificate for `logs` naming `AGG_ADDR` - ([Security](security.md#enabling-mtls)). +1. Certificates: `lw tls ca --dir PKI`, then + `lw tls cert --ca-dir PKI --name logs --hosts AGG_ADDR --server`, which + writes `PKI/logs.crt` and `PKI/logs.key` ([CLI](cli.md#lw-tls)). 2. The aggregator, from the host: ```sh deploy/lw-deploy.sh --yes --runtime native --jail logs --role aggregator \ --file-dir /var/log/logwisp --http-port 8080 \ - --tls --cert-file logs.crt --key-file logs.key \ + --tls --cert-file PKI/logs.crt --key-file PKI/logs.key \ --auth scram --add-user web1 --add-user db1 \ --sink-auth scram --add-viewer ops ``` @@ -142,13 +143,13 @@ already, and the `logs` jail has an address the others reach (`AGG_ADDR`). ```sh deploy/lw-deploy.sh --yes --runtime native --jail web1 --role edge \ --log-dir /var/log/nginx --aggregator AGG_ADDR \ - --tls --ca-file ca.crt --auth scram --password-file ~/logwisp-secrets/web1.pass + --tls --ca-file PKI/ca.crt --auth scram --password-file ~/logwisp-secrets/web1.pass ``` 4. Check: `jexec logs service logwisp status`, the files in the `logs` jail's `/var/log/logwisp`, and `https://AGG_ADDR:8080/status` with a token from `lw auth token --url https://AGG_ADDR:8080 --user ops --password-file - ~/logwisp-secrets/ops.pass --ca-file ca.crt` ([CLI](cli.md#lw-auth)). + ~/logwisp-secrets/ops.pass --ca-file PKI/ca.crt` ([CLI](cli.md#lw-auth)). Variants: @@ -187,7 +188,7 @@ every key. - `acl` on the listeners: address rules, PROXY headers, per-client limits - mtls identities other than the certificate CN; `node_binding` other than `force` -- creating certificates +- creating certificates (`lw tls`) - several native instances on one host: use one per host or jail ## Testing the script diff --git a/doc/filters.md b/doc/filters.md index f4d909c..2ceeef7 100644 --- a/doc/filters.md +++ b/doc/filters.md @@ -18,8 +18,9 @@ patterns = ["ERROR", "WARN"] pattern matches). - `patterns`, []string, default `[]`: Go RE2 regular expressions. -A filter with no patterns passes everything. Invalid patterns fail at startup -with the filter index and the offending pattern in the message. +A filter with no patterns passes everything. An invalid pattern is refused +when the configuration loads, named by its path +(`pipelines[0].flow.filters[0].patterns[1]`). ## What Gets Matched @@ -34,7 +35,8 @@ Empty parts are omitted, so an entry with no detected level matches level as easily as the message body: - `"^app\\.log "` matches entries whose source is `app.log`. -- `"ERROR"` matches level `ERROR`, or the word `ERROR` anywhere in the message. +- `"ERROR"` matches `ERROR` anywhere in the text: the level, the message, the + source name, or inside a word (`NOERROR`). Use `"\\bERROR\\b"` for the word. The structured `fields` payload is **not** part of the match text. @@ -124,16 +126,15 @@ time, which is exactly what you want in a log hot path. - Whole word: `\berror\b` - Alternation: `ERROR|WARN|FATAL` - Character class: `[0-9]{3}` -- Anchors: `^ERROR`, `ERROR$` +- Anchors: `^app\.log ` (the text starts with the source), `ERROR$` - Any characters: `.*exception.*` Remember that TOML basic strings process escapes, so a regex backslash needs doubling: `"\\berror\\b"`. TOML literal strings avoid the issue: `'\berror\b'`. -Anchors apply to the assembled match text, which begins with the source name — -so `^ERROR` will not match an entry whose source is non-empty. Use -`\bERROR\b` instead unless you mean to anchor on the source. +Anchors apply to the assembled match text, which begins with the source name, +so `^ERROR` never matches an entry that has a source; use `\bERROR\b`. ## Common Recipes @@ -153,8 +154,8 @@ type = "exclude" patterns = ["/healthz", "/metrics", "\\bping\\b"] ``` -**Secret suppression** — see [Security](security.md); filters are the only -redaction mechanism LogWisp currently offers. +**Secret suppression**: LogWisp has no redaction; an exclude filter drops each +entry that matches ([Security](security.md)). ```toml [[pipelines.flow.filters]] @@ -162,8 +163,6 @@ type = "exclude" patterns = ["password", "api[_-]?key", "authorization", "bearer ", "secret", "token"] ``` -Note this drops the whole entry, it does not redact part of it. - **Per-application routing** — run one pipeline per application, each with its own include filter, rather than trying to route inside one pipeline. Sinks fan out to *all* sinks in a pipeline; there is no conditional routing. @@ -182,7 +181,6 @@ evaluated against every filter until one rejects it, so cost scales with the number of patterns and their complexity. Prefer literal substrings and simple alternations over broad `.*` wildcards. -Filters log at DEBUG on every entry — pattern text, match results, and the -final decision. That is invaluable when a filter is not behaving as expected and -very expensive in production; keep `logging.level` at `info` or higher on a busy -pipeline. +Filters log every pattern match and decision at DEBUG: useful when a filter +misbehaves, expensive on a busy pipeline, where `logging.level` stays at +`info` or higher. diff --git a/doc/formatters.md b/doc/formatters.md index f40e99e..4df8a25 100644 --- a/doc/formatters.md +++ b/doc/formatters.md @@ -12,8 +12,9 @@ timestamp_format = "" ``` One formatter serves the whole pipeline. Sinks receive an identical payload, -one record ending in exactly one newline; there is no per-sink formatting. When you need two shapes of the same data, run -two pipelines, or chain to a node that formats differently. +one record ending in exactly one newline; there is no per-sink formatting. +When you need two shapes of the same data, run two pipelines, or chain to a +node that formats differently. Omitting `[pipelines.flow.format]` entirely selects `raw`. @@ -32,27 +33,25 @@ Each as type and default: ### raw -Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches -the sink as the source produced it, with no timestamp, level or source prefix -added, and a newline appended unless the message ends in one. An entry that also -carries `fields` gets the fields JSON appended verbatim after a single space — -`raw` never drops data and never re-encodes it. +Passthrough, the fastest type, for text already in its final form. `FlagRaw` +bypasses formatting and sanitization: the message reaches the sink as the +source produced it, control characters included, with no timestamp, level or +source prefix added, and a newline appended unless the message ends in one. +An entry that also carries `fields` gets the fields JSON appended verbatim +after a single space; `raw` never drops data and never re-encodes it. ```toml [pipelines.flow.format] type = "raw" ``` -Fastest option, and the right one when you are relaying text that is already in -its final form. Note that it also bypasses sanitization, so control characters -in the source data reach your sinks intact. - -Byte-exact transport needs a source that does not split the line: the `console` -source, or the `file` source with `raw = true`. Both put the whole line, its -terminator removed, in the message and leave `fields` empty; `raw` writes it -back as one `\n`. The `file` source's -JSON branch splits a line into message and fields, so `raw` reassembles it as -` ` rather than reproducing the original object. +The closest to verbatim transport is the `console` source, or the `file` +source with `raw = true`. The whole line, terminator removed, becomes the +message and `fields` stays empty; `raw` writes it back with one `\n`. It is not +byte-exact: blank lines are skipped, CRLF becomes LF, and a line over 1 MiB +continues in the next entry. The `file` source's JSON branch splits a line +into message and fields, so `raw` writes ` ` rather than the +original object. ### txt @@ -81,9 +80,13 @@ Output has the shape: {"time":"2026-01-02T15:04:05.123Z","level":"ERROR","trace":"edge-01/app.log","fields":["connection refused"]} ``` -The exact key names and structure come from the `lixenwraith/log` formatter, not -from LogWisp; they are stable for a given dependency version but are not part of -LogWisp's own configuration surface. +`level` is DEBUG, INFO, WARN or ERROR, here and in `txt`. Only `DEBUG`, +`INFO`, `WARN`, `WARNING` and `ERROR`, all upper or all lower case, are read as +such; any other level, `TRACE` included, and an entry without one print as +INFO. Filters see the level as the source stored it. + +The key names and structure come from the `lixenwraith/log` formatter, not +from LogWisp's configuration. ## Flags @@ -93,13 +96,15 @@ you need to override the defaults. - `1` Raw: bypass formatting and sanitization entirely. - `2` ShowTimestamp: emit the timestamp. - `4` ShowLevel: emit the level. -- `8` StructuredJSON: render attached fields as a JSON object. +- `8` StructuredJSON: in `json`, the fields become a `fields` object beside + `message`. - `16` NoTimestamp: suppress the timestamp. - `32` NoLevel: suppress the level. With `flags = 0` the formatter selects `1` for `type = "raw"` and `6` -(timestamp + level) for every other type. `8` is added automatically whenever an -entry carries parseable `fields` and `1` is not set; `1` always wins. +(timestamp + level) for every other type. `8` is added automatically whenever +an entry carries parseable `fields` and `1` is not set (it shapes `json` only); +`1` always wins. `txt` and `json` show timestamp and level unless a suppress bit says otherwise (suppress wins over show), so `2` and `4` alone change nothing. Examples: @@ -115,45 +120,43 @@ Each policy, then what to use it with: - `raw`: no-op passthrough. - `type = "raw"` where you control the data. -- `txt`: escapes non-printable characters. +- `txt`: writes `<` and non-printable characters as `` of their bytes + (`` becomes `<3c>b>`). - File and console sinks. - `json`: escapes control characters for safe JSON embedding. - `type = "json"`. - `shell`: strips shell metacharacters, whitespace, and control characters. - Data that will be passed to a command. -When `sanitizer_policy` is omitted, it is the policy of the same name as -`type`, so the safe pairing is the default. +Every policy but `raw` replaces invalid UTF-8 with U+FFFD. With +`type = "raw"` and `flags = 0` no policy applies (`1` bypasses it). When +`sanitizer_policy` is omitted, it is the policy of the same name as `type`, so +the safe pairing is the default. > `shell` strips dangerous characters but is **not** sufficient to make a string > safe for shell construction. Pass arguments through `exec` argv instead of > building command lines. To see a policy working, point a pipeline at the `random` source with -`special = true`, which injects control bytes and multi-byte Unicode into every -message. +`special = true` and `format` `raw` or `txt`, which mix control and non-ASCII +bytes, often invalid UTF-8, into its messages ([random](sources.md#random)). ## Node Identity in Output -Entries that arrived over a chain link carry a `Node` label. The formatter -renders it as a syslog-style prefix on the source field (`trace` in `json` -output): - -``` -edge-01/app.log -``` - -Entries with no node label show the bare source. Node identity therefore appears -*inside* the source field rather than as a separate output key — worth knowing -when writing downstream parsers or grep patterns. +Entries that arrived over a chain link carry a `Node` label, written as a +syslog-style prefix on the source (`trace` in `json`): `edge-01/app.log`. +Entries without one show the bare source. There is no separate node key, which +downstream parsers and grep patterns need to know. ## Structured Fields When an entry carries `Fields` (raw JSON) and `FlagRaw` is not set, the -formatter parses it and switches to structured rendering by adding the -`StructuredJSON` flag automatically. Fields reach a pipeline in two ways: from -the `file` source when a tailed line parses as JSON with a `fields` key, and -from the heartbeat generator when `include_stats = true`. +formatter parses it and adds the `StructuredJSON` flag. Fields reach a pipeline +from the `file` source when a tailed line parses as JSON with a `fields` key, +from the `tcp_chain` and `http_chain` sources when the upstream entry carried +them, and from the heartbeat when `include_stats = true`. In `json` output +such an entry is `{"time":…,"level":…,"trace":…,"message":…,"fields":{…}}`; +without fields the message sits in `"fields":[…]`. ## Choosing a Configuration @@ -165,22 +168,19 @@ By goal: - Downstream ingestion (Loki, Elasticsearch, jq): `type = "json"`, `sanitizer_policy = "json"`. - Compact console output: `type = "txt"`, `flags = 16`. -- Untrusted log content: never `raw`; pick `txt` or `json` and set the - matching policy. +- Untrusted log content: never `raw`; `txt` or `json`, whose policies are the + default. ## Formatting and Chain Links -Chain sinks (`tcp_chain`, `http_chain`) do **not** ship the formatted payload. -They re-serialize the structured entry into the canonical chain encoding, which -makes them independent of the local formatter. +Chain sinks (`tcp_chain`, `http_chain`) do **not** ship the formatted payload: +they send the structured entry in the chain encoding. `flow.format` on an edge +node therefore changes only that node's own local sinks; the shape a human or +a downstream system sees is decided on the node that owns the sink they read. -The practical consequence: setting `flow.format` on an edge node changes only -that node's own local sinks. The output shape seen by a human or a downstream -system is decided on the node that owns the sink they read. - -If an event ever reaches a chain sink without a structured entry, the sink wraps -the formatted payload into a synthetic entry and counts it in `synthesized`. -A non-zero `synthesized` count means something upstream lost structure. +An event that reaches a chain sink without a structured entry is wrapped from +its formatted payload and counted in `synthesized`; a non-zero count means +something upstream lost structure. ## Performance @@ -188,7 +188,5 @@ Relative cost, cheapest first: `raw` (passthrough) → `txt` (line assembly) → `json` (serialization). Sanitization adds a scan of the message; the `raw` policy skips it. -The formatter holds a mutex because the underlying implementation reuses an -internal buffer and is not goroutine-safe. It is the only shared serialization -point in the hot path, and the reason a single pipeline formats entries one at -a time. +The formatter reuses an internal buffer under a mutex, so a pipeline formats +its entries one at a time. diff --git a/doc/installation.md b/doc/installation.md index 19ce192..18d9e18 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -6,8 +6,9 @@ - Architecture: amd64; arm64 builds but is untested - To build: Go 1.27.2 or newer (FreeBSD: the `go127` package), and GNU make or BSD make -- For `make e2e`: bash 5+, coreutils, curl, openssl and go (the proxy test - builds its reverse proxy). Optional, else those checks SKIP: node with +- For `make e2e`: bash 5+, coreutils, curl, openssl, go (the proxy and acl + tests build helpers) and nc (acl). Optional, else those checks SKIP: + script(1) and setsid(1) (`preset-test.sh`'s terminal checks), node with playwright and the Chromium builds `playwright install chromium` fetches (browser checks), an IPv6 loopback (`ipv6-test.sh`), docker as root with the `logwisp:dev` image (`deploy-test.sh`'s container run) @@ -31,7 +32,8 @@ The Makefile works with GNU make and BSD make alike. Targets: `lwconf.js` that loads and calls it, `schema.json` (`lw --schema`) and `lwconf.json`, their manifest: version, commit, Go version and each file's sha256. Stamped with the commit's time, a commit's files are the - same on every build + same on every build. Its calls: + [Architecture](architecture.md#configuration-engine) - `make version`: the metadata a build would embed - `make completion`: regenerate the bash, zsh and fish completion in `deploy/package/completion/` from `cmd/lw`'s `shorts` and `commands` @@ -60,6 +62,8 @@ Variables: - `GO_BUILDFLAGS`, `GO_LDFLAGS`: extra `go build` and linker flags for packagers. They are distinct from `GOFLAGS` and `LDFLAGS`, which the go command and the C linker read themselves. +- `VERSION`, `GIT_COMMIT`, `BUILD_TIME`: what a build embeds, by default from + git and the clock; a package sets them. - `SOURCE_DATE_EPOCH`: when set, the embedded build time comes from it, so package builds are reproducible. @@ -73,8 +77,9 @@ checkout. `make install` copies what a package ships and compiles nothing, so build first. It honours `DESTDIR`, `PREFIX`, `SYSCONFDIR`, `BINDIR` (default -`$PREFIX/bin`, the path the systemd unit and the rc.d script run `lw` from) -and `SERVICE`: +`$PREFIX/bin`, the path the systemd unit and the rc.d script run `lw` from), +`MANDIR`, `DOCDIR` and `LICENSEDIR` (under `$PREFIX/share` by default) and +`SERVICE`: - Linux defaults: `PREFIX=/usr`, `SYSCONFDIR=/etc`. Outside a package manager, prefer `PREFIX=/usr/local`; systemd reads units, sysusers and @@ -83,7 +88,7 @@ and `SERVICE`: - `INSTALL_OS` (default: `uname -s`) picks the layout, so `INSTALL_OS=FreeBSD DESTDIR=...` stages the FreeBSD one on another system. - `SERVICE`: `yes` (default) adds the service and its configuration; `no` - installs `lw` alone ([Without a Service](#without-a-service)). + leaves them out ([Without a Service](#without-a-service)). - `BASHCOMPDIR`, `ZSHCOMPDIR`, `FISHCOMPDIR`: where the shell completion goes, by default `$PREFIX/share/bash-completion/completions`, `$PREFIX/share/zsh/site-functions` (`/usr/share/zsh/vendor-completions` @@ -97,16 +102,16 @@ sudo make install PREFIX=/usr/local Every install gets: -- `$BINDIR/lw` and the manual `$PREFIX/share/man/man1/lw.1` +- `$BINDIR/lw` and the manual `$MANDIR/man1/lw.1` - completion for `lw`, its commands, flags, files and preset names: `lw`, `_lw` and `lw.fish` in the three directories above -- `$PREFIX/share/doc/logwisp/` (this documentation) and - `$PREFIX/share/licenses/logwisp/LICENSE` +- `$DOCDIR` (this documentation) and `$LICENSEDIR/LICENSE` A service install also gets the service files below and, on Linux and FreeBSD alike, `$SYSCONFDIR/logwisp/logwisp.toml`: the annotated sample -configuration, placed when none is there and otherwise kept. Edit it before -starting the service. +configuration, placed when none is there and otherwise kept. As shipped it +defines no pipeline, so the service would copy an empty stdin and exit: edit it +before starting the service. ### Linux (systemd) @@ -139,7 +144,7 @@ Create a credentials file with its owner and mode first; `lw auth` keeps both: ```bash -sudo install -m 0600 -o logwisp -g logwisp /dev/null /etc/logwisp/users.toml +sudo install -m 0640 -o root -g logwisp /dev/null /etc/logwisp/users.toml sudo lw auth add-user --credentials /etc/logwisp/users.toml --user \ --password-file sudo systemctl reload logwisp @@ -256,10 +261,11 @@ For every user: `sudo make install SERVICE=no PREFIX=/usr/local`. existing file, where you choose: - `lw config init`: `~/.config/logwisp/logwisp.toml`, which `lw` reads - without `-c` unless a `LOGWISP_CONFIG_*` variable names another file. With - it in place, `lw` without pipeline options runs its - pipelines instead of copying stdin to stdout; pipeline options and presets - still replace them. + without `-c` unless a `LOGWISP_CONFIG_*` variable names another file. As + shipped it defines no pipeline, so `lw` still copies stdin to stdout, though + its own log on stderr is at the file's default `info` rather than `warn`; + once it defines `[[pipelines]]`, `lw` without pipeline options runs them, + and pipeline options and presets still replace them. - `sudo lw config init --out /usr/local/etc/logwisp/logwisp.toml`: a file for every user, named with `-c`, or without it by `LOGWISP_CONFIG_DIR=/usr/local/etc/logwisp` @@ -324,11 +330,11 @@ docker run -d --name logwisp \ -e LOGWISP_SINK=console ``` - Without a file lw logs warnings only and runs no status reporter; add - `-e LOGWISP_LOGGING_LEVEL=info` (and `-e LOGWISP_STATUS_REPORTER=true`) for - the service defaults. The image cannot set them: a variable would override - a mounted file's value. Without `-i`, stdin is `/dev/null`, so a container - needs a source other than `console`. + Without a file lw logs less ([CLI](cli.md#built-in-defaults)): add + `-e LOGWISP_LOGGING_LEVEL=info -e LOGWISP_STATUS_REPORTER=true`; the image + sets neither, as a variable would override a mounted file's value. Without + `-i`, stdin is `/dev/null`, so a container needs a source other than + `console`. - Secrets: only as mounted files, at `/run/secrets` (Docker or Compose secrets, a Kubernetes secret volume), named by `tls.key_file`, @@ -413,36 +419,41 @@ or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target. ``` 15801-15804 chain-test.sh, chain-aggregate-test.sh test/run/ (no ports) passthrough-test.sh test/run/passthrough/ +(no ports) pipe-test.sh test/run/pipe/ 15811-15814 mtls-chain-test.sh test/run-mtls/ 15821-15825 scram-chain-test.sh test/run-scram/ -15831-15832 scram-proxy-test.sh test/run-proxy/ +15831-15833 scram-proxy-test.sh test/run-proxy/ 15851-15855 ipv6-test.sh test/run-ipv6/ 15871-15879 deploy-test.sh test/run-deploy/ 15881-15889 acl-test.sh test/run/acl/ +15890-15898 preset-test.sh test/run/preset/ ``` -- `chain-test.sh`: two edges into a relay, one pipeline per chain transport; - ports 15801-15804 +- `chain-test.sh`: two edges into a relay, one pipeline per chain transport - `chain-aggregate-test.sh`: the same edges, with the relay fanning both into - both sinks; ports 15801-15804 + both sinks - `passthrough-test.sh`: `file` source to `file` sink, a wide envelope byte - for byte (`raw = true`) and, parsed, with no key dropped; no ports -- `mtls-chain-test.sh`: mTLS chain edges and viewers, allow lists, node - binding; ports 15811-15814 + for byte (`raw = true`) and, parsed, with no key dropped +- `pipe-test.sh`: lw without a configuration as a filter, stdin to stdout + line for line without loss, quiet on stderr, exit 0 at the end of input +- `mtls-chain-test.sh`: mTLS chain edges and viewers, allow lists, node binding - `scram-chain-test.sh`: SCRAM chain edges and viewers with `lw auth` - credentials; ports 15821-15825 + credentials - `scram-proxy-test.sh`: browser and CLI logins to an `http` sink behind a - TLS-terminating proxy; ports 15831-15832; the browser checks skip without - node and playwright's Chromium: the headless shell for the scenarios with - cookies, full Chromium for the profile that blocks them + TLS-terminating proxy; the browser checks skip without node and + playwright's Chromium: the headless shell for the scenarios with cookies, + full Chromium for the profile that blocks them - `ipv6-test.sh`: a relay on `::1` over both chain transports, IPv6 SANs, - per-family listeners; ports 15851-15855; skips (exit 77) without an IPv6 - loopback + per-family listeners; skips (exit 77) without an IPv6 loopback - `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed - and checked; ports 15871-15879 + and checked - `acl-test.sh`: each listener admitting or refusing peers by address before TLS, with clients from 127.0.0.1 and 127.0.0.2, then by the client a PROXY header names behind a stub proxy on 127.0.0.11-13 (and nginx `stream` on 127.0.0.14 when installed) or an L7 proxy's `X-Forwarded-For`, with - per-client connection caps, request rates and login throttling; ports - 15881-15889; skips (exit 77) where 127.0.0.2 cannot be bound + per-client connection caps, request rates and login throttling; skips + (exit 77) where 127.0.0.2 cannot be bound +- `preset-test.sh`: colors, presets, `--dump`, `lw tls`, listeners that make + their certificate, the serve preset's viewer, network sinks delivering a + finite input whole, a SCRAM pipe whose password is on no disk; the + terminal checks skip without script(1) or setsid(1) diff --git a/doc/lw.1 b/doc/lw.1 index c17231a..bd53796 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -1,4 +1,4 @@ -.Dd October 10, 2026 +.Dd October 11, 2026 .Dt LW 1 .Os .Sh NAME @@ -68,12 +68,10 @@ source runs until shutdown. When every pipeline has finished, .Nm shuts down gracefully and exits 0. -Every sink first writes what it holds; the -.Cm http -and -.Cm tcp -sinks give each client -.Cm write_timeout_ms , +Every sink first writes what it holds; a network sink gives its clients or +its downstream +.Cm write_timeout_ms +.Po Cm request_timeout_ms No for Cm http_chain Pc , at most 2 seconds, to take it. Under a supervisor, standard input is usually .Pa /dev/null : @@ -116,8 +114,12 @@ A value follows as the next argument or after .Ql = ; a value that starts with .Ql - , -and a switch's, follows -.Ql = . +and a boolean key's, follows +.Ql = ; +.Fl t , Fl \-dump , Fl T , Fl \-schema +and +.Fl V +take none. Short options do not combine. .Nm takes options only: an unknown option, or a word no option takes, is a usage @@ -347,9 +349,12 @@ make a pipeline named after its preset, or .Dq cli without one. .It Fl p Ar spec , Fl \-preset Ar spec -Start the pipeline with a preset, described in +Begin the pipeline with a preset, as its first option, described in .Sx Presets ; -later options add sources, sinks and filters to it, or replace its format. +later options add sources, sinks and filters to it, set its rate limit and +heartbeat, or replace its format. +A second preset needs its own +.Fl \-pipeline . .It Fl \-source Ar spec , Fl \-sink Ar spec , Fl \-filter Ar spec Add a source, sink or filter; each is repeatable. .It Fl \-format Ar spec , Fl \-rate\-limit Ar spec , Fl \-heartbeat Ar spec @@ -410,9 +415,21 @@ specification, and is .Fl \-user . A -.Ar path -is a file, a directory, whose files are followed, or a glob; without one, a -preset that takes it reads standard input. +.Cm path +is a file, a directory, whose files are followed, or a glob, and this host +decides which: an existing directory is followed whole, any other path is a +file name or glob in its directory; for a directory not here, as in a +.Fl \-dump +for another host, write +.Ql dir/* . +Without one, +.Cm serve +and +.Cm edge +read standard input. +.Cm from +.Pq Cm end No or Cm start +is where reading a file starts. .Cm user names one SCRAM user, an edge's login or a listener's only user in place of .Cm users ; @@ -431,7 +448,7 @@ runs without a configuration. .It Cm tail path Op Cm from , format Follow files to standard output, like .Ql tail \-F . -.It Cm serve Oo Cm path , listen , format , tls , users , user , password_file , proxy , viewer , allow , deny Oc +.It Cm serve Oo Cm path , from , listen , format , tls , cert , key , issuer_cert , issuer_key , hosts , users , user , password_file , proxy , viewer , allow , deny Oc Serve a live HTTP stream on .Cm listen .Pq default Li 127.0.0.1:8080 , @@ -446,6 +463,8 @@ is or .Cm files .Pq Cm cert No and Cm key ; +.Cm hosts +adds names and addresses to a certificate made at startup; .Cm users names a credentials file, or .Cm user @@ -463,26 +482,38 @@ since a browser cannot bind its login to the TLS channel. and .Cm deny list the addresses or CIDRs the listener admits and refuses. -.It Cm edge to Oo Cm path , transport , ca | pin , user , password_file , cert , key , node Oc +.It Cm edge to Oo Cm path , from , transport , ca | pin , server_name , user , password_file , cert , key , node Oc Forward to an aggregator at .Cm to over TLS, by .Cm tcp_chain or, with .Cm transport=http , -.Cm http_chain . +.Cm http_chain , +verifying it by +.Cm ca , +by default the system roots, or +.Cm pin ; +.Cm server_name +is the name its certificate carries, by default the host of +.Cm to . It refuses to run without credentials: .Cm user , or a client .Cm cert and .Cm key . -.It Cm aggregator Oo Cm listen , transport , tls , users , user , password_file , client_ca , out , format , allow , deny Oc +.It Cm aggregator Oo Cm listen , transport , tls , cert , key , issuer_cert , issuer_key , hosts , users , user , password_file , client_ca , out , format , allow , deny Oc Receive from edges on .Cm listen .Pq default Li 0.0.0.0:9000 over TLS, never off -.Pq default Cm tls=self , +.Po default Cm tls=self ; +.Cm cert , key , issuer_cert , issuer_key +and +.Cm hosts +as for +.Cm serve Pc , authenticated by the .Cm users credentials file, the one @@ -523,6 +554,11 @@ lists the DNS names and addresses a server certificate carries is also the subject, the identity .Cm auth.type=mtls checks. +One of +.Fl \-server +and +.Fl \-client +is required; both may be given. .El .Pp A listener needs no files: @@ -646,7 +682,7 @@ The directory holding .It Ev LOGWISP_PIPELINE The name of the one pipeline the variables below define. .It Ev LOGWISP_PRESET -The preset that starts it. +The preset that begins it. All of these are ignored when a pipeline option is on the command line. .It Ev LOGWISP_SOURCE , LOGWISP_SINK , LOGWISP_FILTER Source, sink and filter specifications, in the syntax of the matching @@ -820,7 +856,7 @@ $ lw \-\-preset tail:path=/var/log/app .Pp Print the configuration a set of options makes, to keep as a file: .Bd -literal -offset indent -$ lw \-\-preset serve:path=/var/log/app,tls=self \-\-dump > serve.toml +$ lw \-\-preset 'serve:path=/var/log/app/*,tls=self' \-\-dump > serve.toml .Ed .Pp Ship a directory from an edge to an aggregator: on the aggregator, create @@ -873,7 +909,7 @@ letting .Nm generate the password into a file, then reload the server: .Bd -literal -offset indent -# install \-m 0600 \-o logwisp \-g logwisp \e +# install \-m 0640 \-o root \-g logwisp \e /dev/null /etc/logwisp/users.toml # lw auth add-user \-\-credentials /etc/logwisp/users.toml \e \-\-user alice \-\-password-file /root/alice.pass @@ -894,6 +930,8 @@ under the installation prefix and kept at A listener without an .Cm auth block serves or accepts logs from anyone who can reach it. -Keep key, credentials and password files at mode 0600, owned by the account +Keep key, credentials and password files unreadable by others: mode 0600, +owned by the account .Nm -runs as; startup warns about world-readable ones. +runs as, or 0640, owned by root and a group that account is in; startup +warns about world-readable ones. diff --git a/doc/mtls-auth-plan.md b/doc/mtls-auth-plan.md deleted file mode 100644 index 69eda35..0000000 --- a/doc/mtls-auth-plan.md +++ /dev/null @@ -1,335 +0,0 @@ -# mTLS as Authentication - -**Status:** implemented. Phases 1–3 of the original proposal, plus dialer-side -server identity pinning from phase 4, are in the tree and covered by -`test/mtls-chain-test.sh`. The remaining phase-4 items are listed under -[Not Implemented](#not-implemented). - -**Scope:** turn transport-level mutual TLS into an authentication and -authorization mechanism. - -## Problem - -LogWisp already did mutual TLS at the transport layer. A listener with -`client_auth = true` refused any peer that could not present a certificate -chaining to `client_ca_file`, and `internal/tlsx` extracted the peer's Common -Name into session metadata as `tls_peer_cn`. - -Nothing read it back. The result was a CA-wide membership check with no notion -of *which* peer connected: - -1. **No per-identity authorization.** Every certificate the CA issued was - equivalent. There was no way to say "only `edge-01` and `edge-02` may write - to this ingest port", so one CA could not serve several trust domains, and - withdrawing one peer meant rotating the CA bundle for all of them. -2. **Node labels were unauthenticated.** With `trust_node = true` (the default) - a peer declares its own `node` label in the chain hello or the - `X-Logwisp-Node` header. Any certificate holder could claim any label, - including another host's, and every downstream consumer would attribute - those entries accordingly. The only defence, `trust_node = false`, replaced - the label with a remote address — unforgeable, but useless for identifying a - host behind NAT or a load balancer. -3. **The `http` sink had no authentication at all**, even with TLS on. Its - stream and status endpoints were readable by anyone who could reach the port. - -Password, token, and SCRAM authentication were removed during the plugin/flow -restructure and the move to standard-library networking. Certificates are the -one credential the transport already carries, which made mTLS the cheapest path -back to authenticated peers. - -## Goals - -- Authorize peers by certificate identity, per listener. ✅ -- Bind the chain `node` label to the authenticated identity, so origin - attribution is trustworthy. ✅ -- Gate the `http` sink's endpoints on client certificates. ✅ -- Make identity visible in sessions, statistics, and logs. ✅ -- Change nothing for existing configurations that omit the new block. ✅ - -## Non-Goals - -- Password authentication. It is implemented separately as `type = "scram"` - ([scram-auth-plan.md](scram-auth-plan.md)) on the same `auth` block and - `internal/authz` seam, carried by `chain.Hello` on TCP and a bearer token on - HTTP. -- IP allow/deny lists and per-peer rate limits. Related, but a separate feature - with its own config surface. -- OCSP. See [Revocation](#revocation) for what is done instead. -- Authorization *within* a stream — the unit of decision is a connection (TCP) - or a request (HTTP), never an individual entry. - -## Design - -### Identity - -The authenticated identity is a single string derived from the peer's verified -leaf certificate. Because `tls.RequireAndVerifyClientCert` has already validated -the chain, signature, and validity window by the time we look, extraction is -pure field selection — `tlsx.PeerIdentity`. The `identity` modes: - -- `cn` (default): `Subject.CommonName`, matching the existing `tls_peer_cn` - metadata. -- `san_dns`: the first `DNSNames` entry; preferred for host identities. -- `san_uri`: the first `URIs` entry; SPIFFE-style IDs. -- `san_email`: the first `EmailAddresses` entry; operator identities. - -An empty identity is a rejection, not an empty match: a certificate with no -usable identity field cannot satisfy any policy. - -Identities are not secrets, so ordinary string comparison is used; there is no -timing side channel worth defending here. - -### Configuration - -An `auth` table sits beside `tls` in every network plugin's `config`. Keeping it -separate from `tls` matters: TLS answers "is this channel private and does the -peer chain to a CA", auth answers "may *this* peer do *this*", and a later -non-certificate method can reuse the block. - -```toml -[pipelines.plugin_sources.config.auth] -type = "mtls" # none (default) | mtls -identity = "cn" # cn | san_dns | san_uri | san_email -allow = ["edge-01", "edge-02"] # exact identities -allow_patterns = ["^edge-\\d{2}$"] # RE2, anchored by the author -node_binding = "force" # none | assert | force -``` - -- `type`, string, default `none`: `none` preserves pre-auth behaviour - exactly; `mtls` enables the policy. -- `identity`, string, default `cn`: which certificate field is the identity. -- `allow`, []string, default `[]`: exact identity matches. -- `allow_patterns`, []string, default `[]`: RE2 patterns matched against the - identity. -- `node_binding`, string, default `force` when `type = "mtls"`: chain sources - only; see below. - -Empty `allow` **and** empty `allow_patterns` under `type = "mtls"` means "any -identity the CA vouches for" — that is, the pre-auth behaviour, but with the -identity now recorded and node binding available. It is a deliberate, documented -default rather than a silent deny-all, and the plugin logs a WARN at startup -saying so. - -`node_binding` applies only to the chain sources, where a `node` label is -declared. Setting it to anything but `none` on any other plugin is a -configuration error. Its values: - -- `none`: `trust_node` governs the connection label, as before, and the - per-entry `node` field. -- `assert`: the connection label must equal the identity, and a mismatch or an - omission is rejected; `trust_node` governs the per-entry `node` field. -- `force`: the declared label is ignored and the identity is used; every - per-entry `node` field is overwritten with the identity. - -The split between `assert` and `force` is what makes both worth having: - -- **`force`** is for an ingest boundary that does not trust its peer. Every - entry is relabeled, so a compromised edge cannot smuggle a foreign origin - through the per-entry `node` field either. It is the default under - `type = "mtls"` because it is the only setting where a misconfigured or - hostile edge cannot mislabel its entries. -- **`assert`** is for a relay-to-relay hop. The relay must prove *its own* - identity — a mismatch is loud rather than silently corrected — but the entries - it forwards keep the origin labels stamped at the first hop, so multi-hop - attribution survives. - -`node_binding` overrides `trust_node` for the connection label. Under `force` -the constructor logs that `trust_node` is ignored; under `assert`, that it still -governs the per-entry labels. - -Dialer-side plugins (`tcp_chain` and `http_chain` sinks) accept the same block -to pin the *server's* identity beyond hostname verification. There -`node_binding` does not apply, and `tls.insecure_skip_verify` is rejected: -identity read from an unverified chain is a claim, not a fact. - -### Validation - -At plugin construction, before anything binds: - -- `type = "mtls"` on a listener requires `tls.enabled = true` and - `tls.client_auth = true`; on a dialer it requires `tls.enabled = true` and - forbids `tls.insecure_skip_verify`. Silently accepting an auth policy the - transport cannot enforce is the failure mode worth designing out. -- `identity` must be one of the four modes. -- Every entry in `allow_patterns` must compile. -- `node_binding` must be one of the three values, and must be absent or `none` - outside the chain sources. - -Errors follow existing style: `auth: type "mtls" requires tls.client_auth`. - -### The `internal/authz` package - -One small package, mirroring `internal/tlsx`, is the single seam between the -`auth` block and the plugins. `New(o, tlsCfg, role, transport)` compiles a policy -against the sibling TLS config and returns `(nil, nil)` when auth is disabled; -every method tolerates a nil receiver, so call sites never branch on config. -Listeners admit only through `Admit` (TCP) and `AuthorizeRequest` (HTTP); the -current API, including SCRAM, is in -[scram-auth-plan.md](scram-auth-plan.md#the-authz-seam). - -### Enforcement Points - -**`tcp_chain` source** (`internal/source/tcpchain/tcpchain.go`, `handleConn`) - -Handshake → `Admit` (authorize, then read hello) → `ResolveNode` → create -session. Under `mtls` the certificate is checked before the hello is read, so -an unauthorized peer never gets a preamble parsed on its behalf. -`chain.DecodeEntry` is then called with `auth.TrustsEntryNode(trust_node)` -rather than `trust_node` itself. - -**`http_chain` source** (`internal/source/httpchain/httpchain.go`, `handleIngest`) - -Per request, through `AuthorizeRequest`, before the body is read — an -unauthorized sender does not get to stream `max_body_bytes` into the process. -Rejection is `403`, distinct from the `400` used for protocol errors, so a -sender can tell "you are not allowed" from "your batch was malformed". -`ResolveNode` then governs the `X-Logwisp-Node` header exactly as it governs the -TCP hello. The session cache key includes the identity, so two peers sharing a -remote address never share a session. - -**`tcp` sink** (`internal/sink/tcp/tcp.go`, `handleConn`) - -After the explicit handshake, before the session is created and the client is -registered — so an unauthorized peer never appears in the client map and never -receives a broadcast. - -**`http` sink** (`internal/sink/http/http.go`, `authMiddleware`) - -A middleware around the mux covers both the stream and the status endpoint with -one wrapper and keeps the handlers themselves unaware of authorization. The -authorized identity is passed down through the request context for session -metadata. Rejections are `403` with no body detail — the status endpoint leaks -host, port, and throughput counters, so a rejection should not leak policy shape -on top of that. - -**`tcp_chain` / `http_chain` sinks** (dialers) - -The policy is installed as `tls.Config.VerifyConnection`, which runs after the -standard chain and hostname checks. A server whose identity the policy rejects -fails the handshake itself rather than the first write, and the chain sink's -existing backoff loop handles it as any other connect failure. - -### Capabilities - -`core.CapAuth` now means "this plugin authorizes peers" — it is derived from the -policy, not from `tlsConfig.ClientAuth`. Transport-level mTLS without a policy -still reports `CapTLS`, the `mtls=true` field on the startup log line, and the -`tls` statistic. - -`Pipeline.initSourceCapabilities` and `initSinkCapabilities` treat this as a -cross-cutting check: a plugin advertising `CapAuth` without `CapTLS` is a -contradiction and fails pipeline construction rather than starting. An `http` -sink in SCRAM proxy mode also reports `CapProxyTLS`, which satisfies the check -without `tls`: TLS ends at a trusted proxy. - -### Observability - -Every authorization decision is visible, because a silent deny is -indistinguishable from a network fault at 3am. - -- **Session metadata** gains `auth_method` and `auth_identity` alongside the - existing `tls` and `tls_peer_cn`. -- **Statistics** gain `auth`, `auth_identity` (the mode), `auth_unrestricted`, - `auth_allowed`, `auth_rejected`, and — on chain sources — `node_binding`, in - the `details` map of every affected source and sink. They surface in the - status reporter and in the `http` sink's status endpoint. -- **Logs** record a WARN per rejection with the remote address and the reason. - The startup line carries a rendered policy summary - (`auth="mtls identity=cn allow=[1 exact, 0 pattern(s)] node_binding=force"`), - a WARN when the allow list is empty, and an INFO when node binding overrides - `trust_node`. - -### Revocation - -Certificate revocation is handled by the allow-list rather than by CRL or OCSP: - -1. Remove the identity from `allow` / `allow_patterns`. -2. `kill -HUP`. - -The reload path rebuilds every pipeline, so the policy takes effect on the next -connection and existing connections are dropped by the rebuild itself. This is -one moving part instead of three, it needs no network calls on the handshake -path, and it is exact — no window between revocation and the next CRL -publication. - -## Compatibility - -No configuration breaks. Omitting the `auth` block, or setting `type = "none"`, -reproduces the previous behaviour exactly: `Policy` is nil, every call site -short-circuits, and `trust_node` continues to govern node labels. - -The one behavioural note for adopters: turning on `type = "mtls"` defaults -`node_binding` to `force`, so entries from a peer whose certificate identity -differs from its configured `node` label will be relabelled. That is the point -of the feature, but it moves data between labels in a dashboard, so plan for it. -Use `node_binding = "assert"` on relay-to-relay hops where upstream origin -labels must survive. - -## Verification - -`test/mtls-chain-test.sh` builds a full PKI with `openssl` and exercises both -target topologies end to end: - -``` -./test/mtls-chain-test.sh --auto -``` - -Scenario 1 — chained instances, client authenticating with mTLS: - -- an authorized edge (`edge-01`) delivers entries through both the `tcp_chain` - and `http_chain` ingest ports into a file sink -- `node_binding = "force"` overrides the label the sender configured -- an identity outside the allow list (`edge-99`) is refused, even while claiming - to be `edge-01` -- a peer presenting no certificate fails the handshake -- a dialer that pins a server identity the relay does not hold refuses to - connect, even though the server certificate chains to the trusted CA - -Scenario 2 — a viewer client reading a streaming sink over mTLS: - -- an authorized viewer streams from the `tcp` sink and from the `http` sink's - SSE endpoint, and reads `/status` -- a CA-valid but unauthorized viewer gets nothing from the `tcp` sink and `403` - from both `http` sink endpoints -- a client with no certificate fails the handshake -- the status endpoint reports the policy and its rejection count - -`test/chain-test.sh` and `test/chain-aggregate-test.sh` continue to pass -unchanged, which is the regression check for the auth-disabled path. - -## Not Implemented - -The remaining phase-4 items, in rough order of value: - -1. **CRL file support** alongside `client_ca_file`, re-read on reload, for - operators with existing CRL infrastructure. The allow-list covers the same - ground with fewer moving parts, so this is only worth doing for a fleet whose - revocation already flows through a CRL. -2. **Per-identity rate limits.** The natural follow-on now that identity exists: - the `acl` block's per-client limits count addresses - ([Security](security.md#per-client-limits)), and would be keyed by identity - once a peer is authenticated. -3. **Per-client identity in the `http` sink's status output.** The endpoint - reports the policy and counters, but not which identities are currently - connected; session metadata has the data. -4. **A list of `identity` modes** (try `san_uri`, fall back to `cn`) for - heterogeneous PKI. A single mode is simpler and covers a uniform CA. - -## Decisions Taken - -Four questions were left open by the proposal. What was chosen, and why: - -1. **An empty allow-list allows rather than denies.** Deny-by-default is the - safer instinct, but `type = "mtls"` with no list is a legitimate - configuration — "any peer this CA issued, but bind the node labels" — and - node binding alone is worth enabling without enumerating every node. - Erroring on it would force operators to list their whole fleet to get - trustworthy attribution. The compromise is a WARN at startup naming the - condition and the fix. -2. **`identity` takes a single mode.** Simpler, and a uniform CA is the common - case. Listed above as a possible extension. -3. **Per-identity rate limits are out of scope**, as proposed. -4. **`assert` rejects rather than warns and corrects.** A certificate/config - mismatch under `assert` is an outage, which is the point: `force` is the - forgiving option and it is the default, so an operator reaches for `assert` - precisely when they want the mismatch to be loud. diff --git a/doc/networking.md b/doc/networking.md index 292b3f7..3ff73d4 100644 --- a/doc/networking.md +++ b/doc/networking.md @@ -57,11 +57,11 @@ Dialers: - `http_chain` sink: chain v1; forwards entries as batched POSTs. There is no port registry and no default port: `port` is required on every -network plugin. There is also no cross-pipeline conflict detection — two sinks -on the same port fail at bind time when the pipeline starts: +network plugin. Two sinks on the same port pass `lw --check` and fail at bind +when the service starts; lw exits 1: ``` -ERROR msg="Failed to start sink" error="tcp sink bind 0.0.0.0:9090: listen tcp4 0.0.0.0:9090: bind: address already in use" +ERROR msg "Failed to initialize service" error "failed to start service pipelines: failed to start pipeline : failed to start sink : tcp sink bind :: listen tcp4 :: bind: address already in use" ``` ## Timeouts @@ -91,7 +91,8 @@ Fixed, non-configurable bounds: - TLS handshake: 10 s, on all TLS listeners and dialers. - HTTP read-header timeout: 10 s, on the `http` sink and `http_chain` source. -- HTTP server shutdown grace: 2 s, on the `http` sink and `http_chain` source. +- HTTP server shutdown grace: 2 s on the `http_chain` source and the `http` + sink; on the `http` sink, `write_timeout_ms` instead when set below 2 s. - Single entry line: at most 1 MiB, on chain listeners. - SCRAM login: 10 s, on the `tcp` sink, chain sinks, `lw auth`, and each `/auth` request. @@ -103,15 +104,16 @@ would terminate long-lived SSE streams; per-event deadlines come from ## Connection Limits -`max_connections` caps concurrent connections on the `tcp` sink, the `http` -sink, and the `tcp_chain` source. `0` means unlimited. +`max_connections` caps concurrent connections on the `tcp` sink and the +`tcp_chain` source, and concurrent streams on the `http` sink (other requests +are not counted); `0` means unlimited. -- Admission is a load-then-check, so a burst can over-admit by roughly one - connection. This is accepted, not a bug to work around. +- Admission is a load-then-check, so a burst can over-admit by roughly one. - On the `tcp` sink and `tcp_chain` source the count is taken at accept, so it bounds concurrent TLS handshakes as well as established sessions. -- Over-limit connections are closed immediately and counted in `rejected_conns` - (TCP) or `rejected_clients` (HTTP, which first answers `503`). +- Over the limit a TCP connection is closed and counted in `rejected_conns`; + an `http` stream request is answered `503` and counted in + `rejected_clients`, its connection kept. The `http_chain` source has no connection cap; it bounds work with `max_body_bytes` and `read_timeout_ms` instead. @@ -123,39 +125,24 @@ pipeline-wide entry rate limit: it cannot distinguish or throttle a peer. ## Keep-Alive -TCP keep-alive is available on the `tcp` sink (for accepted connections) and the -`tcp_chain` sink (for its outbound connection): +The `tcp` sink (accepted connections) and the `tcp_chain` sink (its outbound +connection) configure TCP keep-alive: ```toml -keep_alive = true +keep_alive = true # false sends no probes keep_alive_period_ms = 30000 ``` -This is kernel-level keep-alive; it detects a dead peer but does not keep an -application-level stream flowing. For that, use a heartbeat. +Probes start after `keep_alive_period_ms` idle. Other listeners and the +`http_chain` sink use Go's default TCP keep-alive (15 s); it is not +configurable there. Keep-alive detects a dead peer but does not keep an +application-level stream flowing: that is a heartbeat's job. ## Heartbeats -Heartbeats are a **flow-level** feature, not a per-sink one. Enabling one -injects a synthetic entry into the pipeline at a fixed interval; it reaches -every sink and traverses chain links as an ordinary structured entry. - -```toml -[pipelines.flow.heartbeat] -enabled = true -interval_ms = 30000 -include_timestamp = true -include_stats = false -format = "txt" # txt | json | raw -``` - -Use it to keep idle SSE clients, TCP clients, and chain links from being -reaped by intermediate NAT or proxy timeouts, and to make an idle pipeline -visibly alive. - -> `format = "comment"` (SSE `:` comment framing) is rejected by validation -> despite appearing in older documentation and in a still-present code branch. -> A pipeline configured with it fails to start. +A flow heartbeat keeps idle TCP clients and chain links alive through NAT and +proxy timeouts; idle SSE streams already get a comment every 15 s. See +[Configuration](configuration.md#heartbeat). ## Reconnection @@ -169,35 +156,34 @@ backoff_max_ms = 30000 ``` The connection is established lazily, so an edge node starts cleanly even when -its relay is down and connects as soon as the relay appears. Reconnect counts -are reported in the sink's `reconnects` statistic. +its relay is down and connects as soon as the relay appears. The `tcp_chain` +sink reports `connected` and `reconnects`; the `http_chain` sink reports +`request_errors` and `dropped_batches`. Server-side sinks (`tcp`, `http`) do not reconnect; clients are expected to retry. Browsers reconnect SSE streams automatically. ## Protocol Details -**HTTP sink (SSE)** — HTTP/1.1 in plaintext; HTTP/2 is negotiated via ALPN when -TLS is enabled. Only `GET` is routed to the stream and status paths. Each event -is framed as one `data:` line per newline in the payload, so multi-line entries -stream intact. Response headers set `Cache-Control: no-cache`, -`X-Accel-Buffering: no`, and, without an auth policy, -`Access-Control-Allow-Origin: *`. Under `scram`, `POST /auth` serves the login -and both paths take `Authorization: Bearer `. +**HTTP sink (SSE)**: HTTP/1.1 in plaintext, HTTP/2 via ALPN under TLS; `GET` +on the stream path, `GET` or `HEAD` on the status path. Framing and headers: +[Sinks](sinks.md#http). Under `scram`, `POST /auth` logs in and both paths +take `Authorization: Bearer `. -**TCP sink** — raw payload bytes, no framing added by the sink. Whether entries -are newline-delimited depends on the formatter. Under `scram` the client's hello -and login lines come first, as on a [chain link](chaining.md#tcp-transport). +**TCP sink**: the formatted payloads, each ending in a newline, with no other +framing; under `json` that is one entry per line, while `raw` passes a +message's own line breaks through. Under `scram` the client's hello and login +lines come first, as on a [chain link](chaining.md#tcp-transport). -**Chain transports** — see [Chaining](chaining.md) for the hello preamble, -headers, and entry encoding. +**Chain transports**: see [Chaining](chaining.md#wire-protocol) for the hello +preamble, headers, and entry encoding. ## Troubleshooting -**Plugin fails to start with `unknown key "..."`** -- The key is misspelled or belongs to another plugin type. Every plugin rejects - keys it does not declare, nested tables included, so a typo in `tls` or `auth` - cannot leave a protection silently off. +**`unknown key "..."` when the configuration loads** +- The key is misspelled or belongs to another plugin type. A plugin's keys are + checked at load (`lw --check` too), nested tables included, so a typo in + `tls` or `auth` cannot leave a protection silently off. **Connection refused** - Confirm the pipeline started; a bind failure is logged at ERROR. @@ -252,18 +238,23 @@ headers, and entry encoding. `tls_handshake_errors`. **Rejected after a successful handshake** -- `auth: identity "..." is not allowed` — the certificate is valid and chains to - the CA, but the identity is not in `auth.allow` / `auth.allow_patterns`. On - TCP the connection is closed; on HTTP the answer is `403`. -- `auth: peer certificate carries no identity` — `auth.identity` names a - field the certificate does not populate, e.g. `san_dns` on a CN-only leaf. -- `auth: node_binding "assert": declared node "..." does not match identity` — - the sender's `node` option and its certificate disagree. Fix one, or use +- `auth: refused: identity "" is not allowed` — the certificate is valid + and chains to the CA, but the identity is not in `auth.allow` / + `auth.allow_patterns`. On TCP the connection is closed; on HTTP the answer + is `403`. +- `auth: refused: peer certificate carries no identity` — + `auth.identity` names a field the certificate does not populate, e.g. + `san_dns` on a CN-only leaf. +- `auth: node_binding "assert": declared node "" does not match identity ""` + — the sender's `node` option and its certificate disagree. Fix one, or use `node_binding = "force"` to let the certificate win silently. - On a `tcp_chain` sink the same message inside `Chain connect refused` means the *server* was refused: its certificate identity is not in the sink's `auth.allow`. -- Rejections appear as WARN and increment `auth_rejected`. +- Allow-list and login rejections log WARN and count `auth_rejected`; a + node-binding refusal logs `Connection rejected by node binding` + (`tcp_chain`, `rejected_conns`) or `Request rejected by node binding` + (`http_chain`, `rejected_requests`). **SCRAM login fails** - `authentication failed` — the dialer is not told why, by design. The @@ -334,11 +325,14 @@ headers, and entry encoding. send the header. **Entries not arriving over a chain link** -- Check the sink's `connected` statistic and its `reconnects` count. +- `tcp_chain`: check the sink's `connected` and `reconnects`; `http_chain`: + its `request_errors` and `dropped_batches`. - Check the source's `auth_rejected` — an allow-list miss or a failed login looks exactly like a network fault from the sender's side — and the sink's `last_auth_error`. -- Check the source's `parse_errors` — a version skew shows up here. +- A version skew: `tcp_chain` logs WARN `Rejected chain connection` + (unsupported protocol version), counted only under `scram`, in + `auth_rejected`; `http_chain` answers `400` and counts `rejected_requests`. - On `http_chain`, remember entries wait up to `flush_interval_ms` before a batch is sent. diff --git a/doc/operations.md b/doc/operations.md index 35729f3..a52e342 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -13,15 +13,9 @@ tail -F app.log | lw --filter include:patterns=ERROR lw -c /etc/logwisp/logwisp.toml ``` -lw exits on its own only at the end of input: once every pipeline has -finished, a pipeline finishing when all its sources have ended. Only a console -source ends, when stdin does; file and network sources run until a signal, and -lw shuts down gracefully, exit 0, either way. -Every sink writes what is queued before the exit; a network sink gives its -clients or its downstream its `write_timeout_ms` (`request_timeout_ms` for -`http_chain`), at most 2 s, to take it. A reload stops the old sinks the same -way, so a stalled client or downstream delays it by that much. See -[CLI](cli.md#built-in-defaults). +lw exits on its own only at the end of input ([CLI](cli.md#built-in-defaults)). +A reload stops the old sinks as shutdown does, so a stalled client or +downstream delays it by up to 2 s. There is no built-in daemon mode. Run LogWisp in the foreground under a supervisor — systemd, rc.d, or a container runtime — which is where restart, @@ -63,13 +57,13 @@ kill -HUP $(pidof lw) ``` Signals reread the selected TOML file even when `auto_reload` is disabled. -Reload validates a detached snapshot and constructs a new service **before** -tearing the old one down, so invalid configuration leaves the running service -intact and logs the failure: - -``` -ERROR msg="Failed to bootstrap new service, keeping old service running" error=... -``` +Reload validates a detached snapshot and builds a new service **before** +tearing the old one down, so a bad configuration leaves the running service +intact. An invalid one logs `Failed to read reload configuration` with the +validator's `validate configuration: : ...` error. One whose files +fail when its plugins are built (a missing certificate, an unreadable +credentials file) logs `Failed to bootstrap new service, keeping old service +running`. What reload does *not* do: @@ -84,32 +78,16 @@ What reload does *not* do: Standard input survives a reload: one reader serves the process, so the new console source continues where the old one stopped, and no line is read twice. -Plan reloads on a busy relay the way you would plan a restart. -Listener bind/start failures happen after the old service stops; these can leave -the application without working pipelines until a corrected configuration reloads. -File-watch errors do not restart services, and queued changes are combined. +Plan reloads on a busy relay as you would a restart: a listener that fails to +bind does so after the old service stopped, leaving lw without working +pipelines until a corrected configuration reloads +([Hot Reload](configuration.md#hot-reload)). ### Checking a configuration -`lw --check` builds every pipeline and plugin as a start would, without binding -a port or opening, reading or creating a log file, prints the startup warnings, -and exits 0, 1 on an invalid configuration, or 2 when a named file is missing: - -```bash -lw --check -c candidate.toml -``` - -Before a reload, check the edited file this way: a failed reload keeps the -running service, but `--check` says why without touching it. It prints the -first problem of each setting, flow stage, source, sink and pipeline, a line -each named by its key's path; once none is left, the files they name are -opened as a start opens them: - -```text -Error: Failed to load config: invalid configuration: -pipelines[0].plugin_sources[in].config.directory: must be set -pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000 -``` +Before a reload, run `lw --check -c FILE` on the edited file: a failed reload +keeps the running service, and `--check` says why without touching it +([CLI](cli.md#usage-patterns)). ## Monitoring @@ -143,27 +121,35 @@ curl -s http://127.0.0.1:8080/status | jq . ```json { - "service": "LogWisp", - "version": "v0.16.0", + "endpoints": { + "status": "/status", + "stream": "/stream" + }, "instance_id": "sse", "server": { - "type": "http", - "host": "0.0.0.0", - "port": 8080, - "tls": false, + "acl": "none", "active_clients": 3, + "auth": "none", "buffer_size": 1000, "client_buffer_size": 256, + "host": "0.0.0.0", "max_connections": 32, - "write_timeout_ms": 5000, - "uptime_seconds": 8130 + "port": 8080, + "tls": false, + "type": "http", + "uptime_seconds": 8130, + "write_timeout_ms": 5000 }, - "endpoints": { "stream": "/stream", "status": "/status" }, + "service": "LogWisp", "statistics": { - "total_processed": 15234, + "acl_denied": 0, + "acl_limited": 0, + "auth_rejected": 0, "dropped_writes": 12, - "rejected_clients": 0 - } + "rejected_clients": 0, + "total_processed": 15234 + }, + "version": "vX.Y.Z" } ``` @@ -188,18 +174,24 @@ Each counter, where it is reported, and what a rise means: auth refusals - `tls_handshake_errors` (`tcp` sink, `tcp_chain` source): a certificate or version mismatch, or scanning -- `parse_errors` (chain sources): protocol or version skew upstream +- `parse_errors` (chain sources): malformed entry lines from upstream, each + skipped - `reconnects` (`tcp_chain` sink): an unstable link or a flapping downstream - `dropped_batches` (`http_chain` sink): downstream is rejecting batches permanently - `synthesized` (chain sinks): events reach the sink without structure +- `acl_denied`, `acl_limited` (a listener whose `acl` block sets a rule or + limit; the `http` status always): connections or requests the address + rules or PROXY-header checks refused, and those over a per-client limit; + `acl_proxy_headers` counts the PROXY headers read ## Log Management -LogWisp's own operational log goes to stderr by default, at `info` with a -configuration file and at `warn` without one; explicit values win. A container -configured only through variables or flags therefore logs at `warn`: pass -`-e LOGWISP_LOGGING_LEVEL=info`. `lw-deploy.sh` always passes `-c`, so its +LogWisp's own log goes to stderr by default. Without a configuration file lw +logs less ([CLI](cli.md#built-in-defaults)), so a container configured only +through variables passes `LOGWISP_LOGGING_LEVEL=info` and +`LOGWISP_STATUS_REPORTER=true`; the image sets neither, as a variable would +override a mounted file's value. `lw-deploy.sh` always passes `-c`, so its nodes log at `info`. To files: ```toml @@ -226,19 +218,14 @@ the filter stage logs several lines per entry evaluated. ### Buffers -Raise `buffer_size` when `total_dropped_by_sink` is climbing but the sink itself -is healthy — that is a burst-absorption problem. - -`dropped_writes` on a network sink has two causes that a counter alone does not -separate. A consumer slower than the sustained rate cannot be bought off with -buffer, and drops are the intended outcome. A burst the consumer would have -drained, arriving faster than it reads, is configuration: the sink queues a -whole burst while the client writes one frame at a time, so the part of a burst -above `client_buffer_size` is lost even to a loopback reader. -Where a `rate_limit` bounds the pipeline, its `burst` is that number — keep -`client_buffer_size` at or above it and the second cause disappears. The HTTP -status endpoint reports both queue bounds alongside the counters so an operator -can tell which one is in play. +- `total_dropped_by_sink` climbing while the sink is healthy: a burst + outran its queue; raise `buffer_size` +- `dropped_writes` on a network sink: + - a client slower than the sustained rate: drops are intended + - a burst above `client_buffer_size`, lost even to a fast reader: keep + `client_buffer_size` at or above the burst a `drop` rate limit lets + through, its `burst_entries` (`entries_per_second` when 0) +- the HTTP status endpoint reports both queue bounds beside the counters ```toml [pipelines.plugin_sinks.config] @@ -256,10 +243,8 @@ policy = "drop" max_entry_size_bytes = 65536 ``` -Two behaviours to keep in mind: the limiter does not exist at all when -`entries_per_second <= 0`, and `policy = "pass"` short-circuits the size cap as -well as the rate check. Enforcing `max_entry_size_bytes` therefore requires -`entries_per_second > 0` and `policy = "drop"`. +Enforcing `max_entry_size_bytes` needs a rate above 0 and `policy = "drop"` +(a file's default is `pass`); see [Rate limiting](configuration.md#rate-limiting). ### Formatting @@ -314,10 +299,10 @@ problems see [Networking](networking.md#troubleshooting). **Environment variable override has no effect** -LogWisp reads `LOGWISP_QUIET`, `LOGWISP_LOGGING_LEVEL`, and other prefixed -names. Bare names used by older versions must be renamed. Array-indexed paths, -such as a key inside `[[pipelines]]`, cannot be set from the environment or the -command line; define whole pipelines there with the +Only `LOGWISP_`-prefixed names are read (`LOGWISP_QUIET`, +`LOGWISP_LOGGING_LEVEL`). Array-indexed paths, such as a key inside +`[[pipelines]]`, cannot be set from the environment or the command line; +define whole pipelines there with the [pipeline flags](cli.md#pipelines) or [pipeline variables](cli.md#pipeline-variables). @@ -355,11 +340,10 @@ than masking parts of them. See [Filters](filters.md#common-recipes). **Upgrades** -1. Read the changelog for configuration-schema changes. -2. Start the new binary against the current configuration in a scratch - environment. -3. Stop the old process, install the new binary, start it. -4. Confirm each pipeline started and that counters are advancing. +1. Run the new binary's `lw --check -c FILE` against the current + configuration: unknown or renamed keys fail there. +2. Stop the old process, install the new binary, start it. +3. Confirm each pipeline started and that counters are advancing. **Backup** diff --git a/doc/scram-auth-plan.md b/doc/scram-auth-plan.md deleted file mode 100644 index 634cc5b..0000000 --- a/doc/scram-auth-plan.md +++ /dev/null @@ -1,419 +0,0 @@ -# Password Authentication (Argon2id-SCRAM) - -**Status:** implemented, in three batches: `lixenwraith/auth` gained SCRAM -channel binding and the mTLS layer was hardened (see -[mTLS Hardening](#mtls-hardening)); then the SCRAM integration described here, -on the `auth` release that carries channel binding; then browser logins behind -a TLS-terminating proxy. Operator documentation lives in -[Security](security.md#password-authentication-scram) and -[`lw auth`](cli.md#lw-auth). - -**Scope:** optional username/password authentication on every network plugin, -next to the existing certificate method, without weakening what mTLS gives. - -## Goals - -- Password authentication for chain links (`tcp_chain`, `http_chain`: sources - verify, sinks present) and for viewers of the `tcp` and `http` sinks. -- No password or password-equivalent on the wire, no KDF work per login on - listeners, and no credential relay through a TLS-terminating MITM. -- A temporary pipe needs no file: one listener user from a descriptor, a pipe - or the terminal, with a certificate made at startup. -- Default stays `auth.type = "none"`; existing configurations are unaffected. -- Config in TOML, credential management and viewer clients in a `lw auth` - CLI. - -## Non-Goals - -- Standard SASL SCRAM-SHA-256 interoperability. The `auth` exchange is - logwisp-to-logwisp (or logwisp CLI) only. -- Browser viewers of a logwisp that terminates TLS itself: a browser cannot see - the server certificate, so its proof cannot be channel-bound. Browsers log in - through [proxy mode](#browser-viewers-behind-a-tls-terminating-proxy), or use - mTLS. -- Multiple backends behind a non-sticky L7 balancer: handshake state and the - token key live in one process. - -## Mechanism - -`lixenwraith/auth` Argon2id-SCRAM. The listener stores a verifier per user: -`StoredKey = SHA-256(HMAC(K, "Client Key"))`, `ServerKey = HMAC(K, "Server Key")`, -`K = Argon2id(password, salt)`. The Argon2id digest itself is never stored: it is -the salted password, from which both keys derive. Verifying a proof is HMAC only; -Argon2 runs on the dialer or CLI (t=3, 64 MiB, 4 threads, the `auth` defaults). -The dialer refuses a challenge below that cost, so a rogue server cannot ask for -a cheaply guessable proof, and trusts the link only after the server's final -signature proves it holds the user's `ServerKey`. - -**Channel binding.** Both peers pass `auth.WithChannelBinding(SHA-256(server leaf -DER))` at the proof step. The server hashes its configured certificate; the -client hashes the certificate it saw. A relay presenting any other certificate, -even one the CA trusts, makes the server reject the proof, so it never obtains a -link or a token. A mismatch is indistinguishable from a wrong password on the -wire by design; the client also sends its hash as a diagnostic field, so the -listener logs "TLS interception or a terminating proxy" and counts it. TLS must -terminate at logwisp, except in [proxy mode](#browser-viewers-behind-a-tls-terminating-proxy). - -**HTTP pinning.** HTTP auth takes two requests and every later request may use a -new pooled connection, so binding the exchange is not enough. Once the challenge -arrives the dialer pins the certificate that answered on its `VerifyConnection`: -the proof and every ingest request must present the same leaf, or the handshake -fails before anything is sent. The `http_chain` sink then clears token and pin -and logs in again, which also covers certificate rotation. A token handed to -curl by the CLI is unpinned. - -## Configuration - -```toml -# listener (tcp_chain/http_chain source, tcp/http sink) -[pipelines.plugin_sources.config.auth] -type = "scram" -credentials_file = "/etc/logwisp/users.toml" # or username + password_file -token_lifetime_ms = 900000 # http sink and http_chain source only -node_binding = "force" # chain sources; binds to the username - -# dialer (tcp_chain/http_chain sink) -[pipelines.plugin_sinks.config.auth] -type = "scram" -username = "edge-01" -password_file = "/etc/logwisp/edge-01.pass" -``` - -Validation at construction: - -- `scram` requires TLS, except on an `http` sink in proxy mode; a dialer also - forbids `insecure_skip_verify`. -- Listeners require `credentials_file`, or `username` and `password_file` - (one user, its verifier derived in memory at construction), not both, and - reject `allow`/`allow_patterns`: the users are the allow list. Use one file - per listener when listeners admit different users. -- Dialers require `username` + `password_file`, and reject `credentials_file`, - `token_lifetime_ms`, `allow*` and `identity` (server pinning stays an mTLS - feature). -- A password is 1–1024 bytes after trimming one line break, 8 or more on a - listener. A regular file is read at every construction; `/dev/fd/N` (the - inherited descriptor, then closed), a pipe, a FIFO or another device once, at - startup, and kept for reloads; a terminal is asked once, at startup, without - echo, and on a listener Enter alone generates a password shown only there. - The open file decides, not stat: FreeBSD stats `/dev/stdin` as a device - whatever it holds. No terminal fails closed. -- `identity` on a listener requires `tls.client_auth` and **binds the certificate - to the user**: the certificate's identity field must equal the SCRAM username, - so a peer needs its own certificate and its own password (PostgreSQL's - `clientcert=verify-full`). Without it the factors are independent: any - CA-issued certificate plus any valid password (`clientcert=verify-ca`). -- `token_lifetime_ms` only on HTTP listeners; default 15 minutes. -- `/auth` and every path under it are reserved. -- `mtls` rejects every SCRAM key, `trusted_proxies` included. A block that - names peers or credentials with `type = "none"` is refused: the type was - forgotten, not the block. - -**Credentials file** (TOML, written by the CLI, mode 0600): - -```toml -decoy_key = "" -[[users]] -argon_memory = 65536 -argon_threads = 4 -argon_time = 3 -salt = "" -server_key = "" -stored_key = "" -username = "edge-01" -``` - -`decoy_key` keeps unknown-user challenges stable across restarts and edits, so -a single probe cannot tell a real user from an unknown one (a real user's salt -changes when it is added, rotated or removed). It is created by the first -`add-user` and preserved by every rewrite. An empty user set, a duplicate -user, a missing or short `decoy_key`, an unknown key, or users differing in KDF -profile or salt length are configuration errors. The file is read at -construction; the SCRAM server starts in the plugin's `Start` and stops in -`Stop`, so a rejected reload leaks nothing. - -## Wire Protocol - -`chain.Hello` gains `scram` (raw JSON, decoded by `authz`); the protocol version -stays 1 because the field is additive. Every message after the hello is one -`authStep`: `error`, `challenge`, `proof` (+ `binding`), `final` (+ `token`, -`expires_in` on HTTP). Pre-auth lines and `/auth` bodies are capped at 4 KiB. - -**TCP** (`tcp_chain`, `tcp` sink): hello carrying the client-first message, -challenge, proof, final — all under one deadline (`hello_timeout_ms` on the -`tcp_chain` source, 10 s on the `tcp` sink and every dialer, with the dialer's -context able to cut it short). The final is written only after every admission -check (certificate binding, node binding); a later refusal is an `error` line -instead. The stream then continues on the same buffered reader. A listener -without SCRAM that reads a hello (the `tcp_chain` source) answers one carrying -credentials with `error: authentication not enabled`; an old binary stays -silent, and the dialer gives up after its deadline with "no challenge within -10s". - -**HTTP** (`http_chain` source, `http` sink): `POST /auth`, outside the auth -middleware, with its own 10 s read/write deadline. It answers `404` unless the -policy is `scram`. Each step, by its request body: - -- Step 1, the hello - - Success: `200` with the challenge. - - Failure: `429` throttled, `503` busy, `400` malformed, `413` oversized. -- Step 2, the proof - - Success: `200` with the final, `token` and `expires_in`. - - Failure: `401 {"error":"authentication failed"}`, or `503` if the server - stopped. - -Protected endpoints take `Authorization: Bearer `: missing or invalid is -`401` with `WWW-Authenticate: Bearer realm="logwisp"`, a certificate-binding miss -is `403`. Tokens are HS256 JWTs from `auth.NewJWT` (issuer `logwisp`, no leeway) -with a random per-instance key, so every reload revokes them. An SSE stream is -checked at connect and outlives its token; a reload ends it. - -**Dialer outcomes** (`http_chain` sink): every `/auth` failure is transient (the -batch is held under backoff); `404`/`405` from `/auth` reads "no auth endpoint -at (older logwisp, or not a scram listener)"; an ingest `401` clears the -token and is retried (tokens are renewed ahead of expiry, so this follows a -listener reload); `403` drops the batch; a pin mismatch clears token and pin. -Refusals are logged at WARN on every attempt, paced by the backoff (`Chain -connect refused` on `tcp_chain`, `Chain batch delivery failed` on -`http_chain`), and failed logins are counted in `auth_failures` / -`last_auth_error`. - -## Throttling - -Per remote IP (the socket address; in proxy mode the forwarded client, an IPv6 -one per /64), on handshake starts: a token bucket (burst 10, 1/s) refunded on -success, and at most 4 unfinished exchanges. An unanswered HTTP challenge holds -its slot for the `auth` handshake timeout (30 s); abandoned TCP exchanges -release their slot, and their entry in the `auth` handshake table, -immediately. The address table, `netacl.Table` as for the `acl` block's -per-client limits, holds 65,536 entries, drops one once its bucket has -refilled and its challenges have expired, and fails closed when full; the SCRAM -server itself caps in-flight handshakes at 4,096 (`busy`). - -Counters: `auth_allowed` counts logins; `auth_rejected` every refusal — failed -proofs, malformed requests, missing credentials and, on HTTP, refused tokens; -`auth_binding_mismatch` is the subset of failed proofs whose client saw another -certificate. `auth_throttled` and `auth_busy` count starts refused before a -challenge and are not in `auth_rejected`. - -## The `authz` Seam - -Listeners never call `Authorize` directly any more; one entry point per transport -decides, and `Authorize` itself refuses under `scram` so a missed call site fails -closed. - -- `Admit(conn, cs, wantHello, timeout)` returns an admission (identity, hello, - reader) whose `Accept` or `Reject` writes the final message. -- `AuthorizeRequest(r)` covers mTLS certificates and bearer tokens, returning - the refusal status for `Refuse`; `ServeAuth(w, r)` serves `/auth`. -- Dialers: `Greet(ctx, conn, node)` writes the hello and, under SCRAM, runs the - TCP exchange. Over HTTP `Prepare` logs in when no token is held and sets the - bearer header, `Invalidate` drops the token after a `401` or a pin mismatch, - `Token` is the login itself (behind `Prepare`, and for the CLI), and - `VerifyConnection` pins. -- `Start`/`Close` bracket the SCRAM server and are nil-safe like everything else. - -## CLI - -``` -lw auth add-user --credentials F --user U [--password-file P] [--generate] -lw auth remove-user --credentials F --user U -lw auth token --url https://host:port[/path] --user U [--password-file P] [--unbound] [TLS flags] -lw auth stream --addr host:port --user U [--password-file P] [TLS flags] -``` - -- `add-user` takes the password from `--password-file` when it exists (at least - 8 bytes); a new user without one gets a generated 130-bit password, written to - that file (0600) or printed once. Rotating an existing user needs the file or - `--generate`, so a mistyped path cannot replace a password. The file's KDF - profile and decoy key are reused, the result is validated with the daemon's - own parser, and the rewrite (temp file beside the file or a symlink's target, - rename) keeps the existing mode and owner, or fails. -- `remove-user` refuses to remove the last user. -- Both print that changes apply on `SIGHUP`: `auto_reload` does not watch the - credentials file. -- `token` prints a bearer token for curl (`-H @<(...)` keeps it out of argv); - `stream` authenticates to a `tcp` sink and copies the stream to stdout until - interrupted. `--url` takes a path only with `--unbound`, and redirects are not - followed. `--password-file` defaults to `/dev/tty`, which asks. -- TLS flags: `--ca-file` or `--pin-sha256`, `--server-name`, `--cert-file`, - `--key-file`. There is no insecure flag. -- Exit status: 0 success, 1 failure, 2 usage error. - -## Rollout and Rotation - -- Turning SCRAM on is per listener and cuts off unconfigured dialers. To migrate - without a gap, upgrade binaries, add a second SCRAM listener, move dialers, - then remove the old listener. -- Password rotation: `add-user --generate`, `SIGHUP` the listener, deploy the new - password file, `SIGHUP` the dialer. The dialer retries under backoff in - between. -- Revocation: `remove-user`, `SIGHUP`. All tokens die with the reload. - -## Browser Viewers Behind a TLS-Terminating Proxy - -The deployer serves the log stream inside a website whose TLS ends at its -reverse proxy; LogWisp holds no private key and listens on a trusted hop. -Viewers log in from a browser with the client LogWisp ships, and the password -reaches neither the proxy nor LogWisp. Operator view and example: -[Security](security.md#browsers-behind-a-tls-terminating-proxy). - -**Trust model.** `auth.trusted_proxies` (addresses or CIDRs) switches the `http` -sink into proxy mode; it is refused on every other plugin and with `identity`. -- Only listed peers may connect: `403` on every path, the pages included. -- `X-Forwarded-Proto` must be `https` on every hop, so a site served in - plaintext by mistake fails closed instead of exposing sessions. -- The client is the rightmost `X-Forwarded-For` hop that is not a proxy (hops to - its left are the client's own claims), or the leftmost when every hop is - inside a proxy range; throttling (an IPv6 client per /64, as one host holds - it whole), sessions and logs use it. Outside proxy mode forwarded headers - stay ignored. -- SCRAM runs unbound: a browser cannot read the site's certificate, and LogWisp - sees none. The proof and the session ride on the site's TLS; a party on the - proxy-to-LogWisp hop could relay a login, hence the address restriction and a - startup warning for a plaintext hop to a proxy off this host. `tls` may stay - off. Outside proxy mode nothing changes: proofs stay bound, and a bound proof - in proxy mode fails with a hint naming `--unbound`. - -**Session.** The proof step may ask for a cookie (`"session": "cookie"`, proxy -mode only). The answer sets `logwisp_session` (`HttpOnly; Secure; -SameSite=Strict; Max-Age=`) and carries no token in the body. It has -no `Path`: RFC 6265 defaults it to the directory of `POST /auth`, which -is the mount behind any proxy prefix. Stream and status accept the cookie or -`Authorization: Bearer`; another `Authorization` scheme, such as the site's own -Basic auth, leaves the cookie in charge. Logout is `{"logout": true}` on -`POST /auth` itself, because a clearing cookie set from `/auth/logout` would -default to another path and miss the login cookie. It clears the cookie and -revokes a valid token until it expires (a set of at most 65,536 hashes, gone on -reload like every token). An open stream outlives its token; a reconnect after -expiry gets `401`. `/auth` takes only `application/json`, which a cross-origin -page cannot send without a preflight nobody answers. - -**Token mode.** A proof step without `"session"` gets the token in the answer -body, in proxy mode as everywhere else, so a browser that keeps no cookies needs -no server change. `scram.js` asks for it with `{session: "token"}`, reads the -stream through `fetch` with the bearer (`EventSource` cannot send one), and -revokes it with `logout(base, {token})`. The viewer falls back to it when a -cookie would not stick or a fresh cookie login still gets `401`, and keeps the -token in a page variable only: it lasts no longer than the page or its lifetime. - -**Endpoints.** `/auth` and every path under it are reserved; every URL in the -pages is relative, so a proxy prefix works unchanged. - -- `POST /auth`: SCRAM hello, proof or logout; unbound in proxy mode. -- `GET /auth/scram.js`, `style.css`: the client library and the pages' style, - always served in proxy mode, and with the viewer. -- `GET /`: `303` to `auth/view`, else `auth/login`, when either is served. -- `GET /auth/login`, `login.js`: the login page (`login_page`). -- `GET /auth/view`, `view.js`: a minimal live viewer (`viewer_page`, needs - `login_page`). - -Files carry `default-src 'none'; script-src 'self'; connect-src 'self'; -style-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`, -`nosniff` and `no-referrer`; nothing is inline. The viewer learns a custom -`status_path` from a meta tag the sink fills in, renders entries as text, and -sends a signed-out visitor to `login?next=view%23signed-in` (the fragment marks -a fresh cookie login; without cookies it shows its own sign-in form); the login -page follows `next` only within its origin. The `proxy_tls` capability -satisfies the pipeline's "auth needs TLS" check. - -**Client library.** `internal/sink/http/web/scram.js`, embedded with `go:embed`, -is one dependency-free ES module, so a site with its own CSP can copy it into -its bundle. It exports `login(base, username, password, {onProgress, session})`, -`logout(base, {token})`, `stream(url, {token, signal, onEvent})` (an event -stream parsed as the HTML standard does), `cookiesUsable()` and -`loginUnavailable()`. BLAKE2b and Argon2id are plain JS (no WebAssembly, which -would need `'wasm-unsafe-eval'`), yielding to the page every 50 ms; SHA-256 and -HMAC come from WebCrypto, so it needs a secure context. It checks the challenge -as the Go client does (nonce, salt, cost limits and the dialer's Argon2 floor), -refuses redirects, and reports success only after verifying the server's -signature. A 64 MiB login takes about 2 s on a desktop. The wire protocol above -is the contract: a site may implement its own client. - -**CLI.** `lw auth token --unbound` logs in through the proxy, still pinning -its certificate across the two requests; only then may `--url` carry the mount -path. - -## Verification - -Go tests, one per rule: binding (match, mismatch, one-sided, length) in `auth`. -In `authz`: the validation table (each row valid but for its rule, the error -naming it) and credentials file validation; the TCP exchange over `net.Pipe` -(admission with the first entry in the same read, wrong password -indistinguishable from an unknown user, another certificate refused and counted, -hello and policy disagreeing either way, a dialer cut short by its context, an -abandoned exchange freeing its handshake and limiter slots, a refusal after the -exchange sent as a reason rather than a final, certificate-to-user binding for -logins and tokens); only failed attempts throttled (the limiter's bounds are -`netacl`'s, under the per-client limits); pre-auth -input capped at 4 KiB; the decoy salt stable across restarts; a dialer refusing -a challenge below its Argon2 floor; the HTTP exchange on a TLS test server -(token accepted, garbage and foreign tokens `401`, forged final yields no token, -a different certificate after the challenge refused before the proof is sent, -renewal ahead of expiry); `Authorize` failing closed under SCRAM; password -trimming; a one-user listener admitting its password, refusing another and an -unknown user alike; a descriptor's password surviving a reload, closed and -refused to a second user; a descriptor lw opened itself never read; refused -hellos freeing their slot; unbound logins pinned too. In `tlsx`, terminals, -pipes and descriptors never warned as world-readable. Proxy -mode: trusted peers, https and the client hop; throttling on the forwarded -client and IPv6 /64; an unbound browser login with the cookie's attributes, the -cookie opening endpoints (beside Basic auth) until logout revokes it, cookie -sessions refused elsewhere; only unbound logins behind a proxy; `/auth` taking -only JSON. In the `http` sink, every SSE line break framed as data. -In the plugins: `/auth` outside the `http` sink's gate, the pages served with -their types and CSP, direct peers refused, pages only in proxy mode, and the -`http_chain` sink logging in again after a `401` and delivering once. In the -CLI: private files with a matching verifier, no password replaced without a -source, mode, decoy key and symlink kept across rewrites, an empty file filled, -last-user refusal. Under node, `scram.js` against the RFC 9106 vector, Go's -`argon2.IDKey`, and `auth`'s known-answer proof; the event-stream line breaks -and fields, the bearer, refusals and abort; a token-mode login and logout. - -`test/scram-chain-test.sh --auto` (ports 15821-15825): a relay with SCRAM on all -four listener types plus one `client_auth` + SCRAM port; authorized edges over -both chain transports; node labels forced to usernames; wrong password, unknown -user and no-auth edges deliver nothing; a `tcp` sink viewer through `lw auth -stream` held past the exchange deadline; a raw TLS client without a hello gets -nothing; the `http` sink through `lw auth token` and curl; missing and bad -tokens get `401`; `remove-user` + `SIGHUP` revokes token and login; throttling -last. `test/scram-proxy-test.sh --auto` (ports 15831-15832): a Go reverse proxy -ending TLS in front of a plaintext `http` sink mounted at `/logs/`; headless -Chromium sent to the login page, refused a wrong password, streaming events -after login, with the cookie scoped to `/logs` and hidden from scripts, cleared -by sign-out, and no CSP violations; a cookie the browser drops leading to the -viewer's own form, not a loop, its token kept out of storage and the URL; a -profile blocking every cookie signing in on the viewer, its proofs asking for -no cookie, streaming with a bearer, sign-out revoking the token, and a reload's -`401` asking again; `token --unbound` and curl through the proxy; direct peers -and plaintext-forwarded requests `403`. `test/preset-test.sh --auto` (ports -15897-15898): an aggregator and an edge given one password on `/dev/fd/3` -deliver, another password is refused, both survive `SIGHUP` and deliver again, -and the password is in no file, argv or environment; under `script(1)` a -listener's generated password shows on its terminal and not in its log, and -an edge and `lw auth token` log in with it. The existing scripts keep passing. - -## Not Implemented - -None. The PROXY protocol behind TLS passthrough, deferred here, shipped with -the ACL work: [Security](security.md#proxy-protocol). - -## mTLS Hardening - -Shipped with this plan, independent of SCRAM: - -- The `http_chain` sink no longer follows redirects. Go's client resends the - body on a 307/308, including from `https` to plaintext `http`. -- Its response drain is bounded (64 KiB). -- The `tcp` sink tracks connections from accept, so `Stop` and reloads no longer - wait out the 10 s handshake timeout of a silent peer. -- Unknown keys in any plugin's config, at any depth, fail construction: a typo - such as `[...tls] enabeld = true` used to switch protection off silently. The - configuration file itself is checked the same way, so a misspelled table path - above a plugin's config fails too. -- Startup and every reload warn about certificates that expired, are not yet - valid, or expire within 30 days; about `insecure_skip_verify`; about - `allow_patterns` that are not anchored at both ends of every alternative; and, - once per path, about key, credentials and password files every local user can - read. -- An `http` sink with an auth policy no longer sends - `Access-Control-Allow-Origin: *`. diff --git a/doc/security.md b/doc/security.md index df41e29..bd415fb 100644 --- a/doc/security.md +++ b/doc/security.md @@ -33,10 +33,8 @@ configure it, and — equally important — what it does not yet do. - Startup warnings for expiring certificates and risky settings: see [Startup Warnings](#startup-warnings). - Unknown configuration keys rejected: a typo in `tls`, `auth`, `acl` or a - table path fails startup. -- Configuration input hardened (lixenwraith/config v0.2.2 and - lixenwraith/toml v0.1.3, reviewed with adversarial tests and fuzzing like - `auth`): + table path is refused when the configuration loads. +- Configuration input hardened (lixenwraith/config, lixenwraith/toml): - a document creates at most 65536 tables, so a small file cannot exhaust memory, and unescaped control characters are refused in strings and comments; @@ -58,6 +56,10 @@ configure it, and — equally important — what it does not yet do. - Per-client stream caps: under HTTP/2 `max_connections_per_client` bounds a client's connections, not the streams each carries; the request rate only slows how fast it opens them. +- A per-identity listing in the `http` status endpoint: it shows the policy + and `auth_rejected` only. +- More than one identity mode per listener: `auth.identity` takes one of + `cn`, `san_dns`, `san_uri`, `san_email`. Two credentials are supported. Certificates (`mtls`) are the one the transport already carries: the `tls` block establishes that a peer chains to your CA, and @@ -96,7 +98,9 @@ Options by role, each with its default: - `cert_file`: local certificate; a listener needs it, `self_signed` or the issuer files, a dialer may present it as its client identity. - `key_file`: private key for `cert_file`; set the two together. - - `min_version` (`"1.3"`): `"1.2"` or `"1.3"`. + - `min_version` (`"1.3"`): `"1.2"` or `"1.3"`. There is no `max_version` + and no `cipher_suites`: Go fixes the 1.3 suites, and the 1.2 ones are + its defaults. - Listeners - `self_signed` (`false`): make a self-signed certificate at startup. - `issuer_cert_file`, `issuer_key_file`: make one at startup, signed by this @@ -118,18 +122,16 @@ Options by role, each with its default: Listeners are the `tcp` and `http` sinks and the `tcp_chain` and `http_chain` sources; dialers are the `tcp_chain` and `http_chain` sinks. -> `min_version` takes `"1.2"` or `"1.3"`. The older `"TLS1.2"` spelling from -> pre-restructure releases is rejected. There is no `max_version` and no -> `cipher_suites` option; TLS 1.3 suites are not configurable in Go, and the -> 1.2 defaults are the standard library's. - ### Validation -Misconfiguration fails at plugin construction, before the pipeline starts: +Misconfiguration is refused when the configuration loads +([Validation](configuration.md#validation)), unknown keys always and the other +rules only when `enabled`; certificate and key files fail when the plugin is +built: - a listener with `enabled = true` and none of `cert_file`/`key_file`, - `self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one; an - issuer that is no CA (`lw tls ca` makes one) + `self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one +- `hosts` on a listener without `self_signed` or the issuer files - `pin_sha256` with `ca_file` or `insecure_skip_verify`, or not of the `sha256//BASE64` form - a key of the other role: `self_signed`, `issuer_cert_file`, @@ -137,12 +139,13 @@ Misconfiguration fails at plugin construction, before the pipeline starts: `ca_file`, `server_name`, `insecure_skip_verify` or `pin_sha256` on a listener - `client_auth = true` with no `client_ca_file` -- a dialer with only one of `cert_file` / `key_file` -- a certificate or key that will not load, or a CA file containing no - certificates +- only one of `cert_file` / `key_file` - a `min_version` that is neither `"1.2"` nor `"1.3"` - a key the plugin does not know, at any depth: `unknown key "tls.enabeld"`. A misspelled option must not silently leave a protection switched off +- when the plugin is built: a certificate or key that will not load, a CA + file containing no certificates, or an issuer that is no CA (`lw tls ca` + makes one) ## The Auth Block @@ -286,7 +289,7 @@ you listed. ```toml [pipelines.plugin_sinks.config.auth] type = "mtls" -allow = ["relay.internal"] +allow = ["relay.example.org"] ``` The check runs as part of the handshake, so a server the policy rejects never @@ -296,7 +299,8 @@ was shown instead; see [Channel binding](#channel-binding). ### Validation -Misconfiguration fails at plugin construction, before the pipeline starts: +As for `tls`, the rules are checked when the configuration loads, and the +files (`credentials_file`, `password_file`) when the plugin is built: - `type` `mtls` or `scram` without `tls.enabled` (proxy mode aside), or on a dialer with `tls.insecure_skip_verify`: an identity read from an unverified @@ -320,14 +324,16 @@ Misconfiguration fails at plugin construction, before the pipeline starts: - `trusted_proxies` on anything but the `http` sink, with `identity`, or with an entry that is neither an address nor a CIDR; `login_page` or `viewer_page` without `trusted_proxies`, `viewer_page` without `login_page` -- an `identity` that is not one of the four modes +- an `identity` that is not one of the four modes, or a `node_binding` that + is not one of the three values, whatever `type` is, `none` included +- under `mtls` or `scram`, a `node_binding` other than `none` on a plugin + other than a chain source - an `allow_patterns` entry that does not compile -- a `node_binding` that is not one of the three values, or one set on a plugin - that has no node concept -- an HTTP path (`stream_path`, `status_path`, `ingest_path`) equal to `/auth`, - which is reserved for logins +- `/auth`, which is reserved for logins: `ingest_path` may not equal it; + `stream_path` and `status_path` may not be it or lie under it -Errors read like `auth: type "mtls" requires tls.client_auth`. +Errors name their path: +`pipelines[0].plugin_sinks[tcp].config.auth: type "mtls" requires tls.client_auth`. ## The ACL Block @@ -352,9 +358,9 @@ requests_per_second_per_client = 10 # HTTP listeners; 0 (default): none on an IPv4 listener, IPv6 on an IPv6 one, either behind a hostname (only the family it binds ever matches). Behind `proxy_from` or the `http` sink's `trusted_proxies` they take either, as the proxy may name a client of - either. An entry of the other family, an - IPv4-mapped address, one with a zone, or anything but an address or CIDR - fails construction. A link-local peer matches without its zone. + either. An entry of the other family, an IPv4-mapped address, one with a + zone, or anything but an address or CIDR fails at load. A link-local peer + matches without its zone. - A refused connection is closed at once and counted in `acl_denied`, or `acl_limited` past a [limit](#per-client-limits) (plugin stats, the `http` sink's status). A WARN names the client, the reason and the counts, at most @@ -374,7 +380,7 @@ requests_per_second_per_client = 10 # HTTP listeners; 0 (default): none - HTTP keep-alive connections count, idle ones too, and a browser opens up to six; under HTTP/2 one connection carries a client's many requests. - Behind `trusted_proxies` every connection is the proxy's, so the key - fails construction: cap clients at the proxy. + fails at load: cap clients at the proxy. - `requests_per_second_per_client`, on the `http` sink and the `http_chain` source, admits a second's worth of requests at once (at least one), refilled at that rate; past it a request gets `429`, with `Retry-After` the @@ -437,6 +443,11 @@ when it is constructed, so startup and every reload repeat them: - `insecure_skip_verify` on a dialer - an `allow_patterns` entry not anchored at both ends of every alternative (`^a|b$` admits `a…` and `…b`) +- `type = "mtls"` on a listener with neither `allow` nor `allow_patterns`: it + admits any identity the CA vouches for ([The allow list](#the-allow-list)) +- `auth.trusted_proxies` with an entry beyond loopback and `tls` off: the hop + from the proxy is plaintext + ([Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy)) - a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every local user can read (once per path per process); terminals, pipes and descriptors hold nothing at rest and are not reported @@ -506,10 +517,10 @@ openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \ # Relay (server) certificate — SAN must match how clients address it openssl req -newkey rsa:2048 -nodes -keyout relay.key -out relay.csr \ - -subj "/CN=relay.internal" + -subj "/CN=relay.example.org" openssl x509 -req -in relay.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ -out relay.crt -days 825 \ - -extfile <(printf "subjectAltName=DNS:relay.internal\nextendedKeyUsage=serverAuth") + -extfile <(printf "subjectAltName=DNS:relay.example.org\nextendedKeyUsage=serverAuth") # Edge (client) certificate — CN identifies the node openssl req -newkey rsa:2048 -nodes -keyout edge-01.key -out edge-01.csr \ @@ -557,7 +568,7 @@ min_version = "1.3" [pipelines.plugin_sinks.config.auth] type = "mtls" -allow = ["relay.internal"] +allow = ["relay.example.org"] ``` ### 4. Verify @@ -685,9 +696,10 @@ TCP connection that ends mid-exchange frees it at once. A refused start answers `too many attempts` (HTTP `429`) and counts `auth_throttled`; the table holds 65,536 addresses and refuses new ones when full. Separately, at most 4,096 exchanges may be in flight per listener; beyond that, or while the plugin -stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`. Peers behind -one NAT, or one passthrough proxy that sends no PROXY header, share a bucket: -there, one client can exhaust every other client's logins. +stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`; neither +counts in `auth_rejected`. Peers behind one NAT, or one passthrough proxy that +sends no PROXY header, share a bucket: there, one client can exhaust every +other client's logins. ### Credentials file @@ -857,9 +869,11 @@ location /logs/ { A page not served over HTTPS keeps no `Secure` cookie either: both pages name HTTPS instead and offer no sign-in. - **Integrating `scram.js`.** A site with its own CSP can copy `scram.js` from - `internal/sink/http/web/` into its bundle: one dependency-free ES module, - where `base` is the mount URL ending in `/`. It needs a secure context for - WebCrypto and takes about 2 s of Argon2 per login on a desktop. + `internal/sink/http/web/` into its bundle: one dependency-free ES module in + plain JS (no WebAssembly, so no `'wasm-unsafe-eval'`), where `base` is the + mount URL ending in `/`. It needs a secure context for WebCrypto and takes + about 2 s of Argon2 per login on a desktop. `argon2id` and `_proofForTest` + are exported for `scram.test.mjs` only; do not rely on them. - Cookie mode, the default: `login(base, username, password, {onProgress})`, then `EventSource` and `fetch` as for any same-origin resource; `logout(base)`. @@ -872,12 +886,12 @@ location /logs/ { - `cookiesUsable()` tells which applies: false when the session cookie would not stick. `loginUnavailable()`, asked first, names why neither can run (no secure context), or is empty. -- **Pages.** Under `/auth/` the sink serves `scram.js` and `style.css` always, - and with `login_page` / `viewer_page` the login page and a minimal live - viewer with their scripts; `GET /` leads to the viewer, else the login page. - All are served under `default-src 'none'; script-src 'self'; - connect-src 'self'; style-src 'self'; form-action 'self'; frame-ancestors - 'none'; base-uri 'none'`. +- **Pages.** Under `/auth/` the sink serves `scram.js`, `style.css` and + `favicon.svg` always, and with `login_page` / `viewer_page` the login page + and a minimal live viewer with their scripts; `GET /` leads to the viewer, + else the login page. All are served with `nosniff`, `no-referrer` and the CSP + `default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; + img-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`. `test/scram-proxy-test.sh --auto` logs in from headless Chromium through such a proxy, with cookies and with every cookie blocked. @@ -1011,9 +1025,7 @@ peer is. identity or user and send `SIGHUP`: the reload rebuilds every pipeline, so the change takes effect on the next connection and existing ones are dropped by the rebuild. No network call on the handshake path, and no window between revocation -and the next CRL publication. See -[mtls-auth-plan.md](mtls-auth-plan.md#not-implemented) for what CRL support -would add. +and the next CRL publication. ## Surfaces Without Access Control @@ -1026,20 +1038,21 @@ What each exposes when `auth.type = "none"`: - `http` sink `stream_path`: the full log stream, with `Access-Control-Allow-Origin: *`, so any browser origin can read it (the header is omitted once an auth policy is set). -- `http` sink `status_path`: host, port, TLS flag, uptime, client counts, - throughput counters. -- `http` sink `GET /` (`303` to `/auth/view`) and the viewer's files: static - pages that read the two endpoints above under the CSP and expose nothing - beyond them. Under `mtls` they are served before the allow list, while the - stream and status stay gated. +- `http` sink `status_path`: version and instance id, host, port, TLS flag, + the auth and acl policy summary, buffer sizes and limits, endpoints, + uptime, client counts, throughput and rejection counters. +- `http` sink `GET /` (`303` to `auth/view`, relative to the mount) and the + viewer's files: static pages that read the two endpoints above under the + CSP and expose nothing beyond them. Under `mtls` they are served before the + allow list, while the stream and status stay gated. - `tcp` sink: the full log stream to any client that connects. - `tcp_chain` and `http_chain` sources: ingest from any peer that can connect (with `client_auth`, any the CA vouches for), under any node label it claims. -`max_connections` bounds concurrency on all but the `http_chain` source; the -`acl` block's [per-client limits](#per-client-limits) bound each caller's -share. +`max_connections` bounds connections on the `tcp` sink and `tcp_chain` source +and streams on the `http` sink; the `acl` block's +[per-client limits](#per-client-limits) bound each caller's share. Both methods require TLS on the listener, except an `http` sink in proxy mode, where TLS ends at the trusted proxy; `mtls` also requires `client_auth`. @@ -1050,9 +1063,10 @@ where TLS ends at the trusted proxy; `mtls` also requires `client_auth`. - Use a dedicated CA for LogWisp so its trust decisions stay independent. - Keep leaf lifetimes short (90–825 days) and automate renewal. -- Key, credentials and password files should be `0600` and owned by the service - account; a world-readable one is reported at startup. `lw auth` creates - them `0600` and keeps the mode of an existing file. +- Keep key, credentials and password files unreadable by others: `0600` + owned by the account lw runs as, or `0640` `root:GROUP` for a group it is + in; startup warns about world-readable ones. `lw auth` and `lw tls` create + them `0600`, and `lw auth` keeps the mode of an existing file. - Rotation requires a reload (`SIGHUP`), because certificates and credentials are loaded once at plugin construction; there is no on-disk watch for them. - Startup and every reload warn 30 days before a certificate lapses. Still diff --git a/doc/sinks.md b/doc/sinks.md index 03d3227..d41afdf 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -8,9 +8,10 @@ type-specific `config` table. Registered types: `console`, `file`, `http`, `tcp`, `null`, `tcp_chain`, `http_chain`. -Dispatch into a sink is non-blocking. A sink whose input queue is full drops the -event *for itself only* and the pipeline counts it in `total_dropped_by_sink`; -sibling sinks are unaffected. +Dispatch waits for a `console` sink, so a slow reader paces the whole pipeline, +sibling sinks included (`null` waits too but discards at once). Any other sink +whose input queue is full drops the event for itself only +(`total_dropped_by_sink`). --- @@ -32,7 +33,8 @@ color = "auto" Options, each as type and default: -- `target` (string, `stdout`): `stdout` or `stderr`. +- `target` (string, `stdout`): `stdout` or `stderr`; `split` exists only for + lw's own log (`logging.output`). - `buffer_size` (int, `1000`): sink input queue depth. - `escape` (string, `auto`): write control characters as `` of their bytes, `ESC` as `<1b>`: `auto` when the output is a terminal, `always`, or @@ -41,16 +43,12 @@ Options, each as type and default: `auto` on a terminal with `NO_COLOR` unset and `TERM` not `dumb`, `always`, or `never`. -> `split` is **not** a valid target for this sink and is rejected at startup. -> Level-based splitting exists only for LogWisp's own application log -> (`logging.output = "split"`). - Payloads are written as the formatter made them, one record per line. - Escaping: a log line must not drive the terminal that shows it (cursor and - screen control, title changes, OSC 52 clipboard writes, bidi spoofing). C0 - and C1 controls other than tab, DEL, the bidi controls, the line and - paragraph separators and invalid UTF-8 are escaped; other invisible + screen control, title changes, OSC 52 clipboard writes, bidi spoofing). + Escaped: every C0 control but tab; DEL and the C1 controls; the bidi + controls; the line and paragraph separators; invalid UTF-8. Other invisible characters, such as the joiners emoji and Indic scripts need, are kept. `never` keeps an application's own ANSI colours; under `auto`, pipes and files get the bytes unchanged. @@ -91,20 +89,23 @@ flush_interval_ms = 100 Options, each as type and default: -- `directory` (string, **required**): output directory. -- `name` (string, **required**): base filename. +- `directory` (string, **required**): output directory. The sink writes + `NAME.log` and rotates it to `NAME_YYMMDD_HHMMSS.log`. The limits below + delete *any* `.log` file in the directory, so give each file sink a + directory of its own. +- `name` (string, **required**): file name base. - `max_size_mb` (int, `100`): rotate when the active file reaches this size. -- `max_total_size_mb` (int, `1000`): cap across all rotated files. -- `min_disk_free_mb` (int, `0`): free-space floor before writing; `0` = none. -- `retention_hours` (float, `168.0`): delete rotated files older than this. +- `max_total_size_mb` (int, `1000`): above this total, the oldest `.log` files + there are deleted. +- `min_disk_free_mb` (int, `0`): the oldest `.log` files there are deleted to + keep this much space free, and nothing is written while that fails. `0` + means none and a negative value means `100`. +- `retention_hours` (float, `168.0`): `.log` files there older than this are + deleted; `0` keeps them. - `buffer_size` (int, `1000`): sink input queue depth, and the depth of the internal writer's queue, which drops and counts when full. - `flush_interval_ms` (int, `100`): forced flush interval. -> `min_disk_free_mb` has an unusual default. The constructor replaces only -> *negative* values with `100`; leaving the key unset yields `0`, which means no -> free-space floor. Set it explicitly if you want one. - The sink drives an internal writer configured for raw output with timestamps and levels disabled, so what lands on disk is exactly the formatted payload. On stop it hands the writer what is queued before closing the file. @@ -113,8 +114,9 @@ it hands the writer what is queued before closing the file. ## null -Discards everything, counting entries and bytes. Useful for benchmarking a -source or flow in isolation. +Discards; holds a sink's place until one is chosen, so a pipeline can be set +up before its sink is. It counts entries and bytes, so it also isolates a +source or flow when measuring it. ```toml [[pipelines.plugin_sinks]] @@ -190,9 +192,10 @@ Options, each as type and default: **Behaviour** -- Only `GET` is routed to either path; anything else gets `405`, `HEAD` on - `stream_path` included — a stream is a body, and a client registered to have - its body discarded never reads and never leaves. +- Only `GET` (and `HEAD` on `status_path`) is routed to the two paths; + anything else gets `405`, including `HEAD` on `stream_path`. A stream is a + body, and a client registered to have its body discarded never reads and + never leaves. - With an `auth` block, one middleware gates **both** endpoints, with no body detail in a refusal. Under `mtls` a refused certificate gets `403`. Under `scram` a client logs in at `POST /auth`, which sits outside the gate (see @@ -225,15 +228,17 @@ Options, each as type and default: - Refusals are logged at WARN and counted in `auth_rejected`. The authorized identity is recorded in the client's session as `auth_method` / `auth_identity`. -- On connect the client receives an `event: connected` frame carrying its - client id, session id, sink instance id, endpoint paths, and buffer size. +- Stream responses carry `Content-Type: text/event-stream`, + `Cache-Control: no-cache` and `X-Accel-Buffering: no`. On connect the client + receives an `event: connected` frame carrying its client id, session id, + sink instance id, endpoint paths, and buffer size. - Payloads are framed per the SSE spec, one `data:` line per line break in the payload (CRLF, LF or a lone CR), so multi-line entries stream correctly and no entry can inject an `event:`, `id:` or `retry:` field. - The server sets no `WriteTimeout` (that would kill long-lived streams); per-write deadlines come from `write_timeout_ms` via `http.ResponseController` and cover the connected frame, every payload, and the idle comment. -- A quiet stream emits an SSE comment every 15 s. It refreshes the client's +- Every stream gets an SSE comment every 15 s. It refreshes the client's session and is how a peer that stopped reading is noticed. - A client whose send queue is full has that event dropped (`dropped_writes`); it is not disconnected. A `dropped_writes` that rises while @@ -258,11 +263,9 @@ buffer sizes, connection limit, write timeout, uptime, endpoint paths, and the `total_processed` / `dropped_writes` / `rejected_clients` / `auth_rejected` / `acl_denied` / `acl_limited` counters. -**Statistics**: `dropped_writes`, `rejected_clients`, `auth`, `auth_allowed`, -`auth_rejected`; under `scram` also `auth_users`, `auth_throttled`, -`auth_busy`, `auth_binding_mismatch` and `auth_token_lifetime_ms`, and in -proxy mode `auth_trusted_proxies`; with a non-empty `acl` block also `acl`, -`acl_denied`, `acl_limited` and `acl_proxy_headers`. +**Statistics**: `host`, `port`, `tls`, `buffer_size`, `client_buffer_size`, +`max_connections`, `write_timeout_ms`, `endpoints`, `dropped_writes`, +`rejected_clients`, and the [auth and acl keys](sources.md#source-statistics). > Without an `auth` block both endpoints are unauthenticated, and the stream > response carries `Access-Control-Allow-Origin: *`, so any web origin can read @@ -303,22 +306,14 @@ allow = ["viewer-01"] Options, each as type and default: -- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the listener - keeps to its family ([Networking](networking.md#address-family)). -- `port` (int, **required**): listen port. -- `buffer_size` (int, `1000`): sink input queue depth. -- `client_buffer_size` (int, `256`): per-client send queue depth. +- `host`, `port`, `buffer_size`, `client_buffer_size`: as on [http](#http). - `write_timeout_ms` (int, `5000`): per-write deadline. -- `keep_alive` (bool, `true`): TCP keep-alive on accepted connections. +- `keep_alive` (bool, `true`): TCP keep-alive probes after + `keep_alive_period_ms` idle; `false` sends none. - `keep_alive_period_ms` (int, `30000`): keep-alive idle period. - `max_connections` (int, `0`): concurrent connection cap; `0` = unlimited. -- `tls` (table): listener TLS. -- `auth` (table): client authentication (`mtls` or `scram`); see - [Security](security.md#the-auth-block). -- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused - before TLS, `proxy_protocol` with `proxy_from` for the client a PROXY header - names, and `max_connections_per_client`; see - [Security](security.md#the-acl-block). +- `tls`, `auth`: as on `http`. +- `acl` (table): as on `http`, without `requests_per_second_per_client`. **Behaviour** @@ -342,11 +337,9 @@ Options, each as type and default: receives a broadcast. Its connection is closed, the rejection logged at WARN, and `rejected_conns` incremented. -**Statistics**: `write_errors`, `dropped_writes`, `rejected_conns`, -`tls_handshake_errors`, `auth`, `auth_allowed`, `auth_rejected`; under `scram` -also `auth_users`, `auth_throttled`, `auth_busy` and `auth_binding_mismatch`; -with a non-empty `acl` block also `acl`, `acl_denied`, `acl_limited` and -`acl_proxy_headers`. +**Statistics**: `host`, `port`, `tls`, `buffer_size`, `write_errors`, +`dropped_writes`, `rejected_conns`, `tls_handshake_errors`, and the +[auth and acl keys](sources.md#source-statistics). --- @@ -360,7 +353,7 @@ persistent connection. See [Chaining](chaining.md). id = "to_relay" type = "tcp_chain" [pipelines.plugin_sinks.config] -host = "relay.internal" +host = "relay.example.org" port = 15801 node = "edge-01" buffer_size = 1000 @@ -388,8 +381,10 @@ Options, each as type and default: - `dial_timeout_ms` (int, `5000`): TCP connect timeout. - `write_timeout_ms` (int, `5000`): per-write deadline. - `backoff_min_ms` (int, `500`): reconnect backoff floor. -- `backoff_max_ms` (int, `30000`): reconnect backoff ceiling. -- `keep_alive` (bool, `true`): TCP keep-alive. +- `backoff_max_ms` (int, `30000`): reconnect backoff ceiling; below + `backoff_min_ms` it is `30000`. +- `keep_alive` (bool, `true`): TCP keep-alive probes after + `keep_alive_period_ms` idle; `false` sends none. - `keep_alive_period_ms` (int, `30000`): keep-alive idle period. - `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity. - `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see @@ -401,12 +396,13 @@ Options, each as type and default: downstream being up. - Under `scram` the hello carries the login, and the link counts as established only once the server's final message proves it holds the user's verifier. The - exchange is bounded at 10 s and cut short by shutdown. A refusal — - `authentication failed`, `authentication not enabled`, `no challenge within - 10s` — is logged at WARN as `Chain connect refused` on every attempt; other - connect failures log at DEBUG. Both retry under the normal backoff. + exchange is bounded at 10 s and cut short by shutdown. +- A refusal (`authentication failed`, `authentication not enabled`, `no + challenge within 10s`) or a server failing its CA or pin is logged at WARN + as `Chain connect refused` on every attempt; other connect failures log at + DEBUG as `Chain connect failed`. Both retry under the normal backoff. - Each entry is serialized as one canonical JSON line. Delivery holds the line - across reconnects until it is written or the process shuts down, with + across reconnects until the socket accepts it or the process shuts down, with exponential backoff plus ±20 % jitter between attempts. The delay keeps growing until a link outlives `backoff_min_ms`, so a source that drops every link right after accepting it is not retried in a tight loop. @@ -425,12 +421,9 @@ Options, each as type and default: - Events arriving without a structured entry are wrapped from the formatted payload and counted in `synthesized`. -Under `mtls` the policy pins the server's identity as part of the handshake, so -a server it rejects is refused like a failed login above. - -**Statistics**: `target`, `node`, `tls`, `auth`, `connected`, `reconnects`, -`write_errors`, `synthesized`; under `scram` also `auth_username`, -`auth_failures` and `last_auth_error` (empty after a successful login). +**Statistics**: `target`, `node`, `tls`, `connected`, `reconnects`, +`write_errors`, `synthesized`, and the +[auth keys](sources.md#source-statistics). --- @@ -444,7 +437,7 @@ Batches structured entries as NDJSON and POSTs them to a downstream LogWisp id = "to_collector" type = "http_chain" [pipelines.plugin_sinks.config] -host = "collector.internal" +host = "collector.example.org" port = 15802 ingest_path = "/ingest" node = "edge-01" @@ -465,29 +458,22 @@ key_file = "/etc/logwisp/tls/client.key" Options, each as type and default: -- `host` (string, **required**): downstream host or address; an IPv6 one goes - bare (`::1`). -- `port` (int, **required**): downstream port. -- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`. -- `node` (string, `os.Hostname()`): origin label stamped on first-hop entries. -- `buffer_size` (int, `1000`): sink input queue depth. +- `host`, `port`, `node`, `buffer_size`, `tls`, `auth`: as on + [tcp_chain](#tcp_chain). +- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`, and + `/auth` is reserved for the login. - `max_batch_count` (int, `100`): flush after this many entries. - `max_batch_bytes` (int, `1048576`): flush after this many bytes (1 MiB). - `flush_interval_ms` (int, `1000`): flush after this long. - `request_timeout_ms` (int, `10000`): covers dial, write, and response. - `backoff_min_ms` (int, `500`): retry backoff floor. -- `backoff_max_ms` (int, `30000`): retry backoff ceiling. -- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity. -- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see - [Security](security.md#the-auth-block). +- `backoff_max_ms` (int, `30000`): retry backoff ceiling; below + `backoff_min_ms` it is `30000`. **Behaviour** -- Delivery is at-least-once per batch: a retried batch can be delivered twice if - the first attempt succeeded but the response was lost. -- Retries apply to transport errors, `408`, `429`, and `5xx`. Any other - non-2xx response is treated as permanent, and the batch is dropped and counted - in `dropped_batches`. A retry is logged at WARN, a dropped batch at ERROR. +- Delivery: [Chaining](chaining.md#delivery-semantics); a retry logs WARN, a + dropped batch ERROR. - Under `scram` the sink logs in at `POST /auth` whenever it holds no token and sends `Authorization: Bearer `. A failed login is retried like a transport error, holding the batch. The token is renewed ahead of its expiry; @@ -503,9 +489,9 @@ Options, each as type and default: within `request_timeout_ms`, at most 2 s; each batch still undelivered then is dropped and logged at WARN. -**Statistics**: `target`, `node`, `tls`, `auth`, `batches_sent`, -`request_errors`, `dropped_batches`, `synthesized`; under `scram` also -`auth_username`, `auth_failures` and `last_auth_error`. +**Statistics**: `target`, `node`, `tls`, `batches_sent`, `request_errors`, +`dropped_batches`, `synthesized`, and the +[auth keys](sources.md#source-statistics). --- @@ -513,4 +499,5 @@ Options, each as type and default: Every sink reports: `id`, `type`, `total_processed`, `active_connections`, `start_time`, `last_processed`, and a type-specific `details` map. The DEBUG -status report spreads `details` into each plugin's line. +status report spreads `details` into each plugin's line, network sinks' with +the [auth and acl keys](sources.md#source-statistics). diff --git a/doc/sources.md b/doc/sources.md index b38512a..72d963f 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -2,21 +2,12 @@ Sources produce `core.LogEntry` values for a pipeline. Every source is declared as a `[[pipelines.plugin_sources]]` entry with an `id`, a `type`, and a -type-specific `config` table. +type-specific `config` table. Registered types: `file`, `console`, `random`, +`null`, `tcp_chain`, `http_chain`. -```toml -[[pipelines.plugin_sources]] -id = "app_logs" -type = "file" -[pipelines.plugin_sources.config] -directory = "/var/log/myapp" -``` - -Registered types: `file`, `console`, `random`, `null`, `tcp_chain`, -`http_chain`. - -Publication from any source is non-blocking. When a subscriber channel is full -the entry is dropped and counted in `dropped_entries`. +Publication is non-blocking for every source but `console`: a full subscriber +channel drops the entry and counts it in `dropped_entries`. The `console` +source waits instead (see [console](#console)). --- @@ -41,30 +32,32 @@ Options, each as type and default: - `directory` (string, required): directory to scan; not recursive. - `pattern` (string, `*`): glob over filenames; `*` and `?` only. - `check_interval_ms` (int, `100`): directory rescan interval; minimum `10`. + An open file is polled every 100 ms whatever this is. - `raw` (bool, `false`): never parse a line: the whole line is the message. - `from` (string, `end`): where a new watcher starts, `end` or `start` of the file. **Behaviour** -- `check_interval_ms` governs how often the directory is rescanned for new or - removed files. Tailing an already-open file polls on a **fixed 100 ms** - interval that this option does not change. -- Each matched file gets its own watcher. Watchers for files that disappear are - stopped and removed on the next scan. +- Each matched file gets its own watcher, and its entries the file's base + name as `Source`. Watchers for files that disappear are stopped and removed + on the next scan. - The file lw's stdout or stderr is redirected to is never followed, with a warning: `lw > out.log` in the directory would read its own output back. - A line longer than 1 MiB continues in the next entry, as on the console source. - A new watcher seeks to end-of-file. Positions live in memory only, so a - restart resumes from the current end of each file and content written while - LogWisp was down is not read. `from = "start"` reads each file whole when its - watcher is created instead — what a process writing beside LogWisp needs, at - the cost of replaying a file already on disk at every restart. + restart or a reload resumes from the current end of each file, and content + written meanwhile is not read. `from = "start"` instead reads each file whole + when its watcher is created, which a process writing beside LogWisp needs, at + the cost of replaying files already on disk at every restart and reload. - Rotation is detected from size decrease, modification-time reset, a position beyond end-of-file, or an inode change. An inode change where the new file is - already larger than the recorded position is treated as an atomic save, not a - rotation, and the position is preserved. + at least as large as the recorded position is treated as an atomic save, not + a rotation, and the position is preserved. Each rotation also publishes an + INFO entry `Log rotation detected (#N): REASON` with the file's name as + `Source`; it reaches every sink, so exclude it with a filter + (`Log rotation detected`) where unwanted. - A rotation that renames in place — what a size-capped writer does — puts the same inode back under a name `pattern` also matches. Its watcher resumes at the position the original reached, so `from = "start"` reads the tail an @@ -72,17 +65,19 @@ Options, each as type and default: - A line is parsed as JSON only when it is an object whose top-level keys are all drawn from `time`, `level`, `msg` and `fields` — the four an entry can carry. `time` is read as RFC3339Nano. Any other key, and any non-object line, - is kept whole as text, because parsing it would drop the rest. + is kept whole as text, because parsing it would drop the rest. `level` is + kept as written (a filter sees `Error` or `fatal` as is), and a line without + `level` has none; only text lines get the inference below. - A text line's level is its first whole word, in any case, naming one: `TRACE`; `DEBUG`, `DBG`; `INFO`, `INF`; `WARN`, `WARNING`; `ERROR`, `ERR`, `FATAL`. `warning: disk full` is WARN, `INFO retry after error` INFO. The console sink paints that word and the http sink's viewer filters on it. -- `raw = true` skips the JSON branch entirely. The line, plus its newline, - becomes the message; `fields` stays empty, the time is the read time, and the - level is inferred from the text as for any unparsed line. Paired with - `format.type = "raw"` this is byte-exact transport for records LogWisp's - envelope cannot hold — see [Formatters](formatters.md#raw). -- `Source` is set to the file's base name. +- `raw = true` skips the JSON branch. The line, with its terminator (`\n` or + `\r\n`) removed, becomes the message. `fields` stays empty, the time is the + read time, and the level is inferred from the text. Blank lines are skipped. + With `format.type = "raw"` each line is written back with one `\n`, which + carries records the envelope cannot hold but is not byte-exact; see + [Formatters](formatters.md#raw). **Statistics**: per-watcher size, position, entries read, rotation count, and last read time, plus `active_watchers`. @@ -128,7 +123,8 @@ Delivery and lifetime: ## random -Synthetic entry generator for development, smoke tests, and sanitizer testing. +Generated entries, to check a deployment before real logs are wired: paths, +permissions, sinks, chains, and sanitizer policies with `special = true`. ```toml [[pipelines.plugin_sources]] @@ -145,23 +141,25 @@ special = false Options, each as type and default: - `interval_ms` (int, `500`): emission period. -- `jitter_ms` (int, `0`): symmetric jitter; must be non-negative, clamped to - `interval_ms`. +- `jitter_ms` (int, `0`): each interval varies by up to ±`jitter_ms`/2; + non-negative, capped at `interval_ms`. - `format` (string, `txt`): `raw` (message only), `txt` (bracketed line) or `json` (JSON object as the message). -- `length` (int, `20`): message length in characters. -- `special` (bool, `false`): inject control and non-ASCII characters. +- `length` (int, `20`): bytes of random text; `txt` and `json` wrap it in a + longer message. +- `special` (bool, `false`): with `length` ≥ 3, mix in up to one control byte + (`\n` and `\r` among them) and a few non-ASCII bytes, often invalid UTF-8. + `raw` and `txt` carry them as they are; `json` escapes them inside its + message. -`special = true` is the intended way to exercise sanitizer policies: it inserts -control bytes and multi-byte Unicode into otherwise ordinary messages. Levels -are chosen at random from DEBUG, INFO, WARN, ERROR. +Levels (`txt` and `json` only) are random among DEBUG, INFO, WARN, ERROR. --- ## null -Produces nothing. Useful as a placeholder so a sink-only pipeline satisfies the -"at least one source" requirement. +Nothing; holds a source's place until one is chosen, so a pipeline can be set +up, sinks first, before its source is. ```toml [[pipelines.plugin_sources]] @@ -216,8 +214,8 @@ Options, each as type and default: - `hello_timeout_ms` (int, `10000`): deadline for the hello preamble. - `trust_node` (bool, `true`): `false` overrides the sender's node label with its remote address. `auth.node_binding` `assert` or `force` takes over the - connection label; only `force` also overrides the per-entry labels; `none` - leaves both to `trust_node`. + connection label, and `force` the per-entry labels too + ([Chaining](chaining.md#node-identity)). - `tls` (table): listener TLS; see [Security](security.md). - `auth` (table): peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block). @@ -235,33 +233,31 @@ and a `credentials_file`, or one `username` with its `password_file`, and - TLS handshakes run explicitly with a 10 s bound before the preamble is read, after the `max_connections` admission check. -- A connection is rejected if the first line is not a valid hello with a - matching protocol version. - Under `mtls` the certificate is checked before the hello is read, so an unauthorized peer never gets a preamble parsed on its behalf. Under `scram` the hello opens the login: challenge, proof and final follow on the same connection, the whole exchange within `hello_timeout_ms`, and the final is sent only once node binding agrees. A source without `scram` answers a hello - offering credentials with `authentication not enabled`. A rejection is logged - at WARN and counted in `rejected_conns`. -- The node label is then resolved: under `auth.node_binding` it comes from the - peer's certificate identity or username, otherwise `trust_node` governs. - `force` also overrides the `node` field on every individual entry; `assert` - leaves per-entry labels to `trust_node`, so a relay can forward other nodes' - entries while proving its own identity. -- Each accepted connection gets a session recording the remote address, node - label, — under TLS — `tls` and `tls_peer_cn`, — under auth — - `auth_method` and `auth_identity`, and behind a PROXY header the proxy as - `peer_addr`. + offering credentials with `authentication not enabled`. +- An auth or node-binding refusal is logged at WARN and counted in + `rejected_conns`. A missing, malformed or wrong-version hello is logged at + WARN (`Rejected chain connection`) and not counted in `rejected_conns`; + under `scram` a malformed or wrong-version one counts in `auth_rejected`. A + connection over `max_connections` is closed and counted without a log. +- The node label is then resolved by `trust_node` and `auth.node_binding` (see + [Chaining](chaining.md#node-identity)). +- Each accepted connection gets a session recording its remote address and: + - `node`, the resolved label + - under TLS: `tls`, `tls_peer_cn` + - under auth: `auth_method`, `auth_identity` + - behind a PROXY header: the proxy as `peer_addr` - A malformed entry line increments `parse_errors` and is skipped; the connection survives. A line over 1 MiB is a protocol violation and terminates the connection. -**Statistics**: `active_connections`, `rejected_conns`, `parse_errors`, -`tls_handshake_errors`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`, -`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy` -and `auth_binding_mismatch`; with a non-empty `acl` block also `acl`, -`acl_denied`, `acl_limited` and `acl_proxy_headers`. +**Statistics**: `host`, `port`, `tls`, `active_connections`, `rejected_conns`, +`parse_errors`, `tls_handshake_errors`, `trust_node`, and the +[auth and acl keys](#source-statistics). --- @@ -297,25 +293,13 @@ node_binding = "force" Options, each as type and default: -- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the - listener keeps to its family ([Networking](networking.md#address-family)). -- `port` (int, required): listen port, 1–65535. +- `host`, `port`, `trust_node`, `tls`, `auth`: as on [tcp_chain](#tcp_chain). - `ingest_path` (string, `/ingest`): endpoint path; must start with `/`, and `/auth` is reserved for the login. - `buffer_size` (int, `1000`): subscriber channel depth. - `max_body_bytes` (int, `8388608`): per-request body cap (8 MiB). - `read_timeout_ms` (int, `30000`): full request read deadline. -- `trust_node` (bool, `true`): `false` overrides the sender's node label with - its remote address. `auth.node_binding` `assert` or `force` takes over the - connection label; only `force` also overrides the per-entry labels; `none` - leaves both to `trust_node`. -- `tls` (table): listener TLS; see [Security](security.md). -- `auth` (table): peer authentication (`mtls` or `scram`) and node binding; - see [Security](security.md#the-auth-block). -- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused - before TLS, `proxy_protocol` with `proxy_from` for the client a PROXY header - names, and the per-client `max_connections_per_client` and - `requests_per_second_per_client`; see [Security](security.md#the-acl-block). +- `acl` (table): as on `tcp_chain`, plus `requests_per_second_per_client`. **Behaviour** @@ -329,21 +313,23 @@ Options, each as type and default: node-binding failure answer `403`. Both are distinct from the `400` used for protocol errors, so a sender can tell "not allowed" from "malformed batch". - A missing or mismatched `X-Logwisp-Protocol` header is rejected with `400`. +- With `acl.requests_per_second_per_client`, a request over the rate gets `429` + with `Retry-After`; the sink retries it. - Batch acceptance is atomic: entries are published only after the body reads - cleanly end to end. A transfer error rejects the whole batch (`400`, or `413` - when the body cap is hit) so the sender retries it. A malformed *line* inside - an otherwise clean transfer is skipped and counted in `parse_errors`. + cleanly end to end. A transfer error, a line over 1 MiB included, rejects the + whole batch with `400`, or `413` when the body cap is hit; an `http_chain` + sink drops such a batch (`dropped_batches`) rather than retry it. A malformed + *line* inside an otherwise clean transfer is skipped and counted in + `parse_errors`. - Success is `204 No Content` with `X-Logwisp-Accepted` set to the number of entries ingested. - Sessions are cached per remote host + node + authenticated identity, and recreated after idle expiry. Including the identity in the key means two peers sharing a remote address never share a session. -**Statistics**: `total_requests`, `rejected_requests`, `parse_errors`, -`cached_sessions`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`, -`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`, -`auth_binding_mismatch` and `auth_token_lifetime_ms`; with a non-empty `acl` -block also `acl`, `acl_denied`, `acl_limited` and `acl_proxy_headers`. +**Statistics**: `host`, `port`, `ingest_path`, `tls`, `total_requests`, +`rejected_requests`, `parse_errors`, `cached_sessions`, `trust_node`, and the +[auth and acl keys](#source-statistics). --- @@ -352,3 +338,16 @@ block also `acl`, `acl_denied`, `acl_limited` and `acl_proxy_headers`. Every source reports: `id`, `type`, `total_entries`, `dropped_entries`, `start_time`, `last_entry_time`, and a type-specific `details` map. The DEBUG status report spreads `details` into each plugin's line. + +Network sources and sinks add the auth and acl keys to `details`: + +- `auth`: the method, `none` without an `auth` block; with one also + - `auth_allowed`, `auth_rejected`; chain sources `node_binding` + - `mtls`: `auth_identity`, `auth_unrestricted` + - `scram` listeners: `auth_users`, `auth_throttled`, `auth_busy`, + `auth_binding_mismatch`, `auth_identity` when bound; on HTTP + `auth_token_lifetime_ms`; in proxy mode `auth_trusted_proxies` + - `scram` dialers: `auth_username`, `auth_failures`, `last_auth_error` +- with a non-empty `acl` block: `acl`, `acl_denied`, `acl_limited`, + `acl_proxy_headers` + diff --git a/doc/todo.md b/doc/todo.md index 458c386..1b338b8 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -6,23 +6,11 @@ they belong to. ## 1. Packaging: AUR, FreeBSD ports, Debian -The foundation exists: -- The `lw` binary name, free in Arch (official repositories and AUR), Ubuntu - 24.04 and 26.04, and the FreeBSD 15.1 ports tree. -- The canonical module path `github.com/lixenwraith/logwisp`, which - `go install`, FreeBSD's `USES=go:modules` and Debian's dh-golang expect. -- `make install` with `DESTDIR`, `PREFIX`, `SYSCONFDIR` and `SERVICE`: the - binary, the `doc/lw.1` manual, bash, zsh and fish completion generated from - `cmd/lw`'s `shorts` and `commands` tables, and the service files in - `deploy/package/` (systemd unit, sysusers, tmpfiles, FreeBSD rc.d script) - with the configuration, kept on reinstall. -- `make deb`, a binary package built with `dpkg-deb`, and `make arch`, the - PKGBUILD built from the working tree. -- Skeletons in `deploy/package/arch/` and `deploy/package/freebsd/`. -- `.github/workflows/package.yml`: `make arch`, the `.deb` and BSD make on - FreeBSD 15.1, each reinstalled over an edited configuration; the systemd - and rc.d services run; a home install with `SERVICE=no` and - `lw config init`; and `make image image-check`. +[Installation](installation.md#packaging-status) lists what is in place. The +`lw` name is free in Arch (official repositories and AUR), Ubuntu 24.04 and +26.04, and the FreeBSD 15.1 ports tree; `go install`, FreeBSD's +`USES=go:modules` and Debian's dh-golang expect the module path +`github.com/lixenwraith/logwisp`. What remains is the maintainer's: it needs push rights, an identity or a release tag, and both skeletons download the tagged source. @@ -62,8 +50,8 @@ release tag, and both skeletons download the tagged source. - `make deb` serves direct installs. A PPA builds from a signed source package without network access: a `debian/` directory (changelog, control, rules) and the Go modules vendored into the source tarball. - - The archive wants every Go dependency packaged (dh-golang); the four - `lixenwraith` libraries are not. + - The archive wants every Go dependency packaged (dh-golang); the + `lixenwraith` libraries in `go.mod` are not. - `lintian` on the `make deb` package reports `no-changelog` (a source package's `debian/changelog` provides it), `statically-linked-binary` (lw is static by design) and `maintainer-script-calls-systemctl` (dh's @@ -72,95 +60,11 @@ release tag, and both skeletons download the tagged source. from a ports tree (`make stage check-plist` in a copy of `deploy/package/freebsd/`), rather than BSD make on the checkout. -## 2. `lw --tui` and the configuration engine it shares with lixen.com +## 2. What `lw --tui` and its engine leave for later -`lw --tui` composes pipelines in a full-screen terminal UI, runs them under -the screen with a live view (`t`), and on exit runs them at once or prints one -form of them: a command line, environment variables, or a configuration file. -Its logic is a pure engine that lixen.com's planned Configure tab (lixencom -`doc/todo.md`) runs as WebAssembly, so no part of LogWisp is rewritten in -JavaScript. - -What exists: -- Option structs with toml tags for every plugin and flow stage - (`internal/config/config.go`), presets as rows with typed parameters - (`preset.go`), the command-line and environment grammar (`spec.go`), - `ValidateConfig`, `--check` and `--dump`. -- The option schema, in `internal/config`: `default:`, `help:` and `lw:` tags - on the option structs, one catalogue row per plugin type, `Decode` (every - constructor decodes through it and keeps only I/O), `Coerce` for typed - command-line values, and a `ValidateConfig` that decodes every plugin - offline and names the first problem of each part by its key's path. -- `internal/config` already compiles for `GOOS=js`; it links no net/http, - crypto/tls or terminal code. -- The engine, `internal/compose`: starts empty, from a preset (whose path - `config.IsDir` resolves, from answers off the host), a loaded - configuration or pasted text of any form; adds, removes and moves sources, - sinks and filters, sets and unsets keys, validates; writes a command line, - environment variables or a file, each reading back to the same pipelines. - Its test fails when it links TLS, HTTP, process, signal or terminal code. -- `lw --schema`: settings, pipeline flags, flow stages in order with what - turns each on, sources and sinks with their tables' keys of their side, - presets and forms, as one JSON document. -- `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions, - layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal, - ConfirmDialog, scrolling, mouse; themed Radio, Field, TextInput, - OptionList and menus, Frame, Rule, Wires and Window, which `lw --tui` - draws with. `github.com/lixenwraith/color`: RGB, - blending, RGBTo256, RGBTo16. - -Steps: -1. Schema, in `internal/config`: done. -2. Engine, `internal/compose`: done. -3. `lw --schema`: done. The site generates its catalogue from it. -4. `lixenwraith/terminal` v0.2.0: done: 16 and no color, the terminal's - default colors, `/dev/tty`, bracketed paste, themed form controls. Still - optional, in `color`: export the 16 xterm reference shades (RGBTo16 maps - the site's hues badly, so the 16-color tier is chosen by hand). v0.2.1: - radio buttons, narrow form fields and option lists, menus, frames, rules, - wires and windows, moved from `lw --tui`. Pinned at the untagged v0.2.2 - commit: one glyph per cell at output, a reverse cursor bar in one style. -5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)). -6. WebAssembly: done. `make wasm` writes `bin/lwconf/`, which a site drops - in whole: - - `lwconf.wasm`, version stamped: 7.1 MB, 1.9 MB at gzip -9. - - `wasm_exec.js`, the toolchain's, which uses no eval: the page needs only - `'wasm-unsafe-eval'` in its `script-src`. - - `lwconf.js`, one dependency-free ES module that knows the calls and - shapes, never a key: `load({execURL, execSRI, wasmURL, wasmSRI})` - fetches both by their integrity and resolves to the engine; - `fields(schema, role, type)` is a node's form rows, a table's a group; - `value`, `at` and `group` read a node's value and sort problems by node. - - `schema.json`, equal to `schema()`, and `lwconf.json`, the manifest: - version, commit, Go version and each file's sha256. - - The engine's calls take strings (`lwconf.js` writes objects as JSON) and - answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`, - as far as the engine places the refusal: - - `schema()`: what `lw --schema` prints. - - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key - to a string, a list key to strings. - - `parse(text, form, dirs)`: the pipelines of a form's pasted text. - - `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled - (defaults, kinds, ids) and the first problem of each part (a value of - the wrong kind too) as `{path, pipeline, node, key, text}`. - - `emit(pipelines, form)`: the text of `command`, `environment` or `file`. - - `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is - `{op, pipeline, ...}`: `add_pipeline` and `rename_pipeline` (`name`), - `remove_pipeline`, `add` (`role`, `type`), `retype` (`node`, `type`), - `remove` (`node`), `move_filter` (`from`, `to`), `set` (`node`, `key`, - `values`), `unset` (`node`, `key`). - - `dirs` answers whether a path a preset names is a directory, - `{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, - one with a glob a pattern, and any other is refused at `path`. - - Pipelines are a JSON list in the file's shape (`name`, `flow`, - `plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node - is `{role, id, index}`: a source's or sink's id, a filter's index. -7. `t`, the tap: done ([doc/cli.md](cli.md#lw---tui)): `core.Tap` takes each - flow verdict, `internal/logline` reads any line shape, and the live view - shows them under the diagram. - -Not yet: +`lw --tui` ([CLI](cli.md#lw---tui)) and the configuration engine it shares +with the website ([Architecture](architecture.md#configuration-engine)) are +in place. Not yet: - Pins and export in the live view, as vif-log has them: pinned rows survive filter changes and show alone on a key; the pins or the shown rows go to a new file. @@ -172,7 +76,11 @@ Not yet: `lixenwraith/terminal`'s one-cell rule goes once terminal exports that rule. - A scripted session on a pseudo-terminal for Run, each output and `t` (keys, the password dialog, `SIGHUP`), on a text console and over ssh. -- `AuthOptions.Check` alone limits `auth.node_binding` to chain sources and - `auth.trusted_proxies` to the http sink, so the schema lists both on every - listener; a side tag naming those sides would filter them as `http` - filters HTTP's keys. +- The schema lists `auth.node_binding` on every listener and + `auth.trusted_proxies` on both HTTP listeners, though `AuthOptions.Check` + refuses a binding off the chain sources (under `mtls` or `scram`) and + `trusted_proxies` off the `http` sink; side tags for chain sources and the + `http` sink would filter them as `http` filters HTTP's keys. +- Optional, in `lixenwraith/color`: export the 16 xterm reference shades + (`RGBTo16` maps the site's hues badly, so the 16-color tier is chosen by + hand). diff --git a/internal/config/config.go b/internal/config/config.go index f55db89..9791b22 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,7 +15,6 @@ type Config struct { // Selected file path is runtime metadata, never a configurable override. ConfigFile string `toml:"-"` - // Existing fields Logging *LogConfig `toml:"logging"` Pipelines []PipelineConfig `toml:"pipelines"` } @@ -79,15 +78,15 @@ type HeartbeatConfig struct { Enabled bool `toml:"enabled" help:"send heartbeats"` IntervalMS int64 `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"` IncludeTimestamp bool `toml:"include_timestamp" help:"stamp each heartbeat with its time"` - IncludeStats bool `toml:"include_stats" help:"add the pipeline's counters"` - Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"heartbeat format"` + IncludeStats bool `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"` + Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"no effect: heartbeats use the flow's format"` } // FormatConfig is how a pipeline writes its entries type FormatConfig struct { Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"entry format"` Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"formatter flags"` - TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339" help:"Go time layout of timestamps"` + TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"Go time layout of timestamps"` SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"` } @@ -143,7 +142,7 @@ type PluginSourceConfig struct { ID string `toml:"id"` Type string `toml:"type"` Config map[string]any `toml:"config"` - ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config + ConfigFile string `toml:"config_file,omitempty"` // accepted, never read } // NullSourceOptions defines settings for a null source (no configuration needed) @@ -152,10 +151,10 @@ type NullSourceOptions struct{} // RandomSourceOptions defines settings for a random log generator source type RandomSourceOptions struct { IntervalMS int64 `toml:"interval_ms" default:"500" lw:"min=1" help:"milliseconds between entries"` - JitterMS int64 `toml:"jitter_ms" lw:"min=0,zero=none" help:"random milliseconds added to the interval, at most the interval"` + JitterMS int64 `toml:"jitter_ms" lw:"min=0,zero=none" help:"milliseconds the interval varies by, half each way; at most the interval"` Format string `toml:"format" default:"txt" lw:"enum=raw|txt|json" help:"entry format"` - Length int64 `toml:"length" default:"20" lw:"min=1" help:"message characters"` - Special bool `toml:"special" help:"mix in control characters"` + Length int64 `toml:"length" default:"20" lw:"min=1" help:"bytes of random text"` + Special bool `toml:"special" help:"mix in control and non-ASCII bytes"` } // FileSourceOptions defines settings for a file-based source @@ -209,7 +208,7 @@ type PluginSinkConfig struct { ID string `toml:"id"` Type string `toml:"type"` Config map[string]any `toml:"config"` - ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config + ConfigFile string `toml:"config_file,omitempty"` // accepted, never read } // NullSinkOptions defines settings for a null sink (no configuration needed) @@ -230,7 +229,7 @@ type FileSinkOptions struct { MaxSizeMB int64 `toml:"max_size_mb" default:"100" lw:"min=1" help:"size at which a file rotates"` MaxTotalSizeMB int64 `toml:"max_total_size_mb" default:"1000" lw:"min=1" help:"size of all files, beyond which the oldest go"` MinDiskFreeMB int64 `toml:"min_disk_free_mb" lw:"zero=none" help:"free space kept on the disk; negative means 100"` - RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0" help:"hours a file is kept"` + RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0,zero=kept" help:"hours a .log file is kept; 0 keeps them"` BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"` FlushIntervalMs int64 `toml:"flush_interval_ms" default:"100" lw:"min=1" help:"milliseconds between flushes"` } diff --git a/internal/config/schema.go b/internal/config/schema.go index 8c0b1ff..bf06d2d 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -42,13 +42,13 @@ type Plugin struct { var plugins = []Plugin{ {"source", "console", Local, false, true, "Standard input, line by line", reflect.TypeFor[ConsoleSourceOptions]()}, {"source", "file", Local, false, false, "Follow the files of a directory that match a pattern", reflect.TypeFor[FileSourceOptions]()}, - {"source", "random", Local, false, false, "Random entries, for tests", reflect.TypeFor[RandomSourceOptions]()}, - {"source", "null", Local, false, false, "Nothing, ever", reflect.TypeFor[NullSourceOptions]()}, + {"source", "random", Local, false, false, "Generated entries, to check a deployment before real logs", reflect.TypeFor[RandomSourceOptions]()}, + {"source", "null", Local, false, false, "Nothing; holds a source's place until one is chosen", reflect.TypeFor[NullSourceOptions]()}, {"source", "tcp_chain", ChainListener, false, false, "Entries from tcp_chain sinks of other instances", reflect.TypeFor[TCPChainSourceOptions]()}, {"source", "http_chain", ChainListener, true, false, "Entry batches from http_chain sinks of other instances", reflect.TypeFor[HTTPChainSourceOptions]()}, {"sink", "console", Local, false, false, "Standard output or standard error", reflect.TypeFor[ConsoleSinkOptions]()}, {"sink", "file", Local, false, false, "Rotating files", reflect.TypeFor[FileSinkOptions]()}, - {"sink", "null", Local, false, false, "Discard", reflect.TypeFor[NullSinkOptions]()}, + {"sink", "null", Local, false, false, "Discards; holds a sink's place until one is chosen", reflect.TypeFor[NullSinkOptions]()}, {"sink", "tcp", Listener, false, false, "Serve entries to TCP clients, one per line", reflect.TypeFor[TCPSinkOptions]()}, {"sink", "http", Listener, true, false, "Serve entries as server-sent events, with a browser viewer", reflect.TypeFor[HTTPSinkOptions]()}, {"sink", "tcp_chain", Dialer, false, false, "Forward entries to a tcp_chain source", reflect.TypeFor[TCPChainSinkOptions]()}, diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index bda93ad..b06b3a0 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -32,10 +32,11 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) { } } -// A zero number takes the default unless zero means something there, a switch -// that defaults on stays off when set off, a negative number is refused, and -// what moved from the constructors keeps its meaning: min_disk_free_mb below -// zero is the default, a backoff maximum below the minimum the default maximum +// A zero number takes the default unless zero means something there (a file +// sink's retention_hours 0 keeps files), a switch that defaults on stays off +// when set off, a negative number is refused, and what moved from the +// constructors keeps its meaning: min_disk_free_mb below zero is the default, +// a backoff maximum below the minimum the default maximum func TestZeroDefaultsAndNegatives(t *testing.T) { tcp, err := Decode[TCPSinkOptions]("sink", "tcp", map[string]any{"port": int64(1), "buffer_size": int64(0), "keep_alive": false}) if err != nil || tcp.BufferSize != 1000 || tcp.KeepAlive || tcp.Host != "0.0.0.0" { @@ -55,6 +56,16 @@ func TestZeroDefaultsAndNegatives(t *testing.T) { t.Errorf("min_disk_free_mb %v: %+v %v, want %d", in, file, err, want) } } + for in, want := range map[any]float64{nil: 168, int64(0): 0, 0.0: 0, 2.5: 2.5} { + m := map[string]any{"directory": "d", "name": "n", "retention_hours": in} + if in == nil { + delete(m, "retention_hours") + } + file, err := Decode[FileSinkOptions]("sink", "file", m) + if err != nil || file.RetentionHours != want { + t.Errorf("retention_hours %v: %+v %v, want %v", in, file, err, want) + } + } chain, err := Decode[TCPChainSinkOptions]("sink", "tcp_chain", map[string]any{"host": "h", "port": int64(1), "backoff_min_ms": int64(800), "backoff_max_ms": int64(700)}) if err != nil || chain.BackoffMaxMS != 30000 { t.Fatalf("backoff maximum below the minimum: %+v %v", chain, err) diff --git a/internal/sink/http/web/scram.js b/internal/sink/http/web/scram.js index b1a21aa..aa3bcd4 100644 --- a/internal/sink/http/web/scram.js +++ b/internal/sink/http/web/scram.js @@ -16,9 +16,10 @@ const encoder = new TextEncoder(); /** * Logs in to the logwisp http sink mounted at base (a URL ending in "/"), per - * logwisp's doc/scram-auth-plan.md; it resolves only once the server has proved - * it holds this user's verifier. The session is an HttpOnly cookie, or with - * session "token" the returned token. onProgress(fraction) follows Argon2. + * logwisp's doc/security.md (Password authentication) and doc/chaining.md + * (Wire protocol); it resolves only once the server has proved it holds this + * user's verifier. The session is an HttpOnly cookie, or with session "token" + * the returned token. onProgress(fraction) follows Argon2. */ export async function login(base, username, password, { onProgress, session = "cookie" } = {}) { if (session !== "cookie" && session !== "token") { diff --git a/internal/sink/null/null.go b/internal/sink/null/null.go index 85bb801..55ae985 100644 --- a/internal/sink/null/null.go +++ b/internal/sink/null/null.go @@ -15,35 +15,30 @@ import ( "github.com/lixenwraith/log" ) -// init registers the component in plugin factory func init() { if err := plugin.RegisterSink("null", NewNullSinkPlugin); err != nil { panic(fmt.Sprintf("failed to register null sink: %v", err)) } } -// NullSink discards all received transport events, used for testing +// NullSink discards every event: it holds a sink's place until one is chosen, +// or isolates a source or flow while it is measured type NullSink struct { - // Plugin identity and session management id string proxy *session.Proxy session *session.Session - // Application input chan core.TransportEvent logger *log.Logger - // Runtime done chan struct{} startTime time.Time - // Statistics totalReceived atomic.Uint64 totalBytes atomic.Uint64 lastReceived atomic.Value // time.Time } -// NewNullSinkPlugin creates a null sink through plugin factory func NewNullSinkPlugin( id string, configMap map[string]any, @@ -62,7 +57,6 @@ func NewNullSinkPlugin( } ns.lastReceived.Store(time.Time{}) - // Create session for null sink ns.session = proxy.CreateSession( "null://devnull", map[string]any{ @@ -78,7 +72,6 @@ func NewNullSinkPlugin( return ns, nil } -// Capabilities returns supported capabilities func (ns *NullSink) Capabilities() []core.Capability { return []core.Capability{ core.CapSessionAware, @@ -86,14 +79,11 @@ func (ns *NullSink) Capabilities() []core.Capability { } } -// Input returns the channel for sending transport events func (ns *NullSink) Input() chan<- core.TransportEvent { return ns.input } -// Start begins the processing loop func (ns *NullSink) Start(ctx context.Context) error { - ns.startTime = time.Now() go ns.processLoop(ctx) ns.logger.Debug("msg", "Null sink started", @@ -102,7 +92,6 @@ func (ns *NullSink) Start(ctx context.Context) error { return nil } -// Stop gracefully shuts down the sink func (ns *NullSink) Stop() { if ns.session != nil { ns.proxy.RemoveSession(ns.session.ID) @@ -113,7 +102,6 @@ func (ns *NullSink) Stop() { "total_received", ns.totalReceived.Load()) } -// GetStats returns sink statistics func (ns *NullSink) GetStats() sink.SinkStats { lastRcv, _ := ns.lastReceived.Load().(time.Time) @@ -129,7 +117,6 @@ func (ns *NullSink) GetStats() sink.SinkStats { } } -// processLoop reads transport events and discards them func (ns *NullSink) processLoop(ctx context.Context) { for { select { @@ -137,7 +124,6 @@ func (ns *NullSink) processLoop(ctx context.Context) { if !ok { return } - // Discard the event, only update stats ns.totalReceived.Add(1) ns.totalBytes.Add(uint64(len(event.Payload))) ns.lastReceived.Store(time.Now()) diff --git a/internal/source/file/file_watcher.go b/internal/source/file/file_watcher.go index 32c99c0..dddcc43 100644 --- a/internal/source/file/file_watcher.go +++ b/internal/source/file/file_watcher.go @@ -310,8 +310,6 @@ func (w *fileWatcher) checkFile() error { return nil } -// initPosition records the file's metadata and, unless the watcher was created -// to read from the start, sets the initial read position to the end // scanLinesCapped is bufio.ScanLines, except that a line longer than // core.MaxLogEntryBytes continues in the next one, as on the console source: // the scanner would otherwise stop at it and every poll re-read the lines before. @@ -323,6 +321,8 @@ func scanLinesCapped(data []byte, atEOF bool) (int, []byte, error) { return advance, token, err } +// initPosition records the file's metadata and, unless the watcher was created +// to read from the start, sets the initial read position to the end. func (w *fileWatcher) initPosition() error { file, err := os.Open(w.directory) if err != nil { diff --git a/internal/source/null/null.go b/internal/source/null/null.go index e112238..746613a 100644 --- a/internal/source/null/null.go +++ b/internal/source/null/null.go @@ -14,34 +14,28 @@ import ( "github.com/lixenwraith/log" ) -// init registers the component in plugin factory func init() { if err := plugin.RegisterSource("null", NewNullSourcePlugin); err != nil { panic(fmt.Sprintf("failed to register null source: %v", err)) } } -// NullSource generates no log entries, used for testing +// NullSource generates nothing: it holds a source's place until one is chosen type NullSource struct { - // Plugin identity and session management id string proxy *session.Proxy session *session.Session - // Application subscribers []chan core.LogEntry logger *log.Logger - // Runtime done chan struct{} - // Statistics totalEntries atomic.Uint64 startTime time.Time lastEntryTime atomic.Value // time.Time } -// NewNullSourcePlugin creates a null source through plugin factory func NewNullSourcePlugin( id string, configMap map[string]any, @@ -60,7 +54,6 @@ func NewNullSourcePlugin( } ns.lastEntryTime.Store(time.Time{}) - // Create session for null source ns.session = proxy.CreateSession( "null://void", map[string]any{ @@ -76,21 +69,18 @@ func NewNullSourcePlugin( return ns, nil } -// Capabilities returns supported capabilities func (ns *NullSource) Capabilities() []core.Capability { return []core.Capability{ core.CapSessionAware, } } -// Subscribe returns a channel for receiving log entries func (ns *NullSource) Subscribe() <-chan core.LogEntry { ch := make(chan core.LogEntry, 1000) ns.subscribers = append(ns.subscribers, ch) return ch } -// Start begins the source operation (no-op for null source) func (ns *NullSource) Start() error { ns.startTime = time.Now() ns.proxy.UpdateActivity(ns.session.ID) @@ -100,7 +90,6 @@ func (ns *NullSource) Start() error { return nil } -// Stop signals the source to stop func (ns *NullSource) Stop() { close(ns.done) if ns.session != nil { @@ -114,7 +103,6 @@ func (ns *NullSource) Stop() { "instance_id", ns.id) } -// GetStats returns the source's statistics func (ns *NullSource) GetStats() source.SourceStats { lastEntry, _ := ns.lastEntryTime.Load().(time.Time) diff --git a/internal/source/random/random.go b/internal/source/random/random.go index d997e87..f7248f3 100644 --- a/internal/source/random/random.go +++ b/internal/source/random/random.go @@ -17,49 +17,41 @@ import ( "github.com/lixenwraith/log" ) -// init registers the component in plugin factory func init() { if err := plugin.RegisterSource("random", NewRandomSourcePlugin); err != nil { panic(fmt.Sprintf("failed to register random source: %v", err)) } } -// RandomSource generates random log entries for testing +// RandomSource generates entries, to check a deployment before real logs are wired type RandomSource struct { - // Plugin identity and session management id string proxy *session.Proxy session *session.Session - // Configuration config *config.RandomSourceOptions - // Application subscribers []chan core.LogEntry logger *log.Logger rng *rand.Rand mu sync.RWMutex - // Runtime done chan struct{} wg sync.WaitGroup cancel chan struct{} - // Statistics totalEntries atomic.Uint64 droppedEntries atomic.Uint64 startTime time.Time lastEntryTime atomic.Value // time.Time } -// NewRandomSourcePlugin creates a random source through plugin factory func NewRandomSourcePlugin( id string, configMap map[string]any, logger *log.Logger, proxy *session.Proxy, ) (source.Source, error) { - // Step 1: Create empty config struct with defaults opts, err := config.Decode[config.RandomSourceOptions]("source", "random", configMap) if err != nil { return nil, err @@ -77,7 +69,6 @@ func NewRandomSourcePlugin( } rs.lastEntryTime.Store(time.Time{}) - // Create session for random source rs.session = proxy.CreateSession( fmt.Sprintf("random://%s", id), map[string]any{ @@ -98,14 +89,12 @@ func NewRandomSourcePlugin( return rs, nil } -// Capabilities returns supported capabilities func (rs *RandomSource) Capabilities() []core.Capability { return []core.Capability{ core.CapSessionAware, } } -// Subscribe returns a channel for receiving log entries func (rs *RandomSource) Subscribe() <-chan core.LogEntry { rs.mu.Lock() defer rs.mu.Unlock() @@ -114,7 +103,6 @@ func (rs *RandomSource) Subscribe() <-chan core.LogEntry { return ch } -// Start begins generating random log entries func (rs *RandomSource) Start() error { rs.startTime = time.Now() rs.wg.Add(1) @@ -127,7 +115,6 @@ func (rs *RandomSource) Start() error { return nil } -// Stop signals the source to stop generating func (rs *RandomSource) Stop() { close(rs.cancel) rs.wg.Wait() @@ -148,7 +135,6 @@ func (rs *RandomSource) Stop() { "total_entries", rs.totalEntries.Load()) } -// GetStats returns the source's statistics func (rs *RandomSource) GetStats() source.SourceStats { lastEntry, _ := rs.lastEntryTime.Load().(time.Time) @@ -169,12 +155,10 @@ func (rs *RandomSource) GetStats() source.SourceStats { } } -// generateLoop continuously generates random log entries at configured intervals func (rs *RandomSource) generateLoop() { defer rs.wg.Done() for { - // Calculate next interval with jitter interval := time.Duration(rs.config.IntervalMS) * time.Millisecond if rs.config.JitterMS > 0 { jitter := time.Duration(rs.rng.Intn(int(rs.config.JitterMS))) * time.Millisecond @@ -194,7 +178,6 @@ func (rs *RandomSource) generateLoop() { } } -// generateEntry creates a random log entry based on configured format func (rs *RandomSource) generateEntry() core.LogEntry { now := time.Now() @@ -247,7 +230,6 @@ func (rs *RandomSource) generateEntry() core.LogEntry { } } -// generateRandomString creates a random string of specified length func (rs *RandomSource) generateRandomString(length int) string { const normalChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 " const specialChars = "\t\n\r\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F" @@ -259,17 +241,14 @@ func (rs *RandomSource) generateRandomString(length int) string { // Reserve space for at least one special and one unicode char normalLength := length - 2 - // Generate normal characters for i := 0; i < normalLength; i++ { result = append(result, normalChars[rs.rng.Intn(len(normalChars))]) } - // Insert special character at random position specialPos := rs.rng.Intn(len(result) + 1) specialChar := specialChars[rs.rng.Intn(len(specialChars))] result = append(result[:specialPos], append([]byte{specialChar}, result[specialPos:]...)...) - // Insert unicode character at random position unicodePos := rs.rng.Intn(len(result) + 1) unicodeChar := unicodeChars[rs.rng.Intn(len(unicodeChars)/3)*3:] if len(unicodeChar) >= 3 { @@ -281,12 +260,10 @@ func (rs *RandomSource) generateRandomString(length int) string { } } - // Trim to exact length if needed if len(result) > length { result = result[:length] } } else { - // Normal generation without special characters for i := 0; i < length; i++ { result = append(result, normalChars[rs.rng.Intn(len(normalChars))]) } @@ -295,13 +272,11 @@ func (rs *RandomSource) generateRandomString(length int) string { return string(result) } -// randomLogLevel returns a random log level func (rs *RandomSource) randomLogLevel() string { levels := []string{"DEBUG", "INFO", "WARN", "ERROR"} return levels[rs.rng.Intn(len(levels))] } -// publish sends a log entry to all subscribers func (rs *RandomSource) publish(entry core.LogEntry) { rs.mu.RLock() defer rs.mu.RUnlock() diff --git a/internal/tui/inspector.go b/internal/tui/inspector.go index c8fbbcf..8a7870f 100644 --- a/internal/tui/inspector.go +++ b/internal/tui/inspector.go @@ -109,7 +109,7 @@ func (a *app) items(n compose.Node, path string) []string { // placeholder says what an unset key means: its default, what zero means, or // the kind of value it takes func placeholder(k config.Key) string { - if k.Zero != "" { + if k.Zero != "" && k.Default == "" { return "(" + k.Zero + ")" } return cmp.Or(k.Default, k.Hint) diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index f7cf527..24b346e 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -249,6 +249,16 @@ func TestInspectorHelpSitsUnderItsKeys(t *testing.T) { } } +// An unset key shows its default, what zero means when it has none, or the +// kind of value it takes +func TestAnUnsetKeyShowsItsDefaultFirst(t *testing.T) { + for want, k := range map[string]config.Key{"168": {Default: "168", Zero: "kept"}, "(off)": {Zero: "off"}, "dir": {Hint: "dir"}} { + if got := placeholder(k); got != want { + t.Errorf("%+v shows %q, want %q", k, got, want) + } + } +} + // Hints follow their text, three columns on, not the screen's far edge: the // keys after the status, the bar's verbs after the pipelines func TestHintsFollowTheirText(t *testing.T) {