lixen/logwisp
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.34.0 doc sweep and update

Browse Source
This commit is contained in:
lixen committed 2026-10-11 03:27:10 -04:00
1 parent 87f5576ccf
commit 1abffcec6c
41 files changed
+1447 -2287
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/package.yml
+8 -5
View File
@@ -66,14 +66,15 @@ jobs:
test ! -e /etc/logwisp/logwisp.toml
lw config init
lw --check
lw --dump | grep -q 'id = "default_source"'
lw --dump | grep -q '^status_reporter = true'
if lw config init; then exit 1; fi
make uninstall PREFIX="$HOME/.local"
test ! -e "$HOME/.local/bin/lw"
test -f "$HOME/.config/logwisp/logwisp.toml"
# make deb on a host with systemd: the service runs from the package's
# configuration and survives a reinstall that keeps an edited one; remove
# stops and disables it and leaves the conffile, purge removes that too
# make deb on a host with systemd: the service runs once the package's
# configuration, which ships no pipeline, gets one, and survives a reinstall
# that keeps it; remove stops and disables it and leaves the conffile, purge
# removes that too
deb:
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -87,6 +88,7 @@ jobs:
sudo apt-get install -y ./bin/logwisp_*.deb
lw --version
test -f /usr/share/zsh/vendor-completions/_lw
printf '[[pipelines]]\nname = "ci"\n[[pipelines.plugin_sources]]\nid = "in"\ntype = "null"\n[[pipelines.plugin_sinks]]\nid = "out"\ntype = "null"\n' | sudo tee -a /etc/logwisp/logwisp.toml
sudo systemctl enable --now logwisp
sleep 3
systemctl is-active logwisp || { sudo journalctl -u logwisp --no-pager; exit 1; }
@@ -104,7 +106,7 @@ jobs:
sudo apt-get purge -y logwisp
test ! -e /etc/logwisp/logwisp.toml
# BSD make on FreeBSD: build, install twice keeping an edited configuration,
# run the rc.d service, uninstall
# run the rc.d service once the configuration has a pipeline, uninstall
freebsd:
runs-on: ubuntu-latest
timeout-minutes: 20
@@ -125,6 +127,7 @@ jobs:
lw --version
pw groupadd logwisp
pw useradd logwisp -g logwisp -d /nonexistent -s /usr/sbin/nologin
printf '[[pipelines]]\nname = "ci"\n[[pipelines.plugin_sources]]\nid = "in"\ntype = "null"\n[[pipelines.plugin_sinks]]\nid = "out"\ntype = "null"\n' >> /usr/local/etc/logwisp/logwisp.toml
service logwisp onestart
sleep 3
service logwisp onestatus
Makefile
+2 -2
View File
@@ -46,7 +46,7 @@ IMAGE_RUN = $(CONTAINER_ENGINE) run --read-only --user 65532:65532 \
--network none --cap-drop ALL --security-opt no-new-privileges
DESTDIR ?=
# SERVICE=no installs lw alone: no service files and no system configuration,
# SERVICE=no leaves out the service files and the system configuration,
# which lw config init then places where the user wants it.
SERVICE ?= yes
INSTALL_OS_DEFAULT != uname -s
@@ -234,7 +234,7 @@ install:
put 0644 $(PKG_DIR)/completion/_lw "$$root$(ZSHCOMPDIR)/_lw"; \
put 0644 $(PKG_DIR)/completion/lw.fish "$$root$(FISHCOMPDIR)/lw.fish"; \
put 0644 LICENSE "$$root$(LICENSEDIR)/LICENSE"; \
for src in README.md doc/*.md; do put 0644 "$$src" "$$root$(DOCDIR)/$$src"; done; \
for src in README.md logwisp-logo.svg doc/*.md doc/lw.1 config/logwisp.toml; do put 0644 "$$src" "$$root$(DOCDIR)/$$src"; done; \
if [ '$(SERVICE)' = no ]; then echo "skip the service and $$etc (SERVICE=no): lw config init writes a configuration"; exit 0; fi; \
if [ -e "$$etc/logwisp.toml" ]; then echo "keep $$etc/logwisp.toml"; \
else put 0644 config/logwisp.toml "$$etc/logwisp.toml"; fi; \
README.md
+24 -20
View File
@@ -40,16 +40,18 @@ streams, or downstream LogWisp nodes.
### Inputs
`file` (directory tail with rotation detection and JSON line parsing),
`console` (stdin, one reader per process), `random` (synthetic generator),
`null`, and the chain ingest listeners `tcp_chain` and `http_chain`.
`console` (stdin, one reader per process), `random` (generated entries, to
check a deployment before real logs), `null` (nothing; holds a source's place
until one is chosen), and the chain ingest listeners `tcp_chain` and
`http_chain`.
### Outputs
`console` (level names in color and control characters escaped on a
terminal), `file` (rotating with
retention), `http` (Server-Sent Events, a JSON status endpoint and a browser
viewer), `tcp` (broadcast server), `null`, and the chain
forwarders `tcp_chain` and `http_chain`.
viewer), `tcp` (broadcast server), `null` (discards; holds a sink's place
until one is chosen), and the chain forwarders `tcp_chain` and `http_chain`.
### Processing
@@ -87,13 +89,14 @@ exponential backoff and jitter.
- Node binding: a chain source can label entries from the sender's certificate
identity or username instead of from what the sender claims, so origin
attribution is not forgeable
- Fail-closed configuration: unknown keys are rejected, and startup warns about
expiring certificates, disabled verification, unanchored allow patterns and
world-readable secrets
- Address rules: an `acl` block admits or refuses peers by address, or by the
client an L4 proxy's PROXY header names, and limits each client
- Fail-closed configuration: every option is validated when the file loads,
unknown keys included, and startup warns about expiring certificates,
disabled verification, unanchored allow patterns and world-readable secrets
See [Security](doc/security.md) for configuration and the exact boundary, and
the [mTLS](doc/mtls-auth-plan.md) and [SCRAM](doc/scram-auth-plan.md)
authentication designs for the rationale and what is deliberately left out.
[Current State](doc/security.md#current-state) for what is left out.
## Documentation
@@ -102,7 +105,7 @@ authentication designs for the rationale and what is deliberately left out.
- [Deployment](doc/deployment.md): `deploy/lw-deploy.sh` for edges,
aggregators, containers, services, jails
- [Architecture](doc/architecture.md): component model, data flow,
concurrency, back-pressure
concurrency, back-pressure, the configuration engine's WebAssembly API
- [Configuration](doc/configuration.md): TOML structure, precedence,
environment and CLI overrides
- [Sources](doc/sources.md) and [Sinks](doc/sinks.md): every plugin and its
@@ -111,15 +114,16 @@ authentication designs for the rationale and what is deliberately left out.
matching, output shaping and sanitization
- [Chaining](doc/chaining.md): multi-node topologies and the wire protocol
- [Networking](doc/networking.md): listeners, dialers, timeouts, limits
- [Security](doc/security.md): TLS, mTLS, peer authorization, threat model
- Design notes: [mTLS](doc/mtls-auth-plan.md) and
[password](doc/scram-auth-plan.md) authentication
- [CLI](doc/cli.md): filter use, flags, pipeline specifications, signals,
exit codes, `lw --tui`, `lw auth`; also the `lw(1)` manual,
- [Security](doc/security.md): TLS, mTLS, password (SCRAM) authentication,
address ACLs, certificates made at startup; threat model and current limits
- [CLI](doc/cli.md): filter use, flags, pipeline specifications, presets,
environment, signals, exit codes, `lw --tui`, `--schema`, `lw auth`,
`lw tls`, `lw preset`, `lw config`; also the `lw(1)` manual,
[`doc/lw.1`](doc/lw.1)
- [Operations](doc/operations.md): running, monitoring, tuning,
troubleshooting
- [To Do](doc/todo.md): planned work: packaging, `lw --tui` and its shared configuration engine
- [To Do](doc/todo.md): the maintainer's release and packaging steps, and
what `lw --tui` and the schema leave for later
A fully annotated configuration covering every option ships as
[`config/logwisp.toml`](config/logwisp.toml).
@@ -191,10 +195,10 @@ docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
`sudo make install` installs the binary, the manual, shell completion, a
systemd unit or FreeBSD rc.d script and its configuration; `make deb` and
`make arch` package them for Debian, Ubuntu and Arch. Without a service,
`make install SERVICE=no PREFIX="$HOME/.local"` installs `lw` alone and
`lw config init` writes the annotated configuration it reads. See
[Installation](doc/installation.md).
`make arch` package them for Debian, Ubuntu and Arch.
`make install SERVICE=no PREFIX="$HOME/.local"` installs `lw`, its manual,
completion and documentation, and no service; `lw config init` writes the
annotated configuration it reads. See [Installation](doc/installation.md).
## System Requirements
cmd/lw/cli.go
+1 -1
View File
@@ -198,7 +198,7 @@ func parseCommandLine(argv []string) (inv invocation, err error) {
}
specFlags, settings := config.SpecFlags(), config.Settings()
unexpected := func(word string) error {
return fmt.Errorf("unexpected argument %q: lw takes options only (a configuration file is -c FILE, a switch's value follows '=')", word)
return fmt.Errorf("unexpected argument %q: lw takes options only (a configuration file is -c FILE, a boolean key's value follows '=')", word)
}
next := func(i *int) string {
if *i+1 < len(argv) && !strings.HasPrefix(argv[*i+1], "-") {
cmd/lw/help.go
+7 -5
View File
@@ -26,8 +26,9 @@ Usage:
Commands (lw COMMAND -h lists its subcommands):
%s
Options: a short option is its long one; a value follows as the next argument
or after '=' (-c FILE, --config=FILE), one that starts with '-' or a switch's
after '='; short options do not combine. lw takes options only.
or after '=' (-c FILE, --config=FILE), one starting with '-' after '='; a
BOOL key's only after '=', and -t, --dump, -T, -V, --schema take none.
Short options do not combine. lw takes options only.
-c, --config FILE Configuration file (default:
~/.config/logwisp/logwisp.toml if it exists,
else ./logwisp.toml)
@@ -82,9 +83,10 @@ key alone means true:
Pipelines (they replace the configuration file's pipelines):
-p, --preset NAME[:KEY=VALUE,...]
Start a pipeline with a preset: pipe, tail,
serve, edge, aggregator; lw preset NAME -h
lists its keys, lw preset NAME prints it
Begin a pipeline with a preset (a second
needs its own --pipeline): pipe, tail, serve,
edge, aggregator; lw preset NAME -h lists
its keys, lw preset NAME prints it
--pipeline NAME Start a pipeline named NAME, which the
pipeline options after it configure; those
before any --pipeline make one named after
cmd/lw/reload_test.go
+4 -15
View File
@@ -39,26 +39,15 @@ func TestWatchErrorsDoNotTriggerReloadAndChangesAreCoalesced(t *testing.T) {
}
}
func TestShippedConfigurationBuildsWithCheckedPluginDecoder(t *testing.T) {
testLogger(t)
// The shipped file defines no pipeline, so lw keeps its built-in pipe
func TestShippedConfigurationKeepsTheBuiltInPipeline(t *testing.T) {
path, err := filepath.Abs(filepath.Join("..", "..", "config", "logwisp.toml"))
if err != nil {
t.Fatal(err)
}
cfg := loadTestConfig(t, path)
if len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "default" {
t.Fatalf("sample array-of-tables decoded incorrectly: %+v", cfg.Pipelines)
if cfg := loadTestConfig(t, path); len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "pipe" {
t.Fatalf("the sample defines pipelines: %+v", cfg.Pipelines)
}
svc, err := bootstrapService(context.Background(), cfg, nil)
if err != nil {
t.Fatal(err)
}
defer svc.Shutdown()
pl, err := svc.GetPipeline("default")
if err != nil || len(pl.Sources) != 1 || len(pl.Sinks) != 1 {
t.Fatalf("sample plugins failed to build: %v", err)
}
svc.Shutdown() // A later reload failure or process exit can close it again.
}
// Once the first service is built a reload reads no descriptor, pipe or
cmd/lwconf/lwconf.js
+1 -1
View File
@@ -55,7 +55,7 @@ export function fields(schema, role, type) {
const rows = (keys, prefix) => keys.map(key => {
const path = prefix + key.key;
if (Array.isArray(key.keys)) return { path, key, rows: rows(key.keys, path + ".") };
return { path, key, placeholder: key.zero ? `(${key.zero})` : (key.default ?? key.hint ?? "") };
return { path, key, placeholder: key.default ?? (key.zero ? `(${key.zero})` : (key.hint ?? "")) };
});
return rows(plugin?.keys ?? schema.flow.find(s => s.key === role)?.keys ?? [], "");
}
cmd/lwconf/lwconf.test.mjs
+1
View File
@@ -152,6 +152,7 @@ test("fields are a node's keys, its tables' of the plugin's side as groups", ()
assert.deepEqual(rows.find(r => r.path === "port").key.max, 65535);
assert.equal(fields(schema, "rate_limit").find(r => r.path === "entries_per_second").placeholder, "(off)");
assert.equal(fields(schema, "heartbeat").find(r => r.path === "interval_ms").placeholder, "1000");
assert.equal(fields(schema, "sink", "file").find(r => r.path === "retention_hours").placeholder, "168");
});
test("a refused call answers why and where, and the engine keeps answering", () => {
config/logwisp.toml
+316 -273
View File
@@ -1,310 +1,333 @@
###############################################################################
### LogWisp Configuration
### Default location: ~/.config/logwisp/logwisp.toml
### Precedence: CLI flags > Environment > File > Defaults
### LogWisp configuration
### lw reads -c FILE, else LOGWISP_CONFIG_FILE / LOGWISP_CONFIG_DIR, else
### ~/.config/logwisp/logwisp.toml if it exists, else ./logwisp.toml
### Precedence: flags > environment > file > built-in defaults; pipeline flags
### or variables replace the file's pipelines whole
###
### Commented values are the built-in defaults unless marked "example".
### Uncommenting a default is a no-op.
### Every key is listed with its built-in default, commented: uncommenting a
### default is a no-op. Values marked "example" are not defaults, and keys
### marked "Required" have none. As shipped the file defines no pipeline, so
### lw runs its built-in one, stdin to stdout.
###
### NOTE: http/tcp ingest sources and http_client/tcp_client sinks are not
### available in this version. TLS is available on every network source and
### sink (the [...tls] blocks below), the [...auth] blocks authenticate peers by
### certificate (mtls) or password (scram; credentials via `lw auth`), and the
### listeners' [...acl] blocks admit peers by address, or by the client an L4
### proxy's PROXY header names, and limit each client. Unknown keys anywhere in
### this file or in any plugin config are rejected at startup.
### Every setting is also a flag (--logging.level=debug) and a variable
### (LOGWISP_LOGGING_LEVEL=debug). A pipeline is set whole by pipeline flags
### (--source, --sink, --preset, ...) or variables (LOGWISP_SOURCE, ...), never
### key by key.
###
### Environment overrides use the LOGWISP_ prefix
### (LOGWISP_QUIET, LOGWISP_LOGGING_LEVEL, ...). Bare names are ignored.
### LOGWISP_CONFIG_FILE and LOGWISP_CONFIG_DIR select the file. Paths such as
### pipelines.0.name cannot be set from the CLI or environment at all.
### TLS is available on every network source and sink (the [...tls] tables),
### [...auth] authenticates peers by certificate (mtls) or password (scram;
### credentials via `lw auth`), and the listeners' [...acl] admits peers by
### address, or by the client an L4 proxy's PROXY header names, and limits
### each client. Unknown keys are refused, except a top-level config_file,
### version, check or dump, which is ignored.
###############################################################################
###############################################################################
### Global Settings
### Settings
###############################################################################
quiet = false # No log of lw's own; pipeline output still flows
status_reporter = true # Periodic status logging (30s, DEBUG level); false without a file
auto_reload = false # Watch file; read at startup (signals always reread it)
color = "auto" # auto|always|never: level names on console sinks; auto on a terminal, NO_COLOR unset
# quiet = false # No log of lw's own; pipeline output still flows
# status_reporter = true # Periodic status logging (30s, DEBUG level); false without a file
# auto_reload = false # Watch file; read at startup (signals always reread it)
# color = "auto" # auto|always|never: level names on console sinks; auto on a terminal, NO_COLOR unset
###############################################################################
### Logging (LogWisp's internal operational logging)
### Logging (lw's own log)
###############################################################################
[logging]
output = "stderr" # file|stdout|stderr|split|all|none; stdout is the console sink's
level = "info" # debug|info|warn|error; warn without a file
# [logging]
# output = "stderr" # file|stdout|stderr|split|all|none; stdout is the console sink's
# level = "info" # debug|info|warn|error; warn without a file
# format = "txt" # raw|txt|json
# sanitization = "" # raw|json|txt|shell (empty = logger default)
# sanitization = "" # raw|json|txt|shell; empty = the log default
# [logging.file] # Used when output is "file" or "all"
# directory = "./log"
# directory = "./log" # lw's alone: the limits below delete any .log file in it
# name = "logwisp"
# max_size_mb = 100
# max_total_size_mb = 1000
# retention_hours = 168.0 # 7 days
# max_total_size_mb = 1000 # past it the oldest .log files there are deleted
# retention_hours = 168.0 # .log files there older than this are deleted; 0 keeps them
## Validated but NOT applied: the console destination comes from `output` above.
# [logging.console]
# target = "stdout" # stdout|stderr|split
# target = "stderr" # stdout|stderr|split
###############################################################################
### Pipelines
### Each pipeline: plugin_sources -> flow (rate_limit|filters|format) -> plugin_sinks
### Names must be unique. 1+ source and 1+ sink required.
### Pipelines: plugin_sources -> flow -> plugin_sinks
### A pipeline has a unique name, one source and one sink at least. The blocks
### below form one pipeline holding every source and sink: uncomment
### [[pipelines]], its name and the blocks it needs.
###############################################################################
[[pipelines]]
name = "default"
# [[pipelines]]
# name = "default" # example
###============================================================================
### Flow (processing between sources and sinks)
### Flow: the stages between sources and sinks, validated whether on or off
###============================================================================
## policy="pass" short-circuits the size check too: enforcing a size cap needs
## entries_per_second > 0 AND policy = "drop".
## policy = "pass" lets every entry through, the size check included: a limit
## needs entries_per_second > 0 and policy = "drop".
# [pipelines.flow.rate_limit]
# entries_per_second = 0.0 # 0 = limiter disabled
# burst_entries = 0.0 # Bucket capacity (0 = entries_per_second)
# entries_per_second = 0.0 # 0 = off
# burst_entries = 0.0 # Bucket capacity; 0 = entries_per_second
# policy = "pass" # pass|drop
# max_entry_size_bytes = 0 # 0 = unlimited
## Filters: sequential include/exclude chain, matched against "<source> <level> <message>"
## Filters run in order, each matching "<source> <level> <message>"
# [[pipelines.flow.filters]]
# type = "include" # include|exclude
# logic = "or" # or|and
# patterns = [".*ERROR.*", ".*WARN.*"] # example; RE2 syntax
# logic = "or" # or|and: any pattern, or all
# patterns = [] # RE2, e.g. [".*ERROR.*", ".*WARN.*"]; none passes every entry
# [pipelines.flow.format]
# type = "raw" # raw|json|txt
# flags = 0 # Formatter flags (0 = defaults per type)
# timestamp_format = "" # Go time layout (formatter default if empty)
# sanitizer_policy = "" # raw|json|txt|shell (defaults per type)
# type = "raw" # raw|txt|json
# flags = 0 # Formatter flags; 0 = by type
# timestamp_format = "" # Go time layout; empty = RFC 3339 with nanoseconds
# sanitizer_policy = "" # raw|json|txt|shell; empty = by type
## Flow-level heartbeat (fan-out to all sinks, traverses chain links)
## Heartbeat: sent to every sink, through chain links
# [pipelines.flow.heartbeat]
# enabled = false
# interval_ms = 1000 # Minimum 100
# include_timestamp = false
# include_stats = false
# format = "txt" # accepted, no effect: heartbeats use flow.format
# interval_ms = 1000 # At least 100
# include_timestamp = false # false: level only; the raw format shows neither
# include_stats = false # Adds type, beat_count and, from the second beat, interval_ms
# format = "txt" # txt|json|raw; no effect: heartbeats use the flow's format
###============================================================================
### TLS (shared shape; applies to every network source and sink)
### TLS (a [...tls] table on every network source and sink)
### Listeners (tcp/http sinks, tcp_chain/http_chain sources) take one of
### cert_file + key_file, self_signed, or issuer_cert_file + issuer_key_file;
### client_auth + client_ca_file make it mTLS. Dialers (tcp_chain/http_chain
### sinks) verify the server by ca_file + server_name, or by pin_sha256, and
### present cert_file + key_file for mTLS. `lw tls ca` / `lw tls cert` make a
### CA and certificates. A table not enabled has no effect; only its keys
### and kinds are checked.
###
### Listeners (tcp/http sinks, tcp_chain/http_chain sources):
### cert_file + key_file, self_signed, or issuer_cert_file + issuer_key_file
### (exactly one); client_auth + client_ca_file enable mTLS.
### Dialers (tcp_chain/http_chain sinks):
### ca_file + server_name, or pin_sha256, verify the server; cert_file +
### key_file present a client identity for mTLS.
### `lw tls ca` / `lw tls cert` make a CA and certificates.
###
### enabled = false Master switch
### cert_file = "" Local certificate
### key_file = "" Private key for cert_file (set together)
### client_auth = false Listener: require and verify a client certificate
### client_ca_file = "" Listener: CA bundle verifying client certs
### ca_file = "" Dialer: CA bundle verifying the server (empty = system)
### server_name = "" Dialer: SNI / name to verify (empty = configured host)
### self_signed = false Listener: make a self-signed certificate at startup;
### it logs the pin_sha256 dialers set (changes on restart)
### issuer_cert_file = "" Listener: make one at startup signed by this CA ...
### issuer_key_file = "" ... and its key (lw tls ca writes both)
### hosts = [] Listener: names/addresses of a made certificate, beyond
### host, os.Hostname(), localhost, 127.0.0.1, ::1
### insecure_skip_verify = false Dialer: disable verification (never in prod)
### pin_sha256 = "" Dialer: sha256//BASE64 of the server key, in place of
### ca_file; ';' between several
### min_version = "1.3" "1.2" or "1.3". No max_version, no cipher_suites.
### enabled master switch
### cert_file, key_file the certificate presented and its key, set together
### self_signed listener: make a self-signed certificate at startup;
### it logs the pin_sha256 dialers set (new each start)
### issuer_cert_file, issuer_key_file
### listener: make one at startup, signed by this CA
### (lw tls ca writes both)
### hosts listener: names/addresses of a made certificate,
### beyond host, os.Hostname(), localhost, 127.0.0.1, ::1
### client_auth listener: require and verify a client certificate
### client_ca_file listener: CA bundle verifying client certificates
### ca_file dialer: server CA bundle; empty = the system roots
### server_name dialer: name to verify; empty = the configured host
### insecure_skip_verify dialer: verify nothing (never in production)
### pin_sha256 dialer: sha256//BASE64 of the server key, in place of
### ca_file; ';' between several
### min_version "1.2" or "1.3"; no max_version, no cipher_suites
###
### TLS alone is a CA membership check: ANY certificate the CA signed is
### accepted. Add an [...auth] block to decide WHICH of them may connect.
### accepted. An [...auth] table decides WHICH of them may connect.
###============================================================================
###============================================================================
### AUTH (shared shape; sits beside [...tls] on every network source and sink)
###
### Both methods REQUIRE tls.enabled; dialers forbid tls.insecure_skip_verify.
### On the http sink the block gates stream_path AND status_path.
### Listeners (tcp/http sinks, tcp_chain/http_chain sources):
### AUTH (an [...auth] table beside [...tls] on every network source and sink)
### Both methods REQUIRE tls.enabled, except scram behind trusted_proxies on
### the http sink; dialers forbid tls.insecure_skip_verify. A table naming
### peers or credentials with type = "none" is refused.
### Listeners (tcp/http sinks, tcp_chain/http_chain sources):
### mtls authorizes the client certificate; REQUIRES tls.client_auth.
### scram admits the users in credentials_file (`lw auth add-user`), or
### the one username whose password is in password_file.
### Dialers (tcp_chain/http_chain sinks):
### Dialers (tcp_chain/http_chain sinks):
### mtls pins the server identity.
### scram logs in as username with the password in password_file.
### Chain sources additionally bind the node label to the identity or username.
### A block naming peers or credentials with type = "none" is refused.
###
### type = "none" none | mtls | scram
### identity = "cn" cn | san_dns | san_uri | san_email. Unset under scram;
### set on a scram listener (needs tls.client_auth), the
### certificate's field must equal the username.
### allow = [] mtls: exact identities. Empty allow AND allow_patterns
### admits any identity the CA vouches for (logged WARN).
### allow_patterns = [] mtls: RE2 patterns; anchor them yourself (^...$)
### node_binding = "" Chain sources only, default "force" with mtls or scram:
### none - trust_node governs, as before
### assert - declared label must equal the identity;
### per-entry node labels still follow trust_node
### force - label AND every entry take the identity
### Overrides trust_node. See doc/security.md.
### credentials_file = "" scram listeners: verifiers; read at (re)load, not watched
### token_lifetime_ms = 0 scram, http sink and http_chain source: 10 s to 24 h, 0 = 15 minutes
### username = "" scram dialers, or a listener's one user
### password_file = "" scram, with username: one trailing line break is
### trimmed; /dev/fd/N or a pipe is read once and kept
### for reloads, /dev/tty asks once
### trusted_proxies = [] scram http sink: TLS-terminating reverse proxies
### (addresses or CIDRs); browsers log in through them
### type none | mtls | scram
### identity cn | san_dns | san_uri | san_email; unset = cn under
### mtls. Set on a scram listener, it binds the client
### certificate to the user (needs tls.client_auth)
### allow mtls: exact identities. Empty allow AND
### allow_patterns admits any identity the CA vouches
### for (logged WARN)
### allow_patterns mtls: RE2 patterns; anchor them yourself (^...$)
### node_binding chain sources: binds the node label to the identity
### or username, overriding trust_node; unset = force
### none - trust_node governs
### assert - declared label must equal the identity;
### per-entry node labels follow trust_node
### force - label AND every entry take the identity
### credentials_file scram listener: verifiers, read at (re)load only
### token_lifetime_ms scram, HTTP listeners: 10 s to 24 h; 0 = 15 minutes
### username scram: a dialer's user, or a listener's one user
### password_file scram, with username: one trailing line break is
### trimmed; /dev/fd/N or a pipe is read once and kept
### for reloads, /dev/tty asks once
### trusted_proxies scram, http sink: TLS-terminating reverse proxies
### (addresses or CIDRs); browsers log in through them
###============================================================================
###============================================================================
### ACL (beside [...tls] and [...auth] on the listeners; dialers have none)
###
### ACL (an [...acl] table beside [...tls] and [...auth] on the listeners)
### Applied as a connection is accepted, before TLS: a refused peer is closed
### at once and counted in acl_denied (acl_limited past a limit), with a WARN
### at most once a minute.
### allow = [] addresses or CIDRs of the listener's family (either behind
### proxy_from or trusted_proxies); empty = all
### deny = [] refused, listed in allow or not
### proxy_protocol = "off" off | optional | required: PROXY v1/v2 headers,
### read before TLS; the rules then see the client.
### optional lets a headerless stream pass as the
### proxy: use it only while rolling headers out
### proxy_from = [] the proxies that may send one; a header from any
### other peer is refused
### max_connections_per_client = 0 per client (an IPv6 one by its /64);
### refused behind trusted_proxies
### requests_per_second_per_client = 0 HTTP listeners, /auth aside: past it,
### 429; behind trusted_proxies, per
### forwarded client
###
### allow addresses or CIDRs of the listener's family (either
### behind proxy_from or trusted_proxies); empty = all
### deny refused, listed in allow or not
### proxy_protocol off | optional | required: PROXY v1/v2 headers, read
### before TLS; the rules then see the client. optional
### lets a headerless stream pass as the proxy: use it
### only while rolling headers out
### proxy_from the proxies that may send one; a header from any
### other peer is refused
### max_connections_per_client
### per client (an IPv6 one by its /64); 0 = unlimited;
### refused behind trusted_proxies
### requests_per_second_per_client
### HTTP listeners, /auth aside: past it, 429; behind
### trusted_proxies, per forwarded client; 0 = unlimited
###============================================================================
###============================================================================
### Sources (1+ required)
### Sources (one at least; ids are examples, unique among a pipeline's sources)
###============================================================================
## Null source (testing)
## Null source: nothing; holds a source's place until one is chosen
# [[pipelines.plugin_sources]]
# id = "null_in"
# type = "null"
[[pipelines.plugin_sources]]
id = "default_source"
type = "file"
[pipelines.plugin_sources.config]
directory = "./" # Directory to monitor (required, not recursive)
pattern = "*.log" # Glob pattern (* and ? only)
## Tailing an already-open file polls at a fixed 100ms, regardless of this value.
check_interval_ms = 100 # Directory rescan interval (min 10)
## raw = true never parses a line; with format type "raw" the file is relayed byte for byte.
raw = false # Keep the whole line as the message
from = "end" # "end" or "start" of a newly discovered file
## File source: follows the files of a directory that match a pattern
# [[pipelines.plugin_sources]]
# id = "file_in"
# type = "file"
# [pipelines.plugin_sources.config]
# directory = "/var/log/app" # Required (example); not recursive
# pattern = "*" # Glob of the whole file name (* and ? only)
# check_interval_ms = 100 # Directory rescan, at least 10; open files are polled every 100 ms
## raw = true never parses a line; with format type "raw" each line is relayed
## as written, except that empty lines are skipped and CRLF becomes LF.
# raw = false
# from = "end" # end|start: where a newly found file is read from
## Console source (stdin; one in the whole configuration, ends lw's pipeline at end of input)
## Console source (stdin; one in the whole configuration). lw exits at end of
## input only when it is the configuration's only source.
# [[pipelines.plugin_sources]]
# id = "console_in"
# type = "console"
# [pipelines.plugin_sources.config]
# buffer_size = 1000
## Random source (testing; special=true exercises sanitizer policies)
## Random source: generated entries, to check a deployment before real logs
## (paths, permissions, sinks, chains; special = true for sanitizer policies)
# [[pipelines.plugin_sources]]
# id = "random_in"
# type = "random"
# [pipelines.plugin_sources.config]
# interval_ms = 500
# jitter_ms = 0 # Clamped to interval_ms
# jitter_ms = 0 # The interval varies by it, half each way; at most interval_ms
# format = "txt" # raw|txt|json
# length = 20
# special = false
# length = 20 # Bytes of random text
# special = false # Mix in control and non-ASCII bytes
## TCP chain source (stdlib listener; receives NDJSON from upstream tcp_chain sinks)
## Topology: jail [file source -> tcp_chain sink] -> host [tcp_chain source -> aggregate sink]
## TCP chain source (receives NDJSON from upstream tcp_chain sinks)
## Topology: edge [file source -> tcp_chain sink] -> aggregator [tcp_chain source -> sinks]
# [[pipelines.plugin_sources]]
# id = "chain_in"
# type = "tcp_chain"
# [pipelines.plugin_sources.config]
# host = "0.0.0.0" # IPv4 only; "::" or "::1": IPv6 only
# port = 9440 # Required
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only
# port = 9440 # Required (example)
# buffer_size = 1000
# max_connections = 0 # 0 = unlimited
# read_timeout_ms = 0 # idle deadline, 0 = none
# read_timeout_ms = 0 # Idle deadline; 0 = none
# hello_timeout_ms = 10000 # Protocol preamble deadline
# trust_node = true # false: label entries by remote address
# [pipelines.plugin_sources.config.tls]
# enabled = true # example: mTLS listener
# cert_file = "/etc/logwisp/tls/server.crt"
# key_file = "/etc/logwisp/tls/server.key"
# client_auth = true
# client_ca_file = "/etc/logwisp/tls/client-ca.crt"
# enabled = false
# cert_file = ""
# key_file = ""
# self_signed = false
# issuer_cert_file = ""
# issuer_key_file = ""
# hosts = []
# client_auth = false
# client_ca_file = ""
# min_version = "1.3"
# [pipelines.plugin_sources.config.auth] # authenticate senders
# type = "none" # none | mtls | scram; mtls requires client_auth
# identity = "cn" # mtls default; under scram unset, set binds the cert to the user
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# node_binding = "force" # none | assert | force; overrides trust_node
# type = "none"
# identity = ""
# allow = []
# allow_patterns = []
# node_binding = ""
# credentials_file = "" # scram: written by `lw auth add-user`
# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# username = ""
# password_file = ""
# [pipelines.plugin_sources.config.acl]
# allow = []
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
# max_connections_per_client = 0 # per client at once; 0 = unlimited
# proxy_protocol = "off"
# proxy_from = []
# max_connections_per_client = 0
## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks)
## HTTP chain source (receives NDJSON batches from upstream http_chain sinks)
# [[pipelines.plugin_sources]]
# id = "hchain_in"
# type = "http_chain"
# [pipelines.plugin_sources.config]
# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only)
# port = 9441 # Required
# ingest_path = "/ingest" # Must start with "/"
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only
# port = 9441 # Required (example)
# ingest_path = "/ingest" # Must start with "/"; not "/auth"
# buffer_size = 1000
# max_body_bytes = 8388608 # per-request cap (8 MiB)
# max_body_bytes = 8388608 # Per-request cap (8 MiB)
# read_timeout_ms = 30000
# trust_node = true # false: label entries by remote address
# [pipelines.plugin_sources.config.tls]
# enabled = true # example: mTLS listener
# cert_file = "/etc/logwisp/tls/server.crt"
# key_file = "/etc/logwisp/tls/server.key"
# client_auth = true
# client_ca_file = "/etc/logwisp/tls/client-ca.crt"
# enabled = false
# cert_file = ""
# key_file = ""
# self_signed = false
# issuer_cert_file = ""
# issuer_key_file = ""
# hosts = []
# client_auth = false
# client_ca_file = ""
# min_version = "1.3"
# [pipelines.plugin_sources.config.auth] # authenticate senders
# type = "none" # none | mtls | scram; mtls requires client_auth
# identity = "cn" # mtls default; under scram unset, set binds the cert to the user
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# node_binding = "force" # none | assert | force; overrides trust_node
# type = "none"
# identity = ""
# allow = []
# allow_patterns = []
# node_binding = ""
# credentials_file = "" # scram: written by `lw auth add-user`
# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes
# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# token_lifetime_ms = 0
# trusted_proxies = [] # the http sink only
# username = ""
# password_file = ""
# [pipelines.plugin_sources.config.acl]
# allow = []
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
# max_connections_per_client = 0 # per client at once; 0 = unlimited
# requests_per_second_per_client = 0 # per client, a second's worth at once; 0 = unlimited
# proxy_protocol = "off"
# proxy_from = []
# max_connections_per_client = 0
# requests_per_second_per_client = 0
###============================================================================
### Sinks (1+ required, fan-out)
### Sinks (one at least, each getting every entry; ids are examples, unique
### among a pipeline's sinks)
###============================================================================
## Null sink (testing)
## Null sink: discards; holds a sink's place until one is chosen, or isolates a
## source or flow while it is measured
# [[pipelines.plugin_sinks]]
# id = "null_out"
# type = "null"
[[pipelines.plugin_sinks]]
id = "default_sink"
type = "console"
[pipelines.plugin_sinks.config]
target = "stdout" # stdout|stderr ("split" NOT supported)
## Console sink
# [[pipelines.plugin_sinks]]
# id = "console_out"
# type = "console"
# [pipelines.plugin_sinks.config]
# target = "stdout" # stdout|stderr ("split" NOT supported)
# buffer_size = 1000
# escape = "auto" # auto|always|never: control characters as <hex>, auto on a terminal
# color = "auto" # auto|always|never: level names in color (default: top-level color)
@@ -314,146 +337,166 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# id = "file_out"
# type = "file"
# [pipelines.plugin_sinks.config]
# directory = "./logs" # Required
# name = "output" # Required
# directory = "/var/log/logwisp" # Required (example)
# name = "output" # Required (example)
# max_size_mb = 100
# max_total_size_mb = 1000
# min_disk_free_mb = 0 # 0 = no floor (only negatives become 100)
# retention_hours = 168.0
# min_disk_free_mb = 0 # 0 = no floor; a negative value means 100
# retention_hours = 168.0 # .log files there older than this are deleted; 0 keeps them
# buffer_size = 1000
# flush_interval_ms = 100
## HTTP sink (SSE streaming server + JSON status endpoint)
## Without an auth block both endpoints are UNAUTHENTICATED and the stream sends
## Access-Control-Allow-Origin: *. Bind to a trusted interface.
## HTTP sink (server-sent events, a JSON status document and a browser viewer)
## Without an auth table both endpoints are UNAUTHENTICATED and the stream
## sends Access-Control-Allow-Origin: *. Bind to a trusted interface.
# [[pipelines.plugin_sinks]]
# id = "http_out"
# type = "http"
# [pipelines.plugin_sinks.config]
# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only)
# port = 8081 # Required
# stream_path = "/stream" # Exact; starts with "/", clean, none of { } % ? #
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only
# port = 8081 # Required (example)
# stream_path = "/stream" # Exact; starts with "/", clean (one trailing "/" allowed), none of { } % ? #; not /auth or under it
# status_path = "/status" # Same rules; differs from stream_path
# buffer_size = 1000 # Sink input queue
# client_buffer_size = 256 # Per-client send queue
# write_timeout_ms = 0 # Per-event deadline, 0 = none
# write_timeout_ms = 0 # Per-event deadline; 0 = none
# max_connections = 0 # 0 = unlimited
# replay_lines = 0 # Last entries a new stream gets first, 0 = none
# login_page = false # GET /auth/login; scram behind auth.trusted_proxies
# viewer_page = false # GET /auth/view, needs login_page; scram + trusted_proxies only
# (without auth or under mtls the viewer is always on)
# replay_lines = 0 # Last entries a new stream gets first; 0 = none
# login_page = false # GET /auth/login; scram behind auth.trusted_proxies only
# viewer_page = false # GET /auth/view, needs login_page; without auth or under mtls the viewer is always at /
# [pipelines.plugin_sinks.config.tls]
# enabled = true # example
# cert_file = "/etc/logwisp/tls/server.crt"
# key_file = "/etc/logwisp/tls/server.key"
# enabled = false
# cert_file = ""
# key_file = ""
# self_signed = false
# issuer_cert_file = ""
# issuer_key_file = ""
# hosts = []
# client_auth = false
# client_ca_file = ""
# min_version = "1.3"
# [pipelines.plugin_sinks.config.auth] # gates BOTH stream_path and status_path
# type = "none" # none | mtls | scram; browsers: mtls, or scram behind a proxy
# identity = "cn" # mtls default; under scram unset, set binds the cert to the user
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# type = "none" # browsers: mtls, or scram behind a proxy
# identity = ""
# allow = []
# allow_patterns = []
# node_binding = "" # chain sources only
# credentials_file = "" # scram: log in with `lw auth token`
# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes
# token_lifetime_ms = 0
# trusted_proxies = [] # scram: proxies ending the browsers' TLS; tls may then be off
# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# username = ""
# password_file = ""
# [pipelines.plugin_sinks.config.acl]
# allow = []
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
# max_connections_per_client = 0 # per client at once; 0 = unlimited
# requests_per_second_per_client = 0 # per client, a second's worth at once; 0 = unlimited
# proxy_protocol = "off"
# proxy_from = []
# max_connections_per_client = 0
# requests_per_second_per_client = 0
## TCP sink (streaming server)
## TCP sink (serves entries to TCP clients, one per line)
# [[pipelines.plugin_sinks]]
# id = "tcp_out"
# type = "tcp"
# [pipelines.plugin_sinks.config]
# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only)
# port = 9090 # Required
# host = "0.0.0.0" # IPv4 only; "::" for IPv6 only
# port = 9090 # Required (example)
# buffer_size = 1000
# client_buffer_size = 256
# write_timeout_ms = 5000 # Missed deadline disconnects the client
# keep_alive = true
# write_timeout_ms = 5000 # A missed deadline disconnects the client
# keep_alive = true # false: no TCP keep-alive
# keep_alive_period_ms = 30000
# max_connections = 0
# max_connections = 0 # 0 = unlimited
# [pipelines.plugin_sinks.config.tls]
# enabled = true # example
# cert_file = "/etc/logwisp/tls/server.crt"
# key_file = "/etc/logwisp/tls/server.key"
# client_auth = true
# client_ca_file = "/etc/logwisp/tls/client-ca.crt"
# enabled = false
# cert_file = ""
# key_file = ""
# self_signed = false
# issuer_cert_file = ""
# issuer_key_file = ""
# hosts = []
# client_auth = false
# client_ca_file = ""
# min_version = "1.3"
# [pipelines.plugin_sinks.config.auth] # authenticate stream readers
# type = "none" # none | mtls | scram; mtls requires client_auth
# identity = "cn" # mtls default; under scram unset, set binds the cert to the user
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# type = "none"
# identity = ""
# allow = []
# allow_patterns = []
# node_binding = "" # chain sources only
# credentials_file = "" # scram: readers use `lw auth stream`
# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# username = ""
# password_file = ""
# [pipelines.plugin_sinks.config.acl]
# allow = []
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
# max_connections_per_client = 0 # per client at once; 0 = unlimited
# proxy_protocol = "off"
# proxy_from = []
# max_connections_per_client = 0
## TCP chain sink (stdlib client; forwards to downstream tcp_chain source)
## TCP chain sink (forwards to a downstream tcp_chain source)
## Do NOT point a chain sink at a chain source in the SAME pipeline: entries
## loop back in and amplify without bound.
# [[pipelines.plugin_sinks]]
# id = "chain_out"
# type = "tcp_chain"
# [pipelines.plugin_sinks.config]
# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1"
# port = 9440 # Required
# node = "" # origin label, default hostname; preserved across hops
# host = "aggregator.example.org" # Required (example); an IPv6 literal bare, e.g. "2001:db8::1"
# port = 9440 # Required (example)
# node = "" # Origin label, kept across hops; empty = the host name
# buffer_size = 1000
# dial_timeout_ms = 5000
# write_timeout_ms = 5000
# backoff_min_ms = 500
# backoff_max_ms = 30000
# keep_alive = true
# backoff_max_ms = 30000 # Below backoff_min_ms: 30000
# keep_alive = true # false: no TCP keep-alive
# keep_alive_period_ms = 30000
# [pipelines.plugin_sinks.config.tls]
# enabled = true # example: mTLS dialer
# ca_file = "/etc/logwisp/tls/ca.crt"
# enabled = false
# cert_file = ""
# key_file = ""
# ca_file = ""
# server_name = ""
# insecure_skip_verify = false
# cert_file = "/etc/logwisp/tls/client.crt"
# key_file = "/etc/logwisp/tls/client.key"
# pin_sha256 = ""
# min_version = "1.3"
# [pipelines.plugin_sinks.config.auth] # pin the server (mtls) or log in (scram)
# type = "none" # none | mtls | scram; both require tls.enabled
# identity = "cn" # mtls: cn | san_dns | san_uri | san_email
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# username = "" # scram
# password_file = "" # scram
# type = "none"
# identity = "" # mtls only; unset = cn
# allow = []
# allow_patterns = []
# username = ""
# password_file = ""
## HTTP chain sink (stdlib client; batched NDJSON POST to downstream http_chain source)
## HTTP chain sink (posts NDJSON batches to a downstream http_chain source)
# [[pipelines.plugin_sinks]]
# id = "hchain_out"
# type = "http_chain"
# [pipelines.plugin_sinks.config]
# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1"
# port = 9441 # Required
# ingest_path = "/ingest"
# node = "" # origin label, default hostname; preserved across hops
# host = "aggregator.example.org" # Required (example); an IPv6 literal bare, e.g. "2001:db8::1"
# port = 9441 # Required (example)
# ingest_path = "/ingest" # Must start with "/"; not "/auth"
# node = "" # Origin label, kept across hops; empty = the host name
# buffer_size = 1000
# max_batch_count = 100
# max_batch_bytes = 1048576
# flush_interval_ms = 1000
# request_timeout_ms = 10000 # Covers dial + write + response
# request_timeout_ms = 10000 # Covers dial, write and response
# backoff_min_ms = 500
# backoff_max_ms = 30000
# backoff_max_ms = 30000 # Below backoff_min_ms: 30000
# [pipelines.plugin_sinks.config.tls]
# enabled = true # example: mTLS dialer
# ca_file = "/etc/logwisp/tls/ca.crt"
# cert_file = "/etc/logwisp/tls/client.crt"
# key_file = "/etc/logwisp/tls/client.key"
# enabled = false
# cert_file = ""
# key_file = ""
# ca_file = ""
# server_name = ""
# insecure_skip_verify = false
# pin_sha256 = ""
# min_version = "1.3"
# [pipelines.plugin_sinks.config.auth] # pin the server (mtls) or log in (scram)
# type = "none" # none | mtls | scram; both require tls.enabled
# identity = "cn" # mtls: cn | san_dns | san_uri | san_email
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# username = "" # scram
# password_file = "" # scram
# type = "none"
# identity = "" # mtls only; unset = cn
# allow = []
# allow_patterns = []
# username = ""
# password_file = ""
config/sample_test.go
+108
View File
@@ -0,0 +1,108 @@
package config
import (
"cmp"
"fmt"
"strings"
"testing"
schema "github.com/lixenwraith/logwisp/internal/config"
"github.com/lixenwraith/toml"
)
// The sample lists every key of the settings, the flow stages and each
// catalogued plugin, tls, auth and acl included, with the schema's default;
// a line marked example shows another value
func TestSampleListsEveryKeyWithItsDefault(t *testing.T) {
all, shown := uncommented(t, true), uncommented(t, false)
for _, k := range schema.SettingKeys() {
i := strings.LastIndex(k.Name, ".") + 1
at := k.Name[:i]
k.Name = k.Name[i:]
checkKeys(t, at, []schema.Key{k}, table(all, at), table(shown, at))
}
pipeline, pipelineShown := first(all["pipelines"]), first(shown["pipelines"])
for _, stage := range schema.FlowStages() {
checkKeys(t, "flow."+stage.Key+".", stage.Keys, first(table(pipeline, "flow")[stage.Key]), first(table(pipelineShown, "flow")[stage.Key]))
}
for _, p := range schema.Plugins() {
list := "plugin_" + p.Role + "s"
block := ofType(pipeline[list], p.Type)
if block == nil {
t.Errorf("%s %s: no block in the sample", p.Type, p.Role)
continue
}
checkKeys(t, p.Type+" "+p.Role+": ", p.Keys(), table(block, "config"), table(ofType(pipelineShown[list], p.Type), "config"))
}
}
// uncommented is the sample with its commented keys and tables made active
// (prose stays under "##"); without examples it drops the lines marked so
func uncommented(t *testing.T, examples bool) map[string]any {
t.Helper()
var b strings.Builder
for line := range strings.Lines(string(Sample)) {
line = strings.TrimPrefix(line, "# ")
if _, comment, _ := strings.Cut(line, " # "); examples || !strings.Contains(comment, "example") {
b.WriteString(line)
}
}
root, err := toml.NewParser([]byte(b.String())).Parse()
if err != nil {
t.Fatalf("the sample uncommented: %v", err)
}
return root
}
// zeros is how the sample writes a kind's zero, the default of a key without one
var zeros = map[string]string{"integer": "0", "number": "0", "bool": "false", "list": "[]"}
// checkKeys finds each key in all, the sample, and compares the value shown,
// without examples, with the default
func checkKeys(t *testing.T, path string, keys []schema.Key, all, shown map[string]any) {
t.Helper()
for _, k := range keys {
_, ok := all[k.Name]
switch want := cmp.Or(k.Default, zeros[k.Kind]); {
case !ok:
t.Errorf("%s%s: not in the sample", path, k.Name)
case k.Kind == "table":
checkKeys(t, path+k.Name+".", k.Keys, table(all, k.Name), table(shown, k.Name))
default:
if v, ok := shown[k.Name]; ok && fmt.Sprint(v) != want {
t.Errorf("%s%s = %v, the default is %q", path, k.Name, v, want)
}
}
}
}
// table descends a dotted path of tables
func table(m map[string]any, path string) map[string]any {
for key := range strings.SplitSeq(path, ".") {
if key != "" {
m, _ = m[key].(map[string]any)
}
}
return m
}
// first is a table, or the first of an array of tables
func first(v any) map[string]any {
if list, ok := v.([]map[string]any); ok && len(list) > 0 {
return list[0]
}
m, _ := v.(map[string]any)
return m
}
// ofType is the block of an array of plugin tables with the type given
func ofType(v any, typ string) map[string]any {
list, _ := v.([]map[string]any)
for _, m := range list {
if m["type"] == typ {
return m
}
}
return nil
}
deploy/README.md
+1 -17
View File
@@ -12,20 +12,4 @@ deploy/lw-deploy.sh --dry-run ... # print the configuration and commands only
deploy/lw-deploy.sh --yes ... # no questions: defaults, or fail on a missing value
```
- Roles
- `edge`: tails log directories, forwards the entries to an aggregator
- `aggregator`: receives from edges; writes files, serves HTTP and TCP streams
- `standalone`: tails log directories; writes files, serves streams
- Runtimes
- `docker`, `podman`: a hardened container of the `logwisp` image (Linux)
- `native`: systemd on Linux, rc.d on FreeBSD, `--jail NAME` for a jail
- `manual`: the configuration only, for `lw -c`
- Every question has a flag. Before changing anything the script prints its
plan and the command line that repeats it without questions, ready for the
next node. It starts nothing that `lw --check` rejects.
- The native runtimes install the service files of `deploy/package/`.
- Not covered, and printed by the script: filters, rate limits, heartbeat,
several pipelines, proxy mode, ACLs, certificate creation. Edit the generated
`logwisp.toml` for those.
Scenarios and what each runtime writes: [doc/deployment.md](../doc/deployment.md).
Roles, runtimes, scenarios and what is not covered: [doc/deployment.md](../doc/deployment.md).
deploy/lw-deploy.sh
+3 -3
View File
@@ -70,7 +70,7 @@ Outputs (aggregator, standalone)
--tcp-port PORT serve a plain TCP stream
--format raw|txt|json entry format (default txt)
TLS (this script creates no certificates: doc/security.md#enabling-mtls)
TLS (this script creates no certificates: lw tls makes them, doc/cli.md#lw-tls)
--tls, --no-tls TLS on every network link of the node
--cert-file FILE this node's certificate: listeners, and edges under mtls
--key-file FILE its private key
@@ -122,7 +122,7 @@ Not covered by this script; edit the generated configuration instead
- proxy mode: browser logins through a TLS-terminating proxy (trusted_proxies)
- acl on the listeners: address rules, PROXY headers, per-client limits
- mtls identities other than the certificate CN; node_binding other than force
- creating certificates: openssl or your PKI (doc/security.md#enabling-mtls)
- creating certificates: lw tls, openssl or your PKI (doc/cli.md#lw-tls)
- several native instances on one host: use one per host or jail
EOF
}
@@ -612,7 +612,7 @@ resolve() {
[ -z "$SINK_ALLOW" ] || _isa=mtls
[ -z "$ADD_VIEWERS" ] || _isa=scram
if [ "$TLS" = yes ]; then
[ "$INTERACTIVE" = 0 ] || note "Certificates come from openssl or your PKI: doc/security.md#enabling-mtls" >&2
[ "$INTERACTIVE" = 0 ] || note "Certificates come from lw tls, openssl or your PKI: doc/cli.md#lw-tls" >&2
if [ "$ROLE" = edge ]; then
ask_opt CA_FILE --ca-file "CA bundle verifying the aggregator (empty: the system roots)" v_file
else
deploy/package/freebsd/Makefile
+4 -3
View File
@@ -46,8 +46,9 @@ post-install:
${INSTALL_DATA} ${WRKSRC}/config/logwisp.toml ${STAGEDIR}${ETCDIR}/logwisp.toml.sample
post-install-DOCS-on:
${MKDIR} ${STAGEDIR}${DOCSDIR}/doc
${INSTALL_DATA} ${WRKSRC}/README.md ${STAGEDIR}${DOCSDIR}
${INSTALL_DATA} ${WRKSRC}/doc/*.md ${STAGEDIR}${DOCSDIR}/doc
${MKDIR} ${STAGEDIR}${DOCSDIR}/doc ${STAGEDIR}${DOCSDIR}/config
${INSTALL_DATA} ${WRKSRC}/README.md ${WRKSRC}/logwisp-logo.svg ${STAGEDIR}${DOCSDIR}
${INSTALL_DATA} ${WRKSRC}/doc/*.md ${WRKSRC}/doc/lw.1 ${STAGEDIR}${DOCSDIR}/doc
${INSTALL_DATA} ${WRKSRC}/config/logwisp.toml ${STAGEDIR}${DOCSDIR}/config
.include <bsd.port.mk>
doc/README.md
+27 -130
View File
@@ -1,135 +1,32 @@
# LogWisp Documentation
LogWisp is a pipeline-based log transport and processing system written in Go.
It collects log entries from files, stdin, or other LogWisp nodes; rate-limits,
filters, and formats them; and distributes them to files, consoles, live network
streams, or downstream LogWisp nodes.
The [README](../README.md) has LogWisp's features, a quick start and the
system requirements.
## Documentation Map
- [Installation](installation.md): building, installing, services,
the container image, packaging
- [Deployment](deployment.md): `deploy/lw-deploy.sh` for edges,
aggregators, containers, services, jails
- [Architecture](architecture.md): component model, data flow,
concurrency, back-pressure, the configuration engine's WebAssembly API
- [Configuration](configuration.md): TOML structure, precedence,
environment and CLI overrides
- [Sources](sources.md) and [Sinks](sinks.md): every plugin and its
options
- [Filters](filters.md) and [Formatters](formatters.md): pattern
matching, output shaping and sanitization
- [Chaining](chaining.md): multi-node topologies and the wire protocol
- [Networking](networking.md): listeners, dialers, timeouts, limits
- [Security](security.md): TLS, mTLS, password (SCRAM) authentication,
address ACLs, certificates made at startup; threat model and current limits
- [CLI](cli.md): filter use, flags, pipeline specifications, presets,
environment, signals, exit codes, `lw --tui`, `--schema`, `lw auth`,
`lw tls`, `lw preset`, `lw config`; also the `lw(1)` manual,
[`lw.1`](lw.1)
- [Operations](operations.md): running, monitoring, tuning,
troubleshooting
- [To Do](todo.md): the maintainer's release and packaging steps, and
what `lw --tui` and the schema leave for later
- [Installation](installation.md): building, installing, and running as a
service
- [Deployment](deployment.md): `deploy/lw-deploy.sh` for edges, aggregators,
containers, services, jails
- [Architecture](architecture.md): component model, data flow, concurrency,
back-pressure
- [Configuration](configuration.md): TOML structure, precedence, environment
and CLI overrides
- [Sources](sources.md): every input plugin and its options
- [Sinks](sinks.md): every output plugin and its options
- [Filters](filters.md): pattern-based inclusion and exclusion
- [Formatters](formatters.md): output shaping and sanitization
- [Chaining](chaining.md): multi-node topologies and the chain wire protocol
- [Networking](networking.md): listeners, dialers, timeouts, connection limits
- [Security](security.md): TLS, mTLS, and peer authorization; threat model and
current limits
- [mTLS Authentication](mtls-auth-plan.md): design and rationale for
certificate-based authorization
- [Password Authentication](scram-auth-plan.md): design and rationale for
Argon2id-SCRAM authentication; mTLS hardening
- [CLI](cli.md): flags, signals, exit codes, `lw --tui`, `lw auth`; also
the `lw(1)` manual, [`lw.1`](lw.1)
- [Operations](operations.md): running, monitoring, tuning, troubleshooting
- [To Do](todo.md): planned work: packaging, `lw --tui` and its shared configuration engine
A fully annotated configuration covering every option lives at
A fully annotated configuration covering every option ships as
[`config/logwisp.toml`](../config/logwisp.toml).
## Capabilities
### Pipeline
- A Unix filter without configuration: stdin to stdout, line for line
- Independent named pipelines, each `sources → flow → sinks`
- Fan-in (many sources per pipeline) and fan-out (many sinks per pipeline)
- Isolated sinks: a stalled sink drops its own events rather than stall the
pipeline; only the console sink waits, so a filter loses no line
- Hot reload of pipeline configuration via `SIGHUP`/`SIGUSR1` or a file watch
### Inputs
`file` (directory tail with rotation detection), `console` (stdin, one
reader per process),
`random` (synthetic generator), `null`, and the chain ingest listeners
`tcp_chain` and `http_chain`.
### Outputs
`console` (control characters escaped on a terminal), `file` (rotating),
`http` (Server-Sent Events plus a JSON status
endpoint), `tcp` (broadcast server), `null`, and the chain forwarders
`tcp_chain` and `http_chain`.
### Processing
- Token-bucket rate limiting with an optional per-entry size cap
- Chainable include/exclude regex filters with `or`/`and` logic
- `raw`, `txt`, and `json` formatting with selectable sanitizer policies
- Optional flow-level heartbeat entries
### Transport security and authentication
- TLS 1.2/1.3 on every network source and sink, listener and dialer alike
- Mutual TLS: listeners can require and verify client certificates; dialers can
present a client identity
- Authorization by certificate identity, per listener: named peers rather than
everything the CA issued, with the `http` sink's endpoints gated too
- Password authentication (Argon2id-SCRAM) bound to the listener's certificate,
with bearer tokens on HTTP listeners and a `lw auth` CLI, and browser
logins behind a TLS-terminating proxy
- Node binding, so a chain source labels entries from the sender's certificate
identity or username rather than from what the sender claims
See [Security](security.md) for what each layer does and does not give you.
## Quick Start
Without a configuration file `lw` is a filter, exiting 0 at the end of input;
a pipeline without `--source` reads stdin and one without `--sink` writes
stdout:
```bash
lw < app.log > copy.log
tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN
```
As a service, a file holds the pipelines:
```toml
[[pipelines]]
name = "app"
[pipelines.flow.format]
type = "json"
sanitizer_policy = "json"
[[pipelines.plugin_sources]]
id = "app_logs"
type = "file"
[pipelines.plugin_sources.config]
directory = "/var/log/myapp"
pattern = "*.log"
[[pipelines.plugin_sinks]]
id = "stdout"
type = "console"
[pipelines.plugin_sinks.config]
target = "stdout"
```
```bash
lw -c config.toml
```
## System Requirements
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
- **Architecture**: amd64
- **Go**: 1.27.2+ to build from source
Network sources and sinks bind and dial IPv4 or IPv6, each keeping strictly to
the family of its host ([Networking](networking.md#address-family)).
## License
BSD 3-Clause.
doc/architecture.md
+108 -51
View File
@@ -7,27 +7,36 @@ plugins, sessions, TLS, statistics — hangs off that spine.
```
main
└── Service
├── Pipeline "app"
│ ├── Registry instance tracking, single-instance enforcement
│ ├── Session Manager per-pipeline connection/session bookkeeping
│ ├── Sources[] plugin instances, keyed by id
│ ├── Flow
│ │ ├── Rate Limiter optional, token bucket
│ │ ├── Filter Chain optional, ordered
│ │ ├── Formatter raw | txt | json, with sanitizer
│ │ └── Heartbeat optional generator
│ └── Sinks[] plugin instances, keyed by id
├── Pipeline "audit"
│ └── ...
└── Status Reporter optional, 30s interval
├── Service
│ ├── Pipeline "app"
│ │ ├── Registry creates each plugin from its factory
│ │ ├── Session Manager per-pipeline connection/session bookkeeping
│ │ ├── Sources[] plugin instances, keyed by id
│ │ ├── Flow
│ │ │ ├── Rate Limiter optional, token bucket
│ │ │ ├── Filter Chain optional, ordered
│ │ │ ├── Formatter raw | txt | json, with sanitizer
│ │ │ └── Heartbeat optional generator
│ │ └── Sinks[] plugin instances, keyed by id
│ └── Pipeline "audit"
│ └── ...
└── Status Reporter cmd/lw, optional, 30 s: Service.GetGlobalStats
```
Package map:
- `cmd/lw`: entry point, help, logger bootstrap, signal loop, status
reporter.
- `internal/config`: typed config schema, loading, top-level validation.
- `cmd/lw`: entry point: the command line (`cli.go`: lw's flags and the
`auth`, `tls`, `preset` and `config` commands), help, completion, logger
bootstrap, signal and reload loop, status reporter.
- `cmd/lwconf`: `internal/compose` as WebAssembly, and `lwconf.js`, which
loads it: see [Configuration Engine](#configuration-engine).
- `internal/config`: option schema (tags, catalogue) and `Decode`; loading
from file, environment and flags; presets; offline validation of every
setting and plugin option.
- `internal/compose`: the configuration engine `lw --tui` and `cmd/lwconf`
share. It edits pipelines and writes and reads back the command-line,
environment and file forms; it reads no file and links no TLS, HTTP or
terminal code.
- `internal/service`: owns the pipeline set; start, stop, shutdown, global
stats.
- `internal/pipeline`: pipeline runtime and per-pipeline plugin registry.
@@ -41,7 +50,8 @@ Package map:
- `internal/chain`: chain wire protocol: hello preamble, entry codec, backoff.
- `internal/tlsx`: the single seam between `TLSOptions` and `crypto/tls`.
- `internal/authz`: the single seam between `AuthOptions` and the network
plugins: `Admit`, `AuthorizeRequest`, `Greet`, `Prepare`.
plugins: `Admit`, `AuthorizeRequest`, `Greet`, `Prepare`; `Authorize`
refuses under `scram`.
- `internal/netacl`: a listener's `acl` block: address rules, PROXY headers
and per-client limits; its `Table` is SCRAM throttling's limiter too.
- `internal/session`: session manager and per-instance proxy.
@@ -53,15 +63,16 @@ Package map:
- `internal/logline`: reads a line of any common shape into time, level,
tag, message and fields for the live view; holds the one rule for runes a
terminal acts on, which the console sink shares.
- `internal/version`: version, commit and build time, stamped at build.
- `internal/testutil`: fixtures for tests only.
## Plugin Registration
Every plugin registers itself in an `init()` function, and
`cmd/lw/bootstrap.go` blank-imports each package to trigger those
`init()`s. Adding a plugin therefore means writing the package, calling
`plugin.RegisterSource` / `plugin.RegisterSink`, adding one blank import, and
adding the type's row to the catalogue in `internal/config/schema.go`, which
registration requires.
`cmd/lw/bootstrap.go` blank-imports each package to trigger it. Adding a
plugin means writing the package, calling `plugin.RegisterSource` /
`plugin.RegisterSink`, adding one blank import, and adding the type's row to
the catalogue in `internal/config/schema.go`, which registration requires.
The row holds the type's options struct, whose tags declare its defaults and
rules, and whether the type is single. The `console` source is single,
@@ -77,10 +88,11 @@ instance of a single type fails, naming both pipelines.
starts. Publication is non-blocking: a full subscriber channel increments
the source's `dropped_entries` counter. The console source alone waits,
since stdin can simply be read later.
2. **Flow** applies, in order: rate limit → filter chain → formatter. A drop at
any stage ends the entry's life and increments `flow.total_dropped`. The
verdict, passed or the stage that dropped it (a filter by its index), goes
to the pipeline's tap when a viewer set one, as `lw --tui`'s live view does.
2. **Flow** applies, in order: rate limit → filter chain → formatter. A rate
limit or filter drop ends the entry's life and increments
`flow.total_dropped`. The verdict, passed or the stage that dropped it (a
filter by its index), goes to the pipeline's tap when a viewer set one, as
`lw --tui`'s live view does.
3. The formatter output becomes a `core.TransportEvent`, which carries both the
formatted `Payload` and the original structured `Entry`.
4. **Dispatch** sends the event to every sink's input channel with a
@@ -109,8 +121,7 @@ loses nothing. Each stage drops and counts instead of waiting:
- Source → subscriber: a full channel drops the entry; source
`dropped_entries`. The console source waits instead and reads stdin later.
- Flow: the rate limiter, a filter rejection or a format error drops it;
`flow.total_dropped`.
- Flow: the rate limiter or a filter rejection drops it; `flow.total_dropped`.
- Pipeline → sink: a full sink input drops it for that sink only; pipeline
`total_dropped_by_sink`. A sink declaring `core.CapBackpressure`, the console
and null sinks, makes dispatch wait instead, so a filter loses no line to a
@@ -124,11 +135,12 @@ loses nothing. Each stage drops and counts instead of waiting:
the view shows them.
The `tcp_chain` sink is the one deliberate exception. It holds a line across
reconnects until it is written or the process shuts down, so a downstream
outage propagates backwards as a full input buffer and surfaces as
`total_dropped_by_sink` on the pipeline rather than as silent data loss inside
the sink. The `http_chain` sink retries a batch with backoff, and drops it only
on a non-retryable response or on shutdown (`dropped_batches`).
reconnects until it is written or the sink stops (shutdown or reload, after
`sink.FlushBound`), so a downstream outage propagates backwards as a full
input buffer and surfaces as `total_dropped_by_sink` on the pipeline rather
than as silent data loss inside the sink. The `http_chain` sink retries a
batch with backoff, and drops it only on a non-retryable response or on
shutdown (`dropped_batches`).
## Concurrency Model
@@ -138,20 +150,23 @@ on a non-retryable response or on shutdown (`dropped_batches`).
- Each network sink runs one broadcast/broker goroutine plus, per connection, a
writer goroutine (and for TCP, a reader goroutine that exists only to detect
disconnects and refresh session activity).
- Chain sinks run a single run-loop goroutine that exclusively owns the
connection or the pending batch, so no locking is needed around either.
- Chain sinks run one run-loop goroutine that owns the pending line or batch;
the `tcp_chain` sink adds a reader goroutine per link that closes the link
when the source writes anything or closes it.
- Statistics are atomics; configuration and registries use RW mutexes;
shutdown is context cancellation plus wait groups.
### Shutdown ordering
`Pipeline.Stop` is deliberately ordered so in-flight data drains:
`Pipeline.Stop` is ordered so in-flight data drains:
1. Release dispatch waits on backpressure sinks; stop all sources
concurrently, each closing its subscriber channels.
2. Wait for the run loop, which ends when every subscription channel closes.
3. Stop all sinks concurrently; console and file sinks write their queue
first. Then cancel the pipeline context.
3. Stop all sinks concurrently: console and file sinks write their queue;
network sinks deliver theirs within `sink.FlushBound`, their write or
request timeout when set below 2 s, else 2 s. Then cancel the pipeline
context.
A finite source ends on its own: the console source at the end of stdin. The
run loop then returns and the pipeline's `Finished` channel closes; once every
@@ -176,7 +191,7 @@ The network plugins, by role:
- `http_chain` sink: chain protocol, batched POST with retry.
TLS is built in exactly one place, `internal/tlsx`, which exposes
`Server(opts)` for listeners and `Client(opts, host)` for dialers. See
`Server(opts, host)` for listeners and `Client(opts, host)` for dialers. See
[Security](security.md).
## Sessions
@@ -194,18 +209,17 @@ the corresponding SSE client.
Authorization decisions do not read session metadata — `internal/authz` makes
them from the handshake, the SCRAM exchange or a bearer token, at the point of
connection or request, and their outcome is *recorded* in the session. That
ordering matters: a session exists only for a peer that was already admitted.
See the [mTLS](mtls-auth-plan.md) and [SCRAM](scram-auth-plan.md)
authentication designs.
connection or request, and their outcome is *recorded* in the session, so a
session exists only for a peer that was already admitted. See
[Security](security.md#the-auth-block) and
[Chaining](chaining.md#wire-protocol).
## Configuration Reload
Reload (signal or file watch) rebuilds the entire service:
1. Signals reread the selected file; watches consume a newly published snapshot.
The application owns its manager and detached snapshots; no global mutable
target is shared with the running service. Validate every candidate again.
1. Signals (`SIGHUP`, `SIGUSR1`) reread the file; a watch takes the published
snapshot. Every candidate is validated again.
2. Build a **new** service from it. If construction fails, the old service keeps
running untouched.
3. Shut the old service down, start the new one, and restart the status
@@ -222,15 +236,58 @@ certificate rotation. Failed candidates release their session workers. Once the
old service has stopped, a bind/start failure requires another corrected reload
or process restart.
## Configuration Engine
`internal/compose` is what `lw --tui` edits with and what the website runs as
WebAssembly, so no part of LogWisp is rewritten in JavaScript. `make wasm`
writes the files a site drops in
([Installation](installation.md#building-from-source)): `lwconf.wasm` is
7.1 MB, 1.9 MB at gzip -9, and the toolchain's `wasm_exec.js` uses no eval,
so the page needs only `'wasm-unsafe-eval'` in its `script-src`.
`lwconf.js` is one dependency-free ES module that knows the calls and shapes,
never a key:
- `load({execURL, execSRI, wasmURL, wasmSRI})` fetches both by their
integrity and resolves to the engine.
- `fields(schema, role, type)` is a node's form rows, a table's a group.
- `value`, `at` and `group` read a node's value and sort problems by node.
The engine's calls take strings (`lwconf.js` writes objects as JSON) and
answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`, as
far as the engine places the refusal:
- `schema()`: what `lw --schema` prints.
- `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key to
a string, a list key to strings.
- `parse(text, form, dirs)`: the pipelines of a form's pasted text.
- `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled
(defaults, kinds, ids) and the first problem of each part (a value of the
wrong kind too) as `{path, pipeline, node, key, text}`.
- `emit(pipelines, form)`: the text of `command`, `environment` or `file`.
- `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is
`{op, pipeline, ...}`:
- `add_pipeline` and `rename_pipeline` (`name`), `remove_pipeline`
- `add` (`role`, `type`), `retype` (`node`, `type`), `remove` (`node`)
- `move_filter` (`from`, `to`)
- `set` (`node`, `key`, `values`), `unset` (`node`, `key`)
- `dirs` answers whether a path a preset names is a directory,
`{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, one
with a glob a pattern, and any other is refused at `path`.
- Pipelines are a JSON list in the file's shape (`name`, `flow`,
`plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node is
`{role, id, index}`: a source's or sink's id, a filter's index.
## Resource Management
- Every buffer is bounded; the drop-not-block policy keeps memory flat under
load.
- The `tcp` and `http` sinks and the `tcp_chain` source accept a
`max_connections` cap. Admission is a load-then-check, so a burst can
over-admit by roughly one connection.
- `max_connections` caps connections on the `tcp` sink and the `tcp_chain`
source, and streams on the `http` sink. Admission is a load-then-check, so a
burst can over-admit by roughly one.
- Chain listeners bound a single line at `core.MaxLogEntryBytes` (1 MiB); an
oversized line is a protocol violation and terminates the connection.
oversized line terminates a `tcp_chain` link and fails an `http_chain`
batch with `400`.
- The `http_chain` source caps each request body at `max_body_bytes`.
- File sinks rotate on size, cap total rotated size, and honour a retention
window.
doc/chaining.md
+39 -40
View File
@@ -43,13 +43,14 @@ Chained entries carry a `node` label identifying where they originated.
- `false`: always overwrite with the sender's remote address.
Relays preserve `node`, so a label survives any number of hops and identifies
the original producer rather than the last relay.
the original producer, unless a hop's source overwrites it: `trust_node =
false`, or `node_binding = "force"` (the default with an `auth` block).
Formatters write the label as a prefix on the source
([Formatters](formatters.md#node-identity-in-output)).
With an `auth` block the source can instead bind the label to the sender's
authenticated identity — its certificate identity under `mtls`, its username
under `scram`. `assert` and `force` take the connection label over from
`trust_node`; only `force` also overrides the per-entry labels.
`auth.node_binding`:
authenticated identity: its certificate identity under `mtls`, its username
under `scram`. `auth.node_binding`:
- `none`: `trust_node` governs the connection label and the per-entry `node`
field.
@@ -64,10 +65,6 @@ per-entry `node` field. Pick `assert` on a relay-to-relay hop, where the relay
should prove its own identity but the origin labels it forwards must survive.
See [Security](security.md#node-binding).
Formatters render node identity as a syslog-style prefix on the source field:
`edge-01/app.log`. In JSON output the node therefore appears inside the source
field, not as a separate top-level key.
> `trust_node = true` with no `auth` block means any peer that can connect can
> claim **any** node label, including one belonging to another host. On an
> untrusted network set an `auth` block (`mtls` or `scram`) and keep
@@ -96,6 +93,11 @@ A persistent connection carrying newline-delimited JSON.
Line size is bounded at 1 MiB. An oversized line is a protocol violation and
terminates the connection, because the scanner cannot resynchronize afterwards.
The cap applies to the encoded line, envelope and JSON escapes included, so a
message near 1 MiB exceeds it (the console and file sources cut lines at
1 MiB). On `tcp_chain` such an entry ends the link and is lost, together with
any line the sink writes before it notices the close. On `http_chain` it fails
its whole batch with `400`, which the sink drops.
Under `auth.type = "scram"` the hello also carries the SCRAM client-first
message, and three more lines precede the entries, each one JSON object of at
@@ -127,15 +129,19 @@ Batches of NDJSON delivered by `POST`, with the preamble expressed as headers.
- Response header
- `X-Logwisp-Accepted`: the number of entries ingested.
Responses: `204` on success, `400` for a bad protocol version or a malformed
body, `413` when the body cap is exceeded, `405` for a non-`POST` method, `401`
for a missing or expired token, `403` for a refused certificate or node label.
Responses: `204` on success; `400` for a bad protocol version or a malformed
body, a line over 1 MiB included; `413` past `max_body_bytes`; `404` for
another path; `405` for a non-`POST` method; `429` (with `Retry-After`) past
`acl.requests_per_second_per_client`; `401` for a missing, invalid or expired
token; `403` for a refused certificate or node label.
Under `scram` the sink first logs in with two `POST /auth` requests carrying the
same JSON as the TCP lines: the hello (`200` and the challenge), then the proof
(`200` with the final message, `token` and `expires_in` in seconds). A refusal is
`{"error":"…"}`: `401` for a failed proof, `429` when throttled, `503` when the
listener is busy, `400` or `413` for a malformed or oversized body.
Under `scram` the sink first logs in with two `POST /auth` requests, each
`Content-Type: application/json` and carrying the same JSON as the TCP lines:
the hello (`200` and the challenge), then the proof (`200` with the final
message, `token` and `expires_in` in seconds). A refusal is `{"error":"…"}`:
`401` for a failed proof, `429` when throttled, `503` when the listener is
busy, `415` without the JSON content type, `400` or `413` for a malformed or
oversized body.
### Entry encoding
@@ -155,19 +161,16 @@ in at ingest.
## Delivery Semantics
- `tcp_chain`: per line, each held across reconnects.
- On failure it retries with exponential backoff plus ±20 % jitter until
the line is written or the sink shuts down.
- Back-pressure appears upstream as `total_dropped_by_sink`.
- `http_chain`: at least once per batch.
- It retries transport errors, `408`, `429`, `5xx` and, under `scram`, failed
logins and `401`.
- It drops the batch on any other non-2xx, counted in `dropped_batches`.
`http_chain` batches can be delivered twice when a successful request's response
is lost. There is no de-duplication downstream; design your consumers to
tolerate it, or use `tcp_chain` where each line is written once per successful
write.
- `tcp_chain`: per line, held across reconnects until the socket accepts it,
with exponential backoff plus ±20 % jitter. There is no acknowledgement:
lines in flight when a link breaks are lost (at most once).
- `http_chain`: at least once per batch. It retries transport errors, `408`,
`429`, `5xx` and, under `scram`, failed logins and `401`. It drops the batch
on any other non-2xx, counted in `dropped_batches`. A batch can arrive twice
when a successful request's response is lost; there is no de-duplication
downstream.
- Both: while a link is down or a batch retries, the sink's input queue fills
and the pipeline counts drops in `total_dropped_by_sink`.
Neither transport persists anything to disk. Entries buffered in memory during
an outage are lost if the process exits.
@@ -191,7 +194,7 @@ pattern = "*.log"
id = "forward"
type = "tcp_chain"
[pipelines.plugin_sinks.config]
host = "relay.internal"
host = "relay.example.org"
port = 15801
node = "edge-01"
[pipelines.plugin_sinks.config.tls]
@@ -201,7 +204,7 @@ cert_file = "/etc/logwisp/tls/edge-01.crt"
key_file = "/etc/logwisp/tls/edge-01.key"
[pipelines.plugin_sinks.config.auth]
type = "mtls"
allow = ["relay.internal"] # pin the relay, not just its hostname
allow = ["relay.example.org"] # pin the relay, not just its hostname
```
**Relay** — ingest, keep errors only, archive and stream:
@@ -252,27 +255,23 @@ port = 8080
Entries arriving on this relay are labelled `edge-01` or `edge-02` because that
is what their certificates say, regardless of the `node` each edge configured.
`test/mtls-chain-test.sh` builds exactly this shape against a throwaway PKI.
`test/mtls-chain-test.sh` runs a relay of this kind (mTLS ingest with
`node_binding = "force"`) against a throwaway PKI.
To use passwords instead of client certificates, replace both `auth` blocks:
`type = "scram"` with `username` and `password_file` on the edge, and on the
relay a `credentials_file`, or one `username` with its `password_file`.
`client_auth` and the edge's certificate become
optional, and entries are labelled with the username. See
`client_auth` and the edge's certificate become optional, and entries are
labelled with the username. See
[Password Authentication](security.md#password-authentication-scram).
## Operational Notes
- **Formatting is a relay decision.** Because chain links carry structured
entries, the edge node's `flow.format` affects only its own local sinks. Set
the output shape on the node that owns the human-facing sink.
- **Filtering early saves bandwidth.** A filter on the edge drops entries before
they cross the network; a filter on the relay is easier to change centrally.
- **Rate limits are per pipeline.** An edge limit protects the link; a relay
limit protects the relay from a noisy edge.
- **Heartbeats traverse chain links** as ordinary structured entries and keep
otherwise-idle links and their sessions warm.
- **Ports** used by the bundled test scripts: `15801` tcp_chain ingest, `15802`
http_chain ingest, `15803` tcp sink, `15804` http sink.
- **Dial the listener's own literal** when testing locally: a listener on
`127.0.0.1` takes IPv4 only, and `localhost` may resolve to `::1`.
doc/cli.md
+84 -98
View File
@@ -33,11 +33,13 @@ first argument.
one-letter short form that stands for it in every command that has it:
- `-c` `--config`, `-p` `--preset`, `-q` `--quiet`, `-t` `--check`,
`-T` `--tui`, `-V` `--version`, `-h` `--help`
- `-u` `--user` in `lw auth` and `lw preset edge`
- `-u` `--user` in `lw auth` and in `lw preset serve`, `edge` and
`aggregator`
- A value follows as the next argument or after `=` (`-c FILE`, `-c=FILE`,
`--config FILE`, `--config=FILE`); a value that starts with `-` follows `=`,
and `--` ends the options.
- A switch (`--check`, `--quiet`, any boolean key) takes no separate value:
- `--check`, `--dump`, `--tui`, `--version` and `--schema` take no value
(`--check=false` is a usage error); a boolean key takes one only after `=`:
`--quiet=false`, not `--quiet false`.
- Short options do not combine (`-q -t`, not `-qt`) or take an attached value
(`-cFILE`).
@@ -75,8 +77,8 @@ pipelines have [their own flags](#pipelines):
- a key: its kind, default, rules and help; a table (`tls`, `auth`, `acl`)
holds its own keys, those of the plugin's side (listener or dialer),
HTTP's only on a plugin that speaks HTTP. Every listener lists
`auth.node_binding` and `auth.trusted_proxies`, which validation admits
only on chain sources and on the http sink
`auth.node_binding`, which only chain sources use, and every HTTP
listener `auth.trusted_proxies`, which only the `http` sink admits
- a flow stage: its keys, and `on`, the values that turn it on
- a preset key: its default and help, and the choices (`enum`) and `hint`
of the option it sets
@@ -116,19 +118,22 @@ destination is derived from `--logging.output`.
### Pipelines
Pipeline flags define whole pipelines. They replace the configuration file's
pipelines and the built-in default; every other key keeps its precedence.
pipelines and the built-in default, and a reload keeps them; every other key
keeps its [precedence](configuration.md#configuration-precedence).
- `--pipeline NAME` starts a pipeline; specs before the first one go to a
pipeline named `cli`, or after the preset that starts them
- `-p SPEC`, `--preset SPEC` starts a pipeline with a [preset](#presets);
later flags add sources, sinks and filters to it, or replace its format,
rate limit and heartbeat
- `--pipeline NAME` starts a pipeline named NAME; specs before the first one
go to one pipeline, named after its preset, else `cli`
- `-p SPEC`, `--preset SPEC` begins the pipeline with a [preset](#presets),
as its first spec; a second preset needs its own `--pipeline NAME`. Later
flags add sources, sinks and filters, set a rate limit and heartbeat, or
replace its format
- `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable
- `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline
- a pipeline without `--source` reads stdin (console source `stdin`), one
without `--sink` writes stdout (console sink `stdout`)
- stdin has one reader, so the whole configuration holds at most one console
source: `pipeline "b": a console source already reads stdin in pipeline "a"`
source: `pipelines[1].plugin_sources[stdin]: one console source may run,
and pipeline "a" has it`
A SPEC is `TYPE:key=value,...`, or `key=value,...` without a TYPE:
@@ -161,8 +166,8 @@ Errors name the flag and the offending part, and nothing starts:
--source file,directory=/x: TYPE ends at ':', as in file:directory=/x
```
A misspelled plugin key fails when the plugin is built, before any listener
opens: `failed to create sink http: ... unknown key "tls.enabeld"`.
A misspelled plugin key fails validation, before anything is built or opened:
`pipelines[0].plugin_sinks[http].config: unknown key "tls.enabeld"`.
```bash
# tail a directory, serve it over SSE on every interface; a browser at
@@ -189,17 +194,23 @@ Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns).
`--preset NAME:key=value,...` takes the [SPEC](#pipelines) syntax. A list
key (`hosts`, `proxy`, `allow`, `deny`) takes several values: repeated
(`hosts=a,hosts=b`) or `\,`-separated in a SPEC, `,`-separated in `lw preset`
flags and the TUI; any other key takes one.
`lw preset NAME [--key value ...]` prints the pipeline a preset expands to,
ready for a configuration file; its flags are the keys, spelled as in the SPEC
(`--password_file`), and `-u` is `--user`;
`lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An
unknown key fails and lists the valid ones. A `path` is a file, a directory
(its files) or a glob; a preset that takes one reads stdin without it, and
`from` (`end`, or `start`) is where reading a file starts. `user` names one
SCRAM user, an edge's login or a listener's only user in place of `users`;
`password_file` holds its password and defaults to `/dev/tty`, which asks
([password sources](security.md#one-user-without-a-credentials-file)).
flags (a repeated flag keeps only its last value) and the TUI; any other key
takes one. `lw preset NAME [--key value ...]` prints the pipeline a preset
expands to, ready for a configuration file; its flags are the keys, spelled as
in the SPEC (`--password_file`). `lw preset NAME -h` lists its keys and
defaults, `lw preset -h` the presets. An unknown key fails and lists the valid
ones.
- `path`: a file, a directory (its files) or a glob, and this host decides
which: an existing directory is followed whole, any other path is a file
name or glob in its directory; for a directory not here, as in a `--dump`
for another host, write `DIR/*`. Without one, `serve` and `edge` read stdin
- `from` (`end`, or `start`): where reading a file starts
- `user`: one SCRAM user, an edge's login or a listener's only user in place
of `users`; `password_file` holds its password and defaults to `/dev/tty`,
which asks ([password sources](security.md#one-user-without-a-credentials-file))
The presets, each with its keys:
- `pipe`: stdin to stdout, the built-in default
- `format` (`raw`)
@@ -242,7 +253,7 @@ lw --preset tail:path=/var/log/app
lw --preset serve:path=/var/log/app
# serve it over HTTPS, self-signed, to SCRAM users; keep the result as a file
lw --preset serve:path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml
lw --preset 'serve:path=/var/log/app/*,tls=self,users=/etc/logwisp/users.toml' --dump > serve.toml
# an aggregator with a certificate made at startup, and an edge pinning it
lw --preset aggregator:users=/etc/logwisp/users.toml,out=/var/log/edges
@@ -264,8 +275,10 @@ changes when lw restarts; a fleet signs with an issuer instead, and edges set
## Environment Variables
Configuration paths map to environment variables by replacing `.` with `_`,
uppercasing and adding `LOGWISP_`:
Every setting is also a variable: its path with `.` as `_`, uppercased, after
`LOGWISP_`. An empty variable, a setting's or a pipeline's, counts as unset.
They rank below flags and above the file
([precedence](configuration.md#configuration-precedence)).
```bash
export LOGWISP_QUIET=true
@@ -273,10 +286,7 @@ export LOGWISP_LOGGING_LEVEL=debug
export LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
```
Bare names used by older versions are now ignored. Rename them to the prefixed
forms when upgrading; see [Configuration](configuration.md#environment-variables).
The path resolver reads these variables directly:
The path resolver reads two more:
- `LOGWISP_CONFIG_FILE`: the configuration file path, joined onto
`LOGWISP_CONFIG_DIR` when both are set
@@ -290,14 +300,16 @@ Any pipeline flag on the command line makes LogWisp ignore all of them.
- `LOGWISP_PIPELINE`: the pipeline's name, default the `LOGWISP_PRESET` name,
else `cli`
- `LOGWISP_PRESET`: the [preset](#presets) that starts it
- `LOGWISP_PRESET`: the [preset](#presets) that begins it
- `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each
- `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered
one in numeric order; likewise `LOGWISP_SINK_N` and `LOGWISP_FILTER_N`
- `LOGWISP_FORMAT`, `LOGWISP_RATE_LIMIT`, `LOGWISP_HEARTBEAT`
- an empty variable counts as unset
A container needs no configuration file:
A container needs no configuration file. Without one lw logs less
([Built-in Defaults](#built-in-defaults)), so pass `LOGWISP_LOGGING_LEVEL=info`
and `LOGWISP_STATUS_REPORTER=true`; the image sets neither, as a variable would
override a mounted file's value:
```bash
docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \
@@ -306,45 +318,21 @@ docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \
-e LOGWISP_FILTER='exclude:patterns=DEBUG' \
-e LOGWISP_SINK='http:host=0.0.0.0,port=8080' \
-e LOGWISP_SINK_1=console \
-e LOGWISP_LOGGING_LEVEL=info \
logwisp
-e LOGWISP_LOGGING_LEVEL=info -e LOGWISP_STATUS_REPORTER=true \
logwisp:dev
```
Without a file, lw logs at `warn` with no status reporter, so a container
configured through the environment passes `LOGWISP_LOGGING_LEVEL=info` (and
`LOGWISP_STATUS_REPORTER=true`) to keep the service defaults. The image does
not set them: environment variables would override a mounted file's values.
## Precedence
- Pipelines
- pipeline flags, if any are given
- else pipeline variables, if any are set
- else the configuration file's `[[pipelines]]`
- else the built-in `pipe` ([Built-in Defaults](#built-in-defaults))
- Every other key
1. command-line flags
2. environment variables
3. configuration file
4. built-in defaults; `logging.level` and `status_reporter` depend on
whether a file was found
A reload rereads the file and keeps the command-line or environment pipelines.
## Signals
- `SIGINT`, `SIGTERM`: graceful shutdown
- `SIGHUP`, `SIGUSR1`: reload the configuration
- `SIGHUP`, `SIGUSR1`: reload: the file is reread and the whole service
rebuilt; a configuration error leaves the running service untouched
([Hot Reload](configuration.md#hot-reload))
- `SIGHUP` before the signal handler is installed is ignored, so LogWisp
survives a terminal hang-up like `nohup`
- On the `lw --tui` screen, `SIGTERM` and `SIGHUP` close it, stopping what
`t` runs, and lw exits 0
`SIGHUP` is ignored during startup, before the signal handler is installed, so
LogWisp survives a terminal hang-up like `nohup`. Once running, it triggers a
reload rather than terminating.
Reload rebuilds the whole service. A configuration error leaves the running
service untouched; see [Configuration](configuration.md#hot-reload).
## Exit Codes
- `0`: the end of input, a clean shutdown (`SIGINT`, `SIGTERM`), `--version` /
@@ -353,15 +341,12 @@ service untouched; see [Configuration](configuration.md#hot-reload).
included, and a bad value on the command line), a logger init failure, a
service bootstrap failure
- `2`: a usage error (an unknown option, a word no option takes, a missing
value or `-c` path), or an explicitly requested configuration file that is
not found
value or `-c` path), or a configuration file named by `-c`, `--config=` or
a `LOGWISP_CONFIG_*` variable that does not exist
([File Location](configuration.md#file-location))
- killed by `SIGPIPE` (shell status `141`): the reader of stdout went away, as
in `lw | head`; `cat` ends the same way
A missing file is exit code 2 only when it was named explicitly (`-c`,
`--config=`, or the `LOGWISP_CONFIG_*` variables). A missing discovered default
is not an error, and LogWisp starts on built-in defaults.
## Built-in Defaults
Without a configuration file and without pipeline flags or variables, lw runs
@@ -417,9 +402,6 @@ journalctl -f | lw --sink http:host=127.0.0.1,port=8080
lw --source file:directory=/var/log/app,pattern='*.log' > all.log
```
A console sink never drops: when its reader is slow, the pipeline waits, and
so does the reading of stdin.
**Development**
```bash
@@ -439,7 +421,7 @@ exits 1 (2 when a named file is missing, see [Exit Codes](#exit-codes)):
```bash
lw --check -c /etc/logwisp/logwisp.toml
lw --check --source file:directory=/var/log/app --sink http:port=8080
lw --check --source file --sink tcp:port=70000
```
```text
@@ -490,11 +472,13 @@ WebAssembly; `t` runs them under the screen and shows what flows.
required key, then edits a value (empty returns it to its default), opens
a list's choices or adds on an add row; in a value, Enter sets it, and Tab
or Up/Down set it and move. Left/Right step a choice or set a switch;
Space flips it; Esc goes back. `a` adds a source, sink or filter; `c`
changes a source's or sink's type in its place; `d` deletes, or returns a
value to its default; Space turns a flow stage on or off; `J`/`K` move a
filter; `[` `]` switch pipelines; `p` presets; `!` shows every problem,
moving to the first; `r` runs; `t` taps; `o` output; `?` keys; `q` quits.
Space flips it; Esc goes back; Tab moves to the next column. `a` adds a
source, sink or filter; `c` changes a source's or sink's type in its place,
and moves to a filter's (in the inspector) or the format's `type` row; `d`
deletes, or returns a value to its default; Space turns a flow stage on or
off; `J`/`K` move a filter; `[` `]` switch pipelines; `p` presets; `!` shows
every problem, moving to the first; `r` runs; `t` taps; `o` output; `?`
keys; `q` or Ctrl-C quits.
- Pipelines: Up from a column's top row reaches the bar, where `[` `]` or
Left/Right switch, `a` adds one (empty or from a preset), Enter renames and
`d` deletes, asking first when it has parts.
@@ -520,7 +504,8 @@ WebAssembly; `t` runs them under the screen and shows what flows.
dropped by sinks, and missed by the screen: the view never slows a
pipeline.
- Live view: from 40 rows it sits under the diagram; shorter, it takes the
screen while it has the keys. `t` or Esc gives them back to the diagram.
screen while it has the keys. `t` or Esc gives them back to the diagram
(Esc first closes an open detail).
- The canvas cursor picks its point, so the view opens on the first
source: a source shows its entries, a flow stage (or the filter the
inspector is on) what reaches it, the heartbeat its own, the sinks (or
@@ -533,13 +518,13 @@ WebAssembly; `t` runs them under the screen and shows what flows.
screen drops columns; a dropped entry is marked; a rune a terminal would
act on or draw wider than a cell shows as a placeholder.
- Keys: arrows or `jk` move (up pauses, down onto the newest follows),
PgUp/PgDn; `g` the oldest, `G` or `f` the newest; Enter the row's
detail, `J`/`K` scroll it; `/` only rows matching a pattern, ignoring
case when it is all lower case (clearing it, or a level, keeps the
focused row in place, paused there unless it is the newest); `1`-`5`
trace to error and up, `0` all; `s` chooses the point;
`i` inspects the stage that dropped the row; `c` clears; `q`, `?`, `r`,
`o`, `[` and `]` act as on the diagram.
PgUp/PgDn; `g` or Home the oldest, `G`, End or `f` the newest; Enter the
row's detail, `J`/`K` scroll it; `/` only rows matching a pattern,
ignoring case when it is all lower case (clearing it, or a level, keeps
the focused row in place, paused there unless it is the newest); `1`-`5`
trace to error and up, `0` all; `s` chooses the point; `i` inspects the
stage that dropped the row; `c` clears; `q`, Ctrl-C, `?`, `r`, `o`, `[`
and `]` act as on the diagram.
- The detail, beside the rows from 160 columns and under them otherwise:
time and the stamp as written, level, origin, the verdict with a
filter's patterns, the layout read, message, fields, and the payload as
@@ -574,8 +559,6 @@ lw auth token --url https://HOST:PORT[/PATH] -u NAME [--password-file FILE
lw auth stream --addr HOST:PORT -u NAME [--password-file FILE] [TLS flags]
```
`-u` is short for `--user`.
- `add-user`: adds a user, or replaces its password
- `remove-user`: removes a user; refuses the last one
- `token`: logs in to an `http` sink or `http_chain` source and prints a
@@ -611,8 +594,9 @@ echo; a file, `/dev/fd/N` or a pipe is read as by a
`--ca-file` (default: system roots) or `--pin-sha256` (the `tls.pin_sha256` a
self-signed listener logs), `--server-name` (default: the host), and
`--cert-file` / `--key-file` for a listener with `tls.client_auth`. There is no
flag to skip verification: an unverified server could relay the login. Redirects are not followed. `--unbound` logs in to an `http` sink behind
a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's
flag to skip verification: an unverified server could relay the login.
Redirects are not followed. `--unbound` logs in to an `http` sink behind a
TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's
certificate across the two requests; only then may `--url` carry the path the
proxy mounts LogWisp at. `stream` exits `0` on `SIGINT` or `SIGTERM`, and `1`
when the server ends the stream (a reload or shutdown).
@@ -665,8 +649,8 @@ lw tls cert --ca-dir DIR --name NAME [--hosts NAME,...] [--server] [--client]
`tls.issuer_cert_file` and `tls.issuer_key_file`.
- `cert`: `NAME.crt` and `NAME.key`, in `--out` (default: `--ca-dir`).
- `--server` for a listener: `--hosts` lists the DNS names and IP addresses
it carries, default `NAME` (`--host`, its earlier name, still works);
`--client` for a dialer presenting a certificate; both may be given.
it carries, default `NAME`; `--host` is an alias. `--client` for a dialer
presenting a certificate; one of the two is required, both may be given.
- `NAME` is the subject CN, the identity `auth.type = "mtls"` matches.
- Valid 397 days (the longest browsers accept), never past the CA. It prints
the certificate's `pin_sha256`.
@@ -687,10 +671,12 @@ lw config init [--out FILE]
`config/logwisp.toml`, every option with its default, to edit and run. It
never replaces a file (remove one to replace it). Without `--out` it writes
`~/.config/logwisp/logwisp.toml`, the file lw reads without `-c` unless a
`LOGWISP_CONFIG_*` variable names another; with that file in place, lw
without pipeline options runs its pipelines rather than filtering stdin.
Any other path is run with `-c FILE`; lw prints which applies. `lw --dump` prints the
effective configuration instead, flags and environment included.
`LOGWISP_CONFIG_*` variable names another; any other path is run with
`-c FILE`, and lw prints which applies. As shipped the file defines no
pipeline, so lw still filters stdin, though its own log on stderr is at the
file's default `info` rather than `warn`; once it defines `[[pipelines]]`, lw
without pipeline options runs them instead. `lw --dump` prints the effective
configuration, flags and environment included.
```bash
lw config init
doc/configuration.md
+104 -166
View File
@@ -6,12 +6,12 @@ and its default ships as [`config/logwisp.toml`](../config/logwisp.toml).
## Configuration Precedence
- Pipelines come from the first of these that defines any, replacing the rest
wholesale (the default pipeline's rate limit and formatter included):
wholesale:
1. pipeline flags: `--source`, `--sink`, ... ([CLI](cli.md#pipelines))
2. pipeline variables: `LOGWISP_SOURCE`, ...
([CLI](cli.md#pipeline-variables))
3. the file's `[[pipelines]]`
4. the built-in default pipeline
4. the built-in `pipe` pipeline ([CLI](cli.md#built-in-defaults))
- Every other key merges, highest priority first:
1. command-line flags
2. environment variables
@@ -28,33 +28,28 @@ The path is resolved before any other configuration is read:
4. `~/.config/logwisp/logwisp.toml`, if it exists
5. `./logwisp.toml`
Missing file behaviour differs by how it was chosen. An explicitly requested
file that does not exist is a fatal error (exit code 2); a missing discovered
default is not an error, and LogWisp starts on built-in defaults.
File-selection flags are consumed before schema overrides are parsed. A missing
or empty path is a usage error; `--` stops option parsing. The selected path is
runtime metadata and cannot be redirected by a `config_file` key inside the file.
A file named by `-c` or a variable must exist (exit code 2); a missing
discovered one is not an error, and lw runs on built-in defaults. A relative
path that climbs with `..` is refused; give it absolute. A file may hold at
most 10 MiB.
## Global Settings
Top-level keys, each as type and default:
- `quiet` (bool, `false`): disable all application logging and console
diagnostics; pipeline output still flows.
diagnostics; pipeline output still flows. It overrides every logging
setting.
- `status_reporter` (bool, `true` with a configuration file, `false` without):
emit a status report every 30 s at DEBUG level.
a status report every 30 s, shown only at `logging.level = "debug"`.
- `auto_reload` (bool, `false`): watch the config file and reload pipelines on
change.
- `color` (string, `"auto"`): level names in color on console sinks, which may
set their own: `"auto"` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`),
`"always"` or `"never"`; see the [console sink](sinks.md#console).
`--version`, `--check`, `--dump` and `--schema` print and exit; they are not
persistent settings.
Note that `status_reporter` writes at DEBUG level, so it produces nothing unless
`logging.level = "debug"`.
`--version`, `--check`, `--dump`, `--schema` and `--tui` are switches, not
settings; the matching variables are ignored.
## Application Logging
@@ -76,7 +71,7 @@ max_total_size_mb = 1000
retention_hours = 168.0
```
### Output modes
`output` modes:
- `file`: files only.
- `stdout`: standard output only.
@@ -85,23 +80,12 @@ retention_hours = 168.0
- `all`: files plus split console.
- `none`: no application logging.
`[logging.file]` applies only to the `file` and `all` modes. The `stdout`,
`split` and `all` modes mix the log into a console sink's stdout.
Without a configuration file (none named, none found) lw is a command-line
tool: `level` defaults to `warn` and `status_reporter` to `false`. Explicit
values from the environment or flags still win. A container configured only
through variables or flags passes `LOGWISP_LOGGING_LEVEL=info` to log as a
service; the image does not set it, since a variable would override a mounted
file's value.
> `[logging.console].target` is accepted and validated (`stdout`, `stderr`,
> `split`) but **not applied**. The console destination is derived from
> `logging.output`. The key is retained for compatibility; setting it has no
> effect.
`quiet = true` overrides every logging setting and disables both file and
console output.
`[logging.file]` applies only to the `file` and `all` modes. Its limits delete
any `.log` file in `directory`, lw's or not: give lw a directory of its own.
`retention_hours = 0` keeps files. The `stdout`, `split` and `all` modes mix
the log into a console sink's stdout. `[logging.console].target` is validated
(`stdout`, `stderr`, `split`) but not applied: `logging.output` chooses the
console.
## Pipeline Configuration
@@ -131,7 +115,7 @@ interval_ms = 30000
# --- sources: one or more ---
[[pipelines.plugin_sources]]
id = "app_logs" # unique within the pipeline
id = "app_logs" # unique among the pipeline's sources
type = "file"
[pipelines.plugin_sources.config]
directory = "/var/log/myapp"
@@ -146,14 +130,13 @@ port = 8080
Every source and sink is a plugin instance with three keys:
- `id`: instance identifier, unique within the pipeline; appears in logs and
stats.
- `id`: instance identifier, unique among the pipeline's sources, and among
its sinks; appears in logs and stats.
- `type`: registered plugin type.
- `config`: plugin-specific table; see [Sources](sources.md) and
[Sinks](sinks.md).
`config_file` is reserved on both structures for a future include mechanism and
is not implemented.
`config_file` on a source or sink is accepted and ignored.
### Flow stages
@@ -168,19 +151,16 @@ Every stage is optional:
Options, each as type and default:
- `entries_per_second` (float, `0`): `<= 0` disables the limiter entirely.
- `entries_per_second` (float, `0`): `0` disables the limiter; a negative value
is refused.
- `burst_entries` (float, `entries_per_second`): token bucket capacity.
- `policy` (string, `pass`): `pass` allows everything through, `drop` discards
over-limit entries.
- `policy` (string, `pass`): `pass` or `drop`, in lower case like every enum.
`drop` discards entries over the rate or the size cap; `pass` lets
everything through.
- `max_entry_size_bytes` (int, `0`): per-entry byte cap; `0` = unlimited.
Two behaviours are easy to trip over:
- The limiter is constructed only when `entries_per_second > 0`. With `0`,
`max_entry_size_bytes` is never enforced.
- `policy = "pass"` short-circuits the whole check, including the size cap.
To enforce a size cap you need `entries_per_second > 0` **and**
`policy = "drop"`.
The limiter, its size cap included, acts only with `entries_per_second > 0`
and `policy = "drop"`.
#### Heartbeat
@@ -188,138 +168,98 @@ Options, each as type and default:
- `enabled` (bool, `false`): enable heartbeat generation.
- `interval_ms` (int, `1000`): interval; minimum `100`.
- `include_timestamp` (bool, `false`): `false` formats with level only, no
timestamp.
- `include_stats` (bool, `false`): attach `beat_count` and measured
`interval_ms` as fields.
- `format` (string, `txt`): `txt`, `json` or `raw` are accepted but have no
effect: heartbeats go through the flow's `format`, like every entry.
- `include_timestamp` (bool, `false`): `false` writes the level only, no
timestamp; the `raw` format shows neither.
- `include_stats` (bool, `false`): add `type = "heartbeat"`, `beat_count` and,
from the second beat, the measured `interval_ms` as fields.
- `format` (string, `txt`): `txt`, `json` or `raw`; no effect: heartbeats go
through the flow's `format`.
Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They
are generated after the flow's filter and rate-limit stages, so filters do not
suppress them, and they reach every sink in the pipeline.
> `format = "comment"` (SSE comment framing) appears in older documentation and
> in a code path in the generator, but the validator rejects it and the pipeline
> fails to start. Use `txt`, `json`, or `raw`.
## Environment Variables and Flags
## Environment Variables
Environment overrides are derived from the TOML path: `.` becomes `_`, the
result is uppercased, and `LOGWISP_` is prepended:
```
quiet LOGWISP_QUIET
status_reporter LOGWISP_STATUS_REPORTER
logging.level LOGWISP_LOGGING_LEVEL
logging.file.directory LOGWISP_LOGGING_FILE_DIRECTORY
```
Migration: the old custom transform accidentally read bare names such as `QUIET`
and `LOGGING_LEVEL`. Rename those variables to their prefixed forms; bare names
are now ignored. `LOGWISP_CONFIG_FILE` and `LOGWISP_CONFIG_DIR` still select the
file directly.
Only scalar paths that exist in the configuration schema can be set this way.
Array elements cannot: `LOGWISP_PIPELINES_0_NAME` has no effect. A whole
pipeline can: see [pipeline variables](cli.md#pipeline-variables).
## Command-Line Overrides
Any scalar configuration path is settable as a flag using its TOML path:
```bash
lw --logging.level=debug --status_reporter=false
lw --logging.level debug # space form also works
lw --quiet # bare flag means true
```
A bare flag is accepted only for a boolean key, which takes its value only
after `=`: `--logging.file.directory --quiet` fails with
`--logging.file.directory needs a value`, and `--quiet false` with `unexpected
argument "false"`. An unknown key is an error too (`unknown option
--nosuch.key`); each is a usage error, exit code 2. Single-dash options are
lw's own ([CLI](cli.md#options)).
An empty `LOGWISP_<PATH>` variable counts as unset, as the pipeline variables
do.
An array-indexed path such as `--pipelines.0.name=x` is an error (exit 1);
whole pipelines have [their own flags](cli.md#pipelines).
Every setting is also a variable and a flag by its path: `logging.level` is
`LOGWISP_LOGGING_LEVEL` and `--logging.level`. Pipelines are not: their flags
and variables replace the file's whole, and an array-indexed path such as
`--pipelines.0.name=x` is an unknown option (exit code 2). See
[CLI](cli.md#options) and [Environment Variables](cli.md#environment-variables).
## Validation
Startup validation is intentionally split.
The configuration is checked offline when it loads, at every reload and by
`lw --check`. Each check reports the first problem of each table (the top
level, `logging`, each flow stage, each source and sink) by its key's path:
`internal/config` validates only global structure:
```
Error: Failed to load config: invalid configuration:
pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "Drop"
pipelines[0].plugin_sinks[http].config: unknown key "tls.enabeld"
```
- no key the file schema does not declare, at any depth outside a plugin's
`config` table: `config file "…": unknown key
"pipelines[0].plugin_sinks[0].confg"`. A misspelled table path would otherwise
drop the whole table it heads. A top-level `config_file` is ignored
- pipeline [specs](cli.md#pipelines): syntax, and unknown flow-stage keys such
as `--rate-limit entries_per_second=1,polcy=drop`
- at least one pipeline
- unique, non-empty pipeline names
- at least one source and one sink per pipeline
- `logging.output`, `logging.level`, `logging.format`, `logging.sanitization`,
and `logging.console.target` enum membership
- Pipeline flags and variables come first, alone: their syntax and keys
(`--rate-limit entries_per_second=1,polcy=drop: unknown key "polcy"`).
- Settings: the `color` enum and the `logging` ones (`output`, `level`,
`format`, `sanitization`, `console.target`).
- Pipelines: at least one; names set and unique; a source and a sink each;
ids unique among a pipeline's sources and among its sinks; one `console`
source in the whole configuration.
- Plugin options are decoded in full: unknown keys at any depth, kinds,
required keys, enums, ranges, paths, and the `tls`, `auth` and `acl` rules
of the plugin's side; a `tls` table's rules only when it is `enabled`.
- Enums are case-sensitive: `policy = "Drop"` and `target = "STDERR"` are
refused.
- In plugin options and flow stages, `0`, or `""` for a string, takes the
key's default, or the meaning its help gives zero (off, none, unlimited). A
number below its key's minimum is refused, a negative one included where
zero means the default (`buffer_size = -1`).
- Flow stages are checked whether enabled or not: kinds, enums and ranges,
and filter patterns must compile ([Filters](filters.md)).
- `auth` enums (`identity`, `node_binding`) are checked whatever `type` is; a
block naming peers or credentials with `type` `none` or unset is refused
([Security](security.md#validation-1)).
- Once the rest is valid, the file itself: the first key the schema does not
declare outside a plugin's `config` is refused (`config file "…": unknown
key "pipelines[0].plugin_sinks[0].confg"`), since a misspelled table path
would drop the whole table it heads. A top-level `config_file`, `version`,
`check` or `dump` is ignored.
- Files a plugin names (certificates, keys, CA files, credentials, password
files) are read when it is built: at start, at reload and by `lw --check`.
Those failures name the pipeline and plugin.
Everything else is validated by the plugin constructor that owns it — port
range, required paths, path prefixes, enum values, regex compilation, TLS and
credentials file loading. A failure there aborts pipeline construction with a
message naming the pipeline, plugin id, and offending key. A key the plugin does
not declare, at any depth, is such a failure (`unknown key "tls.enabeld"`): a
misspelled option must not silently fall back to its default. For the same
reason an `auth` block that names peers or credentials (`allow`,
`allow_patterns`, `credentials_file`, `token_lifetime_ms`, `username`,
`password_file`, `trusted_proxies`) while its `type` is `none` or unset is refused: auth was
intended and the type forgotten. See [Security](security.md#the-auth-block).
There is **no** cross-pipeline port-conflict detection. Two sinks bound to the
same port fail at listener bind time, when the pipeline starts.
Two listeners on one port pass validation; the second fails at bind and lw
exits 1 ([Networking](networking.md#network-plugins)).
## Hot Reload
```toml
auto_reload = true
```
`auto_reload = true` watches the configuration file. `SIGHUP` or `SIGUSR1`
rereads it even when watching is off and always rebuilds, so certificates and
credentials rotate without a TOML edit. CLI and environment overrides,
pipelines included, are captured at startup and keep their precedence. A
discovered default file that does not exist is not an error, so an instance
configured by flags or variables alone still rebuilds; once read, though, its
removal fails the reload and the running service stays as it is; restart to
run without the file.
or send `SIGHUP` / `SIGUSR1`. Signals reread the selected file even when watching
is disabled and always rebuild, allowing certificate and credentials rotation
without TOML edits; `auto_reload` watches only the configuration file.
CLI and environment overrides, pipelines included, are captured at startup and
keep their precedence. As at startup, a discovered default file that does not
exist is not an error, so an instance configured by flags or variables alone
still rebuilds. Once read, though, its removal fails the reload and the running
service stays as it is; restart to run without the file.
Reload builds a new service from a detached snapshot of the configuration,
validated again, and shuts the old one down only if that succeeds: parse,
validation and construction errors leave the running service intact. Listener
bind failures occur after the old service has stopped and can leave lw without
working pipelines; correct the file and signal again.
Reload rebuilds the whole service: a new service is constructed from the new
configuration first, and only if that succeeds is the old one shut down. Each
candidate is a detached snapshot and runs top-level validation again. Parse,
conversion, validation and construction errors leave the running service intact.
Listener bind/start failures occur after shutdown of the old service and can leave
the application without working pipelines; correct the file and signal again.
Watch errors (deletion, permission changes, timeout) are logged without
rebuilding. Queued changes are combined, and unchanged pipelines and status
settings do not trigger another rebuild. Removed file keys fall back to the
remaining sources on the next successful load.
Watch errors (including deletion, permission changes and timeout) are logged
without rebuilding. Queued path changes are combined, and unchanged pipelines
and status settings do not trigger another rebuild. Removed file keys fall back
to the remaining sources on the next successful load.
- Reloaded: pipelines, sources, sinks, flow stages, `status_reporter`, `color`
- Not reloaded: `logging.*` (applied once at startup), `quiet`, `auto_reload`
(the watcher is not restarted)
What a reload applies:
- Reloaded:
- pipelines, sources, sinks
- filters, formatters, rate limits, heartbeats
- `status_reporter`
- `color`
- Not reloaded:
- `logging.*`: applied once at startup
- `quiet`
- `auto_reload`: the watcher is not restarted
Because the rebuild is total, listeners close and reopen and every connected
client is disconnected. Chain sinks reconnect on their own backoff schedule.
The rebuild is total: listeners close and reopen, every connected client is
disconnected, and chain sinks reconnect on their own backoff schedule.
## Type Reference
@@ -332,7 +272,5 @@ Each TOML type, its Go type, and its command-line / environment form:
- Array (`[]T`): only in [pipeline specs](cli.md#pipelines), as a repeated key.
- Table (struct): nested path with `.` (flags) or `_` (environment).
Integer fields reject fractions, overflow and negative unsigned values. Non-finite
floats and integer-to-float precision loss are rejected too. This applies to plugin
maps when their constructors decode them. Only TOML configuration files are accepted;
JSON log formatting and network payloads are unaffected.
Integer keys refuse fractions and out-of-range values; number keys refuse inf
and NaN. Configuration files are TOML only.
doc/deployment.md
+8 -7
View File
@@ -5,7 +5,7 @@ for what its flags leave open, prints a plan with the equivalent non-interactive
command, then writes the files, runs `lw --check` on them (the node's binary or
image, with its mounts) and starts the node. `--dry-run` stops at printing;
`--yes` never asks and fails on a missing required value. It creates no
certificates: see [Security](security.md#enabling-mtls).
certificates: `lw tls` makes them ([CLI](cli.md#lw-tls)).
## Roles
@@ -123,14 +123,15 @@ needs `--force`.
Application jails run an edge each; a `logs` jail aggregates. Every jail runs
already, and the `logs` jail has an address the others reach (`AGG_ADDR`).
1. Certificates: a CA, and a server certificate for `logs` naming `AGG_ADDR`
([Security](security.md#enabling-mtls)).
1. Certificates: `lw tls ca --dir PKI`, then
`lw tls cert --ca-dir PKI --name logs --hosts AGG_ADDR --server`, which
writes `PKI/logs.crt` and `PKI/logs.key` ([CLI](cli.md#lw-tls)).
2. The aggregator, from the host:
```sh
deploy/lw-deploy.sh --yes --runtime native --jail logs --role aggregator \
--file-dir /var/log/logwisp --http-port 8080 \
--tls --cert-file logs.crt --key-file logs.key \
--tls --cert-file PKI/logs.crt --key-file PKI/logs.key \
--auth scram --add-user web1 --add-user db1 \
--sink-auth scram --add-viewer ops
```
@@ -142,13 +143,13 @@ already, and the `logs` jail has an address the others reach (`AGG_ADDR`).
```sh
deploy/lw-deploy.sh --yes --runtime native --jail web1 --role edge \
--log-dir /var/log/nginx --aggregator AGG_ADDR \
--tls --ca-file ca.crt --auth scram --password-file ~/logwisp-secrets/web1.pass
--tls --ca-file PKI/ca.crt --auth scram --password-file ~/logwisp-secrets/web1.pass
```
4. Check: `jexec logs service logwisp status`, the files in the `logs` jail's
`/var/log/logwisp`, and `https://AGG_ADDR:8080/status` with a token from
`lw auth token --url https://AGG_ADDR:8080 --user ops --password-file
~/logwisp-secrets/ops.pass --ca-file ca.crt` ([CLI](cli.md#lw-auth)).
~/logwisp-secrets/ops.pass --ca-file PKI/ca.crt` ([CLI](cli.md#lw-auth)).
Variants:
@@ -187,7 +188,7 @@ every key.
- `acl` on the listeners: address rules, PROXY headers, per-client limits
- mtls identities other than the certificate CN; `node_binding` other than
`force`
- creating certificates
- creating certificates (`lw tls`)
- several native instances on one host: use one per host or jail
## Testing the script
doc/filters.md
+13 -15
View File
@@ -18,8 +18,9 @@ patterns = ["ERROR", "WARN"]
pattern matches).
- `patterns`, []string, default `[]`: Go RE2 regular expressions.
A filter with no patterns passes everything. Invalid patterns fail at startup
with the filter index and the offending pattern in the message.
A filter with no patterns passes everything. An invalid pattern is refused
when the configuration loads, named by its path
(`pipelines[0].flow.filters[0].patterns[1]`).
## What Gets Matched
@@ -34,7 +35,8 @@ Empty parts are omitted, so an entry with no detected level matches
level as easily as the message body:
- `"^app\\.log "` matches entries whose source is `app.log`.
- `"ERROR"` matches level `ERROR`, or the word `ERROR` anywhere in the message.
- `"ERROR"` matches `ERROR` anywhere in the text: the level, the message, the
source name, or inside a word (`NOERROR`). Use `"\\bERROR\\b"` for the word.
The structured `fields` payload is **not** part of the match text.
@@ -124,16 +126,15 @@ time, which is exactly what you want in a log hot path.
- Whole word: `\berror\b`
- Alternation: `ERROR|WARN|FATAL`
- Character class: `[0-9]{3}`
- Anchors: `^ERROR`, `ERROR$`
- Anchors: `^app\.log ` (the text starts with the source), `ERROR$`
- Any characters: `.*exception.*`
Remember that TOML basic strings process escapes, so a regex backslash needs
doubling: `"\\berror\\b"`. TOML literal strings avoid the issue:
`'\berror\b'`.
Anchors apply to the assembled match text, which begins with the source name —
so `^ERROR` will not match an entry whose source is non-empty. Use
`\bERROR\b` instead unless you mean to anchor on the source.
Anchors apply to the assembled match text, which begins with the source name,
so `^ERROR` never matches an entry that has a source; use `\bERROR\b`.
## Common Recipes
@@ -153,8 +154,8 @@ type = "exclude"
patterns = ["/healthz", "/metrics", "\\bping\\b"]
```
**Secret suppression** — see [Security](security.md); filters are the only
redaction mechanism LogWisp currently offers.
**Secret suppression**: LogWisp has no redaction; an exclude filter drops each
entry that matches ([Security](security.md)).
```toml
[[pipelines.flow.filters]]
@@ -162,8 +163,6 @@ type = "exclude"
patterns = ["password", "api[_-]?key", "authorization", "bearer ", "secret", "token"]
```
Note this drops the whole entry, it does not redact part of it.
**Per-application routing** — run one pipeline per application, each with its
own include filter, rather than trying to route inside one pipeline. Sinks fan
out to *all* sinks in a pipeline; there is no conditional routing.
@@ -182,7 +181,6 @@ evaluated against every filter until one rejects it, so cost scales with the
number of patterns and their complexity. Prefer literal substrings and simple
alternations over broad `.*` wildcards.
Filters log at DEBUG on every entry — pattern text, match results, and the
final decision. That is invaluable when a filter is not behaving as expected and
very expensive in production; keep `logging.level` at `info` or higher on a busy
pipeline.
Filters log every pattern match and decision at DEBUG: useful when a filter
misbehaves, expensive on a busy pipeline, where `logging.level` stays at
`info` or higher.
doc/formatters.md
+57 -59
View File
@@ -12,8 +12,9 @@ timestamp_format = ""
```
One formatter serves the whole pipeline. Sinks receive an identical payload,
one record ending in exactly one newline; there is no per-sink formatting. When you need two shapes of the same data, run
two pipelines, or chain to a node that formats differently.
one record ending in exactly one newline; there is no per-sink formatting.
When you need two shapes of the same data, run two pipelines, or chain to a
node that formats differently.
Omitting `[pipelines.flow.format]` entirely selects `raw`.
@@ -32,27 +33,25 @@ Each as type and default:
### raw
Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches
the sink as the source produced it, with no timestamp, level or source prefix
added, and a newline appended unless the message ends in one. An entry that also
carries `fields` gets the fields JSON appended verbatim after a single space —
`raw` never drops data and never re-encodes it.
Passthrough, the fastest type, for text already in its final form. `FlagRaw`
bypasses formatting and sanitization: the message reaches the sink as the
source produced it, control characters included, with no timestamp, level or
source prefix added, and a newline appended unless the message ends in one.
An entry that also carries `fields` gets the fields JSON appended verbatim
after a single space; `raw` never drops data and never re-encodes it.
```toml
[pipelines.flow.format]
type = "raw"
```
Fastest option, and the right one when you are relaying text that is already in
its final form. Note that it also bypasses sanitization, so control characters
in the source data reach your sinks intact.
Byte-exact transport needs a source that does not split the line: the `console`
source, or the `file` source with `raw = true`. Both put the whole line, its
terminator removed, in the message and leave `fields` empty; `raw` writes it
back as one `\n`. The `file` source's
JSON branch splits a line into message and fields, so `raw` reassembles it as
`<msg> <fields>` rather than reproducing the original object.
The closest to verbatim transport is the `console` source, or the `file`
source with `raw = true`. The whole line, terminator removed, becomes the
message and `fields` stays empty; `raw` writes it back with one `\n`. It is not
byte-exact: blank lines are skipped, CRLF becomes LF, and a line over 1 MiB
continues in the next entry. The `file` source's JSON branch splits a line
into message and fields, so `raw` writes `<msg> <fields>` rather than the
original object.
### txt
@@ -81,9 +80,13 @@ Output has the shape:
{"time":"2026-01-02T15:04:05.123Z","level":"ERROR","trace":"edge-01/app.log","fields":["connection refused"]}
```
The exact key names and structure come from the `lixenwraith/log` formatter, not
from LogWisp; they are stable for a given dependency version but are not part of
LogWisp's own configuration surface.
`level` is DEBUG, INFO, WARN or ERROR, here and in `txt`. Only `DEBUG`,
`INFO`, `WARN`, `WARNING` and `ERROR`, all upper or all lower case, are read as
such; any other level, `TRACE` included, and an entry without one print as
INFO. Filters see the level as the source stored it.
The key names and structure come from the `lixenwraith/log` formatter, not
from LogWisp's configuration.
## Flags
@@ -93,13 +96,15 @@ you need to override the defaults.
- `1` Raw: bypass formatting and sanitization entirely.
- `2` ShowTimestamp: emit the timestamp.
- `4` ShowLevel: emit the level.
- `8` StructuredJSON: render attached fields as a JSON object.
- `8` StructuredJSON: in `json`, the fields become a `fields` object beside
`message`.
- `16` NoTimestamp: suppress the timestamp.
- `32` NoLevel: suppress the level.
With `flags = 0` the formatter selects `1` for `type = "raw"` and `6`
(timestamp + level) for every other type. `8` is added automatically whenever an
entry carries parseable `fields` and `1` is not set; `1` always wins.
(timestamp + level) for every other type. `8` is added automatically whenever
an entry carries parseable `fields` and `1` is not set (it shapes `json` only);
`1` always wins.
`txt` and `json` show timestamp and level unless a suppress bit says otherwise
(suppress wins over show), so `2` and `4` alone change nothing. Examples:
@@ -115,45 +120,43 @@ Each policy, then what to use it with:
- `raw`: no-op passthrough.
- `type = "raw"` where you control the data.
- `txt`: escapes non-printable characters.
- `txt`: writes `<` and non-printable characters as `<hex>` of their bytes
(`<b>` becomes `<3c>b>`).
- File and console sinks.
- `json`: escapes control characters for safe JSON embedding.
- `type = "json"`.
- `shell`: strips shell metacharacters, whitespace, and control characters.
- Data that will be passed to a command.
When `sanitizer_policy` is omitted, it is the policy of the same name as
`type`, so the safe pairing is the default.
Every policy but `raw` replaces invalid UTF-8 with U+FFFD. With
`type = "raw"` and `flags = 0` no policy applies (`1` bypasses it). When
`sanitizer_policy` is omitted, it is the policy of the same name as `type`, so
the safe pairing is the default.
> `shell` strips dangerous characters but is **not** sufficient to make a string
> safe for shell construction. Pass arguments through `exec` argv instead of
> building command lines.
To see a policy working, point a pipeline at the `random` source with
`special = true`, which injects control bytes and multi-byte Unicode into every
message.
`special = true` and `format` `raw` or `txt`, which mix control and non-ASCII
bytes, often invalid UTF-8, into its messages ([random](sources.md#random)).
## Node Identity in Output
Entries that arrived over a chain link carry a `Node` label. The formatter
renders it as a syslog-style prefix on the source field (`trace` in `json`
output):
```
edge-01/app.log
```
Entries with no node label show the bare source. Node identity therefore appears
*inside* the source field rather than as a separate output key — worth knowing
when writing downstream parsers or grep patterns.
Entries that arrived over a chain link carry a `Node` label, written as a
syslog-style prefix on the source (`trace` in `json`): `edge-01/app.log`.
Entries without one show the bare source. There is no separate node key, which
downstream parsers and grep patterns need to know.
## Structured Fields
When an entry carries `Fields` (raw JSON) and `FlagRaw` is not set, the
formatter parses it and switches to structured rendering by adding the
`StructuredJSON` flag automatically. Fields reach a pipeline in two ways: from
the `file` source when a tailed line parses as JSON with a `fields` key, and
from the heartbeat generator when `include_stats = true`.
formatter parses it and adds the `StructuredJSON` flag. Fields reach a pipeline
from the `file` source when a tailed line parses as JSON with a `fields` key,
from the `tcp_chain` and `http_chain` sources when the upstream entry carried
them, and from the heartbeat when `include_stats = true`. In `json` output
such an entry is `{"time":…,"level":…,"trace":…,"message":…,"fields":{…}}`;
without fields the message sits in `"fields":[…]`.
## Choosing a Configuration
@@ -165,22 +168,19 @@ By goal:
- Downstream ingestion (Loki, Elasticsearch, jq): `type = "json"`,
`sanitizer_policy = "json"`.
- Compact console output: `type = "txt"`, `flags = 16`.
- Untrusted log content: never `raw`; pick `txt` or `json` and set the
matching policy.
- Untrusted log content: never `raw`; `txt` or `json`, whose policies are the
default.
## Formatting and Chain Links
Chain sinks (`tcp_chain`, `http_chain`) do **not** ship the formatted payload.
They re-serialize the structured entry into the canonical chain encoding, which
makes them independent of the local formatter.
Chain sinks (`tcp_chain`, `http_chain`) do **not** ship the formatted payload:
they send the structured entry in the chain encoding. `flow.format` on an edge
node therefore changes only that node's own local sinks; the shape a human or
a downstream system sees is decided on the node that owns the sink they read.
The practical consequence: setting `flow.format` on an edge node changes only
that node's own local sinks. The output shape seen by a human or a downstream
system is decided on the node that owns the sink they read.
If an event ever reaches a chain sink without a structured entry, the sink wraps
the formatted payload into a synthetic entry and counts it in `synthesized`.
A non-zero `synthesized` count means something upstream lost structure.
An event that reaches a chain sink without a structured entry is wrapped from
its formatted payload and counted in `synthesized`; a non-zero count means
something upstream lost structure.
## Performance
@@ -188,7 +188,5 @@ Relative cost, cheapest first: `raw` (passthrough) → `txt` (line assembly) →
`json` (serialization). Sanitization adds a scan of the message; the `raw`
policy skips it.
The formatter holds a mutex because the underlying implementation reuses an
internal buffer and is not goroutine-safe. It is the only shared serialization
point in the hot path, and the reason a single pipeline formats entries one at
a time.
The formatter reuses an internal buffer under a mutex, so a pipeline formats
its entries one at a time.
doc/installation.md
+48 -37
View File
@@ -6,8 +6,9 @@
- Architecture: amd64; arm64 builds but is untested
- To build: Go 1.27.2 or newer (FreeBSD: the `go127` package), and GNU make
or BSD make
- For `make e2e`: bash 5+, coreutils, curl, openssl and go (the proxy test
builds its reverse proxy). Optional, else those checks SKIP: node with
- For `make e2e`: bash 5+, coreutils, curl, openssl, go (the proxy and acl
tests build helpers) and nc (acl). Optional, else those checks SKIP:
script(1) and setsid(1) (`preset-test.sh`'s terminal checks), node with
playwright and the Chromium builds `playwright install chromium` fetches
(browser checks), an IPv6 loopback (`ipv6-test.sh`), docker as root with the
`logwisp:dev` image (`deploy-test.sh`'s container run)
@@ -31,7 +32,8 @@ The Makefile works with GNU make and BSD make alike. Targets:
`lwconf.js` that loads and calls it, `schema.json` (`lw --schema`) and
`lwconf.json`, their manifest: version, commit, Go version and each
file's sha256. Stamped with the commit's time, a commit's files are the
same on every build
same on every build. Its calls:
[Architecture](architecture.md#configuration-engine)
- `make version`: the metadata a build would embed
- `make completion`: regenerate the bash, zsh and fish completion in
`deploy/package/completion/` from `cmd/lw`'s `shorts` and `commands`
@@ -60,6 +62,8 @@ Variables:
- `GO_BUILDFLAGS`, `GO_LDFLAGS`: extra `go build` and linker flags for
packagers. They are distinct from `GOFLAGS` and `LDFLAGS`, which the go
command and the C linker read themselves.
- `VERSION`, `GIT_COMMIT`, `BUILD_TIME`: what a build embeds, by default from
git and the clock; a package sets them.
- `SOURCE_DATE_EPOCH`: when set, the embedded build time comes from it, so
package builds are reproducible.
@@ -73,8 +77,9 @@ checkout.
`make install` copies what a package ships and compiles nothing, so build
first. It honours `DESTDIR`, `PREFIX`, `SYSCONFDIR`, `BINDIR` (default
`$PREFIX/bin`, the path the systemd unit and the rc.d script run `lw` from)
and `SERVICE`:
`$PREFIX/bin`, the path the systemd unit and the rc.d script run `lw` from),
`MANDIR`, `DOCDIR` and `LICENSEDIR` (under `$PREFIX/share` by default) and
`SERVICE`:
- Linux defaults: `PREFIX=/usr`, `SYSCONFDIR=/etc`. Outside a package
manager, prefer `PREFIX=/usr/local`; systemd reads units, sysusers and
@@ -83,7 +88,7 @@ and `SERVICE`:
- `INSTALL_OS` (default: `uname -s`) picks the layout, so
`INSTALL_OS=FreeBSD DESTDIR=...` stages the FreeBSD one on another system.
- `SERVICE`: `yes` (default) adds the service and its configuration; `no`
installs `lw` alone ([Without a Service](#without-a-service)).
leaves them out ([Without a Service](#without-a-service)).
- `BASHCOMPDIR`, `ZSHCOMPDIR`, `FISHCOMPDIR`: where the shell completion
goes, by default `$PREFIX/share/bash-completion/completions`,
`$PREFIX/share/zsh/site-functions` (`/usr/share/zsh/vendor-completions`
@@ -97,16 +102,16 @@ sudo make install PREFIX=/usr/local
Every install gets:
- `$BINDIR/lw` and the manual `$PREFIX/share/man/man1/lw.1`
- `$BINDIR/lw` and the manual `$MANDIR/man1/lw.1`
- completion for `lw`, its commands, flags, files and preset names:
`lw`, `_lw` and `lw.fish` in the three directories above
- `$PREFIX/share/doc/logwisp/` (this documentation) and
`$PREFIX/share/licenses/logwisp/LICENSE`
- `$DOCDIR` (this documentation) and `$LICENSEDIR/LICENSE`
A service install also gets the service files below and, on Linux and
FreeBSD alike, `$SYSCONFDIR/logwisp/logwisp.toml`: the annotated sample
configuration, placed when none is there and otherwise kept. Edit it before
starting the service.
configuration, placed when none is there and otherwise kept. As shipped it
defines no pipeline, so the service would copy an empty stdin and exit: edit it
before starting the service.
### Linux (systemd)
@@ -139,7 +144,7 @@ Create a credentials file with its owner and mode first; `lw auth` keeps
both:
```bash
sudo install -m 0600 -o logwisp -g logwisp /dev/null /etc/logwisp/users.toml
sudo install -m 0640 -o root -g logwisp /dev/null /etc/logwisp/users.toml
sudo lw auth add-user --credentials /etc/logwisp/users.toml --user <name> \
--password-file <file>
sudo systemctl reload logwisp
@@ -256,10 +261,11 @@ For every user: `sudo make install SERVICE=no PREFIX=/usr/local`.
existing file, where you choose:
- `lw config init`: `~/.config/logwisp/logwisp.toml`, which `lw` reads
without `-c` unless a `LOGWISP_CONFIG_*` variable names another file. With
it in place, `lw` without pipeline options runs its
pipelines instead of copying stdin to stdout; pipeline options and presets
still replace them.
without `-c` unless a `LOGWISP_CONFIG_*` variable names another file. As
shipped it defines no pipeline, so `lw` still copies stdin to stdout, though
its own log on stderr is at the file's default `info` rather than `warn`;
once it defines `[[pipelines]]`, `lw` without pipeline options runs them,
and pipeline options and presets still replace them.
- `sudo lw config init --out /usr/local/etc/logwisp/logwisp.toml`: a file
for every user, named with `-c`, or without it by
`LOGWISP_CONFIG_DIR=/usr/local/etc/logwisp`
@@ -324,11 +330,11 @@ docker run -d --name logwisp \
-e LOGWISP_SINK=console
```
Without a file lw logs warnings only and runs no status reporter; add
`-e LOGWISP_LOGGING_LEVEL=info` (and `-e LOGWISP_STATUS_REPORTER=true`) for
the service defaults. The image cannot set them: a variable would override
a mounted file's value. Without `-i`, stdin is `/dev/null`, so a container
needs a source other than `console`.
Without a file lw logs less ([CLI](cli.md#built-in-defaults)): add
`-e LOGWISP_LOGGING_LEVEL=info -e LOGWISP_STATUS_REPORTER=true`; the image
sets neither, as a variable would override a mounted file's value. Without
`-i`, stdin is `/dev/null`, so a container needs a source other than
`console`.
- Secrets: only as mounted files, at `/run/secrets` (Docker or Compose
secrets, a Kubernetes secret volume), named by `tls.key_file`,
@@ -413,36 +419,41 @@ or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target.
```
15801-15804 chain-test.sh, chain-aggregate-test.sh test/run/
(no ports) passthrough-test.sh test/run/passthrough/
(no ports) pipe-test.sh test/run/pipe/
15811-15814 mtls-chain-test.sh test/run-mtls/
15821-15825 scram-chain-test.sh test/run-scram/
15831-15832 scram-proxy-test.sh test/run-proxy/
15831-15833 scram-proxy-test.sh test/run-proxy/
15851-15855 ipv6-test.sh test/run-ipv6/
15871-15879 deploy-test.sh test/run-deploy/
15881-15889 acl-test.sh test/run/acl/
15890-15898 preset-test.sh test/run/preset/
```
- `chain-test.sh`: two edges into a relay, one pipeline per chain transport;
ports 15801-15804
- `chain-test.sh`: two edges into a relay, one pipeline per chain transport
- `chain-aggregate-test.sh`: the same edges, with the relay fanning both into
both sinks; ports 15801-15804
both sinks
- `passthrough-test.sh`: `file` source to `file` sink, a wide envelope byte
for byte (`raw = true`) and, parsed, with no key dropped; no ports
- `mtls-chain-test.sh`: mTLS chain edges and viewers, allow lists, node
binding; ports 15811-15814
for byte (`raw = true`) and, parsed, with no key dropped
- `pipe-test.sh`: lw without a configuration as a filter, stdin to stdout
line for line without loss, quiet on stderr, exit 0 at the end of input
- `mtls-chain-test.sh`: mTLS chain edges and viewers, allow lists, node binding
- `scram-chain-test.sh`: SCRAM chain edges and viewers with `lw auth`
credentials; ports 15821-15825
credentials
- `scram-proxy-test.sh`: browser and CLI logins to an `http` sink behind a
TLS-terminating proxy; ports 15831-15832; the browser checks skip without
node and playwright's Chromium: the headless shell for the scenarios with
cookies, full Chromium for the profile that blocks them
TLS-terminating proxy; the browser checks skip without node and
playwright's Chromium: the headless shell for the scenarios with cookies,
full Chromium for the profile that blocks them
- `ipv6-test.sh`: a relay on `::1` over both chain transports, IPv6 SANs,
per-family listeners; ports 15851-15855; skips (exit 77) without an IPv6
loopback
per-family listeners; skips (exit 77) without an IPv6 loopback
- `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed
and checked; ports 15871-15879
and checked
- `acl-test.sh`: each listener admitting or refusing peers by address before
TLS, with clients from 127.0.0.1 and 127.0.0.2, then by the client a PROXY
header names behind a stub proxy on 127.0.0.11-13 (and nginx `stream` on
127.0.0.14 when installed) or an L7 proxy's `X-Forwarded-For`, with
per-client connection caps, request rates and login throttling; ports
15881-15889; skips (exit 77) where 127.0.0.2 cannot be bound
per-client connection caps, request rates and login throttling; skips
(exit 77) where 127.0.0.2 cannot be bound
- `preset-test.sh`: colors, presets, `--dump`, `lw tls`, listeners that make
their certificate, the serve preset's viewer, network sinks delivering a
finite input whole, a SCRAM pipe whose password is on no disk; the
terminal checks skip without script(1) or setsid(1)
doc/lw.1
+62 -24
View File
@@ -1,4 +1,4 @@
.Dd October 10, 2026
.Dd October 11, 2026
.Dt LW 1
.Os
.Sh NAME
@@ -68,12 +68,10 @@ source runs until shutdown.
When every pipeline has finished,
.Nm
shuts down gracefully and exits 0.
Every sink first writes what it holds; the
.Cm http
and
.Cm tcp
sinks give each client
.Cm write_timeout_ms ,
Every sink first writes what it holds; a network sink gives its clients or
its downstream
.Cm write_timeout_ms
.Po Cm request_timeout_ms No for Cm http_chain Pc ,
at most 2 seconds, to take it.
Under a supervisor, standard input is usually
.Pa /dev/null :
@@ -116,8 +114,12 @@ A value follows as the next argument or after
.Ql = ;
a value that starts with
.Ql - ,
and a switch's, follows
.Ql = .
and a boolean key's, follows
.Ql = ;
.Fl t , Fl \-dump , Fl T , Fl \-schema
and
.Fl V
take none.
Short options do not combine.
.Nm
takes options only: an unknown option, or a word no option takes, is a usage
@@ -347,9 +349,12 @@ make a pipeline named after its preset, or
.Dq cli
without one.
.It Fl p Ar spec , Fl \-preset Ar spec
Start the pipeline with a preset, described in
Begin the pipeline with a preset, as its first option, described in
.Sx Presets ;
later options add sources, sinks and filters to it, or replace its format.
later options add sources, sinks and filters to it, set its rate limit and
heartbeat, or replace its format.
A second preset needs its own
.Fl \-pipeline .
.It Fl \-source Ar spec , Fl \-sink Ar spec , Fl \-filter Ar spec
Add a source, sink or filter; each is repeatable.
.It Fl \-format Ar spec , Fl \-rate\-limit Ar spec , Fl \-heartbeat Ar spec
@@ -410,9 +415,21 @@ specification, and
is
.Fl \-user .
A
.Ar path
is a file, a directory, whose files are followed, or a glob; without one, a
preset that takes it reads standard input.
.Cm path
is a file, a directory, whose files are followed, or a glob, and this host
decides which: an existing directory is followed whole, any other path is a
file name or glob in its directory; for a directory not here, as in a
.Fl \-dump
for another host, write
.Ql dir/* .
Without one,
.Cm serve
and
.Cm edge
read standard input.
.Cm from
.Pq Cm end No or Cm start
is where reading a file starts.
.Cm user
names one SCRAM user, an edge's login or a listener's only user in place of
.Cm users ;
@@ -431,7 +448,7 @@ runs without a configuration.
.It Cm tail path Op Cm from , format
Follow files to standard output, like
.Ql tail \-F .
.It Cm serve Oo Cm path , listen , format , tls , users , user , password_file , proxy , viewer , allow , deny Oc
.It Cm serve Oo Cm path , from , listen , format , tls , cert , key , issuer_cert , issuer_key , hosts , users , user , password_file , proxy , viewer , allow , deny Oc
Serve a live HTTP stream on
.Cm listen
.Pq default Li 127.0.0.1:8080 ,
@@ -446,6 +463,8 @@ is
or
.Cm files
.Pq Cm cert No and Cm key ;
.Cm hosts
adds names and addresses to a certificate made at startup;
.Cm users
names a credentials file, or
.Cm user
@@ -463,26 +482,38 @@ since a browser cannot bind its login to the TLS channel.
and
.Cm deny
list the addresses or CIDRs the listener admits and refuses.
.It Cm edge to Oo Cm path , transport , ca | pin , user , password_file , cert , key , node Oc
.It Cm edge to Oo Cm path , from , transport , ca | pin , server_name , user , password_file , cert , key , node Oc
Forward to an aggregator at
.Cm to
over TLS, by
.Cm tcp_chain
or, with
.Cm transport=http ,
.Cm http_chain .
.Cm http_chain ,
verifying it by
.Cm ca ,
by default the system roots, or
.Cm pin ;
.Cm server_name
is the name its certificate carries, by default the host of
.Cm to .
It refuses to run without credentials:
.Cm user ,
or a client
.Cm cert
and
.Cm key .
.It Cm aggregator Oo Cm listen , transport , tls , users , user , password_file , client_ca , out , format , allow , deny Oc
.It Cm aggregator Oo Cm listen , transport , tls , cert , key , issuer_cert , issuer_key , hosts , users , user , password_file , client_ca , out , format , allow , deny Oc
Receive from edges on
.Cm listen
.Pq default Li 0.0.0.0:9000
over TLS, never off
.Pq default Cm tls=self ,
.Po default Cm tls=self ;
.Cm cert , key , issuer_cert , issuer_key
and
.Cm hosts
as for
.Cm serve Pc ,
authenticated by the
.Cm users
credentials file, the one
@@ -523,6 +554,11 @@ lists the DNS names and addresses a server certificate carries
is also the subject, the identity
.Cm auth.type=mtls
checks.
One of
.Fl \-server
and
.Fl \-client
is required; both may be given.
.El
.Pp
A listener needs no files:
@@ -646,7 +682,7 @@ The directory holding
.It Ev LOGWISP_PIPELINE
The name of the one pipeline the variables below define.
.It Ev LOGWISP_PRESET
The preset that starts it.
The preset that begins it.
All of these are ignored when a pipeline option is on the command line.
.It Ev LOGWISP_SOURCE , LOGWISP_SINK , LOGWISP_FILTER
Source, sink and filter specifications, in the syntax of the matching
@@ -820,7 +856,7 @@ $ lw \-\-preset tail:path=/var/log/app
.Pp
Print the configuration a set of options makes, to keep as a file:
.Bd -literal -offset indent
$ lw \-\-preset serve:path=/var/log/app,tls=self \-\-dump > serve.toml
$ lw \-\-preset 'serve:path=/var/log/app/*,tls=self' \-\-dump > serve.toml
.Ed
.Pp
Ship a directory from an edge to an aggregator: on the aggregator, create
@@ -873,7 +909,7 @@ letting
.Nm
generate the password into a file, then reload the server:
.Bd -literal -offset indent
# install \-m 0600 \-o logwisp \-g logwisp \e
# install \-m 0640 \-o root \-g logwisp \e
/dev/null /etc/logwisp/users.toml
# lw auth add-user \-\-credentials /etc/logwisp/users.toml \e
\-\-user alice \-\-password-file /root/alice.pass
@@ -894,6 +930,8 @@ under the installation prefix and kept at
A listener without an
.Cm auth
block serves or accepts logs from anyone who can reach it.
Keep key, credentials and password files at mode 0600, owned by the account
Keep key, credentials and password files unreadable by others: mode 0600,
owned by the account
.Nm
runs as; startup warns about world-readable ones.
runs as, or 0640, owned by root and a group that account is in; startup
warns about world-readable ones.
doc/mtls-auth-plan.md
-335
View File
@@ -1,335 +0,0 @@
# mTLS as Authentication
**Status:** implemented. Phases 1–3 of the original proposal, plus dialer-side
server identity pinning from phase 4, are in the tree and covered by
`test/mtls-chain-test.sh`. The remaining phase-4 items are listed under
[Not Implemented](#not-implemented).
**Scope:** turn transport-level mutual TLS into an authentication and
authorization mechanism.
## Problem
LogWisp already did mutual TLS at the transport layer. A listener with
`client_auth = true` refused any peer that could not present a certificate
chaining to `client_ca_file`, and `internal/tlsx` extracted the peer's Common
Name into session metadata as `tls_peer_cn`.
Nothing read it back. The result was a CA-wide membership check with no notion
of *which* peer connected:
1. **No per-identity authorization.** Every certificate the CA issued was
equivalent. There was no way to say "only `edge-01` and `edge-02` may write
to this ingest port", so one CA could not serve several trust domains, and
withdrawing one peer meant rotating the CA bundle for all of them.
2. **Node labels were unauthenticated.** With `trust_node = true` (the default)
a peer declares its own `node` label in the chain hello or the
`X-Logwisp-Node` header. Any certificate holder could claim any label,
including another host's, and every downstream consumer would attribute
those entries accordingly. The only defence, `trust_node = false`, replaced
the label with a remote address — unforgeable, but useless for identifying a
host behind NAT or a load balancer.
3. **The `http` sink had no authentication at all**, even with TLS on. Its
stream and status endpoints were readable by anyone who could reach the port.
Password, token, and SCRAM authentication were removed during the plugin/flow
restructure and the move to standard-library networking. Certificates are the
one credential the transport already carries, which made mTLS the cheapest path
back to authenticated peers.
## Goals
- Authorize peers by certificate identity, per listener. ✅
- Bind the chain `node` label to the authenticated identity, so origin
attribution is trustworthy. ✅
- Gate the `http` sink's endpoints on client certificates. ✅
- Make identity visible in sessions, statistics, and logs. ✅
- Change nothing for existing configurations that omit the new block. ✅
## Non-Goals
- Password authentication. It is implemented separately as `type = "scram"`
([scram-auth-plan.md](scram-auth-plan.md)) on the same `auth` block and
`internal/authz` seam, carried by `chain.Hello` on TCP and a bearer token on
HTTP.
- IP allow/deny lists and per-peer rate limits. Related, but a separate feature
with its own config surface.
- OCSP. See [Revocation](#revocation) for what is done instead.
- Authorization *within* a stream — the unit of decision is a connection (TCP)
or a request (HTTP), never an individual entry.
## Design
### Identity
The authenticated identity is a single string derived from the peer's verified
leaf certificate. Because `tls.RequireAndVerifyClientCert` has already validated
the chain, signature, and validity window by the time we look, extraction is
pure field selection — `tlsx.PeerIdentity`. The `identity` modes:
- `cn` (default): `Subject.CommonName`, matching the existing `tls_peer_cn`
metadata.
- `san_dns`: the first `DNSNames` entry; preferred for host identities.
- `san_uri`: the first `URIs` entry; SPIFFE-style IDs.
- `san_email`: the first `EmailAddresses` entry; operator identities.
An empty identity is a rejection, not an empty match: a certificate with no
usable identity field cannot satisfy any policy.
Identities are not secrets, so ordinary string comparison is used; there is no
timing side channel worth defending here.
### Configuration
An `auth` table sits beside `tls` in every network plugin's `config`. Keeping it
separate from `tls` matters: TLS answers "is this channel private and does the
peer chain to a CA", auth answers "may *this* peer do *this*", and a later
non-certificate method can reuse the block.
```toml
[pipelines.plugin_sources.config.auth]
type = "mtls" # none (default) | mtls
identity = "cn" # cn | san_dns | san_uri | san_email
allow = ["edge-01", "edge-02"] # exact identities
allow_patterns = ["^edge-\\d{2}$"] # RE2, anchored by the author
node_binding = "force" # none | assert | force
```
- `type`, string, default `none`: `none` preserves pre-auth behaviour
exactly; `mtls` enables the policy.
- `identity`, string, default `cn`: which certificate field is the identity.
- `allow`, []string, default `[]`: exact identity matches.
- `allow_patterns`, []string, default `[]`: RE2 patterns matched against the
identity.
- `node_binding`, string, default `force` when `type = "mtls"`: chain sources
only; see below.
Empty `allow` **and** empty `allow_patterns` under `type = "mtls"` means "any
identity the CA vouches for" — that is, the pre-auth behaviour, but with the
identity now recorded and node binding available. It is a deliberate, documented
default rather than a silent deny-all, and the plugin logs a WARN at startup
saying so.
`node_binding` applies only to the chain sources, where a `node` label is
declared. Setting it to anything but `none` on any other plugin is a
configuration error. Its values:
- `none`: `trust_node` governs the connection label, as before, and the
per-entry `node` field.
- `assert`: the connection label must equal the identity, and a mismatch or an
omission is rejected; `trust_node` governs the per-entry `node` field.
- `force`: the declared label is ignored and the identity is used; every
per-entry `node` field is overwritten with the identity.
The split between `assert` and `force` is what makes both worth having:
- **`force`** is for an ingest boundary that does not trust its peer. Every
entry is relabeled, so a compromised edge cannot smuggle a foreign origin
through the per-entry `node` field either. It is the default under
`type = "mtls"` because it is the only setting where a misconfigured or
hostile edge cannot mislabel its entries.
- **`assert`** is for a relay-to-relay hop. The relay must prove *its own*
identity — a mismatch is loud rather than silently corrected — but the entries
it forwards keep the origin labels stamped at the first hop, so multi-hop
attribution survives.
`node_binding` overrides `trust_node` for the connection label. Under `force`
the constructor logs that `trust_node` is ignored; under `assert`, that it still
governs the per-entry labels.
Dialer-side plugins (`tcp_chain` and `http_chain` sinks) accept the same block
to pin the *server's* identity beyond hostname verification. There
`node_binding` does not apply, and `tls.insecure_skip_verify` is rejected:
identity read from an unverified chain is a claim, not a fact.
### Validation
At plugin construction, before anything binds:
- `type = "mtls"` on a listener requires `tls.enabled = true` and
`tls.client_auth = true`; on a dialer it requires `tls.enabled = true` and
forbids `tls.insecure_skip_verify`. Silently accepting an auth policy the
transport cannot enforce is the failure mode worth designing out.
- `identity` must be one of the four modes.
- Every entry in `allow_patterns` must compile.
- `node_binding` must be one of the three values, and must be absent or `none`
outside the chain sources.
Errors follow existing style: `auth: type "mtls" requires tls.client_auth`.
### The `internal/authz` package
One small package, mirroring `internal/tlsx`, is the single seam between the
`auth` block and the plugins. `New(o, tlsCfg, role, transport)` compiles a policy
against the sibling TLS config and returns `(nil, nil)` when auth is disabled;
every method tolerates a nil receiver, so call sites never branch on config.
Listeners admit only through `Admit` (TCP) and `AuthorizeRequest` (HTTP); the
current API, including SCRAM, is in
[scram-auth-plan.md](scram-auth-plan.md#the-authz-seam).
### Enforcement Points
**`tcp_chain` source** (`internal/source/tcpchain/tcpchain.go`, `handleConn`)
Handshake → `Admit` (authorize, then read hello) → `ResolveNode` → create
session. Under `mtls` the certificate is checked before the hello is read, so
an unauthorized peer never gets a preamble parsed on its behalf.
`chain.DecodeEntry` is then called with `auth.TrustsEntryNode(trust_node)`
rather than `trust_node` itself.
**`http_chain` source** (`internal/source/httpchain/httpchain.go`, `handleIngest`)
Per request, through `AuthorizeRequest`, before the body is read — an
unauthorized sender does not get to stream `max_body_bytes` into the process.
Rejection is `403`, distinct from the `400` used for protocol errors, so a
sender can tell "you are not allowed" from "your batch was malformed".
`ResolveNode` then governs the `X-Logwisp-Node` header exactly as it governs the
TCP hello. The session cache key includes the identity, so two peers sharing a
remote address never share a session.
**`tcp` sink** (`internal/sink/tcp/tcp.go`, `handleConn`)
After the explicit handshake, before the session is created and the client is
registered — so an unauthorized peer never appears in the client map and never
receives a broadcast.
**`http` sink** (`internal/sink/http/http.go`, `authMiddleware`)
A middleware around the mux covers both the stream and the status endpoint with
one wrapper and keeps the handlers themselves unaware of authorization. The
authorized identity is passed down through the request context for session
metadata. Rejections are `403` with no body detail — the status endpoint leaks
host, port, and throughput counters, so a rejection should not leak policy shape
on top of that.
**`tcp_chain` / `http_chain` sinks** (dialers)
The policy is installed as `tls.Config.VerifyConnection`, which runs after the
standard chain and hostname checks. A server whose identity the policy rejects
fails the handshake itself rather than the first write, and the chain sink's
existing backoff loop handles it as any other connect failure.
### Capabilities
`core.CapAuth` now means "this plugin authorizes peers" — it is derived from the
policy, not from `tlsConfig.ClientAuth`. Transport-level mTLS without a policy
still reports `CapTLS`, the `mtls=true` field on the startup log line, and the
`tls` statistic.
`Pipeline.initSourceCapabilities` and `initSinkCapabilities` treat this as a
cross-cutting check: a plugin advertising `CapAuth` without `CapTLS` is a
contradiction and fails pipeline construction rather than starting. An `http`
sink in SCRAM proxy mode also reports `CapProxyTLS`, which satisfies the check
without `tls`: TLS ends at a trusted proxy.
### Observability
Every authorization decision is visible, because a silent deny is
indistinguishable from a network fault at 3am.
- **Session metadata** gains `auth_method` and `auth_identity` alongside the
existing `tls` and `tls_peer_cn`.
- **Statistics** gain `auth`, `auth_identity` (the mode), `auth_unrestricted`,
`auth_allowed`, `auth_rejected`, and — on chain sources — `node_binding`, in
the `details` map of every affected source and sink. They surface in the
status reporter and in the `http` sink's status endpoint.
- **Logs** record a WARN per rejection with the remote address and the reason.
The startup line carries a rendered policy summary
(`auth="mtls identity=cn allow=[1 exact, 0 pattern(s)] node_binding=force"`),
a WARN when the allow list is empty, and an INFO when node binding overrides
`trust_node`.
### Revocation
Certificate revocation is handled by the allow-list rather than by CRL or OCSP:
1. Remove the identity from `allow` / `allow_patterns`.
2. `kill -HUP`.
The reload path rebuilds every pipeline, so the policy takes effect on the next
connection and existing connections are dropped by the rebuild itself. This is
one moving part instead of three, it needs no network calls on the handshake
path, and it is exact — no window between revocation and the next CRL
publication.
## Compatibility
No configuration breaks. Omitting the `auth` block, or setting `type = "none"`,
reproduces the previous behaviour exactly: `Policy` is nil, every call site
short-circuits, and `trust_node` continues to govern node labels.
The one behavioural note for adopters: turning on `type = "mtls"` defaults
`node_binding` to `force`, so entries from a peer whose certificate identity
differs from its configured `node` label will be relabelled. That is the point
of the feature, but it moves data between labels in a dashboard, so plan for it.
Use `node_binding = "assert"` on relay-to-relay hops where upstream origin
labels must survive.
## Verification
`test/mtls-chain-test.sh` builds a full PKI with `openssl` and exercises both
target topologies end to end:
```
./test/mtls-chain-test.sh --auto
```
Scenario 1 — chained instances, client authenticating with mTLS:
- an authorized edge (`edge-01`) delivers entries through both the `tcp_chain`
and `http_chain` ingest ports into a file sink
- `node_binding = "force"` overrides the label the sender configured
- an identity outside the allow list (`edge-99`) is refused, even while claiming
to be `edge-01`
- a peer presenting no certificate fails the handshake
- a dialer that pins a server identity the relay does not hold refuses to
connect, even though the server certificate chains to the trusted CA
Scenario 2 — a viewer client reading a streaming sink over mTLS:
- an authorized viewer streams from the `tcp` sink and from the `http` sink's
SSE endpoint, and reads `/status`
- a CA-valid but unauthorized viewer gets nothing from the `tcp` sink and `403`
from both `http` sink endpoints
- a client with no certificate fails the handshake
- the status endpoint reports the policy and its rejection count
`test/chain-test.sh` and `test/chain-aggregate-test.sh` continue to pass
unchanged, which is the regression check for the auth-disabled path.
## Not Implemented
The remaining phase-4 items, in rough order of value:
1. **CRL file support** alongside `client_ca_file`, re-read on reload, for
operators with existing CRL infrastructure. The allow-list covers the same
ground with fewer moving parts, so this is only worth doing for a fleet whose
revocation already flows through a CRL.
2. **Per-identity rate limits.** The natural follow-on now that identity exists:
the `acl` block's per-client limits count addresses
([Security](security.md#per-client-limits)), and would be keyed by identity
once a peer is authenticated.
3. **Per-client identity in the `http` sink's status output.** The endpoint
reports the policy and counters, but not which identities are currently
connected; session metadata has the data.
4. **A list of `identity` modes** (try `san_uri`, fall back to `cn`) for
heterogeneous PKI. A single mode is simpler and covers a uniform CA.
## Decisions Taken
Four questions were left open by the proposal. What was chosen, and why:
1. **An empty allow-list allows rather than denies.** Deny-by-default is the
safer instinct, but `type = "mtls"` with no list is a legitimate
configuration — "any peer this CA issued, but bind the node labels" — and
node binding alone is worth enabling without enumerating every node.
Erroring on it would force operators to list their whole fleet to get
trustworthy attribution. The compromise is a WARN at startup naming the
condition and the fix.
2. **`identity` takes a single mode.** Simpler, and a uniform CA is the common
case. Listed above as a possible extension.
3. **Per-identity rate limits are out of scope**, as proposed.
4. **`assert` rejects rather than warns and corrects.** A certificate/config
mismatch under `assert` is an outage, which is the point: `force` is the
forgiving option and it is the default, so an operator reaches for `assert`
precisely when they want the mismatch to be loud.
doc/networking.md
+57 -63
View File
@@ -57,11 +57,11 @@ Dialers:
- `http_chain` sink: chain v1; forwards entries as batched POSTs.
There is no port registry and no default port: `port` is required on every
network plugin. There is also no cross-pipeline conflict detection — two sinks
on the same port fail at bind time when the pipeline starts:
network plugin. Two sinks on the same port pass `lw --check` and fail at bind
when the service starts; lw exits 1:
```
ERROR msg="Failed to start sink" error="tcp sink bind 0.0.0.0:9090: listen tcp4 0.0.0.0:9090: bind: address already in use"
ERROR msg "Failed to initialize service" error "failed to start service pipelines: failed to start pipeline <name>: failed to start sink <id>: tcp sink bind <host>:<port>: listen tcp4 <host>:<port>: bind: address already in use"
```
## Timeouts
@@ -91,7 +91,8 @@ Fixed, non-configurable bounds:
- TLS handshake: 10 s, on all TLS listeners and dialers.
- HTTP read-header timeout: 10 s, on the `http` sink and `http_chain` source.
- HTTP server shutdown grace: 2 s, on the `http` sink and `http_chain` source.
- HTTP server shutdown grace: 2 s on the `http_chain` source and the `http`
sink; on the `http` sink, `write_timeout_ms` instead when set below 2 s.
- Single entry line: at most 1 MiB, on chain listeners.
- SCRAM login: 10 s, on the `tcp` sink, chain sinks, `lw auth`, and each
`/auth` request.
@@ -103,15 +104,16 @@ would terminate long-lived SSE streams; per-event deadlines come from
## Connection Limits
`max_connections` caps concurrent connections on the `tcp` sink, the `http`
sink, and the `tcp_chain` source. `0` means unlimited.
`max_connections` caps concurrent connections on the `tcp` sink and the
`tcp_chain` source, and concurrent streams on the `http` sink (other requests
are not counted); `0` means unlimited.
- Admission is a load-then-check, so a burst can over-admit by roughly one
connection. This is accepted, not a bug to work around.
- Admission is a load-then-check, so a burst can over-admit by roughly one.
- On the `tcp` sink and `tcp_chain` source the count is taken at accept, so it
bounds concurrent TLS handshakes as well as established sessions.
- Over-limit connections are closed immediately and counted in `rejected_conns`
(TCP) or `rejected_clients` (HTTP, which first answers `503`).
- Over the limit a TCP connection is closed and counted in `rejected_conns`;
an `http` stream request is answered `503` and counted in
`rejected_clients`, its connection kept.
The `http_chain` source has no connection cap; it bounds work with
`max_body_bytes` and `read_timeout_ms` instead.
@@ -123,39 +125,24 @@ pipeline-wide entry rate limit: it cannot distinguish or throttle a peer.
## Keep-Alive
TCP keep-alive is available on the `tcp` sink (for accepted connections) and the
`tcp_chain` sink (for its outbound connection):
The `tcp` sink (accepted connections) and the `tcp_chain` sink (its outbound
connection) configure TCP keep-alive:
```toml
keep_alive = true
keep_alive = true # false sends no probes
keep_alive_period_ms = 30000
```
This is kernel-level keep-alive; it detects a dead peer but does not keep an
application-level stream flowing. For that, use a heartbeat.
Probes start after `keep_alive_period_ms` idle. Other listeners and the
`http_chain` sink use Go's default TCP keep-alive (15 s); it is not
configurable there. Keep-alive detects a dead peer but does not keep an
application-level stream flowing: that is a heartbeat's job.
## Heartbeats
Heartbeats are a **flow-level** feature, not a per-sink one. Enabling one
injects a synthetic entry into the pipeline at a fixed interval; it reaches
every sink and traverses chain links as an ordinary structured entry.
```toml
[pipelines.flow.heartbeat]
enabled = true
interval_ms = 30000
include_timestamp = true
include_stats = false
format = "txt" # txt | json | raw
```
Use it to keep idle SSE clients, TCP clients, and chain links from being
reaped by intermediate NAT or proxy timeouts, and to make an idle pipeline
visibly alive.
> `format = "comment"` (SSE `:` comment framing) is rejected by validation
> despite appearing in older documentation and in a still-present code branch.
> A pipeline configured with it fails to start.
A flow heartbeat keeps idle TCP clients and chain links alive through NAT and
proxy timeouts; idle SSE streams already get a comment every 15 s. See
[Configuration](configuration.md#heartbeat).
## Reconnection
@@ -169,35 +156,34 @@ backoff_max_ms = 30000
```
The connection is established lazily, so an edge node starts cleanly even when
its relay is down and connects as soon as the relay appears. Reconnect counts
are reported in the sink's `reconnects` statistic.
its relay is down and connects as soon as the relay appears. The `tcp_chain`
sink reports `connected` and `reconnects`; the `http_chain` sink reports
`request_errors` and `dropped_batches`.
Server-side sinks (`tcp`, `http`) do not reconnect; clients are expected to
retry. Browsers reconnect SSE streams automatically.
## Protocol Details
**HTTP sink (SSE)** — HTTP/1.1 in plaintext; HTTP/2 is negotiated via ALPN when
TLS is enabled. Only `GET` is routed to the stream and status paths. Each event
is framed as one `data:` line per newline in the payload, so multi-line entries
stream intact. Response headers set `Cache-Control: no-cache`,
`X-Accel-Buffering: no`, and, without an auth policy,
`Access-Control-Allow-Origin: *`. Under `scram`, `POST /auth` serves the login
and both paths take `Authorization: Bearer <token>`.
**HTTP sink (SSE)**: HTTP/1.1 in plaintext, HTTP/2 via ALPN under TLS; `GET`
on the stream path, `GET` or `HEAD` on the status path. Framing and headers:
[Sinks](sinks.md#http). Under `scram`, `POST /auth` logs in and both paths
take `Authorization: Bearer <token>`.
**TCP sink** — raw payload bytes, no framing added by the sink. Whether entries
are newline-delimited depends on the formatter. Under `scram` the client's hello
and login lines come first, as on a [chain link](chaining.md#tcp-transport).
**TCP sink**: the formatted payloads, each ending in a newline, with no other
framing; under `json` that is one entry per line, while `raw` passes a
message's own line breaks through. Under `scram` the client's hello and login
lines come first, as on a [chain link](chaining.md#tcp-transport).
**Chain transports** — see [Chaining](chaining.md) for the hello preamble,
headers, and entry encoding.
**Chain transports**: see [Chaining](chaining.md#wire-protocol) for the hello
preamble, headers, and entry encoding.
## Troubleshooting
**Plugin fails to start with `unknown key "..."`**
- The key is misspelled or belongs to another plugin type. Every plugin rejects
keys it does not declare, nested tables included, so a typo in `tls` or `auth`
cannot leave a protection silently off.
**`unknown key "..."` when the configuration loads**
- The key is misspelled or belongs to another plugin type. A plugin's keys are
checked at load (`lw --check` too), nested tables included, so a typo in
`tls` or `auth` cannot leave a protection silently off.
**Connection refused**
- Confirm the pipeline started; a bind failure is logged at ERROR.
@@ -252,18 +238,23 @@ headers, and entry encoding.
`tls_handshake_errors`.
**Rejected after a successful handshake**
- `auth: identity "..." is not allowed` — the certificate is valid and chains to
the CA, but the identity is not in `auth.allow` / `auth.allow_patterns`. On
TCP the connection is closed; on HTTP the answer is `403`.
- `auth: peer certificate carries no <mode> identity` — `auth.identity` names a
field the certificate does not populate, e.g. `san_dns` on a CN-only leaf.
- `auth: node_binding "assert": declared node "..." does not match identity` —
the sender's `node` option and its certificate disagree. Fix one, or use
- `auth: refused: identity "<id>" is not allowed` — the certificate is valid
and chains to the CA, but the identity is not in `auth.allow` /
`auth.allow_patterns`. On TCP the connection is closed; on HTTP the answer
is `403`.
- `auth: refused: peer certificate carries no <mode> identity` —
`auth.identity` names a field the certificate does not populate, e.g.
`san_dns` on a CN-only leaf.
- `auth: node_binding "assert": declared node "<node>" does not match identity "<id>"`
— the sender's `node` option and its certificate disagree. Fix one, or use
`node_binding = "force"` to let the certificate win silently.
- On a `tcp_chain` sink the same message inside `Chain connect refused` means
the *server* was refused: its certificate identity is not in the sink's
`auth.allow`.
- Rejections appear as WARN and increment `auth_rejected`.
- Allow-list and login rejections log WARN and count `auth_rejected`; a
node-binding refusal logs `Connection rejected by node binding`
(`tcp_chain`, `rejected_conns`) or `Request rejected by node binding`
(`http_chain`, `rejected_requests`).
**SCRAM login fails**
- `authentication failed` — the dialer is not told why, by design. The
@@ -334,11 +325,14 @@ headers, and entry encoding.
send the header.
**Entries not arriving over a chain link**
- Check the sink's `connected` statistic and its `reconnects` count.
- `tcp_chain`: check the sink's `connected` and `reconnects`; `http_chain`:
its `request_errors` and `dropped_batches`.
- Check the source's `auth_rejected` — an allow-list miss or a failed login
looks exactly like a network fault from the sender's side — and the sink's
`last_auth_error`.
- Check the source's `parse_errors` — a version skew shows up here.
- A version skew: `tcp_chain` logs WARN `Rejected chain connection`
(unsupported protocol version), counted only under `scram`, in
`auth_rejected`; `http_chain` answers `400` and counts `rejected_requests`.
- On `http_chain`, remember entries wait up to `flush_interval_ms` before a
batch is sent.
doc/operations.md
+66 -82
View File
@@ -13,15 +13,9 @@ tail -F app.log | lw --filter include:patterns=ERROR
lw -c /etc/logwisp/logwisp.toml
```
lw exits on its own only at the end of input: once every pipeline has
finished, a pipeline finishing when all its sources have ended. Only a console
source ends, when stdin does; file and network sources run until a signal, and
lw shuts down gracefully, exit 0, either way.
Every sink writes what is queued before the exit; a network sink gives its
clients or its downstream its `write_timeout_ms` (`request_timeout_ms` for
`http_chain`), at most 2 s, to take it. A reload stops the old sinks the same
way, so a stalled client or downstream delays it by that much. See
[CLI](cli.md#built-in-defaults).
lw exits on its own only at the end of input ([CLI](cli.md#built-in-defaults)).
A reload stops the old sinks as shutdown does, so a stalled client or
downstream delays it by up to 2 s.
There is no built-in daemon mode. Run LogWisp in the foreground under a
supervisor — systemd, rc.d, or a container runtime — which is where restart,
@@ -63,13 +57,13 @@ kill -HUP $(pidof lw)
```
Signals reread the selected TOML file even when `auto_reload` is disabled.
Reload validates a detached snapshot and constructs a new service **before**
tearing the old one down, so invalid configuration leaves the running service
intact and logs the failure:
```
ERROR msg="Failed to bootstrap new service, keeping old service running" error=...
```
Reload validates a detached snapshot and builds a new service **before**
tearing the old one down, so a bad configuration leaves the running service
intact. An invalid one logs `Failed to read reload configuration` with the
validator's `validate configuration: <key path>: ...` error. One whose files
fail when its plugins are built (a missing certificate, an unreadable
credentials file) logs `Failed to bootstrap new service, keeping old service
running`.
What reload does *not* do:
@@ -84,32 +78,16 @@ What reload does *not* do:
Standard input survives a reload: one reader serves the process, so the new
console source continues where the old one stopped, and no line is read twice.
Plan reloads on a busy relay the way you would plan a restart.
Listener bind/start failures happen after the old service stops; these can leave
the application without working pipelines until a corrected configuration reloads.
File-watch errors do not restart services, and queued changes are combined.
Plan reloads on a busy relay as you would a restart: a listener that fails to
bind does so after the old service stopped, leaving lw without working
pipelines until a corrected configuration reloads
([Hot Reload](configuration.md#hot-reload)).
### Checking a configuration
`lw --check` builds every pipeline and plugin as a start would, without binding
a port or opening, reading or creating a log file, prints the startup warnings,
and exits 0, 1 on an invalid configuration, or 2 when a named file is missing:
```bash
lw --check -c candidate.toml
```
Before a reload, check the edited file this way: a failed reload keeps the
running service, but `--check` says why without touching it. It prints the
first problem of each setting, flow stage, source, sink and pipeline, a line
each named by its key's path; once none is left, the files they name are
opened as a start opens them:
```text
Error: Failed to load config: invalid configuration:
pipelines[0].plugin_sources[in].config.directory: must be set
pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000
```
Before a reload, run `lw --check -c FILE` on the edited file: a failed reload
keeps the running service, and `--check` says why without touching it
([CLI](cli.md#usage-patterns)).
## Monitoring
@@ -143,27 +121,35 @@ curl -s http://127.0.0.1:8080/status | jq .
```json
{
"service": "LogWisp",
"version": "v0.16.0",
"endpoints": {
"status": "/status",
"stream": "/stream"
},
"instance_id": "sse",
"server": {
"type": "http",
"host": "0.0.0.0",
"port": 8080,
"tls": false,
"acl": "none",
"active_clients": 3,
"auth": "none",
"buffer_size": 1000,
"client_buffer_size": 256,
"host": "0.0.0.0",
"max_connections": 32,
"write_timeout_ms": 5000,
"uptime_seconds": 8130
"port": 8080,
"tls": false,
"type": "http",
"uptime_seconds": 8130,
"write_timeout_ms": 5000
},
"endpoints": { "stream": "/stream", "status": "/status" },
"service": "LogWisp",
"statistics": {
"total_processed": 15234,
"acl_denied": 0,
"acl_limited": 0,
"auth_rejected": 0,
"dropped_writes": 12,
"rejected_clients": 0
}
"rejected_clients": 0,
"total_processed": 15234
},
"version": "vX.Y.Z"
}
```
@@ -188,18 +174,24 @@ Each counter, where it is reported, and what a rise means:
auth refusals
- `tls_handshake_errors` (`tcp` sink, `tcp_chain` source): a certificate or
version mismatch, or scanning
- `parse_errors` (chain sources): protocol or version skew upstream
- `parse_errors` (chain sources): malformed entry lines from upstream, each
skipped
- `reconnects` (`tcp_chain` sink): an unstable link or a flapping downstream
- `dropped_batches` (`http_chain` sink): downstream is rejecting batches
permanently
- `synthesized` (chain sinks): events reach the sink without structure
- `acl_denied`, `acl_limited` (a listener whose `acl` block sets a rule or
limit; the `http` status always): connections or requests the address
rules or PROXY-header checks refused, and those over a per-client limit;
`acl_proxy_headers` counts the PROXY headers read
## Log Management
LogWisp's own operational log goes to stderr by default, at `info` with a
configuration file and at `warn` without one; explicit values win. A container
configured only through variables or flags therefore logs at `warn`: pass
`-e LOGWISP_LOGGING_LEVEL=info`. `lw-deploy.sh` always passes `-c`, so its
LogWisp's own log goes to stderr by default. Without a configuration file lw
logs less ([CLI](cli.md#built-in-defaults)), so a container configured only
through variables passes `LOGWISP_LOGGING_LEVEL=info` and
`LOGWISP_STATUS_REPORTER=true`; the image sets neither, as a variable would
override a mounted file's value. `lw-deploy.sh` always passes `-c`, so its
nodes log at `info`. To files:
```toml
@@ -226,19 +218,14 @@ the filter stage logs several lines per entry evaluated.
### Buffers
Raise `buffer_size` when `total_dropped_by_sink` is climbing but the sink itself
is healthy — that is a burst-absorption problem.
`dropped_writes` on a network sink has two causes that a counter alone does not
separate. A consumer slower than the sustained rate cannot be bought off with
buffer, and drops are the intended outcome. A burst the consumer would have
drained, arriving faster than it reads, is configuration: the sink queues a
whole burst while the client writes one frame at a time, so the part of a burst
above `client_buffer_size` is lost even to a loopback reader.
Where a `rate_limit` bounds the pipeline, its `burst` is that number — keep
`client_buffer_size` at or above it and the second cause disappears. The HTTP
status endpoint reports both queue bounds alongside the counters so an operator
can tell which one is in play.
- `total_dropped_by_sink` climbing while the sink is healthy: a burst
outran its queue; raise `buffer_size`
- `dropped_writes` on a network sink:
- a client slower than the sustained rate: drops are intended
- a burst above `client_buffer_size`, lost even to a fast reader: keep
`client_buffer_size` at or above the burst a `drop` rate limit lets
through, its `burst_entries` (`entries_per_second` when 0)
- the HTTP status endpoint reports both queue bounds beside the counters
```toml
[pipelines.plugin_sinks.config]
@@ -256,10 +243,8 @@ policy = "drop"
max_entry_size_bytes = 65536
```
Two behaviours to keep in mind: the limiter does not exist at all when
`entries_per_second <= 0`, and `policy = "pass"` short-circuits the size cap as
well as the rate check. Enforcing `max_entry_size_bytes` therefore requires
`entries_per_second > 0` and `policy = "drop"`.
Enforcing `max_entry_size_bytes` needs a rate above 0 and `policy = "drop"`
(a file's default is `pass`); see [Rate limiting](configuration.md#rate-limiting).
### Formatting
@@ -314,10 +299,10 @@ problems see [Networking](networking.md#troubleshooting).
**Environment variable override has no effect**
LogWisp reads `LOGWISP_QUIET`, `LOGWISP_LOGGING_LEVEL`, and other prefixed
names. Bare names used by older versions must be renamed. Array-indexed paths,
such as a key inside `[[pipelines]]`, cannot be set from the environment or the
command line; define whole pipelines there with the
Only `LOGWISP_`-prefixed names are read (`LOGWISP_QUIET`,
`LOGWISP_LOGGING_LEVEL`). Array-indexed paths, such as a key inside
`[[pipelines]]`, cannot be set from the environment or the command line;
define whole pipelines there with the
[pipeline flags](cli.md#pipelines) or
[pipeline variables](cli.md#pipeline-variables).
@@ -355,11 +340,10 @@ than masking parts of them. See [Filters](filters.md#common-recipes).
**Upgrades**
1. Read the changelog for configuration-schema changes.
2. Start the new binary against the current configuration in a scratch
environment.
3. Stop the old process, install the new binary, start it.
4. Confirm each pipeline started and that counters are advancing.
1. Run the new binary's `lw --check -c FILE` against the current
configuration: unknown or renamed keys fail there.
2. Stop the old process, install the new binary, start it.
3. Confirm each pipeline started and that counters are advancing.
**Backup**
doc/scram-auth-plan.md
-419
View File
@@ -1,419 +0,0 @@
# Password Authentication (Argon2id-SCRAM)
**Status:** implemented, in three batches: `lixenwraith/auth` gained SCRAM
channel binding and the mTLS layer was hardened (see
[mTLS Hardening](#mtls-hardening)); then the SCRAM integration described here,
on the `auth` release that carries channel binding; then browser logins behind
a TLS-terminating proxy. Operator documentation lives in
[Security](security.md#password-authentication-scram) and
[`lw auth`](cli.md#lw-auth).
**Scope:** optional username/password authentication on every network plugin,
next to the existing certificate method, without weakening what mTLS gives.
## Goals
- Password authentication for chain links (`tcp_chain`, `http_chain`: sources
verify, sinks present) and for viewers of the `tcp` and `http` sinks.
- No password or password-equivalent on the wire, no KDF work per login on
listeners, and no credential relay through a TLS-terminating MITM.
- A temporary pipe needs no file: one listener user from a descriptor, a pipe
or the terminal, with a certificate made at startup.
- Default stays `auth.type = "none"`; existing configurations are unaffected.
- Config in TOML, credential management and viewer clients in a `lw auth`
CLI.
## Non-Goals
- Standard SASL SCRAM-SHA-256 interoperability. The `auth` exchange is
logwisp-to-logwisp (or logwisp CLI) only.
- Browser viewers of a logwisp that terminates TLS itself: a browser cannot see
the server certificate, so its proof cannot be channel-bound. Browsers log in
through [proxy mode](#browser-viewers-behind-a-tls-terminating-proxy), or use
mTLS.
- Multiple backends behind a non-sticky L7 balancer: handshake state and the
token key live in one process.
## Mechanism
`lixenwraith/auth` Argon2id-SCRAM. The listener stores a verifier per user:
`StoredKey = SHA-256(HMAC(K, "Client Key"))`, `ServerKey = HMAC(K, "Server Key")`,
`K = Argon2id(password, salt)`. The Argon2id digest itself is never stored: it is
the salted password, from which both keys derive. Verifying a proof is HMAC only;
Argon2 runs on the dialer or CLI (t=3, 64 MiB, 4 threads, the `auth` defaults).
The dialer refuses a challenge below that cost, so a rogue server cannot ask for
a cheaply guessable proof, and trusts the link only after the server's final
signature proves it holds the user's `ServerKey`.
**Channel binding.** Both peers pass `auth.WithChannelBinding(SHA-256(server leaf
DER))` at the proof step. The server hashes its configured certificate; the
client hashes the certificate it saw. A relay presenting any other certificate,
even one the CA trusts, makes the server reject the proof, so it never obtains a
link or a token. A mismatch is indistinguishable from a wrong password on the
wire by design; the client also sends its hash as a diagnostic field, so the
listener logs "TLS interception or a terminating proxy" and counts it. TLS must
terminate at logwisp, except in [proxy mode](#browser-viewers-behind-a-tls-terminating-proxy).
**HTTP pinning.** HTTP auth takes two requests and every later request may use a
new pooled connection, so binding the exchange is not enough. Once the challenge
arrives the dialer pins the certificate that answered on its `VerifyConnection`:
the proof and every ingest request must present the same leaf, or the handshake
fails before anything is sent. The `http_chain` sink then clears token and pin
and logs in again, which also covers certificate rotation. A token handed to
curl by the CLI is unpinned.
## Configuration
```toml
# listener (tcp_chain/http_chain source, tcp/http sink)
[pipelines.plugin_sources.config.auth]
type = "scram"
credentials_file = "/etc/logwisp/users.toml" # or username + password_file
token_lifetime_ms = 900000 # http sink and http_chain source only
node_binding = "force" # chain sources; binds to the username
# dialer (tcp_chain/http_chain sink)
[pipelines.plugin_sinks.config.auth]
type = "scram"
username = "edge-01"
password_file = "/etc/logwisp/edge-01.pass"
```
Validation at construction:
- `scram` requires TLS, except on an `http` sink in proxy mode; a dialer also
forbids `insecure_skip_verify`.
- Listeners require `credentials_file`, or `username` and `password_file`
(one user, its verifier derived in memory at construction), not both, and
reject `allow`/`allow_patterns`: the users are the allow list. Use one file
per listener when listeners admit different users.
- Dialers require `username` + `password_file`, and reject `credentials_file`,
`token_lifetime_ms`, `allow*` and `identity` (server pinning stays an mTLS
feature).
- A password is 1–1024 bytes after trimming one line break, 8 or more on a
listener. A regular file is read at every construction; `/dev/fd/N` (the
inherited descriptor, then closed), a pipe, a FIFO or another device once, at
startup, and kept for reloads; a terminal is asked once, at startup, without
echo, and on a listener Enter alone generates a password shown only there.
The open file decides, not stat: FreeBSD stats `/dev/stdin` as a device
whatever it holds. No terminal fails closed.
- `identity` on a listener requires `tls.client_auth` and **binds the certificate
to the user**: the certificate's identity field must equal the SCRAM username,
so a peer needs its own certificate and its own password (PostgreSQL's
`clientcert=verify-full`). Without it the factors are independent: any
CA-issued certificate plus any valid password (`clientcert=verify-ca`).
- `token_lifetime_ms` only on HTTP listeners; default 15 minutes.
- `/auth` and every path under it are reserved.
- `mtls` rejects every SCRAM key, `trusted_proxies` included. A block that
names peers or credentials with `type = "none"` is refused: the type was
forgotten, not the block.
**Credentials file** (TOML, written by the CLI, mode 0600):
```toml
decoy_key = "<base64, 32 random bytes>"
[[users]]
argon_memory = 65536
argon_threads = 4
argon_time = 3
salt = "<base64>"
server_key = "<base64>"
stored_key = "<base64>"
username = "edge-01"
```
`decoy_key` keeps unknown-user challenges stable across restarts and edits, so
a single probe cannot tell a real user from an unknown one (a real user's salt
changes when it is added, rotated or removed). It is created by the first
`add-user` and preserved by every rewrite. An empty user set, a duplicate
user, a missing or short `decoy_key`, an unknown key, or users differing in KDF
profile or salt length are configuration errors. The file is read at
construction; the SCRAM server starts in the plugin's `Start` and stops in
`Stop`, so a rejected reload leaks nothing.
## Wire Protocol
`chain.Hello` gains `scram` (raw JSON, decoded by `authz`); the protocol version
stays 1 because the field is additive. Every message after the hello is one
`authStep`: `error`, `challenge`, `proof` (+ `binding`), `final` (+ `token`,
`expires_in` on HTTP). Pre-auth lines and `/auth` bodies are capped at 4 KiB.
**TCP** (`tcp_chain`, `tcp` sink): hello carrying the client-first message,
challenge, proof, final — all under one deadline (`hello_timeout_ms` on the
`tcp_chain` source, 10 s on the `tcp` sink and every dialer, with the dialer's
context able to cut it short). The final is written only after every admission
check (certificate binding, node binding); a later refusal is an `error` line
instead. The stream then continues on the same buffered reader. A listener
without SCRAM that reads a hello (the `tcp_chain` source) answers one carrying
credentials with `error: authentication not enabled`; an old binary stays
silent, and the dialer gives up after its deadline with "no challenge within
10s".
**HTTP** (`http_chain` source, `http` sink): `POST /auth`, outside the auth
middleware, with its own 10 s read/write deadline. It answers `404` unless the
policy is `scram`. Each step, by its request body:
- Step 1, the hello
- Success: `200` with the challenge.
- Failure: `429` throttled, `503` busy, `400` malformed, `413` oversized.
- Step 2, the proof
- Success: `200` with the final, `token` and `expires_in`.
- Failure: `401 {"error":"authentication failed"}`, or `503` if the server
stopped.
Protected endpoints take `Authorization: Bearer <token>`: missing or invalid is
`401` with `WWW-Authenticate: Bearer realm="logwisp"`, a certificate-binding miss
is `403`. Tokens are HS256 JWTs from `auth.NewJWT` (issuer `logwisp`, no leeway)
with a random per-instance key, so every reload revokes them. An SSE stream is
checked at connect and outlives its token; a reload ends it.
**Dialer outcomes** (`http_chain` sink): every `/auth` failure is transient (the
batch is held under backoff); `404`/`405` from `/auth` reads "no auth endpoint
at <url> (older logwisp, or not a scram listener)"; an ingest `401` clears the
token and is retried (tokens are renewed ahead of expiry, so this follows a
listener reload); `403` drops the batch; a pin mismatch clears token and pin.
Refusals are logged at WARN on every attempt, paced by the backoff (`Chain
connect refused` on `tcp_chain`, `Chain batch delivery failed` on
`http_chain`), and failed logins are counted in `auth_failures` /
`last_auth_error`.
## Throttling
Per remote IP (the socket address; in proxy mode the forwarded client, an IPv6
one per /64), on handshake starts: a token bucket (burst 10, 1/s) refunded on
success, and at most 4 unfinished exchanges. An unanswered HTTP challenge holds
its slot for the `auth` handshake timeout (30 s); abandoned TCP exchanges
release their slot, and their entry in the `auth` handshake table,
immediately. The address table, `netacl.Table` as for the `acl` block's
per-client limits, holds 65,536 entries, drops one once its bucket has
refilled and its challenges have expired, and fails closed when full; the SCRAM
server itself caps in-flight handshakes at 4,096 (`busy`).
Counters: `auth_allowed` counts logins; `auth_rejected` every refusal — failed
proofs, malformed requests, missing credentials and, on HTTP, refused tokens;
`auth_binding_mismatch` is the subset of failed proofs whose client saw another
certificate. `auth_throttled` and `auth_busy` count starts refused before a
challenge and are not in `auth_rejected`.
## The `authz` Seam
Listeners never call `Authorize` directly any more; one entry point per transport
decides, and `Authorize` itself refuses under `scram` so a missed call site fails
closed.
- `Admit(conn, cs, wantHello, timeout)` returns an admission (identity, hello,
reader) whose `Accept` or `Reject` writes the final message.
- `AuthorizeRequest(r)` covers mTLS certificates and bearer tokens, returning
the refusal status for `Refuse`; `ServeAuth(w, r)` serves `/auth`.
- Dialers: `Greet(ctx, conn, node)` writes the hello and, under SCRAM, runs the
TCP exchange. Over HTTP `Prepare` logs in when no token is held and sets the
bearer header, `Invalidate` drops the token after a `401` or a pin mismatch,
`Token` is the login itself (behind `Prepare`, and for the CLI), and
`VerifyConnection` pins.
- `Start`/`Close` bracket the SCRAM server and are nil-safe like everything else.
## CLI
```
lw auth add-user --credentials F --user U [--password-file P] [--generate]
lw auth remove-user --credentials F --user U
lw auth token --url https://host:port[/path] --user U [--password-file P] [--unbound] [TLS flags]
lw auth stream --addr host:port --user U [--password-file P] [TLS flags]
```
- `add-user` takes the password from `--password-file` when it exists (at least
8 bytes); a new user without one gets a generated 130-bit password, written to
that file (0600) or printed once. Rotating an existing user needs the file or
`--generate`, so a mistyped path cannot replace a password. The file's KDF
profile and decoy key are reused, the result is validated with the daemon's
own parser, and the rewrite (temp file beside the file or a symlink's target,
rename) keeps the existing mode and owner, or fails.
- `remove-user` refuses to remove the last user.
- Both print that changes apply on `SIGHUP`: `auto_reload` does not watch the
credentials file.
- `token` prints a bearer token for curl (`-H @<(...)` keeps it out of argv);
`stream` authenticates to a `tcp` sink and copies the stream to stdout until
interrupted. `--url` takes a path only with `--unbound`, and redirects are not
followed. `--password-file` defaults to `/dev/tty`, which asks.
- TLS flags: `--ca-file` or `--pin-sha256`, `--server-name`, `--cert-file`,
`--key-file`. There is no insecure flag.
- Exit status: 0 success, 1 failure, 2 usage error.
## Rollout and Rotation
- Turning SCRAM on is per listener and cuts off unconfigured dialers. To migrate
without a gap, upgrade binaries, add a second SCRAM listener, move dialers,
then remove the old listener.
- Password rotation: `add-user --generate`, `SIGHUP` the listener, deploy the new
password file, `SIGHUP` the dialer. The dialer retries under backoff in
between.
- Revocation: `remove-user`, `SIGHUP`. All tokens die with the reload.
## Browser Viewers Behind a TLS-Terminating Proxy
The deployer serves the log stream inside a website whose TLS ends at its
reverse proxy; LogWisp holds no private key and listens on a trusted hop.
Viewers log in from a browser with the client LogWisp ships, and the password
reaches neither the proxy nor LogWisp. Operator view and example:
[Security](security.md#browsers-behind-a-tls-terminating-proxy).
**Trust model.** `auth.trusted_proxies` (addresses or CIDRs) switches the `http`
sink into proxy mode; it is refused on every other plugin and with `identity`.
- Only listed peers may connect: `403` on every path, the pages included.
- `X-Forwarded-Proto` must be `https` on every hop, so a site served in
plaintext by mistake fails closed instead of exposing sessions.
- The client is the rightmost `X-Forwarded-For` hop that is not a proxy (hops to
its left are the client's own claims), or the leftmost when every hop is
inside a proxy range; throttling (an IPv6 client per /64, as one host holds
it whole), sessions and logs use it. Outside proxy mode forwarded headers
stay ignored.
- SCRAM runs unbound: a browser cannot read the site's certificate, and LogWisp
sees none. The proof and the session ride on the site's TLS; a party on the
proxy-to-LogWisp hop could relay a login, hence the address restriction and a
startup warning for a plaintext hop to a proxy off this host. `tls` may stay
off. Outside proxy mode nothing changes: proofs stay bound, and a bound proof
in proxy mode fails with a hint naming `--unbound`.
**Session.** The proof step may ask for a cookie (`"session": "cookie"`, proxy
mode only). The answer sets `logwisp_session` (`HttpOnly; Secure;
SameSite=Strict; Max-Age=<lifetime>`) and carries no token in the body. It has
no `Path`: RFC 6265 defaults it to the directory of `POST <mount>/auth`, which
is the mount behind any proxy prefix. Stream and status accept the cookie or
`Authorization: Bearer`; another `Authorization` scheme, such as the site's own
Basic auth, leaves the cookie in charge. Logout is `{"logout": true}` on
`POST /auth` itself, because a clearing cookie set from `/auth/logout` would
default to another path and miss the login cookie. It clears the cookie and
revokes a valid token until it expires (a set of at most 65,536 hashes, gone on
reload like every token). An open stream outlives its token; a reconnect after
expiry gets `401`. `/auth` takes only `application/json`, which a cross-origin
page cannot send without a preflight nobody answers.
**Token mode.** A proof step without `"session"` gets the token in the answer
body, in proxy mode as everywhere else, so a browser that keeps no cookies needs
no server change. `scram.js` asks for it with `{session: "token"}`, reads the
stream through `fetch` with the bearer (`EventSource` cannot send one), and
revokes it with `logout(base, {token})`. The viewer falls back to it when a
cookie would not stick or a fresh cookie login still gets `401`, and keeps the
token in a page variable only: it lasts no longer than the page or its lifetime.
**Endpoints.** `/auth` and every path under it are reserved; every URL in the
pages is relative, so a proxy prefix works unchanged.
- `POST /auth`: SCRAM hello, proof or logout; unbound in proxy mode.
- `GET /auth/scram.js`, `style.css`: the client library and the pages' style,
always served in proxy mode, and with the viewer.
- `GET /`: `303` to `auth/view`, else `auth/login`, when either is served.
- `GET /auth/login`, `login.js`: the login page (`login_page`).
- `GET /auth/view`, `view.js`: a minimal live viewer (`viewer_page`, needs
`login_page`).
Files carry `default-src 'none'; script-src 'self'; connect-src 'self';
style-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`,
`nosniff` and `no-referrer`; nothing is inline. The viewer learns a custom
`status_path` from a meta tag the sink fills in, renders entries as text, and
sends a signed-out visitor to `login?next=view%23signed-in` (the fragment marks
a fresh cookie login; without cookies it shows its own sign-in form); the login
page follows `next` only within its origin. The `proxy_tls` capability
satisfies the pipeline's "auth needs TLS" check.
**Client library.** `internal/sink/http/web/scram.js`, embedded with `go:embed`,
is one dependency-free ES module, so a site with its own CSP can copy it into
its bundle. It exports `login(base, username, password, {onProgress, session})`,
`logout(base, {token})`, `stream(url, {token, signal, onEvent})` (an event
stream parsed as the HTML standard does), `cookiesUsable()` and
`loginUnavailable()`. BLAKE2b and Argon2id are plain JS (no WebAssembly, which
would need `'wasm-unsafe-eval'`), yielding to the page every 50 ms; SHA-256 and
HMAC come from WebCrypto, so it needs a secure context. It checks the challenge
as the Go client does (nonce, salt, cost limits and the dialer's Argon2 floor),
refuses redirects, and reports success only after verifying the server's
signature. A 64 MiB login takes about 2 s on a desktop. The wire protocol above
is the contract: a site may implement its own client.
**CLI.** `lw auth token --unbound` logs in through the proxy, still pinning
its certificate across the two requests; only then may `--url` carry the mount
path.
## Verification
Go tests, one per rule: binding (match, mismatch, one-sided, length) in `auth`.
In `authz`: the validation table (each row valid but for its rule, the error
naming it) and credentials file validation; the TCP exchange over `net.Pipe`
(admission with the first entry in the same read, wrong password
indistinguishable from an unknown user, another certificate refused and counted,
hello and policy disagreeing either way, a dialer cut short by its context, an
abandoned exchange freeing its handshake and limiter slots, a refusal after the
exchange sent as a reason rather than a final, certificate-to-user binding for
logins and tokens); only failed attempts throttled (the limiter's bounds are
`netacl`'s, under the per-client limits); pre-auth
input capped at 4 KiB; the decoy salt stable across restarts; a dialer refusing
a challenge below its Argon2 floor; the HTTP exchange on a TLS test server
(token accepted, garbage and foreign tokens `401`, forged final yields no token,
a different certificate after the challenge refused before the proof is sent,
renewal ahead of expiry); `Authorize` failing closed under SCRAM; password
trimming; a one-user listener admitting its password, refusing another and an
unknown user alike; a descriptor's password surviving a reload, closed and
refused to a second user; a descriptor lw opened itself never read; refused
hellos freeing their slot; unbound logins pinned too. In `tlsx`, terminals,
pipes and descriptors never warned as world-readable. Proxy
mode: trusted peers, https and the client hop; throttling on the forwarded
client and IPv6 /64; an unbound browser login with the cookie's attributes, the
cookie opening endpoints (beside Basic auth) until logout revokes it, cookie
sessions refused elsewhere; only unbound logins behind a proxy; `/auth` taking
only JSON. In the `http` sink, every SSE line break framed as data.
In the plugins: `/auth` outside the `http` sink's gate, the pages served with
their types and CSP, direct peers refused, pages only in proxy mode, and the
`http_chain` sink logging in again after a `401` and delivering once. In the
CLI: private files with a matching verifier, no password replaced without a
source, mode, decoy key and symlink kept across rewrites, an empty file filled,
last-user refusal. Under node, `scram.js` against the RFC 9106 vector, Go's
`argon2.IDKey`, and `auth`'s known-answer proof; the event-stream line breaks
and fields, the bearer, refusals and abort; a token-mode login and logout.
`test/scram-chain-test.sh --auto` (ports 15821-15825): a relay with SCRAM on all
four listener types plus one `client_auth` + SCRAM port; authorized edges over
both chain transports; node labels forced to usernames; wrong password, unknown
user and no-auth edges deliver nothing; a `tcp` sink viewer through `lw auth
stream` held past the exchange deadline; a raw TLS client without a hello gets
nothing; the `http` sink through `lw auth token` and curl; missing and bad
tokens get `401`; `remove-user` + `SIGHUP` revokes token and login; throttling
last. `test/scram-proxy-test.sh --auto` (ports 15831-15832): a Go reverse proxy
ending TLS in front of a plaintext `http` sink mounted at `/logs/`; headless
Chromium sent to the login page, refused a wrong password, streaming events
after login, with the cookie scoped to `/logs` and hidden from scripts, cleared
by sign-out, and no CSP violations; a cookie the browser drops leading to the
viewer's own form, not a loop, its token kept out of storage and the URL; a
profile blocking every cookie signing in on the viewer, its proofs asking for
no cookie, streaming with a bearer, sign-out revoking the token, and a reload's
`401` asking again; `token --unbound` and curl through the proxy; direct peers
and plaintext-forwarded requests `403`. `test/preset-test.sh --auto` (ports
15897-15898): an aggregator and an edge given one password on `/dev/fd/3`
deliver, another password is refused, both survive `SIGHUP` and deliver again,
and the password is in no file, argv or environment; under `script(1)` a
listener's generated password shows on its terminal and not in its log, and
an edge and `lw auth token` log in with it. The existing scripts keep passing.
## Not Implemented
None. The PROXY protocol behind TLS passthrough, deferred here, shipped with
the ACL work: [Security](security.md#proxy-protocol).
## mTLS Hardening
Shipped with this plan, independent of SCRAM:
- The `http_chain` sink no longer follows redirects. Go's client resends the
body on a 307/308, including from `https` to plaintext `http`.
- Its response drain is bounded (64 KiB).
- The `tcp` sink tracks connections from accept, so `Stop` and reloads no longer
wait out the 10 s handshake timeout of a silent peer.
- Unknown keys in any plugin's config, at any depth, fail construction: a typo
such as `[...tls] enabeld = true` used to switch protection off silently. The
configuration file itself is checked the same way, so a misspelled table path
above a plugin's config fails too.
- Startup and every reload warn about certificates that expired, are not yet
valid, or expire within 30 days; about `insecure_skip_verify`; about
`allow_patterns` that are not anchored at both ends of every alternative; and,
once per path, about key, credentials and password files every local user can
read.
- An `http` sink with an auth policy no longer sends
`Access-Control-Allow-Origin: *`.
doc/security.md
+72 -58
View File
@@ -33,10 +33,8 @@ configure it, and — equally important — what it does not yet do.
- Startup warnings for expiring certificates and risky settings: see
[Startup Warnings](#startup-warnings).
- Unknown configuration keys rejected: a typo in `tls`, `auth`, `acl` or a
table path fails startup.
- Configuration input hardened (lixenwraith/config v0.2.2 and
lixenwraith/toml v0.1.3, reviewed with adversarial tests and fuzzing like
`auth`):
table path is refused when the configuration loads.
- Configuration input hardened (lixenwraith/config, lixenwraith/toml):
- a document creates at most 65536 tables, so a small file cannot exhaust
memory, and unescaped control characters are refused in strings and
comments;
@@ -58,6 +56,10 @@ configure it, and — equally important — what it does not yet do.
- Per-client stream caps: under HTTP/2 `max_connections_per_client` bounds a
client's connections, not the streams each carries; the request rate only
slows how fast it opens them.
- A per-identity listing in the `http` status endpoint: it shows the policy
and `auth_rejected` only.
- More than one identity mode per listener: `auth.identity` takes one of
`cn`, `san_dns`, `san_uri`, `san_email`.
Two credentials are supported. Certificates (`mtls`) are the one the transport
already carries: the `tls` block establishes that a peer chains to your CA, and
@@ -96,7 +98,9 @@ Options by role, each with its default:
- `cert_file`: local certificate; a listener needs it, `self_signed` or the
issuer files, a dialer may present it as its client identity.
- `key_file`: private key for `cert_file`; set the two together.
- `min_version` (`"1.3"`): `"1.2"` or `"1.3"`.
- `min_version` (`"1.3"`): `"1.2"` or `"1.3"`. There is no `max_version`
and no `cipher_suites`: Go fixes the 1.3 suites, and the 1.2 ones are
its defaults.
- Listeners
- `self_signed` (`false`): make a self-signed certificate at startup.
- `issuer_cert_file`, `issuer_key_file`: make one at startup, signed by this
@@ -118,18 +122,16 @@ Options by role, each with its default:
Listeners are the `tcp` and `http` sinks and the `tcp_chain` and `http_chain`
sources; dialers are the `tcp_chain` and `http_chain` sinks.
> `min_version` takes `"1.2"` or `"1.3"`. The older `"TLS1.2"` spelling from
> pre-restructure releases is rejected. There is no `max_version` and no
> `cipher_suites` option; TLS 1.3 suites are not configurable in Go, and the
> 1.2 defaults are the standard library's.
### Validation
Misconfiguration fails at plugin construction, before the pipeline starts:
Misconfiguration is refused when the configuration loads
([Validation](configuration.md#validation)), unknown keys always and the other
rules only when `enabled`; certificate and key files fail when the plugin is
built:
- a listener with `enabled = true` and none of `cert_file`/`key_file`,
`self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one; an
issuer that is no CA (`lw tls ca` makes one)
`self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one
- `hosts` on a listener without `self_signed` or the issuer files
- `pin_sha256` with `ca_file` or `insecure_skip_verify`, or not of the
`sha256//BASE64` form
- a key of the other role: `self_signed`, `issuer_cert_file`,
@@ -137,12 +139,13 @@ Misconfiguration fails at plugin construction, before the pipeline starts:
`ca_file`, `server_name`, `insecure_skip_verify` or `pin_sha256` on a
listener
- `client_auth = true` with no `client_ca_file`
- a dialer with only one of `cert_file` / `key_file`
- a certificate or key that will not load, or a CA file containing no
certificates
- only one of `cert_file` / `key_file`
- a `min_version` that is neither `"1.2"` nor `"1.3"`
- a key the plugin does not know, at any depth: `unknown key "tls.enabeld"`. A
misspelled option must not silently leave a protection switched off
- when the plugin is built: a certificate or key that will not load, a CA
file containing no certificates, or an issuer that is no CA (`lw tls ca`
makes one)
## The Auth Block
@@ -286,7 +289,7 @@ you listed.
```toml
[pipelines.plugin_sinks.config.auth]
type = "mtls"
allow = ["relay.internal"]
allow = ["relay.example.org"]
```
The check runs as part of the handshake, so a server the policy rejects never
@@ -296,7 +299,8 @@ was shown instead; see [Channel binding](#channel-binding).
### Validation
Misconfiguration fails at plugin construction, before the pipeline starts:
As for `tls`, the rules are checked when the configuration loads, and the
files (`credentials_file`, `password_file`) when the plugin is built:
- `type` `mtls` or `scram` without `tls.enabled` (proxy mode aside), or on a
dialer with `tls.insecure_skip_verify`: an identity read from an unverified
@@ -320,14 +324,16 @@ Misconfiguration fails at plugin construction, before the pipeline starts:
- `trusted_proxies` on anything but the `http` sink, with `identity`, or with
an entry that is neither an address nor a CIDR; `login_page` or
`viewer_page` without `trusted_proxies`, `viewer_page` without `login_page`
- an `identity` that is not one of the four modes
- an `identity` that is not one of the four modes, or a `node_binding` that
is not one of the three values, whatever `type` is, `none` included
- under `mtls` or `scram`, a `node_binding` other than `none` on a plugin
other than a chain source
- an `allow_patterns` entry that does not compile
- a `node_binding` that is not one of the three values, or one set on a plugin
that has no node concept
- an HTTP path (`stream_path`, `status_path`, `ingest_path`) equal to `/auth`,
which is reserved for logins
- `/auth`, which is reserved for logins: `ingest_path` may not equal it;
`stream_path` and `status_path` may not be it or lie under it
Errors read like `auth: type "mtls" requires tls.client_auth`.
Errors name their path:
`pipelines[0].plugin_sinks[tcp].config.auth: type "mtls" requires tls.client_auth`.
## The ACL Block
@@ -352,9 +358,9 @@ requests_per_second_per_client = 10 # HTTP listeners; 0 (default): none
on an IPv4 listener, IPv6 on an IPv6 one, either behind a hostname (only the
family it binds ever matches). Behind `proxy_from` or the `http` sink's
`trusted_proxies` they take either, as the proxy may name a client of
either. An entry of the other family, an
IPv4-mapped address, one with a zone, or anything but an address or CIDR
fails construction. A link-local peer matches without its zone.
either. An entry of the other family, an IPv4-mapped address, one with a
zone, or anything but an address or CIDR fails at load. A link-local peer
matches without its zone.
- A refused connection is closed at once and counted in `acl_denied`, or
`acl_limited` past a [limit](#per-client-limits) (plugin stats, the `http`
sink's status). A WARN names the client, the reason and the counts, at most
@@ -374,7 +380,7 @@ requests_per_second_per_client = 10 # HTTP listeners; 0 (default): none
- HTTP keep-alive connections count, idle ones too, and a browser opens up
to six; under HTTP/2 one connection carries a client's many requests.
- Behind `trusted_proxies` every connection is the proxy's, so the key
fails construction: cap clients at the proxy.
fails at load: cap clients at the proxy.
- `requests_per_second_per_client`, on the `http` sink and the `http_chain`
source, admits a second's worth of requests at once (at least one),
refilled at that rate; past it a request gets `429`, with `Retry-After` the
@@ -437,6 +443,11 @@ when it is constructed, so startup and every reload repeat them:
- `insecure_skip_verify` on a dialer
- an `allow_patterns` entry not anchored at both ends of every alternative
(`^a|b$` admits `a…` and `…b`)
- `type = "mtls"` on a listener with neither `allow` nor `allow_patterns`: it
admits any identity the CA vouches for ([The allow list](#the-allow-list))
- `auth.trusted_proxies` with an entry beyond loopback and `tls` off: the hop
from the proxy is plaintext
([Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy))
- a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every
local user can read (once per path per process); terminals, pipes and
descriptors hold nothing at rest and are not reported
@@ -506,10 +517,10 @@ openssl req -x509 -newkey rsa:4096 -nodes -days 3650 \
# Relay (server) certificate — SAN must match how clients address it
openssl req -newkey rsa:2048 -nodes -keyout relay.key -out relay.csr \
-subj "/CN=relay.internal"
-subj "/CN=relay.example.org"
openssl x509 -req -in relay.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
-out relay.crt -days 825 \
-extfile <(printf "subjectAltName=DNS:relay.internal\nextendedKeyUsage=serverAuth")
-extfile <(printf "subjectAltName=DNS:relay.example.org\nextendedKeyUsage=serverAuth")
# Edge (client) certificate — CN identifies the node
openssl req -newkey rsa:2048 -nodes -keyout edge-01.key -out edge-01.csr \
@@ -557,7 +568,7 @@ min_version = "1.3"
[pipelines.plugin_sinks.config.auth]
type = "mtls"
allow = ["relay.internal"]
allow = ["relay.example.org"]
```
### 4. Verify
@@ -685,9 +696,10 @@ TCP connection that ends mid-exchange frees it at once. A refused start answers
`too many attempts` (HTTP `429`) and counts `auth_throttled`; the table holds
65,536 addresses and refuses new ones when full. Separately, at most 4,096
exchanges may be in flight per listener; beyond that, or while the plugin
stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`. Peers behind
one NAT, or one passthrough proxy that sends no PROXY header, share a bucket:
there, one client can exhaust every other client's logins.
stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`; neither
counts in `auth_rejected`. Peers behind one NAT, or one passthrough proxy that
sends no PROXY header, share a bucket: there, one client can exhaust every
other client's logins.
### Credentials file
@@ -857,9 +869,11 @@ location /logs/ {
A page not served over HTTPS keeps no `Secure` cookie either: both pages
name HTTPS instead and offer no sign-in.
- **Integrating `scram.js`.** A site with its own CSP can copy `scram.js` from
`internal/sink/http/web/` into its bundle: one dependency-free ES module,
where `base` is the mount URL ending in `/`. It needs a secure context for
WebCrypto and takes about 2 s of Argon2 per login on a desktop.
`internal/sink/http/web/` into its bundle: one dependency-free ES module in
plain JS (no WebAssembly, so no `'wasm-unsafe-eval'`), where `base` is the
mount URL ending in `/`. It needs a secure context for WebCrypto and takes
about 2 s of Argon2 per login on a desktop. `argon2id` and `_proofForTest`
are exported for `scram.test.mjs` only; do not rely on them.
- Cookie mode, the default: `login(base, username, password, {onProgress})`,
then `EventSource` and `fetch` as for any same-origin resource;
`logout(base)`.
@@ -872,12 +886,12 @@ location /logs/ {
- `cookiesUsable()` tells which applies: false when the session cookie would
not stick. `loginUnavailable()`, asked first, names why neither can run
(no secure context), or is empty.
- **Pages.** Under `/auth/` the sink serves `scram.js` and `style.css` always,
and with `login_page` / `viewer_page` the login page and a minimal live
viewer with their scripts; `GET /` leads to the viewer, else the login page.
All are served under `default-src 'none'; script-src 'self';
connect-src 'self'; style-src 'self'; form-action 'self'; frame-ancestors
'none'; base-uri 'none'`.
- **Pages.** Under `/auth/` the sink serves `scram.js`, `style.css` and
`favicon.svg` always, and with `login_page` / `viewer_page` the login page
and a minimal live viewer with their scripts; `GET /` leads to the viewer,
else the login page. All are served with `nosniff`, `no-referrer` and the CSP
`default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self';
img-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`.
`test/scram-proxy-test.sh --auto` logs in from headless Chromium through such a
proxy, with cookies and with every cookie blocked.
@@ -1011,9 +1025,7 @@ peer is.
identity or user and send `SIGHUP`: the reload rebuilds every pipeline, so the
change takes effect on the next connection and existing ones are dropped by the
rebuild. No network call on the handshake path, and no window between revocation
and the next CRL publication. See
[mtls-auth-plan.md](mtls-auth-plan.md#not-implemented) for what CRL support
would add.
and the next CRL publication.
## Surfaces Without Access Control
@@ -1026,20 +1038,21 @@ What each exposes when `auth.type = "none"`:
- `http` sink `stream_path`: the full log stream, with
`Access-Control-Allow-Origin: *`, so any browser origin can read it (the
header is omitted once an auth policy is set).
- `http` sink `status_path`: host, port, TLS flag, uptime, client counts,
throughput counters.
- `http` sink `GET /` (`303` to `/auth/view`) and the viewer's files: static
pages that read the two endpoints above under the CSP and expose nothing
beyond them. Under `mtls` they are served before the allow list, while the
stream and status stay gated.
- `http` sink `status_path`: version and instance id, host, port, TLS flag,
the auth and acl policy summary, buffer sizes and limits, endpoints,
uptime, client counts, throughput and rejection counters.
- `http` sink `GET /` (`303` to `auth/view`, relative to the mount) and the
viewer's files: static pages that read the two endpoints above under the
CSP and expose nothing beyond them. Under `mtls` they are served before the
allow list, while the stream and status stay gated.
- `tcp` sink: the full log stream to any client that connects.
- `tcp_chain` and `http_chain` sources: ingest from any peer that can connect
(with `client_auth`, any the CA vouches for), under any node label it
claims.
`max_connections` bounds concurrency on all but the `http_chain` source; the
`acl` block's [per-client limits](#per-client-limits) bound each caller's
share.
`max_connections` bounds connections on the `tcp` sink and `tcp_chain` source
and streams on the `http` sink; the `acl` block's
[per-client limits](#per-client-limits) bound each caller's share.
Both methods require TLS on the listener, except an `http` sink in proxy mode,
where TLS ends at the trusted proxy; `mtls` also requires `client_auth`.
@@ -1050,9 +1063,10 @@ where TLS ends at the trusted proxy; `mtls` also requires `client_auth`.
- Use a dedicated CA for LogWisp so its trust decisions stay independent.
- Keep leaf lifetimes short (90–825 days) and automate renewal.
- Key, credentials and password files should be `0600` and owned by the service
account; a world-readable one is reported at startup. `lw auth` creates
them `0600` and keeps the mode of an existing file.
- Keep key, credentials and password files unreadable by others: `0600`
owned by the account lw runs as, or `0640` `root:GROUP` for a group it is
in; startup warns about world-readable ones. `lw auth` and `lw tls` create
them `0600`, and `lw auth` keeps the mode of an existing file.
- Rotation requires a reload (`SIGHUP`), because certificates and credentials
are loaded once at plugin construction; there is no on-disk watch for them.
- Startup and every reload warn 30 days before a certificate lapses. Still
doc/sinks.md
+72 -85
View File
@@ -8,9 +8,10 @@ type-specific `config` table.
Registered types: `console`, `file`, `http`, `tcp`, `null`, `tcp_chain`,
`http_chain`.
Dispatch into a sink is non-blocking. A sink whose input queue is full drops the
event *for itself only* and the pipeline counts it in `total_dropped_by_sink`;
sibling sinks are unaffected.
Dispatch waits for a `console` sink, so a slow reader paces the whole pipeline,
sibling sinks included (`null` waits too but discards at once). Any other sink
whose input queue is full drops the event for itself only
(`total_dropped_by_sink`).
---
@@ -32,7 +33,8 @@ color = "auto"
Options, each as type and default:
- `target` (string, `stdout`): `stdout` or `stderr`.
- `target` (string, `stdout`): `stdout` or `stderr`; `split` exists only for
lw's own log (`logging.output`).
- `buffer_size` (int, `1000`): sink input queue depth.
- `escape` (string, `auto`): write control characters as `<hex>` of their
bytes, `ESC` as `<1b>`: `auto` when the output is a terminal, `always`, or
@@ -41,16 +43,12 @@ Options, each as type and default:
`auto` on a terminal with `NO_COLOR` unset and `TERM` not `dumb`, `always`,
or `never`.
> `split` is **not** a valid target for this sink and is rejected at startup.
> Level-based splitting exists only for LogWisp's own application log
> (`logging.output = "split"`).
Payloads are written as the formatter made them, one record per line.
- Escaping: a log line must not drive the terminal that shows it (cursor and
screen control, title changes, OSC 52 clipboard writes, bidi spoofing). C0
and C1 controls other than tab, DEL, the bidi controls, the line and
paragraph separators and invalid UTF-8 are escaped; other invisible
screen control, title changes, OSC 52 clipboard writes, bidi spoofing).
Escaped: every C0 control but tab; DEL and the C1 controls; the bidi
controls; the line and paragraph separators; invalid UTF-8. Other invisible
characters, such as the joiners emoji and Indic scripts need, are kept.
`never` keeps an application's own ANSI colours; under `auto`, pipes and
files get the bytes unchanged.
@@ -91,20 +89,23 @@ flush_interval_ms = 100
Options, each as type and default:
- `directory` (string, **required**): output directory.
- `name` (string, **required**): base filename.
- `directory` (string, **required**): output directory. The sink writes
`NAME.log` and rotates it to `NAME_YYMMDD_HHMMSS.log`. The limits below
delete *any* `.log` file in the directory, so give each file sink a
directory of its own.
- `name` (string, **required**): file name base.
- `max_size_mb` (int, `100`): rotate when the active file reaches this size.
- `max_total_size_mb` (int, `1000`): cap across all rotated files.
- `min_disk_free_mb` (int, `0`): free-space floor before writing; `0` = none.
- `retention_hours` (float, `168.0`): delete rotated files older than this.
- `max_total_size_mb` (int, `1000`): above this total, the oldest `.log` files
there are deleted.
- `min_disk_free_mb` (int, `0`): the oldest `.log` files there are deleted to
keep this much space free, and nothing is written while that fails. `0`
means none and a negative value means `100`.
- `retention_hours` (float, `168.0`): `.log` files there older than this are
deleted; `0` keeps them.
- `buffer_size` (int, `1000`): sink input queue depth, and the depth of the
internal writer's queue, which drops and counts when full.
- `flush_interval_ms` (int, `100`): forced flush interval.
> `min_disk_free_mb` has an unusual default. The constructor replaces only
> *negative* values with `100`; leaving the key unset yields `0`, which means no
> free-space floor. Set it explicitly if you want one.
The sink drives an internal writer configured for raw output with timestamps and
levels disabled, so what lands on disk is exactly the formatted payload. On stop
it hands the writer what is queued before closing the file.
@@ -113,8 +114,9 @@ it hands the writer what is queued before closing the file.
## null
Discards everything, counting entries and bytes. Useful for benchmarking a
source or flow in isolation.
Discards; holds a sink's place until one is chosen, so a pipeline can be set
up before its sink is. It counts entries and bytes, so it also isolates a
source or flow when measuring it.
```toml
[[pipelines.plugin_sinks]]
@@ -190,9 +192,10 @@ Options, each as type and default:
**Behaviour**
- Only `GET` is routed to either path; anything else gets `405`, `HEAD` on
`stream_path` included — a stream is a body, and a client registered to have
its body discarded never reads and never leaves.
- Only `GET` (and `HEAD` on `status_path`) is routed to the two paths;
anything else gets `405`, including `HEAD` on `stream_path`. A stream is a
body, and a client registered to have its body discarded never reads and
never leaves.
- With an `auth` block, one middleware gates **both** endpoints, with no body
detail in a refusal. Under `mtls` a refused certificate gets `403`. Under
`scram` a client logs in at `POST /auth`, which sits outside the gate (see
@@ -225,15 +228,17 @@ Options, each as type and default:
- Refusals are logged at WARN and counted in `auth_rejected`. The authorized
identity is recorded in the client's session as `auth_method` /
`auth_identity`.
- On connect the client receives an `event: connected` frame carrying its
client id, session id, sink instance id, endpoint paths, and buffer size.
- Stream responses carry `Content-Type: text/event-stream`,
`Cache-Control: no-cache` and `X-Accel-Buffering: no`. On connect the client
receives an `event: connected` frame carrying its client id, session id,
sink instance id, endpoint paths, and buffer size.
- Payloads are framed per the SSE spec, one `data:` line per line break in the
payload (CRLF, LF or a lone CR), so multi-line entries stream correctly and
no entry can inject an `event:`, `id:` or `retry:` field.
- The server sets no `WriteTimeout` (that would kill long-lived streams);
per-write deadlines come from `write_timeout_ms` via `http.ResponseController`
and cover the connected frame, every payload, and the idle comment.
- A quiet stream emits an SSE comment every 15 s. It refreshes the client's
- Every stream gets an SSE comment every 15 s. It refreshes the client's
session and is how a peer that stopped reading is noticed.
- A client whose send queue is full has that event dropped
(`dropped_writes`); it is not disconnected. A `dropped_writes` that rises while
@@ -258,11 +263,9 @@ buffer sizes, connection limit, write timeout, uptime, endpoint paths, and the
`total_processed` / `dropped_writes` / `rejected_clients` / `auth_rejected` /
`acl_denied` / `acl_limited` counters.
**Statistics**: `dropped_writes`, `rejected_clients`, `auth`, `auth_allowed`,
`auth_rejected`; under `scram` also `auth_users`, `auth_throttled`,
`auth_busy`, `auth_binding_mismatch` and `auth_token_lifetime_ms`, and in
proxy mode `auth_trusted_proxies`; with a non-empty `acl` block also `acl`,
`acl_denied`, `acl_limited` and `acl_proxy_headers`.
**Statistics**: `host`, `port`, `tls`, `buffer_size`, `client_buffer_size`,
`max_connections`, `write_timeout_ms`, `endpoints`, `dropped_writes`,
`rejected_clients`, and the [auth and acl keys](sources.md#source-statistics).
> Without an `auth` block both endpoints are unauthenticated, and the stream
> response carries `Access-Control-Allow-Origin: *`, so any web origin can read
@@ -303,22 +306,14 @@ allow = ["viewer-01"]
Options, each as type and default:
- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the listener
keeps to its family ([Networking](networking.md#address-family)).
- `port` (int, **required**): listen port.
- `buffer_size` (int, `1000`): sink input queue depth.
- `client_buffer_size` (int, `256`): per-client send queue depth.
- `host`, `port`, `buffer_size`, `client_buffer_size`: as on [http](#http).
- `write_timeout_ms` (int, `5000`): per-write deadline.
- `keep_alive` (bool, `true`): TCP keep-alive on accepted connections.
- `keep_alive` (bool, `true`): TCP keep-alive probes after
`keep_alive_period_ms` idle; `false` sends none.
- `keep_alive_period_ms` (int, `30000`): keep-alive idle period.
- `max_connections` (int, `0`): concurrent connection cap; `0` = unlimited.
- `tls` (table): listener TLS.
- `auth` (table): client authentication (`mtls` or `scram`); see
[Security](security.md#the-auth-block).
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
before TLS, `proxy_protocol` with `proxy_from` for the client a PROXY header
names, and `max_connections_per_client`; see
[Security](security.md#the-acl-block).
- `tls`, `auth`: as on `http`.
- `acl` (table): as on `http`, without `requests_per_second_per_client`.
**Behaviour**
@@ -342,11 +337,9 @@ Options, each as type and default:
receives a broadcast. Its connection is closed, the rejection logged at WARN,
and `rejected_conns` incremented.
**Statistics**: `write_errors`, `dropped_writes`, `rejected_conns`,
`tls_handshake_errors`, `auth`, `auth_allowed`, `auth_rejected`; under `scram`
also `auth_users`, `auth_throttled`, `auth_busy` and `auth_binding_mismatch`;
with a non-empty `acl` block also `acl`, `acl_denied`, `acl_limited` and
`acl_proxy_headers`.
**Statistics**: `host`, `port`, `tls`, `buffer_size`, `write_errors`,
`dropped_writes`, `rejected_conns`, `tls_handshake_errors`, and the
[auth and acl keys](sources.md#source-statistics).
---
@@ -360,7 +353,7 @@ persistent connection. See [Chaining](chaining.md).
id = "to_relay"
type = "tcp_chain"
[pipelines.plugin_sinks.config]
host = "relay.internal"
host = "relay.example.org"
port = 15801
node = "edge-01"
buffer_size = 1000
@@ -388,8 +381,10 @@ Options, each as type and default:
- `dial_timeout_ms` (int, `5000`): TCP connect timeout.
- `write_timeout_ms` (int, `5000`): per-write deadline.
- `backoff_min_ms` (int, `500`): reconnect backoff floor.
- `backoff_max_ms` (int, `30000`): reconnect backoff ceiling.
- `keep_alive` (bool, `true`): TCP keep-alive.
- `backoff_max_ms` (int, `30000`): reconnect backoff ceiling; below
`backoff_min_ms` it is `30000`.
- `keep_alive` (bool, `true`): TCP keep-alive probes after
`keep_alive_period_ms` idle; `false` sends none.
- `keep_alive_period_ms` (int, `30000`): keep-alive idle period.
- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity.
- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see
@@ -401,12 +396,13 @@ Options, each as type and default:
downstream being up.
- Under `scram` the hello carries the login, and the link counts as established
only once the server's final message proves it holds the user's verifier. The
exchange is bounded at 10 s and cut short by shutdown. A refusal —
`authentication failed`, `authentication not enabled`, `no challenge within
10s` — is logged at WARN as `Chain connect refused` on every attempt; other
connect failures log at DEBUG. Both retry under the normal backoff.
exchange is bounded at 10 s and cut short by shutdown.
- A refusal (`authentication failed`, `authentication not enabled`, `no
challenge within 10s`) or a server failing its CA or pin is logged at WARN
as `Chain connect refused` on every attempt; other connect failures log at
DEBUG as `Chain connect failed`. Both retry under the normal backoff.
- Each entry is serialized as one canonical JSON line. Delivery holds the line
across reconnects until it is written or the process shuts down, with
across reconnects until the socket accepts it or the process shuts down, with
exponential backoff plus ±20 % jitter between attempts. The delay keeps
growing until a link outlives `backoff_min_ms`, so a source that drops every
link right after accepting it is not retried in a tight loop.
@@ -425,12 +421,9 @@ Options, each as type and default:
- Events arriving without a structured entry are wrapped from the formatted
payload and counted in `synthesized`.
Under `mtls` the policy pins the server's identity as part of the handshake, so
a server it rejects is refused like a failed login above.
**Statistics**: `target`, `node`, `tls`, `auth`, `connected`, `reconnects`,
`write_errors`, `synthesized`; under `scram` also `auth_username`,
`auth_failures` and `last_auth_error` (empty after a successful login).
**Statistics**: `target`, `node`, `tls`, `connected`, `reconnects`,
`write_errors`, `synthesized`, and the
[auth keys](sources.md#source-statistics).
---
@@ -444,7 +437,7 @@ Batches structured entries as NDJSON and POSTs them to a downstream LogWisp
id = "to_collector"
type = "http_chain"
[pipelines.plugin_sinks.config]
host = "collector.internal"
host = "collector.example.org"
port = 15802
ingest_path = "/ingest"
node = "edge-01"
@@ -465,29 +458,22 @@ key_file = "/etc/logwisp/tls/client.key"
Options, each as type and default:
- `host` (string, **required**): downstream host or address; an IPv6 one goes
bare (`::1`).
- `port` (int, **required**): downstream port.
- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`.
- `node` (string, `os.Hostname()`): origin label stamped on first-hop entries.
- `buffer_size` (int, `1000`): sink input queue depth.
- `host`, `port`, `node`, `buffer_size`, `tls`, `auth`: as on
[tcp_chain](#tcp_chain).
- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`, and
`/auth` is reserved for the login.
- `max_batch_count` (int, `100`): flush after this many entries.
- `max_batch_bytes` (int, `1048576`): flush after this many bytes (1 MiB).
- `flush_interval_ms` (int, `1000`): flush after this long.
- `request_timeout_ms` (int, `10000`): covers dial, write, and response.
- `backoff_min_ms` (int, `500`): retry backoff floor.
- `backoff_max_ms` (int, `30000`): retry backoff ceiling.
- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity.
- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see
[Security](security.md#the-auth-block).
- `backoff_max_ms` (int, `30000`): retry backoff ceiling; below
`backoff_min_ms` it is `30000`.
**Behaviour**
- Delivery is at-least-once per batch: a retried batch can be delivered twice if
the first attempt succeeded but the response was lost.
- Retries apply to transport errors, `408`, `429`, and `5xx`. Any other
non-2xx response is treated as permanent, and the batch is dropped and counted
in `dropped_batches`. A retry is logged at WARN, a dropped batch at ERROR.
- Delivery: [Chaining](chaining.md#delivery-semantics); a retry logs WARN, a
dropped batch ERROR.
- Under `scram` the sink logs in at `POST /auth` whenever it holds no token and
sends `Authorization: Bearer <token>`. A failed login is retried like a
transport error, holding the batch. The token is renewed ahead of its expiry;
@@ -503,9 +489,9 @@ Options, each as type and default:
within `request_timeout_ms`, at most 2 s; each batch still undelivered then
is dropped and logged at WARN.
**Statistics**: `target`, `node`, `tls`, `auth`, `batches_sent`,
`request_errors`, `dropped_batches`, `synthesized`; under `scram` also
`auth_username`, `auth_failures` and `last_auth_error`.
**Statistics**: `target`, `node`, `tls`, `batches_sent`, `request_errors`,
`dropped_batches`, `synthesized`, and the
[auth keys](sources.md#source-statistics).
---
@@ -513,4 +499,5 @@ Options, each as type and default:
Every sink reports: `id`, `type`, `total_processed`, `active_connections`,
`start_time`, `last_processed`, and a type-specific `details` map. The DEBUG
status report spreads `details` into each plugin's line.
status report spreads `details` into each plugin's line, network sinks' with
the [auth and acl keys](sources.md#source-statistics).
doc/sources.md
+83 -84
View File
@@ -2,21 +2,12 @@
Sources produce `core.LogEntry` values for a pipeline. Every source is declared
as a `[[pipelines.plugin_sources]]` entry with an `id`, a `type`, and a
type-specific `config` table.
type-specific `config` table. Registered types: `file`, `console`, `random`,
`null`, `tcp_chain`, `http_chain`.
```toml
[[pipelines.plugin_sources]]
id = "app_logs"
type = "file"
[pipelines.plugin_sources.config]
directory = "/var/log/myapp"
```
Registered types: `file`, `console`, `random`, `null`, `tcp_chain`,
`http_chain`.
Publication from any source is non-blocking. When a subscriber channel is full
the entry is dropped and counted in `dropped_entries`.
Publication is non-blocking for every source but `console`: a full subscriber
channel drops the entry and counts it in `dropped_entries`. The `console`
source waits instead (see [console](#console)).
---
@@ -41,30 +32,32 @@ Options, each as type and default:
- `directory` (string, required): directory to scan; not recursive.
- `pattern` (string, `*`): glob over filenames; `*` and `?` only.
- `check_interval_ms` (int, `100`): directory rescan interval; minimum `10`.
An open file is polled every 100 ms whatever this is.
- `raw` (bool, `false`): never parse a line: the whole line is the message.
- `from` (string, `end`): where a new watcher starts, `end` or `start` of the
file.
**Behaviour**
- `check_interval_ms` governs how often the directory is rescanned for new or
removed files. Tailing an already-open file polls on a **fixed 100 ms**
interval that this option does not change.
- Each matched file gets its own watcher. Watchers for files that disappear are
stopped and removed on the next scan.
- Each matched file gets its own watcher, and its entries the file's base
name as `Source`. Watchers for files that disappear are stopped and removed
on the next scan.
- The file lw's stdout or stderr is redirected to is never followed, with a
warning: `lw > out.log` in the directory would read its own output back.
- A line longer than 1 MiB continues in the next entry, as on the console
source.
- A new watcher seeks to end-of-file. Positions live in memory only, so a
restart resumes from the current end of each file and content written while
LogWisp was down is not read. `from = "start"` reads each file whole when its
watcher is created instead — what a process writing beside LogWisp needs, at
the cost of replaying a file already on disk at every restart.
restart or a reload resumes from the current end of each file, and content
written meanwhile is not read. `from = "start"` instead reads each file whole
when its watcher is created, which a process writing beside LogWisp needs, at
the cost of replaying files already on disk at every restart and reload.
- Rotation is detected from size decrease, modification-time reset, a position
beyond end-of-file, or an inode change. An inode change where the new file is
already larger than the recorded position is treated as an atomic save, not a
rotation, and the position is preserved.
at least as large as the recorded position is treated as an atomic save, not
a rotation, and the position is preserved. Each rotation also publishes an
INFO entry `Log rotation detected (#N): REASON` with the file's name as
`Source`; it reaches every sink, so exclude it with a filter
(`Log rotation detected`) where unwanted.
- A rotation that renames in place — what a size-capped writer does — puts the
same inode back under a name `pattern` also matches. Its watcher resumes at
the position the original reached, so `from = "start"` reads the tail an
@@ -72,17 +65,19 @@ Options, each as type and default:
- A line is parsed as JSON only when it is an object whose top-level keys are
all drawn from `time`, `level`, `msg` and `fields` — the four an entry can
carry. `time` is read as RFC3339Nano. Any other key, and any non-object line,
is kept whole as text, because parsing it would drop the rest.
is kept whole as text, because parsing it would drop the rest. `level` is
kept as written (a filter sees `Error` or `fatal` as is), and a line without
`level` has none; only text lines get the inference below.
- A text line's level is its first whole word, in any case, naming one:
`TRACE`; `DEBUG`, `DBG`; `INFO`, `INF`; `WARN`, `WARNING`; `ERROR`, `ERR`,
`FATAL`. `warning: disk full` is WARN, `INFO retry after error` INFO. The
console sink paints that word and the http sink's viewer filters on it.
- `raw = true` skips the JSON branch entirely. The line, plus its newline,
becomes the message; `fields` stays empty, the time is the read time, and the
level is inferred from the text as for any unparsed line. Paired with
`format.type = "raw"` this is byte-exact transport for records LogWisp's
envelope cannot hold — see [Formatters](formatters.md#raw).
- `Source` is set to the file's base name.
- `raw = true` skips the JSON branch. The line, with its terminator (`\n` or
`\r\n`) removed, becomes the message. `fields` stays empty, the time is the
read time, and the level is inferred from the text. Blank lines are skipped.
With `format.type = "raw"` each line is written back with one `\n`, which
carries records the envelope cannot hold but is not byte-exact; see
[Formatters](formatters.md#raw).
**Statistics**: per-watcher size, position, entries read, rotation count, and
last read time, plus `active_watchers`.
@@ -128,7 +123,8 @@ Delivery and lifetime:
## random
Synthetic entry generator for development, smoke tests, and sanitizer testing.
Generated entries, to check a deployment before real logs are wired: paths,
permissions, sinks, chains, and sanitizer policies with `special = true`.
```toml
[[pipelines.plugin_sources]]
@@ -145,23 +141,25 @@ special = false
Options, each as type and default:
- `interval_ms` (int, `500`): emission period.
- `jitter_ms` (int, `0`): symmetric jitter; must be non-negative, clamped to
`interval_ms`.
- `jitter_ms` (int, `0`): each interval varies by up to ±`jitter_ms`/2;
non-negative, capped at `interval_ms`.
- `format` (string, `txt`): `raw` (message only), `txt` (bracketed line) or
`json` (JSON object as the message).
- `length` (int, `20`): message length in characters.
- `special` (bool, `false`): inject control and non-ASCII characters.
- `length` (int, `20`): bytes of random text; `txt` and `json` wrap it in a
longer message.
- `special` (bool, `false`): with `length` ≥ 3, mix in up to one control byte
(`\n` and `\r` among them) and a few non-ASCII bytes, often invalid UTF-8.
`raw` and `txt` carry them as they are; `json` escapes them inside its
message.
`special = true` is the intended way to exercise sanitizer policies: it inserts
control bytes and multi-byte Unicode into otherwise ordinary messages. Levels
are chosen at random from DEBUG, INFO, WARN, ERROR.
Levels (`txt` and `json` only) are random among DEBUG, INFO, WARN, ERROR.
---
## null
Produces nothing. Useful as a placeholder so a sink-only pipeline satisfies the
"at least one source" requirement.
Nothing; holds a source's place until one is chosen, so a pipeline can be set
up, sinks first, before its source is.
```toml
[[pipelines.plugin_sources]]
@@ -216,8 +214,8 @@ Options, each as type and default:
- `hello_timeout_ms` (int, `10000`): deadline for the hello preamble.
- `trust_node` (bool, `true`): `false` overrides the sender's node label with
its remote address. `auth.node_binding` `assert` or `force` takes over the
connection label; only `force` also overrides the per-entry labels; `none`
leaves both to `trust_node`.
connection label, and `force` the per-entry labels too
([Chaining](chaining.md#node-identity)).
- `tls` (table): listener TLS; see [Security](security.md).
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
see [Security](security.md#the-auth-block).
@@ -235,33 +233,31 @@ and a `credentials_file`, or one `username` with its `password_file`, and
- TLS handshakes run explicitly with a 10 s bound before the preamble is read,
after the `max_connections` admission check.
- A connection is rejected if the first line is not a valid hello with a
matching protocol version.
- Under `mtls` the certificate is checked before the hello is read, so an
unauthorized peer never gets a preamble parsed on its behalf. Under `scram`
the hello opens the login: challenge, proof and final follow on the same
connection, the whole exchange within `hello_timeout_ms`, and the final is
sent only once node binding agrees. A source without `scram` answers a hello
offering credentials with `authentication not enabled`. A rejection is logged
at WARN and counted in `rejected_conns`.
- The node label is then resolved: under `auth.node_binding` it comes from the
peer's certificate identity or username, otherwise `trust_node` governs.
`force` also overrides the `node` field on every individual entry; `assert`
leaves per-entry labels to `trust_node`, so a relay can forward other nodes'
entries while proving its own identity.
- Each accepted connection gets a session recording the remote address, node
label, — under TLS — `tls` and `tls_peer_cn`, — under auth —
`auth_method` and `auth_identity`, and behind a PROXY header the proxy as
`peer_addr`.
offering credentials with `authentication not enabled`.
- An auth or node-binding refusal is logged at WARN and counted in
`rejected_conns`. A missing, malformed or wrong-version hello is logged at
WARN (`Rejected chain connection`) and not counted in `rejected_conns`;
under `scram` a malformed or wrong-version one counts in `auth_rejected`. A
connection over `max_connections` is closed and counted without a log.
- The node label is then resolved by `trust_node` and `auth.node_binding` (see
[Chaining](chaining.md#node-identity)).
- Each accepted connection gets a session recording its remote address and:
- `node`, the resolved label
- under TLS: `tls`, `tls_peer_cn`
- under auth: `auth_method`, `auth_identity`
- behind a PROXY header: the proxy as `peer_addr`
- A malformed entry line increments `parse_errors` and is skipped; the
connection survives. A line over 1 MiB is a protocol violation and terminates
the connection.
**Statistics**: `active_connections`, `rejected_conns`, `parse_errors`,
`tls_handshake_errors`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`
and `auth_binding_mismatch`; with a non-empty `acl` block also `acl`,
`acl_denied`, `acl_limited` and `acl_proxy_headers`.
**Statistics**: `host`, `port`, `tls`, `active_connections`, `rejected_conns`,
`parse_errors`, `tls_handshake_errors`, `trust_node`, and the
[auth and acl keys](#source-statistics).
---
@@ -297,25 +293,13 @@ node_binding = "force"
Options, each as type and default:
- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the
listener keeps to its family ([Networking](networking.md#address-family)).
- `port` (int, required): listen port, 1–65535.
- `host`, `port`, `trust_node`, `tls`, `auth`: as on [tcp_chain](#tcp_chain).
- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`, and
`/auth` is reserved for the login.
- `buffer_size` (int, `1000`): subscriber channel depth.
- `max_body_bytes` (int, `8388608`): per-request body cap (8 MiB).
- `read_timeout_ms` (int, `30000`): full request read deadline.
- `trust_node` (bool, `true`): `false` overrides the sender's node label with
its remote address. `auth.node_binding` `assert` or `force` takes over the
connection label; only `force` also overrides the per-entry labels; `none`
leaves both to `trust_node`.
- `tls` (table): listener TLS; see [Security](security.md).
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
see [Security](security.md#the-auth-block).
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
before TLS, `proxy_protocol` with `proxy_from` for the client a PROXY header
names, and the per-client `max_connections_per_client` and
`requests_per_second_per_client`; see [Security](security.md#the-acl-block).
- `acl` (table): as on `tcp_chain`, plus `requests_per_second_per_client`.
**Behaviour**
@@ -329,21 +313,23 @@ Options, each as type and default:
node-binding failure answer `403`. Both are distinct from the `400` used for
protocol errors, so a sender can tell "not allowed" from "malformed batch".
- A missing or mismatched `X-Logwisp-Protocol` header is rejected with `400`.
- With `acl.requests_per_second_per_client`, a request over the rate gets `429`
with `Retry-After`; the sink retries it.
- Batch acceptance is atomic: entries are published only after the body reads
cleanly end to end. A transfer error rejects the whole batch (`400`, or `413`
when the body cap is hit) so the sender retries it. A malformed *line* inside
an otherwise clean transfer is skipped and counted in `parse_errors`.
cleanly end to end. A transfer error, a line over 1 MiB included, rejects the
whole batch with `400`, or `413` when the body cap is hit; an `http_chain`
sink drops such a batch (`dropped_batches`) rather than retry it. A malformed
*line* inside an otherwise clean transfer is skipped and counted in
`parse_errors`.
- Success is `204 No Content` with `X-Logwisp-Accepted` set to the number of
entries ingested.
- Sessions are cached per remote host + node + authenticated identity, and
recreated after idle expiry. Including the identity in the key means two peers
sharing a remote address never share a session.
**Statistics**: `total_requests`, `rejected_requests`, `parse_errors`,
`cached_sessions`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`,
`auth_binding_mismatch` and `auth_token_lifetime_ms`; with a non-empty `acl`
block also `acl`, `acl_denied`, `acl_limited` and `acl_proxy_headers`.
**Statistics**: `host`, `port`, `ingest_path`, `tls`, `total_requests`,
`rejected_requests`, `parse_errors`, `cached_sessions`, `trust_node`, and the
[auth and acl keys](#source-statistics).
---
@@ -352,3 +338,16 @@ block also `acl`, `acl_denied`, `acl_limited` and `acl_proxy_headers`.
Every source reports: `id`, `type`, `total_entries`, `dropped_entries`,
`start_time`, `last_entry_time`, and a type-specific `details` map. The DEBUG
status report spreads `details` into each plugin's line.
Network sources and sinks add the auth and acl keys to `details`:
- `auth`: the method, `none` without an `auth` block; with one also
- `auth_allowed`, `auth_rejected`; chain sources `node_binding`
- `mtls`: `auth_identity`, `auth_unrestricted`
- `scram` listeners: `auth_users`, `auth_throttled`, `auth_busy`,
`auth_binding_mismatch`, `auth_identity` when bound; on HTTP
`auth_token_lifetime_ms`; in proxy mode `auth_trusted_proxies`
- `scram` dialers: `auth_username`, `auth_failures`, `last_auth_error`
- with a non-empty `acl` block: `acl`, `acl_denied`, `acl_limited`,
`acl_proxy_headers`
doc/todo.md
+19 -111
View File
@@ -6,23 +6,11 @@ they belong to.
## 1. Packaging: AUR, FreeBSD ports, Debian
The foundation exists:
- The `lw` binary name, free in Arch (official repositories and AUR), Ubuntu
24.04 and 26.04, and the FreeBSD 15.1 ports tree.
- The canonical module path `github.com/lixenwraith/logwisp`, which
`go install`, FreeBSD's `USES=go:modules` and Debian's dh-golang expect.
- `make install` with `DESTDIR`, `PREFIX`, `SYSCONFDIR` and `SERVICE`: the
binary, the `doc/lw.1` manual, bash, zsh and fish completion generated from
`cmd/lw`'s `shorts` and `commands` tables, and the service files in
`deploy/package/` (systemd unit, sysusers, tmpfiles, FreeBSD rc.d script)
with the configuration, kept on reinstall.
- `make deb`, a binary package built with `dpkg-deb`, and `make arch`, the
PKGBUILD built from the working tree.
- Skeletons in `deploy/package/arch/` and `deploy/package/freebsd/`.
- `.github/workflows/package.yml`: `make arch`, the `.deb` and BSD make on
FreeBSD 15.1, each reinstalled over an edited configuration; the systemd
and rc.d services run; a home install with `SERVICE=no` and
`lw config init`; and `make image image-check`.
[Installation](installation.md#packaging-status) lists what is in place. The
`lw` name is free in Arch (official repositories and AUR), Ubuntu 24.04 and
26.04, and the FreeBSD 15.1 ports tree; `go install`, FreeBSD's
`USES=go:modules` and Debian's dh-golang expect the module path
`github.com/lixenwraith/logwisp`.
What remains is the maintainer's: it needs push rights, an identity or a
release tag, and both skeletons download the tagged source.
@@ -62,8 +50,8 @@ release tag, and both skeletons download the tagged source.
- `make deb` serves direct installs. A PPA builds from a signed source
package without network access: a `debian/` directory (changelog,
control, rules) and the Go modules vendored into the source tarball.
- The archive wants every Go dependency packaged (dh-golang); the four
`lixenwraith` libraries are not.
- The archive wants every Go dependency packaged (dh-golang); the
`lixenwraith` libraries in `go.mod` are not.
- `lintian` on the `make deb` package reports `no-changelog` (a source
package's `debian/changelog` provides it), `statically-linked-binary`
(lw is static by design) and `maintainer-script-calls-systemctl` (dh's
@@ -72,95 +60,11 @@ release tag, and both skeletons download the tagged source.
from a ports tree (`make stage check-plist` in a copy of
`deploy/package/freebsd/`), rather than BSD make on the checkout.
## 2. `lw --tui` and the configuration engine it shares with lixen.com
## 2. What `lw --tui` and its engine leave for later
`lw --tui` composes pipelines in a full-screen terminal UI, runs them under
the screen with a live view (`t`), and on exit runs them at once or prints one
form of them: a command line, environment variables, or a configuration file.
Its logic is a pure engine that lixen.com's planned Configure tab (lixencom
`doc/todo.md`) runs as WebAssembly, so no part of LogWisp is rewritten in
JavaScript.
What exists:
- Option structs with toml tags for every plugin and flow stage
(`internal/config/config.go`), presets as rows with typed parameters
(`preset.go`), the command-line and environment grammar (`spec.go`),
`ValidateConfig`, `--check` and `--dump`.
- The option schema, in `internal/config`: `default:`, `help:` and `lw:` tags
on the option structs, one catalogue row per plugin type, `Decode` (every
constructor decodes through it and keeps only I/O), `Coerce` for typed
command-line values, and a `ValidateConfig` that decodes every plugin
offline and names the first problem of each part by its key's path.
- `internal/config` already compiles for `GOOS=js`; it links no net/http,
crypto/tls or terminal code.
- The engine, `internal/compose`: starts empty, from a preset (whose path
`config.IsDir` resolves, from answers off the host), a loaded
configuration or pasted text of any form; adds, removes and moves sources,
sinks and filters, sets and unsets keys, validates; writes a command line,
environment variables or a file, each reading back to the same pipelines.
Its test fails when it links TLS, HTTP, process, signal or terminal code.
- `lw --schema`: settings, pipeline flags, flow stages in order with what
turns each on, sources and sinks with their tables' keys of their side,
presets and forms, as one JSON document.
- `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions,
layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal,
ConfirmDialog, scrolling, mouse; themed Radio, Field, TextInput,
OptionList and menus, Frame, Rule, Wires and Window, which `lw --tui`
draws with. `github.com/lixenwraith/color`: RGB,
blending, RGBTo256, RGBTo16.
Steps:
1. Schema, in `internal/config`: done.
2. Engine, `internal/compose`: done.
3. `lw --schema`: done. The site generates its catalogue from it.
4. `lixenwraith/terminal` v0.2.0: done: 16 and no color, the terminal's
default colors, `/dev/tty`, bracketed paste, themed form controls. Still
optional, in `color`: export the 16 xterm reference shades (RGBTo16 maps
the site's hues badly, so the 16-color tier is chosen by hand). v0.2.1:
radio buttons, narrow form fields and option lists, menus, frames, rules,
wires and windows, moved from `lw --tui`. Pinned at the untagged v0.2.2
commit: one glyph per cell at output, a reverse cursor bar in one style.
5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)).
6. WebAssembly: done. `make wasm` writes `bin/lwconf/`, which a site drops
in whole:
- `lwconf.wasm`, version stamped: 7.1 MB, 1.9 MB at gzip -9.
- `wasm_exec.js`, the toolchain's, which uses no eval: the page needs only
`'wasm-unsafe-eval'` in its `script-src`.
- `lwconf.js`, one dependency-free ES module that knows the calls and
shapes, never a key: `load({execURL, execSRI, wasmURL, wasmSRI})`
fetches both by their integrity and resolves to the engine;
`fields(schema, role, type)` is a node's form rows, a table's a group;
`value`, `at` and `group` read a node's value and sort problems by node.
- `schema.json`, equal to `schema()`, and `lwconf.json`, the manifest:
version, commit, Go version and each file's sha256.
The engine's calls take strings (`lwconf.js` writes objects as JSON) and
answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`,
as far as the engine places the refusal:
- `schema()`: what `lw --schema` prints.
- `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key
to a string, a list key to strings.
- `parse(text, form, dirs)`: the pipelines of a form's pasted text.
- `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled
(defaults, kinds, ids) and the first problem of each part (a value of
the wrong kind too) as `{path, pipeline, node, key, text}`.
- `emit(pipelines, form)`: the text of `command`, `environment` or `file`.
- `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is
`{op, pipeline, ...}`: `add_pipeline` and `rename_pipeline` (`name`),
`remove_pipeline`, `add` (`role`, `type`), `retype` (`node`, `type`),
`remove` (`node`), `move_filter` (`from`, `to`), `set` (`node`, `key`,
`values`), `unset` (`node`, `key`).
- `dirs` answers whether a path a preset names is a directory,
`{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one,
one with a glob a pattern, and any other is refused at `path`.
- Pipelines are a JSON list in the file's shape (`name`, `flow`,
`plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node
is `{role, id, index}`: a source's or sink's id, a filter's index.
7. `t`, the tap: done ([doc/cli.md](cli.md#lw---tui)): `core.Tap` takes each
flow verdict, `internal/logline` reads any line shape, and the live view
shows them under the diagram.
Not yet:
`lw --tui` ([CLI](cli.md#lw---tui)) and the configuration engine it shares
with the website ([Architecture](architecture.md#configuration-engine)) are
in place. Not yet:
- Pins and export in the live view, as vif-log has them: pinned rows survive
filter changes and show alone on a key; the pins or the shown rows go to a
new file.
@@ -172,7 +76,11 @@ Not yet:
`lixenwraith/terminal`'s one-cell rule goes once terminal exports that rule.
- A scripted session on a pseudo-terminal for Run, each output and `t` (keys,
the password dialog, `SIGHUP`), on a text console and over ssh.
- `AuthOptions.Check` alone limits `auth.node_binding` to chain sources and
`auth.trusted_proxies` to the http sink, so the schema lists both on every
listener; a side tag naming those sides would filter them as `http`
filters HTTP's keys.
- The schema lists `auth.node_binding` on every listener and
`auth.trusted_proxies` on both HTTP listeners, though `AuthOptions.Check`
refuses a binding off the chain sources (under `mtls` or `scram`) and
`trusted_proxies` off the `http` sink; side tags for chain sources and the
`http` sink would filter them as `http` filters HTTP's keys.
- Optional, in `lixenwraith/color`: export the 16 xterm reference shades
(`RGBTo16` maps the site's hues badly, so the 16-color tier is chosen by
hand).
internal/config/config.go
+9 -10
View File
@@ -15,7 +15,6 @@ type Config struct {
// Selected file path is runtime metadata, never a configurable override.
ConfigFile string `toml:"-"`
// Existing fields
Logging *LogConfig `toml:"logging"`
Pipelines []PipelineConfig `toml:"pipelines"`
}
@@ -79,15 +78,15 @@ type HeartbeatConfig struct {
Enabled bool `toml:"enabled" help:"send heartbeats"`
IntervalMS int64 `toml:"interval_ms" default:"1000" lw:"min=100" help:"milliseconds between heartbeats"`
IncludeTimestamp bool `toml:"include_timestamp" help:"stamp each heartbeat with its time"`
IncludeStats bool `toml:"include_stats" help:"add the pipeline's counters"`
Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"heartbeat format"`
IncludeStats bool `toml:"include_stats" help:"add type, beat_count and, from the second beat, the measured interval_ms"`
Format string `toml:"format" default:"txt" lw:"enum=txt|json|raw" help:"no effect: heartbeats use the flow's format"`
}
// FormatConfig is how a pipeline writes its entries
type FormatConfig struct {
Type string `toml:"type" default:"raw" lw:"enum=raw|txt|json" help:"entry format"`
Flags int64 `toml:"flags,omitempty" lw:"zero=by type" help:"formatter flags"`
TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339" help:"Go time layout of timestamps"`
TimestampFormat string `toml:"timestamp_format,omitempty" lw:"zero=RFC 3339 with nanoseconds" help:"Go time layout of timestamps"`
SanitizerPolicy string `toml:"sanitizer_policy,omitempty" lw:"enum=raw|json|txt|shell,zero=by type" help:"how control characters are escaped"`
}
@@ -143,7 +142,7 @@ type PluginSourceConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
ConfigFile string `toml:"config_file,omitempty"` // accepted, never read
}
// NullSourceOptions defines settings for a null source (no configuration needed)
@@ -152,10 +151,10 @@ type NullSourceOptions struct{}
// RandomSourceOptions defines settings for a random log generator source
type RandomSourceOptions struct {
IntervalMS int64 `toml:"interval_ms" default:"500" lw:"min=1" help:"milliseconds between entries"`
JitterMS int64 `toml:"jitter_ms" lw:"min=0,zero=none" help:"random milliseconds added to the interval, at most the interval"`
JitterMS int64 `toml:"jitter_ms" lw:"min=0,zero=none" help:"milliseconds the interval varies by, half each way; at most the interval"`
Format string `toml:"format" default:"txt" lw:"enum=raw|txt|json" help:"entry format"`
Length int64 `toml:"length" default:"20" lw:"min=1" help:"message characters"`
Special bool `toml:"special" help:"mix in control characters"`
Length int64 `toml:"length" default:"20" lw:"min=1" help:"bytes of random text"`
Special bool `toml:"special" help:"mix in control and non-ASCII bytes"`
}
// FileSourceOptions defines settings for a file-based source
@@ -209,7 +208,7 @@ type PluginSinkConfig struct {
ID string `toml:"id"`
Type string `toml:"type"`
Config map[string]any `toml:"config"`
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
ConfigFile string `toml:"config_file,omitempty"` // accepted, never read
}
// NullSinkOptions defines settings for a null sink (no configuration needed)
@@ -230,7 +229,7 @@ type FileSinkOptions struct {
MaxSizeMB int64 `toml:"max_size_mb" default:"100" lw:"min=1" help:"size at which a file rotates"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb" default:"1000" lw:"min=1" help:"size of all files, beyond which the oldest go"`
MinDiskFreeMB int64 `toml:"min_disk_free_mb" lw:"zero=none" help:"free space kept on the disk; negative means 100"`
RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0" help:"hours a file is kept"`
RetentionHours float64 `toml:"retention_hours" default:"168" lw:"min=0,zero=kept" help:"hours a .log file is kept; 0 keeps them"`
BufferSize int64 `toml:"buffer_size" default:"1000" lw:"min=1" help:"entries queued"`
FlushIntervalMs int64 `toml:"flush_interval_ms" default:"100" lw:"min=1" help:"milliseconds between flushes"`
}
internal/config/schema.go
+3 -3
View File
@@ -42,13 +42,13 @@ type Plugin struct {
var plugins = []Plugin{
{"source", "console", Local, false, true, "Standard input, line by line", reflect.TypeFor[ConsoleSourceOptions]()},
{"source", "file", Local, false, false, "Follow the files of a directory that match a pattern", reflect.TypeFor[FileSourceOptions]()},
{"source", "random", Local, false, false, "Random entries, for tests", reflect.TypeFor[RandomSourceOptions]()},
{"source", "null", Local, false, false, "Nothing, ever", reflect.TypeFor[NullSourceOptions]()},
{"source", "random", Local, false, false, "Generated entries, to check a deployment before real logs", reflect.TypeFor[RandomSourceOptions]()},
{"source", "null", Local, false, false, "Nothing; holds a source's place until one is chosen", reflect.TypeFor[NullSourceOptions]()},
{"source", "tcp_chain", ChainListener, false, false, "Entries from tcp_chain sinks of other instances", reflect.TypeFor[TCPChainSourceOptions]()},
{"source", "http_chain", ChainListener, true, false, "Entry batches from http_chain sinks of other instances", reflect.TypeFor[HTTPChainSourceOptions]()},
{"sink", "console", Local, false, false, "Standard output or standard error", reflect.TypeFor[ConsoleSinkOptions]()},
{"sink", "file", Local, false, false, "Rotating files", reflect.TypeFor[FileSinkOptions]()},
{"sink", "null", Local, false, false, "Discard", reflect.TypeFor[NullSinkOptions]()},
{"sink", "null", Local, false, false, "Discards; holds a sink's place until one is chosen", reflect.TypeFor[NullSinkOptions]()},
{"sink", "tcp", Listener, false, false, "Serve entries to TCP clients, one per line", reflect.TypeFor[TCPSinkOptions]()},
{"sink", "http", Listener, true, false, "Serve entries as server-sent events, with a browser viewer", reflect.TypeFor[HTTPSinkOptions]()},
{"sink", "tcp_chain", Dialer, false, false, "Forward entries to a tcp_chain source", reflect.TypeFor[TCPChainSinkOptions]()},
internal/config/schema_test.go
+15 -4
View File
@@ -32,10 +32,11 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) {
}
}
// A zero number takes the default unless zero means something there, a switch
// that defaults on stays off when set off, a negative number is refused, and
// what moved from the constructors keeps its meaning: min_disk_free_mb below
// zero is the default, a backoff maximum below the minimum the default maximum
// A zero number takes the default unless zero means something there (a file
// sink's retention_hours 0 keeps files), a switch that defaults on stays off
// when set off, a negative number is refused, and what moved from the
// constructors keeps its meaning: min_disk_free_mb below zero is the default,
// a backoff maximum below the minimum the default maximum
func TestZeroDefaultsAndNegatives(t *testing.T) {
tcp, err := Decode[TCPSinkOptions]("sink", "tcp", map[string]any{"port": int64(1), "buffer_size": int64(0), "keep_alive": false})
if err != nil || tcp.BufferSize != 1000 || tcp.KeepAlive || tcp.Host != "0.0.0.0" {
@@ -55,6 +56,16 @@ func TestZeroDefaultsAndNegatives(t *testing.T) {
t.Errorf("min_disk_free_mb %v: %+v %v, want %d", in, file, err, want)
}
}
for in, want := range map[any]float64{nil: 168, int64(0): 0, 0.0: 0, 2.5: 2.5} {
m := map[string]any{"directory": "d", "name": "n", "retention_hours": in}
if in == nil {
delete(m, "retention_hours")
}
file, err := Decode[FileSinkOptions]("sink", "file", m)
if err != nil || file.RetentionHours != want {
t.Errorf("retention_hours %v: %+v %v, want %v", in, file, err, want)
}
}
chain, err := Decode[TCPChainSinkOptions]("sink", "tcp_chain", map[string]any{"host": "h", "port": int64(1), "backoff_min_ms": int64(800), "backoff_max_ms": int64(700)})
if err != nil || chain.BackoffMaxMS != 30000 {
t.Fatalf("backoff maximum below the minimum: %+v %v", chain, err)
internal/sink/http/web/scram.js
+4 -3
View File
@@ -16,9 +16,10 @@ const encoder = new TextEncoder();
/**
* Logs in to the logwisp http sink mounted at base (a URL ending in "/"), per
* logwisp's doc/scram-auth-plan.md; it resolves only once the server has proved
* it holds this user's verifier. The session is an HttpOnly cookie, or with
* session "token" the returned token. onProgress(fraction) follows Argon2.
* logwisp's doc/security.md (Password authentication) and doc/chaining.md
* (Wire protocol); it resolves only once the server has proved it holds this
* user's verifier. The session is an HttpOnly cookie, or with session "token"
* the returned token. onProgress(fraction) follows Argon2.
*/
export async function login(base, username, password, { onProgress, session = "cookie" } = {}) {
if (session !== "cookie" && session !== "token") {
internal/sink/null/null.go
+2 -16
View File
@@ -15,35 +15,30 @@ import (
"github.com/lixenwraith/log"
)
// init registers the component in plugin factory
func init() {
if err := plugin.RegisterSink("null", NewNullSinkPlugin); err != nil {
panic(fmt.Sprintf("failed to register null sink: %v", err))
}
}
// NullSink discards all received transport events, used for testing
// NullSink discards every event: it holds a sink's place until one is chosen,
// or isolates a source or flow while it is measured
type NullSink struct {
// Plugin identity and session management
id string
proxy *session.Proxy
session *session.Session
// Application
input chan core.TransportEvent
logger *log.Logger
// Runtime
done chan struct{}
startTime time.Time
// Statistics
totalReceived atomic.Uint64
totalBytes atomic.Uint64
lastReceived atomic.Value // time.Time
}
// NewNullSinkPlugin creates a null sink through plugin factory
func NewNullSinkPlugin(
id string,
configMap map[string]any,
@@ -62,7 +57,6 @@ func NewNullSinkPlugin(
}
ns.lastReceived.Store(time.Time{})
// Create session for null sink
ns.session = proxy.CreateSession(
"null://devnull",
map[string]any{
@@ -78,7 +72,6 @@ func NewNullSinkPlugin(
return ns, nil
}
// Capabilities returns supported capabilities
func (ns *NullSink) Capabilities() []core.Capability {
return []core.Capability{
core.CapSessionAware,
@@ -86,14 +79,11 @@ func (ns *NullSink) Capabilities() []core.Capability {
}
}
// Input returns the channel for sending transport events
func (ns *NullSink) Input() chan<- core.TransportEvent {
return ns.input
}
// Start begins the processing loop
func (ns *NullSink) Start(ctx context.Context) error {
ns.startTime = time.Now()
go ns.processLoop(ctx)
ns.logger.Debug("msg", "Null sink started",
@@ -102,7 +92,6 @@ func (ns *NullSink) Start(ctx context.Context) error {
return nil
}
// Stop gracefully shuts down the sink
func (ns *NullSink) Stop() {
if ns.session != nil {
ns.proxy.RemoveSession(ns.session.ID)
@@ -113,7 +102,6 @@ func (ns *NullSink) Stop() {
"total_received", ns.totalReceived.Load())
}
// GetStats returns sink statistics
func (ns *NullSink) GetStats() sink.SinkStats {
lastRcv, _ := ns.lastReceived.Load().(time.Time)
@@ -129,7 +117,6 @@ func (ns *NullSink) GetStats() sink.SinkStats {
}
}
// processLoop reads transport events and discards them
func (ns *NullSink) processLoop(ctx context.Context) {
for {
select {
@@ -137,7 +124,6 @@ func (ns *NullSink) processLoop(ctx context.Context) {
if !ok {
return
}
// Discard the event, only update stats
ns.totalReceived.Add(1)
ns.totalBytes.Add(uint64(len(event.Payload)))
ns.lastReceived.Store(time.Now())
internal/source/file/file_watcher.go
+2 -2
View File
@@ -310,8 +310,6 @@ func (w *fileWatcher) checkFile() error {
return nil
}
// initPosition records the file's metadata and, unless the watcher was created
// to read from the start, sets the initial read position to the end
// scanLinesCapped is bufio.ScanLines, except that a line longer than
// core.MaxLogEntryBytes continues in the next one, as on the console source:
// the scanner would otherwise stop at it and every poll re-read the lines before.
@@ -323,6 +321,8 @@ func scanLinesCapped(data []byte, atEOF bool) (int, []byte, error) {
return advance, token, err
}
// initPosition records the file's metadata and, unless the watcher was created
// to read from the start, sets the initial read position to the end.
func (w *fileWatcher) initPosition() error {
file, err := os.Open(w.directory)
if err != nil {
internal/source/null/null.go
+1 -13
View File
@@ -14,34 +14,28 @@ import (
"github.com/lixenwraith/log"
)
// init registers the component in plugin factory
func init() {
if err := plugin.RegisterSource("null", NewNullSourcePlugin); err != nil {
panic(fmt.Sprintf("failed to register null source: %v", err))
}
}
// NullSource generates no log entries, used for testing
// NullSource generates nothing: it holds a source's place until one is chosen
type NullSource struct {
// Plugin identity and session management
id string
proxy *session.Proxy
session *session.Session
// Application
subscribers []chan core.LogEntry
logger *log.Logger
// Runtime
done chan struct{}
// Statistics
totalEntries atomic.Uint64
startTime time.Time
lastEntryTime atomic.Value // time.Time
}
// NewNullSourcePlugin creates a null source through plugin factory
func NewNullSourcePlugin(
id string,
configMap map[string]any,
@@ -60,7 +54,6 @@ func NewNullSourcePlugin(
}
ns.lastEntryTime.Store(time.Time{})
// Create session for null source
ns.session = proxy.CreateSession(
"null://void",
map[string]any{
@@ -76,21 +69,18 @@ func NewNullSourcePlugin(
return ns, nil
}
// Capabilities returns supported capabilities
func (ns *NullSource) Capabilities() []core.Capability {
return []core.Capability{
core.CapSessionAware,
}
}
// Subscribe returns a channel for receiving log entries
func (ns *NullSource) Subscribe() <-chan core.LogEntry {
ch := make(chan core.LogEntry, 1000)
ns.subscribers = append(ns.subscribers, ch)
return ch
}
// Start begins the source operation (no-op for null source)
func (ns *NullSource) Start() error {
ns.startTime = time.Now()
ns.proxy.UpdateActivity(ns.session.ID)
@@ -100,7 +90,6 @@ func (ns *NullSource) Start() error {
return nil
}
// Stop signals the source to stop
func (ns *NullSource) Stop() {
close(ns.done)
if ns.session != nil {
@@ -114,7 +103,6 @@ func (ns *NullSource) Stop() {
"instance_id", ns.id)
}
// GetStats returns the source's statistics
func (ns *NullSource) GetStats() source.SourceStats {
lastEntry, _ := ns.lastEntryTime.Load().(time.Time)
internal/source/random/random.go
+1 -26
View File
@@ -17,49 +17,41 @@ import (
"github.com/lixenwraith/log"
)
// init registers the component in plugin factory
func init() {
if err := plugin.RegisterSource("random", NewRandomSourcePlugin); err != nil {
panic(fmt.Sprintf("failed to register random source: %v", err))
}
}
// RandomSource generates random log entries for testing
// RandomSource generates entries, to check a deployment before real logs are wired
type RandomSource struct {
// Plugin identity and session management
id string
proxy *session.Proxy
session *session.Session
// Configuration
config *config.RandomSourceOptions
// Application
subscribers []chan core.LogEntry
logger *log.Logger
rng *rand.Rand
mu sync.RWMutex
// Runtime
done chan struct{}
wg sync.WaitGroup
cancel chan struct{}
// Statistics
totalEntries atomic.Uint64
droppedEntries atomic.Uint64
startTime time.Time
lastEntryTime atomic.Value // time.Time
}
// NewRandomSourcePlugin creates a random source through plugin factory
func NewRandomSourcePlugin(
id string,
configMap map[string]any,
logger *log.Logger,
proxy *session.Proxy,
) (source.Source, error) {
// Step 1: Create empty config struct with defaults
opts, err := config.Decode[config.RandomSourceOptions]("source", "random", configMap)
if err != nil {
return nil, err
@@ -77,7 +69,6 @@ func NewRandomSourcePlugin(
}
rs.lastEntryTime.Store(time.Time{})
// Create session for random source
rs.session = proxy.CreateSession(
fmt.Sprintf("random://%s", id),
map[string]any{
@@ -98,14 +89,12 @@ func NewRandomSourcePlugin(
return rs, nil
}
// Capabilities returns supported capabilities
func (rs *RandomSource) Capabilities() []core.Capability {
return []core.Capability{
core.CapSessionAware,
}
}
// Subscribe returns a channel for receiving log entries
func (rs *RandomSource) Subscribe() <-chan core.LogEntry {
rs.mu.Lock()
defer rs.mu.Unlock()
@@ -114,7 +103,6 @@ func (rs *RandomSource) Subscribe() <-chan core.LogEntry {
return ch
}
// Start begins generating random log entries
func (rs *RandomSource) Start() error {
rs.startTime = time.Now()
rs.wg.Add(1)
@@ -127,7 +115,6 @@ func (rs *RandomSource) Start() error {
return nil
}
// Stop signals the source to stop generating
func (rs *RandomSource) Stop() {
close(rs.cancel)
rs.wg.Wait()
@@ -148,7 +135,6 @@ func (rs *RandomSource) Stop() {
"total_entries", rs.totalEntries.Load())
}
// GetStats returns the source's statistics
func (rs *RandomSource) GetStats() source.SourceStats {
lastEntry, _ := rs.lastEntryTime.Load().(time.Time)
@@ -169,12 +155,10 @@ func (rs *RandomSource) GetStats() source.SourceStats {
}
}
// generateLoop continuously generates random log entries at configured intervals
func (rs *RandomSource) generateLoop() {
defer rs.wg.Done()
for {
// Calculate next interval with jitter
interval := time.Duration(rs.config.IntervalMS) * time.Millisecond
if rs.config.JitterMS > 0 {
jitter := time.Duration(rs.rng.Intn(int(rs.config.JitterMS))) * time.Millisecond
@@ -194,7 +178,6 @@ func (rs *RandomSource) generateLoop() {
}
}
// generateEntry creates a random log entry based on configured format
func (rs *RandomSource) generateEntry() core.LogEntry {
now := time.Now()
@@ -247,7 +230,6 @@ func (rs *RandomSource) generateEntry() core.LogEntry {
}
}
// generateRandomString creates a random string of specified length
func (rs *RandomSource) generateRandomString(length int) string {
const normalChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "
const specialChars = "\t\n\r\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F"
@@ -259,17 +241,14 @@ func (rs *RandomSource) generateRandomString(length int) string {
// Reserve space for at least one special and one unicode char
normalLength := length - 2
// Generate normal characters
for i := 0; i < normalLength; i++ {
result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
}
// Insert special character at random position
specialPos := rs.rng.Intn(len(result) + 1)
specialChar := specialChars[rs.rng.Intn(len(specialChars))]
result = append(result[:specialPos], append([]byte{specialChar}, result[specialPos:]...)...)
// Insert unicode character at random position
unicodePos := rs.rng.Intn(len(result) + 1)
unicodeChar := unicodeChars[rs.rng.Intn(len(unicodeChars)/3)*3:]
if len(unicodeChar) >= 3 {
@@ -281,12 +260,10 @@ func (rs *RandomSource) generateRandomString(length int) string {
}
}
// Trim to exact length if needed
if len(result) > length {
result = result[:length]
}
} else {
// Normal generation without special characters
for i := 0; i < length; i++ {
result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
}
@@ -295,13 +272,11 @@ func (rs *RandomSource) generateRandomString(length int) string {
return string(result)
}
// randomLogLevel returns a random log level
func (rs *RandomSource) randomLogLevel() string {
levels := []string{"DEBUG", "INFO", "WARN", "ERROR"}
return levels[rs.rng.Intn(len(levels))]
}
// publish sends a log entry to all subscribers
func (rs *RandomSource) publish(entry core.LogEntry) {
rs.mu.RLock()
defer rs.mu.RUnlock()
internal/tui/inspector.go
+1 -1
View File
@@ -109,7 +109,7 @@ func (a *app) items(n compose.Node, path string) []string {
// placeholder says what an unset key means: its default, what zero means, or
// the kind of value it takes
func placeholder(k config.Key) string {
if k.Zero != "" {
if k.Zero != "" && k.Default == "" {
return "(" + k.Zero + ")"
}
return cmp.Or(k.Default, k.Hint)
internal/tui/tui_test.go
+10
View File
@@ -249,6 +249,16 @@ func TestInspectorHelpSitsUnderItsKeys(t *testing.T) {
}
}
// An unset key shows its default, what zero means when it has none, or the
// kind of value it takes
func TestAnUnsetKeyShowsItsDefaultFirst(t *testing.T) {
for want, k := range map[string]config.Key{"168": {Default: "168", Zero: "kept"}, "(off)": {Zero: "off"}, "dir": {Hint: "dir"}} {
if got := placeholder(k); got != want {
t.Errorf("%+v shows %q, want %q", k, got, want)
}
}
}
// Hints follow their text, three columns on, not the screen's far edge: the
// keys after the status, the bar's verbs after the pipelines
func TestHintsFollowTheirText(t *testing.T) {