From 71d1b82f83b7fa5658dc4875279e141092783130ba164b4c3d9db28969d7ea30 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sat, 3 Oct 2026 14:52:16 -0400 Subject: [PATCH] v0.22.0 several enhancements and fixes for security, network, doc, test, packaging --- .dockerignore | 21 + .github/workflows/test.yml | 41 +- .gitignore | 3 + AGENTS.md | 15 +- Dockerfile | 31 +- Makefile | 255 +++- README.md | 77 +- cmd/{logwisp => lw}/auth.go | 35 +- cmd/{logwisp => lw}/auth_test.go | 18 +- cmd/{logwisp => lw}/bootstrap.go | 17 + cmd/{logwisp => lw}/help.go | 29 +- cmd/{logwisp => lw}/main.go | 11 +- cmd/{logwisp => lw}/output.go | 2 +- cmd/{logwisp => lw}/reload.go | 0 cmd/{logwisp => lw}/reload_test.go | 44 + cmd/{logwisp => lw}/status.go | 0 cmd/{logwisp => lw}/status_test.go | 0 cmd/{logwisp => lw}/test_helpers_test.go | 4 +- config/logwisp.toml | 30 +- deploy/README.md | 31 + deploy/lw-deploy.sh | 1331 +++++++++++++++++++++ deploy/package/arch/PKGBUILD | 46 + deploy/package/freebsd/Makefile | 44 + deploy/package/freebsd/files/logwisp.in | 1 + deploy/package/freebsd/pkg-descr | 5 + deploy/package/freebsd/pkg-plist | 3 + deploy/package/logwisp.rc | 47 + deploy/package/logwisp.service | 54 + deploy/package/logwisp.sysusers | 2 + deploy/package/logwisp.tmpfiles | 3 + doc/README.md | 51 +- doc/architecture.md | 112 +- doc/chaining.md | 53 +- doc/cli.md | 263 ++-- doc/configuration.md | 171 +-- doc/deployment.md | 198 +++ doc/filters.md | 34 +- doc/formatters.md | 70 +- doc/installation.md | 433 ++++--- doc/lw.1 | 328 +++++ doc/mtls-auth-plan.md | 69 +- doc/networking.md | 135 ++- doc/operations.md | 69 +- doc/scram-auth-plan.md | 143 ++- doc/security.md | 399 ++++-- doc/sinks.md | 161 +-- doc/sources.md | 92 +- doc/todo.md | 202 ++++ internal/authz/authz.go | 27 +- internal/authz/proxy_test.go | 2 +- internal/authz/scram.go | 35 +- internal/authz/scram_test.go | 86 ++ internal/config/config.go | 3 +- internal/config/loader.go | 50 +- internal/config/loader_test.go | 8 + internal/config/spec.go | 272 +++++ internal/config/spec_test.go | 190 +++ internal/config/validate.go | 20 +- internal/core/const.go | 24 +- internal/core/const_test.go | 53 + internal/core/flow.go | 1 - internal/core/listen.go | 49 + internal/core/listen_test.go | 34 + internal/filter/chain.go | 2 +- internal/filter/filter.go | 2 +- internal/flow/flow.go | 1 - internal/flow/heartbeat.go | 8 +- internal/flow/heartbeat_test.go | 29 + internal/flow/ratelimiter.go | 2 +- internal/format/format.go | 2 +- internal/pipeline/registry.go | 2 +- internal/plugin/factory.go | 2 +- internal/sanitize/sanitize.go | 2 +- internal/sink/file/file.go | 32 +- internal/sink/http/http.go | 13 +- internal/sink/http/web/login.js | 19 +- internal/sink/http/web/scram.js | 136 ++- internal/sink/http/web/scram.test.mjs | 100 +- internal/sink/http/web/style.css | 9 + internal/sink/http/web/view.html | 8 + internal/sink/http/web/view.js | 149 ++- internal/sink/httpchain/httpchain.go | 18 +- internal/sink/httpchain/httpchain_test.go | 74 ++ internal/sink/sink.go | 2 +- internal/sink/tcp/tcp.go | 13 +- internal/sink/tcpchain/tcpchain.go | 10 +- internal/source/httpchain/httpchain.go | 15 +- internal/source/source.go | 2 +- internal/source/tcpchain/tcpchain.go | 18 +- internal/testutil/fixtures.go | 19 +- internal/tlsx/tlsx.go | 5 +- internal/tlsx/tlsx_test.go | 16 + internal/version/version.go | 2 +- test/chain-aggregate-test.sh | 223 +--- test/chain-test.sh | 216 +--- test/deploy-test.sh | 347 ++++++ test/ipv6-test.sh | 244 ++++ test/lib.sh | 288 +++++ test/mtls-chain-test.sh | 434 ++----- test/passthrough-test.sh | 58 +- test/scram-chain-test.sh | 329 ++--- test/scram-proxy-test.sh | 392 +++--- 102 files changed, 7102 insertions(+), 2178 deletions(-) create mode 100644 .dockerignore rename cmd/{logwisp => lw}/auth.go (90%) rename cmd/{logwisp => lw}/auth_test.go (87%) rename cmd/{logwisp => lw}/bootstrap.go (89%) rename cmd/{logwisp => lw}/help.go (55%) rename cmd/{logwisp => lw}/main.go (93%) rename cmd/{logwisp => lw}/output.go (99%) rename cmd/{logwisp => lw}/reload.go (100%) rename cmd/{logwisp => lw}/reload_test.go (67%) rename cmd/{logwisp => lw}/status.go (100%) rename cmd/{logwisp => lw}/status_test.go (100%) rename cmd/{logwisp => lw}/test_helpers_test.go (71%) create mode 100644 deploy/README.md create mode 100755 deploy/lw-deploy.sh create mode 100644 deploy/package/arch/PKGBUILD create mode 100644 deploy/package/freebsd/Makefile create mode 120000 deploy/package/freebsd/files/logwisp.in create mode 100644 deploy/package/freebsd/pkg-descr create mode 100644 deploy/package/freebsd/pkg-plist create mode 100644 deploy/package/logwisp.rc create mode 100644 deploy/package/logwisp.service create mode 100644 deploy/package/logwisp.sysusers create mode 100644 deploy/package/logwisp.tmpfiles create mode 100644 doc/deployment.md create mode 100644 doc/lw.1 create mode 100644 doc/todo.md create mode 100644 internal/config/spec.go create mode 100644 internal/config/spec_test.go create mode 100644 internal/core/const_test.go create mode 100644 internal/core/listen.go create mode 100644 internal/core/listen_test.go create mode 100644 internal/flow/heartbeat_test.go create mode 100755 test/deploy-test.sh create mode 100755 test/ipv6-test.sh create mode 100755 test/lib.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f209e5e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +# The builder needs the module and its sources only; local state and keys stay out. +.git +.github +.idea +.claude +worktrees +.worktrees +bin +build +cert +data +dev +log +logs +script +test/run* +**/node_modules +*.log +*.pass +*.toml +!config/*.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04da8dc..0d3c188 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,44 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod + - uses: actions/setup-node@v4 + with: + node-version: 22 - run: go test -timeout 60s -race ./... - run: go vet ./... - - run: go build ./cmd/logwisp - - run: GOOS=freebsd GOARCH=amd64 go build -o /tmp/logwisp-freebsd ./cmd/logwisp + - run: go build ./cmd/lw + - run: GOOS=freebsd GOARCH=amd64 go build -o /tmp/lw-freebsd ./cmd/lw + - run: node --test + working-directory: internal/sink/http/web + # The end-to-end scripts on a host with an IPv6 loopback and playwright's + # Chromium; the docker run needs root, so it skips + e2e: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + FORCE_COLOR: "1" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - uses: actions/setup-node@v4 + with: + node-version: 22 + # "chromium" fetches the headless shell and full Chromium, which the + # cookie-blocking profile needs; the driver's require reads NODE_PATH + - run: | + npm i -g playwright + npx playwright install --with-deps chromium + echo "NODE_PATH=$(npm root -g)" >> "$GITHUB_ENV" + - run: make build + - run: make e2e 2>&1 | tee "$RUNNER_TEMP/e2e.out" + shell: bash # adds pipefail: the step fails with make, not with tee + # ipv6-test.sh skips without ::1, which would leave the job green untested + - name: Require an IPv6 loopback + run: grep -q '^0\{31\}1 ' /proc/net/if_inet6 + # scram-proxy-test.sh skips its browser checks without playwright's Chromium + - name: Require the browser checks + run: | + grep -q 'scram-proxy-test.sh --auto' "$RUNNER_TEMP/e2e.out" + if grep 'SKIP.*browser checks' "$RUNNER_TEMP/e2e.out"; then exit 1; fi diff --git a/.gitignore b/.gitignore index 9000880..95de01e 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ bin/ script/ build/ *.log +*.pass *.toml !config/*.toml build.sh @@ -17,3 +18,5 @@ test/run/ test/run-mtls/ test/run-scram/ test/run-proxy/ +test/run-ipv6/ +test/run-deploy/ diff --git a/AGENTS.md b/AGENTS.md index 519abad..fe0b9ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,9 @@ It is the contract, not a suggestion. - `doc/` is already long. Condense when you touch it; append only for a new concept or scope. +- No tables in docs, never a large one: docs are read in terminals and in editors + that do not render Markdown. Use hierarchical bullet lists (item, then its + details indented beneath); fenced blocks for config, commands and port maps. - A gap you are deferring goes in the "Not Implemented" list of the design doc it belongs to: extend the item, or add a concise one in the list's pattern. - The repository describes the architecture, never a machine: docs and commands @@ -64,7 +67,7 @@ It is the contract, not a suggestion. `go build ./...`, `gofmt -l` on changed files, and `go test` on the packages the change touches. - A change to platform-sensitive code also cross-builds the production target: - `GOOS=freebsd GOARCH=amd64 go build ./cmd/logwisp`. + `GOOS=freebsd GOARCH=amd64 go build ./cmd/lw`. - Do not run `-race` tests unless investigating a known/suspected race issue; CI runs them. @@ -91,8 +94,12 @@ change touches. dependency-free ES module (sites copy it into their bundles) and the pages stay free of inline script and style (their CSP); `node --test` there checks it against Go's Argon2 and `auth`'s known answer. -- Listeners and dialers are IPv4-only (`tcp4`). E2E scripts in `test/` need - `bin/logwisp` and `--auto`, and each owns a port range and a gitignored run - directory. +- Listeners and dialers take their network from `core.Network`, strictly per + family: an IPv4 literal `tcp4`, an IPv6 literal (`::` too) IPv6-only `tcp6`, + a hostname `tcp`. Join addresses with `net.JoinHostPort`, build URLs with + `net/url`. E2E scripts in `test/` need `bin/lw` and `--auto`, and each owns + a port range and a gitignored run directory. They source `test/lib.sh` + (checks, skips, summary; daemons log only to files) and exit 77 when the host + lacks what they test. - Many older files lack a trailing newline and fail `gofmt -l`; format the files you change, not the tree. diff --git a/Dockerfile b/Dockerfile index 58af5e6..fc976e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,23 +2,30 @@ # an older builder reports the mismatch only after downloading the module graph. ARG GO_VERSION=1.27.1 -FROM docker.io/library/golang:${GO_VERSION}-alpine AS build +# The builder runs natively and cross-compiles, so foreign platforms need no emulation. +FROM --platform=$BUILDPLATFORM docker.io/library/golang:${GO_VERSION}-alpine AS build -# Git supplies Go's VCS build information; only /out/logwisp crosses stages. -RUN apk add --no-cache git WORKDIR /src COPY go.mod go.sum ./ -RUN go mod download +# The optional build_ca secret adds a CA (an intercepting proxy's) for this step +# only: Go reads SSL_CERT_DIR besides the system bundle, and a secret mount +# never reaches a layer. Proxy variables are Docker's predefined build args. +RUN --mount=type=secret,id=build_ca,target=/run/build-ca/ca.pem \ + SSL_CERT_DIR=/run/build-ca go mod download + COPY . . -ARG TARGETOS=linux -ARG TARGETARCH=amd64 +# No defaults: a declared default would override the platform BuildKit passes. +ARG TARGETOS +ARG TARGETARCH ARG VERSION=dev ARG REVISION=unknown +ARG BUILD_TIME=unknown +# REVISION identifies the commit: the context has no .git, so no VCS stamping. RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -trimpath \ - -ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION}" \ - -o /out/logwisp ./cmd/logwisp + go build -trimpath -buildvcs=false \ + -ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION} -X logwisp/internal/version.BuildTime=${BUILD_TIME}" \ + -o /out/lw ./cmd/lw FROM scratch @@ -32,9 +39,11 @@ LABEL org.opencontainers.image.title="logwisp" \ org.opencontainers.image.version="${VERSION}" \ org.opencontainers.image.licenses="BSD-3-Clause" -COPY --from=build /out/logwisp /logwisp +# Dialers without tls.ca_file, and `lw auth` without -ca-file, verify against system roots. +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=build /out/lw /lw # Numeric identity is required in scratch and satisfies a restricted pod spec. USER 65532:65532 -ENTRYPOINT ["/logwisp"] +ENTRYPOINT ["/lw"] diff --git a/Makefile b/Makefile index aa2fc5d..c9201c5 100644 --- a/Makefile +++ b/Makefile @@ -1,52 +1,227 @@ -# LogWisp Makefile -# Compatible with GNU Make (Linux) and BSD Make (FreeBSD) +# LogWisp. GNU make and BSD make alike: no conditionals or functions specific +# to either; decisions run in the shell. Plain `make` prints the targets. -BINARY_NAME = logwisp -BUILD_DIR = bin -BINARY_PATH = $(BUILD_DIR)/$(BINARY_NAME) -VERSION != git describe --tags --always --dirty 2>/dev/null || echo "dev" -GIT_COMMIT != git rev-parse --short HEAD 2>/dev/null || echo "unknown" -BUILD_TIME != date -u '+%Y-%m-%d_%H:%M:%S' +BINARY := lw +SRC := ./cmd/lw +BIN_DIR := bin +PKG_DIR := deploy/package +WEB_DIR := internal/sink/http/web -# Go build variables -GO = go -GOFLAGS = -LDFLAGS = -X 'logwisp/internal/version.Version=$(VERSION)' \ - -X 'logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \ - -X 'logwisp/internal/version.BuildTime=$(BUILD_TIME)' +VERSION_DEFAULT != git describe --tags --always --dirty 2>/dev/null || echo dev +VERSION ?= $(VERSION_DEFAULT) +GIT_COMMIT_DEFAULT != git rev-parse --short HEAD 2>/dev/null || echo unknown +GIT_COMMIT ?= $(GIT_COMMIT_DEFAULT) +# SOURCE_DATE_EPOCH makes package builds reproducible; GNU date takes -d, BSD date -r. +BUILD_TIME_DEFAULT != f='+%Y-%m-%d_%H:%M:%S'; e="$${SOURCE_DATE_EPOCH:-}"; \ + if [ -n "$$e" ]; then date -u -d "@$$e" "$$f" 2>/dev/null || date -u -r "$$e" "$$f"; else date -u "$$f"; fi +BUILD_TIME ?= $(BUILD_TIME_DEFAULT) +# The image stamps the commit time, so a rebuild of one commit stays cached +IMAGE_BUILD_TIME_DEFAULT != TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%d_%H:%M:%S 2>/dev/null || echo unknown +IMAGE_BUILD_TIME ?= $(IMAGE_BUILD_TIME_DEFAULT) +VERSION_LDFLAGS = -X 'logwisp/internal/version.Version=$(VERSION)' \ + -X 'logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \ + -X 'logwisp/internal/version.BuildTime=$(BUILD_TIME)' +# Not LDFLAGS/GOFLAGS: packagers export those for the C linker and go itself. +GO_BUILDFLAGS ?= +GO_LDFLAGS ?= -# Installation directories -PREFIX ?= /usr/local -BINDIR = $(PREFIX)/bin +# FreeBSD packages each Go release under its own name: go.mod's 1.27 is go127. +GO_DEFAULT != uname -s | grep -q FreeBSD && sed -n 's/^go \([0-9]*\)\.\([0-9]*\).*/go\1\2/p' go.mod 2>/dev/null | grep . || echo go +GO ?= $(GO_DEFAULT) -# Default target -all: build +E2E ?= test/*-test.sh -# Build the binary -build: - mkdir -p $(BUILD_DIR) - $(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/logwisp +CONTAINER_ENGINE ?= docker +IMAGE ?= logwisp +IMAGE_TAG ?= dev +IMAGE_REVISION_DEFAULT != git rev-parse HEAD 2>/dev/null || echo unknown +IMAGE_REVISION ?= $(IMAGE_REVISION_DEFAULT) +# A PEM bundle the builder trusts while downloading modules (an intercepting +# proxy); it reaches the build as a BuildKit secret, never a layer. +BUILD_CA ?= +IMAGE_BUILD_FLAGS ?= +IMAGE_CHECK_CONFIG ?= config/logwisp.toml +IMAGE_RUN = $(CONTAINER_ENGINE) run --read-only --user 65532:65532 \ + --network none --cap-drop ALL --security-opt no-new-privileges -# Install the binary -install: build - install -m 755 $(BINARY_PATH) $(BINDIR)/ +DESTDIR ?= +INSTALL_OS_DEFAULT != uname -s +INSTALL_OS ?= $(INSTALL_OS_DEFAULT) +# FreeBSD keeps everything outside the base system under /usr/local. +PREFIX_DEFAULT != [ '$(INSTALL_OS)' = FreeBSD ] && echo /usr/local || echo /usr +PREFIX ?= $(PREFIX_DEFAULT) +SYSCONFDIR_DEFAULT != [ '$(INSTALL_OS)' = FreeBSD ] && echo /usr/local/etc || echo /etc +SYSCONFDIR ?= $(SYSCONFDIR_DEFAULT) +BINDIR ?= $(PREFIX)/bin +MANDIR ?= $(PREFIX)/share/man +DOCDIR ?= $(PREFIX)/share/doc/logwisp +LICENSEDIR ?= $(PREFIX)/share/licenses/logwisp + +.DEFAULT_GOAL := help + +.PHONY: help check-go build release dev test verify e2e image image-check install uninstall clean version + +help: + @echo "Usage: make [VARIABLE=value ...]" + @echo "" + @echo "Build:" + @echo " build Build $(BIN_DIR)/$(BINARY) with version metadata" + @echo " release Build it static, stripped and trimmed (CGO_ENABLED=0)" + @echo " dev Build it with the race detector" + @echo " version Print the version metadata a build embeds" + @echo " clean Remove $(BIN_DIR)/" + @echo "Check:" + @echo " test Run the Go tests" + @echo " verify vet, gofmt, tests, linux and freebsd cross builds, web client tests" + @echo " e2e Build, then run each of $(E2E) with --auto and report" + @echo "Container:" + @echo " image Build $(IMAGE):$(IMAGE_TAG) with $(CONTAINER_ENGINE) (scratch, static, UID 65532)" + @echo " image-check Run it read-only, offline, without capabilities: --version, then" + @echo " --check on $(IMAGE_CHECK_CONFIG)" + @echo "Install (build or release first):" + @echo " install Binary, manual, sample config, service files, licence and docs" + @echo " uninstall Remove them, keeping $(SYSCONFDIR)/logwisp" + @echo "" + @echo "Variables: GO=$(GO) PREFIX=$(PREFIX) SYSCONFDIR=$(SYSCONFDIR) DESTDIR=$(DESTDIR)" + @echo " INSTALL_OS=$(INSTALL_OS) (the layout install uses: FreeBSD or any other)" + @echo " GO_BUILDFLAGS, GO_LDFLAGS (extra go build and linker flags), E2E (scripts)," + @echo " CONTAINER_ENGINE, IMAGE, IMAGE_TAG, BUILD_CA, IMAGE_BUILD_FLAGS, IMAGE_CHECK_CONFIG" + +check-go: + @command -v $(GO) >/dev/null 2>&1 || { \ + echo "Go toolchain '$(GO)' not found; go.mod requires go $$(sed -n 's/^go //p' go.mod)."; \ + if [ "$$(uname -s)" = FreeBSD ]; then echo " FreeBSD: pkg install $(GO_DEFAULT)"; \ + elif [ -f /etc/arch-release ]; then echo " Arch Linux: pacman -S go"; \ + else echo " Elsewhere: https://go.dev/dl/ (distribution packages often lag)"; fi; \ + echo " Or name a toolchain: make GO=/path/to/go ..."; \ + exit 1; } + +build: check-go + $(GO) build $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC) + +release: check-go + CGO_ENABLED=0 $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "-s -w $(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC) + +dev: check-go + $(GO) build -race $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC) + +test: check-go + $(GO) test ./... + +# gofmt checks the Go files changed since the branch forked from GOFMT_BASE, +# exact renames aside, or the whole tree without git or that ref: older files +# predate the gate, and AGENTS.md formats what a change touches. +GOFMT_BASE ?= main +verify: test + $(GO) vet ./... + @base='$(GOFMT_BASE)'; \ + if [ -n "$$base" ] && fork=$$(git merge-base "$$base" HEAD 2>/dev/null); then \ + files=$$( { git diff --name-only --find-renames=100% --diff-filter=dr "$$fork" -- cmd internal; \ + git ls-files --others --exclude-standard -- cmd internal; } | grep '\.go$$' || true); \ + else \ + files=$$(find cmd internal -name '*.go'); \ + fi; \ + bad=$$( [ -z "$$files" ] || "$$($(GO) env GOROOT)/bin/gofmt" -l $$files ); \ + if [ -n "$$bad" ]; then echo "gofmt -l reports:"; echo "$$bad"; exit 1; fi + @for target in linux/amd64 linux/arm64 freebsd/amd64 freebsd/arm64; do \ + echo "cross build $$target"; \ + CGO_ENABLED=0 GOOS=$${target%/*} GOARCH=$${target#*/} $(GO) build -o /dev/null $(SRC) || exit 1; \ + done + @if command -v node >/dev/null 2>&1; then \ + cd $(WEB_DIR) && node --test; \ + else \ + echo "node not found: web client tests skipped"; \ + fi + +# A script exits 0 (passed), 77 (skipped: the host lacks what it tests) or +# anything else (failed); only failures fail the target. +e2e: build + @if [ "$${FORCE_COLOR:-0}" != 0 ] || { [ -t 1 ] && [ -z "$${NO_COLOR:-}" ]; }; then \ + g=$$(printf '\033[1;32m'); r=$$(printf '\033[1;31m'); y=$$(printf '\033[1;33m'); \ + b=$$(printf '\033[1m'); o=$$(printf '\033[0m'); \ + else g=; r=; y=; b=; o=; fi; \ + pass=0; fail=0; skip=0; results=; \ + for t in $(E2E); do \ + echo; "$$t" --auto; rc=$$?; \ + case $$rc in \ + 0) pass=$$((pass + 1)); results="$$results PASS:$$t" ;; \ + 77) skip=$$((skip + 1)); results="$$results SKIP:$$t" ;; \ + *) fail=$$((fail + 1)); results="$$results FAIL:$$t:$$rc" ;; \ + esac; \ + done; \ + printf '\n%se2e summary%s\n' "$$b" "$$o"; \ + for x in $$results; do \ + st=$${x%%:*}; t=$${x#*:}; c=$$g; note=; \ + case $$st in SKIP) c=$$y ;; FAIL) c=$$r; note=" (exit $${t##*:})"; t=$${t%:*} ;; esac; \ + printf ' %s%s%s %s%s\n' "$$c" "$$st" "$$o" "$$t" "$$note"; \ + done; \ + c=$$g; [ "$$fail" -eq 0 ] || c=$$r; \ + printf '%se2e: %d passed, %d failed, %d skipped%s\n' "$$c" "$$pass" "$$fail" "$$skip" "$$o"; \ + [ "$$fail" -eq 0 ] + +image: + @set --; if [ -n '$(BUILD_CA)' ]; then set -- --secret 'id=build_ca,src=$(BUILD_CA)'; fi; \ + set -x; \ + $(CONTAINER_ENGINE) build "$$@" $(IMAGE_BUILD_FLAGS) \ + --build-arg VERSION='$(VERSION)' \ + --build-arg REVISION='$(IMAGE_REVISION)' \ + --build-arg BUILD_TIME='$(IMAGE_BUILD_TIME)' \ + -t '$(IMAGE):$(IMAGE_TAG)' . + +# --check builds every pipeline and plugin, as a start would, and binds nothing +image-check: + $(IMAGE_RUN) --rm '$(IMAGE):$(IMAGE_TAG)' --version + @set -eu; cfg='$(IMAGE_CHECK_CONFIG)'; case "$$cfg" in /*) ;; *) cfg="$$(pwd)/$$cfg" ;; esac; \ + $(IMAGE_RUN) --rm --mount "type=bind,src=$$cfg,dst=/etc/logwisp/logwisp.toml,readonly" \ + '$(IMAGE):$(IMAGE_TAG)' --check -c /etc/logwisp/logwisp.toml + +# install stages what a package ships and compiles nothing, so a packager owns +# the build flags. The service files get the installed paths substituted; a +# configuration already in place is kept. +install: + @set -eu; \ + [ -x $(BIN_DIR)/$(BINARY) ] || { echo "no $(BIN_DIR)/$(BINARY): run make build or make release first" >&2; exit 1; }; \ + put() { install -d -m 0755 "$${3%/*}"; install -m "$$1" "$$2" "$$3"; echo "install $$3"; }; \ + sub() { install -d -m 0755 "$${3%/*}"; \ + sed -e 's|@BINDIR@|$(BINDIR)|g' -e 's|@SYSCONFDIR@|$(SYSCONFDIR)|g' -e 's|%%BINDIR%%|$(BINDIR)|g' \ + -e 's|%%PREFIX%%/etc|$(SYSCONFDIR)|g' -e 's|%%PREFIX%%|$(PREFIX)|g' "$$2" > "$$3"; \ + chmod "$$1" "$$3"; echo "install $$3"; }; \ + root='$(DESTDIR)'; etc="$$root$(SYSCONFDIR)/logwisp"; \ + put 0755 $(BIN_DIR)/$(BINARY) "$$root$(BINDIR)/$(BINARY)"; \ + put 0644 doc/lw.1 "$$root$(MANDIR)/man1/lw.1"; \ + put 0644 LICENSE "$$root$(LICENSEDIR)/LICENSE"; \ + for src in README.md doc/*.md; do put 0644 "$$src" "$$root$(DOCDIR)/$$src"; done; \ + case '$(INSTALL_OS)' in \ + FreeBSD) \ + put 0644 config/logwisp.toml "$$etc/logwisp.toml.sample"; \ + if [ -z "$$root" ] && [ ! -e "$$etc/logwisp.toml" ]; then put 0644 config/logwisp.toml "$$etc/logwisp.toml"; fi; \ + sub 0755 $(PKG_DIR)/logwisp.rc "$$root$(PREFIX)/etc/rc.d/logwisp" ;; \ + *) \ + if [ -e "$$etc/logwisp.toml" ]; then echo "keep $$etc/logwisp.toml"; \ + else put 0644 config/logwisp.toml "$$etc/logwisp.toml"; fi; \ + sub 0644 $(PKG_DIR)/logwisp.service "$$root$(PREFIX)/lib/systemd/system/logwisp.service"; \ + sub 0644 $(PKG_DIR)/logwisp.sysusers "$$root$(PREFIX)/lib/sysusers.d/logwisp.conf"; \ + sub 0644 $(PKG_DIR)/logwisp.tmpfiles "$$root$(PREFIX)/lib/tmpfiles.d/logwisp.conf" ;; \ + esac -# Uninstall the binary uninstall: - rm -f $(BINDIR)/$(BINARY_NAME) + @set -eu; root='$(DESTDIR)'; \ + for f in "$$root$(BINDIR)/$(BINARY)" "$$root$(MANDIR)/man1/lw.1" \ + "$$root$(PREFIX)/lib/systemd/system/logwisp.service" \ + "$$root$(PREFIX)/lib/sysusers.d/logwisp.conf" "$$root$(PREFIX)/lib/tmpfiles.d/logwisp.conf" \ + "$$root$(PREFIX)/etc/rc.d/logwisp" "$$root$(SYSCONFDIR)/logwisp/logwisp.toml.sample"; do \ + if [ -e "$$f" ]; then rm -f "$$f"; echo "remove $$f"; fi; \ + done; \ + for d in "$$root$(DOCDIR)" "$$root$(LICENSEDIR)"; do \ + case "$$d" in */logwisp) ;; *) echo "keep $$d (not a logwisp directory)"; continue ;; esac; \ + if [ -d "$$d" ]; then rm -rf "$$d"; echo "remove $$d"; fi; \ + done; \ + echo "keep $$root$(SYSCONFDIR)/logwisp (configuration and credentials)" -# Clean build artifacts clean: - rm -f $(BINARY_PATH) + rm -rf $(BIN_DIR) -# Development build with race detector -dev: - $(GO) build $(GOFLAGS) -race -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/logwisp - -# Show current version version: - @echo "Version: $(VERSION)" - @echo "Commit: $(GIT_COMMIT)" - @echo "Build Time: $(BUILD_TIME)" - -.PHONY: all build install uninstall clean dev version + @echo "Version: $(VERSION)" + @echo "Commit: $(GIT_COMMIT)" + @echo "Build time: $(BUILD_TIME)" diff --git a/README.md b/README.md index ed69bff..7232252 100644 --- a/README.md +++ b/README.md @@ -64,11 +64,11 @@ exponential backoff and jitter. stream and status endpoints, and lets a dialer pin the server it talks to - Password authentication (Argon2id-SCRAM) on the same block: listeners hold verifiers, never passwords, and do no KDF work; every login outside proxy - mode is bound to the - listener's certificate, so no relay presenting another one can use it; HTTP - listeners issue short-lived bearer tokens. `logwisp auth` manages credentials - files and logs viewers in; behind a site's TLS-terminating proxy, browsers - log in through a shipped login page and a dependency-free JS client + mode is bound to the listener's certificate, so no relay presenting another + one can use it; HTTP listeners issue short-lived bearer tokens. `lw auth` + manages credentials files and logs viewers in; behind a site's + TLS-terminating proxy, browsers log in through a shipped login page and a + dependency-free JS client - Node binding: a chain source can label entries from the sender's certificate identity or username instead of from what the sender claims, so origin attribution is not forgeable @@ -82,22 +82,30 @@ authentication designs for the rationale and what is deliberately left out. ## Documentation -| Document | Contents | -|----------|----------| -| [Installation](doc/installation.md) | Building, installing, running as a service | -| [Architecture](doc/architecture.md) | Component model, data flow, concurrency, back-pressure | -| [Configuration](doc/configuration.md) | TOML structure, precedence, environment and CLI overrides | -| [Sources](doc/sources.md) | Every input plugin and its options | -| [Sinks](doc/sinks.md) | Every output plugin and its options | -| [Filters](doc/filters.md) | Pattern-based inclusion and exclusion | -| [Formatters](doc/formatters.md) | Output shaping and sanitization | -| [Chaining](doc/chaining.md) | Multi-node topologies and the chain wire protocol | -| [Networking](doc/networking.md) | Listeners, dialers, timeouts, connection limits | -| [Security](doc/security.md) | TLS, mTLS, and peer authorization; threat model and current limits | -| [mTLS Authentication](doc/mtls-auth-plan.md) | Design and rationale for certificate-based authorization | -| [Password Authentication](doc/scram-auth-plan.md) | Design and rationale for Argon2id-SCRAM authentication; mTLS hardening | -| [CLI](doc/cli.md) | Flags, signals, exit codes, `logwisp auth` | -| [Operations](doc/operations.md) | Running, monitoring, tuning, troubleshooting | +- [Installation](doc/installation.md): building, installing, services, + the container image, packaging +- [Deployment](doc/deployment.md): `deploy/lw-deploy.sh` for edges, + aggregators, containers, services, jails +- [Architecture](doc/architecture.md): component model, data flow, + concurrency, back-pressure +- [Configuration](doc/configuration.md): TOML structure, precedence, + environment and CLI overrides +- [Sources](doc/sources.md) and [Sinks](doc/sinks.md): every plugin and its + options +- [Filters](doc/filters.md) and [Formatters](doc/formatters.md): pattern + matching, output shaping and sanitization +- [Chaining](doc/chaining.md): multi-node topologies and the wire protocol +- [Networking](doc/networking.md): listeners, dialers, timeouts, limits +- [Security](doc/security.md): TLS, mTLS, peer authorization, threat model +- Design notes: [mTLS](doc/mtls-auth-plan.md) and + [password](doc/scram-auth-plan.md) authentication +- [CLI](doc/cli.md): flags, pipeline specifications, signals, exit codes, + `lw auth`; also the `lw(1)` manual, [`doc/lw.1`](doc/lw.1) +- [Operations](doc/operations.md): running, monitoring, tuning, + troubleshooting +- [To Do](doc/todo.md): planned work in priority order: network access + control and the PROXY protocol, a hardening review of the config libraries, + packaging A fully annotated configuration covering every option ships as [`config/logwisp.toml`](config/logwisp.toml). @@ -105,9 +113,13 @@ A fully annotated configuration covering every option ships as ## Quick Start ```bash -make +make build # plain `make` lists the targets +./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \ + --format json,sanitizer_policy=json --sink console ``` +Or as a configuration file: + ```toml # logwisp.toml [[pipelines]] @@ -132,11 +144,23 @@ target = "stdout" ``` ```bash -logwisp -c logwisp.toml +./bin/lw -c logwisp.toml ``` -Running with no configuration file starts a self-demonstrating pipeline: a -synthetic generator writing JSON to stdout. +Running with neither starts a self-demonstrating pipeline: a synthetic +generator writing JSON to stdout. + +In a container, read-only and without capabilities: + +```bash +make image +docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \ + -v /etc/logwisp:/etc/logwisp:ro logwisp:dev -c /etc/logwisp/logwisp.toml +``` + +`sudo make install` installs the binary, the manual, a sample configuration +and a systemd unit or FreeBSD rc.d script; see +[Installation](doc/installation.md). ## System Requirements @@ -144,7 +168,8 @@ synthetic generator writing JSON to stdout. - **Architecture**: amd64 - **Go**: 1.27.1+ to build from source -Network sources and sinks bind and dial over IPv4 only. +Network sources and sinks use IPv4 or IPv6, following the address family of +the configured host. ## License diff --git a/cmd/logwisp/auth.go b/cmd/lw/auth.go similarity index 90% rename from cmd/logwisp/auth.go rename to cmd/lw/auth.go index f1fdd9f..5125426 100644 --- a/cmd/logwisp/auth.go +++ b/cmd/lw/auth.go @@ -20,6 +20,7 @@ import ( "logwisp/internal/authz" "logwisp/internal/config" + "logwisp/internal/core" "logwisp/internal/tlsx" "github.com/lixenwraith/auth" @@ -27,7 +28,7 @@ import ( const reloadHint = "send SIGHUP to logwisp to apply (auto_reload does not watch the credentials file)" -// authCommand is one `logwisp auth` subcommand: define registers its flags +// authCommand is one `lw auth` subcommand: define registers its flags // and returns the command, which runs once they are parsed and checked. type authCommand struct { name, synopsis, summary string @@ -55,7 +56,7 @@ type usageError string func (e usageError) Error() string { return string(e) } -// runAuth runs `logwisp auth` and returns the exit status: 0 success or +// runAuth runs `lw auth` and returns the exit status: 0 success or // help, 1 failure, 2 usage error. func runAuth(args []string, stdout, stderr io.Writer) int { if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) { @@ -67,12 +68,12 @@ func runAuth(args []string, stdout, stderr io.Writer) int { } i := slices.IndexFunc(authCommands, func(c authCommand) bool { return c.name == args[0] }) if i < 0 { - fmt.Fprintf(stderr, "logwisp auth: unknown command %q\n\n", args[0]) + fmt.Fprintf(stderr, "lw auth: unknown command %q\n\n", args[0]) printAuthUsage(stderr) return 2 } c := authCommands[i] - fs := flag.NewFlagSet("logwisp auth "+c.name, flag.ContinueOnError) + fs := flag.NewFlagSet("lw auth "+c.name, flag.ContinueOnError) fs.SetOutput(stderr) fs.Usage = func() { fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), c.synopsis, c.summary) @@ -101,11 +102,11 @@ func runAuth(args []string, stdout, stderr io.Writer) int { } func printAuthUsage(w io.Writer) { - fmt.Fprint(w, "Usage: logwisp auth [flags]\n\n") + fmt.Fprint(w, "Usage: lw auth [flags]\n\n") for _, c := range authCommands { fmt.Fprintf(w, " %-12s %s\n", c.name, c.summary) } - fmt.Fprint(w, "\nRun logwisp auth -h for its flags. Credential changes apply on SIGHUP.\n"+ + fmt.Fprint(w, "\nRun lw auth -h for its flags. Credential changes apply on SIGHUP.\n"+ "Exit status: 0 success, 1 failure, 2 usage error.\n") } @@ -231,7 +232,7 @@ func defineRemoveUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { } func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { - rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT (with -unbound, a proxy's mount URL may carry a path)") + rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT or https://[IPV6]:PORT (with -unbound, a proxy's mount URL may carry a path)") user := fs.String("user", "", "user `name`") passwordFile := fs.String("password-file", "", "`file` holding the password") unbound := fs.Bool("unbound", false, "log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)") @@ -245,6 +246,10 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { // TLS passthrough cannot route by path: only a terminating proxy can return usageError("-url takes a path only with -unbound") } + network, err := core.Network(u.Hostname()) + if err != nil { + return usageError("-url: " + err.Error()) + } tlsCfg, policy, err := dialPolicy(tlsOpts, u.Hostname(), *user, *passwordFile, authz.HTTP) if err != nil { return err @@ -256,7 +261,7 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { Transport: &http.Transport{ DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { var d net.Dialer - return d.DialContext(ctx, "tcp4", addr) + return d.DialContext(ctx, network, addr) }, TLSClientConfig: tlsCfg, TLSHandshakeTimeout: tlsx.HandshakeTimeout, @@ -264,7 +269,10 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { // A redirect would carry the login to another endpoint CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, } - token, err := policy.Token(context.Background(), client, "https://"+u.Host+u.Path) + // String escapes an IPv6 zone, which u.Host holds raw; the parts drop + // an empty "?" that would swallow the /auth path + base := url.URL{Scheme: u.Scheme, Host: u.Host, Path: u.Path, RawPath: u.RawPath} + token, err := policy.Token(context.Background(), client, base.String()) if err != nil { return err } @@ -274,14 +282,15 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { } func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { - addr := fs.String("addr", "", "tcp sink `address`, HOST:PORT") + addr := fs.String("addr", "", "tcp sink `address`, HOST:PORT or [IPV6]:PORT") user := fs.String("user", "", "user `name`") passwordFile := fs.String("password-file", "", "`file` holding the password") tlsOpts := tlsFlags(fs) return func(stdout, _ io.Writer) error { host, _, err := net.SplitHostPort(*addr) - if err != nil || host == "" { - return usageError("-addr must be HOST:PORT") + network, nerr := core.Network(host) + if err != nil || nerr != nil || host == "" { + return usageError("-addr must be HOST:PORT, with an IPv6 address in brackets") } tlsCfg, policy, err := dialPolicy(tlsOpts, host, *user, *passwordFile, authz.TCP) if err != nil { @@ -290,7 +299,7 @@ func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() dctx, cancel := context.WithTimeout(ctx, tlsx.HandshakeTimeout) - conn, err := (&tls.Dialer{Config: tlsCfg}).DialContext(dctx, "tcp4", *addr) + conn, err := (&tls.Dialer{Config: tlsCfg}).DialContext(dctx, network, *addr) cancel() if err == nil { defer conn.Close() diff --git a/cmd/logwisp/auth_test.go b/cmd/lw/auth_test.go similarity index 87% rename from cmd/logwisp/auth_test.go rename to cmd/lw/auth_test.go index 3f1d336..57e281c 100644 --- a/cmd/logwisp/auth_test.go +++ b/cmd/lw/auth_test.go @@ -190,7 +190,23 @@ func TestRemoveUserRefusesTheLastUser(t *testing.T) { func TestAuthCommandHelpPrintsAuthUsage(t *testing.T) { code, _, stderr := runAuthTest(t, "add-user", "-h") - if code != 0 || !strings.Contains(stderr, "Usage: logwisp auth add-user -credentials FILE") { + if code != 0 || !strings.Contains(stderr, "Usage: lw auth add-user -credentials FILE") { t.Fatalf("exit %d, stderr: %s", code, stderr) } } + +// The login is posted to the -url given plus /auth: a link-local URL keeps +// its zone escaped, not turned into one that no longer parses +func TestTokenLogsInAtTheURLGiven(t *testing.T) { + pass := filepath.Join(t.TempDir(), "edge-01.pass") + if err := os.WriteFile(pass, []byte("correct horse battery\n"), 0o600); err != nil { + t.Fatal(err) + } + // No such interface or listener: the dial fails at once, after the URL is formed + for _, rawURL := range []string{"https://[fe80::1%25nosuchif0]:1", "https://127.0.0.1:1?"} { + code, _, stderr := runAuthTest(t, "token", "-url", rawURL, "-user", "edge-01", "-password-file", pass) + if want := `"` + strings.TrimSuffix(rawURL, "?") + `/auth"`; code != 1 || !strings.Contains(stderr, want) { + t.Errorf("%s: exit %d, stderr: %s", rawURL, code, stderr) + } + } +} diff --git a/cmd/logwisp/bootstrap.go b/cmd/lw/bootstrap.go similarity index 89% rename from cmd/logwisp/bootstrap.go rename to cmd/lw/bootstrap.go index 0812cfe..0a1841b 100644 --- a/cmd/logwisp/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "os" _ "logwisp/internal/source/console" _ "logwisp/internal/source/file" @@ -108,6 +109,22 @@ func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service return svc, nil } +// checkConfig builds every pipeline and plugin as a start would, so TLS files, +// credentials and options are loaded and their warnings print, then exits +// before anything binds or reads: plugins start goroutines only in Start. +func checkConfig(cfg *config.Config) int { + cfg.Logging.Output, cfg.Logging.Level = "stderr", "warn" + if err := initializeLogger(cfg); err == nil && logger.Start() == nil { + defer shutdownLogger() + } + if _, err := service.NewService(context.Background(), cfg, logger); err != nil { + fmt.Fprintf(os.Stderr, "configuration invalid: %v\n", err) + return 1 + } + fmt.Printf("configuration ok: %d pipeline(s)\n", len(cfg.Pipelines)) + return 0 +} + // initializeLogger sets up the global logger based on the application's configuration func initializeLogger(cfg *config.Config) error { logger = log.NewLogger() diff --git a/cmd/logwisp/help.go b/cmd/lw/help.go similarity index 55% rename from cmd/logwisp/help.go rename to cmd/lw/help.go index c8372d8..5e6eb67 100644 --- a/cmd/logwisp/help.go +++ b/cmd/lw/help.go @@ -6,17 +6,18 @@ import ( "os" ) -// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths. +// helpText is the CLI usage reference. Scalar flags map 1:1 to TOML config paths. const helpText = `LogWisp %s - log collection, processing, and distribution Usage: - logwisp [options] - logwisp help | -h | --help - logwisp --version + lw [options] + lw --check [options] Build every pipeline and plugin, report, exit + lw help | -h | --help + lw --version Subcommands: - logwisp auth SCRAM credentials files, bearer tokens and tcp - sink viewing; logwisp auth -h lists commands + lw auth SCRAM credentials files, bearer tokens and tcp + sink viewing; lw auth -h lists commands Scalar configuration keys are settable as flags using their TOML paths: --= e.g. --logging.level=debug @@ -32,14 +33,26 @@ Logging: --logging.level= debug|info|warn|error --logging.file.directory= -Pipelines, plugins and filters are configured in TOML; array-indexed CLI and -environment overrides are unsupported. -- ends option parsing. +Pipelines (replace the file's pipelines; see doc/cli.md): + --pipeline Start a pipeline; earlier specs go to "cli" + --source TYPE[,key=value...], repeatable + --sink e.g. http,host=0.0.0.0,port=8080, repeatable + --filter include|exclude,patterns=RE, repeatable + --format json|txt|raw[,key=value...] + --rate-limit rate=N[,burst=N,policy=drop|pass] + --heartbeat interval_ms=N[,include_stats=true,...] + Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and + '\' escapes ',' '=' '\' in values. -- ends option parsing. Environment: LOGWISP_ Config path, '.' -> '_', uppercase e.g. LOGWISP_LOGGING_LEVEL=debug LOGWISP_CONFIG_FILE Configuration file path LOGWISP_CONFIG_DIR Configuration directory + LOGWISP_PIPELINE, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N], LOGWISP_FILTER[_N], + LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT + One pipeline, specs as the flags (_N adds + more); ignored when a pipeline flag is given Signals: SIGINT, SIGTERM Graceful shutdown diff --git a/cmd/logwisp/main.go b/cmd/lw/main.go similarity index 93% rename from cmd/logwisp/main.go rename to cmd/lw/main.go index 8db6571..694158a 100644 --- a/cmd/logwisp/main.go +++ b/cmd/lw/main.go @@ -19,7 +19,7 @@ import ( var logger *log.Logger func main() { - // Before handleHelp, so `logwisp auth -h` prints the auth usage + // Before handleHelp, so `lw auth -h` prints the auth usage if len(os.Args) > 1 && os.Args[1] == "auth" { os.Exit(runAuth(os.Args[2:], os.Stdout, os.Stderr)) } @@ -53,6 +53,9 @@ func main() { fmt.Println(version.String()) os.Exit(0) } + if cfg.Check { + os.Exit(checkConfig(cfg)) + } if err := initializeLogger(cfg); err != nil { FatalError(1, "Failed to initialize logger: %v\n", err) @@ -63,9 +66,13 @@ func main() { FatalError(1, "Failed to start logger: %v\n", err) } + configFile := cfg.ConfigFile + if _, err := os.Stat(configFile); err != nil { + configFile = "none" // flags, environment and built-in defaults only + } logger.Info("msg", "LogWisp starting", "version", version.String(), - "config_file", cfg.ConfigFile, + "config_file", configFile, "log_output", cfg.Logging.Output, "status_reporter", cfg.StatusReporter, "auto_reload", cfg.ConfigAutoReload) diff --git a/cmd/logwisp/output.go b/cmd/lw/output.go similarity index 99% rename from cmd/logwisp/output.go rename to cmd/lw/output.go index 0005fdb..f93bf36 100644 --- a/cmd/logwisp/output.go +++ b/cmd/lw/output.go @@ -90,4 +90,4 @@ func (o *OutputHandler) SetQuiet(quiet bool) { o.mu.Lock() defer o.mu.Unlock() o.quiet = quiet -} \ No newline at end of file +} diff --git a/cmd/logwisp/reload.go b/cmd/lw/reload.go similarity index 100% rename from cmd/logwisp/reload.go rename to cmd/lw/reload.go diff --git a/cmd/logwisp/reload_test.go b/cmd/lw/reload_test.go similarity index 67% rename from cmd/logwisp/reload_test.go rename to cmd/lw/reload_test.go index 467e512..bf86dec 100644 --- a/cmd/logwisp/reload_test.go +++ b/cmd/lw/reload_test.go @@ -2,6 +2,10 @@ package main import ( "context" + "errors" + "io/fs" + "net" + "os" "path/filepath" "strings" "testing" @@ -106,3 +110,43 @@ func TestEveryPluginRejectsUnknownKeys(t *testing.T) { check("sink", name, err) } } + +// A misspelled spec key reaches the plugin as the nested key it names, so the +// sink refuses to build instead of starting without the TLS it was asked for. +func TestMisspelledSpecKeyFailsPluginConstruction(t *testing.T) { + testLogger(t) + path := filepath.Join(t.TempDir(), "empty.toml") + testutil.WriteFile(t, path, "") + cfg := loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15861,tls.enabeld=true") + svc, err := bootstrapService(context.Background(), cfg) + if err == nil { + svc.Shutdown() + } + if err == nil || !strings.Contains(err.Error(), `unknown key "tls.enabeld"`) { + t.Fatalf("misspelled spec key accepted: %v", err) + } +} + +// lw --check builds every plugin, so a bad option fails it, but starts none: +// the port a valid sink names stays free and its log directory uncreated. +func TestCheckBuildsWithoutStarting(t *testing.T) { + testLogger(t) + dir := t.TempDir() + path, out := filepath.Join(dir, "empty.toml"), filepath.Join(dir, "out") + testutil.WriteFile(t, path, "") + if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15862", + "--sink", "file,name=check,directory="+out)); code != 0 { + t.Fatalf("valid configuration: exit %d", code) + } + ln, err := net.Listen("tcp4", "127.0.0.1:15862") + if err != nil { + t.Fatalf("check bound the sink's port: %v", err) + } + ln.Close() + if _, err := os.Stat(out); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("check created the file sink's directory: %v", err) + } + if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,port=15862,tls.enabled=true")); code != 1 { + t.Fatalf("listener TLS without a certificate: exit %d, want 1", code) + } +} diff --git a/cmd/logwisp/status.go b/cmd/lw/status.go similarity index 100% rename from cmd/logwisp/status.go rename to cmd/lw/status.go diff --git a/cmd/logwisp/status_test.go b/cmd/lw/status_test.go similarity index 100% rename from cmd/logwisp/status_test.go rename to cmd/lw/status_test.go diff --git a/cmd/logwisp/test_helpers_test.go b/cmd/lw/test_helpers_test.go similarity index 71% rename from cmd/logwisp/test_helpers_test.go rename to cmd/lw/test_helpers_test.go index 3b91c88..a271ec0 100644 --- a/cmd/logwisp/test_helpers_test.go +++ b/cmd/lw/test_helpers_test.go @@ -16,10 +16,10 @@ func testLogger(t *testing.T) { t.Cleanup(func() { logger = previous }) } -func loadTestConfig(t *testing.T, path string) *config.Config { +func loadTestConfig(t *testing.T, path string, args ...string) *config.Config { t.Helper() testutil.ClearEnvPrefix(t, "LOGWISP_") - m, err := config.Load([]string{"-c", path, "--quiet", "--status_reporter=false"}) + m, err := config.Load(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...)) if err != nil { t.Fatal(err) } diff --git a/config/logwisp.toml b/config/logwisp.toml index ef9d0fe..414a6f2 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -10,7 +10,7 @@ ### http_client/tcp_client sinks are not available in this version. TLS is ### available on every network source and sink (the [...tls] blocks below), and ### the [...auth] blocks authenticate peers by certificate (mtls) or password -### (scram; credentials via `logwisp auth`). Unknown keys anywhere in this file +### (scram; credentials via `lw auth`). Unknown keys anywhere in this file ### or in any plugin config are rejected at startup. ### ### Environment overrides use the LOGWISP_ prefix @@ -87,7 +87,7 @@ name = "default" # interval_ms = 1000 # Minimum 100 # include_timestamp = false # include_stats = false -# format = "txt" # txt|json|raw ("comment" is rejected) +# format = "txt" # accepted, no effect: heartbeats use flow.format ###============================================================================ ### TLS (shared shape; applies to every network source and sink) @@ -119,7 +119,7 @@ name = "default" ### On the http sink the block gates stream_path AND status_path. ### Listeners (tcp/http sinks, tcp_chain/http_chain sources): ### mtls authorizes the client certificate; REQUIRES tls.client_auth. -### scram admits the users in credentials_file (`logwisp auth add-user`). +### scram admits the users in credentials_file (`lw auth add-user`). ### Dialers (tcp_chain/http_chain sinks): ### mtls pins the server identity. ### scram logs in as username with the password in password_file. @@ -192,7 +192,7 @@ from = "end" # "end" or "start" of a newly discovered # id = "chain_in" # type = "tcp_chain" # [pipelines.plugin_sources.config] -# host = "0.0.0.0" # IPv4 only +# host = "0.0.0.0" # IPv4 only; "::" or "::1": IPv6 only # port = 9440 # Required # buffer_size = 1000 # max_connections = 0 # 0 = unlimited @@ -212,14 +212,14 @@ from = "end" # "end" or "start" of a newly discovered # allow = [] # mtls: exact identities; empty = any the CA issued # allow_patterns = [] # mtls: RE2, anchor them yourself # node_binding = "force" # none | assert | force; overrides trust_node -# credentials_file = "" # scram: written by `logwisp auth add-user` +# credentials_file = "" # scram: written by `lw auth add-user` ## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks) # [[pipelines.plugin_sources]] # id = "hchain_in" # type = "http_chain" # [pipelines.plugin_sources.config] -# host = "0.0.0.0" +# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only) # port = 9441 # Required # ingest_path = "/ingest" # Must start with "/" # buffer_size = 1000 @@ -238,7 +238,7 @@ from = "end" # "end" or "start" of a newly discovered # allow = [] # mtls: exact identities; empty = any the CA issued # allow_patterns = [] # mtls: RE2, anchor them yourself # node_binding = "force" # none | assert | force; overrides trust_node -# credentials_file = "" # scram: written by `logwisp auth add-user` +# credentials_file = "" # scram: written by `lw auth add-user` # token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes ###============================================================================ @@ -271,14 +271,14 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # buffer_size = 1000 # flush_interval_ms = 100 -## HTTP sink (SSE streaming server + JSON status endpoint; IPv4 clients only) +## HTTP sink (SSE streaming server + JSON status endpoint) ## Without an auth block both endpoints are UNAUTHENTICATED and the stream sends ## Access-Control-Allow-Origin: *. Bind to a trusted interface. # [[pipelines.plugin_sinks]] # id = "http_out" # type = "http" # [pipelines.plugin_sinks.config] -# host = "0.0.0.0" +# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only) # port = 8081 # Required # stream_path = "/stream" # Must start with "/" # status_path = "/status" # Must differ from stream_path @@ -299,16 +299,16 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # identity = "cn" # mtls default; under scram unset, set binds the cert to the user # allow = [] # mtls: exact identities; empty = any the CA issued # allow_patterns = [] # mtls: RE2, anchor them yourself -# credentials_file = "" # scram: log in with `logwisp auth token` +# credentials_file = "" # scram: log in with `lw auth token` # token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes # trusted_proxies = [] # scram: proxies ending the browsers' TLS; tls may then be off -## TCP sink (streaming server, IPv4 clients only) +## TCP sink (streaming server) # [[pipelines.plugin_sinks]] # id = "tcp_out" # type = "tcp" # [pipelines.plugin_sinks.config] -# host = "0.0.0.0" +# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only) # port = 9090 # Required # buffer_size = 1000 # client_buffer_size = 256 @@ -327,7 +327,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # identity = "cn" # mtls default; under scram unset, set binds the cert to the user # allow = [] # mtls: exact identities; empty = any the CA issued # allow_patterns = [] # mtls: RE2, anchor them yourself -# credentials_file = "" # scram: readers use `logwisp auth stream` +# credentials_file = "" # scram: readers use `lw auth stream` ## TCP chain sink (stdlib client; forwards to downstream tcp_chain source) ## Do NOT point a chain sink at a chain source in the SAME pipeline: entries @@ -336,7 +336,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # id = "chain_out" # type = "tcp_chain" # [pipelines.plugin_sinks.config] -# host = "10.0.0.1" # Required +# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1" # port = 9440 # Required # node = "" # origin label, default hostname; preserved across hops # buffer_size = 1000 @@ -367,7 +367,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # id = "hchain_out" # type = "http_chain" # [pipelines.plugin_sinks.config] -# host = "10.0.0.1" # Required +# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1" # port = 9441 # Required # ingest_path = "/ingest" # node = "" # origin label, default hostname; preserved across hops diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..28ea759 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,31 @@ +# deploy + +`lw-deploy.sh` writes a LogWisp configuration for one node and deploys it: in a +docker or podman container, as a systemd or FreeBSD rc.d service (on the host +or inside a jail), or as a configuration you run yourself. It is POSIX sh, +written for FreeBSD's `/bin/sh` as much as for dash. + +```sh +deploy/lw-deploy.sh # asks, with inline menus +deploy/lw-deploy.sh -h # every flag +deploy/lw-deploy.sh --dry-run ... # print the configuration and commands only +deploy/lw-deploy.sh --yes ... # no questions: defaults, or fail on a missing value +``` + +- Roles + - `edge`: tails log directories, forwards the entries to an aggregator + - `aggregator`: receives from edges; writes files, serves HTTP and TCP streams + - `standalone`: tails log directories; writes files, serves streams +- Runtimes + - `docker`, `podman`: a hardened container of the `logwisp` image (Linux) + - `native`: systemd on Linux, rc.d on FreeBSD, `--jail NAME` for a jail + - `manual`: the configuration only, for `lw -c` +- Every question has a flag. Before changing anything the script prints its + plan and the command line that repeats it without questions, ready for the + next node. It starts nothing that `lw --check` rejects. +- The native runtimes install the service files of `deploy/package/`. +- Not covered, and printed by the script: filters, rate limits, heartbeat, + several pipelines, proxy mode, ACLs, certificate creation. Edit the generated + `logwisp.toml` for those. + +Scenarios and what each runtime writes: [doc/deployment.md](../doc/deployment.md). diff --git a/deploy/lw-deploy.sh b/deploy/lw-deploy.sh new file mode 100755 index 0000000..5b5138f --- /dev/null +++ b/deploy/lw-deploy.sh @@ -0,0 +1,1331 @@ +#!/bin/sh +# Deploys one LogWisp node: an edge (log files -> chain), an aggregator (chain +# -> files, streams) or a standalone node (log files -> files, streams), as a +# docker or podman container, a systemd or FreeBSD rc.d service, or a plain +# configuration. Asks for what the flags leave open; -h lists the flags and +# doc/deployment.md walks through the scenarios. + +set -u +set -f # globs such as *.log are values here, never file names +umask 022 + +PROG=${0##*/} +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P) || exit 1 +REPO=${SCRIPT_DIR%/*} +PACKAGE_DIR=$SCRIPT_DIR/package +NL=' +' +OIFS=$IFS +IMAGE_USER=65532:65532 +PKG_UNIT=/usr/lib/systemd/system/logwisp.service # a package's, used as it is +HOME_DIR=${HOME:-$PWD} # keeps keys and passwords out of a checkout run from its root + +ROLE='' RUNTIME='' ENGINE='' TARGET_OS='' NAME='' CONF_DIR='' JAIL='' +LOG_DIRS='' LOG_PATTERN='' FROM='' LOG_GROUP='' +CHAIN='' AGGREGATOR='' CHAIN_PORT='' NODE='' LISTEN='' +FILE_DIR='' FILE_NAME='' HTTP_PORT='' TCP_PORT='' FORMAT='' +TLS='' CERT_FILE='' KEY_FILE='' CA_FILE='' SERVER_NAME='' +AUTH='' ALLOW='' USERNAME='' PASSWORD_FILE='' ADD_USERS='' +SINK_AUTH='' SINK_ALLOW='' ADD_VIEWERS='' SECRETS_DIR='' +IMAGE='' BUILD='' BUILD_CA='' BUILD_FLAGS='' NETWORK='' BIN='' +START='' FORCE='' YES=0 DRY_RUN=0 + +usage() { + cat <&2; exit 1; } +warn() { printf '%s: warning: %s\n' "$PROG" "$*" >&2; } +note() { printf '%s\n' "$*"; } + +add_item() { if [ -z "$1" ]; then printf '%s' "$2"; else printf '%s\n%s' "$1" "$2"; fi; } + +shquote() { + case $1 in + '') printf "''" ;; + *[!A-Za-z0-9_./:=,@%+-]*) printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" ;; + *) printf '%s' "$1" ;; + esac +} + +tq() { printf '"%s"' "$(printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g')"; } + +tlist() { # newline-separated list -> TOML array + _tl='' + IFS=$NL + for _i in $1; do _tl="$_tl${_tl:+, }$(tq "$_i")"; done + IFS=$OIFS + printf '[%s]' "$_tl" +} + +urlhost() { case $1 in *:*) printf '[%s]' "$1" ;; *) printf '%s' "$1" ;; esac; } +# in a URL a zone's % is escaped as %25 (RFC 6874) +urihost() { case $1 in *%*) urlhost "${1%%\%*}%25${1#*%}" ;; *) urlhost "$1" ;; esac; } + +# --- commands: printed, and run unless --dry-run --- + +show_cmd() { # a long command breaks before each option up to its first operand + printf '+' + _brk='' _n=0 _prev=- + [ $# -le 12 ] || _brk=yes + for _a in "$@"; do + _n=$((_n + 1)) + _b=$_brk + case $_a in -*) ;; *) case $_prev in -*) _b='' ;; *) [ "$_n" -le 2 ] || _brk='' ;; esac ;; esac + [ "$_b" != yes ] || [ "$_n" -le 2 ] || printf ' \\\n ' + printf ' %s' "$(shquote "$_a")" + _prev=$_a + done +} + +run() { + show_cmd "$@" + printf '\n' + [ "$DRY_RUN" = 1 ] || "$@" +} + +run_in() { # FILE COMMAND...: COMMAND reads FILE on stdin + _in=$1 + shift + show_cmd "$@" + printf ' < %s\n' "$(shquote "$_in")" + [ "$DRY_RUN" = 1 ] || "$@" <"$_in" +} + +# trun runs inside the target jail when there is one; tprobe also, unprinted. +trun() { if [ -n "$JAIL" ]; then run jexec "$JAIL" "$@"; else run "$@"; fi; } +tprobe() { if [ -n "$JAIL" ]; then jexec "$JAIL" "$@"; else "$@"; fi; } + +# put_file writes stdin, or SOURCE, to PATH in the target: a jail's own +# symlinks then resolve inside the jail, never on the host. +put_file() { # PATH MODE [SOURCE] + if [ "$DRY_RUN" = 1 ]; then + if [ -n "${3:-}" ]; then + printf '+ copy %s to %s (mode %s)%s\n' "$(shquote "$3")" "$(shquote "$1")" "$2" "${JAIL:+ in jail $JAIL}" + else + printf '+ write %s (mode %s)%s:\n' "$(shquote "$1")" "$2" "${JAIL:+ in jail $JAIL}" + sed 's/^/ | /' + fi + return 0 + fi + [ -z "${3:-}" ] || { put_file "$1" "$2" <"$3"; return; } + # shellcheck disable=SC2016 # expanded by the inner shell + tprobe /bin/sh -c 'umask 077 && t=$1.lw-deploy.$$ && + { cat >"$t" && chmod "$2" "$t" && mv -f "$t" "$1" || { rm -f "$t"; exit 1; }; }' put_file "$1" "$2" || + die "cannot write $1${JAIL:+ in jail $JAIL}" + note "wrote $1${JAIL:+ in jail $JAIL}" +} + +# --- prompts: a value given as a flag is validated, never asked again --- + +prompt_line() { # PROMPT DEFAULT -> REPLY + if [ -n "$2" ]; then printf '%s [%s]: ' "$1" "$2" >&2; else printf '%s: ' "$1" >&2; fi + IFS= read -r REPLY || die "input ended" +} + +ask() { # VAR FLAG PROMPT DEFAULT [VALIDATOR] + _var=$1 _flag=$2 _prompt=$3 _def=$4 _check=${5:-} + eval "_val=\$$_var" + if [ -z "$_val" ] && [ "$INTERACTIVE" = 0 ]; then + [ -n "$_def" ] || die "missing value: $_flag" + _val=$_def + fi + while [ -z "$_val" ] || { [ -n "$_check" ] && ! "$_check" "$_val"; }; do + [ "$INTERACTIVE" = 1 ] || die "invalid $_flag: $_val" + prompt_line "$_prompt" "$_def" + _val=${REPLY:-$_def} + done + eval "$_var=\$_val" +} + +ask_opt() { # VAR FLAG PROMPT [VALIDATOR]: empty is an answer + _var=$1 _flag=$2 _prompt=$3 _check=${4:-} + eval "_val=\$$_var" + if [ -z "$_val" ] && [ "$INTERACTIVE" = 1 ]; then + prompt_line "$_prompt" '' + _val=$REPLY + fi + while [ -n "$_val" ] && [ -n "$_check" ] && ! "$_check" "$_val"; do + [ "$INTERACTIVE" = 1 ] || die "invalid $_flag: $_val" + prompt_line "$_prompt" '' + _val=$REPLY + done + eval "$_var=\$_val" +} + +ask_list() { # VAR FLAG PROMPT VALIDATOR REQUIRED(yes|no); answers split on spaces + _var=$1 _flag=$2 _prompt=$3 _check=$4 _req=$5 + eval "_val=\$$_var" + while :; do + if [ -z "$_val" ] && [ "$INTERACTIVE" = 1 ]; then + prompt_line "$_prompt" '' + _val=$(printf '%s\n' "$REPLY" | tr '\t' ' ' | tr -s ' ' '\n' | sed '/^$/d') + fi + if [ -z "$_val" ] && [ "$_req" = yes ]; then + [ "$INTERACTIVE" = 1 ] || die "missing value: $_flag" + continue + fi + _bad=0 + IFS=$NL + for _i in $_val; do "$_check" "$_i" || _bad=1; done + IFS=$OIFS + [ "$_bad" = 0 ] && break + [ "$INTERACTIVE" = 1 ] || die "invalid $_flag" + _val='' + done + eval "$_var=\$_val" +} + +choose() { # VAR FLAG PROMPT DEFAULT VALUE DESCRIPTION [VALUE DESCRIPTION]... + _var=$1 _flag=$2 _prompt=$3 _def=$4 + shift 4 + eval "_val=\$$_var" + _vals='' _menu='' _n=0 _defn='' + while [ $# -ge 2 ]; do + _n=$((_n + 1)) + _vals=$_vals$1$NL + _menu=$_menu$(printf ' %d) %-11s %s' "$_n" "$1" "$2")$NL + [ "$1" = "$_def" ] && _defn=$_n + shift 2 + done + while :; do + if [ -n "$_val" ]; then + case $NL$_vals in *"$NL$_val$NL"*) break ;; esac + [ "$INTERACTIVE" = 1 ] || die "invalid $_flag: $_val (one of: $(printf '%s' "$_vals" | tr '\n' ' '))" + printf ' not one of the choices: %s\n' "$_val" >&2 + elif [ "$INTERACTIVE" = 0 ]; then + [ -n "$_def" ] || die "missing value: $_flag" + _val=$_def + continue + fi + printf '%s\n%s' "$_prompt" "$_menu" >&2 + prompt_line ' choice' "$_defn" + _val=${REPLY:-$_defn} + case $_val in + [1-9] | [1-9][0-9]) [ "$_val" -le "$_n" ] && _val=$(printf '%s' "$_vals" | sed -n "${_val}p") ;; + esac + done + eval "$_var=\$_val" +} + +ask_yn() { # VAR PROMPT DEFAULT(yes|no) + _var=$1 _prompt=$2 _def=$3 + eval "_val=\$$_var" + if [ -z "$_val" ] && [ "$INTERACTIVE" = 1 ]; then + while :; do + if [ "$_def" = yes ]; then prompt_line "$_prompt [Y/n]" ''; else prompt_line "$_prompt [y/N]" ''; fi + case $REPLY in + '') _val=$_def ;; + [Yy] | [Yy][Ee][Ss]) _val=yes ;; + [Nn] | [Nn][Oo]) _val=no ;; + *) continue ;; + esac + break + done + fi + eval "$_var=\${_val:-\$_def}" +} + +# With NEEDS, an auth flag: --yes stops, as dropping it would leave the node open. +drop() { # VAR FLAG [NEEDS] + eval "_val=\$$1" + [ -z "$_val" ] || [ -z "${3:-}" ] || [ "$INTERACTIVE" = 1 ] || die "$2 needs $3" + [ -z "$_val" ] || warn "$2 does not apply to these choices; ignored" + eval "$1=''" +} + +# --- validators: print why and fail; free text rejects control characters, which +# would break out of a TOML string --- + +bad() { printf ' %s\n' "$*" >&2; return 1; } +v_port() { + case $1 in '' | *[!0-9]* | 0* | ??????*) ;; *) [ "$1" -le 65535 ] && return 0 ;; esac + bad "not a port (1-65535): $1" +} +v_abs() { + case $1 in *:* | *[[:cntrl:]]*) bad "':' (a mount separator) or a control character in a path: $1" ;; /*) return 0 ;; *) bad "not an absolute path: $1" ;; esac +} +v_ident() { case $1 in '' | *[!A-Za-z0-9._@-]*) bad "use letters, digits and . _ @ -: $1" ;; esac; } +v_cname() { case $1 in [A-Za-z0-9]*) case $1 in *[!A-Za-z0-9_.-]*) ;; *) return 0 ;; esac ;; esac; bad "not a container name: $1"; } +v_host() { case $1 in '' | *[!A-Za-z0-9.:%_-]*) bad "not a host name or address: $1" ;; esac; } +v_glob() { case $1 in */* | *[[:cntrl:]]* | '') bad "a file name pattern, without '/' or control characters: $1" ;; esac; } +v_word() { case $1 in '' | *[!A-Za-z0-9_./:@-]*) bad "unexpected characters: $1" ;; esac; } +v_any() { case $1 in '' | *[[:cntrl:]]*) bad "an empty value or a control character: $1" ;; esac; } +v_file() { + [ -f "$1" ] && [ -r "$1" ] && return 0 + [ "$DRY_RUN" = 1 ] && { warn "not a readable file here: $1"; return 0; } + bad "not a readable file: $1" +} +v_dir() { # a directory the service reads or writes + v_abs "$1" || return 1 + [ "$RUNTIME/$TARGET_OS" = native/linux ] || return 0 + case $1/ in /tmp/* | /var/tmp/*) bad "the systemd unit gives lw its own /tmp and /var/tmp (PrivateTmp=yes): $1" ;; esac +} +v_logdir() { + v_dir "$1" || return 1 + [ -d "$ROOT$1" ] && return 0 + [ "$DRY_RUN" = 1 ] && { warn "no such directory here: $ROOT$1"; return 0; } + bad "no such directory: $ROOT$1" +} + +# --- arguments --- + +need_arg() { [ $# -ge 2 ] || die "$1 needs a value"; } + +while [ $# -gt 0 ]; do + case $1 in + --*=*) + _o=${1%%=*} _v=${1#*=} + shift + set -- "$_o" "$_v" "$@" + ;; + esac + case $1 in + -h | --help) usage; exit 0 ;; + -y | --yes) YES=1 ;; + -n | --dry-run) DRY_RUN=1 ;; + --force) FORCE=yes ;; + --start) START=yes ;; + --no-start) START=no ;; + --tls) TLS=yes ;; + --no-tls) TLS=no ;; + --build) BUILD=yes ;; + --no-build) BUILD=no ;; + --log-dir) need_arg "$@"; LOG_DIRS=$(add_item "$LOG_DIRS" "$2"); shift ;; + --allow) need_arg "$@"; ALLOW=$(add_item "$ALLOW" "$2"); shift ;; + --add-user) need_arg "$@"; ADD_USERS=$(add_item "$ADD_USERS" "$2"); shift ;; + --sink-allow) need_arg "$@"; SINK_ALLOW=$(add_item "$SINK_ALLOW" "$2"); shift ;; + --add-viewer) need_arg "$@"; ADD_VIEWERS=$(add_item "$ADD_VIEWERS" "$2"); shift ;; + --*) + case $1 in + --role) _var=ROLE ;; --runtime) _var=RUNTIME ;; --engine) _var=ENGINE ;; + --os) _var=TARGET_OS ;; --name) _var=NAME ;; --config-dir) _var=CONF_DIR ;; + --jail) _var=JAIL ;; --log-pattern) _var=LOG_PATTERN ;; --from) _var=FROM ;; + --log-group) _var=LOG_GROUP ;; --chain) _var=CHAIN ;; --aggregator) _var=AGGREGATOR ;; + --chain-port) _var=CHAIN_PORT ;; --node) _var=NODE ;; --listen) _var=LISTEN ;; + --file-dir) _var=FILE_DIR ;; --file-name) _var=FILE_NAME ;; --http-port) _var=HTTP_PORT ;; + --tcp-port) _var=TCP_PORT ;; --format) _var=FORMAT ;; --cert-file) _var=CERT_FILE ;; + --key-file) _var=KEY_FILE ;; --ca-file) _var=CA_FILE ;; --server-name) _var=SERVER_NAME ;; + --auth) _var=AUTH ;; --username) _var=USERNAME ;; --password-file) _var=PASSWORD_FILE ;; + --sink-auth) _var=SINK_AUTH ;; --secrets-dir) _var=SECRETS_DIR ;; --image) _var=IMAGE ;; + --build-ca) _var=BUILD_CA ;; --build-flags) _var=BUILD_FLAGS ;; --network) _var=NETWORK ;; + --bin) _var=BIN ;; --package-dir) _var=PACKAGE_DIR ;; + *) die "unknown option: $1 (see $PROG -h)" ;; + esac + need_arg "$@" + [ -n "$2" ] || die "$1 needs a value" + eval "$_var=\$2" + shift + ;; + *) die "unexpected argument: $1 (see $PROG -h)" ;; + esac + shift +done + +if [ "$YES" = 1 ]; then + INTERACTIVE=0 +elif [ -t 0 ]; then + INTERACTIVE=1 +elif [ "$DRY_RUN" = 1 ]; then + INTERACTIVE=0 +else + die "stdin is not a terminal: pass --yes and the values as flags" +fi + +# --- questions --- + +host_os() { + case $(uname -s) in + Linux) echo linux ;; + FreeBSD) echo freebsd ;; + *) echo unknown ;; + esac +} + +short_host() { _h=$(uname -n); printf '%s' "${_h%%.*}"; } + +freebsd_no_containers() { + cat >&2 <<'EOF' +FreeBSD has no Docker. Podman on FreeBSD runs FreeBSD OCI images only, and the +LogWisp image is a Linux image, so containers are not supported there. Run +LogWisp natively instead: on the host, or inside a jail with --jail NAME. +EOF +} + +jail_guide() { + cat >&2 </dev/null) || ROOT='' + if [ -z "$ROOT" ]; then + jail_guide + [ "$DRY_RUN" = 1 ] || exit 1 + ROOT=/usr/local/jails/$JAIL + warn "showing paths under $ROOT" + fi + fi + CONF_LW=/usr/local/etc/logwisp + else + CONF_LW=/etc/logwisp + fi + CONF_HOST=$ROOT$CONF_LW + ;; + manual) + ask CONF_DIR --config-dir "Directory for the configuration" "$HOME_DIR/logwisp-$ROLE" v_abs + CONF_HOST=$CONF_DIR CONF_LW=$CONF_DIR + ;; + esac + + if [ "$ROLE" != aggregator ]; then + if [ -z "$LOG_DIRS" ] && [ "$INTERACTIVE" = 1 ]; then + note "Log directories to tail, one per line; an empty line ends the list." >&2 + while :; do + prompt_line " directory" '' + [ -n "$REPLY" ] || { [ -n "$LOG_DIRS" ] && break; continue; } + v_logdir "$REPLY" && LOG_DIRS=$(add_item "$LOG_DIRS" "$REPLY") + done + fi + ask_list LOG_DIRS --log-dir "Log directories, space-separated" v_logdir yes + ask LOG_PATTERN --log-pattern "File name pattern in each directory" '*.log' v_glob + FROM=${FROM:-end} + case $FROM in start | end) ;; *) die "invalid --from: $FROM (start|end)" ;; esac + [ "$RUNTIME" = manual ] || ask_opt LOG_GROUP --log-group "Group that may read these logs, for the service (empty: none)" v_ident + fi + + if [ "$ROLE" != standalone ]; then + choose CHAIN --chain "Chain transport:" tcp \ + tcp "tcp_chain: one long-lived connection" \ + http "http_chain: batched HTTP POSTs" + _dv=9440 + [ "$CHAIN" = http ] && _dv=9441 + if [ "$ROLE" = edge ]; then + ask AGGREGATOR --aggregator "Aggregator host or address" '' v_host + ask CHAIN_PORT --chain-port "Aggregator chain port" "$_dv" v_port + _dv=$(short_host) + [ -z "$JAIL" ] || _dv=$JAIL + ask NODE --node "Node label for this edge's entries" "$_dv" v_ident + else + ask CHAIN_PORT --chain-port "Chain listener port" "$_dv" v_port + fi + fi + + if [ "$ROLE" != edge ]; then + ask LISTEN --listen "Listen address" 0.0.0.0 v_host + if [ -z "$FILE_DIR$HTTP_PORT$TCP_PORT" ]; then + [ "$INTERACTIVE" = 1 ] || die "no output: give --file-dir, --http-port or --tcp-port" + _want='' _dv=/var/log/logwisp + [ "$RUNTIME" = docker ] && _dv=/var/log/$NAME + [ "$RUNTIME" = manual ] && _dv=$CONF_DIR/out + ask_yn _want "Write the entries to rotating files?" yes + [ "$_want" = no ] || ask FILE_DIR --file-dir "Output directory" "$_dv" v_dir + _want='' + ask_yn _want "Serve a live HTTP stream (SSE) with a status endpoint?" no + [ "$_want" = no ] || ask HTTP_PORT --http-port "HTTP port" 8080 v_port + _want='' + ask_yn _want "Serve a plain TCP stream?" no + [ "$_want" = no ] || ask TCP_PORT --tcp-port "TCP port" 9090 v_port + [ -n "$FILE_DIR$HTTP_PORT$TCP_PORT" ] || die "no output chosen" + fi + [ -z "$FILE_DIR" ] || ask FILE_DIR --file-dir "Output directory" '' v_dir + [ -z "$FILE_DIR" ] || ask FILE_NAME --file-name "Output file name stem" logwisp v_ident + [ -z "$HTTP_PORT" ] || ask HTTP_PORT --http-port "HTTP port" '' v_port + [ -z "$TCP_PORT" ] || ask TCP_PORT --tcp-port "TCP port" '' v_port + choose FORMAT --format "Entry format:" txt \ + txt "timestamp, level, node/source, message" \ + json "one JSON object per entry" \ + raw "the message only" + _dup=$(printf '%s\n' "$CHAIN_PORT" "$HTTP_PORT" "$TCP_PORT" | sed '/^$/d' | sort | uniq -d) + [ -z "$_dup" ] || die "two listeners on port $_dup" + fi + + # A standalone node without network outputs has no link to secure. + if [ "$ROLE" = standalone ] && [ -z "$HTTP_PORT$TCP_PORT" ]; then + [ "$TLS" != yes ] || warn "ignoring --tls: no http or tcp output" + TLS=no + fi + _dv=no + [ -z "$CERT_FILE$CA_FILE" ] || _dv=yes + ask_yn TLS "Use TLS on this node's network links?" "$_dv" + # Auth-only flags imply their mode, the default --yes takes. + _ia=none _isa=none + [ -z "$ALLOW" ] || _ia=mtls + [ -z "$ADD_USERS$USERNAME$PASSWORD_FILE" ] || _ia=scram + [ -z "$SINK_ALLOW" ] || _isa=mtls + [ -z "$ADD_VIEWERS" ] || _isa=scram + if [ "$TLS" = yes ]; then + [ "$INTERACTIVE" = 0 ] || note "Certificates come from openssl or your PKI: doc/security.md#enabling-mtls" >&2 + if [ "$ROLE" = edge ]; then + ask_opt CA_FILE --ca-file "CA bundle verifying the aggregator (empty: the system roots)" v_file + else + ask CERT_FILE --cert-file "This node's certificate (PEM)" '' v_file + ask KEY_FILE --key-file "Its private key (PEM)" '' v_file + fi + [ "$ROLE" = standalone ] || choose AUTH --auth "Authenticate the chain link:" "$_ia" \ + none "TLS only" \ + mtls "client certificates, admitted by CN" \ + scram "username and password" + else + [ -z "$CERT_FILE$KEY_FILE$CA_FILE$SERVER_NAME" ] || die "certificate options need --tls" + [ -z "$AUTH" ] || [ "$AUTH" = none ] || die "--auth $AUTH needs --tls" + [ -z "$SINK_AUTH" ] || [ "$SINK_AUTH" = none ] || die "--sink-auth $SINK_AUTH needs --tls" + fi + [ "$ROLE" != standalone ] || drop AUTH --auth + AUTH=${AUTH:-none} + + case $ROLE/$AUTH in + edge/mtls) + ask CERT_FILE --cert-file "This edge's client certificate (PEM)" '' v_file + ask KEY_FILE --key-file "Its private key (PEM)" '' v_file + ask_list ALLOW --allow "Aggregator certificate CN to pin (empty: any the CA issued)" v_any no + ;; + edge/scram) + ask PASSWORD_FILE --password-file "Password file (from the aggregator's secrets directory)" '' v_file + # The aggregator names each generated password file after its user. + _dv=${PASSWORD_FILE##*/} + _dv=${_dv%.pass} + case $_dv in '' | *[!A-Za-z0-9._@-]*) _dv=$NODE ;; esac + ask USERNAME --username "SCRAM username" "$_dv" v_ident + ;; + aggregator/mtls) + ask CA_FILE --ca-file "CA bundle verifying the edges' certificates" '' v_file + ask_list ALLOW --allow "Edge certificate CNs to admit, space-separated (empty: any the CA issued)" v_any no + ;; + aggregator/scram) + _req=yes + [ -s "$CONF_HOST/users.toml" ] && _req=no + ask_list ADD_USERS --add-user "Edge usernames to create or update, space-separated" v_ident "$_req" + ;; + esac + _c=${CERT_FILE:+1} _k=${KEY_FILE:+1} + [ "$_c" = "$_k" ] || die "--cert-file and --key-file go together" + + _viewers=no + [ "$ROLE" != edge ] && [ -n "$HTTP_PORT$TCP_PORT" ] && [ "$TLS" = yes ] && _viewers=yes + if [ "$_viewers" = yes ]; then + choose SINK_AUTH --sink-auth "Authenticate the stream viewers (http/tcp outputs):" "$_isa" \ + none "TLS only" \ + mtls "client certificates, admitted by CN" \ + scram "username and password (lw auth token / stream)" + case $SINK_AUTH in + mtls) + ask CA_FILE --ca-file "CA bundle verifying the viewers' certificates" '' v_file + ask_list SINK_ALLOW --sink-allow "Viewer certificate CNs to admit (empty: any the CA issued)" v_any no + ;; + scram) + _req=yes + [ -s "$CONF_HOST/viewers.toml" ] && _req=no + ask_list ADD_VIEWERS --add-viewer "Viewer usernames to create or update, space-separated" v_ident "$_req" + ;; + esac + else + SINK_AUTH=none + fi + + # Values the choices made moot are dropped, so the plan shows what applies. + _n='--auth scram (TLS, on an edge)' + [ "$ROLE/$AUTH" = edge/mtls ] || [ "$ROLE/$AUTH" = aggregator/mtls ] || + drop ALLOW --allow '--auth mtls (TLS, on an edge or aggregator)' + [ "$ROLE/$AUTH" = edge/scram ] || { drop USERNAME --username "$_n"; drop PASSWORD_FILE --password-file "$_n"; } + [ "$ROLE/$AUTH" = aggregator/scram ] || drop ADD_USERS --add-user '--auth scram (TLS, on an aggregator)' + [ "$SINK_AUTH" = mtls ] || drop SINK_ALLOW --sink-allow '--sink-auth mtls (TLS, an http or tcp output)' + [ "$SINK_AUTH" = scram ] || drop ADD_VIEWERS --add-viewer '--sink-auth scram (TLS, an http or tcp output)' + [ "$ROLE" = edge ] || { drop SERVER_NAME --server-name; drop AGGREGATOR --aggregator; drop NODE --node; } + [ "$ROLE" != edge ] || { drop FILE_DIR --file-dir; drop HTTP_PORT --http-port; drop TCP_PORT --tcp-port; } + [ "$ROLE" != aggregator ] || drop LOG_DIRS --log-dir + [ -z "$SERVER_NAME" ] || v_host "$SERVER_NAME" || exit 1 + if [ -n "$ADD_USERS$ADD_VIEWERS" ]; then + ask SECRETS_DIR --secrets-dir "Directory for the generated passwords" "$HOME_DIR/logwisp-secrets" v_abs + else + drop SECRETS_DIR --secrets-dir + fi + + if [ "$RUNTIME" = docker ]; then + if [ -z "$BUILD" ]; then + if "$ENGINE" image inspect "$IMAGE" >/dev/null 2>&1; then + BUILD=no + elif [ "$INTERACTIVE" = 1 ]; then + ask_yn BUILD "Image $IMAGE is not here; build it from $REPO?" yes + else + BUILD=no + warn "image $IMAGE is not here; $ENGINE will try to pull it (--build builds it)" + fi + fi + [ -z "$BUILD_CA" ] || v_file "$BUILD_CA" || exit 1 + [ -z "$NETWORK" ] || v_word "$NETWORK" || exit 1 + if [ "$NETWORK" != host ] && [ "$ROLE" != edge ]; then + case $LISTEN in *:*) ;; *[!0-9.]*) die "--listen $LISTEN: a container publishes its ports on an IP address, not a host name" ;; esac + fi + fi + [ -z "$BIN" ] || v_file "$BIN" || exit 1 + for _var in CERT_FILE KEY_FILE CA_FILE PASSWORD_FILE BUILD_CA BIN; do + eval "_val=\$$_var" + case $_val in '' | /*) ;; *) eval "$_var=\$PWD/\$_val" ;; esac + done + BUILD=${BUILD:-no} + [ "$RUNTIME" = manual ] && START=no + ask_yn START "Start it now?" yes +} + +# --- derived values --- + +derive() { + C_CERT=$CONF_LW/tls/node.crt C_KEY=$CONF_LW/tls/node.key C_CA=$CONF_LW/tls/ca.crt + C_PASS=$CONF_LW/chain.pass C_USERS=$CONF_LW/users.toml C_VIEWERS=$CONF_LW/viewers.toml + CONF_T=${CONF_HOST#"$ROOT"} # CONF_HOST as seen from inside the target + LISTEN=${LISTEN:-0.0.0.0} + BIND=$LISTEN + [ "$RUNTIME" = docker ] && [ "$NETWORK" != host ] && BIND=0.0.0.0 + FILE_MODE=0640 SECRET_MODE=0600 + [ "$RUNTIME" = native ] && SECRET_MODE=0640 + CHOWN_IMAGE=0 USERNS='' + if [ "$RUNTIME" = docker ]; then + if [ "$(id -u)" = 0 ]; then + CHOWN_IMAGE=1 + elif [ "$ENGINE" = podman ]; then + USERNS=keep-id:uid=65532,gid=65532 + elif [ "$DRY_RUN" = 0 ]; then + die "run as root: the configuration must belong to the image user $IMAGE_USER" + fi + fi + LOG_GID='' + if [ "$RUNTIME" = docker ] && [ -n "$LOG_GROUP" ]; then + LOG_GID=$(getent group "$LOG_GROUP" 2>/dev/null | cut -d: -f3) + [ -n "$LOG_GID" ] || die "no such group here: $LOG_GROUP" + fi + case $RUNTIME in + docker) DESC="$ENGINE container $NAME" ;; + native) DESC="systemd service logwisp" ;; + manual) DESC="configuration for lw -c" ;; + esac + [ "$RUNTIME/$TARGET_OS" = native/freebsd ] && DESC="rc.d service logwisp${JAIL:+ in jail $JAIL}" + OLD_CONF='' OLD_CTR='' + [ ! -f "$CONF_HOST/logwisp.toml" ] || OLD_CONF=$CONF_HOST/logwisp.toml + if [ "$RUNTIME/$START" = docker/yes ] && "$ENGINE" container inspect "$NAME" >/dev/null 2>&1; then + OLD_CTR=$NAME + fi +} + +# --- the configuration --- + +tls_listener() { # PREFIX AUTH + [ "$TLS" = yes ] || return 0 + printf '[%s.tls]\nenabled = true\ncert_file = %s\nkey_file = %s\n' "$1" "$(tq "$C_CERT")" "$(tq "$C_KEY")" + [ "$2" != mtls ] || printf 'client_auth = true\nclient_ca_file = %s\n' "$(tq "$C_CA")" +} + +auth_listener() { # PREFIX AUTH ALLOW CREDENTIALS + case $2 in + mtls) printf '[%s.auth]\ntype = "mtls"\nallow = %s\n' "$1" "$(tlist "$3")" ;; + scram) printf '[%s.auth]\ntype = "scram"\ncredentials_file = %s\n' "$1" "$(tq "$4")" ;; + esac +} + +gen_config() { + printf '# LogWisp %s node (%s), written by deploy/lw-deploy.sh.\n#\n' "$ROLE" "$DESC" + not_covered '# ' + printf '\n[logging]\noutput = "stdout"\nlevel = "info"\n\n[[pipelines]]\nname = "%s"\n' "$ROLE" + [ "$ROLE" = edge ] || printf '\n[pipelines.flow.format]\ntype = "%s"\n' "$FORMAT" + + _n=0 + IFS=$NL + for _d in $LOG_DIRS; do + IFS=$OIFS + _n=$((_n + 1)) + _id=logs + [ "$_n" = 1 ] || _id=logs_$_n + printf '\n[[pipelines.plugin_sources]]\nid = "%s"\ntype = "file"\n[pipelines.plugin_sources.config]\n' "$_id" + printf 'directory = %s\npattern = %s\nfrom = "%s"\n' "$(tq "$_d")" "$(tq "$LOG_PATTERN")" "$FROM" + done + IFS=$OIFS + + _p=pipelines.plugin_sources.config + if [ "$ROLE" = aggregator ]; then + printf '\n[[pipelines.plugin_sources]]\nid = "from_edges"\ntype = "%s_chain"\n[%s]\n' "$CHAIN" "$_p" + printf 'host = %s\nport = %s\n' "$(tq "$BIND")" "$CHAIN_PORT" + tls_listener "$_p" "$AUTH" + auth_listener "$_p" "$AUTH" "$ALLOW" "$C_USERS" + fi + + _p=pipelines.plugin_sinks.config + if [ "$ROLE" = edge ]; then + printf '\n[[pipelines.plugin_sinks]]\nid = "to_aggregator"\ntype = "%s_chain"\n[%s]\n' "$CHAIN" "$_p" + printf 'host = %s\nport = %s\nnode = %s\n' "$(tq "$AGGREGATOR")" "$CHAIN_PORT" "$(tq "$NODE")" + if [ "$TLS" = yes ]; then + printf '[%s.tls]\nenabled = true\n' "$_p" + [ -z "$CA_FILE" ] || printf 'ca_file = %s\n' "$(tq "$C_CA")" + [ -z "$SERVER_NAME" ] || printf 'server_name = %s\n' "$(tq "$SERVER_NAME")" + [ -z "$CERT_FILE" ] || printf 'cert_file = %s\nkey_file = %s\n' "$(tq "$C_CERT")" "$(tq "$C_KEY")" + fi + case $AUTH in + mtls) printf '[%s.auth]\ntype = "mtls"\nallow = %s\n' "$_p" "$(tlist "$ALLOW")" ;; + scram) printf '[%s.auth]\ntype = "scram"\nusername = %s\npassword_file = %s\n' "$_p" "$(tq "$USERNAME")" "$(tq "$C_PASS")" ;; + esac + return 0 + fi + if [ -n "$FILE_DIR" ]; then + printf '\n[[pipelines.plugin_sinks]]\nid = "files"\ntype = "file"\n[%s]\n' "$_p" + printf 'directory = %s\nname = %s\n' "$(tq "$FILE_DIR")" "$(tq "$FILE_NAME")" + fi + for _t in http tcp; do + _port=$HTTP_PORT + [ "$_t" = tcp ] && _port=$TCP_PORT + [ -n "$_port" ] || continue + printf '\n[[pipelines.plugin_sinks]]\nid = "%s"\ntype = "%s"\n[%s]\n' "$_t" "$_t" "$_p" + printf 'host = %s\nport = %s\n' "$(tq "$BIND")" "$_port" + tls_listener "$_p" "$SINK_AUTH" + auth_listener "$_p" "$SINK_AUTH" "$SINK_ALLOW" "$C_VIEWERS" + done +} + +# --- the plan, and the command that repeats it --- + +EQ='' +eq() { + EQ="$EQ \\$NL $1" + [ $# -lt 2 ] || EQ="$EQ $(shquote "$2")" +} +eq_opt() { [ -z "$2" ] || eq "$1" "$2"; } +eq_list() { + IFS=$NL + for _e in $2; do eq "$1" "$_e"; done + IFS=$OIFS +} + +equivalent() { + eq --role "$ROLE" + if [ "$RUNTIME" = docker ] && [ "$ENGINE" = podman ]; then eq --runtime podman; else eq --runtime "$RUNTIME"; fi + [ "$TARGET_OS" = "$HOST_OS" ] || eq --os "$TARGET_OS" + eq_opt --jail "$JAIL" + if [ "$RUNTIME" = docker ]; then + eq --name "$NAME" + eq --image "$IMAGE" + [ "$BUILD" = no ] || eq --build + eq_opt --build-ca "$BUILD_CA" + eq_opt --build-flags "$BUILD_FLAGS" + eq_opt --network "$NETWORK" + fi + [ "$RUNTIME" = native ] || eq --config-dir "$CONF_DIR" + [ "$RUNTIME" != native ] || [ "$BUILD" = no ] || eq --build + eq_opt --bin "$BIN" + eq_list --log-dir "$LOG_DIRS" + if [ "$ROLE" != aggregator ]; then + eq --log-pattern "$LOG_PATTERN" + eq --from "$FROM" + fi + eq_opt --log-group "$LOG_GROUP" + if [ "$ROLE" != standalone ]; then + eq --chain "$CHAIN" + eq_opt --aggregator "$AGGREGATOR" + eq --chain-port "$CHAIN_PORT" + eq_opt --node "$NODE" + fi + if [ "$ROLE" != edge ]; then + eq --listen "$LISTEN" + eq_opt --file-dir "$FILE_DIR" + [ -z "$FILE_DIR" ] || eq --file-name "$FILE_NAME" + eq_opt --http-port "$HTTP_PORT" + eq_opt --tcp-port "$TCP_PORT" + eq --format "$FORMAT" + fi + if [ "$TLS" = yes ]; then eq --tls; else eq --no-tls; fi + eq_opt --cert-file "$CERT_FILE" + eq_opt --key-file "$KEY_FILE" + eq_opt --ca-file "$CA_FILE" + eq_opt --server-name "$SERVER_NAME" + [ "$ROLE" = standalone ] || eq --auth "$AUTH" + eq_list --allow "$ALLOW" + eq_opt --username "$USERNAME" + eq_opt --password-file "$PASSWORD_FILE" + eq_list --add-user "$ADD_USERS" + [ "$ROLE" = edge ] || [ "$TLS" = no ] || [ -z "$HTTP_PORT$TCP_PORT" ] || eq --sink-auth "$SINK_AUTH" + eq_list --sink-allow "$SINK_ALLOW" + eq_list --add-viewer "$ADD_VIEWERS" + eq_opt --secrets-dir "$SECRETS_DIR" + [ "$RUNTIME" = manual ] || { if [ "$START" = yes ]; then eq --start; else eq --no-start; fi; } + printf '%s%s \\\n --yes\n' "$(shquote "$0")" "$EQ" +} + +plan() { + note "LogWisp $ROLE node: $DESC" + note " configuration $CONF_HOST/logwisp.toml" + [ "$CONF_HOST" = "$CONF_LW" ] || note " (lw reads it as $CONF_LW/logwisp.toml)" + [ -z "$OLD_CONF" ] || note " replacing the existing one, kept as logwisp.toml.bak" + [ -z "$OLD_CTR" ] || note " replaces the existing container $OLD_CTR" + IFS=$NL + for _d in $LOG_DIRS; do note " tails $_d/$LOG_PATTERN"; done + IFS=$OIFS + _sec=plain + [ "$TLS" = yes ] && _sec="TLS, auth $AUTH" + [ "$ROLE" = edge ] && note " forwards to $(urlhost "$AGGREGATOR"):$CHAIN_PORT over ${CHAIN}_chain ($_sec) as node $NODE" + [ "$ROLE" = aggregator ] && note " receives on $(urlhost "$LISTEN"):$CHAIN_PORT over ${CHAIN}_chain ($_sec)" + _sec=plain + [ "$TLS" = yes ] && _sec="TLS, auth $SINK_AUTH" + [ -z "$FILE_DIR" ] || note " writes $FILE_DIR/$FILE_NAME.log ($FORMAT)" + [ -z "$HTTP_PORT" ] || note " serves http $(urlhost "$LISTEN"):$HTTP_PORT /stream /status ($FORMAT, $_sec)" + [ -z "$TCP_PORT" ] || note " serves tcp $(urlhost "$LISTEN"):$TCP_PORT ($FORMAT, $_sec)" + [ -z "$ADD_USERS$ADD_VIEWERS" ] || note " passwords $SECRETS_DIR/USER.pass" + note "" + note "The same deployment without prompts:" + equivalent + note "" + not_covered ' ' + note "" +} + +# --- files --- + +OWNED='' +own() { OWNED=$(add_item "$OWNED" "$1"); } + +install_copy() { # SRC REL MODE: a copy inside the configuration directory + own "$2" + [ "$1" = "$CONF_HOST/$2" ] || put_file "$CONF_T/$2" "$3" "$1" +} + +apply_owner() { + _base=$CONF_T + set -- + IFS=$NL + for _r in $OWNED; do set -- "$@" "$_base/$_r"; done + IFS=$OIFS + case $RUNTIME in + docker) [ "$CHOWN_IMAGE" = 0 ] || run chown "$IMAGE_USER" "$_base" "$@" || die "chown failed" ;; + native) trun chown root:logwisp "$_base" "$@" || die "chown failed" ;; + esac +} + +output_dir() { + [ -n "$FILE_DIR" ] || return 0 + case $RUNTIME in + docker) _owner=$IMAGE_USER ;; + native) _owner=logwisp:logwisp ;; + *) _owner='' ;; + esac + if [ -d "$ROOT$FILE_DIR" ]; then + # An existing directory keeps its owner: it may be shared. + case $RUNTIME in + docker) [ "$CHOWN_IMAGE" = 0 ] || [ -n "$(find "$FILE_DIR" -prune -user "${IMAGE_USER%:*}")" ] || + warn "$FILE_DIR must be writable by $IMAGE_USER: chown $IMAGE_USER $FILE_DIR" ;; + native) [ "$DRY_RUN" = 1 ] || [ -n "$(tprobe find "$FILE_DIR" -prune -user logwisp)" ] || + warn "$FILE_DIR must be writable by logwisp: chown logwisp:logwisp $FILE_DIR" ;; + esac + return 0 + fi + trun install -d -m 0750 "$FILE_DIR" || die "cannot create $FILE_DIR" + case $RUNTIME in + docker) [ "$CHOWN_IMAGE" = 0 ] || run chown "$_owner" "$FILE_DIR" ;; + native) trun chown "$_owner" "$FILE_DIR" ;; + esac +} + +write_files() { + if [ -d "$CONF_HOST" ] && [ ! -f "$CONF_HOST/logwisp.toml" ] && [ -n "$(ls -A "$CONF_HOST" 2>/dev/null)" ] && [ "$RUNTIME" != native ]; then + die "$CONF_HOST holds other files; choose an empty or a LogWisp --config-dir" + fi + trun install -d -m 0750 "$CONF_T" || die "cannot create $CONF_HOST" + if [ -n "$CERT_FILE$CA_FILE" ]; then + own tls + trun install -d -m 0750 "$CONF_T/tls" || die "cannot create $CONF_HOST/tls" + fi + [ -z "$CERT_FILE" ] || install_copy "$CERT_FILE" tls/node.crt "$FILE_MODE" + [ -z "$KEY_FILE" ] || install_copy "$KEY_FILE" tls/node.key "$SECRET_MODE" + [ -z "$CA_FILE" ] || install_copy "$CA_FILE" tls/ca.crt "$FILE_MODE" + [ -z "$PASSWORD_FILE" ] || install_copy "$PASSWORD_FILE" chain.pass "$SECRET_MODE" + _creds='' + [ "$ROLE/$AUTH" != aggregator/scram ] || _creds=users.toml + [ "$SINK_AUTH" != scram ] || _creds="$_creds viewers.toml" + for _f in $_creds; do + own "$_f" + # lw auth keeps an existing file's owner and mode. + [ -f "$CONF_HOST/$_f" ] || put_file "$CONF_T/$_f" "$SECRET_MODE" "$1" && echo >>"$1") || die "cannot write $1" +} + +keep_passwords() { # CREDENTIALS(rel) NAMES: a new password would lock out an existing user + IFS=$NL + for _u in $2; do + [ -s "$SECRETS_DIR/$_u.pass" ] || ! grep -qxF "username = \"$_u\"" "$CONF_HOST/$1" 2>/dev/null || + die "user $_u exists in $1, but $SECRETS_DIR/$_u.pass does not: give the --secrets-dir holding it, or remove the user (lw auth remove-user)" + done + IFS=$OIFS +} + +add_users() { # CREDENTIALS(rel) NAMES + [ -n "$2" ] || return 0 + _cred=$1 + IFS=$NL + for _u in $2; do + IFS=$OIFS + _pw=$SECRETS_DIR/$_u.pass + gen_password "$_pw" + case $RUNTIME in + docker) + set -- "$ENGINE" run --rm -i --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges -u "$IMAGE_USER" + [ -z "$USERNS" ] || set -- "$@" --userns "$USERNS" + set -- "$@" -v "$CONF_HOST:/etc/logwisp" "$IMAGE" + ;; + native) + set -- "$BINDIR/lw" + [ -z "$JAIL" ] || set -- jexec "$JAIL" "$@" + ;; + manual) set -- "$LW" ;; + esac + run_in "$_pw" "$@" auth add-user -credentials "$CONF_LW/$_cred" -user "$_u" -password-file /dev/stdin || + die "adding user $_u failed" + done + IFS=$OIFS +} + +# --- runtimes --- + +pkg_text() { # FILE SED-SCRIPT + if [ -f "$PACKAGE_DIR/$1" ]; then + sed "$2" "$PACKAGE_DIR/$1" + elif [ "$DRY_RUN" = 1 ]; then + warn "missing $PACKAGE_DIR/$1" + printf '(%s/%s, with %s applied)\n' "$PACKAGE_DIR" "$1" "$2" + else + die "missing $PACKAGE_DIR/$1 (--package-dir names another copy)" + fi +} + +pick_bin() { # CANDIDATE-DIRS: sets BINDIR, and BIN_SRC when lw must be installed + BIN_SRC=$BIN BINDIR=/usr/local/bin + [ -z "$BIN_SRC" ] || return 0 + for _d in $1; do + [ -x "$ROOT$_d/lw" ] && { BINDIR=$_d; return 0; } + done + BIN_SRC=$REPO/bin/lw + [ -x "$BIN_SRC" ] || [ "$DRY_RUN" = 1 ] || die "no lw binary: build it (make build, or --build) or pass --bin FILE" +} + +linux_prepare() { + command -v systemctl >/dev/null 2>&1 || [ "$DRY_RUN" = 1 ] || die "a native Linux install needs systemd; use --runtime docker or manual" + if [ -f "$PKG_UNIT" ]; then + # lw auth and lw --check run the binary the packaged unit starts. + BINDIR=$(sed -n 's|^ExecStart=\(/[^ ]*\)/lw\( .*\)\{0,1\}$|\1|p' "$PKG_UNIT") + [ -n "$BINDIR" ] || die "no lw in the ExecStart of $PKG_UNIT" + note "using the packaged logwisp.service and $BINDIR/lw" + else + pick_bin '/usr/local/bin /usr/bin' + [ -z "$BIN_SRC" ] || run install -m 0755 "$BIN_SRC" "$BINDIR/lw" || die "cannot install lw" + _unit=$(pkg_text logwisp.service "s|@BINDIR@|$BINDIR|g; s|@SYSCONFDIR@|/etc|g") || exit 1 + put_file /etc/systemd/system/logwisp.service 0644 </dev/null 2>&1; then + trun pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin -c "LogWisp log transport" || + die "cannot create the logwisp user" + fi + [ -z "$LOG_GROUP" ] || trun pw groupmod "$LOG_GROUP" -m logwisp || die "cannot add logwisp to $LOG_GROUP" + _rc=$(pkg_text logwisp.rc "s|%%BINDIR%%|$BINDIR|g; s|%%PREFIX%%|/usr/local|g") || exit 1 + put_file /usr/local/etc/rc.d/logwisp 0555 </dev/null) || _ver=dev + _rev=$(git -C "$REPO" rev-parse HEAD 2>/dev/null) || _rev=unknown + run "$@" --build-arg "VERSION=$_ver" --build-arg "REVISION=$_rev" -t "$IMAGE" "$REPO" || + die "the image build failed (behind a proxy: --build-ca FILE, --build-flags \"--network host\")" +} + +# --check builds every plugin as a start would, reading its TLS and credentials files. +conf_check() { # COMMAND...: lw, or the container that runs it + run "$@" --check -c "$CONF_LW/logwisp.toml" || + die "lw --check rejects $CONF_HOST/logwisp.toml${OLD_CONF:+ (the previous one is logwisp.toml.bak)}" +} + +docker_start() { + set -- --read-only --cap-drop ALL --security-opt no-new-privileges -u "$IMAGE_USER" + [ -z "$USERNS" ] || set -- "$@" --userns "$USERNS" + [ -z "$LOG_GID" ] || set -- "$@" --group-add "$LOG_GID" + set -- "$@" -v "$CONF_HOST:/etc/logwisp:ro" + IFS=$NL + for _d in $LOG_DIRS; do set -- "$@" -v "$_d:$_d:ro"; done + IFS=$OIFS + [ -z "$FILE_DIR" ] || set -- "$@" -v "$FILE_DIR:$FILE_DIR" + conf_check "$ENGINE" run --rm --network none "$@" "$IMAGE" + _net='' + case $NETWORK in + '' | host | bridge | none) ;; + *) "$ENGINE" network inspect "$NETWORK" >/dev/null 2>&1 || _net=$NETWORK ;; + esac + if [ "$START" = yes ]; then + [ -z "$_net" ] || run "$ENGINE" network create "$_net" || die "cannot create network $_net" + [ -z "$OLD_CTR" ] || run "$ENGINE" rm -f "$NAME" || die "cannot remove container $NAME" + fi + [ -z "$NETWORK" ] || set -- "$@" --network "$NETWORK" + if [ "$NETWORK" != host ] && [ "$ROLE" != edge ]; then + for _p in $CHAIN_PORT $HTTP_PORT $TCP_PORT; do + set -- "$@" -p "$(urlhost "$LISTEN"):$_p:$_p" + done + fi + set -- "$ENGINE" run -d --name "$NAME" --restart unless-stopped "$@" "$IMAGE" -c /etc/logwisp/logwisp.toml + if [ "$START" = no ]; then + note "Start it with:" + [ -z "$_net" ] || note "+ $ENGINE network create $_net" + show_cmd "$@" + printf '\n' + return 0 + fi + run "$@" || die "the container did not start" + [ "$DRY_RUN" = 1 ] && return 0 + sleep 3 + _state=$("$ENGINE" inspect -f '{{.State.Running}} {{.RestartCount}}' "$NAME" 2>/dev/null) + "$ENGINE" logs --tail 15 "$NAME" 2>&1 | sed 's/^/ | /' + [ "$_state" = "true 0" ] || die "container $NAME is not running cleanly; see: $ENGINE logs $NAME" + note "container $NAME is running" +} + +native_start() { + if [ "$TARGET_OS" = freebsd ]; then + if [ "$START" = no ]; then + note "Start it with: ${JAIL:+jexec $JAIL }sysrc logwisp_enable=YES && ${JAIL:+jexec $JAIL }service logwisp start" + return 0 + fi + trun sysrc logwisp_enable=YES || die "sysrc failed" + trun service logwisp restart || die "the service did not start; see /var/log/messages" + [ "$DRY_RUN" = 1 ] || { sleep 2; tprobe service logwisp status || die "the service is not running; see /var/log/messages"; } + return 0 + fi + run systemctl daemon-reload || exit 1 + if [ "$START" = no ]; then + note "Start it with: systemctl enable --now logwisp" + return 0 + fi + run systemctl enable logwisp || exit 1 + run systemctl restart logwisp || die "the service did not start: journalctl -u logwisp" + [ "$DRY_RUN" = 1 ] && return 0 + sleep 2 + systemctl is-active --quiet logwisp || { journalctl -u logwisp -n 20 --no-pager; die "the service is not running"; } + note "service logwisp is running" +} + +next_steps() { + note "" + note "Next:" + case $RUNTIME in + docker) + note " logs: $ENGINE logs -f $NAME" + note " reload: edit $CONF_HOST/logwisp.toml, then $ENGINE kill -s HUP $NAME" + ;; + native) + if [ "$TARGET_OS" = freebsd ]; then note " logs: /var/log/messages${JAIL:+ in jail $JAIL} (tag logwisp)"; else note " logs: journalctl -u logwisp -f"; fi + if [ "$TARGET_OS" = freebsd ]; then _r="${JAIL:+jexec $JAIL }service logwisp reload"; else _r="systemctl reload logwisp"; fi + note " reload: edit $CONF_LW/logwisp.toml, then $_r" + ;; + manual) note " run: ${LW:-lw} -c $CONF_LW/logwisp.toml" ;; + esac + if [ -n "$HTTP_PORT" ]; then + _h=$LISTEN + [ "$_h" = 0.0.0.0 ] && _h=127.0.0.1 + [ "$_h" = :: ] && _h=::1 + if [ "$TLS" = yes ]; then _u=https; else _u=http; fi + _u=$_u://$(urihost "$_h"):$HTTP_PORT + case $SINK_AUTH in + none) + _c='' + [ "$TLS" = no ] || _c='--cacert CA ' + note " status: curl $_c$_u/status" + ;; + mtls) note " status: curl --cacert CA --cert CERT --key KEY $_u/status" ;; + scram) note " status: curl with a bearer token from: lw auth token -url $_u -user USER -password-file FILE -ca-file CA (doc/cli.md)" ;; + esac + note " stream: $_u/stream" + fi + if [ -n "$ADD_USERS" ]; then + note " edges: copy $SECRETS_DIR/USER.pass to each edge over a safe channel, then run" + note " there: $PROG --role edge --tls --auth scram --username USER --password-file FILE ..." + fi + [ -z "$ADD_VIEWERS" ] || note " viewers: lw auth token / lw auth stream with $SECRETS_DIR/USER.pass (doc/cli.md)" + if [ "$ROLE" != standalone ] && [ "$AUTH" = none ]; then + note " without chain auth the aggregator admits any peer and trusts the node label it claims" + fi + if [ "$SINK_AUTH" = none ] && [ -n "$HTTP_PORT$TCP_PORT" ]; then + note " the http/tcp outputs admit anyone who reaches them: bind --listen to a trusted address" + fi +} + +main() { + resolve + derive + plan + if [ "$DRY_RUN" = 1 ]; then + note "Dry run: nothing below is changed." + note "" + elif [ "$INTERACTIVE" = 1 ]; then + _go='' + ask_yn _go "Apply?" yes + [ "$_go" = yes ] || exit 0 + elif [ -n "$OLD_CONF$OLD_CTR" ] && [ "$FORCE" != yes ]; then + die "${OLD_CONF:-container $NAME} exists (--force replaces it)" + fi + + keep_passwords users.toml "$ADD_USERS" + keep_passwords viewers.toml "$ADD_VIEWERS" + case $RUNTIME in + docker) + [ "$DRY_RUN" = 1 ] || "$ENGINE" info >/dev/null 2>&1 || die "cannot reach $ENGINE" + [ "$BUILD" = no ] || build_image + ;; + native) + [ "$DRY_RUN" = 1 ] || [ "$(id -u)" = 0 ] || die "a native install needs root" + if [ "$BUILD" = yes ]; then + run make -C "$REPO" build || die "make build failed" + BIN=$REPO/bin/lw + fi + if [ "$TARGET_OS" = freebsd ]; then freebsd_prepare; else linux_prepare; fi + ;; + manual) + LW=$BIN + [ -n "$LW" ] || LW=$(command -v lw 2>/dev/null) || LW='' + [ -n "$LW" ] || LW=$REPO/bin/lw + [ -z "$ADD_USERS$ADD_VIEWERS" ] || [ -x "$LW" ] || [ "$DRY_RUN" = 1 ] || die "no lw binary to add users with: --bin FILE" + ;; + esac + + write_files + [ -z "$ADD_USERS$ADD_VIEWERS" ] || run install -d -m 0700 "$SECRETS_DIR" || die "cannot create $SECRETS_DIR" + add_users users.toml "$ADD_USERS" + add_users viewers.toml "$ADD_VIEWERS" + + case $RUNTIME in + docker) docker_start ;; + native) + if [ -n "$JAIL" ]; then conf_check jexec "$JAIL" "$BINDIR/lw"; else conf_check "$BINDIR/lw"; fi + native_start + ;; + manual) [ ! -x "$LW" ] || conf_check "$LW" ;; + esac + next_steps +} + +main diff --git a/deploy/package/arch/PKGBUILD b/deploy/package/arch/PKGBUILD new file mode 100644 index 0000000..b939baa --- /dev/null +++ b/deploy/package/arch/PKGBUILD @@ -0,0 +1,46 @@ +# Maintainer: +# Skeleton: source and checksum wait for a tagged release; run namcap and +# makepkg --printsrcinfo > .SRCINFO before publishing to the AUR. + +pkgname=logwisp +pkgver=0.21.0 +pkgrel=1 +pkgdesc='Pipeline-based log transport with TLS and SCRAM authentication' +arch=('x86_64' 'aarch64') +url='https://github.com/lixenwraith/logwisp' +license=('BSD-3-Clause') +depends=('glibc') +makedepends=('go') +backup=('etc/logwisp/logwisp.toml') +source=("$pkgname-$pkgver.tar.gz::$url/archive/v$pkgver.tar.gz") +sha256sums=('SKIP') + +prepare() { + cd "$pkgname-$pkgver" + export GOPATH="$srcdir/gopath" + go mod download -modcacherw +} + +# The Go package guidelines' flags; -linkmode=external goes through GO_LDFLAGS +# because the Makefile's own -ldflags (the version) would override GOFLAGS'. +build() { + cd "$pkgname-$pkgver" + export GOPATH="$srcdir/gopath" + export CGO_CPPFLAGS="$CPPFLAGS" + export CGO_CFLAGS="$CFLAGS" + export CGO_CXXFLAGS="$CXXFLAGS" + export CGO_LDFLAGS="$LDFLAGS" + export GOFLAGS='-buildmode=pie -trimpath -mod=readonly -modcacherw' + make build VERSION="$pkgver" GIT_COMMIT="v$pkgver" GO_LDFLAGS='-linkmode=external' +} + +check() { + cd "$pkgname-$pkgver" + export GOPATH="$srcdir/gopath" + go test ./... +} + +package() { + cd "$pkgname-$pkgver" + make install DESTDIR="$pkgdir" PREFIX=/usr SYSCONFDIR=/etc +} diff --git a/deploy/package/freebsd/Makefile b/deploy/package/freebsd/Makefile new file mode 100644 index 0000000..04991ed --- /dev/null +++ b/deploy/package/freebsd/Makefile @@ -0,0 +1,44 @@ +# Skeleton for sysutils/logwisp. GO_MODULE fetches the module by its canonical +# path, so it waits for the module rename to github.com/lixenwraith/logwisp +# (the -X paths below follow it), a tagged release, a maintainer, and a +# UIDs/GIDs entry for logwisp. files/logwisp.in links to ../logwisp.rc: cp -L. + +PORTNAME= logwisp +DISTVERSIONPREFIX= v +DISTVERSION= 0.21.0 +CATEGORIES= sysutils + +MAINTAINER= maintainer@example.org +COMMENT= Pipeline-based log transport with TLS and SCRAM authentication +WWW= https://github.com/lixenwraith/logwisp + +LICENSE= BSD3CLAUSE +LICENSE_FILE= ${WRKSRC}/LICENSE + +USES= go:1.27,modules +USE_RC_SUBR= logwisp +SUB_LIST= BINDIR=${PREFIX}/bin + +GO_MODULE= github.com/lixenwraith/logwisp +GO_TARGET= ./cmd/lw +GO_BUILDFLAGS= -ldflags="-s -w \ + -X logwisp/internal/version.Version=${DISTVERSION} \ + -X logwisp/internal/version.GitCommit=${DISTVERSIONFULL}" + +USERS= logwisp +GROUPS= logwisp + +OPTIONS_DEFINE= DOCS +PORTDOCS= * + +post-install: + ${INSTALL_MAN} ${WRKSRC}/doc/lw.1 ${STAGEDIR}${PREFIX}/share/man/man1 + ${MKDIR} ${STAGEDIR}${ETCDIR} + ${INSTALL_DATA} ${WRKSRC}/config/logwisp.toml ${STAGEDIR}${ETCDIR}/logwisp.toml.sample + +post-install-DOCS-on: + ${MKDIR} ${STAGEDIR}${DOCSDIR}/doc + ${INSTALL_DATA} ${WRKSRC}/README.md ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/doc/*.md ${STAGEDIR}${DOCSDIR}/doc + +.include diff --git a/deploy/package/freebsd/files/logwisp.in b/deploy/package/freebsd/files/logwisp.in new file mode 120000 index 0000000..89d0b02 --- /dev/null +++ b/deploy/package/freebsd/files/logwisp.in @@ -0,0 +1 @@ +../../logwisp.rc \ No newline at end of file diff --git a/deploy/package/freebsd/pkg-descr b/deploy/package/freebsd/pkg-descr new file mode 100644 index 0000000..386b559 --- /dev/null +++ b/deploy/package/freebsd/pkg-descr @@ -0,0 +1,5 @@ +LogWisp collects log entries from files, standard input or other LogWisp +nodes; rate-limits, filters and formats them; and distributes them to files, +the console, live HTTP (Server-Sent Events) and TCP streams, or downstream +nodes. Every network source and sink supports TLS, mutual TLS and +Argon2id-SCRAM password authentication, and reloads on SIGHUP. diff --git a/deploy/package/freebsd/pkg-plist b/deploy/package/freebsd/pkg-plist new file mode 100644 index 0000000..76ebf2b --- /dev/null +++ b/deploy/package/freebsd/pkg-plist @@ -0,0 +1,3 @@ +bin/lw +@sample %%ETCDIR%%/logwisp.toml.sample +share/man/man1/lw.1.gz diff --git a/deploy/package/logwisp.rc b/deploy/package/logwisp.rc new file mode 100644 index 0000000..a9ce586 --- /dev/null +++ b/deploy/package/logwisp.rc @@ -0,0 +1,47 @@ +#!/bin/sh + +# PROVIDE: logwisp +# REQUIRE: LOGIN NETWORKING +# KEYWORD: shutdown + +# Add to /etc/rc.conf (sysrc logwisp_enable=YES): +# logwisp_enable "YES" starts LogWisp at boot (default "NO") +# logwisp_config configuration file (default %%PREFIX%%/etc/logwisp/logwisp.toml) +# logwisp_user account lw runs as (default "logwisp") +# logwisp_chdir working directory (default /var/db/logwisp) +# logwisp_args extra lw options, e.g. --logging.level=debug (default empty); +# rc.subr hands logwisp_flags to daemon(8), not to lw + +. /etc/rc.subr + +name=logwisp +rcvar=logwisp_enable +desc="LogWisp log transport" + +load_rc_config $name + +: ${logwisp_enable:="NO"} +: ${logwisp_config:="%%PREFIX%%/etc/logwisp/logwisp.toml"} +: ${logwisp_user:="logwisp"} +: ${logwisp_chdir:="/var/db/logwisp"} +: ${logwisp_args:=""} + +# daemon(8) records the child's pid, so stop and reload (SIGHUP) signal lw +# itself, and sends its output to syslog under the tag logwisp. +pidfile="/var/run/${name}/${name}.pid" +procname="%%BINDIR%%/lw" +command="/usr/sbin/daemon" +command_args="-f -S -T ${name} -p ${pidfile} ${procname} -c ${logwisp_config} ${logwisp_args}" +required_files="${logwisp_config}" +extra_commands="reload" +start_precmd="${name}_prestart" + +# rc.subr starts daemon(8) as logwisp_user, which must own the pidfile directory. +# An existing logwisp_chdir keeps its owner: it may be a shared directory. +logwisp_prestart() +{ + install -d -o "${logwisp_user}" -m 0755 "/var/run/${name}" + [ -d "${logwisp_chdir}" ] || install -d -o "${logwisp_user}" -m 0750 "${logwisp_chdir}" +} + +run_rc_command "$1" diff --git a/deploy/package/logwisp.service b/deploy/package/logwisp.service new file mode 100644 index 0000000..29bff39 --- /dev/null +++ b/deploy/package/logwisp.service @@ -0,0 +1,54 @@ +[Unit] +Description=LogWisp log transport +Documentation=man:lw(1) https://github.com/lixenwraith/logwisp +Wants=network-online.target +After=network-online.target +# A broken configuration stops after five tries instead of looping forever. +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +Type=exec +User=logwisp +Group=logwisp +ExecStart=@BINDIR@/lw -c @SYSCONFDIR@/logwisp/logwisp.toml +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartSec=5s +TimeoutStopSec=15s + +# logwisp.tmpfiles creates both directories. A file sink elsewhere needs a +# drop-in with ReadWritePaths=, a source under /home one with ProtectHome=read-only, +# and a port below 1024 CapabilityBoundingSet= and AmbientCapabilities= both +# set to CAP_NET_BIND_SERVICE. +WorkingDirectory=/var/lib/logwisp +ReadWritePaths=/var/lib/logwisp /var/log/logwisp +UMask=0027 + +NoNewPrivileges=yes +CapabilityBoundingSet= +AmbientCapabilities= +PrivateDevices=yes +PrivateTmp=yes +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProcSubset=pid +ProtectSystem=strict +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +RemoveIPC=yes +SystemCallArchitectures=native +SystemCallFilter=@system-service +LockPersonality=yes +MemoryDenyWriteExecute=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/package/logwisp.sysusers b/deploy/package/logwisp.sysusers new file mode 100644 index 0000000..6df6b1b --- /dev/null +++ b/deploy/package/logwisp.sysusers @@ -0,0 +1,2 @@ +# Type Name ID Description Home Shell +u logwisp - "LogWisp log transport" /var/lib/logwisp - diff --git a/deploy/package/logwisp.tmpfiles b/deploy/package/logwisp.tmpfiles new file mode 100644 index 0000000..58dd996 --- /dev/null +++ b/deploy/package/logwisp.tmpfiles @@ -0,0 +1,3 @@ +# Type Path Mode User Group Age Argument +d /var/lib/logwisp 0750 logwisp logwisp - - +d /var/log/logwisp 0750 logwisp logwisp - - diff --git a/doc/README.md b/doc/README.md index be43d60..2a8426d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -7,22 +7,32 @@ streams, or downstream LogWisp nodes. ## Documentation Map -| Document | Contents | -|----------|----------| -| [Installation](installation.md) | Building, installing, and running as a service | -| [Architecture](architecture.md) | Component model, data flow, concurrency, back-pressure | -| [Configuration](configuration.md) | TOML structure, precedence, environment and CLI overrides | -| [Sources](sources.md) | Every input plugin and its options | -| [Sinks](sinks.md) | Every output plugin and its options | -| [Filters](filters.md) | Pattern-based inclusion and exclusion | -| [Formatters](formatters.md) | Output shaping and sanitization | -| [Chaining](chaining.md) | Multi-node topologies and the chain wire protocol | -| [Networking](networking.md) | Listeners, dialers, timeouts, connection limits | -| [Security](security.md) | TLS, mTLS, and peer authorization; threat model and current limits | -| [mTLS Authentication](mtls-auth-plan.md) | Design and rationale for certificate-based authorization | -| [Password Authentication](scram-auth-plan.md) | Design and rationale for Argon2id-SCRAM authentication; mTLS hardening | -| [CLI](cli.md) | Flags, signals, exit codes, `logwisp auth` | -| [Operations](operations.md) | Running, monitoring, tuning, troubleshooting | +- [Installation](installation.md): building, installing, and running as a + service +- [Deployment](deployment.md): `deploy/lw-deploy.sh` for edges, aggregators, + containers, services, jails +- [Architecture](architecture.md): component model, data flow, concurrency, + back-pressure +- [Configuration](configuration.md): TOML structure, precedence, environment + and CLI overrides +- [Sources](sources.md): every input plugin and its options +- [Sinks](sinks.md): every output plugin and its options +- [Filters](filters.md): pattern-based inclusion and exclusion +- [Formatters](formatters.md): output shaping and sanitization +- [Chaining](chaining.md): multi-node topologies and the chain wire protocol +- [Networking](networking.md): listeners, dialers, timeouts, connection limits +- [Security](security.md): TLS, mTLS, and peer authorization; threat model and + current limits +- [mTLS Authentication](mtls-auth-plan.md): design and rationale for + certificate-based authorization +- [Password Authentication](scram-auth-plan.md): design and rationale for + Argon2id-SCRAM authentication; mTLS hardening +- [CLI](cli.md): flags, signals, exit codes, `lw auth`; also the `lw(1)` + manual, [`lw.1`](lw.1) +- [Operations](operations.md): running, monitoring, tuning, troubleshooting +- [To Do](todo.md): planned work in priority order: network access control + and the PROXY protocol, a hardening review of the config libraries, + packaging A fully annotated configuration covering every option lives at [`config/logwisp.toml`](../config/logwisp.toml). @@ -64,7 +74,7 @@ endpoint), `tcp` (broadcast server), `null`, and the chain forwarders - Authorization by certificate identity, per listener: named peers rather than everything the CA issued, with the `http` sink's endpoints gated too - Password authentication (Argon2id-SCRAM) bound to the listener's certificate, - with bearer tokens on HTTP listeners and a `logwisp auth` CLI, and browser + with bearer tokens on HTTP listeners and a `lw auth` CLI, and browser logins behind a TLS-terminating proxy - Node binding, so a chain source labels entries from the sender's certificate identity or username rather than from what the sender claims @@ -96,16 +106,17 @@ target = "stdout" ``` ```bash -logwisp -c config.toml +lw -c config.toml ``` ## System Requirements - **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+) - **Architecture**: amd64 -- **Go**: 1.26+ to build from source +- **Go**: 1.27.1+ to build from source -Network sources and sinks bind and dial over IPv4 only. +Network sources and sinks bind and dial IPv4 or IPv6, each keeping strictly to +the family of its host ([Networking](networking.md#address-family)). ## License diff --git a/doc/architecture.md b/doc/architecture.md index 7032946..9316595 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -25,29 +25,33 @@ main Package map: -| Package | Responsibility | -|---------|----------------| -| `cmd/logwisp` | Entry point, help, logger bootstrap, signal loop, status reporter | -| `internal/config` | Typed config schema, loading, top-level validation | -| `internal/service` | Owns the pipeline set; start, stop, shutdown, global stats | -| `internal/pipeline` | Pipeline runtime and per-pipeline plugin registry | -| `internal/flow` | Rate limiter, filter chain invocation, formatting, heartbeat | -| `internal/filter` | Regex filter and filter chain | -| `internal/format` | Adapter over `lixenwraith/log` formatter + sanitizer | -| `internal/source/*` | Source plugins | -| `internal/sink/*` | Sink plugins | -| `internal/plugin` | Global factory registry populated by plugin `init()` | -| `internal/chain` | Chain wire protocol: hello preamble, entry codec, backoff | -| `internal/tlsx` | The single seam between `TLSOptions` and `crypto/tls` | -| `internal/session` | Session manager and per-instance proxy | -| `internal/core` | Shared types (`LogEntry`, `TransportEvent`), capabilities, constants | -| `internal/tokenbucket` | Rate limiter primitive | -| `internal/sanitize` | Standalone hex-escaping helpers | +- `cmd/lw`: entry point, help, logger bootstrap, signal loop, status + reporter. +- `internal/config`: typed config schema, loading, top-level validation. +- `internal/service`: owns the pipeline set; start, stop, shutdown, global + stats. +- `internal/pipeline`: pipeline runtime and per-pipeline plugin registry. +- `internal/flow`: rate limiter, filter chain invocation, formatting, + heartbeat. +- `internal/filter`: regex filter and filter chain. +- `internal/format`: adapter over the `lixenwraith/log` formatter and + sanitizer. +- `internal/source/*`, `internal/sink/*`: the source and sink plugins. +- `internal/plugin`: global factory registry populated by plugin `init()`. +- `internal/chain`: chain wire protocol: hello preamble, entry codec, backoff. +- `internal/tlsx`: the single seam between `TLSOptions` and `crypto/tls`. +- `internal/authz`: the single seam between `AuthOptions` and the network + plugins: `Admit`, `AuthorizeRequest`, `Greet`, `Prepare`. +- `internal/session`: session manager and per-instance proxy. +- `internal/core`: shared types (`LogEntry`, `TransportEvent`), capabilities, + constants. +- `internal/tokenbucket`: rate limiter primitive. +- `internal/sanitize`: standalone hex-escaping helpers. ## Plugin Registration Every plugin registers itself in an `init()` function, and -`cmd/logwisp/bootstrap.go` blank-imports each package to trigger those +`cmd/lw/bootstrap.go` blank-imports each package to trigger those `init()`s. Adding a plugin therefore means writing the package, calling `plugin.RegisterSource` / `plugin.RegisterSink`, and adding one blank import. @@ -71,15 +75,14 @@ registry rejects a second instance of any such type. `LogEntry` fields: -| Field | Purpose | -|-------|---------| -| `Time` | Entry timestamp | -| `Node` | Origin node label for chained topologies; stamped at the first hop, preserved by relays | -| `Source` | Origin identifier within the node (filename, plugin id, …) | -| `Level` | `DEBUG`/`INFO`/`WARN`/`ERROR`/`TRACE`, when detected | -| `Message` | Log content | -| `Fields` | Optional structured metadata as raw JSON | -| `RawSize` | Original byte size, used by the entry-size cap | +- `Time`: entry timestamp. +- `Node`: origin node label for chained topologies; stamped at the first hop, + preserved by relays. +- `Source`: origin identifier within the node (filename, plugin id, …). +- `Level`: `DEBUG`, `INFO`, `WARN`, `ERROR` or `TRACE`, when detected. +- `Message`: log content. +- `Fields`: optional structured metadata as raw JSON. +- `RawSize`: original byte size, used by the entry-size cap. Carrying `Entry` alongside `Payload` is what makes chain sinks format-independent: a `tcp_chain` or `http_chain` sink re-serializes the @@ -88,13 +91,16 @@ structured entry rather than shipping whatever text the local formatter chose. ### Back-pressure and drops There is exactly one drop policy and it is not configurable: **never block**. +Each stage drops and counts instead of waiting: -| Stage | Full-buffer behaviour | Counter | -|-------|----------------------|---------| -| Source → subscriber | Drop the entry | source `dropped_entries` | -| Flow | Drop on rate limit, filter, or format error | `flow.total_dropped` | -| Pipeline → sink | Drop for that sink only | pipeline `total_dropped_by_sink` | -| TCP/HTTP sink → client queue | Drop for that client only | sink `dropped_writes` | +- Source → subscriber: a full channel drops the entry; source + `dropped_entries`. +- Flow: the rate limiter, a filter rejection or a format error drops it; + `flow.total_dropped`. +- Pipeline → sink: a full sink input drops it for that sink only; pipeline + `total_dropped_by_sink`. +- TCP/HTTP sink → client queue: a full queue drops it for that client only; + sink `dropped_writes`. The `tcp_chain` sink is the one deliberate exception. It holds a line across reconnects until it is written or the process shuts down, so a downstream @@ -126,17 +132,20 @@ on a non-retryable response or on shutdown (`dropped_batches`). ## Network Architecture -All listeners bind `tcp4` and all dialers dial `tcp4`. IPv6 clients cannot -connect; this is deliberate, not an oversight. +Each listener and dialer keeps strictly to the family of its host literal: +`tcp4` for IPv4, IPv6-only `tcp6` for IPv6 (`::` too); a hostname resolves. +See [Networking](networking.md#address-family). -| Plugin | Role | Protocol | -|--------|------|----------| -| `tcp` sink | Listener | Raw broadcast of formatted payloads | -| `http` sink | Listener | HTTP/1.1 SSE; HTTP/2 negotiated via ALPN when TLS is on | -| `tcp_chain` source | Listener | Chain protocol, persistent NDJSON stream | -| `http_chain` source | Listener | Chain protocol, NDJSON batches over POST | -| `tcp_chain` sink | Dialer | Chain protocol, persistent stream, auto-reconnect | -| `http_chain` sink | Dialer | Chain protocol, batched POST with retry | +The network plugins, by role: + +- Listeners + - `tcp` sink: raw broadcast of formatted payloads. + - `http` sink: HTTP/1.1 SSE; HTTP/2 negotiated via ALPN when TLS is on. + - `tcp_chain` source: chain protocol, persistent NDJSON stream. + - `http_chain` source: chain protocol, NDJSON batches over POST. +- Dialers + - `tcp_chain` sink: chain protocol, persistent stream, auto-reconnect. + - `http_chain` sink: chain protocol, batched POST with retry. TLS is built in exactly one place, `internal/tlsx`, which exposes `Server(opts)` for listeners and `Client(opts, host)` for dialers. See @@ -176,17 +185,20 @@ Reload (signal or file watch) rebuilds the entire service: Because this is a full rebuild, listening sockets close and reopen and all clients are disconnected. Application logging is configured once at startup and is **not** re-applied on reload. -Watch errors do not rebuild services. Queued changes are combined, and an unchanged -pipeline/status snapshot is skipped. Signals always rebuild for certificate rotation. -Failed candidates release their session workers. Once the old service has stopped, -a bind/start failure requires another corrected reload or process restart. + +Watch errors do not rebuild services. Queued changes are combined, and an +unchanged pipeline/status snapshot is skipped. Signals always rebuild for +certificate rotation. Failed candidates release their session workers. Once the +old service has stopped, a bind/start failure requires another corrected reload +or process restart. ## Resource Management - Every buffer is bounded; the drop-not-block policy keeps memory flat under load. -- Network sinks and chain sources accept a `max_connections` cap. Admission is - a load-then-check, so a burst can over-admit by roughly one connection. +- The `tcp` and `http` sinks and the `tcp_chain` source accept a + `max_connections` cap. Admission is a load-then-check, so a burst can + over-admit by roughly one connection. - Chain listeners bound a single line at `core.MaxLogEntryBytes` (1 MiB); an oversized line is a protocol violation and terminates the connection. - The `http_chain` source caps each request body at `max_body_bytes`. diff --git a/doc/chaining.md b/doc/chaining.md index 100d835..d33c1f9 100644 --- a/doc/chaining.md +++ b/doc/chaining.md @@ -36,24 +36,25 @@ Chained entries carry a `node` label identifying where they originated. The label comes from the `node` option, defaulting to `os.Hostname()`. - A chain **source** either honours the sender's label or overrides it, according to `trust_node`: - -| `trust_node` | Behaviour | -|--------------|-----------| -| `true` (default) | Keep the label the sender declared; fall back to the remote address when absent | -| `false` | Always overwrite with the sender's remote address | + - `true` (default): keep the label the sender declared; fall back to the + remote address when absent. + - `false`: always overwrite with the sender's remote address. Relays preserve `node`, so a label survives any number of hops and identifies the original producer rather than the last relay. With an `auth` block the source can instead bind the label to the sender's authenticated identity — its certificate identity under `mtls`, its username -under `scram` — which overrides `trust_node` entirely: +under `scram`. `assert` and `force` take the connection label over from +`trust_node`; only `force` also overrides the per-entry labels. +`auth.node_binding`: -| `auth.node_binding` | Connection label | Per-entry `node` field | -|---------------------|------------------|------------------------| -| `none` | `trust_node` governs | `trust_node` governs | -| `assert` | Must equal the identity, or the peer is rejected | `trust_node` governs | -| `force` (default with an `auth` block) | The identity | Overwritten with the identity | +- `none`: `trust_node` governs the connection label and the per-entry `node` + field. +- `assert`: the connection label must equal the identity, or the peer is + rejected; `trust_node` governs the per-entry `node` field. +- `force` (default with an `auth` block): the identity is the connection label + and overwrites every per-entry `node` field. Pick `force` at an ingest boundary you do not trust — it is the only setting where a compromised edge cannot mislabel its entries, including through the @@ -116,13 +117,13 @@ binary ignores it and the dialer gives up after 10 s. Batches of NDJSON delivered by `POST`, with the preamble expressed as headers. -| Header | Direction | Meaning | -|--------|-----------|---------| -| `X-Logwisp-Protocol` | request | Protocol version; must be `1` | -| `X-Logwisp-Node` | request | Origin node label | -| `Content-Type` | request | `application/x-ndjson` | -| `Authorization` | request | `Bearer `, under `scram` | -| `X-Logwisp-Accepted` | response | Number of entries ingested | +- Request headers + - `X-Logwisp-Protocol`: the protocol version; must be `1`. + - `X-Logwisp-Node`: the origin node label. + - `Content-Type`: `application/x-ndjson`. + - `Authorization`: `Bearer `, under `scram`. +- Response header + - `X-Logwisp-Accepted`: the number of entries ingested. Responses: `204` on success, `400` for a bad protocol version or a malformed body, `413` when the body cap is exceeded, `405` for a non-`POST` method, `401` @@ -152,10 +153,14 @@ in at ingest. ## Delivery Semantics -| Transport | Guarantee | Failure behaviour | -|-----------|-----------|-------------------| -| `tcp_chain` | Per-line, held across reconnects | Retries with exponential backoff plus ±20 % jitter until written or shutdown; back-pressure appears upstream as `total_dropped_by_sink` | -| `http_chain` | At-least-once per batch | Retries transport errors, `408`, `429`, `5xx` and, under `scram`, failed logins and `401`; drops on any other non-2xx (`dropped_batches`) | +- `tcp_chain`: per line, each held across reconnects. + - On failure it retries with exponential backoff plus ±20 % jitter until + the line is written or the sink shuts down. + - Back-pressure appears upstream as `total_dropped_by_sink`. +- `http_chain`: at least once per batch. + - It retries transport errors, `408`, `429`, `5xx` and, under `scram`, failed + logins and `401`. + - It drops the batch on any other non-2xx, counted in `dropped_batches`. `http_chain` batches can be delivered twice when a successful request's response is lost. There is no de-duplication downstream; design your consumers to @@ -266,5 +271,5 @@ optional, and entries are labelled with the username. See otherwise-idle links and their sessions warm. - **Ports** used by the bundled test scripts: `15801` tcp_chain ingest, `15802` http_chain ingest, `15803` tcp sink, `15804` http sink. -- **Use `127.0.0.1`, not `localhost`**, when testing locally: all listeners and - dialers are IPv4-only, and `localhost` may resolve to `::1`. +- **Dial the listener's own literal** when testing locally: a listener on + `127.0.0.1` takes IPv4 only, and `localhost` may resolve to `::1`. diff --git a/doc/cli.md b/doc/cli.md index f5ad107..5e5baa8 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -1,19 +1,21 @@ # Command Line Interface ``` -logwisp [options] -logwisp auth [flags] -logwisp help | -h | --help -logwisp --version +lw [options] +lw --check [options] +lw auth [flags] +lw help | -h | --help +lw --version ``` -`logwisp auth` manages SCRAM credentials and logs in to `scram` listeners; see -[below](#logwisp-auth). There is no certificate-generation command: use +`lw auth` manages SCRAM credentials and logs in to `scram` listeners; see +[below](#lw-auth). There is no certificate-generation command: use `openssl` or your PKI tooling — see [Security](security.md#enabling-mtls). ## Options -Any scalar configuration key is settable as a flag using its TOML path: +Any scalar configuration key is settable as a flag using its TOML path; +pipelines have [their own flags](#pipelines): ``` --= e.g. --logging.level=debug @@ -23,49 +25,92 @@ Any scalar configuration key is settable as a flag using its TOML path: ### Common -| Flag | Description | Default | -|------|-------------|---------| -| `-c ` | Configuration file | `./logwisp.toml` | -| `--config ` / `--config=` | Configuration file | `./logwisp.toml` | -| `--quiet` | Suppress all application output | `false` | -| `--status_reporter=` | Periodic status logging | `true` | -| `--auto_reload=` | Reload config when the file changes | `false` | -| `--version` | Print version and exit | — | -| `-h`, `--help`, `help` | Print usage and exit | — | +- `-c `, `--config `, `--config=`: the configuration file, + default `./logwisp.toml` +- `--check`: build every pipeline and plugin, bind nothing, report and exit; + see [Usage Patterns](#usage-patterns) +- `--quiet`: suppress all application output, default `false` +- `--status_reporter=`: periodic status logging, default `true` +- `--auto_reload=`: reload when the configuration file changes, default + `false` +- `--version`: print the version and exit +- `-h`, `--help`, `help`: print usage and exit The last file-selection flag wins. `-c=` also works. A missing or empty path returns an error, and `--` ends option parsing. ### Logging -| Flag | Values | -|------|--------| -| `--logging.output` | `file`, `stdout`, `stderr`, `split`, `all`, `none` | -| `--logging.level` | `debug`, `info`, `warn`, `error` | -| `--logging.format` | `raw`, `txt`, `json` | -| `--logging.sanitization` | `raw`, `json`, `txt`, `shell` | -| `--logging.file.directory` | path | -| `--logging.file.name` | string | -| `--logging.file.max_size_mb` | integer | -| `--logging.file.max_total_size_mb` | integer | -| `--logging.file.retention_hours` | float | +- `--logging.output`: `file`, `stdout`, `stderr`, `split`, `all` or `none` +- `--logging.level`: `debug`, `info`, `warn` or `error` +- `--logging.format`: `raw`, `txt` or `json` +- `--logging.sanitization`: `raw`, `json`, `txt` or `shell` +- `--logging.file.directory`: a path +- `--logging.file.name`: a string +- `--logging.file.max_size_mb`, `--logging.file.max_total_size_mb`: integers +- `--logging.file.retention_hours`: a float `--logging.console.target` is accepted but has no effect; the console destination is derived from `--logging.output`. ### Pipelines -Pipelines, sources, sinks, and filters **cannot** be configured from the command -line. Array-indexed paths such as `--pipelines.0.name=app` or -`--pipelines.0.plugin_sinks.0.type=null` are reported as unrecognized and -ignored: +Pipeline flags define whole pipelines. They replace the configuration file's +pipelines and the built-in default; every other key keeps its precedence. + +- `--pipeline NAME` starts a pipeline; specs before the first one go to a + pipeline named `cli` +- `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable +- `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline +- both `--flag SPEC` and `--flag=SPEC` work; `--` ends options + +A SPEC is a comma-separated list: + +- source, sink, filter and format specs start with a TYPE + - the plugin type: [Sources](sources.md), [Sinks](sinks.md) + - `include` or `exclude`: [Filters](filters.md) + - `json`, `txt` or `raw`: [Formatters](formatters.md) +- rate-limit and heartbeat specs have no TYPE +- the rest are `key=value` pairs with the TOML keys of the plugin's `config` + table or of the [flow stage](configuration.md#flow-stages) + - a dotted key reaches a nested table: `tls.cert_file=...`, `auth.type=scram` + - a repeated key makes a list: `patterns=ERROR,patterns=WARN` + - each value is one list entry, commas included: `auth.allow_patterns=^a{1\,3}$` + is one regex, `auth.allow=a,auth.allow=b` two entries + - values convert to the option's type: `port=8080`, `raw=true` +- `\` escapes `,`, `=` and `\` in a value; any other backslash stays, so regex + escapes such as `\d` pass unchanged +- `id=NAME` names a source or sink; the default is its TYPE, then `TYPE_2`, + `TYPE_3`, ... +- naming a stage turns it on: `--rate-limit` defaults `policy` to `drop`, + `--heartbeat` defaults `enabled` to `true` + +Errors name the flag and the offending part, and nothing starts: ``` -Warning: unrecognized flags ignored: [pipelines.0.name] +--sink http,port: missing "=" +--rate-limit rate=100,polcy=drop: unknown key "polcy" ``` -Use a configuration file. Older documentation described CLI pipeline overrides -that the current loader does not implement. +A misspelled plugin key fails when the plugin is built, before any listener +opens: `failed to create sink http: ... unknown key "tls.enabeld"`. + +```bash +# tail a directory, serve it over SSE +lw --source 'file,directory=/var/log/app,pattern=*.log' \ + --sink http,host=0.0.0.0,port=8080 + +# errors and warnings as JSON, over TLS with SCRAM logins +lw --source file,directory=/var/log/app \ + --filter include,patterns=ERROR,patterns=WARN --format json \ + --sink "http,port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\ +tls.enabled=true,tls.cert_file=/etc/logwisp/server.crt,tls.key_file=/etc/logwisp/server.key" + +# two pipelines +lw --pipeline app --source file,directory=/var/log/app --sink console \ + --pipeline relay --source tcp_chain,port=9000 \ + --sink file,directory=/var/log/relay,name=relay +``` ## Environment Variables @@ -83,28 +128,54 @@ forms when upgrading; see [Configuration](configuration.md#environment-variables The path resolver reads these variables directly: -| Variable | Effect | -|----------|--------| -| `LOGWISP_CONFIG_FILE` | Configuration file path; joined onto `LOGWISP_CONFIG_DIR` when both are set | -| `LOGWISP_CONFIG_DIR` | Configuration directory; alone, implies `/logwisp.toml` | +- `LOGWISP_CONFIG_FILE`: the configuration file path, joined onto + `LOGWISP_CONFIG_DIR` when both are set +- `LOGWISP_CONFIG_DIR`: the configuration directory; alone, it implies + `/logwisp.toml` -As with flags, array elements cannot be set this way. +### Pipeline Variables + +One pipeline can come from the environment, with the [SPEC](#pipelines) syntax. +Any pipeline flag on the command line makes LogWisp ignore all of them. + +- `LOGWISP_PIPELINE`: the pipeline's name, default `cli` +- `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each + - `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered + one in numeric order; likewise `LOGWISP_SINK_N` and `LOGWISP_FILTER_N` +- `LOGWISP_FORMAT`, `LOGWISP_RATE_LIMIT`, `LOGWISP_HEARTBEAT` +- an empty variable counts as unset + +A container needs no configuration file: + +```bash +docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \ + -e LOGWISP_PIPELINE=app \ + -e LOGWISP_SOURCE='file,directory=/logs,pattern=*.log' \ + -e LOGWISP_FILTER='exclude,patterns=DEBUG' \ + -e LOGWISP_SINK='http,host=0.0.0.0,port=8080' \ + -e LOGWISP_SINK_1=console \ + logwisp +``` ## Precedence -1. Command-line flags -2. Environment variables -3. Configuration file -4. Built-in defaults +- Pipelines + - pipeline flags, if any are given + - else pipeline variables, if any are set + - else the configuration file's `[[pipelines]]` + - else the built-in default +- Every other key + 1. command-line flags + 2. environment variables + 3. configuration file + 4. built-in defaults + +A reload rereads the file and keeps the command-line or environment pipelines. ## Signals -| Signal | Action | -|--------|--------| -| `SIGINT` | Graceful shutdown | -| `SIGTERM` | Graceful shutdown | -| `SIGHUP` | Reload configuration | -| `SIGUSR1` | Reload configuration | +- `SIGINT`, `SIGTERM`: graceful shutdown +- `SIGHUP`, `SIGUSR1`: reload the configuration `SIGHUP` is ignored during startup, before the signal handler is installed, so LogWisp survives a terminal hang-up like `nohup`. Once running, it triggers a @@ -115,11 +186,10 @@ service untouched; see [Configuration](configuration.md#hot-reload). ## Exit Codes -| Code | Meaning | -|------|---------| -| `0` | Clean shutdown, or `--version` / `--help` | -| `1` | General error: config load or validation failure, logger init failure, service bootstrap failure | -| `2` | Explicitly requested configuration file not found | +- `0`: clean shutdown, `--version` / `--help`, or a valid `--check` +- `1`: general error: a configuration load or validation failure (`--check` + included), a logger init failure, a service bootstrap failure +- `2`: an explicitly requested configuration file is not found Exit code 2 applies only when the file was named explicitly (`-c`, `--config=`, or the `LOGWISP_CONFIG_*` variables). A missing discovered default @@ -133,8 +203,9 @@ a rate limit of 5 entries/second with a burst of 10 and `policy = "drop"`, and a `console` sink on stdout. It is a self-demonstrating idle mode, not a useful production configuration. -Note that as soon as your file defines `[[pipelines]]`, that entire default -pipeline — rate limit included — is replaced rather than merged. +As soon as the file defines `[[pipelines]]`, or pipeline flags or variables are +given, that entire default pipeline — rate limit included — is replaced rather +than merged. ## Usage Patterns @@ -142,26 +213,29 @@ pipeline — rate limit included — is replaced rather than merged. ```bash # verbose, everything to stderr -logwisp -c dev.toml --logging.output=stderr --logging.level=debug +lw -c dev.toml --logging.output=stderr --logging.level=debug # no config at all: synthetic generator to stdout -logwisp +lw ``` **Configuration check** -```bash -# starts the service; a config error exits non-zero before any pipeline runs -logwisp -c /etc/logwisp/logwisp.toml --logging.level=debug -``` +`lw --check` validates without running: it builds every pipeline and plugin as +a start would (options, TLS files, credentials files, startup warnings), but +binds no port and opens, reads or creates no log file; it prints +`configuration ok` and exits 0, or the error and exits 1 (2 when a named file +is missing, see [Exit Codes](#exit-codes)): -There is no dry-run or validate-only mode. The closest approximation is starting -with debug logging and stopping once the pipelines report as started. +```bash +lw --check -c /etc/logwisp/logwisp.toml +lw --check --source file,directory=/var/log/app --sink http,port=8080 +``` **Production** ```bash -logwisp -c /etc/logwisp/logwisp.toml --logging.output=file +lw -c /etc/logwisp/logwisp.toml --logging.output=file ``` Run under a supervisor (systemd, rc.d) rather than backgrounding it — there is @@ -171,29 +245,38 @@ no `--background` flag; earlier releases had one and it was removed. See **Reload** ```bash -kill -HUP $(pidof logwisp) -kill -USR1 $(pidof logwisp) +kill -HUP $(pidof lw) +kill -USR1 $(pidof lw) ``` -## `logwisp auth` +## `lw auth` Manages the credentials files of `auth.type = "scram"` listeners and logs in to them as a viewer. See [Password Authentication](security.md#password-authentication-scram). -| Command | Does | -|---------|------| -| `add-user -credentials FILE -user NAME [-password-file FILE] [-generate]` | Adds a user, or replaces its password | -| `remove-user -credentials FILE -user NAME` | Removes a user; refuses the last one | -| `token -url https://HOST:PORT[/PATH] -user NAME -password-file FILE [-unbound] [TLS flags]` | Logs in to an `http` sink or `http_chain` source and prints a bearer token | -| `stream -addr HOST:PORT -user NAME -password-file FILE [TLS flags]` | Logs in to a `tcp` sink and copies its stream to stdout until interrupted | +``` +lw auth add-user -credentials FILE -user NAME [-password-file FILE] [-generate] +lw auth remove-user -credentials FILE -user NAME +lw auth token -url https://HOST:PORT[/PATH] -user NAME -password-file FILE + [-unbound] [TLS flags] +lw auth stream -addr HOST:PORT -user NAME -password-file FILE [TLS flags] +``` -`logwisp auth -h` lists a command's flags. The exit status is `0` on +- `add-user`: adds a user, or replaces its password +- `remove-user`: removes a user; refuses the last one +- `token`: logs in to an `http` sink or `http_chain` source and prints a + bearer token +- `stream`: logs in to a `tcp` sink and copies its stream to stdout until + interrupted + +`lw auth -h` lists a command's flags. The exit status is `0` on success, `1` on failure and `2` on a usage error. **`add-user`** creates the credentials file, with a fresh `decoy_key`, when it -does not exist or is empty (create it empty first to choose its owner). The password comes from `-password-file` when that file exists -(at least 8 bytes; one trailing line break is trimmed). Otherwise a random +does not exist or is empty (create it empty first to choose its owner). The +password comes from `-password-file` when that file exists (at least 8 bytes; +one trailing line break is trimmed). Otherwise a random 26-character password (130 bits) is generated and written to `-password-file`, or printed once to stdout when there is none. Replacing an existing user's password needs an existing `-password-file` or `-generate`, so a mistyped path @@ -215,29 +298,37 @@ There is no flag to skip verification: an unverified server could relay the login. Redirects are not followed. `-unbound` logs in to an `http` sink behind a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's certificate across the two requests; only then may `-url` carry the path the -proxy mounts LogWisp at. `stream` exits `0` -on `SIGINT` or `SIGTERM`, and `1` when the server ends the stream (a reload or -shutdown). +proxy mounts LogWisp at. `stream` exits `0` on `SIGINT` or `SIGTERM`, and `1` +when the server ends the stream (a reload or shutdown). + +Addresses, as for the plugins ([Networking](networking.md#address-family)): + +- An IPv6 address goes in brackets: `-addr [::1]:PORT`, + `-url https://[::1]:PORT`; in a URL a link-local zone is escaped, + `https://[fe80::1%25eth0]:PORT`. +- The dial keeps to the address's family; a hostname resolves. +- `-server-name` defaults to the host; an address, without brackets or zone, + must then be among the certificate's IP SANs. ```bash # listener host: a file the service user can read, users, then apply install -m 0640 -o root -g logwisp /dev/null /etc/logwisp/users.toml -logwisp auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \ +lw auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \ -password-file /etc/logwisp/edge-01.pass -logwisp auth add-user -credentials /etc/logwisp/users.toml -user viewer \ +lw auth add-user -credentials /etc/logwisp/users.toml -user viewer \ -password-file viewer.pass -kill -HUP $(pidof logwisp) +kill -HUP $(pidof lw) # rotate: new password into the file; SIGHUP, deploy the file, SIGHUP the edge -logwisp auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \ +lw auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \ -password-file /etc/logwisp/edge-01.pass -generate # http sink status, with the token kept out of curl's argv curl --cacert ca.crt -H @<(printf 'Authorization: Bearer %s\n' \ - "$(logwisp auth token -url https://HOST:PORT -user viewer \ + "$(lw auth token -url https://HOST:PORT -user viewer \ -password-file viewer.pass -ca-file ca.crt)") https://HOST:PORT/status # follow a tcp sink -logwisp auth stream -addr HOST:PORT -user viewer -password-file viewer.pass \ +lw auth stream -addr HOST:PORT -user viewer -password-file viewer.pass \ -ca-file ca.crt ``` diff --git a/doc/configuration.md b/doc/configuration.md index 06546e1..e682e08 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -5,16 +5,18 @@ and its default ships as [`config/logwisp.toml`](../config/logwisp.toml). ## Configuration Precedence -Sources are merged in this order, highest priority first: - -1. Command-line flags -2. Environment variables -3. Configuration file -4. Built-in defaults - -The `pipelines` array is replaced wholesale, not merged: as soon as your file -defines `[[pipelines]]`, the built-in default pipeline (and its default rate -limit and formatter) disappears entirely. +- Pipelines come from the first of these that defines any, replacing the rest + wholesale (the default pipeline's rate limit and formatter included): + 1. pipeline flags: `--source`, `--sink`, ... ([CLI](cli.md#pipelines)) + 2. pipeline variables: `LOGWISP_SOURCE`, ... + ([CLI](cli.md#pipeline-variables)) + 3. the file's `[[pipelines]]` + 4. the built-in default pipeline +- Every other key merges, highest priority first: + 1. command-line flags + 2. environment variables + 3. configuration file + 4. built-in defaults ## File Location @@ -36,11 +38,14 @@ runtime metadata and cannot be redirected by a `config_file` key inside the file ## Global Settings -| Setting | Type | Default | Description | -|---------|------|---------|-------------| -| `quiet` | bool | `false` | Disable all application logging and console diagnostics | -| `status_reporter` | bool | `true` | Emit a periodic status report every 30 s at DEBUG level | -| `auto_reload` | bool | `false` | Watch the config file and reload pipelines on change | +Top-level keys, each as type and default: + +- `quiet` (bool, `false`): disable all application logging and console + diagnostics. +- `status_reporter` (bool, `true`): emit a status report every 30 s at DEBUG + level. +- `auto_reload` (bool, `false`): watch the config file and reload pipelines on + change. `--version` prints version information and exits; it is not a persistent setting. @@ -69,14 +74,12 @@ retention_hours = 168.0 ### Output modes -| Mode | Behaviour | -|------|-----------| -| `file` | Files only | -| `stdout` | Standard output only | -| `stderr` | Standard error only | -| `split` | DEBUG/INFO to stdout, WARN/ERROR to stderr | -| `all` | Files plus split console | -| `none` | No application logging | +- `file`: files only. +- `stdout`: standard output only. +- `stderr`: standard error only. +- `split`: DEBUG/INFO to stdout, WARN/ERROR to stderr. +- `all`: files plus split console. +- `none`: no application logging. `[logging.file]` applies only to the `file` and `all` modes. @@ -131,32 +134,34 @@ port = 8080 Every source and sink is a plugin instance with three keys: -| Key | Meaning | -|-----|---------| -| `id` | Instance identifier, unique within the pipeline; appears in logs and stats | -| `type` | Registered plugin type | -| `config` | Plugin-specific table; see [Sources](sources.md) and [Sinks](sinks.md) | +- `id`: instance identifier, unique within the pipeline; appears in logs and + stats. +- `type`: registered plugin type. +- `config`: plugin-specific table; see [Sources](sources.md) and + [Sinks](sinks.md). `config_file` is reserved on both structures for a future include mechanism and is not implemented. ### Flow stages -| Block | Optional | Reference | -|-------|----------|-----------| -| `flow.rate_limit` | yes | below | -| `flow.filters` | yes | [Filters](filters.md) | -| `flow.format` | yes (defaults to `raw`) | [Formatters](formatters.md) | -| `flow.heartbeat` | yes | below | +Every stage is optional: + +- `flow.rate_limit`: see [Rate limiting](#rate-limiting). +- `flow.filters`: see [Filters](filters.md). +- `flow.format`: defaults to `raw`; see [Formatters](formatters.md). +- `flow.heartbeat`: see [Heartbeat](#heartbeat). #### Rate limiting -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `rate` | float | `0` | Entries per second; `<= 0` disables the limiter entirely | -| `burst` | float | `rate` | Token bucket capacity | -| `policy` | string | `pass` | `pass` allows everything through, `drop` discards over-limit entries | -| `max_entry_size_bytes` | int | `0` | Per-entry byte cap; `0` = unlimited | +Options, each as type and default: + +- `rate` (float, `0`): entries per second; `<= 0` disables the limiter + entirely. +- `burst` (float, `rate`): token bucket capacity. +- `policy` (string, `pass`): `pass` allows everything through, `drop` discards + over-limit entries. +- `max_entry_size_bytes` (int, `0`): per-entry byte cap; `0` = unlimited. Two behaviours are easy to trip over: @@ -167,13 +172,16 @@ Two behaviours are easy to trip over: #### Heartbeat -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `enabled` | bool | `false` | Enable heartbeat generation | -| `interval_ms` | int | `1000` | Interval; minimum `100` | -| `include_timestamp` | bool | `false` | `false` formats with level only, no timestamp | -| `include_stats` | bool | `false` | Attach `beat_count` and measured `interval_ms` as fields | -| `format` | string | `txt` | `txt`, `json`, or `raw` | +Options, each as type and default: + +- `enabled` (bool, `false`): enable heartbeat generation. +- `interval_ms` (int, `1000`): interval; minimum `100`. +- `include_timestamp` (bool, `false`): `false` formats with level only, no + timestamp. +- `include_stats` (bool, `false`): attach `beat_count` and measured + `interval_ms` as fields. +- `format` (string, `txt`): `txt`, `json` or `raw` are accepted but have no + effect: heartbeats go through the flow's `format`, like every entry. Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They are generated after the flow's filter and rate-limit stages, so filters do not @@ -186,14 +194,14 @@ suppress them, and they reach every sink in the pipeline. ## Environment Variables Environment overrides are derived from the TOML path: `.` becomes `_`, the -result is uppercased, and `LOGWISP_` is prepended. +result is uppercased, and `LOGWISP_` is prepended: -| TOML path | Environment variable | -|-----------|---------------------| -| `quiet` | `LOGWISP_QUIET` | -| `status_reporter` | `LOGWISP_STATUS_REPORTER` | -| `logging.level` | `LOGWISP_LOGGING_LEVEL` | -| `logging.file.directory` | `LOGWISP_LOGGING_FILE_DIRECTORY` | +``` +quiet LOGWISP_QUIET +status_reporter LOGWISP_STATUS_REPORTER +logging.level LOGWISP_LOGGING_LEVEL +logging.file.directory LOGWISP_LOGGING_FILE_DIRECTORY +``` Migration: the old custom transform accidentally read bare names such as `QUIET` and `LOGGING_LEVEL`. Rename those variables to their prefixed forms; bare names @@ -201,16 +209,17 @@ are now ignored. `LOGWISP_CONFIG_FILE` and `LOGWISP_CONFIG_DIR` still select the file directly. Only scalar paths that exist in the configuration schema can be set this way. -Array elements cannot: `LOGWISP_PIPELINES_0_NAME` has no effect. +Array elements cannot: `LOGWISP_PIPELINES_0_NAME` has no effect. A whole +pipeline can: see [pipeline variables](cli.md#pipeline-variables). ## Command-Line Overrides Any scalar configuration path is settable as a flag using its TOML path: ```bash -logwisp --logging.level=debug --status_reporter=false -logwisp --logging.level debug # space form also works -logwisp --quiet # bare flag means true +lw --logging.level=debug --status_reporter=false +lw --logging.level debug # space form also works +lw --quiet # bare flag means true ``` Unrecognized flags are reported on stderr before the logger exists and are then @@ -220,11 +229,8 @@ ignored: Warning: unrecognized flags ignored: [pipelines.0.name] ``` -> Array-indexed paths are **not** settable from the command line. -> `--pipelines.0.name=x`, `--pipelines.0.plugin_sinks.0.type=null`, and similar -> flags are reported as unrecognized and ignored. Pipelines, sources, sinks, and -> filters can only be defined in the configuration file. Older documentation -> claimed otherwise. +Array-indexed paths such as `--pipelines.0.name=x` are unrecognized; whole +pipelines have [their own flags](cli.md#pipelines). ## Validation @@ -236,6 +242,8 @@ Startup validation is intentionally split. `config` table: `config file "…": unknown key "pipelines[0].plugin_sinks[0].confg"`. A misspelled table path would otherwise drop the whole table it heads. A top-level `config_file` is ignored +- pipeline [specs](cli.md#pipelines): syntax, and unknown flow-stage keys such + as `--rate-limit rate=1,polcy=drop` - at least one pipeline - unique, non-empty pipeline names - at least one source and one sink per pipeline @@ -265,7 +273,11 @@ auto_reload = true or send `SIGHUP` / `SIGUSR1`. Signals reread the selected file even when watching is disabled and always rebuild, allowing certificate and credentials rotation without TOML edits; `auto_reload` watches only the configuration file. -CLI and environment overrides are captured at startup and keep their precedence. +CLI and environment overrides, pipelines included, are captured at startup and +keep their precedence. As at startup, a discovered default file that does not +exist is not an error, so an instance configured by flags or variables alone +still rebuilds. Once read, though, its removal fails the reload and the running +service stays as it is; restart to run without the file. Reload rebuilds the whole service: a new service is constructed from the new configuration first, and only if that succeeds is the old one shut down. Each @@ -279,25 +291,30 @@ without rebuilding. Queued path changes are combined, and unchanged pipelines and status settings do not trigger another rebuild. Removed file keys fall back to the remaining sources on the next successful load. -| Reloaded | Not reloaded | -|----------|--------------| -| Pipelines, sources, sinks | `logging.*` (applied once at startup) | -| Filters, formatters, rate limits, heartbeats | `quiet` | -| `status_reporter` | `auto_reload` (the watcher is not restarted) | +What a reload applies: + +- Reloaded: + - pipelines, sources, sinks + - filters, formatters, rate limits, heartbeats + - `status_reporter` +- Not reloaded: + - `logging.*`: applied once at startup + - `quiet` + - `auto_reload`: the watcher is not restarted Because the rebuild is total, listeners close and reopen and every connected client is disconnected. Chain sinks reconnect on their own backoff schedule. ## Type Reference -| TOML type | Go type | Command-line / environment form | -|-----------|---------|-------------------------------| -| String | `string` | Plain text | -| Integer | `int64` | Decimal string | -| Float | `float64` | Decimal string | -| Boolean | `bool` | `true` / `false`, or a bare flag for `true` | -| Array | `[]T` | Not settable outside the file | -| Table | struct | Nested path with `.` (flags) or `_` (environment) | +Each TOML type, its Go type, and its command-line / environment form: + +- String (`string`): plain text. +- Integer (`int64`): decimal string. +- Float (`float64`): decimal string. +- Boolean (`bool`): `true` / `false`, or a bare flag for `true`. +- Array (`[]T`): only in [pipeline specs](cli.md#pipelines), as a repeated key. +- Table (struct): nested path with `.` (flags) or `_` (environment). Integer fields reject fractions, overflow and negative unsigned values. Non-finite floats and integer-to-float precision loss are rejected too. This applies to plugin diff --git a/doc/deployment.md b/doc/deployment.md new file mode 100644 index 0000000..3510e2a --- /dev/null +++ b/doc/deployment.md @@ -0,0 +1,198 @@ +# Deployment + +`deploy/lw-deploy.sh` configures and deploys one LogWisp node per run. It asks +for what its flags leave open, prints a plan with the equivalent non-interactive +command, then writes the files, runs `lw --check` on them (the node's binary or +image, with its mounts) and starts the node. `--dry-run` stops at printing; +`--yes` never asks and fails on a missing required value. It creates no +certificates: see [Security](security.md#enabling-mtls). + +## Roles + +- `edge` + - tails each `--log-dir` (not recursive) for `--log-pattern` (default `*.log`) + - forwards to `--aggregator` over `tcp_chain` (port 9440) or, with + `--chain http`, `http_chain` (port 9441) + - labels its entries `--node`: the short host name, or the jail name +- `aggregator` + - one chain listener on `--listen` (default `0.0.0.0`) and `--chain-port` + - outputs, at least one: `--file-dir` (rotating `--file-name`.log), + `--http-port` (SSE `/stream`, JSON `/status`), `--tcp-port` + - `--format` `txt` (default), `json` or `raw` +- `standalone` + - the edge's sources and the aggregator's outputs, no chain + +## Runtimes + +- `docker`, `podman` (Linux) + - the container runs `--read-only --cap-drop ALL --security-opt + no-new-privileges -u 65532:65532 --restart unless-stopped` + - `--config-dir` (default `/etc/logwisp/NAME`) holds `logwisp.toml`, `tls/`, + and the credentials files, owned by 65532 and mounted read-only at + `/etc/logwisp` + - log directories mount read-only and the output directory writable, each at + its host path; `--log-group` adds that group's id to the container + - listener ports publish on `--listen`, an IP address; the process inside + binds `0.0.0.0` + - `--network NET` joins a user network, created when missing, where edges + reach an aggregator by container name; `host` publishes nothing + - `--image` defaults to `logwisp:dev`, the `make image` tag; `--build` builds + it from the repository. Behind an HTTPS-intercepting proxy add + `--build-ca FILE` and, for a proxy on loopback, + `--build-flags "--network host"`; the environment's proxy variables pass + through as build arguments + - rootful engines need root to give the files to 65532; rootless podman maps + the caller to 65532 with `--userns keep-id` + - SELinux hosts must label the mounted directories for containers; the + script does not relabel, since `:z` on a shared directory such as + `/var/log` would relabel it for every other service too +- `native`, Linux + - needs systemd; installs `lw` to `/usr/local/bin` unless one is installed + (`--bin FILE`, or `--build` for `make build`), and + `deploy/package/logwisp.service`, `logwisp.sysusers` and + `logwisp.tmpfiles` under `/etc` + - a package's unit (`/usr/lib/systemd/system/logwisp.service`) is used as it + is, with the `lw` its `ExecStart` runs; `--bin` and `--build` then stop the + run, since the service would never run that binary + - writes `/etc/logwisp/logwisp.toml`, `root:logwisp`, mode 0640 + - refuses log and output directories under `/tmp` and `/var/tmp`: the unit's + `PrivateTmp=yes` gives the service its own, empty ones + - adds `logwisp.service.d/deploy.conf` when the node needs it: + `ReadWritePaths` for an output outside `/var/log/logwisp`, + `ProtectHome=read-only` for logs or output under `/home`, + `SupplementaryGroups` for `--log-group`, `CAP_NET_BIND_SERVICE` for a port + below 1024 +- `native`, FreeBSD + - installs `deploy/package/logwisp.rc` as `/usr/local/etc/rc.d/logwisp`, + creates the `logwisp` user with `pw`, enables it with `sysrc`; LogWisp logs + to syslog under the tag `logwisp` + - writes `/usr/local/etc/logwisp/logwisp.toml` + - `--jail NAME` writes the files and runs the commands through `jexec`, so + the jail's own symlinks cannot redirect a write onto the host; the jail + must be running, and the script prints a `jail.conf` example when it is not + - a non-root user binds a port below 1024 only with + `sysctl net.inet.ip.portrange.reservedhigh` lowered or `mac_portacl` +- `manual` + - writes into `--config-dir`, an absolute path (default `~/logwisp-ROLE`), + adds users with the local `lw`, and prints the `lw -c` command + +FreeBSD runs no Docker, and podman there runs FreeBSD images only while the +LogWisp image is Linux: on FreeBSD the script offers `native` instead, on the +host or in a jail. A `native` install needs a host of its `--os`; for another +one, `--dry-run` shows the install and `--runtime manual` writes the +configuration. + +Re-running the same command updates the node: an existing `logwisp.toml` is +kept as `logwisp.toml.bak`, the container is replaced, generated passwords are +reused. A user already in a credentials file whose password file is missing +stops the run before any change, instead of getting a new password that would +lock its edge out. Under `--yes`, replacing a configuration or a container +needs `--force`. + +## Security options + +- `--tls` secures every network link of the node; the files are copied into + the configuration directory as `tls/node.crt`, `tls/node.key`, `tls/ca.crt` + - listeners present `--cert-file` and `--key-file` + - an edge verifies the aggregator with `--ca-file`, or the system roots; the + certificate must name the address dialed (an IP SAN for an IP), or the + name given as `--server-name` +- `--auth` on the chain link + - `mtls`: edges present `--cert-file`; the aggregator admits the `--allow` + CNs (none: any certificate the CA issued); an edge's `--allow` pins the + aggregator's CN + - `scram`: the aggregator creates each `--add-user`, its password generated + into `--secrets-dir`, an absolute path (default `~/logwisp-secrets`), and + handed to `lw auth add-user`, natively or through the image; copy each + password file to its edge as `--password-file`, whose name without + `.pass` is the default `--username` + - either binds the node label to the authenticated identity +- `--sink-auth` on the `http` and `tcp` outputs + - `mtls` with `--sink-allow`, or `scram` with `--add-viewer` (a separate + `viewers.toml`); viewers log in with `lw auth token` and `lw auth stream` + - without it the outputs admit anyone who reaches them: bind `--listen` to a + trusted address +- without `--auth` or `--sink-auth`, the auth flags imply their mode: + `--allow` and `--sink-allow` mtls; `--add-user`, `--username`, + `--password-file` and `--add-viewer` scram. Under `--yes` one that does not + apply, such as `--add-user` with `--auth none` or without `--tls`, stops the + run rather than deploy the node open + +## FreeBSD host, jails and one aggregator + +Application jails run an edge each; a `logs` jail aggregates. Every jail runs +already, and the `logs` jail has an address the others reach (`AGG_ADDR`). + +1. Certificates: a CA, and a server certificate for `logs` naming `AGG_ADDR` + ([Security](security.md#enabling-mtls)). +2. The aggregator, from the host: + + ```sh + deploy/lw-deploy.sh --yes --runtime native --jail logs --role aggregator \ + --file-dir /var/log/logwisp --http-port 8080 \ + --tls --cert-file logs.crt --key-file logs.key \ + --auth scram --add-user web1 --add-user db1 \ + --sink-auth scram --add-viewer ops + ``` + + The passwords land in `~/logwisp-secrets/`: `web1.pass`, `db1.pass` and + `ops.pass`. +3. An edge per application jail: + + ```sh + deploy/lw-deploy.sh --yes --runtime native --jail web1 --role edge \ + --log-dir /var/log/nginx --aggregator AGG_ADDR \ + --tls --ca-file ca.crt --auth scram --password-file ~/logwisp-secrets/web1.pass + ``` + +4. Check: `jexec logs service logwisp status`, the files in the `logs` jail's + `/var/log/logwisp`, and `https://AGG_ADDR:8080/status` with a token from + `lw auth token -url https://AGG_ADDR:8080 -user ops -password-file + ~/logwisp-secrets/ops.pass -ca-file ca.crt` ([CLI](cli.md#lw-auth)). + +Variants: + +- the aggregator on the host itself: drop `--jail logs` +- no daemon per jail: one `standalone` or `edge` on the host tails each jail's + logs through the host path (`--log-dir JAIL_PATH/var/log`); the entries then + carry the host's label and the bare file name, so files of the same name in + two jails cannot be told apart +- a Linux aggregator: run step 2 with `--runtime docker` on that host and point + the edges at it; the chain protocol is the same + +## Linux host with containers + +An aggregator container and an edge container per application, on one user +network: + +```sh +deploy/lw-deploy.sh --yes --runtime docker --role aggregator --network logwisp \ + --file-dir /srv/logwisp --http-port 8080 --listen 127.0.0.1 +deploy/lw-deploy.sh --yes --runtime docker --role edge --name logwisp-edge-app \ + --network logwisp --aggregator logwisp-aggregator --log-dir /var/log/app +``` + +`--listen 127.0.0.1` keeps the published ports on loopback; edges on other +hosts need it on an address they reach. `docker kill -s HUP NAME` reloads an +edited configuration or credentials file. + +## Not covered + +Edit the generated `logwisp.toml` for these; `config/logwisp.toml` documents +every key. + +- filters, rate limits, heartbeat, formatter flags and timestamp layouts +- more than one pipeline; both chain transports, or a further hop, on one node +- proxy mode: browser logins through a TLS-terminating proxy +- network ACLs: not implemented in LogWisp yet +- mtls identities other than the certificate CN; `node_binding` other than + `force` +- creating certificates +- several native instances on one host: use one per host or jail + +## Testing the script + +`test/deploy-test.sh --auto` drives the script with `--yes` and `--dry-run` for +the manual, native and docker runtimes, runs the generated edge, aggregator and +standalone nodes end to end, and checks that `--yes` fails closed and that +`--dry-run` changes nothing. diff --git a/doc/filters.md b/doc/filters.md index 1cf37bf..f4d909c 100644 --- a/doc/filters.md +++ b/doc/filters.md @@ -12,11 +12,11 @@ patterns = ["ERROR", "WARN"] ## Options -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `type` | string | `include` | `include` (only matches pass) or `exclude` (matches are dropped) | -| `logic` | string | `or` | `or` (any pattern matches) or `and` (every pattern matches) | -| `patterns` | []string | `[]` | Go RE2 regular expressions | +- `type`, string, default `include`: `include` (only matches pass) or + `exclude` (matches are dropped). +- `logic`, string, default `or`: `or` (any pattern matches) or `and` (every + pattern matches). +- `patterns`, []string, default `[]`: Go RE2 regular expressions. A filter with no patterns passes everything. Invalid patterns fail at startup with the filter index and the offending pattern in the message. @@ -33,10 +33,8 @@ Empty parts are omitted, so an entry with no detected level matches `" "`. This means a pattern can target the source name or the level as easily as the message body: -| Pattern | Matches | -|---------|---------| -| `"^app\\.log "` | Entries whose source is `app.log` | -| `"ERROR"` | Level `ERROR`, or the word `ERROR` anywhere in the message | +- `"^app\\.log "` matches entries whose source is `app.log`. +- `"ERROR"` matches level `ERROR`, or the word `ERROR` anywhere in the message. The structured `fields` payload is **not** part of the match text. @@ -121,15 +119,13 @@ so later ones evaluate fewer entries. Go's RE2 syntax. No backreferences and no lookaround — RE2 guarantees linear time, which is exactly what you want in a log hot path. -| Need | Pattern | -|------|---------| -| Literal substring | `ERROR` | -| Case-insensitive | `(?i)error` | -| Whole word | `\\berror\\b` | -| Alternation | `ERROR\|WARN\|FATAL` | -| Character class | `[0-9]{3}` | -| Anchors | `^ERROR`, `ERROR$` | -| Any characters | `.*exception.*` | +- Literal substring: `ERROR` +- Case-insensitive: `(?i)error` +- Whole word: `\berror\b` +- Alternation: `ERROR|WARN|FATAL` +- Character class: `[0-9]{3}` +- Anchors: `^ERROR`, `ERROR$` +- Any characters: `.*exception.*` Remember that TOML basic strings process escapes, so a regex backslash needs doubling: `"\\berror\\b"`. TOML literal strings avoid the issue: @@ -137,7 +133,7 @@ doubling: `"\\berror\\b"`. TOML literal strings avoid the issue: Anchors apply to the assembled match text, which begins with the source name — so `^ERROR` will not match an entry whose source is non-empty. Use -`\\bERROR\\b` instead unless you mean to anchor on the source. +`\bERROR\b` instead unless you mean to anchor on the source. ## Common Recipes diff --git a/doc/formatters.md b/doc/formatters.md index 335d6fe..a609cd3 100644 --- a/doc/formatters.md +++ b/doc/formatters.md @@ -19,12 +19,14 @@ Omitting `[pipelines.flow.format]` entirely selects `raw`. ## Options -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `type` | string | `raw` | `raw`, `txt` (alias `text`), or `json` | -| `sanitizer_policy` | string | derived from `type` | `raw`, `txt`, `json`, or `shell` | -| `flags` | int64 | `0` | Bitmask override; `0` selects a per-type default | -| `timestamp_format` | string | formatter default | Go reference layout, e.g. `"2006-01-02T15:04:05Z07:00"` | +Each as type and default: + +- `type` (string, `raw`): `raw`, `txt` (alias `text`), or `json`. +- `sanitizer_policy` (string, derived from `type`): `raw`, `txt`, `json`, or + `shell`. +- `flags` (int64, `0`): bitmask override; `0` selects a per-type default. +- `timestamp_format` (string, `time.RFC3339Nano`): Go reference layout, e.g. + `"2006-01-02T15:04:05Z07:00"`. ## Types @@ -86,33 +88,37 @@ LogWisp's own configuration surface. `flags` is a bitmask passed to the underlying formatter. Leave it at `0` unless you need to override the defaults. -| Value | Name | Effect | -|-------|------|--------| -| `1` | Raw | Bypass formatting and sanitization entirely | -| `2` | ShowTimestamp | Emit the timestamp | -| `4` | ShowLevel | Emit the level | -| `8` | StructuredJSON | Render attached fields as a JSON object | -| `16` | NoTimestamp | Suppress the timestamp | -| `32` | NoLevel | Suppress the level | +- `1` Raw: bypass formatting and sanitization entirely. +- `2` ShowTimestamp: emit the timestamp. +- `4` ShowLevel: emit the level. +- `8` StructuredJSON: render attached fields as a JSON object. +- `16` NoTimestamp: suppress the timestamp. +- `32` NoLevel: suppress the level. With `flags = 0` the formatter selects `1` for `type = "raw"` and `6` (timestamp + level) for every other type. `8` is added automatically whenever an entry carries parseable `fields` and `1` is not set; `1` always wins. -Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp -only, no level. +`txt` and `json` show timestamp and level unless a suppress bit says otherwise +(suppress wins over show), so `2` and `4` alone change nothing. Examples: +`flags = 16` for level only, no timestamp; `flags = 32` for timestamp only, no +level. ## Sanitizer Policies The sanitizer runs before serialization and neutralizes control characters that would otherwise break framing or reach a terminal. -| Policy | Behaviour | Use with | -|--------|-----------|----------| -| `raw` | No-op passthrough | `type = "raw"` where you control the data | -| `txt` | Escapes non-printable characters | File and console sinks | -| `json` | Escapes control characters for safe JSON embedding | `type = "json"`, chain links | -| `shell` | Strips shell metacharacters, whitespace, and control characters | Data that will be passed to a command | +Each policy, then what to use it with: + +- `raw`: no-op passthrough. + - `type = "raw"` where you control the data. +- `txt`: escapes non-printable characters. + - File and console sinks. +- `json`: escapes control characters for safe JSON embedding. + - `type = "json"`. +- `shell`: strips shell metacharacters, whitespace, and control characters. + - Data that will be passed to a command. When `sanitizer_policy` is omitted, the policy is derived from `type`: `json` for `json`, `txt` for `txt`/`text`, and `raw` for anything else — so the safe @@ -129,7 +135,8 @@ message. ## Node Identity in Output Entries that arrived over a chain link carry a `Node` label. The formatter -renders it as a syslog-style prefix on the source field: +renders it as a syslog-style prefix on the source field (`trace` in `json` +output): ``` edge-01/app.log @@ -149,13 +156,16 @@ from the heartbeat generator when `include_stats = true`. ## Choosing a Configuration -| Goal | Configuration | -|------|---------------| -| Maximum throughput, data already formatted | `type = "raw"` | -| Human reading in a terminal or file | `type = "txt"`, `sanitizer_policy = "txt"` | -| Downstream ingestion (Loki, Elasticsearch, jq) | `type = "json"`, `sanitizer_policy = "json"` | -| Compact console output | `type = "txt"`, `flags = 4` | -| Untrusted log content | never `raw`; pick `txt` or `json` and set the matching policy | +By goal: + +- Maximum throughput, data already formatted: `type = "raw"`. +- Human reading in a terminal or file: `type = "txt"`, + `sanitizer_policy = "txt"`. +- Downstream ingestion (Loki, Elasticsearch, jq): `type = "json"`, + `sanitizer_policy = "json"`. +- Compact console output: `type = "txt"`, `flags = 16`. +- Untrusted log content: never `raw`; pick `txt` or `json` and set the + matching policy. ## Formatting and Chain Links diff --git a/doc/installation.md b/doc/installation.md index e2349b5..a86c588 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -2,231 +2,304 @@ ## Requirements -- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+) -- **Architecture**: amd64 -- **Go**: 1.27.1 or newer, to build from source +- Operating systems: Linux (kernel 6.10+), FreeBSD (14.0+) +- Architecture: amd64; arm64 builds but is untested +- To build: Go 1.27.1 or newer (FreeBSD: the `go127` package), and GNU make + or BSD make +- For `make e2e`: bash 5+, coreutils, curl, openssl and go (the proxy test + builds its reverse proxy). Optional, else those checks SKIP: node with + playwright and the Chromium builds `playwright install chromium` fetches + (browser checks), an IPv6 loopback (`ipv6-test.sh`), docker as root with the + `logwisp:dev` image (`deploy-test.sh`'s container run) ## Building from Source ```bash git clone https://github.com/lixenwraith/logwisp.git cd logwisp -make -sudo make install # installs to $PREFIX/bin, default /usr/local/bin +make build # bin/lw; plain `make` only lists the targets ``` -The Makefile works with both GNU make and BSD make. Targets: +The Makefile works with GNU make and BSD make alike. Targets: -| Target | Effect | -|--------|--------| -| `make` / `make build` | Build `bin/logwisp` with version metadata | -| `make dev` | Build with the race detector enabled | -| `make install` | Install the binary to `$(PREFIX)/bin` (default `/usr/local`) | -| `make uninstall` | Remove `$(BINDIR)/logwisp` | -| `make clean` | Remove the built binary | -| `make version` | Print the version, commit, and build time that would be embedded | +- Build + - `make build`: `bin/lw`, with version, commit and build time from git + - `make release`: the same, static (`CGO_ENABLED=0`), `-trimpath`, stripped + - `make dev`: built with the race detector + - `make version`: the metadata a build would embed + - `make clean`: remove `bin/` +- Check + - `make test`: the Go tests + - `make verify`: tests, `go vet`, `gofmt -l` on the Go files changed since + `main`, linux and freebsd cross builds for amd64 and arm64, and the web + client's `node --test` when node is installed + - `make e2e`: builds, then runs every `test/*-test.sh --auto` in turn and + reports passes, failures and skips; `E2E='test/scram-*-test.sh'` runs a + subset; see [Test Scripts](#test-scripts) +- Container + - `make image`, `make image-check`: see [Container Image](#container-image) +- Install + - `make install`, `make uninstall`: see [Installing](#installing) -Version, commit hash, and build time are injected via `-ldflags` from `git -describe` and `git rev-parse`. A plain `go build` produces a working binary that -reports `dev` for all three: +Variables: + +- `GO`: the toolchain. On FreeBSD the default is the versioned package that + `go.mod` asks for (`go127`); `make GO=/path/to/go` overrides it, and + `check-go` names the package to install when it is missing. +- `GO_BUILDFLAGS`, `GO_LDFLAGS`: extra `go build` and linker flags for + packagers. They are distinct from `GOFLAGS` and `LDFLAGS`, which the go + command and the C linker read themselves. +- `SOURCE_DATE_EPOCH`: when set, the embedded build time comes from it, so + package builds are reproducible. + +A plain `go build -o bin/lw ./cmd/lw` works too, and reports `dev` for the +version metadata. `go install github.com/lixenwraith/logwisp/cmd/lw@latest` +does not work until the module is renamed to its canonical path. + +## Installing + +`make install` copies what a package ships and compiles nothing, so build +first. It honours `DESTDIR`, `PREFIX`, `SYSCONFDIR` and `BINDIR` (default +`$PREFIX/bin`, the path the systemd unit and the rc.d script run `lw` from): + +- Linux defaults: `PREFIX=/usr`, `SYSCONFDIR=/etc`. Outside a package + manager, prefer `PREFIX=/usr/local`; systemd reads units, sysusers and + tmpfiles from there too. +- FreeBSD defaults: `PREFIX=/usr/local`, `SYSCONFDIR=/usr/local/etc`. +- `INSTALL_OS` (default: `uname -s`) picks the layout, so + `INSTALL_OS=FreeBSD DESTDIR=...` stages the FreeBSD one on another system. ```bash -go build -o bin/logwisp ./cmd/logwisp +make release +sudo make install PREFIX=/usr/local ``` -`go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with -the same loss of version metadata. +Every system gets: -## Container Image - -The root `Dockerfile` builds the same package into `scratch` under UID 65532, -static and stripped. There is no shell and no config in the image: mount one and -name it, as the binary has no daemon mode and no built-in defaults worth running. - -```bash -REV=$(git rev-parse HEAD) -docker build -t "logwisp:$(git rev-parse --short HEAD)" \ - --build-arg VERSION="$(git describe --tags --always)" \ - --build-arg REVISION="$REV" . -docker run --rm -v /etc/logwisp:/etc/logwisp:ro logwisp:... -c /etc/logwisp/logwisp.toml -``` - -Sinks that listen (`http`, `tcp`) need their ports published; the read-only -root filesystem and dropped capabilities a restricted runtime imposes are all -compatible with it, provided a `file` sink's directory is writable by 65532. - -## Configuration - -Copy the annotated reference configuration and edit it: - -```bash -sudo mkdir -p /etc/logwisp -sudo cp config/logwisp.toml /etc/logwisp/logwisp.toml -``` - -LogWisp searches, in order: `-c ` / `--config ` (also `=`), -`$LOGWISP_CONFIG_DIR`/`$LOGWISP_CONFIG_FILE`, `~/.config/logwisp/logwisp.toml`, -`./logwisp.toml`. See [Configuration](configuration.md). - -## Running as a Service - -LogWisp has no daemon mode; run it in the foreground under a supervisor. +- `$BINDIR/lw` and the manual `$PREFIX/share/man/man1/lw.1` +- `$PREFIX/share/doc/logwisp/` (this documentation) and + `$PREFIX/share/licenses/logwisp/LICENSE` ### Linux (systemd) -`/etc/systemd/system/logwisp.service`: - -```ini -[Unit] -Description=LogWisp Log Transport Service -After=network.target - -[Service] -Type=simple -User=logwisp -Group=logwisp -ExecStart=/usr/local/bin/logwisp -c /etc/logwisp/logwisp.toml -ExecReload=/bin/kill -HUP $MAINPID -Restart=on-failure -RestartSec=10 -WorkingDirectory=/var/lib/logwisp -StandardOutput=journal -StandardError=journal - -# Hardening -NoNewPrivileges=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ReadWritePaths=/var/log/logwisp /var/lib/logwisp - -[Install] -WantedBy=multi-user.target -``` - -`ExecReload` gives you `systemctl reload logwisp` for configuration and -certificate rotation without dropping the process. - -If a pipeline binds a port below 1024, add -`AmbientCapabilities=CAP_NET_BIND_SERVICE` rather than running as root. - -Setup: +- `$SYSCONFDIR/logwisp/logwisp.toml`: the annotated sample configuration, + kept when one is already there. Edit it before starting the service. +- `$PREFIX/lib/systemd/system/logwisp.service`: runs + `lw -c $SYSCONFDIR/logwisp/logwisp.toml` as the `logwisp` user, from + `/var/lib/logwisp`; `systemctl reload` sends `SIGHUP`. +- `$PREFIX/lib/sysusers.d/logwisp.conf`: the `logwisp` account. +- `$PREFIX/lib/tmpfiles.d/logwisp.conf`: `/var/lib/logwisp` and + `/var/log/logwisp`, owned by it. ```bash -sudo useradd -r -s /usr/sbin/nologin logwisp -sudo mkdir -p /etc/logwisp /var/lib/logwisp /var/log/logwisp -sudo chown logwisp:logwisp /var/lib/logwisp /var/log/logwisp +sudo systemd-sysusers logwisp.conf +sudo systemd-tmpfiles --create logwisp.conf sudo systemctl daemon-reload sudo systemctl enable --now logwisp ``` -The service account needs **read** access to every directory a `file` source -watches and **write** access to every directory a `file` sink or -`logging.file` writes to. +The unit is hardened: no capabilities, a read-only view of the system with +only its two directories writable, `/home` hidden, and IPv4, IPv6 and Unix +sockets only. Widen it with a drop-in (`systemctl edit logwisp`) rather than +editing the unit: + +- A `file` sink or `logging.file` elsewhere: `ReadWritePaths=/srv/logs` +- A `file` source under `/home`: `ProtectHome=read-only` +- A port below 1024: `CapabilityBoundingSet=CAP_NET_BIND_SERVICE` and + `AmbientCapabilities=CAP_NET_BIND_SERVICE` + +The service must read every key, credentials and password file it names. +Create a credentials file with its owner and mode first; `lw auth` keeps +both: + +```bash +sudo install -m 0600 -o logwisp -g logwisp /dev/null /etc/logwisp/users.toml +sudo lw auth add-user -credentials /etc/logwisp/users.toml -user \ + -password-file +sudo systemctl reload logwisp +``` ### FreeBSD (rc.d) -`/usr/local/etc/rc.d/logwisp`: - -```sh -#!/bin/sh - -# PROVIDE: logwisp -# REQUIRE: DAEMON NETWORKING -# KEYWORD: shutdown - -. /etc/rc.subr - -name="logwisp" -rcvar="${name}_enable" -pidfile="/var/run/${name}.pid" -procname="/usr/local/bin/logwisp" -command="/usr/sbin/daemon" -command_args="-p ${pidfile} -f ${procname} -c /usr/local/etc/logwisp/logwisp.toml" - -load_rc_config $name -: ${logwisp_enable:="NO"} - -run_rc_command "$1" -``` - -Setup: +- `$SYSCONFDIR/logwisp/logwisp.toml.sample`, copied to `logwisp.toml` when + none exists and `DESTDIR` is empty +- `$PREFIX/etc/rc.d/logwisp`: runs `lw` through `daemon(8)` as + `logwisp_user`, with its output in syslog under the tag `logwisp`; + `service logwisp reload` sends `SIGHUP` ```bash -sudo chmod +x /usr/local/etc/rc.d/logwisp -sudo pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin -sudo mkdir -p /usr/local/etc/logwisp /var/log/logwisp -sudo chown logwisp:logwisp /var/log/logwisp -sudo sysrc logwisp_enable="YES" +sudo pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin -c "LogWisp log transport" +sudo sysrc logwisp_enable=YES sudo service logwisp start ``` -## Directory Layout +rc.conf variables: -| Purpose | Linux | FreeBSD | -|---------|-------|---------| -| Binary | `/usr/local/bin/logwisp` | `/usr/local/bin/logwisp` | -| Configuration | `/etc/logwisp/` | `/usr/local/etc/logwisp/` | -| TLS material | `/etc/logwisp/tls/` | `/usr/local/etc/logwisp/tls/` | -| Working directory | `/var/lib/logwisp/` | `/var/db/logwisp/` | -| Application logs | `/var/log/logwisp/` | `/var/log/logwisp/` | +- `logwisp_enable`: `YES` to start at boot (default `NO`) +- `logwisp_config`: the configuration (default + `/usr/local/etc/logwisp/logwisp.toml`) +- `logwisp_user`: the account (default `logwisp`) +- `logwisp_chdir`: the working directory (default `/var/db/logwisp`, created + for `logwisp_user` when missing) +- `logwisp_args`: extra `lw` options, e.g. `--logging.level=debug` (default + empty); `logwisp_flags` goes to `daemon(8)`, not to `lw` -Key files should be mode `0600` and owned by the service account. +### Uninstall + +`sudo make uninstall` (with the `PREFIX` used to install) removes everything +`make install` wrote except `$SYSCONFDIR/logwisp`, which holds configuration +and credentials. Remove that, the working directories and the account by +hand. + +## Container Image + +The `Dockerfile` builds `lw` in a Go builder and copies it alone, static and +stripped, into `scratch`: `/lw` running as UID 65532, plus the CA bundle that +TLS dialers without a `ca_file` verify against. There is no shell and no +configuration in the image. + +```bash +make image # logwisp:dev; IMAGE=, IMAGE_TAG= rename it +make image-check +``` + +`image-check` runs the image the way it should run in production: read-only, +`--network none`, `--cap-drop ALL`, `no-new-privileges`, as 65532. It prints +`--version`, then runs `--check` on `config/logwisp.toml` (or +`IMAGE_CHECK_CONFIG`) mounted read-only. + +Building behind a proxy: + +- A TLS-intercepting proxy: `make image BUILD_CA=/path/to/ca-bundle.pem`. + The bundle reaches only the module download, as a BuildKit secret. +- An explicit proxy: pass Docker's predefined proxy build arguments, which + never reach an image layer, e.g. + `IMAGE_BUILD_FLAGS='--network host --build-arg HTTPS_PROXY=http://127.0.0.1:'`. +- `CONTAINER_ENGINE=podman` works too. + +### Running it securely + +```bash +docker run -d --name logwisp \ + --read-only --cap-drop ALL --security-opt no-new-privileges \ + -v /etc/logwisp:/etc/logwisp:ro \ + -v /srv/logwisp/secrets:/run/secrets:ro \ + -p 8443:8443 \ + -e LOGWISP_LOGGING_LEVEL=info \ + logwisp: -c /etc/logwisp/logwisp.toml +``` + +- Configuration: mount it read-only at `/etc/logwisp` and name it with `-c`. + The files must be readable by UID 65532. +- Scalar settings: environment variables, `LOGWISP_`. +- Pipelines without a file: the pipeline variables, in the `--source` and + `--sink` syntax (see [CLI](cli.md)); they replace the file's pipelines: + + ```bash + -e LOGWISP_PIPELINE=relay \ + -e LOGWISP_SOURCE=tcp_chain,port=9000,tls.enabled=true,tls.cert_file=/run/secrets/tls.crt,tls.key_file=/run/secrets/tls.key,auth.type=scram,auth.credentials_file=/run/secrets/users.toml \ + -e LOGWISP_SINK=console + ``` + +- Secrets: only as mounted files, at `/run/secrets` (Docker or Compose + secrets, a Kubernetes secret volume), named by `tls.key_file`, + `auth.credentials_file` and `auth.password_file`. Never put a password in + the environment: `docker inspect` and `/proc` show it. Make each file + readable by 65532 only (`chown 65532` and mode `0400`). +- Privileges: keep the image's non-root user, a read-only root filesystem, + `--cap-drop ALL` and `no-new-privileges`. Without capabilities a listener + needs a port of 1024 or above; publish it on any host port. +- Listeners bind `0.0.0.0` inside the container; `127.0.0.1` would be + unreachable through `-p`. +- Writable data: a `file` sink or `logging.file` needs a volume writable by + 65532; nothing else is written. +- Reload: `docker kill -s HUP logwisp`. +- `lw auth` runs from the image too, with directories writable by 65532. + Keep the generated passwords out of the directory the server mounts: it + needs only `users.toml` (verifiers) and the TLS files, while a `.pass` file + is the client's plaintext password. + + ```bash + docker run --rm --read-only --cap-drop ALL --network none \ + -v /srv/logwisp/secrets:/work -v /srv/logwisp/passwords:/pw logwisp: \ + auth add-user -credentials /work/users.toml -user -password-file /pw/.pass + ``` + +## Packaging Status + +The foundation for distribution packages is in place: + +- the `lw` binary name, and `make install` with `DESTDIR`, `PREFIX` and + `SYSCONFDIR` +- the `doc/lw.1` manual +- service files in `deploy/package/`: `logwisp.service`, `logwisp.sysusers`, + `logwisp.tmpfiles` and the FreeBSD `logwisp.rc` +- skeletons: `deploy/package/arch/PKGBUILD` and a `sysutils/logwisp` port in + `deploy/package/freebsd/` + +Still missing: tagged, signed release tarballs; the module rename to +`github.com/lixenwraith/logwisp`, which `go install` and FreeBSD's +`go:modules` need; finishing and submitting the AUR package and the port; +Debian packaging. [To Do](todo.md) has the steps. ## Verification ```bash -logwisp --version - -# start in the foreground with debug logging and watch pipelines come up -logwisp -c /etc/logwisp/logwisp.toml --logging.level=debug --logging.output=stderr - +lw --version +lw -c /etc/logwisp/logwisp.toml --logging.level=debug --logging.output=stderr sudo systemctl status logwisp # Linux sudo service logwisp status # FreeBSD ``` -Expect `Created source instance`, `Created sink instance`, and -`Starting pipeline` for each configured pipeline. There is no validate-only -mode; see [Operations](operations.md#checking-a-configuration). +Expect `Created source instance`, `Created sink instance` and +`Starting pipeline` for each pipeline. To validate a configuration without +starting it, run `lw --check -c FILE`; see +[Operations](operations.md#checking-a-configuration). ## Test Scripts -End-to-end scripts under `test/` run against a local build: +The end-to-end scripts in `test/` run against `bin/lw` and share +`test/lib.sh`. `make e2e` runs each with `--auto` and reports it as PASS, FAIL +or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target. -```bash -make -./test/chain-test.sh --auto # two independent relay pipelines -./test/chain-aggregate-test.sh --auto # fan-in: both edges into one pipeline -./test/mtls-chain-test.sh --auto # the same fan-in under mTLS -./test/scram-chain-test.sh --auto # SCRAM logins on every network plugin -./test/passthrough-test.sh # file source relays a wide envelope intact +- Modes: + - without flags: start the daemons, print a guide of the ports and one-line + commands (run `. test/run-X/env` first for `$LW`, `$CA` and the token + helper), and wait for Ctrl-C + - `--auto`: run the checks and tear down; `--keep` leaves the daemons up + after a pass + - `--follow`: show only WARN, ERROR and key events from the daemons +- Daemons log only to `test/run*/log/NAME.out`, never to the terminal. +- Colour only on a terminal; `NO_COLOR` turns it off, `FORCE_COLOR=1` on. +- Exit status: 0 all passed, 1 a failure, 77 skipped. +- Each script owns a port range and a gitignored run directory: + +``` +15801-15804 chain-test.sh, chain-aggregate-test.sh test/run/ +(no ports) passthrough-test.sh test/run/passthrough/ +15811-15814 mtls-chain-test.sh test/run-mtls/ +15821-15825 scram-chain-test.sh test/run-scram/ +15831-15832 scram-proxy-test.sh test/run-proxy/ +15851-15855 ipv6-test.sh test/run-ipv6/ +15871-15879 deploy-test.sh test/run-deploy/ ``` -Without `--auto` the chain scripts run the relay in the foreground for -interactive inspection. They need bash 5+, coreutils, and curl (the mTLS and -SCRAM scripts also openssl). The plain chain scripts bind ports 15801–15804 and -write to `test/run/`; the mTLS script uses 15811–15814 and `test/run-mtls/`, the -SCRAM script 15821–15825 and `test/run-scram/`. The pass-through test binds -nothing. - -## Uninstall - -### Linux - -```bash -sudo systemctl disable --now logwisp -sudo rm /usr/local/bin/logwisp /etc/systemd/system/logwisp.service -sudo systemctl daemon-reload -sudo rm -rf /etc/logwisp /var/lib/logwisp /var/log/logwisp -sudo userdel logwisp -``` - -### FreeBSD - -```bash -sudo service logwisp stop -sudo sysrc -x logwisp_enable -sudo rm /usr/local/bin/logwisp /usr/local/etc/rc.d/logwisp -sudo rm -rf /usr/local/etc/logwisp /var/db/logwisp /var/log/logwisp -sudo pw userdel logwisp -``` +- `chain-test.sh`: two edges into a relay, one pipeline per chain transport; + ports 15801-15804 +- `chain-aggregate-test.sh`: the same edges, with the relay fanning both into + both sinks; ports 15801-15804 +- `passthrough-test.sh`: `file` source to `file` sink, a wide envelope byte + for byte (`raw = true`) and, parsed, with no key dropped; no ports +- `mtls-chain-test.sh`: mTLS chain edges and viewers, allow lists, node + binding; ports 15811-15814 +- `scram-chain-test.sh`: SCRAM chain edges and viewers with `lw auth` + credentials; ports 15821-15825 +- `scram-proxy-test.sh`: browser and CLI logins to an `http` sink behind a + TLS-terminating proxy; ports 15831-15832; the browser checks skip without + node and playwright's Chromium: the headless shell for the scenarios with + cookies, full Chromium for the profile that blocks them +- `ipv6-test.sh`: a relay on `::1` over both chain transports, IPv6 SANs, + per-family listeners; ports 15851-15855; skips (exit 77) without an IPv6 + loopback +- `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed + and checked; ports 15871-15879 diff --git a/doc/lw.1 b/doc/lw.1 new file mode 100644 index 0000000..01af4ad --- /dev/null +++ b/doc/lw.1 @@ -0,0 +1,328 @@ +.Dd October 2, 2026 +.Dt LW 1 +.Os +.Sh NAME +.Nm lw +.Nd LogWisp log transport +.Sh SYNOPSIS +.Nm +.Op Fl \-check +.Op Fl c Ar file +.Op Fl \-quiet +.Op Fl \- Ns Ar key Ns = Ns Ar value ... +.Op Fl \-pipeline Ar name +.Op Fl \-source Ar spec ... +.Op Fl \-sink Ar spec ... +.Op Fl \-filter Ar spec ... +.Op Fl \-format Ar spec +.Op Fl \-rate\-limit Ar spec +.Op Fl \-heartbeat Ar spec +.Nm +.Cm auth +.Ar command +.Op Ar flags +.Nm +.Fl \-version +.Nm +.Cm help | Fl h | Fl \-help +.Sh DESCRIPTION +.Nm +runs LogWisp: independent pipelines, each reading entries from sources, +passing them through a flow of rate limiting, filters and formatting, and +writing them to sinks. +Sources tail files, read standard input or accept other LogWisp nodes; +sinks write to the console or files, serve live streams over HTTP or TCP, or +forward to other nodes. +Every network source and sink can use TLS, mutual TLS and +Argon2id-SCRAM password authentication. +.Pp +.Nm +stays in the foreground and logs to standard output; run it under a +supervisor such as +.Xr systemd 1 +or +.Xr daemon 8 . +With no configuration file it runs a demonstration pipeline: a synthetic +generator writing JSON to standard output. +.Pp +The options are as follows: +.Bl -tag -width Ds +.It Fl c Ar file , Fl \-config Ar file , Fl \-config= Ns Ar file +Read the configuration from +.Ar file , +which must exist. +The last one given wins. +.It Fl \- Ns Ar key Ns = Ns Ar value , Fl \- Ns Ar key Ar value +Set any scalar configuration key by its TOML path, such as +.Fl \-logging.level=debug . +A bare +.Fl \- Ns Ar key +sets a boolean to true. +.It Fl \-quiet +Suppress console output. +.It Fl \-status_reporter= Ns Ar bool +Log pipeline statistics every 30 seconds at the debug level +.Pq default true . +.It Fl \-auto_reload= Ns Ar bool +Reload when the configuration file changes +.Pq default false . +.It Fl \-logging.output= Ns Ar mode +One of +.Cm file , stdout , stderr , split , all +or +.Cm none . +.It Fl \-logging.level= Ns Ar level +One of +.Cm debug , info , warn +or +.Cm error . +.It Fl \-check +Build every pipeline and plugin as a start would, loading options, TLS files +and credentials files and printing startup warnings, without binding a port +or opening, reading or creating a log file; print +.Dq configuration ok +and exit 0, or the error and exit 1, or 2 when a named configuration file is +missing +.Pq see Sx EXIT STATUS . +.It Fl \-version +Print the version, commit and build time, and exit. +.It Fl h , Fl \-help , Cm help +Print usage and exit. +.It Fl \- +End option processing. +.El +.Ss Pipeline specifications +Pipelines can be given on the command line or in the environment instead of +the configuration file. +They then replace the file's pipelines and the built-in default; every other +key keeps its usual precedence. +A reload rereads the file but keeps these pipelines. +.Bl -tag -width Ds +.It Fl \-pipeline Ar name +Start a new pipeline. +Specifications before the first +.Fl \-pipeline +belong to a pipeline named +.Dq cli . +.It Fl \-source Ar spec , Fl \-sink Ar spec , Fl \-filter Ar spec +Add a source, sink or filter; each is repeatable. +.It Fl \-format Ar spec , Fl \-rate\-limit Ar spec , Fl \-heartbeat Ar spec +Set the pipeline's formatter, rate limit and heartbeat. +.El +.Pp +A source, sink, filter or format +.Ar spec +is a type followed by comma-separated +.Ar key Ns = Ns Ar value +pairs; a rate limit or heartbeat +.Ar spec +has the pairs only: +.Bd -literal -offset indent +\-\-source file,directory=/var/log/app,pattern=*.log +\-\-sink http,host=127.0.0.1,port=8080,id=web +\-\-filter include,patterns=ERROR,patterns=WARN +\-\-format json +\-\-rate\-limit rate=100,burst=200,policy=drop +.Ed +.Pp +Keys are dotted paths into the plugin's options, such as +.Cm tls.cert_file . +A repeated key makes a list. +A backslash escapes +.Ql \&, , +.Ql = +and +.Ql \e +inside a value; quote such a +.Ar spec +so the shell keeps the backslash. +.Cm id= Ns Ar name +names a source or sink; the default is its type, then +.Ar type Ns _2 , +.Ar type Ns _3 +and so on. +Any pipeline flag on the command line makes +.Nm +ignore the pipeline variables in +.Sx ENVIRONMENT . +.Ss Authentication commands +.Nm +.Cm auth +manages SCRAM credentials files and logs in to SCRAM listeners. +Changes to a credentials file apply when the server receives +.Dv SIGHUP . +.Bl -tag -width Ds +.It Cm add-user Fl credentials Ar file Fl user Ar name Oo Fl password-file Ar file Oc Op Fl generate +Add a user, or replace its password. +The password is read from the +.Fl password-file +when it exists; otherwise one is generated and written there, or printed +once to standard output. +.Fl generate +replaces an existing password without reading one. +A missing or empty credentials file is created with mode 0600; an existing +one keeps its owner and mode. +.It Cm remove-user Fl credentials Ar file Fl user Ar name +Remove a user. +The last user cannot be removed. +.It Cm token Fl url Ar https://host:port Fl user Ar name Fl password-file Ar file Oo Ar tls-flags Oc Op Fl unbound +Log in to an +.Cm http +sink or +.Cm http_chain +source and print a bearer token. +.Fl unbound +logs in without channel binding, to an +.Cm http +sink behind a TLS-terminating proxy; the URL may then carry the proxy's path. +.It Cm stream Fl addr Ar host:port Fl user Ar name Fl password-file Ar file Op Ar tls-flags +Log in to a +.Cm tcp +sink and copy its stream to standard output until interrupted. +.El +.Pp +The +.Ar tls-flags +are +.Fl ca-file Ar file +.Pq default: the system roots , +.Fl server-name Ar name , +and +.Fl cert-file Ar file +with +.Fl key-file Ar file +for a listener that requires client certificates. +There is no flag to skip server verification. +An IPv6 +.Ar host +is written in brackets, as in +.Li [2001:db8::1]:8443 . +.Sh ENVIRONMENT +.Bl -tag -width Ds +.It Ev LOGWISP_ Ns Ar KEY +Any scalar configuration key: its TOML path in upper case with +.Ql \&. +replaced by +.Ql _ , +as in +.Ev LOGWISP_LOGGING_LEVEL=debug . +Command-line options take precedence, and both take precedence over the +configuration file. +.It Ev LOGWISP_CONFIG_FILE +The configuration file, relative to +.Ev LOGWISP_CONFIG_DIR +when both are set. +.It Ev LOGWISP_CONFIG_DIR +The directory holding +.Pa logwisp.toml . +.It Ev LOGWISP_PIPELINE +The name of the one pipeline the variables below define. +All of these are ignored when a pipeline option is on the command line. +.It Ev LOGWISP_SOURCE , LOGWISP_SINK , LOGWISP_FILTER +Source, sink and filter specifications, in the syntax of the matching +options; a suffix +.Ar _N , +counting from 1, adds more, as in +.Ev LOGWISP_SINK_1 . +.It Ev LOGWISP_FORMAT , LOGWISP_RATE_LIMIT , LOGWISP_HEARTBEAT +The pipeline's format, rate limit and heartbeat specifications. +.El +.Pp +Secrets never go in the environment: TLS keys, credentials files and +password files are named by path +.Pq Cm tls.key_file , auth.credentials_file , auth.password_file . +.Sh FILES +Without +.Fl c +or the +.Ev LOGWISP_CONFIG_* +variables, +.Nm +reads the first of these that exists: +.Bl -enum -compact +.It +.Pa ~/.config/logwisp/logwisp.toml +.It +.Pa ./logwisp.toml +.El +.Pp +Packages install a sample configuration and service files that pass it with +.Fl c : +.Bl -tag -width Ds +.It Pa /etc/logwisp/logwisp.toml +The configuration on Linux, used by +.Pa logwisp.service . +.It Pa /usr/local/etc/logwisp/logwisp.toml +The configuration on +.Fx , +installed as +.Pa logwisp.toml.sample +and used by +.Pa /usr/local/etc/rc.d/logwisp . +.It Pa /var/lib/logwisp , /var/db/logwisp +The service's working directory on Linux and on +.Fx . +.El +.Sh EXIT STATUS +.Nm +exits 0 after a clean shutdown, +.Fl \-version +or help; 1 on a configuration, logger or startup error; and 2 when a +configuration file named by +.Fl c +or +.Ev LOGWISP_CONFIG_* +does not exist. +.Pp +.Nm +.Cm auth +exits 0 on success or help, 1 on failure, and 2 on a usage error. +.Sh SIGNALS +.Bl -tag -width "SIGINT, SIGTERM" +.It Dv SIGINT , SIGTERM +Shut down gracefully. +.It Dv SIGHUP , SIGUSR1 +Reread the configuration file, certificates and credentials, and rebuild +every pipeline. +An invalid configuration leaves the running service untouched. +.El +.Sh EXAMPLES +Run a configuration in the foreground with debug logging: +.Bd -literal -offset indent +$ lw \-c /etc/logwisp/logwisp.toml \-\-logging.level=debug +.Ed +.Pp +Tail a directory and serve it as a live stream, without a file: +.Bd -literal -offset indent +$ lw \-\-source 'file,directory=/var/log/app,pattern=*.log' \e + \-\-sink http,host=127.0.0.1,port=8080 +.Ed +.Pp +Create a credentials file the service account can read, add a user, +letting +.Nm +generate the password into a file, then reload the server: +.Bd -literal -offset indent +# install \-m 0600 \-o logwisp \-g logwisp /dev/null /etc/logwisp/users.toml +# lw auth add-user \-credentials /etc/logwisp/users.toml \e + \-user alice \-password-file /root/alice.pass +# systemctl reload logwisp +.Ed +.Sh SEE ALSO +.Xr rc.conf 5 , +.Xr systemd.exec 5 , +.Xr sysusers.d 5 , +.Xr tmpfiles.d 5 , +.Xr daemon 8 +.Pp +The documentation installed in +.Pa share/doc/logwisp +under the installation prefix, and +.Lk https://github.com/lixenwraith/logwisp +.Sh SECURITY CONSIDERATIONS +A listener without an +.Cm auth +block serves or accepts logs from anyone who can reach it. +Keep key, credentials and password files at mode 0600, owned by the account +.Nm +runs as; startup warns about world-readable ones. diff --git a/doc/mtls-auth-plan.md b/doc/mtls-auth-plan.md index fa38a6d..b00b08a 100644 --- a/doc/mtls-auth-plan.md +++ b/doc/mtls-auth-plan.md @@ -65,14 +65,13 @@ back to authenticated peers. The authenticated identity is a single string derived from the peer's verified leaf certificate. Because `tls.RequireAndVerifyClientCert` has already validated the chain, signature, and validity window by the time we look, extraction is -pure field selection — `tlsx.PeerIdentity`. +pure field selection — `tlsx.PeerIdentity`. The `identity` modes: -| `identity` mode | Source | Notes | -|-----------------|--------|-------| -| `cn` (default) | `Subject.CommonName` | Matches the existing `tls_peer_cn` metadata | -| `san_dns` | first `DNSNames` entry | Preferred for host identities | -| `san_uri` | first `URIs` entry | SPIFFE-style IDs | -| `san_email` | first `EmailAddresses` entry | Operator identities | +- `cn` (default): `Subject.CommonName`, matching the existing `tls_peer_cn` + metadata. +- `san_dns`: the first `DNSNames` entry; preferred for host identities. +- `san_uri`: the first `URIs` entry; SPIFFE-style IDs. +- `san_email`: the first `EmailAddresses` entry; operator identities. An empty identity is a rejection, not an empty match: a certificate with no usable identity field cannot satisfy any policy. @@ -96,13 +95,14 @@ allow_patterns = ["^edge-\\d{2}$"] # RE2, anchored by the author node_binding = "force" # none | assert | force ``` -| Option | Type | Default | Meaning | -|--------|------|---------|---------| -| `type` | string | `none` | `none` preserves pre-auth behaviour exactly; `mtls` enables the policy | -| `identity` | string | `cn` | Which certificate field is the identity | -| `allow` | []string | `[]` | Exact identity matches | -| `allow_patterns` | []string | `[]` | RE2 patterns matched against the identity | -| `node_binding` | string | `force` when `type = "mtls"` | Chain sources only; see below | +- `type`, string, default `none`: `none` preserves pre-auth behaviour + exactly; `mtls` enables the policy. +- `identity`, string, default `cn`: which certificate field is the identity. +- `allow`, []string, default `[]`: exact identity matches. +- `allow_patterns`, []string, default `[]`: RE2 patterns matched against the + identity. +- `node_binding`, string, default `force` when `type = "mtls"`: chain sources + only; see below. Empty `allow` **and** empty `allow_patterns` under `type = "mtls"` means "any identity the CA vouches for" — that is, the pre-auth behaviour, but with the @@ -111,13 +111,15 @@ default rather than a silent deny-all, and the plugin logs a WARN at startup saying so. `node_binding` applies only to the chain sources, where a `node` label is -declared. Setting it on any other plugin is a configuration error. +declared. Setting it to anything but `none` on any other plugin is a +configuration error. Its values: -| Value | Connection label | Per-entry `node` field | -|-------|------------------|------------------------| -| `none` | `trust_node` governs, as before | `trust_node` governs | -| `assert` | Must equal the identity; a mismatch or an omission is rejected | `trust_node` governs | -| `force` | The declared label is ignored and the identity is used | Overwritten with the identity | +- `none`: `trust_node` governs the connection label, as before, and the + per-entry `node` field. +- `assert`: the connection label must equal the identity, and a mismatch or an + omission is rejected; `trust_node` governs the per-entry `node` field. +- `force`: the declared label is ignored and the identity is used; every + per-entry `node` field is overwritten with the identity. The split between `assert` and `force` is what makes both worth having: @@ -131,8 +133,9 @@ The split between `assert` and `force` is what makes both worth having: it forwards keep the origin labels stamped at the first hop, so multi-hop attribution survives. -`node_binding` overrides `trust_node`; when binding is active the constructor -logs that `trust_node` is being ignored. +`node_binding` overrides `trust_node` for the connection label. Under `force` +the constructor logs that `trust_node` is ignored; under `assert`, that it still +governs the per-entry labels. Dialer-side plugins (`tcp_chain` and `http_chain` sinks) accept the same block to pin the *server's* identity beyond hostname verification. There @@ -161,7 +164,8 @@ One small package, mirroring `internal/tlsx`, is the single seam between the against the sibling TLS config and returns `(nil, nil)` when auth is disabled; every method tolerates a nil receiver, so call sites never branch on config. Listeners admit only through `Admit` (TCP) and `AuthorizeRequest` (HTTP); the -current API, including SCRAM, is in [scram-auth-plan.md](scram-auth-plan.md#the-authz-seam). +current API, including SCRAM, is in +[scram-auth-plan.md](scram-auth-plan.md#the-authz-seam). ### Enforcement Points @@ -169,18 +173,19 @@ current API, including SCRAM, is in [scram-auth-plan.md](scram-auth-plan.md#the- Handshake → `Admit` (authorize, then read hello) → `ResolveNode` → create session. Under `mtls` the certificate is checked before the hello is read, so -an unauthorized peer never gets a preamble parsed on its behalf. `chain.DecodeEntry` is then -called with `auth.TrustsEntryNode(trust_node)` rather than `trust_node` itself. +an unauthorized peer never gets a preamble parsed on its behalf. +`chain.DecodeEntry` is then called with `auth.TrustsEntryNode(trust_node)` +rather than `trust_node` itself. **`http_chain` source** (`internal/source/httpchain/httpchain.go`, `handleIngest`) -Per request, through `AuthorizeRequest`, before the body is read — an unauthorized sender does -not get to stream `max_body_bytes` into the process. Rejection is `403`, -distinct from the `400` used for protocol errors, so a sender can tell "you are -not allowed" from "your batch was malformed". `ResolveNode` then governs the -`X-Logwisp-Node` header exactly as it governs the TCP hello. The session cache -key includes the identity, so two peers sharing a remote address never share a -session. +Per request, through `AuthorizeRequest`, before the body is read — an +unauthorized sender does not get to stream `max_body_bytes` into the process. +Rejection is `403`, distinct from the `400` used for protocol errors, so a +sender can tell "you are not allowed" from "your batch was malformed". +`ResolveNode` then governs the `X-Logwisp-Node` header exactly as it governs the +TCP hello. The session cache key includes the identity, so two peers sharing a +remote address never share a session. **`tcp` sink** (`internal/sink/tcp/tcp.go`, `handleConn`) diff --git a/doc/networking.md b/doc/networking.md index 1e8abe7..ef5e898 100644 --- a/doc/networking.md +++ b/doc/networking.md @@ -6,23 +6,54 @@ see [Chaining](chaining.md). ## Address Family -**All listeners bind `tcp4` and all dialers dial `tcp4`.** IPv6 is not -supported, deliberately. An IPv6 client cannot connect and will simply see a -connection failure. +Every listener and dialer keeps strictly to the family of its `host`: -When testing locally use `127.0.0.1`, not `localhost` — the latter may resolve -to `::1` and appear as an unexplained connection refusal. +- IPv4 literal (`127.0.0.1`, the default `0.0.0.0`): IPv4 only (`tcp4`). An + empty `host` is the IPv4 wildcard, as `0.0.0.0`. +- IPv6 literal (`::1`, `::`, `fe80::1%eth0`): IPv6 only (`tcp6`). + - `::` takes no IPv4 connections, on Linux and FreeBSD alike, whatever the + system's `bindv6only` default. + - To serve both families, run two listeners on the same port, one on + `0.0.0.0` and one on `::`: neither holds the other's port. +- Hostname: resolved. A listener binds one address (IPv4 when there is one) + in that address's family alone, a name for `0.0.0.0` or `::` too, so name + the address to be sure; a dialer tries each address in turn. + +### IPv6 + +- Notation + - `host` takes the bare address: `host = "::1"`. A bracketed host, one + with a port, or an IPv4-mapped one (`::ffff:10.0.0.1`) fails at load. + - Addresses and URLs bracket it: logs print `[::1]:8443`, and `lw auth` + takes `-addr [::1]:8443` and `-url https://[::1]:8443`. Older `curl` + releases read the brackets as a glob; `-g` stops that and is harmless. + - A link-local address carries its zone, `fe80::1%eth0`, escaped in URLs as + `https://[fe80::1%25eth0]:8443`. +- TLS: a dialer verifies an IPv6 target against the certificate's IP SANs + (`subjectAltName = IP:::1`), as it does an IPv4 one. The zone is not part of + the name; `server_name` overrides as usual. +- Peers + - Logs and sessions name an IPv6 peer by its address, a link-local one with + its zone. + - SCRAM throttling counts an IPv6 client by its /64, so the hosts on a link + share one link-local budget per interface (zone): each can pick any + `fe80::/64` address. + - A peer with a zone never matches `auth.trusted_proxies`: put the proxy on + loopback or a routed address. ## Network Plugins -| Plugin | Role | Protocol | Purpose | -|--------|------|----------|---------| -| `tcp` sink | Listener | Raw stream | Broadcast formatted payloads to clients | -| `http` sink | Listener | HTTP SSE | Browser-friendly live stream plus status JSON | -| `tcp_chain` source | Listener | Chain v1 | Ingest a persistent NDJSON stream | -| `http_chain` source | Listener | Chain v1 | Ingest NDJSON batches over POST | -| `tcp_chain` sink | Dialer | Chain v1 | Forward entries over a persistent connection | -| `http_chain` sink | Dialer | Chain v1 | Forward entries as batched POSTs | +Listeners: + +- `tcp` sink: raw stream; broadcasts formatted payloads to clients. +- `http` sink: HTTP SSE; a browser-friendly live stream plus status JSON. +- `tcp_chain` source: chain v1; ingests a persistent NDJSON stream. +- `http_chain` source: chain v1; ingests NDJSON batches over POST. + +Dialers: + +- `tcp_chain` sink: chain v1; forwards entries over a persistent connection. +- `http_chain` sink: chain v1; forwards entries as batched POSTs. There is no port registry and no default port: `port` is required on every network plugin. There is also no cross-pipeline conflict detection — two sinks @@ -34,30 +65,36 @@ ERROR msg="Failed to start sink" error="tcp sink bind 0.0.0.0:9090: listen tcp4 ## Timeouts -Every network plugin exposes the deadlines relevant to its role. Zero means "no -deadline" wherever the table says so. +Every network plugin exposes the deadlines relevant to its role. `0` means no +deadline where the default is `0` (none); elsewhere it selects the default. -| Plugin | Option | Default | Bounds | -|--------|--------|---------|--------| -| `tcp` sink | `write_timeout_ms` | `5000` | One write to one client; a miss disconnects that client | -| `http` sink | `write_timeout_ms` | `0` (none) | One SSE event write | -| `tcp_chain` source | `hello_timeout_ms` | `10000` | Reading the protocol preamble, and under `scram` the whole login | -| `tcp_chain` source | `read_timeout_ms` | `0` (none) | Idle time between entries | -| `http_chain` source | `read_timeout_ms` | `30000` | Reading a whole request body | -| `tcp_chain` sink | `dial_timeout_ms` | `5000` | TCP connect | -| `tcp_chain` sink | `write_timeout_ms` | `5000` | One line write | -| `http_chain` sink | `request_timeout_ms` | `10000` | Dial plus write plus response, and a SCRAM login when one is due | +- `tcp` sink + - `write_timeout_ms`, default `5000`: one write to one client; a miss + disconnects that client. +- `http` sink + - `write_timeout_ms`, default `0` (none): one SSE event write. +- `tcp_chain` source + - `hello_timeout_ms`, default `10000`: reading the protocol preamble, and + under `scram` the whole login. + - `read_timeout_ms`, default `0` (none): idle time between entries. +- `http_chain` source + - `read_timeout_ms`, default `30000`: reading a whole request body. +- `tcp_chain` sink + - `dial_timeout_ms`, default `5000`: TCP connect. + - `write_timeout_ms`, default `5000`: one line write. +- `http_chain` sink + - `request_timeout_ms`, default `10000`: dial plus write plus response, and + a SCRAM login when one is due. Fixed, non-configurable bounds: -| Bound | Value | Applies to | -|-------|-------|------------| -| TLS handshake | 10 s | All TLS listeners and dialers | -| HTTP read-header timeout | 10 s | `http` sink, `http_chain` source | -| HTTP server shutdown grace | 2 s | `http` sink, `http_chain` source | -| Max single entry line | 1 MiB | Chain listeners | -| SCRAM login | 10 s | `tcp` sink, chain sinks, `logwisp auth`, each `/auth` request | -| SCRAM line or `/auth` body | 4 KiB | `scram` listeners and dialers | +- TLS handshake: 10 s, on all TLS listeners and dialers. +- HTTP read-header timeout: 10 s, on the `http` sink and `http_chain` source. +- HTTP server shutdown grace: 2 s, on the `http` sink and `http_chain` source. +- Single entry line: at most 1 MiB, on chain listeners. +- SCRAM login: 10 s, on the `tcp` sink, chain sinks, `lw auth`, and each + `/auth` request. +- SCRAM line or `/auth` body: at most 4 KiB, on `scram` listeners and dialers. The `http` sink deliberately leaves the server's `WriteTimeout` unset, since it would terminate long-lived SSE streams; per-event deadlines come from @@ -163,7 +200,9 @@ headers, and entry encoding. **Connection refused** - Confirm the pipeline started; a bind failure is logged at ERROR. -- Confirm you are dialing IPv4. `localhost` may resolve to `::1`. +- Confirm the dialer uses the listener's family: `[::1]` does not reach a + `127.0.0.1` or `0.0.0.0` listener, nor `127.0.0.1` a `::` one. `localhost` + may resolve to either. - Check the port is not already bound by another pipeline in the same process. **TLS handshake failure** @@ -176,7 +215,8 @@ headers, and entry encoding. is not covered by the server certificate's SANs; set `server_name`. - `protocol version not supported` — one side is pinned to `min_version = "1.3"` and the other cannot negotiate it. -- Handshake failures appear as WARN with the remote address, and increment +- Listeners log handshake failures at WARN with the remote address (the `tcp` + sink at DEBUG); the `tcp` sink and `tcp_chain` source count them in `tls_handshake_errors`. **Rejected after a successful handshake** @@ -206,7 +246,7 @@ headers, and entry encoding. between the peers terminates TLS. Pass TLS through to LogWisp; the login cannot work otherwise, except on an `http` sink in proxy mode (`auth.trusted_proxies`), where `the client bound its proof to the proxy's - certificate` means a client that needs `logwisp auth token -unbound`, and + certificate` means a client that needs `lw auth token -unbound`, and `the client sent an unbound proof` the reverse. On an `http_chain` sink, `server certificate differs from the one the SCRAM login was bound to` means the certificate changed after the login: a rotation (the retry binds anew), @@ -217,15 +257,16 @@ headers, and entry encoding. `X-Forwarded-For`. - `too many attempts` — the address failed or abandoned logins faster than one per second beyond a burst of 10, or has 4 unfinished; it clears within seconds - once the failing peer stops. Peers behind one NAT share the budget. `busy` — - 4,096 logins in flight, or the listener is stopping. + once the failing peer stops. Peers behind one NAT or one passthrough proxy + share the budget. `busy` — 4,096 logins in flight, or the listener is + stopping. - `authentication not enabled` — the dialer has `scram`; the listener's `auth.type` is `none` or `mtls`. - `no challenge within 10s (older logwisp, or not a scram listener)` — the server read the hello and said nothing. Over HTTP the same case reads `no auth endpoint at …/auth`. - `Argon2 parameters below client minimum` — the credentials file holds a - cheaper Argon2 profile than the defaults; recreate it with `logwisp auth`. + cheaper Argon2 profile than the defaults; recreate it with `lw auth`. - `peer offered no credentials` (`authentication required` on the wire) in the listener's log — a peer without `scram` reached a `scram` listener; a client that sends nothing logs `read hello: … i/o timeout` after 10 s instead. A @@ -245,6 +286,22 @@ headers, and entry encoding. - `403` is not the token: it is the `identity` binding, node binding or, in proxy mode, the proxy gate. +**Behind an L4 proxy** +- Every connection through the proxy fails: it sends a PROXY header (nginx + `stream` with `proxy_protocol on`, HAProxy `send-proxy`) to a LogWisp TLS + listener, which cannot read one yet. The header lands in front of the TLS + handshake, which fails; the listener logs the proxy's address and + `first record does not look like a TLS handshake` (the `tcp` sink at DEBUG). + Route LogWisp without it: see + [Behind nginx or another proxy](security.md#behind-nginx-or-another-proxy). +- All peers share one throttling budget behind a passthrough proxy: without + PROXY every `scram` peer arrives from the proxy's address. One client failing + logins exhausts everyone's, and a fifth login while 4 are unfinished (edges + reconnecting together after a restart) is refused; both answer + `too many attempts` ([Throttling](security.md#throttling)). + Expose the LogWisp ports directly, or keep the proxy hop on a trusted + network so only trusted clients share the budget. + **Entries not arriving over a chain link** - Check the sink's `connected` statistic and its `reconnects` count. - Check the source's `auth_rejected` — an allow-list miss or a failed login diff --git a/doc/operations.md b/doc/operations.md index fad235e..bc95ce7 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -6,10 +6,10 @@ Running, monitoring, and maintaining LogWisp. ```bash # foreground, explicit config -logwisp -c /etc/logwisp/logwisp.toml +lw -c /etc/logwisp/logwisp.toml # no config: built-in demo pipeline (random source -> stdout) -logwisp +lw ``` There is no built-in daemon mode. Run LogWisp in the foreground under a @@ -42,7 +42,7 @@ auto_reload = true or send a signal: ```bash -kill -HUP $(pidof logwisp) +kill -HUP $(pidof lw) ``` Signals reread the selected TOML file even when `auto_reload` is disabled. @@ -60,8 +60,9 @@ What reload does *not* do: - Preserve connections. Listeners close and reopen, and every SSE, TCP, and chain client is disconnected. Chain sinks reconnect on their own backoff; browsers reconnect SSE automatically; raw TCP consumers must retry themselves. -- Reload certificates without a reload — certificate files are read at plugin - construction, so rotation requires `SIGHUP`. +- Notice rotated certificates on its own: certificate files are read at plugin + construction and `auto_reload` watches only the configuration file, so + rotation requires `SIGHUP`. Plan reloads on a busy relay the way you would plan a restart. Listener bind/start failures happen after the old service stops; these can leave @@ -70,16 +71,17 @@ File-watch errors do not restart services, and queued changes are combined. ### Checking a configuration -There is no validate-only mode. To check a file, start it with debug logging and -watch for pipeline startup: +`lw --check` builds every pipeline and plugin as a start would, without binding +a port or opening, reading or creating a log file, prints the startup warnings, +and exits 0, 1 on an invalid configuration, or 2 when a named file is missing: ```bash -logwisp -c candidate.toml --logging.level=debug --logging.output=stderr +lw --check -c candidate.toml ``` -Success looks like `Created source instance`, `Created sink instance`, and -`Starting pipeline` for each pipeline. Failures name the pipeline and the -offending key: +Before a reload, check the edited file this way: a failed reload keeps the +running service, but `--check` says why without touching it. Failures name the +pipeline and the offending key: ``` ERROR msg="Failed to create pipeline" pipeline=app error="failed to create sink out: port: must be 1-65535, got 0" @@ -145,22 +147,28 @@ curl -s http://127.0.0.1:8080/status | jq . This endpoint is scoped to one sink, not to the whole process, and without an `auth` block it is **unauthenticated**. Bind it to a trusted interface, or -query it with a client certificate or a token from `logwisp auth token`. +query it with a client certificate or a token from `lw auth token`. ### Metrics worth watching -| Metric | Where | Meaning if rising | -|--------|-------|-------------------| -| `dropped_entries` | source | Downstream cannot keep up with the source | -| `total_dropped` | flow | Rate limit or filters are discarding entries (often intended) | -| `total_dropped_by_sink` | pipeline | A sink's input queue is full | -| `dropped_writes` | tcp/http sink | A client's queue overflowed: either it is too slow, or one burst exceeded `client_buffer_size` | -| `rejected_conns` / `rejected_clients` | tcp/http sink, tcp_chain source | `max_connections` is being hit | -| `tls_handshake_errors` | tcp sink, tcp_chain source | Certificate or version mismatch, or scanning | -| `parse_errors` | chain source | Protocol or version skew upstream | -| `reconnects` | chain sink | Unstable link or a flapping downstream | -| `dropped_batches` | http_chain sink | Downstream rejecting batches permanently | -| `synthesized` | chain sink | Events reaching the sink without structure | +Each counter, where it is reported, and what a rise means: + +- `dropped_entries` (source): downstream cannot keep up with the source +- `total_dropped` (flow): the rate limit or filters are discarding entries, + often by intent +- `total_dropped_by_sink` (pipeline): a sink's input queue is full +- `dropped_writes` (`tcp` and `http` sinks): a client's queue overflowed, + because it is too slow or one burst exceeded `client_buffer_size` +- `rejected_conns` (`tcp` sink, `tcp_chain` source), `rejected_clients` + (`http` sink): `max_connections` is being hit; `rejected_conns` also counts + auth refusals +- `tls_handshake_errors` (`tcp` sink, `tcp_chain` source): a certificate or + version mismatch, or scanning +- `parse_errors` (chain sources): protocol or version skew upstream +- `reconnects` (`tcp_chain` sink): an unstable link or a flapping downstream +- `dropped_batches` (`http_chain` sink): downstream is rejecting batches + permanently +- `synthesized` (chain sinks): events reach the sink without structure ## Log Management @@ -264,15 +272,18 @@ sink blocked on an unreachable downstream also holds its full input queue. **Chain link not delivering** -Check `connected` and `reconnects` on the sink, `parse_errors` on the source, -and remember `http_chain` waits up to `flush_interval_ms`. For TLS problems see -[Networking](networking.md#troubleshooting). +Check `connected` and `reconnects` on a `tcp_chain` sink, `parse_errors` on the +source, and remember `http_chain` waits up to `flush_interval_ms`. For TLS +problems see [Networking](networking.md#troubleshooting). **Environment variable override has no effect** LogWisp reads `LOGWISP_QUIET`, `LOGWISP_LOGGING_LEVEL`, and other prefixed -names. Bare names used by older versions must be renamed. Array-indexed paths cannot be set from the -environment or the command line at all. +names. Bare names used by older versions must be renamed. Array-indexed paths, +such as a key inside `[[pipelines]]`, cannot be set from the environment or the +command line; define whole pipelines there with the +[pipeline flags](cli.md#pipelines) or +[pipeline variables](cli.md#pipeline-variables). ## Security Operations diff --git a/doc/scram-auth-plan.md b/doc/scram-auth-plan.md index bdbcfdb..47ac459 100644 --- a/doc/scram-auth-plan.md +++ b/doc/scram-auth-plan.md @@ -6,7 +6,7 @@ channel binding and the mTLS layer was hardened (see on the `auth` release that carries channel binding; then browser logins behind a TLS-terminating proxy. Operator documentation lives in [Security](security.md#password-authentication-scram) and -[`logwisp auth`](cli.md#logwisp-auth). +[`lw auth`](cli.md#lw-auth). **Scope:** optional username/password authentication on every network plugin, next to the existing certificate method, without weakening what mTLS gives. @@ -18,7 +18,7 @@ next to the existing certificate method, without weakening what mTLS gives. - No password or password-equivalent on the wire, no KDF work on listeners, and no credential relay through a TLS-terminating MITM. - Default stays `auth.type = "none"`; existing configurations are unaffected. -- Config in TOML, credential management and viewer clients in a `logwisp auth` +- Config in TOML, credential management and viewer clients in a `lw auth` CLI. ## Non-Goals @@ -94,8 +94,9 @@ Validation at construction: CA-issued certificate plus any valid password (`clientcert=verify-ca`). - `token_lifetime_ms` only on HTTP listeners; default 15 minutes. - `/auth` and every path under it are reserved. -- `mtls` rejects the four SCRAM keys. A block that names peers or credentials - with `type = "none"` is refused: the type was forgotten, not the block. +- `mtls` rejects every SCRAM key, `trusted_proxies` included. A block that + names peers or credentials with `type = "none"` is refused: the type was + forgotten, not the block. **Credentials file** (TOML, written by the CLI, mode 0600): @@ -113,12 +114,12 @@ username = "edge-01" `decoy_key` keeps unknown-user challenges stable across restarts and edits, so a single probe cannot tell a real user from an unknown one (a real user's salt -changes when it is added, rotated or removed). It is created by the first `add-user` and -preserved by every rewrite. An empty user set, a duplicate user, a missing or -short `decoy_key`, an unknown key, or users differing in KDF profile or salt -length are configuration errors. The file is read at construction; the SCRAM -server starts in the plugin's `Start` and stops in `Stop`, so a rejected reload -leaks nothing. +changes when it is added, rotated or removed). It is created by the first +`add-user` and preserved by every rewrite. An empty user set, a duplicate +user, a missing or short `decoy_key`, an unknown key, or users differing in KDF +profile or salt length are configuration errors. The file is read at +construction; the SCRAM server starts in the plugin's `Start` and stops in +`Stop`, so a rejected reload leaks nothing. ## Wire Protocol @@ -140,12 +141,15 @@ silent, and the dialer gives up after its deadline with "no challenge within **HTTP** (`http_chain` source, `http` sink): `POST /auth`, outside the auth middleware, with its own 10 s read/write deadline. It answers `404` unless the -policy is `scram`. +policy is `scram`. Each step, by its request body: -| Step | Request body | Success | Failure | -|------|--------------|---------|---------| -| 1 | hello | `200` challenge | `429` throttled, `503` busy, `400` malformed, `413` oversized | -| 2 | proof | `200` final, token, expires_in | `401 {"error":"authentication failed"}`, `503` if the server stopped | +- Step 1, the hello + - Success: `200` with the challenge. + - Failure: `429` throttled, `503` busy, `400` malformed, `413` oversized. +- Step 2, the proof + - Success: `200` with the final, `token` and `expires_in`. + - Failure: `401 {"error":"authentication failed"}`, or `503` if the server + stopped. Protected endpoints take `Authorization: Bearer `: missing or invalid is `401` with `WWW-Authenticate: Bearer realm="logwisp"`, a certificate-binding miss @@ -157,23 +161,22 @@ checked at connect and outlives its token; a reload ends it. batch is held under backoff); `404`/`405` from `/auth` reads "no auth endpoint at (older logwisp, or not a scram listener)"; an ingest `401` clears the token and is retried (tokens are renewed ahead of expiry, so this follows a -listener reload); `403` drops -the batch; a pin mismatch clears token and pin. Refusals are logged at WARN on -every attempt, paced by the backoff (`Chain connect refused` on `tcp_chain`, -`Chain batch delivery failed` on `http_chain`), and failed logins are counted -in `auth_failures` / `last_auth_error`. +listener reload); `403` drops the batch; a pin mismatch clears token and pin. +Refusals are logged at WARN on every attempt, paced by the backoff (`Chain +connect refused` on `tcp_chain`, `Chain batch delivery failed` on +`http_chain`), and failed logins are counted in `auth_failures` / +`last_auth_error`. ## Throttling Per remote IP (the socket address; in proxy mode the forwarded client, an IPv6 -one per /64), on handshake -starts: a token bucket (burst 10, 1/s) refunded on success, and at most 4 -unfinished exchanges. An unanswered HTTP challenge holds its slot for the -`auth` handshake timeout (30 s); abandoned TCP exchanges release their slot, -and their entry in the `auth` handshake table, immediately. The address table -holds 65,536 entries, drops one idle for a minute once its challenges have -expired, and fails closed when full; the SCRAM server itself caps in-flight -handshakes at 4,096 (`busy`). +one per /64), on handshake starts: a token bucket (burst 10, 1/s) refunded on +success, and at most 4 unfinished exchanges. An unanswered HTTP challenge holds +its slot for the `auth` handshake timeout (30 s); abandoned TCP exchanges +release their slot, and their entry in the `auth` handshake table, +immediately. The address table holds 65,536 entries, drops one idle for a +minute once its challenges have expired, and fails closed when full; the SCRAM +server itself caps in-flight handshakes at 4,096 (`busy`). Counters: `auth_allowed` counts logins; `auth_rejected` every refusal — failed proofs, malformed requests, missing credentials and, on HTTP, refused tokens; @@ -201,10 +204,10 @@ closed. ## CLI ``` -logwisp auth add-user -credentials F -user U [-password-file P] [-generate] -logwisp auth remove-user -credentials F -user U -logwisp auth token -url https://host:port[/path] -user U -password-file P [-unbound] [TLS flags] -logwisp auth stream -addr host:port -user U -password-file P [TLS flags] +lw auth add-user -credentials F -user U [-password-file P] [-generate] +lw auth remove-user -credentials F -user U +lw auth token -url https://host:port[/path] -user U -password-file P [-unbound] [TLS flags] +lw auth stream -addr host:port -user U -password-file P [TLS flags] ``` - `add-user` takes the password from `-password-file` when it exists (at least @@ -266,44 +269,54 @@ SameSite=Strict; Max-Age=`) and carries no token in the body. It has no `Path`: RFC 6265 defaults it to the directory of `POST /auth`, which is the mount behind any proxy prefix. Stream and status accept the cookie or `Authorization: Bearer`; another `Authorization` scheme, such as the site's own -Basic auth, leaves the cookie in charge. Logout is `{"logout": true}` on `POST /auth` itself, -because a clearing cookie set from `/auth/logout` would default to another path -and miss the login cookie. It clears the cookie and revokes a valid token until -it expires (a set of at most 65,536 hashes, gone on reload like every token). An -open stream outlives its token; a reconnect after expiry gets `401`. `/auth` -takes only `application/json`, which a cross-origin page cannot send without a -preflight nobody answers. +Basic auth, leaves the cookie in charge. Logout is `{"logout": true}` on +`POST /auth` itself, because a clearing cookie set from `/auth/logout` would +default to another path and miss the login cookie. It clears the cookie and +revokes a valid token until it expires (a set of at most 65,536 hashes, gone on +reload like every token). An open stream outlives its token; a reconnect after +expiry gets `401`. `/auth` takes only `application/json`, which a cross-origin +page cannot send without a preflight nobody answers. + +**Token mode.** A proof step without `"session"` gets the token in the answer +body, in proxy mode as everywhere else, so a browser that keeps no cookies needs +no server change. `scram.js` asks for it with `{session: "token"}`, reads the +stream through `fetch` with the bearer (`EventSource` cannot send one), and +revokes it with `logout(base, {token})`. The viewer falls back to it when a +cookie would not stick or a fresh cookie login still gets `401`, and keeps the +token in a page variable only: it lasts no longer than the page or its lifetime. **Endpoints.** `/auth` and every path under it are reserved; every URL in the pages is relative, so a proxy prefix works unchanged. -| Path | Purpose | -|------|---------| -| `POST /auth` | SCRAM hello, proof, or logout; unbound in proxy mode | -| `GET /auth/scram.js` | the client library, always in proxy mode | -| `GET /auth/login`, `login.js`, `style.css` | login page (`login_page`) | -| `GET /auth/view`, `view.js` | minimal live viewer (`viewer_page`, needs `login_page`) | +- `POST /auth`: SCRAM hello, proof or logout; unbound in proxy mode. +- `GET /auth/scram.js`: the client library, always served in proxy mode. +- `GET /auth/login`, `login.js`, `style.css`: the login page (`login_page`). +- `GET /auth/view`, `view.js`: a minimal live viewer (`viewer_page`, needs + `login_page`). Files carry `default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`, `nosniff` and `no-referrer`; nothing is inline. The viewer learns a custom `status_path` from a meta tag the sink fills in, renders entries as text, and -sends a signed-out visitor to `login?next=view`; the login page follows `next` -only within its origin. The `proxy_tls` capability satisfies the pipeline's -"auth needs TLS" check. +sends a signed-out visitor to `login?next=view%23signed-in` (the fragment marks +a fresh cookie login; without cookies it shows its own sign-in form); the login +page follows `next` only within its origin. The `proxy_tls` capability +satisfies the pipeline's "auth needs TLS" check. **Client library.** `internal/sink/http/web/scram.js`, embedded with `go:embed`, is one dependency-free ES module, so a site with its own CSP can copy it into -its bundle. It exports `login(base, username, password, {onProgress})` and -`logout(base)`. BLAKE2b and Argon2id are plain JS (no WebAssembly, which would -need `'wasm-unsafe-eval'`), yielding to the page every 50 ms; SHA-256 and HMAC -come from WebCrypto, so it needs a secure context. It checks the challenge as -the Go client does (nonce, salt, cost limits and the dialer's Argon2 floor), +its bundle. It exports `login(base, username, password, {onProgress, session})`, +`logout(base, {token})`, `stream(url, {token, signal, onEvent})` (an event +stream parsed as the HTML standard does), `cookiesUsable()` and +`loginUnavailable()`. BLAKE2b and Argon2id are plain JS (no WebAssembly, which +would need `'wasm-unsafe-eval'`), yielding to the page every 50 ms; SHA-256 and +HMAC come from WebCrypto, so it needs a secure context. It checks the challenge +as the Go client does (nonce, salt, cost limits and the dialer's Argon2 floor), refuses redirects, and reports success only after verifying the server's signature. A 64 MiB login takes about 2 s on a desktop. The wire protocol above is the contract: a site may implement its own client. -**CLI.** `logwisp auth token -unbound` logs in through the proxy, still pinning +**CLI.** `lw auth token -unbound` logs in through the proxy, still pinning its certificate across the two requests; only then may `-url` carry the mount path. @@ -336,29 +349,31 @@ their types and CSP, direct peers refused, pages only in proxy mode, and the CLI: private files with a matching verifier, no password replaced without a source, mode, decoy key and symlink kept across rewrites, an empty file filled, last-user refusal. Under node, `scram.js` against the RFC 9106 vector, Go's -`argon2.IDKey`, and `auth`'s known-answer proof. +`argon2.IDKey`, and `auth`'s known-answer proof; the event-stream line breaks +and fields, the bearer, refusals and abort; a token-mode login and logout. `test/scram-chain-test.sh --auto` (ports 15821-15825): a relay with SCRAM on all four listener types plus one `client_auth` + SCRAM port; authorized edges over both chain transports; node labels forced to usernames; wrong password, unknown -user and no-auth edges deliver nothing; a `tcp` sink viewer through `logwisp auth +user and no-auth edges deliver nothing; a `tcp` sink viewer through `lw auth stream` held past the exchange deadline; a raw TLS client without a hello gets -nothing; the `http` sink through `logwisp auth token` and curl; missing and bad +nothing; the `http` sink through `lw auth token` and curl; missing and bad tokens get `401`; `remove-user` + `SIGHUP` revokes token and login; throttling last. `test/scram-proxy-test.sh --auto` (ports 15831-15832): a Go reverse proxy ending TLS in front of a plaintext `http` sink mounted at `/logs/`; headless Chromium sent to the login page, refused a wrong password, streaming events after login, with the cookie scoped to `/logs` and hidden from scripts, cleared -by sign-out, and no CSP violations; `token -unbound` and curl through the -proxy; direct peers and plaintext-forwarded requests `403`. The existing -scripts keep passing. +by sign-out, and no CSP violations; a cookie the browser drops leading to the +viewer's own form, not a loop, its token kept out of storage and the URL; a +profile blocking every cookie signing in on the viewer, its proofs asking for +no cookie, streaming with a bearer, sign-out revoking the token, and a reload's +`401` asking again; `token -unbound` and curl through the proxy; direct peers +and plaintext-forwarded requests `403`. The existing scripts keep passing. ## Not Implemented -1. **PROXY protocol behind TLS passthrough.** Every client then shares the - proxy's address, and so one throttling budget that a single client can - exhaust. Accepting PROXY v2 from listed proxies would restore per-client - throttling on the TCP listeners and on HTTP listeners outside proxy mode. +1. **PROXY protocol behind TLS passthrough**: planned in + [To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd). ## mTLS Hardening diff --git a/doc/security.md b/doc/security.md index 63ad7f2..81cb0cd 100644 --- a/doc/security.md +++ b/doc/security.md @@ -5,21 +5,37 @@ configure it, and — equally important — what it does not yet do. ## Current State -| Capability | Status | -|------------|--------| -| TLS 1.2 / 1.3 on all network sources and sinks | Implemented | -| Server certificate verification by dialers | Implemented | -| Mutual TLS (client certificate required and verified) | Implemented at the transport layer | -| Peer identity recorded per session | Implemented | -| Authorization from certificate identity (allow-lists, node binding) | Implemented — see [The Auth Block](#the-auth-block) | -| Password (Argon2id-SCRAM) authentication, bound to the TLS channel | Implemented — see [Password Authentication](#password-authentication-scram) | -| Authentication on the `http` sink's stream and status endpoints | Implemented: client certificate or bearer token | -| Browser logins behind a site's TLS-terminating proxy | Implemented — see [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy) | -| Server pinning by dialers | Implemented: certificate identity (`mtls`), bound certificate (`scram`) | -| Startup warnings for expiring certificates and risky settings | Implemented — see [Startup Warnings](#startup-warnings) | -| Unknown configuration keys rejected | Implemented — a typo in `tls`, `auth` or a table path fails startup | -| Certificate revocation lists (CRL) or OCSP | **Not implemented** — revoke by editing the allow-list or credentials file | -| IP allow/deny lists, per-IP connection or request limits | **Not implemented** — only SCRAM logins are throttled per address | +**Implemented:** + +- TLS 1.2 / 1.3 on all network sources and sinks. +- Server certificate verification by dialers. +- Mutual TLS (client certificate required and verified), at the transport + layer. +- Peer identity recorded per session. +- Authorization from certificate identity (allow-lists, node binding): see + [The Auth Block](#the-auth-block). +- Password (Argon2id-SCRAM) authentication, bound to the TLS channel: see + [Password Authentication](#password-authentication-scram). +- Authentication on the `http` sink's stream and status endpoints: client + certificate or bearer token. +- Browser logins behind a site's TLS-terminating proxy: see + [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy). +- Server pinning by dialers: certificate identity (`mtls`), bound certificate + (`scram`). +- Startup warnings for expiring certificates and risky settings: see + [Startup Warnings](#startup-warnings). +- Unknown configuration keys rejected: a typo in `tls`, `auth` or a table path + fails startup. + +**Not implemented:** + +- Certificate revocation lists (CRL) or OCSP: revoke by editing the allow-list + or credentials file. +- IP allow/deny lists, per-IP connection or request limits: only SCRAM logins + are throttled per address. +- PROXY protocol: behind a proxy that passes TLS through, every peer shares the + proxy's address; see + [Behind nginx or another proxy](#behind-nginx-or-another-proxy). Two credentials are supported. Certificates (`mtls`) are the one the transport already carries: the `tls` block establishes that a peer chains to your CA, and @@ -46,17 +62,23 @@ insecure_skip_verify = false min_version = "1.3" ``` -| Option | Role | Default | Description | -|--------|------|---------|-------------| -| `enabled` | both | `false` | Master switch; when false the whole block is ignored | -| `cert_file` | both | — | Local certificate. **Required** for listeners; optional client identity for dialers | -| `key_file` | both | — | Private key for `cert_file`. Must be set together with it | -| `client_auth` | listener | `false` | Require and verify a client certificate (mTLS) | -| `client_ca_file` | listener | — | CA bundle used to verify client certificates. **Required** when `client_auth` is true | -| `ca_file` | dialer | system store | CA bundle used to verify the server certificate | -| `server_name` | dialer | the configured `host` | SNI and certificate name to verify against | -| `insecure_skip_verify` | dialer | `false` | Disable server verification | -| `min_version` | both | `"1.3"` | `"1.2"` or `"1.3"` | +Options by role, each with its default: + +- Both roles + - `enabled` (`false`): master switch; when false the whole block is ignored. + - `cert_file`: local certificate; **required** for listeners, an optional + client identity for dialers. + - `key_file`: private key for `cert_file`; set the two together. + - `min_version` (`"1.3"`): `"1.2"` or `"1.3"`. +- Listeners + - `client_auth` (`false`): require and verify a client certificate (mTLS). + - `client_ca_file`: CA bundle that verifies client certificates; **required** + when `client_auth` is true. +- Dialers + - `ca_file` (system store): CA bundle that verifies the server certificate. + - `server_name` (the configured `host`): SNI and certificate name to verify + against. + - `insecure_skip_verify` (`false`): disable server verification. Listeners are the `tcp` and `http` sinks and the `tcp_chain` and `http_chain` sources; dialers are the `tcp_chain` and `http_chain` sinks. @@ -99,26 +121,44 @@ password_file = "" # scram dialers trusted_proxies = [] # scram http sink behind a TLS-terminating proxy ``` -| Option | Applies to | Default | Description | -|--------|------------|---------|-------------| -| `type` | all | `none` | `none` ignores the block; `mtls` authorizes by certificate identity; `scram` by password | -| `identity` | `mtls`; `scram` listeners | `cn` (`mtls`), unset (`scram`) | Certificate field carrying the identity. Under `scram` it binds the client certificate to the user | -| `allow` | `mtls` | `[]` | Exact identities to admit | -| `allow_patterns` | `mtls` | `[]` | RE2 patterns matched against the identity; anchor them yourself | -| `node_binding` | chain sources | `force` | `none`, `assert`, or `force` | -| `credentials_file` | `scram` listeners | — | Verifiers written by [`logwisp auth add-user`](cli.md#logwisp-auth) | -| `token_lifetime_ms` | `scram` on the `http` sink and `http_chain` source | 15 minutes | Bearer token lifetime, 10 s to 24 h | -| `username` | `scram` dialers | — | User to log in as | -| `password_file` | `scram` dialers | — | File holding the password; one trailing line break is trimmed | -| `trusted_proxies` | `scram` on the `http` sink | `[]` | Addresses or CIDRs of the reverse proxies that end the browsers' TLS; see [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy) | +Options by where they apply, each with its default: -**Roles by plugin:** +- Every plugin + - `type` (`none`): `none` ignores the block; `mtls` authorizes by + certificate identity, `scram` by password. +- Chain sources + - `node_binding` (`force`): `none`, `assert` or `force`. +- `mtls`, and `scram` listeners + - `identity`: the certificate field carrying the identity; `cn` under + `mtls`, unset under `scram`, where it binds the client certificate to the + user. +- `mtls` + - `allow` (`[]`): exact identities to admit. + - `allow_patterns` (`[]`): RE2 patterns matched against the identity; anchor + them yourself. +- `scram` listeners + - `credentials_file`: verifiers written by + [`lw auth add-user`](cli.md#lw-auth). + - `token_lifetime_ms` (15 minutes): bearer token lifetime, 10 s to 24 h; the + `http` sink and `http_chain` source only. + - `trusted_proxies` (`[]`): addresses or CIDRs of the reverse proxies that + end the browsers' TLS; the `http` sink only, see + [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy). +- `scram` dialers + - `username`: user to log in as. + - `password_file`: file holding the password; one trailing line break is + trimmed. -| Plugin | Role | Decides | -|--------|------|---------| -| `tcp_chain` source, `http_chain` source | Listener | Which senders may ingest, and what node label their entries carry | -| `tcp` sink, `http` sink | Listener | Which clients may read the stream (and, on `http`, the status endpoint) | -| `tcp_chain` sink, `http_chain` sink | Dialer | `mtls`: which server identity to accept, beyond hostname verification. `scram`: which user to log in as | +**Roles by plugin**, and what the block decides: + +- Listeners + - `tcp_chain` and `http_chain` sources: which senders may ingest, and what + node label their entries carry. + - `tcp` and `http` sinks: which clients may read the stream (and, on + `http`, the status endpoint). +- Dialers: the `tcp_chain` and `http_chain` sinks. + - `mtls`: which server identity to accept, beyond hostname verification. + - `scram`: which user to log in as. ### Identity @@ -127,12 +167,13 @@ certificate; under `scram` it is the username. The handshake has already checked the chain, signature, and validity window, so this is pure field selection. -| Mode | Source | Typical use | -|------|--------|-------------| -| `cn` (default) | `Subject.CommonName` | Matches the existing `tls_peer_cn` metadata | -| `san_dns` | first DNS SAN | Host identities | -| `san_uri` | first URI SAN | SPIFFE-style IDs | -| `san_email` | first email SAN | Operator identities | +Modes, each with the field it reads and its typical use: + +- `cn` (default): `Subject.CommonName`; matches the existing `tls_peer_cn` + metadata. +- `san_dns`: the first DNS SAN; host identities. +- `san_uri`: the first URI SAN; SPIFFE-style IDs. +- `san_email`: the first email SAN; operator identities. A certificate with no usable value in the chosen field is rejected. An empty identity is a refusal, not an empty match. @@ -159,13 +200,19 @@ reported at startup. ### Node binding -`node_binding` applies only to the chain sources, and it overrides `trust_node`. +`node_binding` applies only to the chain sources. `assert` and `force` take the +connection label over from `trust_node`; only `force` also overrides the +per-entry labels. -| Value | Connection label | Per-entry `node` field | -|-------|------------------|------------------------| -| `none` | `trust_node` governs | `trust_node` governs | -| `assert` | Must equal the identity; a mismatch or an omission is rejected | `trust_node` governs | -| `force` | Ignored; the identity is used | Overwritten with the identity | +- `none`: `trust_node` governs both the connection label and the per-entry + `node` field. +- `assert` + - Connection label: must equal the identity; a mismatch or an omission is + rejected. + - Per-entry `node` field: `trust_node` governs. +- `force` + - Connection label: ignored; the identity is used. + - Per-entry `node` field: overwritten with the identity. Use **`force`** on an ingest boundary you do not trust. Every entry is relabelled, so a compromised edge cannot smuggle a foreign origin through the @@ -206,14 +253,14 @@ was shown instead; see [Channel binding](#channel-binding). Misconfiguration fails at plugin construction, before the pipeline starts: -- `type` `mtls` or `scram` without `tls.enabled` (proxy mode aside), or on a dialer with - `tls.insecure_skip_verify`: an identity read from an unverified chain is a - claim, and an unverified server could relay a login +- `type` `mtls` or `scram` without `tls.enabled` (proxy mode aside), or on a + dialer with `tls.insecure_skip_verify`: an identity read from an unverified + chain is a claim, and an unverified server could relay a login - `type = "mtls"` on a listener without `tls.client_auth` - a block naming peers or credentials (`allow`, `allow_patterns`, `credentials_file`, `token_lifetime_ms`, `username`, `password_file`, - `trusted_proxies`) whose - `type` is `none` or unset: auth was intended and the type forgotten + `trusted_proxies`) whose `type` is `none` or unset: auth was intended and the + type forgotten - a key of the other method: `allow` or `allow_patterns` under `scram`, a `scram` key under `mtls` - a `scram` listener without a loadable `credentials_file` (see @@ -350,7 +397,7 @@ end to end — run it with `--auto` to see each guarantee asserted. `type = "scram"` authenticates peers by username and password with Argon2id-SCRAM from `lixenwraith/auth`. It needs TLS but no client certificates, works on all six network plugins, and is LogWisp's own protocol, -not standard SASL: viewers use the [`logwisp auth`](cli.md#logwisp-auth) CLI or, +not standard SASL: viewers use the [`lw auth`](cli.md#lw-auth) CLI or, behind a TLS-terminating proxy, the shipped browser client. ```toml @@ -405,7 +452,7 @@ connection. After the challenge the dialer pins the certificate it was bound to: the proof and every ingest request must meet the same certificate, or the TLS handshake fails before anything is sent. The `http_chain` sink then drops token and pin and logs in again, which also covers a rotated server certificate. A -token printed by `logwisp auth token` is not pinned. +token printed by `lw auth token` is not pinned. ### Certificates and users @@ -433,21 +480,22 @@ exchange. Logins are throttled per remote socket address or, on an `http` sink in proxy mode, per forwarded client (an IPv6 client per /64); forwarded headers are read -only from `trusted_proxies`. Each exchange takes a token from a bucket of 10 that refills at one per -second, and a successful login gives it back, so only failed or abandoned -attempts drain it. At most 4 exchanges per address may be unfinished: an HTTP -challenge never answered holds its slot for up to 30 s, a TCP connection that -ends mid-exchange frees it at once. A refused start answers `too many attempts` -(HTTP `429`) and counts `auth_throttled`; the table holds 65,536 addresses and -refuses new ones when full. Separately, at most 4,096 exchanges may be in flight -per listener; beyond that, or while the plugin stops, a login gets `busy` (HTTP -`503`) and counts `auth_busy`. Peers behind one NAT or one passthrough proxy -share a bucket: there, one client can exhaust every other client's logins. +only from `trusted_proxies`. Each exchange takes a token from a bucket of 10 +that refills at one per second, and a successful login gives it back, so only +failed or abandoned attempts drain it. At most 4 exchanges per address may be +unfinished: an HTTP challenge never answered holds its slot for up to 30 s, a +TCP connection that ends mid-exchange frees it at once. A refused start answers +`too many attempts` (HTTP `429`) and counts `auth_throttled`; the table holds +65,536 addresses and refuses new ones when full. Separately, at most 4,096 +exchanges may be in flight per listener; beyond that, or while the plugin +stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`. Peers behind +one NAT or one passthrough proxy share a bucket: there, one client can exhaust +every other client's logins. ### Credentials file ```toml -# logwisp SCRAM verifiers, written by `logwisp auth add-user`. Keep it private. +# logwisp SCRAM verifiers, written by `lw auth add-user`. Keep it private. decoy_key = "" [[users]] @@ -460,7 +508,7 @@ stored_key = "" username = "edge-01" ``` -Write it with [`logwisp auth add-user`](cli.md#logwisp-auth) rather than by +Write it with [`lw auth add-user`](cli.md#lw-auth) rather than by hand. The whole file is validated when the plugin is built: a `decoy_key` of at least 32 bytes, at least one user, unique names, no unknown keys, and one Argon2 profile and salt length for every user, since mixed profiles would tell a prober @@ -477,14 +525,14 @@ different users. - **Rollout.** Turning `scram` on cuts off every dialer of that listener that has no credentials. Upgrade every binary first, then add a second listener with `scram` on another port, move the dialers to it, and remove the old one. -- **Password rotation.** `logwisp auth add-user -generate` with the user's +- **Password rotation.** `lw auth add-user -generate` with the user's `-password-file` replaces verifier and password together. `SIGHUP` the listener, deploy the password file, `SIGHUP` the dialer. Logins fail in between and the dialer retries under backoff, holding its current entry or batch while its input queue fills. To avoid the gap, add a second user, move the dialer to it, then remove the first; under `node_binding = "force"` the node label follows the username. -- **Revocation.** `logwisp auth remove-user`, then `SIGHUP`. The reload drops +- **Revocation.** `lw auth remove-user`, then `SIGHUP`. The reload drops every connection and revokes every token; other dialers log in again on their own. @@ -493,14 +541,15 @@ different users. - TLS must terminate at LogWisp, except on an `http` sink in proxy mode. Behind any other terminating proxy or load balancer every login fails by design; pass TLS through instead (TCP or SNI routing). All clients then share the - proxy's address, and so one throttling budget. + proxy's address, and so one throttling budget: see + [Behind nginx or another proxy](#behind-nginx-or-another-proxy). - One process per HTTP address. Handshake state and the token key live in one instance, so behind a balancer the proof or the token can reach an instance that never saw the login. - Browsers log in only to an `http` sink in proxy mode, below; elsewhere use `mtls` for browsers. - `nc` and `openssl s_client` cannot read a `scram` `tcp` sink; use - `logwisp auth stream`. + `lw auth stream`. `test/scram-chain-test.sh --auto` exercises these guarantees end to end. @@ -541,28 +590,169 @@ location /logs/ { could relay a login. Keep that hop on loopback or a trusted network, or enable `tls` on the sink; a plaintext hop to a proxy off this host is warned about at startup. `identity` cannot be combined with proxy mode. -- **Sessions.** The page asks for a cookie: `logwisp_session`, `HttpOnly`, - `Secure`, `SameSite=Strict`, `Max-Age` the token lifetime, and no `Path`, so - it scopes itself to the mount (`/logs` above). Stream and status accept it or - a bearer token, so `new EventSource("/logs/stream")` works on any page of the - site, beside the site's own Basic auth too, and `logwisp auth token -unbound` - keeps working. `POST /auth` with - `{"logout": true}` clears the cookie and revokes the token until it expires. - `/auth` takes only `application/json`, which a cross-origin page cannot send - without a preflight. +- **Sessions.** The login page asks for a cookie: `logwisp_session`, + `HttpOnly`, `Secure`, `SameSite=Strict`, `Max-Age` the token lifetime, and no + `Path`, so it scopes itself to the mount (`/logs` above). Stream and status + accept it or a bearer token, so `new EventSource("/logs/stream")` works on any + page of the site, beside the site's own Basic auth too, and `lw auth token + -unbound` keeps working. `POST /auth` with `{"logout": true}` clears the + cookie and revokes the token until it expires. `/auth` takes only + `application/json`, which a cross-origin page cannot send without a preflight. +- **Private windows.** Private or incognito windows keep cookies in memory, + apart from normal windows: they sign in on their own, and the session ends at + the token lifetime or when the last private window closes. +- **Cookies disabled.** The viewer runs in token mode when the browser keeps no + cookie for the site. + - It notices from `navigator.cookieEnabled`, from a throwaway cookie with the + session's attributes that does not come back (Chromium blocking every cookie + still reports `cookieEnabled`), or from a `401` right after a cookie login, + and shows its own sign-in form. + - The token lives in a page variable only, never in storage or the URL. It + lasts as long as the page and at most the token lifetime: a reload, another + tab, or a reconnect after expiry asks again; sign out revokes it. + - The login page says that cookies are unavailable and links to the viewer. + A page not served over HTTPS keeps no `Secure` cookie either: both pages + name HTTPS instead and offer no sign-in. +- **Integrating `scram.js`.** A site with its own CSP can copy `scram.js` from + `internal/sink/http/web/` into its bundle: one dependency-free ES module, + where `base` is the mount URL ending in `/`. It needs a secure context for + WebCrypto and takes about 2 s of Argon2 per login on a desktop. + - Cookie mode, the default: `login(base, username, password, {onProgress})`, + then `EventSource` and `fetch` as for any same-origin resource; + `logout(base)`. + - Token mode: `login(..., {session: "token"})` resolves to `{username, + expiresIn, token}`. Keep the token in memory; `stream(url, {token, signal, + onEvent})` reads the event stream through `fetch` with the bearer, status + takes `Authorization: Bearer`, and `logout(base, {token})` revokes it. The + bearer replaces the Basic credentials a browser would send, so token mode + cannot pass a proxy that asks for its own Basic auth. + - `cookiesUsable()` tells which applies: false when the session cookie would + not stick. `loginUnavailable()`, asked first, names why neither can run + (no secure context), or is empty. - **Pages.** Under `/auth/` the sink serves `scram.js` always, and with `login_page` / `viewer_page` the login page and a minimal live viewer with their script and style, under `default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; form-action 'self'; frame-ancestors - 'none'; base-uri 'none'`. A site with its own CSP can copy `scram.js` from - `internal/sink/http/web/` into its bundle: one dependency-free ES module - exporting `login(base, username, password, {onProgress})` and - `logout(base)`, where `base` is the mount URL ending in `/`. It needs a - secure context for WebCrypto and takes about 2 s of Argon2 per login on a - desktop. + 'none'; base-uri 'none'`. `test/scram-proxy-test.sh --auto` logs in from headless Chromium through such a -proxy. +proxy, with cookies and with every cookie blocked. + +### Behind nginx or another proxy + +A proxy in front of LogWisp works at one of two layers: + +- **L7, ending TLS** (nginx `http` block, HAProxy `mode http`): the `http` sink + in [proxy mode](#browsers-behind-a-tls-terminating-proxy). + - `trusted_proxies` lists the proxy's address. + - The proxy sends `X-Forwarded-Proto: https` and `X-Forwarded-For`. It may + overwrite `X-Forwarded-For` with the real client: the rightmost hop that is + not a proxy is the client either way. +- **L4, passing TLS through to LogWisp's own TLS** (nginx `stream` with + `ssl_preread`, HAProxy `mode tcp`): whatever needs TLS to end at LogWisp. + - Chain links (`tcp_chain`, `http_chain`). + - `lw auth stream` viewers of a `tcp` sink. + - An `http` sink that keeps its own TLS, and with it SCRAM channel binding + or client certificates. + +A common shape serves both from port 443: + +``` +:443 nginx stream: ssl_preread routes by SNI, proxy_protocol on + logs.example.org -> 127.0.0.1:8443 nginx http + relay.example.org -> a LogWisp TLS listener, without PROXY (below) +127.0.0.1:8443 nginx http: listens with proxy_protocol, ends TLS + /logs/ -> 127.0.0.1:8081 LogWisp http sink, proxy mode +``` + +**The `http` sink behind the `http` block** works in proxy mode as is: +`trusted_proxies` is the `http` block's address, usually `127.0.0.1`, and the +block names the client from the PROXY header. + +```nginx +server { + listen 127.0.0.1:8443 ssl proxy_protocol; + server_name logs.example.org; + # ssl_certificate, ssl_certificate_key: the site's own + location /logs/ { + proxy_pass http://127.0.0.1:8081/; + proxy_set_header X-Real-IP $proxy_protocol_addr; + proxy_set_header X-Forwarded-For $proxy_protocol_addr; + proxy_set_header X-Forwarded-Proto https; + } +} +``` + +`$proxy_add_x_forwarded_for` does not fit this shape: it appends the stream +server's address, not the client's. LogWisp would name that address as every +client or, when it is trusted (both on `127.0.0.1`), the hop to its left: one +the client wrote, so the client picks its own throttling address. LogWisp +ignores `X-Real-IP`. + +**LogWisp's own TLS listeners take no PROXY header.** LogWisp cannot read the +PROXY protocol yet; it is planned first in the ACL work +([To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd)). +A stream route that sends `proxy_protocol` straight to a LogWisp TLS listener +breaks every connection: the header lands in front of the TLS handshake. nginx +sets `proxy_protocol` per stream `server`, so route LogWisp SNIs without it: + +- Route 1: LogWisp gets its own stream `server` and port, without + `proxy_protocol`. +- Route 2: port 443 sends LogWisp SNIs to an internal stream `server` that + `listen`s with `proxy_protocol` and proxy_passes to LogWisp without it; that + hop consumes the header. +- Either way dialers must send the name: a `host` that is one, or + `tls.server_name` (`-server-name` for `lw auth`); an IP literal sends no SNI. +- With HAProxy, leave `send-proxy` off the LogWisp backends. + +```nginx +stream { + map $ssl_preread_server_name $route { + relay.example.org 127.0.0.1:10443; # route 2: the internal hop + tail.example.org 127.0.0.1:10443; + default 127.0.0.1:8443; # the http block, logs.example.org + } + map $ssl_preread_server_name $logwisp { + relay.example.org 127.0.0.1:9001; # tcp_chain source + tail.example.org 127.0.0.1:9002; # tcp sink, for lw auth stream + } + server { + listen 443; + ssl_preread on; + proxy_protocol on; # on every route of this server + proxy_pass $route; + } + server { # route 1: its own port, no header + listen 9443; + ssl_preread on; + proxy_pass $logwisp; + } + server { # route 2: consumes the header + listen 127.0.0.1:10443 proxy_protocol; + ssl_preread on; + proxy_pass $logwisp; + } +} +``` + +**Passthrough without PROXY** brings every peer from the proxy's address: + +- All peers share one SCRAM [throttling](#throttling) budget: 10 failed or + abandoned logins, then one per second, and 4 unfinished at once. One client + that keeps failing logins makes every other client's logins answer + `too many attempts` until it stops. +- Established `tcp_chain` links and open streams keep flowing; a reconnect + waits for the budget. An `http_chain` sink whose early renewal is put off + (`too many attempts` or `busy`) keeps sending on its token to the server it + pinned, retrying every few seconds, and holds its batches only once that + token has expired; any other refusal ends the token at once. +- Logs and sessions name the proxy, not the peer. +- Proxy mode is unaffected: it throttles on the forwarded client. +- Until LogWisp reads PROXY, mitigate: + - expose the chain ports directly, without the proxy, where you can; + - keep the proxy hop on a trusted network, so only trusted clients share the + budget; + - or accept the shared budget. ## What Each Layer Enforces @@ -598,12 +788,17 @@ would add. An `auth` block (`mtls` or `scram`) closes each of these. Without one, bind them to a trusted interface or front them with an authenticating proxy. -| Surface | Exposure when `auth.type = "none"` | -|---------|-----------------------------------| -| `http` sink `stream_path` | Full log stream, with `Access-Control-Allow-Origin: *`, so any browser origin can read it (the header is omitted once an auth policy is set) | -| `http` sink `status_path` | Host, port, TLS flag, uptime, client counts, throughput counters | -| `tcp` sink | Full log stream to any client that connects | -| `tcp_chain` / `http_chain` source | Ingest from any peer that can connect (with `client_auth`, any the CA vouches for), under any node label it claims | +What each exposes when `auth.type = "none"`: + +- `http` sink `stream_path`: the full log stream, with + `Access-Control-Allow-Origin: *`, so any browser origin can read it (the + header is omitted once an auth policy is set). +- `http` sink `status_path`: host, port, TLS flag, uptime, client counts, + throughput counters. +- `tcp` sink: the full log stream to any client that connects. +- `tcp_chain` and `http_chain` sources: ingest from any peer that can connect + (with `client_auth`, any the CA vouches for), under any node label it + claims. `max_connections` bounds concurrency on all of them but does not distinguish callers. @@ -618,7 +813,7 @@ where TLS ends at the trusted proxy; `mtls` also requires `client_auth`. - Use a dedicated CA for LogWisp so its trust decisions stay independent. - Keep leaf lifetimes short (90–825 days) and automate renewal. - Key, credentials and password files should be `0600` and owned by the service - account; a world-readable one is reported at startup. `logwisp auth` creates + account; a world-readable one is reported at startup. `lw auth` creates them `0600` and keeps the mode of an existing file. - Rotation requires a reload (`SIGHUP`), because certificates and credentials are loaded once at plugin construction; there is no on-disk watch for them. diff --git a/doc/sinks.md b/doc/sinks.md index 80f61ca..c18bdbf 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -27,10 +27,10 @@ target = "stdout" buffer_size = 1000 ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `target` | string | `stdout` | `stdout` or `stderr` | -| `buffer_size` | int | `1000` | Sink input queue depth | +Options, each as type and default: + +- `target` (string, `stdout`): `stdout` or `stderr`. +- `buffer_size` (int, `1000`): sink input queue depth. > `split` is **not** a valid target for this sink and is rejected at startup. > Level-based splitting exists only for LogWisp's own application log @@ -60,16 +60,16 @@ buffer_size = 1000 flush_interval_ms = 100 ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `directory` | string | **required** | Output directory | -| `name` | string | **required** | Base filename | -| `max_size_mb` | int | `100` | Rotate when the active file reaches this size | -| `max_total_size_mb` | int | `1000` | Cap across all rotated files | -| `min_disk_free_mb` | int | `0` | Free-space floor before writing; `0` = no floor | -| `retention_hours` | float | `168.0` | Delete rotated files older than this | -| `buffer_size` | int | `1000` | Sink input queue depth | -| `flush_interval_ms` | int | `100` | Forced flush interval | +Options, each as type and default: + +- `directory` (string, **required**): output directory. +- `name` (string, **required**): base filename. +- `max_size_mb` (int, `100`): rotate when the active file reaches this size. +- `max_total_size_mb` (int, `1000`): cap across all rotated files. +- `min_disk_free_mb` (int, `0`): free-space floor before writing; `0` = none. +- `retention_hours` (float, `168.0`): delete rotated files older than this. +- `buffer_size` (int, `1000`): sink input queue depth. +- `flush_interval_ms` (int, `100`): forced flush interval. > `min_disk_free_mb` has an unusual default. The constructor replaces only > *negative* values with `100`; leaving the key unset yields `0`, which means no @@ -125,20 +125,25 @@ type = "mtls" allow = ["viewer-01"] ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | `0.0.0.0` | Bind address; IPv4 only | -| `port` | int | **required** | Listen port | -| `stream_path` | string | `/stream` | SSE endpoint; must start with `/` | -| `status_path` | string | `/status` | Status endpoint; must start with `/` and differ from `stream_path` | -| `buffer_size` | int | `1000` | Sink input queue depth | -| `client_buffer_size` | int | `256` | Per-client send queue depth | -| `write_timeout_ms` | int | `0` | Per-event write deadline; `0` = none | -| `max_connections` | int | `0` | Concurrent stream cap; `0` = unlimited | -| `login_page` | bool | `false` | Serve the browser login page at `/auth/login`; needs `auth.trusted_proxies` | -| `viewer_page` | bool | `false` | Serve a minimal live viewer at `/auth/view`; needs `login_page` | -| `tls` | table | — | Listener TLS; see [Security](security.md) | -| `auth` | table | — | Client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block) | +Options, each as type and default: + +- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the listener + keeps to its family ([Networking](networking.md#address-family)). +- `port` (int, **required**): listen port. +- `stream_path` (string, `/stream`): SSE endpoint; must start with `/`. +- `status_path` (string, `/status`): status endpoint; must start with `/` and + differ from `stream_path`. +- `buffer_size` (int, `1000`): sink input queue depth. +- `client_buffer_size` (int, `256`): per-client send queue depth. +- `write_timeout_ms` (int, `0`): per-event write deadline; `0` = none. +- `max_connections` (int, `0`): concurrent stream cap; `0` = unlimited. +- `login_page` (bool, `false`): serve the browser login page at `/auth/login`; + needs `auth.trusted_proxies`. +- `viewer_page` (bool, `false`): serve a minimal live viewer at `/auth/view`; + needs `login_page`. +- `tls` (table): listener TLS; see [Security](security.md). +- `auth` (table): client authentication (`mtls` or `scram`); see + [Security](security.md#the-auth-block). **Behaviour** @@ -148,7 +153,7 @@ allow = ["viewer-01"] - With an `auth` block, one middleware gates **both** endpoints, with no body detail in a refusal. Under `mtls` a refused certificate gets `403`. Under `scram` a client logs in at `POST /auth`, which sits outside the gate (see - [`logwisp auth token`](cli.md#logwisp-auth)), and sends + [`lw auth token`](cli.md#lw-auth)), and sends `Authorization: Bearer `; a missing, invalid or expired token gets `401` with `WWW-Authenticate`, a certificate that does not match the token's user `403`. A stream is checked when it connects and outlives its token; a @@ -233,18 +238,20 @@ type = "mtls" allow = ["viewer-01"] ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | `0.0.0.0` | Bind address; IPv4 only | -| `port` | int | **required** | Listen port | -| `buffer_size` | int | `1000` | Sink input queue depth | -| `client_buffer_size` | int | `256` | Per-client send queue depth | -| `write_timeout_ms` | int | `5000` | Per-write deadline | -| `keep_alive` | bool | `true` | Enable TCP keep-alive on accepted connections | -| `keep_alive_period_ms` | int | `30000` | Keep-alive idle period | -| `max_connections` | int | `0` | Concurrent connection cap; `0` = unlimited | -| `tls` | table | — | Listener TLS | -| `auth` | table | — | Client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block) | +Options, each as type and default: + +- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the listener + keeps to its family ([Networking](networking.md#address-family)). +- `port` (int, **required**): listen port. +- `buffer_size` (int, `1000`): sink input queue depth. +- `client_buffer_size` (int, `256`): per-client send queue depth. +- `write_timeout_ms` (int, `5000`): per-write deadline. +- `keep_alive` (bool, `true`): TCP keep-alive on accepted connections. +- `keep_alive_period_ms` (int, `30000`): keep-alive idle period. +- `max_connections` (int, `0`): concurrent connection cap; `0` = unlimited. +- `tls` (table): listener TLS. +- `auth` (table): client authentication (`mtls` or `scram`); see + [Security](security.md#the-auth-block). **Behaviour** @@ -254,7 +261,7 @@ allow = ["viewer-01"] first: a hello carrying the login, then challenge, proof and final, all within 10 s, after which the stream follows on the same connection. `nc` and `openssl s_client` cannot do this; use - [`logwisp auth stream`](cli.md#logwisp-auth). + [`lw auth stream`](cli.md#lw-auth). - A write that misses its deadline means the kernel buffer stayed full for the whole timeout, so the client is disconnected immediately rather than retried. - A client whose send queue is full has that event dropped (`dropped_writes`) @@ -300,20 +307,22 @@ cert_file = "/etc/logwisp/tls/client.crt" key_file = "/etc/logwisp/tls/client.key" ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | **required** | Downstream host | -| `port` | int | **required** | Downstream port | -| `node` | string | `os.Hostname()` | Origin label stamped on first-hop entries | -| `buffer_size` | int | `1000` | Sink input queue depth | -| `dial_timeout_ms` | int | `5000` | TCP connect timeout | -| `write_timeout_ms` | int | `5000` | Per-write deadline | -| `backoff_min_ms` | int | `500` | Reconnect backoff floor | -| `backoff_max_ms` | int | `30000` | Reconnect backoff ceiling | -| `keep_alive` | bool | `true` | Enable TCP keep-alive | -| `keep_alive_period_ms` | int | `30000` | Keep-alive idle period | -| `tls` | table | — | Dialer TLS; `cert_file`/`key_file` present a client identity | -| `auth` | table | — | Server identity pinning (`mtls`) or a login (`scram`); see [Security](security.md#the-auth-block) | +Options, each as type and default: + +- `host` (string, **required**): downstream host or address; an IPv6 one goes + bare (`::1`). +- `port` (int, **required**): downstream port. +- `node` (string, `os.Hostname()`): origin label stamped on first-hop entries. +- `buffer_size` (int, `1000`): sink input queue depth. +- `dial_timeout_ms` (int, `5000`): TCP connect timeout. +- `write_timeout_ms` (int, `5000`): per-write deadline. +- `backoff_min_ms` (int, `500`): reconnect backoff floor. +- `backoff_max_ms` (int, `30000`): reconnect backoff ceiling. +- `keep_alive` (bool, `true`): TCP keep-alive. +- `keep_alive_period_ms` (int, `30000`): keep-alive idle period. +- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity. +- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see + [Security](security.md#the-auth-block). **Behaviour** @@ -380,21 +389,23 @@ cert_file = "/etc/logwisp/tls/client.crt" key_file = "/etc/logwisp/tls/client.key" ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | **required** | Downstream host | -| `port` | int | **required** | Downstream port | -| `ingest_path` | string | `/ingest` | Endpoint path; must start with `/` | -| `node` | string | `os.Hostname()` | Origin label stamped on first-hop entries | -| `buffer_size` | int | `1000` | Sink input queue depth | -| `max_batch_count` | int | `100` | Flush after this many entries | -| `max_batch_bytes` | int | `1048576` | Flush after this many bytes (1 MiB) | -| `flush_interval_ms` | int | `1000` | Flush after this long | -| `request_timeout_ms` | int | `10000` | Covers dial, write, and response | -| `backoff_min_ms` | int | `500` | Retry backoff floor | -| `backoff_max_ms` | int | `30000` | Retry backoff ceiling | -| `tls` | table | — | Dialer TLS; `cert_file`/`key_file` present a client identity | -| `auth` | table | — | Server identity pinning (`mtls`) or a login (`scram`); see [Security](security.md#the-auth-block) | +Options, each as type and default: + +- `host` (string, **required**): downstream host or address; an IPv6 one goes + bare (`::1`). +- `port` (int, **required**): downstream port. +- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`. +- `node` (string, `os.Hostname()`): origin label stamped on first-hop entries. +- `buffer_size` (int, `1000`): sink input queue depth. +- `max_batch_count` (int, `100`): flush after this many entries. +- `max_batch_bytes` (int, `1048576`): flush after this many bytes (1 MiB). +- `flush_interval_ms` (int, `1000`): flush after this long. +- `request_timeout_ms` (int, `10000`): covers dial, write, and response. +- `backoff_min_ms` (int, `500`): retry backoff floor. +- `backoff_max_ms` (int, `30000`): retry backoff ceiling. +- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity. +- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see + [Security](security.md#the-auth-block). **Behaviour** @@ -407,10 +418,10 @@ key_file = "/etc/logwisp/tls/client.key" sends `Authorization: Bearer `. A failed login is retried like a transport error, holding the batch. The token is renewed ahead of its expiry; a `401` on ingest (the source reloaded) drops it and the retry logs in again. - A `403` is permanent. A - connection presenting a certificate other than the one the login was bound - to drops token and pin, and the retry logs in anew. A sink without `scram` - facing a `scram` source gets `401` and drops every batch. + A `403` is permanent. A connection presenting a certificate other than the + one the login was bound to drops token and pin, and the retry logs in anew. + A sink without `scram` facing a `scram` source gets `401` and drops every + batch. - Redirects are never followed; a `3xx` is permanent too. Following one would resend the batch wherever the response points, plaintext `http` included. - HTTP/2 is off by design; batched NDJSON POSTs gain nothing from it. diff --git a/doc/sources.md b/doc/sources.md index 98df9c3..eaebe3f 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -36,13 +36,14 @@ raw = false from = "end" ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `directory` | string | **required** | Directory to scan; not recursive | -| `pattern` | string | `*` | Glob over filenames; `*` and `?` only | -| `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` | -| `raw` | bool | `false` | Never parse a line: the whole line is the message | -| `from` | string | `end` | Where a new watcher starts: `end` or `start` of the file | +Options, each as type and default: + +- `directory` (string, required): directory to scan; not recursive. +- `pattern` (string, `*`): glob over filenames; `*` and `?` only. +- `check_interval_ms` (int, `100`): directory rescan interval; minimum `10`. +- `raw` (bool, `false`): never parse a line: the whole line is the message. +- `from` (string, `end`): where a new watcher starts, `end` or `start` of the + file. **Behaviour** @@ -93,9 +94,9 @@ type = "console" buffer_size = 1000 ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `buffer_size` | int | `1000` | Subscriber channel depth | +Option, as type and default: + +- `buffer_size` (int, `1000`): subscriber channel depth. At most **one** instance per pipeline: the type is registered with `MaxInstances: 1`, and a second instance is rejected at pipeline construction. @@ -119,13 +120,15 @@ length = 20 special = false ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `interval_ms` | int | `500` | Emission period | -| `jitter_ms` | int | `0` | Symmetric jitter; clamped to `interval_ms`, must be non-negative | -| `format` | string | `txt` | `raw` (message only), `txt` (bracketed line), `json` (JSON object as the message) | -| `length` | int | `20` | Message length in characters | -| `special` | bool | `false` | Inject control and non-ASCII characters | +Options, each as type and default: + +- `interval_ms` (int, `500`): emission period. +- `jitter_ms` (int, `0`): symmetric jitter; must be non-negative, clamped to + `interval_ms`. +- `format` (string, `txt`): `raw` (message only), `txt` (bracketed line) or + `json` (JSON object as the message). +- `length` (int, `20`): message length in characters. +- `special` (bool, `false`): inject control and non-ASCII characters. `special = true` is the intended way to exercise sanitizer policies: it inserts control bytes and multi-byte Unicode into otherwise ordinary messages. Levels @@ -180,17 +183,22 @@ allow = ["edge-01", "edge-02"] node_binding = "force" ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | `0.0.0.0` | Bind address; IPv4 only | -| `port` | int | **required** | Listen port, 1–65535 | -| `buffer_size` | int | `1000` | Subscriber channel depth | -| `max_connections` | int | `0` | Concurrent connection cap; `0` = unlimited | -| `read_timeout_ms` | int | `0` | Per-connection idle read deadline; `0` = none | -| `hello_timeout_ms` | int | `10000` | Deadline for the hello preamble | -| `trust_node` | bool | `true` | `false` overrides the sender's node label with its remote address. Ignored when `auth.node_binding` is active | -| `tls` | table | — | Listener TLS; see [Security](security.md) | -| `auth` | table | — | Peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block) | +Options, each as type and default: + +- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the + listener keeps to its family ([Networking](networking.md#address-family)). +- `port` (int, required): listen port, 1–65535. +- `buffer_size` (int, `1000`): subscriber channel depth. +- `max_connections` (int, `0`): concurrent connection cap; `0` = unlimited. +- `read_timeout_ms` (int, `0`): per-connection idle read deadline; `0` = none. +- `hello_timeout_ms` (int, `10000`): deadline for the hello preamble. +- `trust_node` (bool, `true`): `false` overrides the sender's node label with + its remote address. `auth.node_binding` `assert` or `force` takes over the + connection label; only `force` also overrides the per-entry labels; `none` + leaves both to `trust_node`. +- `tls` (table): listener TLS; see [Security](security.md). +- `auth` (table): peer authentication (`mtls` or `scram`) and node binding; + see [Security](security.md#the-auth-block). For passwords instead of certificates, the `auth` block takes `type = "scram"` and a `credentials_file`, and `client_auth` becomes optional; see @@ -258,17 +266,23 @@ allow = ["edge-01", "edge-02"] node_binding = "force" ``` -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | `0.0.0.0` | Bind address; IPv4 only | -| `port` | int | **required** | Listen port | -| `ingest_path` | string | `/ingest` | Endpoint path; must start with `/` | -| `buffer_size` | int | `1000` | Subscriber channel depth | -| `max_body_bytes` | int | `8388608` | Per-request body cap (8 MiB) | -| `read_timeout_ms` | int | `30000` | Full request read deadline | -| `trust_node` | bool | `true` | `false` overrides the sender's node label with its remote address. Ignored when `auth.node_binding` is active | -| `tls` | table | — | Listener TLS | -| `auth` | table | — | Peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block) | +Options, each as type and default: + +- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the + listener keeps to its family ([Networking](networking.md#address-family)). +- `port` (int, required): listen port, 1–65535. +- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`, and + `/auth` is reserved for the login. +- `buffer_size` (int, `1000`): subscriber channel depth. +- `max_body_bytes` (int, `8388608`): per-request body cap (8 MiB). +- `read_timeout_ms` (int, `30000`): full request read deadline. +- `trust_node` (bool, `true`): `false` overrides the sender's node label with + its remote address. `auth.node_binding` `assert` or `force` takes over the + connection label; only `force` also overrides the per-entry labels; `none` + leaves both to `trust_node`. +- `tls` (table): listener TLS; see [Security](security.md). +- `auth` (table): peer authentication (`mtls` or `scram`) and node binding; + see [Security](security.md#the-auth-block). **Behaviour** diff --git a/doc/todo.md b/doc/todo.md new file mode 100644 index 0000000..3344f66 --- /dev/null +++ b/doc/todo.md @@ -0,0 +1,202 @@ +# To Do + +Planned work, in order of priority. Each item names what to build, how it fits +the existing seams, and how to verify it. Finished items move to the design doc +they belong to. + +## 1. Network access control (ACL) + +The next feature push. Today a listener decides *who* a peer is (`tls`, `auth`); +nothing decides *where* a peer may connect from, and every limit except SCRAM +throttling is global. The ACL adds address rules and per-client limits at the +connection level, below TLS, so a refused peer costs no handshake. + +### 1.1 One seam: a filtering listener + +- New package `internal/netacl`, the third network seam beside `tlsx` and + `authz` (AGENTS.md lists the seams; add it there). +- `netacl.New(opts *config.ACLOptions, role) (*Policy, error)` compiles the + block; a nil policy is valid and transparent, like `authz`. +- `(*Policy).Listener(ln net.Listener) net.Listener` wraps the raw listener + before TLS: + - TCP plugins wrap it before `tls.NewListener`. + - HTTP plugins wrap it before `Serve`/`ServeTLS`. +- `Accept` reads an optional PROXY header (1.2), decides allow or deny (1.3), + applies per-address connection caps (1.4), and returns a `net.Conn` whose + `RemoteAddr` is the real client. +- Everything downstream then sees the real client with no further change: + `authz.remoteIP`, SCRAM throttling, sessions, logs and `http.Request.RemoteAddr`. +- Keep the proxy's own address as `peer_addr` in session metadata for audit. +- A denied connection is closed before any byte is read or written, with a + counter (`acl_denied`) and a rate-limited WARN. + +### 1.2 PROXY protocol (deferred gap of SCRAM, see scram-auth-plan.md) + +- Why: behind an L4 proxy that passes TLS through (nginx `stream` with + `ssl_preread`, HAProxy `mode tcp`), every peer arrives from the proxy's + address. + - All peers share one SCRAM throttling budget, so one client can lock out + every login. + - Logs, sessions and the ACL itself see only the proxy. +- What: accept PROXY v1 (text) and v2 (binary) headers, but only from + `proxy_from` addresses or CIDRs. Two config keys in the ACL block: + - `proxy_protocol = "off" | "optional" | "required"`, default off. + - `proxy_from = [...]`. +- Rules: + - A listed peer must send the header when `required`. + - An unlisted peer that sends a header is refused, since the header is + spoofable. + - The header must arrive within a short deadline (reuse + `tlsx.HandshakeTimeout`), with a size cap: v1 at most 107 bytes, v2 with a + bounded TLV length. + - `LOCAL`/`UNKNOWN` commands (proxy health checks) keep the socket address. + - TLVs are ignored. +- The header precedes the TLS ClientHello, so SCRAM channel binding is + unaffected: LogWisp still terminates TLS. +- nginx side: + - In the `stream` server, `proxy_protocol on` on the route to LogWisp (it + sends v1). + - HAProxy: `send-proxy` or `send-proxy-v2`. +- Relation to `auth.trusted_proxies` (HTTP proxy mode): + - That list trusts `X-Forwarded-For` from a TLS-terminating L7 proxy; + `proxy_from` trusts a PROXY header from an L4 proxy. + - Keep both, with distinct names and docs. They answer different layers, and + one deployment can use both: nginx `stream` with `proxy_protocol` into an + nginx `http` block that sets `X-Forwarded-For $proxy_protocol_addr`. + +### 1.3 Address rules + +- `allow = [CIDR...]`, `deny = [CIDR...]`, IPv4 and IPv6 (strict per family, + as the listeners are). +- Order: deny wins, then allow; an empty allow list admits everyone not denied. +- Rules match the real client (after PROXY), never the proxy. + - In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after + the connection-level decision. + - Address rules for forwarded HTTP clients therefore belong in a second, + request-level check inside `authz.ClientAddr`: reuse the compiled `netacl` + rules there rather than duplicating them. +- Startup warnings (in `LogStartup` style): an allow list containing + `0.0.0.0/0` or `::/0`, `proxy_from` covering public ranges, a listener on a + wildcard host with no `allow`. + +### 1.4 Per-client limits + +- Generalize the SCRAM limiter in `internal/authz/scram.go` (token bucket per + key, bounded table that fails closed, idle sweep, IPv6 per /64 via + `throttleKey`) into the shared per-key limiter. One mechanism (AGENTS.md): + SCRAM throttling then uses it too. +- Keys: + - `max_connections_per_client` on every listener (concurrent connections per + address). + - `requests_per_second_per_client` on HTTP listeners (ingest, stream and + status requests). +- Later: per-identity limits once a peer is authenticated (the mtls plan's + deferred item 2), keyed by identity instead of address. + +### 1.5 Config, validation, docs + +- Config shape: `[...config.acl]` beside `tls` and `auth` on the four listener + plugins. + - Decode through `config.Scan`, so unknown keys fail. + - Dialers have no ACL block. +- Intent rule as in `authz`: an ACL key on a dialer, or `proxy_from` without + `proxy_protocol`, is an error. +- Stats: `acl_denied`, `acl_proxy_headers`, `acl_limited`, merged into the + plugin stats like `authz.Stats`. +- Docs: a security.md section, plus networking.md troubleshooting lines for + each refusal. + +### 1.6 Verification + +- Go tests, one per rule: + - PROXY v1 and v2 parse, including malformed, oversized and slow headers. + - A header from an unlisted peer refused; `required` without a header + refused; `LOCAL` kept. + - Deny over allow; IPv6 rules. + - Per-client connection cap; the limiter table failing closed. + - The real client reaching SCRAM throttling and session metadata. +- `test/acl-test.sh`: + - A Go stub proxy that sends v1 and v2 headers in front of each listener + type. + - nginx `stream` with `proxy_protocol on` when nginx is installed. + - Denied and allowed clients; per-client SCRAM budgets behind one proxy + address. + +## 2. Hardening review of `lixenwraith/config` and `lixenwraith/toml` + +Both parse untrusted-shaped input (config files, environment, command line) +and were not reviewed with the auth work. Review them as `auth` was reviewed: +adversarial lenses, reproduction before belief, a fix per confirmed finding in +the library repository, then a dependency bump here. + +- toml parser: + - Limits on nesting depth, key count, string and array length, and number + sizes. + - Behaviour on invalid UTF-8, duplicate keys and tables, dotted-key and + inline-table redefinition. + - Fuzz targets (`go test -fuzz`) seeded with the TOML test suite. +- config: + - The weakly typed decoder: string-to-number overflow, duration and size + parsing, the comma split into slices. + - `maxValueDepth` and `MaxValueSize`, environment and CLI precedence, and + unknown-key reporting. +- Files: + - `PreventPathTraversal`, symlink handling and the file-size cap. + - The watcher: polling, debounce, a file replaced by rename or symlink swap, + TOCTOU between stat and read. +- Errors: values must never echo secrets (a misplaced password in a config + value reported back in an error). +- Deliverables: + - Findings per library. + - Fuzz targets committed in the library repositories. + - A note in this repository's security.md once done. + +## 3. Packaging: AUR, FreeBSD ports, Debian + +The foundation exists: +- The `lw` binary name. +- `make install` with `DESTDIR`, `PREFIX` and `SYSCONFDIR`. +- The `doc/lw.1` manual. +- Service files in `deploy/package/`: systemd unit, sysusers, tmpfiles and the + FreeBSD rc.d script. +- Skeletons in `deploy/package/arch/` and `deploy/package/freebsd/`. + +What remains: + +- Releases: + - The repository has no tags, so versions come from `git describe` and fall + back to a commit hash. Tag releases `vX.Y.Z`. + - Publish source tarballs with checksums, and sign them. +- Module path: + - The module is `logwisp`, so `go install github.com/lixenwraith/logwisp/cmd/lw@latest` + does not work. + - FreeBSD's `USES=go:modules` and Debian's dh-golang expect the canonical + path. + - Rename the module to `github.com/lixenwraith/logwisp`: a mechanical + import rewrite in one commit. +- Name check before submission: + - Arch: `pkgfile -s lw` against the official repositories, and the AUR + package list for `lw` and `logwisp`. + - Already checked: Ubuntu 26.04 and 24.04 contents and the FreeBSD 15.1 + ports tree ship no `bin/lw`. +- Arch (AUR): `deploy/package/arch/PKGBUILD` already has the source URL, the + Arch Go flags (`-trimpath -buildmode=pie -mod=readonly -modcacherw`), + `backup` for the config and `check()`. Remaining: + - the checksum of a tagged tarball in place of `SKIP`, and a maintainer; + - namcap, `.SRCINFO`, and publishing `logwisp` (optionally `logwisp-git`). +- FreeBSD: `deploy/package/freebsd/` already has `USES=go:1.27,modules`, + `USE_RC_SUBR=logwisp`, `USERS`/`GROUPS`, and a pkg-plist with the `.sample` + config and the manual. Remaining: + - `distinfo` (or `GH_TUPLE`), which needs the module rename and a tag; + - a real `MAINTAINER` and a `UIDs`/`GIDs` entry for `logwisp`; + - poudriere on 14.x and 15.x, then submission through Bugzilla + (`sysutils/logwisp`). +- Debian: + - Debian policy wants every Go dependency packaged; the four `lixenwraith` + libraries are not. + - Either package them too (dh-golang), or start with an `.deb` built by the + Makefile `install` target (nfpm or `dpkg-deb`) and a PPA, and move to the + archive later. +- Shell completion for bash, zsh and fish, installed by `make install`. +- A packaging CI job: build the AUR package in an Arch container and the port + in a FreeBSD VM, run `lw --version` and `make image-check`. diff --git a/internal/authz/authz.go b/internal/authz/authz.go index e9624db..20e812c 100644 --- a/internal/authz/authz.go +++ b/internal/authz/authz.go @@ -414,13 +414,26 @@ func (p *Policy) Prepare(ctx context.Context, client *http.Client, baseURL strin if p == nil || p.dialer == nil { return nil } - token := p.dialer.token.Load() - if token == nil || time.Now().UnixNano() >= p.dialer.renewAt.Load() { + d := p.dialer + token := d.token.Load() + if token == nil || time.Now().UnixNano() >= d.renewAt.Load() { + pin, expires := d.pin.Load(), d.expires.Load() t, err := p.Token(ctx, client, baseURL) - if err != nil { + // A renewal put off while the old token holds (a throttled login behind + // a shared address) keeps sending on it and retries in a few seconds. + // Idle connections close first: Token dialed them with no pin, and the + // token may ride only connections the old pin verifies + var later putOff + if left := time.Duration(expires - time.Now().UnixNano()); errors.As(err, &later) && token != nil && left > 0 { + client.CloseIdleConnections() + d.pin.Store(pin) + d.token.Store(token) + d.renewAt.Store(time.Now().Add(min(5*time.Second, left/2)).UnixNano()) + } else if err != nil { return err + } else { + token = &t } - token = &t } req.Header.Set("Authorization", "Bearer "+*token) return nil @@ -561,7 +574,11 @@ func (p *Policy) LogStartup(l *log.Logger, component, id string, trustNode bool) "hint", "set auth.allow or auth.allow_patterns to authorize named peers") } if p.BindsNode() { - l.Info("msg", "Node labels bound to peer identity; trust_node is ignored", + msg := "Connection node label bound to peer identity; trust_node still governs per-entry labels" + if p.binding == BindingForce { + msg = "Node labels bound to peer identity; trust_node is ignored" + } + l.Info("msg", msg, "component", component, "instance_id", id, "node_binding", p.binding, diff --git a/internal/authz/proxy_test.go b/internal/authz/proxy_test.go index 6e85692..d047e70 100644 --- a/internal/authz/proxy_test.go +++ b/internal/authz/proxy_test.go @@ -177,7 +177,7 @@ func TestBrowserSessionBehindProxy(t *testing.T) { } // Behind a proxy logwisp never sees the certificate a client binds to, so -// only unbound proofs (logwisp auth token -unbound) log in. +// only unbound proofs (lw auth token -unbound) log in. func TestOnlyUnboundLoginsBehindProxy(t *testing.T) { f := newFixture(t) l := f.listener(t, config.AuthOptions{TrustedProxies: []string{"127.0.0.1"}}, f.serverTLS, RoleListener, HTTP) diff --git a/internal/authz/scram.go b/internal/authz/scram.go index 778b9b5..c0da9b4 100644 --- a/internal/authz/scram.go +++ b/internal/authz/scram.go @@ -167,12 +167,12 @@ func (c *Credentials) Marshal() ([]byte, error) { if err != nil { return nil, err } - header := "# logwisp SCRAM verifiers, written by `logwisp auth add-user`. Keep it private.\n" + header := "# logwisp SCRAM verifiers, written by `lw auth add-user`. Keep it private.\n" return append([]byte(header), body...), nil } // ReadPassword reads a password file, trimming one trailing line break so a -// file from an editor and one from `logwisp auth add-user` agree. +// file from an editor and one from `lw auth add-user` agree. func ReadPassword(path string) (string, error) { data, err := os.ReadFile(path) if err != nil { @@ -355,7 +355,7 @@ func (p *Policy) finish(ip, nonce string, step authStep, cs *tls.ConnectionState p.rejected.Add(1) switch { case step.Binding != "" && l.proxy != nil: - err = fmt.Errorf("%w; the client bound its proof to the proxy's certificate: behind trusted_proxies, log in unbound (logwisp auth token -unbound)", err) + err = fmt.Errorf("%w; the client bound its proof to the proxy's certificate: behind trusted_proxies, log in unbound (lw auth token -unbound)", err) case step.Binding == "" && l.cb != nil: err = fmt.Errorf("%w; the client sent an unbound proof (-unbound), but this listener binds logins to its certificate", err) case step.Binding != "" && step.Binding != base64.StdEncoding.EncodeToString(l.cb): @@ -566,6 +566,7 @@ type scramDialer struct { unbound bool // HTTP logins to a listener behind a TLS-terminating proxy token atomic.Pointer[string] renewAt atomic.Int64 // unix nanoseconds; Prepare logs in again from then + expires atomic.Int64 // unix nanoseconds; 0 = lifetime unknown pin atomic.Pointer[[]byte] // HTTP: certificate the token's login was bound to failures atomic.Uint64 lastErr atomic.Pointer[string] @@ -722,7 +723,12 @@ func (p *Policy) Token(ctx context.Context, client *http.Client, baseURL string) if step.Token == "" { return "", errors.New("auth: server issued no token") } - d.renewAt.Store(renewAt(time.Now(), time.Duration(step.ExpiresIn)*time.Second).UnixNano()) + now, lifetime := time.Now(), time.Duration(step.ExpiresIn)*time.Second + d.renewAt.Store(renewAt(now, lifetime).UnixNano()) + d.expires.Store(0) + if lifetime > 0 { + d.expires.Store(now.Add(lifetime).UnixNano()) + } d.token.Store(&step.Token) return step.Token, nil } @@ -770,6 +776,12 @@ func (d *scramDialer) stats(m map[string]any) { } } +// putOff marks a login the server deferred (throttled, busy) rather than +// refused: a token still held stays usable meanwhile. +type putOff struct{ error } + +func (e putOff) Unwrap() error { return e.error } + // postStep sends one /auth request; any answer but 200 is a refusal func postStep(ctx context.Context, client *http.Client, url string, body []byte) (authStep, *http.Response, error) { var step authStep @@ -796,7 +808,11 @@ func postStep(ctx context.Context, client *http.Client, url string, body []byte) if msg == "" { msg = resp.Status } - return step, resp, fmt.Errorf("%w: server: %s", ErrRefused, msg) + err := fmt.Errorf("%w: server: %s", ErrRefused, msg) + if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable { + err = putOff{err} + } + return step, resp, err case decodeErr != nil: return step, resp, fmt.Errorf("auth: malformed answer: %w", decodeErr) } @@ -888,13 +904,18 @@ func remoteIP(addr string) string { // --- Throttling --- // throttleKey is what the limiter counts: an address, or for IPv6 its /64, -// which one host usually holds whole +// which one host usually holds whole. Link-local too, per zone: a peer picks +// any fe80::/64 address, so per address it would escape its budget. func throttleKey(ip string) string { addr, err := netip.ParseAddr(ip) if err != nil || addr.Is4() { return ip } - return netip.PrefixFrom(addr, 64).Masked().String() + key := netip.PrefixFrom(addr, 64).Masked().String() + if zone := addr.Zone(); zone != "" { + key += "%" + zone + } + return key } // limiter bounds SCRAM attempts per remote address: failed or abandoned diff --git a/internal/authz/scram_test.go b/internal/authz/scram_test.go index cddf078..7b5874c 100644 --- a/internal/authz/scram_test.go +++ b/internal/authz/scram_test.go @@ -417,6 +417,15 @@ func TestDialerRefusesACheapChallenge(t *testing.T) { } } +// Link-local peers share one budget per link, not one across every link: the +// zone names the link and stays in the key. +func TestLinkLocalBudgetIsPerLink(t *testing.T) { + a, b, other := throttleKey("fe80::1%eth0"), throttleKey("fe80::2%eth0"), throttleKey("fe80::1%eth1") + if a != b || a == other { + t.Fatalf("keys %q, %q on one link, %q on another", a, b, other) + } +} + // Only failed attempts drain an address's budget; beyond it the listener // answers "too many attempts" without running the exchange. func TestOnlyFailedAttemptsAreThrottled(t *testing.T) { @@ -734,3 +743,80 @@ func TestTokenIsRenewedBeforeExpiry(t *testing.T) { t.Fatalf("logins after the renewal point = %d, want 2", n) } } + +// A renewal put off (throttled, or the server busy) keeps the link sending on +// the token it still holds, and retries soon, but only to the pinned server; +// any other refusal, or the token's expiry, ends it. +func TestPutOffRenewalKeepsTheValidToken(t *testing.T) { + f := newFixture(t) + l := f.listener(t, config.AuthOptions{TokenLifetimeMS: MinTokenLifetime.Milliseconds()}, f.serverTLS, RoleListener, HTTP) + srv, _ := f.httpListener(t, l, f.serverTLS) + // An impostor with another certificate of the same CA + var answer, stolen atomic.Int64 + mux := http.NewServeMux() + mux.HandleFunc("POST "+chain.AuthPath, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, int(answer.Load()), authStep{Error: "no"}) + }) + mux.HandleFunc("GET /protected", func(w http.ResponseWriter, r *http.Request) { stolen.Add(1) }) + imp := httptest.NewUnstartedServer(mux) + imp.TLS = f.serverTLS.Clone() + imp.TLS.Certificates = []tls.Certificate{f.serverLeaf(t, "impostor")} + imp.StartTLS() + t.Cleanup(imp.Close) + + d := f.dialer(t, "edge-01", "edge-01-secret") + client := f.httpClient(d) + var reroute atomic.Pointer[string] + client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + if to := reroute.Load(); to != nil { + addr = *to + } + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + prepare := func(renew bool) (*http.Request, error) { + if renew { + d.dialer.renewAt.Store(time.Now().UnixNano()) + } + req, _ := http.NewRequest(http.MethodGet, srv.URL+"/protected", nil) + return req, d.Prepare(t.Context(), client, srv.URL, req) + } + held := func() string { + req, err := prepare(false) + if err != nil { + t.Fatal(err) + } + return req.Header.Get("Authorization") + } + + first := held() + client.CloseIdleConnections() + to := imp.Listener.Addr().String() + reroute.Store(&to) + answer.Store(http.StatusTooManyRequests) + req, err := prepare(true) + if err != nil || req.Header.Get("Authorization") != first { + t.Fatalf("put-off renewal: header %q, %v; want the held token", req.Header.Get("Authorization"), err) + } + if ahead := time.Until(time.Unix(0, d.dialer.renewAt.Load())); ahead <= 0 || ahead > 5*time.Second { + t.Fatalf("next renewal %v ahead, want within 5 s", ahead) + } + if _, err := client.Do(req); !errors.Is(err, errPinMismatch) || stolen.Load() != 0 { + t.Fatalf("the held token went to a server the pin never checked: %v, %d requests", err, stolen.Load()) + } + answer.Store(http.StatusUnauthorized) + if _, err := prepare(true); err == nil || d.dialer.token.Load() != nil { + t.Fatalf("a refused renewal kept the token: %v", err) + } + + client.CloseIdleConnections() + reroute.Store(nil) + first = held() + l.Close() // every further login answers busy + if req, err := prepare(true); err != nil || req.Header.Get("Authorization") != first { + t.Fatalf("busy renewal: %v; want the held token", err) + } + d.dialer.expires.Store(time.Now().UnixNano()) + if _, err := prepare(true); err == nil { + t.Fatal("an expired token was kept after a put-off renewal") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 3470cb6..b2fd783 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,7 @@ package config type Config struct { // Top-level flags for application control ShowVersion bool `toml:"version"` + Check bool `toml:"check"` // build every plugin, report, exit: lw --check Quiet bool `toml:"quiet"` // Runtime behavior flags @@ -347,7 +348,7 @@ type AuthOptions struct { // Overrides trust_node. NodeBinding string `toml:"node_binding"` - // scram listeners: verifier file written by `logwisp auth add-user`, and + // scram listeners: verifier file written by `lw auth add-user`, and // the bearer token lifetime on HTTP listeners (default 15 minutes) CredentialsFile string `toml:"credentials_file"` TokenLifetimeMS int64 `toml:"token_lifetime_ms"` diff --git a/internal/config/loader.go b/internal/config/loader.go index c62e951..ddfd386 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -18,8 +18,11 @@ var ErrConfigNotFound = lconfig.ErrConfigNotFound // Manager owns the configuration sources and watcher for one application instance. // Its snapshots are detached from each other and from the running service. type Manager struct { - config *lconfig.Config - path string + config *lconfig.Config + path string + explicit bool + read bool // the file was loaded once + specs []pipelineSpec // command-line or environment pipelines, kept across reloads } // Load reads the startup sources and validates the initial configuration. @@ -29,13 +32,30 @@ func Load(args []string) (*Manager, error) { if err != nil { return nil, err } + specs, configArgs, err := takePipelineSpecs(configArgs) + if err != nil { + return nil, err + } + if len(specs) == 0 { + specs = envPipelineSpecs() + } + if _, err := buildPipelines(specs); err != nil { + return nil, err + } + m := &Manager{path: configPath, explicit: isExplicit, specs: specs} initial := defaults() cfg, err := lconfig.NewBuilder(). WithTarget(initial). WithEnvPrefix("LOGWISP_"). WithArgs(configArgs). WithFile(configPath). - WithTypedValidator(ValidateConfig). + WithTypedValidator(func(cfg *Config) error { + effective := *cfg // the builder keeps cfg; only the copy is replaced + if err := m.usePipelines(&effective); err != nil { + return err + } + return ValidateConfig(&effective) + }). WithSecurityOptions(lconfig.SecurityOptions{ PreventPathTraversal: true, MaxFileSize: 10 * 1024 * 1024, @@ -50,10 +70,12 @@ func Load(args []string) (*Manager, error) { } // A missing discovered default still permits valid CLI/env/default values. } + m.read = err == nil if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !initial.Quiet { fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown) } - return &Manager{config: cfg, path: configPath}, nil + m.config = cfg + return m, nil } // Snapshot validates a detached candidate before it is used to build a service. @@ -65,6 +87,9 @@ func (m *Manager) Snapshot() (*Config, error) { } cfg := value.(*Config) cfg.ConfigFile = m.path + if err := m.usePipelines(cfg); err != nil { + return nil, err + } if err := checkFileKeys(m.path); err != nil { return nil, fmt.Errorf("config file %q: %w", m.path, err) } @@ -74,12 +99,27 @@ func (m *Manager) Snapshot() (*Config, error) { return cfg, nil } +// usePipelines replaces the file's or default pipelines with the spec ones. +func (m *Manager) usePipelines(cfg *Config) error { + if len(m.specs) == 0 { + return nil + } + pipelines, err := buildPipelines(m.specs) + cfg.Pipelines = pipelines + return err +} + // Reload rereads the selected file, retaining the startup CLI/environment sources. // Signals must call this even when automatic watching is disabled. func (m *Manager) Reload() (*Config, error) { - if err := m.config.LoadFile(m.path); err != nil { + // As at startup, a discovered default that does not exist is no error, so a + // file-less instance still rebuilds and rotates its certificates. Once read, + // its removal is one: the loader would keep the removed file's values. + err := m.config.LoadFile(m.path) + if err != nil && (m.explicit || m.read || !errors.Is(err, ErrConfigNotFound)) { return nil, fmt.Errorf("reload %q: %w", m.path, err) } + m.read = m.read || err == nil return m.Snapshot() } diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go index 371b111..146f495 100644 --- a/internal/config/loader_test.go +++ b/internal/config/loader_test.go @@ -2,6 +2,7 @@ package config import ( "errors" + "os" "path/filepath" "reflect" "strings" @@ -124,6 +125,13 @@ func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) { if err != nil || !recovered.StatusReporter { t.Fatalf("override removal/recovery: %+v %v", recovered, err) } + // Only a missing discovered default is tolerated; a named file must exist. + if err := os.Remove("reload.toml"); err != nil { + t.Fatal(err) + } + if _, err := m.Reload(); !errors.Is(err, ErrConfigNotFound) { + t.Fatalf("reload without the named file: %v", err) + } } func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) { diff --git a/internal/config/spec.go b/internal/config/spec.go new file mode 100644 index 0000000..3e44a8c --- /dev/null +++ b/internal/config/spec.go @@ -0,0 +1,272 @@ +package config + +import ( + "cmp" + "errors" + "fmt" + "os" + "slices" + "strconv" + "strings" +) + +// specKind is one pipeline spec flag. Its environment form is LOGWISP_ plus +// the flag uppercased, '-' as '_'. +type specKind struct { + flag string + typed bool // SPEC starts with TYPE + many bool // repeatable within a pipeline; the environment adds _1.._N + apply func(p *PipelineConfig, typ string, opts map[string]any) error +} + +var specKinds = []specKind{ + {"source", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + ids := make([]string, len(p.PluginSources)) + for i, s := range p.PluginSources { + ids[i] = s.ID + } + id, err := instanceID(typ, opts, ids) + p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts}) + return err + }}, + {"sink", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + ids := make([]string, len(p.PluginSinks)) + for i, s := range p.PluginSinks { + ids[i] = s.ID + } + id, err := instanceID(typ, opts, ids) + p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts}) + return err + }}, + {"filter", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + opts["type"] = typ + f := FilterConfig{} + err := Scan(opts, &f) + p.Flow.Filters = append(p.Flow.Filters, f) + return err + }}, + {"format", true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { + opts["type"] = typ + p.Flow.Format = &FormatConfig{} + return Scan(opts, p.Flow.Format) + }}, + // Naming a stage turns it on: the file defaults (pass, disabled) would not. + {"rate-limit", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { + p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"} + return Scan(opts, p.Flow.RateLimit) + }}, + {"heartbeat", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { + p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true} + return Scan(opts, p.Flow.Heartbeat) + }}, +} + +// pipelineSpec is one pipeline flag or variable; a nil kind starts a pipeline. +type pipelineSpec struct { + name string // "--sink" or "LOGWISP_SINK_2", for errors + kind *specKind + value string +} + +// takePipelineSpecs consumes the pipeline flags, which lixenwraith/config +// would report as unrecognized. +func takePipelineSpecs(args []string) (specs []pipelineSpec, rest []string, err error) { + for i := 0; i < len(args); i++ { + if args[i] == "--" { + return specs, append(rest, args[i:]...), nil + } + name, value, inline := strings.Cut(args[i], "=") + var kind *specKind + if name != "--pipeline" { + k := slices.IndexFunc(specKinds, func(k specKind) bool { return "--"+k.flag == name }) + if k < 0 { + rest = append(rest, args[i]) + continue + } + kind = &specKinds[k] + } + if !inline && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") { + i++ + value = args[i] + } + if value == "" { + return nil, nil, fmt.Errorf("%s requires a value", name) + } + specs = append(specs, pipelineSpec{name, kind, value}) + } + return specs, rest, nil +} + +// envPipelineSpecs reads the one-pipeline environment form. An empty variable +// counts as unset, so container templates may leave optional ones blank. +func envPipelineSpecs() []pipelineSpec { + var specs []pipelineSpec + if name := os.Getenv("LOGWISP_PIPELINE"); name != "" { + specs = append(specs, pipelineSpec{"LOGWISP_PIPELINE", nil, name}) + } + for i := range specKinds { + kind := &specKinds[i] + base := "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(kind.flag, "-", "_")) + type numbered struct { + n uint64 + name, spec string + } + var found []numbered + for _, kv := range os.Environ() { + name, spec, _ := strings.Cut(kv, "=") + suffix, ok := strings.CutPrefix(name, base+"_") + n, err := strconv.ParseUint(suffix, 10, 64) + if name == base || ok && kind.many && err == nil { + found = append(found, numbered{n, name, spec}) + } + } + slices.SortFunc(found, func(a, b numbered) int { return cmp.Or(cmp.Compare(a.n, b.n), cmp.Compare(a.name, b.name)) }) + for _, f := range found { + if f.spec != "" { + specs = append(specs, pipelineSpec{f.name, kind, f.spec}) + } + } + } + return specs +} + +// buildPipelines turns specs, in order, into pipelines; specs before the first +// --pipeline go to one named "cli". Every call returns fresh maps. +func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { + var pipelines []PipelineConfig + var seen map[*specKind]bool + for _, s := range specs { + if s.kind == nil || pipelines == nil { + name := "cli" + if s.kind == nil { + name = s.value + } + pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}}) + seen = map[*specKind]bool{} + if s.kind == nil { + continue + } + } + p := &pipelines[len(pipelines)-1] + if seen[s.kind] && !s.kind.many { + return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name) + } + seen[s.kind] = true + typ, opts, err := parseSpec(s.value, s.kind.typed) + if err != nil { + return nil, fmt.Errorf("%s %w", s.name, err) + } + if err := s.kind.apply(p, typ, opts); err != nil { + return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err) + } + } + return pipelines, nil +} + +// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes +// a list, and a backslash escapes ',', '=' and '\'. Errors name the part. +func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) { + rest, more := spec, true + if typed { + typ, rest, more = cutUnescaped(spec, ',') + if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq { + return "", nil, fmt.Errorf("%s: missing TYPE", spec) + } + typ = unescape(typ) + } + opts = map[string]any{} + for more { + var part string + part, rest, more = cutUnescaped(rest, ',') + key, value, eq := cutUnescaped(part, '=') + switch key = unescape(key); { + case !eq: + err = errors.New(`missing "="`) + case typed && key == "type": + err = errors.New(`TYPE already sets "type"`) + default: + err = setOption(opts, key, unescape(value)) + } + if err != nil { + return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err) + } + } + return typ, opts, nil +} + +// cutUnescaped cuts s at its first unescaped sep, leaving escapes in place. +func cutUnescaped(s string, sep byte) (before, after string, found bool) { + for i := 0; i < len(s); i++ { + if escapes(s, i) { + i++ + } else if s[i] == sep { + return s[:i], s[i+1:], true + } + } + return s, "", false +} + +func unescape(s string) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + if escapes(s, i) { + i++ + } + b.WriteByte(s[i]) + } + return b.String() +} + +// escapes keeps any other backslash literal, so regex escapes such as \d pass. +func escapes(s string, i int) bool { + return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0 +} + +// setOption stores value at a dotted key; a second value turns it into a list. +func setOption(opts map[string]any, key, value string) error { + segments := strings.Split(key, ".") + if slices.Contains(segments, "") { + return fmt.Errorf("empty key in %q", key) + } + table, last := opts, len(segments)-1 + for i, segment := range segments[:last] { + switch existing := table[segment].(type) { + case nil: + next := map[string]any{} + table[segment], table = next, next + case map[string]any: + table = existing + default: + return fmt.Errorf("%q is both a value and a table", strings.Join(segments[:i+1], ".")) + } + } + switch existing := table[segments[last]].(type) { + case nil: + table[segments[last]] = value + case string: + table[segments[last]] = []any{existing, value} + case []any: + table[segments[last]] = append(existing, value) + default: + return fmt.Errorf("%q is both a table and a value", key) + } + return nil +} + +// instanceID takes the id option; an unnamed instance gets the first of TYPE, +// TYPE_2, TYPE_3... that no earlier instance of its role holds. +func instanceID(typ string, opts map[string]any, taken []string) (string, error) { + id, _ := opts["id"].(string) + if _, set := opts["id"]; set && id == "" { + return "", errors.New(`"id" takes one non-empty value`) + } + delete(opts, "id") + if id != "" { + return id, nil + } + id = typ + for n := 2; slices.Contains(taken, id); n++ { + id = typ + "_" + strconv.Itoa(n) + } + return id, nil +} diff --git a/internal/config/spec_test.go b/internal/config/spec_test.go new file mode 100644 index 0000000..56c00bc --- /dev/null +++ b/internal/config/spec_test.go @@ -0,0 +1,190 @@ +package config + +import ( + "errors" + "os" + "reflect" + "testing" + + "logwisp/internal/testutil" +) + +func loadPipelines(t *testing.T, args ...string) []PipelineConfig { + t.Helper() + m, err := Load(args) + if err != nil { + t.Fatal(err) + } + defer m.Close() + cfg, err := m.Snapshot() + if err != nil { + t.Fatal(err) + } + return cfg.Pipelines +} + +func TestPipelineSpecGrammar(t *testing.T) { + isolateConfig(t) + got := loadPipelines(t, + "--source", `file,directory=/var/log/a\,b,pattern=*.log`, + `--source=file,id=app,directory=x\=y`, + "--source", `file,directory=c:\\logs\\`, + "--sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c", + "--filter", "include,patterns=ERROR,patterns=WARN", + "--filter", `exclude,patterns=\d{1\,3}`, + "--format", `txt,timestamp_format=Jan 2\, 2006`, + "--pipeline", "b", "--source", "null", "--sink", "null", "--sink", "null", "--heartbeat", "interval_ms=1000", + "--", "--source", "random") + want := []PipelineConfig{{ + Name: "cli", + Flow: &FlowConfig{ + Filters: []FilterConfig{ + {Type: "include", Patterns: []string{"ERROR", "WARN"}}, + {Type: "exclude", Patterns: []string{`\d{1,3}`}}, + }, + Format: &FormatConfig{Type: "txt", TimestampFormat: "Jan 2, 2006"}, + }, + PluginSources: []PluginSourceConfig{ + {ID: "file", Type: "file", Config: map[string]any{"directory": "/var/log/a,b", "pattern": "*.log"}}, + {ID: "app", Type: "file", Config: map[string]any{"directory": "x=y"}}, + {ID: "file_2", Type: "file", Config: map[string]any{"directory": `c:\logs\`}}, + }, + PluginSinks: []PluginSinkConfig{{ID: "http", Type: "http", Config: map[string]any{ + "port": "8080", + "auth": map[string]any{"type": "scram", "credentials_file": "users.toml"}, + "tls": map[string]any{"cert_file": "c"}, + }}}, + }, { + Name: "b", + Flow: &FlowConfig{Heartbeat: &HeartbeatConfig{Enabled: true, IntervalMS: 1000}}, + PluginSources: []PluginSourceConfig{{ID: "null", Type: "null", Config: map[string]any{}}}, + PluginSinks: []PluginSinkConfig{ + {ID: "null", Type: "null", Config: map[string]any{}}, + {ID: "null_2", Type: "null", Config: map[string]any{}}, + }, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want) + } + for want, args := range map[string][]string{ + `--sink http,port: missing "="`: {"--sink", "http,port"}, + `--source directory=/x: missing TYPE`: {"--source", "directory=/x"}, + `--source file,type=x: TYPE already sets "type"`: {"--source", "file,type=x"}, + `--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"--sink", "http,tls=1,tls.cert_file=c"}, + `--sink requires a value`: {"--sink="}, + `--format txt: pipeline "cli" already has one`: {"--format", "json", "--format", "txt"}, + `--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"--rate-limit", "rate=1,polcy=drop"}, + } { + if _, err := Load(args); err == nil || err.Error() != want { + t.Errorf("%q: err = %v, want %s", args, err, want) + } + } +} + +// A spec value is one list entry, commas included: a regex quantifier or a DN +// must not split into several, wider entries. +func TestSpecValueIsOneListEntry(t *testing.T) { + isolateConfig(t) + sink := loadPipelines(t, "--source", "null", "--sink", + `tcp,auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0] + var opts TCPSinkOptions + if err := Scan(sink.Config, &opts); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(opts.Auth.AllowPatterns, []string{`^edge-\d{1,3}$`}) || + !reflect.DeepEqual(opts.Auth.Allow, []string{"CN=a,O=b", "CN=c"}) { + t.Fatalf("allow_patterns %q, allow %q", opts.Auth.AllowPatterns, opts.Auth.Allow) + } +} + +// One pipeline from LOGWISP_*: numbered variables follow the bare one in +// numeric order, only repeatable kinds are numbered, and empty means unset. +func TestEnvironmentPipeline(t *testing.T) { + isolateConfig(t) + for name, value := range map[string]string{ + "LOGWISP_PIPELINE": "edge", + "LOGWISP_SOURCE": "random", + "LOGWISP_SOURCE_10": "null,id=ten", + "LOGWISP_SOURCE_2": "null,id=two", + "LOGWISP_SINK_1": "console", + "LOGWISP_FILTER": "", + "LOGWISP_FORMAT_1": "raw", + "LOGWISP_RATE_LIMIT": "rate=10", + "LOGWISP_LOGGING_LEVEL": "debug", + } { + t.Setenv(name, value) + } + m, err := Load(nil) + if err != nil { + t.Fatal(err) + } + defer m.Close() + cfg, err := m.Snapshot() + if err != nil { + t.Fatal(err) + } + p := cfg.Pipelines + if len(p) != 1 || p[0].Name != "edge" || len(p[0].PluginSources) != 3 || p[0].PluginSources[1].ID != "two" || + p[0].PluginSources[2].ID != "ten" || len(p[0].PluginSinks) != 1 || p[0].Flow.Filters != nil || p[0].Flow.Format != nil || + *p[0].Flow.RateLimit != (RateLimitConfig{Rate: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" { + t.Fatalf("environment pipeline: %+v %+v", p, cfg.Logging) + } +} + +func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) { + isolateConfig(t) + t.Setenv("LOGWISP_SOURCE", "random") + t.Setenv("LOGWISP_SINK", "console") + got := loadPipelines(t, "--source", "null", "--sink", "null") + if len(got) != 1 || len(got[0].PluginSources) != 1 || got[0].PluginSources[0].Type != "null" || + len(got[0].PluginSinks) != 1 || got[0].PluginSinks[0].Type != "null" { + t.Fatalf("environment specs mixed into command-line pipelines: %+v", got) + } +} + +// The file's pipelines are dropped unvalidated; its other keys still apply. +func TestSpecPipelinesReplaceFilePipelines(t *testing.T) { + isolateConfig(t) + testutil.WriteFile(t, "file.toml", "[logging]\nlevel = \"warn\"\n[[pipelines]]\nname = \"file\"\n") + m, err := Load([]string{"-c", "file.toml", "--source", "null", "--sink", "null", "--status_reporter=false"}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if unknown := m.config.UnknownCLIKeys(); len(unknown) != 0 { + t.Fatalf("spec flags reached the schema parser: %v", unknown) + } + cfg, err := m.Snapshot() + if err != nil || len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "cli" || cfg.Logging.Level != "warn" || cfg.StatusReporter { + t.Fatalf("snapshot: %+v %v", cfg, err) + } +} + +// Without a file a reload still rebuilds; once the discovered default +// appears it is read, and its pipelines still yield to the specs. Its later +// removal fails the reload instead of keeping the removed file's values. +func TestReloadKeepsSpecPipelines(t *testing.T) { + isolateConfig(t) + m, err := Load([]string{"--source", "random,special=true", "--sink", "null"}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + first, err := m.Reload() + if err != nil || !first.StatusReporter { + t.Fatalf("reload without a file: %+v %v", first, err) + } + first.Pipelines[0].PluginSources[0].Config["special"] = "mutated" + testutil.WriteFile(t, "logwisp.toml", "status_reporter = false\n[[pipelines]]\nname = \"file\"\n") + next, err := m.Reload() + if err != nil || next.StatusReporter || len(next.Pipelines) != 1 || next.Pipelines[0].Name != "cli" || + next.Pipelines[0].PluginSources[0].Config["special"] != "true" { + t.Fatalf("reload: %+v %v", next, err) + } + if err := os.Remove("logwisp.toml"); err != nil { + t.Fatal(err) + } + if _, err := m.Reload(); !errors.Is(err, ErrConfigNotFound) { + t.Fatalf("reload after the file was removed: %v", err) + } +} diff --git a/internal/config/validate.go b/internal/config/validate.go index 260b097..f8800d0 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -94,15 +94,17 @@ func validateLogConfig(cfg *LogConfig) error { // not declare: a misspelled tls or auth key must fail, not silently disable // the protection it was meant to configure. func Scan(configMap map[string]any, target any) error { - if err := unknownKeys(configMap, reflect.TypeOf(target), ""); err != nil { + if err := checkKeys(configMap, reflect.TypeOf(target), ""); err != nil { return err } return lconfig.ScanMap(configMap, target) } -// unknownKeys walks nested tables against the toml tags of t. Map-typed fields -// hold free-form keys and are not descended into. -func unknownKeys(m map[string]any, t reflect.Type, prefix string) error { +// checkKeys walks nested tables against the toml tags of t, refusing unknown +// keys and making a lone string for a list field a one-entry list: the weakly +// typed decoder would split it at commas, which a pattern or a DN may hold. +// Map-typed fields hold free-form keys and are not descended into. +func checkKeys(m map[string]any, t reflect.Type, prefix string) error { for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice { t = t.Elem() } @@ -122,8 +124,12 @@ func unknownKeys(m map[string]any, t reflect.Type, prefix string) error { } var tables []map[string]any switch v := m[key].(type) { + case string: + if ft.Kind() == reflect.Slice && ft.Elem().Kind() == reflect.String { + m[key] = []string{v} + } case map[string]any: - if err := unknownKeys(v, ft, prefix+key+"."); err != nil { + if err := checkKeys(v, ft, prefix+key+"."); err != nil { return err } case []map[string]any: @@ -136,7 +142,7 @@ func unknownKeys(m map[string]any, t reflect.Type, prefix string) error { } } for i, table := range tables { - if err := unknownKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil { + if err := checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil { return err } } @@ -160,5 +166,5 @@ func checkFileKeys(path string) error { return err } delete(root, "config_file") // runtime metadata, documented as ignored in the file - return unknownKeys(root, reflect.TypeOf(Config{}), "") + return checkKeys(root, reflect.TypeOf(Config{}), "") } diff --git a/internal/core/const.go b/internal/core/const.go index 4764fe1..eaf8769 100644 --- a/internal/core/const.go +++ b/internal/core/const.go @@ -1,9 +1,31 @@ package core import ( + "fmt" + "net/netip" + "strings" "time" ) +// Network is what a listener on, or a dialer to, host uses: tcp4 for an IPv4 +// literal or "" (the IPv4 wildcard, as the default 0.0.0.0), tcp6 for an IPv6 +// literal, which Go binds IPv6-only, and tcp for a hostname, which resolves. +func Network(host string) (string, error) { + addr, err := netip.ParseAddr(host) + switch { + case host == "" || err == nil && addr.Is4(): + return "tcp4", nil + case err == nil && addr.Is4In6(): + // tcp6 refuses it at bind or dial, long after the config loaded + return "", fmt.Errorf("%q is an IPv4-mapped address; write it as IPv4", host) + case err == nil: + return "tcp6", nil + case strings.ContainsAny(host, "[]:"): + return "", fmt.Errorf("%q is neither an address nor a hostname (write an IPv6 address without brackets or port)", host) + } + return "tcp", nil +} + const ( MaxLogEntryBytes = 1024 * 1024 @@ -30,4 +52,4 @@ const ( ReloadWatchDebounce = 500 * time.Millisecond ReloadWatchTimeout = 30 * time.Second -) \ No newline at end of file +) diff --git a/internal/core/const_test.go b/internal/core/const_test.go new file mode 100644 index 0000000..3b07545 --- /dev/null +++ b/internal/core/const_test.go @@ -0,0 +1,53 @@ +package core + +import ( + "net" + "testing" + + "logwisp/internal/testutil" +) + +// The host literal picks the family, strictly: an IPv4 literal or the empty +// host is IPv4 only, an IPv6 literal (the wildcard :: too) IPv6 only, and a +// hostname resolves. Brackets and ports belong to addresses, not hosts, and +// an IPv4-mapped address is IPv4 written as IPv6. +func TestNetworkFollowsTheHostLiteral(t *testing.T) { + for _, tc := range []struct{ host, want string }{ + {"127.0.0.1", "tcp4"}, {"0.0.0.0", "tcp4"}, {"", "tcp4"}, + {"::1", "tcp6"}, {"::", "tcp6"}, {"fe80::1%eth0", "tcp6"}, + {"relay.example", "tcp"}, {"localhost", "tcp"}, + {"[::1]", ""}, {"::1]", ""}, {"10.0.0.1:80", ""}, {"::ffff:10.0.0.1", ""}, + } { + got, err := Network(tc.host) + if got != tc.want || (err != nil) != (tc.want == "") { + t.Errorf("Network(%q) = %q, %v; want %q", tc.host, got, err, tc.want) + } + } + + testutil.RequireIPv6(t) + network, _ := Network("::") + ln, err := net.Listen(network, "[::]:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + } + }() + _, port, _ := net.SplitHostPort(ln.Addr().String()) + if c, err := net.Dial("tcp4", net.JoinHostPort("127.0.0.1", port)); err == nil { + c.Close() + t.Error(":: accepted an IPv4 connection") + } + c, err := net.Dial("tcp6", net.JoinHostPort("::1", port)) + if err != nil { + t.Fatalf(":: refused [::1]: %v", err) + } + c.Close() +} diff --git a/internal/core/flow.go b/internal/core/flow.go index 0103762..96dce3f 100644 --- a/internal/core/flow.go +++ b/internal/core/flow.go @@ -24,4 +24,3 @@ type TransportEvent struct { // Structured entry for re-serializing sinks (chain links). Zero Time => absent Entry LogEntry } - diff --git a/internal/core/listen.go b/internal/core/listen.go new file mode 100644 index 0000000..78c84ac --- /dev/null +++ b/internal/core/listen.go @@ -0,0 +1,49 @@ +package core + +import ( + "context" + "fmt" + "net" +) + +// lookupNetIP resolves listener hostnames; tests replace it +var lookupNetIP = net.DefaultResolver.LookupNetIP + +// Listen is lc.Listen, but a hostname first resolves to one address, IPv4 when +// it has one (as Go picks), listened on in that address's family alone: Go +// would bind a name for a wildcard address in both families. +func Listen(ctx context.Context, lc *net.ListenConfig, network, addr string) (net.Listener, error) { + network, addr, err := pinFamily(ctx, network, addr) + if err != nil { + return nil, err + } + return lc.Listen(ctx, network, addr) +} + +func pinFamily(ctx context.Context, network, addr string) (string, string, error) { + if network != "tcp" { + return network, addr, nil + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", "", err + } + ips, err := lookupNetIP(ctx, "ip", host) + if err != nil { + return "", "", err + } + if len(ips) == 0 { + return "", "", fmt.Errorf("%s: no address", host) + } + ip := ips[0].Unmap() + for _, a := range ips { + if a.Unmap().Is4() { + ip = a.Unmap() + break + } + } + if network, err = Network(ip.String()); err != nil { + return "", "", err + } + return network, net.JoinHostPort(ip.String(), port), nil +} diff --git a/internal/core/listen_test.go b/internal/core/listen_test.go new file mode 100644 index 0000000..8db9d5d --- /dev/null +++ b/internal/core/listen_test.go @@ -0,0 +1,34 @@ +package core + +import ( + "context" + "net/netip" + "testing" +) + +// A hostname listener binds one of its addresses, IPv4 when it has one, in +// that address's family alone, a name for a wildcard address too. +func TestHostnameListenerKeepsToOneFamily(t *testing.T) { + names := map[string][]netip.Addr{ + "wild4": {netip.MustParseAddr("0.0.0.0")}, + "wild6": {netip.MustParseAddr("::")}, + "both": {netip.MustParseAddr("::1"), netip.MustParseAddr("::ffff:127.0.0.1")}, + } + saved := lookupNetIP + t.Cleanup(func() { lookupNetIP = saved }) + lookupNetIP = func(_ context.Context, _, host string) ([]netip.Addr, error) { return names[host], nil } + for _, tc := range []struct{ network, addr, wantNetwork, wantAddr string }{ + {"tcp", "wild4:80", "tcp4", "0.0.0.0:80"}, + {"tcp", "wild6:80", "tcp6", "[::]:80"}, + {"tcp", "both:80", "tcp4", "127.0.0.1:80"}, + {"tcp6", "[::1]:80", "tcp6", "[::1]:80"}, + } { + network, addr, err := pinFamily(t.Context(), tc.network, tc.addr) + if err != nil || network != tc.wantNetwork || addr != tc.wantAddr { + t.Errorf("%s %s: %s %s, %v; want %s %s", tc.network, tc.addr, network, addr, err, tc.wantNetwork, tc.wantAddr) + } + } + if _, _, err := pinFamily(t.Context(), "tcp", "none:80"); err == nil { + t.Error("a name without addresses was accepted") + } +} diff --git a/internal/filter/chain.go b/internal/filter/chain.go index f30dfa3..0738769 100644 --- a/internal/filter/chain.go +++ b/internal/filter/chain.go @@ -79,4 +79,4 @@ func (c *Chain) GetStats() map[string]any { "total_passed": c.totalPassed.Load(), "filters": filterStats, } -} \ No newline at end of file +} diff --git a/internal/filter/filter.go b/internal/filter/filter.go index bf17a23..b860fd4 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -209,4 +209,4 @@ func (f *Filter) matches(text string) bool { "logic", f.config.Logic) return false } -} \ No newline at end of file +} diff --git a/internal/flow/flow.go b/internal/flow/flow.go index 98014a7..2cfa76a 100644 --- a/internal/flow/flow.go +++ b/internal/flow/flow.go @@ -161,4 +161,3 @@ func (f *Flow) GetStats() map[string]any { return stats } - diff --git a/internal/flow/heartbeat.go b/internal/flow/heartbeat.go index e723aea..09f1b20 100644 --- a/internal/flow/heartbeat.go +++ b/internal/flow/heartbeat.go @@ -137,12 +137,10 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent // Use flow's formatter for consistent formatting if adapter, ok := hg.formatter.(*format.FormatterAdapter); ok { // Customize flags for heartbeat if needed - customFlags := int64(0) + // The formatter shows a timestamp unless told otherwise + customFlags := formatter.FlagDefault if !hg.config.IncludeTimestamp { - // Remove timestamp flag if not wanted - customFlags = formatter.FlagShowLevel - } else { - customFlags = formatter.FlagDefault + customFlags = formatter.FlagShowLevel | formatter.FlagNoTimestamp } payload, err = adapter.FormatWithFlags(entry, customFlags) } else { diff --git a/internal/flow/heartbeat_test.go b/internal/flow/heartbeat_test.go new file mode 100644 index 0000000..0649bf9 --- /dev/null +++ b/internal/flow/heartbeat_test.go @@ -0,0 +1,29 @@ +package flow + +import ( + "bytes" + "testing" + "time" + + "logwisp/internal/config" + "logwisp/internal/format" +) + +// include_timestamp = false drops the timestamp the formatter shows by default +func TestHeartbeatTimestampFollowsIncludeTimestamp(t *testing.T) { + at := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + for _, include := range []bool{false, true} { + f, err := format.NewFormatter(&config.FormatConfig{Type: "txt"}) + if err != nil { + t.Fatal(err) + } + hg, err := NewHeartbeatGenerator(&config.HeartbeatConfig{Enabled: true, IncludeTimestamp: include}, f, nil) + if err != nil { + t.Fatal(err) + } + payload := hg.generateHeartbeat(at).Payload + if got := bytes.Contains(payload, []byte("2026-01-02")); got != include { + t.Errorf("include_timestamp=%v: payload %q", include, payload) + } + } +} diff --git a/internal/flow/ratelimiter.go b/internal/flow/ratelimiter.go index e7f95b9..bdf3c6d 100644 --- a/internal/flow/ratelimiter.go +++ b/internal/flow/ratelimiter.go @@ -130,4 +130,4 @@ func policyString(p config.RateLimitPolicy) string { default: return "unknown" } -} \ No newline at end of file +} diff --git a/internal/format/format.go b/internal/format/format.go index bdb47eb..59177c9 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -25,4 +25,4 @@ func NewFormatter(cfg *config.FormatConfig) (Formatter, error) { } return NewFormatterAdapter(cfg) -} \ No newline at end of file +} diff --git a/internal/pipeline/registry.go b/internal/pipeline/registry.go index bd1bddd..ce74dec 100644 --- a/internal/pipeline/registry.go +++ b/internal/pipeline/registry.go @@ -219,4 +219,4 @@ func (r *Registry) RemoveSink(id string) { } delete(r.sinkInstances, id) -} \ No newline at end of file +} diff --git a/internal/plugin/factory.go b/internal/plugin/factory.go index e530031..d92c6c4 100644 --- a/internal/plugin/factory.go +++ b/internal/plugin/factory.go @@ -186,4 +186,4 @@ func ListSinks() []string { types = append(types, t) } return types -} \ No newline at end of file +} diff --git a/internal/sanitize/sanitize.go b/internal/sanitize/sanitize.go index 77e1a81..365b922 100644 --- a/internal/sanitize/sanitize.go +++ b/internal/sanitize/sanitize.go @@ -67,4 +67,4 @@ func IsSafe(data string) bool { } } return true -} \ No newline at end of file +} diff --git a/internal/sink/file/file.go b/internal/sink/file/file.go index 4b685b4..da8f0bb 100644 --- a/internal/sink/file/file.go +++ b/internal/sink/file/file.go @@ -34,9 +34,10 @@ type FileSink struct { config *config.FileSinkOptions // Application - input chan core.TransportEvent - writer *log.Logger // internal logger for file writing - logger *log.Logger // application logger + input chan core.TransportEvent + writer *log.Logger // internal logger for file writing + writerConfig *log.Config + logger *log.Logger // application logger // Runtime done chan struct{} @@ -117,20 +118,21 @@ func NewFileSinkPlugin( writerConfig.ShowLevel = false writerConfig.Format = "raw" - // Create internal logger for file writing - writer := log.NewLogger() - if err := writer.ApplyConfig(writerConfig); err != nil { + // Validated here, applied in Start: applying creates the directory and + // file, which lw --check and a rejected reload must not do + if err := writerConfig.Validate(); err != nil { return nil, fmt.Errorf("failed to initialize file writer: %w", err) } fs := &FileSink{ - id: id, - proxy: proxy, - config: opts, - input: make(chan core.TransportEvent, opts.BufferSize), - writer: writer, - done: make(chan struct{}), - logger: logger, + id: id, + proxy: proxy, + config: opts, + input: make(chan core.TransportEvent, opts.BufferSize), + writer: log.NewLogger(), + writerConfig: writerConfig, + done: make(chan struct{}), + logger: logger, } fs.lastProcessed.Store(time.Time{}) @@ -168,7 +170,9 @@ func (fs *FileSink) Input() chan<- core.TransportEvent { // Start begins the processing loop for the sink func (fs *FileSink) Start(ctx context.Context) error { - // Start the internal file writer + if err := fs.writer.ApplyConfig(fs.writerConfig); err != nil { + return fmt.Errorf("failed to initialize file writer: %w", err) + } if err := fs.writer.Start(); err != nil { return fmt.Errorf("failed to start file writer: %w", err) } diff --git a/internal/sink/http/http.go b/internal/sink/http/http.go index b97f85a..961d2ea 100644 --- a/internal/sink/http/http.go +++ b/internal/sink/http/http.go @@ -55,8 +55,9 @@ type HTTPSink struct { proxy *session.Proxy // Configuration - config *config.HTTPSinkOptions - addr string + config *config.HTTPSinkOptions + addr string + network string // Network server *http.Server @@ -116,6 +117,10 @@ func NewHTTPSinkPlugin( if err := lconfig.Port(opts.Port); err != nil { return nil, fmt.Errorf("port: %w", err) } + network, err := core.Network(opts.Host) + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } if opts.StreamPath == "" { opts.StreamPath = DefaultHTTPStreamPath } else if !strings.HasPrefix(opts.StreamPath, "/") { @@ -166,6 +171,7 @@ func NewHTTPSinkPlugin( proxy: proxy, config: opts, addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)), + network: network, input: make(chan core.TransportEvent, opts.BufferSize), done: make(chan struct{}), logger: logger, @@ -217,10 +223,9 @@ func (h *HTTPSink) Input() chan<- core.TransportEvent { // Start binds the listener and serves stream/status endpoints func (h *HTTPSink) Start(ctx context.Context) error { - // IPv4-only, parity with existing network sinks. // TLS is applied via server.TLSConfig + ServeTLS below, not by wrapping // ln; net/http then owns handshake, ALPN (h2), and per-conn errors. - ln, err := net.Listen("tcp4", h.addr) + ln, err := core.Listen(ctx, &net.ListenConfig{}, h.network, h.addr) if err != nil { return fmt.Errorf("http sink bind %s: %w", h.addr, err) } diff --git a/internal/sink/http/web/login.js b/internal/sink/http/web/login.js index 8452aea..366617f 100644 --- a/internal/sink/http/web/login.js +++ b/internal/sink/http/web/login.js @@ -1,4 +1,4 @@ -import { login } from "./scram.js"; +import { cookiesUsable, login, loginUnavailable } from "./scram.js"; const base = new URL("..", location.href); const form = document.getElementById("login"); @@ -44,5 +44,20 @@ form.addEventListener("submit", async (event) => { } }); +// A sign-in here only sets a cookie; the viewer, when the sink serves it, +// signs in by itself and keeps a token in memory instead +async function cookiesUnavailable() { + status.textContent = "This browser keeps no cookie for this site, so a sign-in here would not last."; + const viewer = await fetch("view", { method: "HEAD", cache: "no-store", redirect: "error" }).catch(() => null); + if (!viewer?.ok) return; + const link = document.createElement("a"); + link.href = "view"; + link.textContent = "Sign in on the viewer instead."; + status.append(" ", link); +} + // Enabled only once this script runs, so a failed load cannot submit the form natively -button.disabled = false; +const unavailable = loginUnavailable(); +if (unavailable) status.textContent = unavailable; +else if (cookiesUsable()) button.disabled = false; +else cookiesUnavailable(); diff --git a/internal/sink/http/web/scram.js b/internal/sink/http/web/scram.js index 197d751..b1a21aa 100644 --- a/internal/sink/http/web/scram.js +++ b/internal/sink/http/web/scram.js @@ -16,11 +16,14 @@ const encoder = new TextEncoder(); /** * Logs in to the logwisp http sink mounted at base (a URL ending in "/"), per - * the wire protocol in logwisp's doc/scram-auth-plan.md. The session arrives as - * an HttpOnly cookie; the promise resolves only once the server has proved it - * holds this user's verifier. onProgress(fraction) follows the Argon2 pass. + * logwisp's doc/scram-auth-plan.md; it resolves only once the server has proved + * it holds this user's verifier. The session is an HttpOnly cookie, or with + * session "token" the returned token. onProgress(fraction) follows Argon2. */ -export async function login(base, username, password, { onProgress } = {}) { +export async function login(base, username, password, { onProgress, session = "cookie" } = {}) { + if (session !== "cookie" && session !== "token") { + throw new Error('session must be "cookie" or "token"'); + } const pw = checkCredentials(username, password); subtle(); // an insecure page fails before anything is sent const url = new URL("auth", base); @@ -29,7 +32,8 @@ export async function login(base, username, password, { onProgress } = {}) { const first = await post(url, { logwisp: 1, scram: { username, client_nonce: clientNonce } }); const { proof, serverSignature } = await prove(username, pw, clientNonce, first.challenge, MIN_TIME, MIN_MEMORY, onProgress); - const step = await post(url, { proof, session: "cookie" }); + // Without "session" the server answers with the token in the body + const step = await post(url, session === "cookie" ? { proof, session } : { proof }); const final = step.final; if (!final || typeof final !== "object") { throw new Error("server sent no final message"); @@ -40,15 +44,19 @@ export async function login(base, username, password, { onProgress } = {}) { throw new Error("server could not prove it holds this user's verifier"); } const expiresIn = Number.isSafeInteger(step.expires_in) && step.expires_in > 0 ? step.expires_in : 0; - return { username, expiresIn }; + if (session === "cookie") return { username, expiresIn }; + if (typeof step.token !== "string" || step.token === "") { + throw new Error("server issued no token"); + } + return { username, expiresIn, token: step.token }; } // Logout posts to auth itself: a clearing cookie from any other path would -// get another default path and miss the session cookie -export async function logout(base) { +// get another default path and miss the session cookie. A token is revoked. +export async function logout(base, { token } = {}) { const res = await send(new URL("auth", base), { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", ...bearer(token) }, body: JSON.stringify({ logout: true }), }); if (res.status !== 200 && res.status !== 204) { @@ -56,6 +64,101 @@ export async function logout(base) { } } +// Why no login can run on this page, or "": an insecure page refuses the probe's +// Secure cookie too, so a page asks this before cookiesUsable(). +export function loginUnavailable() { + return globalThis.crypto?.subtle ? "" : "WebCrypto unavailable: the page must be served over HTTPS"; +} + +// Whether the session cookie would stick. Chromium blocking every cookie still +// reports navigator.cookieEnabled, so a cookie with the session's attributes +// must also come back. +export function cookiesUsable() { + if (!globalThis.navigator?.cookieEnabled) return false; + document.cookie = "logwisp_probe=1; Secure; SameSite=Strict"; + const usable = document.cookie.split("; ").includes("logwisp_probe=1"); + document.cookie = "logwisp_probe=; Max-Age=0; Secure; SameSite=Strict"; + return usable; +} + +/** + * Reads an event stream through fetch, which can send the bearer EventSource + * cannot. onEvent({type, data, lastEventId}) gets each event EventSource would + * dispatch. Resolves to {lastEventId, retry} when the server ends the stream; + * rejects with err.status on an answer other than 200, or on signal's abort. + */ +export async function stream(url, { token, signal, onEvent } = {}) { + const res = await send(new URL(url, globalThis.location?.href), { + headers: { Accept: "text/event-stream", ...bearer(token) }, + signal, + }); + const type = res.headers.get("Content-Type")?.split(";")[0].trim().toLowerCase(); + if (res.status !== 200 || type !== "text/event-stream") { + res.body?.cancel().catch(() => {}); + const err = new Error(res.status !== 200 ? `stream refused: ${res.status}` : "not an event stream"); + err.status = res.status; + throw err; + } + const parser = eventParser(onEvent); + const reader = res.body.getReader(); + const decoder = new TextDecoder(); // UTF-8, dropping a leading BOM as the spec asks + try { + for (let r = await reader.read(); !r.done; r = await reader.read()) { + parser.feed(decoder.decode(r.value, { stream: true })); + } + } finally { + reader.cancel().catch(() => {}); + } + return parser.result(); +} + +// The event stream interpretation of the HTML standard; a CR that ends one +// chunk may be the first half of a CRLF the next chunk completes. +function eventParser(onEvent) { + let line = "", afterCR = false, data = "", type = "", idBuffer = "", lastEventId = "", retry = null; + const field = (name, value) => { + switch (name) { + case "event": type = value; break; + case "data": data += value + "\n"; break; + case "id": if (!value.includes("\0")) idBuffer = value; break; + case "retry": if (/^[0-9]+$/.test(value)) retry = Number(value); break; + } + }; + const dispatch = () => { + lastEventId = idBuffer; + if (data !== "") onEvent?.({ type: type || "message", data: data.slice(0, -1), lastEventId }); + data = type = ""; + }; + const interpret = (l) => { + if (l === "") return dispatch(); + if (l.startsWith(":")) return; + const colon = l.indexOf(":"); + if (colon < 0) return field(l, ""); + const value = l.slice(colon + 1); + field(l.slice(0, colon), value.startsWith(" ") ? value.slice(1) : value); + }; + const breaks = /[\r\n]/g; + return { + feed(text) { + let i = 0; + if (afterCR && text.startsWith("\n")) i = 1; + if (text !== "") afterCR = false; + for (breaks.lastIndex = i; breaks.exec(text); i = breaks.lastIndex) { + const end = breaks.lastIndex - 1; + interpret(line + text.slice(i, end)); + line = ""; + if (text[end] === "\r") { + if (end + 1 === text.length) afterCR = true; + else if (text[end + 1] === "\n") breaks.lastIndex++; + } + } + line += text.slice(i); + }, + // An event the stream ended before its blank line is discarded + result: () => ({ lastEventId, retry }), + }; +} + // Test hook: the proof for a fixed client nonce, under an explicit Argon2 floor export async function _proofForTest(username, password, clientNonce, challenge, minTime, minMemory) { const { proof, serverSignature } = await prove(username, checkCredentials(username, password), @@ -135,19 +238,24 @@ async function post(url, body) { return step; } -// A redirect would replay the proof to wherever it points +// A redirect would replay the proof or the token to wherever it points async function send(url, init) { try { return await fetch(url, { ...init, credentials: "same-origin", cache: "no-store", redirect: "error" }); - } catch { + } catch (err) { + if (err?.name === "AbortError") throw err; throw new Error(`cannot reach ${url.pathname}`); } } +function bearer(token) { + return token ? { Authorization: `Bearer ${token}` } : {}; +} + function subtle() { - const s = globalThis.crypto?.subtle; - if (!s) throw new Error("WebCrypto unavailable: the page must be served over HTTPS"); - return s; + const reason = loginUnavailable(); + if (reason) throw new Error(reason); + return crypto.subtle; } async function hmac(key, message) { diff --git a/internal/sink/http/web/scram.test.mjs b/internal/sink/http/web/scram.test.mjs index 5b562fe..e94e135 100644 --- a/internal/sink/http/web/scram.test.mjs +++ b/internal/sink/http/web/scram.test.mjs @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { argon2id, _proofForTest } from "./scram.js"; +import { argon2id, _proofForTest, login, logout, stream } from "./scram.js"; const hex = (b) => Buffer.from(b).toString("hex"); const unhex = (s) => new Uint8Array(Buffer.from(s, "hex")); @@ -42,3 +42,101 @@ test("proof and server signature match lixenwraith/auth", async () => { server_signature: "0HKxosIZvdoAzstS4dEkNgoYvcDpFnOgwBjyhiSsK00=", }); }); + +const STREAM = "https://site.test/logs/stream"; + +// A fetch answering with an event stream sent in the given chunks +function sseFetch(t, chunks, { status = 200, type = "text/event-stream" } = {}) { + return t.mock.method(globalThis, "fetch", async () => new Response(new ReadableStream({ + start(c) { + for (const chunk of chunks) c.enqueue(new TextEncoder().encode(chunk)); + c.close(); + }, + }), { status, headers: { "Content-Type": type } })); +} + +async function events(t, chunks) { + sseFetch(t, chunks); + const got = []; + const end = await stream(STREAM, { onEvent: (e) => got.push(e) }); + return { got, end }; +} + +// A split CRLF is one break, so "d" and "e" stay one event; a kept BOM would +// rename the first field and drop "a". +test("stream splits lines at CRLF, LF and a lone CR, across chunks", async (t) => { + const { got } = await events(t, ["\uFEFFdata: a\r\ndata: b\rdata: c\n\r", "\n", "data: d\r", "\ndata: e\r", "\r\n"]); + assert.deepEqual(got, [ + { type: "message", data: "a\nb\nc", lastEventId: "" }, + { type: "message", data: "d\ne", lastEventId: "" }, + ]); +}); + +test("stream interprets fields as the HTML standard does", async (t) => { + const { got, end } = await events(t, [ + ": comment\nevent: connected\ndata:{\"a\":1}\nid: 7\n\n", + "data\n\n", + "data: one space kept\nunknown: x\nid: 8\0\nretry: 3000\nretry: 15x\n\n", + "id: 8\nevent: no-data\n\n", + "id\ndata: never ended\n", + ]); + assert.deepEqual(got, [ + { type: "connected", data: "{\"a\":1}", lastEventId: "7" }, + { type: "message", data: "", lastEventId: "7" }, + { type: "message", data: " one space kept", lastEventId: "7" }, + ]); + assert.deepEqual(end, { lastEventId: "8", retry: 3000 }); +}); + +test("stream presents the bearer and refuses anything but a 200 event stream", async (t) => { + const fetch = sseFetch(t, []); + await stream(STREAM, { token: "tok" }); + const init = fetch.mock.calls[0].arguments[1]; + assert.equal(init.headers.Authorization, "Bearer tok"); + assert.equal(init.redirect, "error"); + fetch.mock.restore(); + sseFetch(t, [], { status: 401 }); + await assert.rejects(stream(STREAM, { token: "tok" }), { status: 401 }); + sseFetch(t, [], { type: "text/html" }); + await assert.rejects(stream(STREAM), { message: "not an event stream" }); +}); + +test("stream rejects with the AbortError once its signal aborts", async (t) => { + t.mock.method(globalThis, "fetch", async (url, { signal }) => signal.throwIfAborted() ?? new Response(new ReadableStream({ + start(c) { + c.enqueue(new TextEncoder().encode("data: first\n\n")); + signal.addEventListener("abort", () => c.error(signal.reason)); + }, + }), { headers: { "Content-Type": "text/event-stream" } })); + const ac = new AbortController(); + await assert.rejects(stream(STREAM, { signal: ac.signal, onEvent: () => ac.abort() }), { name: "AbortError" }); + await assert.rejects(stream(STREAM, { signal: AbortSignal.abort() }), { name: "AbortError" }); +}); + +// The mock server signs with the client's own derivation at the Argon2 floor +test("token mode returns the token, and logout presents it", async (t) => { + const challenge = { salt: Buffer.alloc(16, 9).toString("base64"), argon_time: 3, argon_memory: 65536, argon_threads: 1 }; + const sent = []; + t.mock.method(globalThis, "fetch", async (url, init) => { + const body = JSON.parse(init.body); + sent.push({ body, headers: init.headers }); + if (body.scram) { + challenge.client_nonce = body.scram.client_nonce; + challenge.full_nonce = body.scram.client_nonce + "server-nonce"; + return Response.json({ challenge }); + } + if (body.proof) { + const want = await _proofForTest("alice", "password123", challenge.client_nonce, challenge, 3, 65536); + challenge.proof_ok = body.proof.client_proof === want.client_proof; + return Response.json({ final: { username: "alice", server_signature: want.server_signature }, token: "tok", expires_in: 900 }); + } + return new Response(null, { status: 204 }); + }); + const base = "https://site.test/logs/"; + assert.deepEqual(await login(base, "alice", "password123", { session: "token" }), + { username: "alice", expiresIn: 900, token: "tok" }); + assert.ok(challenge.proof_ok); + assert.equal("session" in sent[1].body, false); + await logout(base, { token: "tok" }); + assert.equal(sent[2].headers.Authorization, "Bearer tok"); +}); diff --git a/internal/sink/http/web/style.css b/internal/sink/http/web/style.css index 5d93ea8..b06ec7a 100644 --- a/internal/sink/http/web/style.css +++ b/internal/sink/http/web/style.css @@ -115,3 +115,12 @@ progress { white-space: pre-wrap; overflow-wrap: anywhere; } + +[hidden] { + display: none !important; +} + +.note { + margin: 0; + color: var(--muted); +} diff --git a/internal/sink/http/web/view.html b/internal/sink/http/web/view.html index 22b898f..200146f 100644 --- a/internal/sink/http/web/view.html +++ b/internal/sink/http/web/view.html @@ -14,6 +14,14 @@ connecting… +
diff --git a/internal/sink/http/web/view.js b/internal/sink/http/web/view.js index 8db2c55..8409688 100644 --- a/internal/sink/http/web/view.js +++ b/internal/sink/http/web/view.js @@ -1,4 +1,4 @@ -import { logout } from "./scram.js"; +import { cookiesUsable, login, loginUnavailable, logout, stream } from "./scram.js"; const base = new URL("..", location.href); // The server writes its status path here, relative to the mount @@ -6,7 +6,22 @@ const statusPath = document.querySelector('meta[name="logwisp-status"]').content const MAX_LINES = 5000; const log = document.getElementById("log"); const state = document.getElementById("state"); -let source = null; +const signOut = document.getElementById("logout"); +const form = document.getElementById("login"); +const progress = document.getElementById("progress"); +const status = document.getElementById("status"); + +const unavailable = loginUnavailable(); +// Without cookies the session is a token in this variable only: never stored, +// so it ends with the page, or earlier with its lifetime or sign-out +let tokenMode = !unavailable && !cookiesUsable(); +let token = null; +// The login page comes back with this fragment: a 401 now means the browser +// dropped the session cookie +let fresh = location.hash === "#signed-in"; +if (fresh) history.replaceState(null, "", location.pathname + location.search); +let stopStream = null; +let timer = null; function append(line) { const atBottom = log.scrollTop + log.clientHeight >= log.scrollHeight - 4; @@ -18,50 +33,140 @@ function append(line) { } // The status answer names the stream path; its 401 is the only way to tell an -// expired session apart, since EventSource hides the status of a failure. +// ended session apart, since EventSource hides the status of a failure. async function connect() { + if (tokenMode && !token) return showLogin(""); let res; try { - res = await fetch(new URL("./" + statusPath, base), { credentials: "same-origin", cache: "no-store", redirect: "error" }); + res = await fetch(new URL("./" + statusPath, base), { + headers: token ? { Authorization: `Bearer ${token}` } : {}, + credentials: "same-origin", cache: "no-store", redirect: "error", + }); } catch { return retry("server unreachable"); } - if (res.status === 401) { - location.replace(new URL("login?next=view", location.href).href); - return; - } + if (res.status === 401) return sessionEnded(); + fresh = false; const path = res.ok ? (await res.json().catch(() => ({})))?.endpoints?.stream : null; if (typeof path !== "string" || !path.startsWith("/")) { return retry(`status unavailable (${res.status})`); } // Stream paths are absolute in logwisp; the proxy may mount it under a prefix - source = new EventSource(new URL("./" + path.replace(/^\/+/, ""), base)); - source.addEventListener("connected", () => { state.textContent = "live"; }); - source.addEventListener("disconnect", () => { - source.close(); + const url = new URL("./" + path.replace(/^\/+/, ""), base); + if (tokenMode) streamWithToken(url); + else streamWithCookie(url); +} + +function onEvent(type, data) { + if (type === "connected") state.textContent = "live"; + if (type === "disconnect") { + stopStream(); retry("server shut down"); - }); - source.onmessage = (event) => event.data.split("\n").forEach(append); + } + if (type === "message") data.split("\n").forEach(append); +} + +function streamWithCookie(url) { + const source = new EventSource(url); + stopStream = () => source.close(); + for (const type of ["connected", "disconnect", "message"]) { + source.addEventListener(type, (event) => onEvent(type, event.data)); + } source.onerror = () => { if (source.readyState === EventSource.CLOSED) retry("disconnected"); else state.textContent = "reconnecting…"; }; } -function retry(reason) { - state.textContent = `${reason}, retrying…`; - setTimeout(connect, 5000); +async function streamWithToken(url) { + const abort = new AbortController(); + stopStream = () => abort.abort(); + let refused = 0; + try { + await stream(url, { token, signal: abort.signal, onEvent: (event) => onEvent(event.type, event.data) }); + } catch (err) { + refused = err.status ?? 0; + } + if (abort.signal.aborted) return; + if (refused === 401) sessionEnded(); + else retry(refused ? `stream refused (${refused})` : "disconnected"); } -document.getElementById("logout").addEventListener("click", async () => { - source?.close(); +function sessionEnded() { + if (!tokenMode && !fresh) { + location.replace(new URL("login?next=" + encodeURIComponent("view#signed-in"), location.href).href); + return; + } + const expired = token !== null; + tokenMode = true; + token = null; + showLogin(expired ? "The session ended; sign in again." : ""); +} + +function showLogin(message) { + stopStream?.(); + clearTimeout(timer); + state.textContent = "signed out"; + signOut.hidden = true; + status.textContent = message; + form.hidden = false; + form.elements.username.focus(); +} + +function retry(reason) { + state.textContent = `${reason}, retrying…`; + clearTimeout(timer); + timer = setTimeout(connect, 5000); +} + +form.addEventListener("submit", async (event) => { + event.preventDefault(); + const fields = new FormData(form); + const button = form.querySelector("button"); + button.disabled = true; + progress.removeAttribute("value"); + progress.hidden = false; + status.textContent = "Deriving the key, this takes a few seconds…"; try { - await logout(base); + ({ token } = await login(base, fields.get("username"), fields.get("password"), { + session: "token", + onProgress: (fraction) => { progress.value = fraction; }, + })); + } catch (err) { + status.textContent = err.message; + return; + } finally { + progress.hidden = true; + button.disabled = false; + } + form.elements.password.value = ""; + form.hidden = true; + signOut.hidden = false; + state.textContent = "connecting…"; + connect(); +}); + +signOut.addEventListener("click", async () => { + stopStream?.(); + clearTimeout(timer); + try { + await logout(base, { token }); } catch (err) { state.textContent = err.message; return; } - location.replace(new URL("login", location.href).href); + if (!tokenMode) { + location.replace(new URL("login", location.href).href); + return; + } + token = null; + showLogin("Signed out."); }); -connect(); +// No session of either kind can start on this page, so it offers no form +if (unavailable) { + state.textContent = unavailable; + signOut.hidden = true; +} else { + connect(); +} diff --git a/internal/sink/httpchain/httpchain.go b/internal/sink/httpchain/httpchain.go index 2434482..34e439a 100644 --- a/internal/sink/httpchain/httpchain.go +++ b/internal/sink/httpchain/httpchain.go @@ -9,6 +9,7 @@ import ( "maps" "net" "net/http" + "net/url" "os" "strconv" "strings" @@ -103,6 +104,10 @@ func NewHTTPChainSinkPlugin( if err := lconfig.Port(opts.Port); err != nil { return nil, fmt.Errorf("port: %w", err) } + network, err := core.Network(opts.Host) + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } if opts.IngestPath == "" { opts.IngestPath = DefaultHTTPChainSinkIngestPath } else if !strings.HasPrefix(opts.IngestPath, "/") { @@ -158,10 +163,9 @@ func NewHTTPChainSinkPlugin( addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)) transport := &http.Transport{ - DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - // IPv4-only, aligns with tcp/http sinks + DialContext: func(ctx context.Context, _, address string) (net.Conn, error) { d := net.Dialer{} - return d.DialContext(ctx, "tcp4", address) + return d.DialContext(ctx, network, address) }, MaxIdleConnsPerHost: 2, IdleConnTimeout: 90 * time.Second, @@ -172,9 +176,9 @@ func NewHTTPChainSinkPlugin( // NDJSON POSTs gain nothing from it } - scheme := "http" + base := url.URL{Scheme: "http", Host: addr} // escapes an IPv6 zone if tlsCfg != nil { - scheme = "https" + base.Scheme = "https" } t := &HTTPChainSink{ @@ -185,8 +189,8 @@ func NewHTTPChainSinkPlugin( tlsEnabled: tlsCfg != nil, mtls: tlsCfg != nil && len(tlsCfg.Certificates) > 0, auth: authPolicy, - baseURL: scheme + "://" + addr, - url: scheme + "://" + addr + opts.IngestPath, + baseURL: base.String(), + url: base.String() + opts.IngestPath, // A redirect would resend the batch, even from https to plaintext http client: &http.Client{ Transport: transport, diff --git a/internal/sink/httpchain/httpchain_test.go b/internal/sink/httpchain/httpchain_test.go index cf7c533..9853551 100644 --- a/internal/sink/httpchain/httpchain_test.go +++ b/internal/sink/httpchain/httpchain_test.go @@ -1,6 +1,7 @@ package httpchain import ( + "context" "crypto/tls" "net" "net/http" @@ -16,6 +17,7 @@ import ( "logwisp/internal/config" "logwisp/internal/core" "logwisp/internal/session" + ingest "logwisp/internal/source/httpchain" "logwisp/internal/testutil" "github.com/lixenwraith/auth" @@ -126,3 +128,75 @@ func TestRefusedTokenLogsInAgainAndDeliversOnce(t *testing.T) { t.Fatalf("logins = %v, want 2", logins) } } + +// The target URL brackets an IPv6 host and escapes its zone: a URL built +// from the raw host could not even form a request. +func TestTargetURLBracketsAnIPv6Host(t *testing.T) { + manager := session.NewManager(time.Hour) + t.Cleanup(manager.Stop) + for host, want := range map[string]string{ + "::1": "http://[::1]:9000/ingest", + "fe80::1%eth0": "http://[fe80::1%25eth0]:9000/ingest", + } { + created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{"host": host, "port": int64(9000)}, + log.NewLogger(), session.NewProxy(manager, "fwd")) + if err != nil { + t.Fatal(err) + } + if got := created.(*HTTPChainSink).url; got != want { + t.Errorf("host %q: target %q, want %q", host, got, want) + } + } +} + +// A link runs over the IPv6 loopback with TLS: the source binds ::1, and the +// sink dials it and verifies the certificate's ::1 address. +func TestLinkOverIPv6LoopbackWithTLS(t *testing.T) { + testutil.RequireIPv6(t) + pki := testutil.NewPKI(t, "edge-01") + probe, err := net.Listen("tcp6", "[::1]:0") + if err != nil { + t.Fatal(err) + } + _, port, _ := net.SplitHostPort(probe.Addr().String()) + probe.Close() + p, _ := strconv.ParseInt(port, 10, 64) + manager := session.NewManager(time.Hour) + t.Cleanup(manager.Stop) + + src, err := ingest.NewHTTPChainSourcePlugin("in", map[string]any{ + "host": "::1", "port": p, + "tls": map[string]any{"enabled": true, "cert_file": pki.ServerCert, "key_file": pki.ServerKey}, + }, log.NewLogger(), session.NewProxy(manager, "in")) + if err != nil { + t.Fatal(err) + } + entries := src.Subscribe() + if err := src.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(src.Stop) + + created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{ + "host": "::1", "port": p, + "tls": map[string]any{"enabled": true, "ca_file": pki.CA}, + }, log.NewLogger(), session.NewProxy(manager, "fwd")) + if err != nil { + t.Fatal(err) + } + sink := created.(*HTTPChainSink) + sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("over IPv6")}) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if !sink.flush(ctx) || sink.droppedBatches.Load() != 0 { + t.Fatalf("batch not delivered: %d request errors", sink.requestErrors.Load()) + } + select { + case e := <-entries: + if e.Message != "over IPv6" { + t.Fatalf("message %q", e.Message) + } + case <-time.After(2 * time.Second): + t.Fatal("the source published nothing") + } +} diff --git a/internal/sink/sink.go b/internal/sink/sink.go index aaecf00..9c7af68 100644 --- a/internal/sink/sink.go +++ b/internal/sink/sink.go @@ -34,4 +34,4 @@ type SinkStats struct { StartTime time.Time LastProcessed time.Time Details map[string]any -} \ No newline at end of file +} diff --git a/internal/sink/tcp/tcp.go b/internal/sink/tcp/tcp.go index 3d2ae24..39df5cc 100644 --- a/internal/sink/tcp/tcp.go +++ b/internal/sink/tcp/tcp.go @@ -48,8 +48,9 @@ type TCPSink struct { proxy *session.Proxy // Configuration - config *config.TCPSinkOptions - addr string + config *config.TCPSinkOptions + addr string + network string // Network listener net.Listener @@ -114,6 +115,10 @@ func NewTCPSinkPlugin( if err := lconfig.Port(opts.Port); err != nil { return nil, fmt.Errorf("port: %w", err) } + network, err := core.Network(opts.Host) + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } if opts.BufferSize <= 0 { opts.BufferSize = DefaultTCPBufferSize } @@ -140,6 +145,7 @@ func NewTCPSinkPlugin( proxy: proxy, config: opts, addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)), + network: network, input: make(chan core.TransportEvent, opts.BufferSize), done: make(chan struct{}), logger: logger, @@ -192,8 +198,7 @@ func (t *TCPSink) listen() (net.Listener, error) { Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond, } } - // IPv4-only, parity with existing network sinks - ln, err := lc.Listen(context.Background(), "tcp4", t.addr) + ln, err := core.Listen(context.Background(), &lc, t.network, t.addr) if err != nil { return nil, err } diff --git a/internal/sink/tcpchain/tcpchain.go b/internal/sink/tcpchain/tcpchain.go index 1039758..fbb2e6d 100644 --- a/internal/sink/tcpchain/tcpchain.go +++ b/internal/sink/tcpchain/tcpchain.go @@ -51,6 +51,7 @@ type TCPChainSink struct { node string addr string + network string tlsConfig *tls.Config // Authorization: pins the downstream server's identity @@ -100,6 +101,10 @@ func NewTCPChainSinkPlugin( if err := lconfig.Port(opts.Port); err != nil { return nil, fmt.Errorf("port: %w", err) } + network, err := core.Network(opts.Host) + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } if opts.BufferSize <= 0 { opts.BufferSize = DefaultChainSinkBufferSize @@ -149,6 +154,7 @@ func NewTCPChainSinkPlugin( config: opts, node: node, addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)), + network: network, tlsConfig: tlsCfg, auth: authPolicy, input: make(chan core.TransportEvent, opts.BufferSize), @@ -377,10 +383,10 @@ func (t *TCPChainSink) connect(ctx context.Context) error { // handshake under ctx, so bound dial + handshake together here dctx, cancel := context.WithTimeout(ctx, t.dialTimeout+tlsx.HandshakeTimeout) td := tls.Dialer{NetDialer: &nd, Config: t.tlsConfig} - conn, err = td.DialContext(dctx, "tcp4", t.addr) // IPv4-only + conn, err = td.DialContext(dctx, t.network, t.addr) cancel() } else { - conn, err = nd.DialContext(ctx, "tcp4", t.addr) // IPv4-only + conn, err = nd.DialContext(ctx, t.network, t.addr) } if err != nil { return err diff --git a/internal/source/httpchain/httpchain.go b/internal/source/httpchain/httpchain.go index 1fc2994..d9922ed 100644 --- a/internal/source/httpchain/httpchain.go +++ b/internal/source/httpchain/httpchain.go @@ -45,9 +45,10 @@ const ( // HTTPChainSource accepts NDJSON batches from upstream http_chain sinks type HTTPChainSource struct { - id string - proxy *session.Proxy - config *config.HTTPChainSourceOptions + id string + proxy *session.Proxy + config *config.HTTPChainSourceOptions + network string subscribers []chan core.LogEntry server *http.Server @@ -91,6 +92,10 @@ func NewHTTPChainSourcePlugin( if err := lconfig.Port(opts.Port); err != nil { return nil, fmt.Errorf("port: %w", err) } + network, err := core.Network(opts.Host) + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } if opts.IngestPath == "" { opts.IngestPath = DefaultHTTPChainSourceIngestPath } else if !strings.HasPrefix(opts.IngestPath, "/") { @@ -120,6 +125,7 @@ func NewHTTPChainSourcePlugin( id: id, proxy: proxy, config: opts, + network: network, subscribers: make([]chan core.LogEntry, 0), sessions: make(map[string]string), logger: logger, @@ -169,8 +175,7 @@ func (s *HTTPChainSource) Start() error { return err } addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10)) - // IPv4-only, aligns with tcp/http sinks - ln, err := net.Listen("tcp4", addr) + ln, err := core.Listen(context.Background(), &net.ListenConfig{}, s.network, addr) if err != nil { s.auth.Close() return fmt.Errorf("listen %s: %w", addr, err) diff --git a/internal/source/source.go b/internal/source/source.go index 7c48cbc..f342b23 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -59,4 +59,4 @@ func ExtractLogLevel(line string) string { } return "" -} \ No newline at end of file +} diff --git a/internal/source/tcpchain/tcpchain.go b/internal/source/tcpchain/tcpchain.go index bd6ca89..afb6930 100644 --- a/internal/source/tcpchain/tcpchain.go +++ b/internal/source/tcpchain/tcpchain.go @@ -39,9 +39,10 @@ const ( // TCPChainSource accepts connections from upstream tcp_chain sinks and ingests NDJSON entries type TCPChainSource struct { - id string - proxy *session.Proxy - config *config.TCPChainSourceOptions + id string + proxy *session.Proxy + config *config.TCPChainSourceOptions + network string subscribers []chan core.LogEntry listener net.Listener @@ -86,6 +87,10 @@ func NewTCPChainSourcePlugin( if err := lconfig.Port(opts.Port); err != nil { return nil, fmt.Errorf("port: %w", err) } + network, err := core.Network(opts.Host) + if err != nil { + return nil, fmt.Errorf("host: %w", err) + } if opts.BufferSize <= 0 { opts.BufferSize = DefaultChainSourceBufferSize } @@ -105,6 +110,7 @@ func NewTCPChainSourcePlugin( id: id, proxy: proxy, config: opts, + network: network, subscribers: make([]chan core.LogEntry, 0), conns: make(map[net.Conn]struct{}), logger: logger, @@ -153,9 +159,9 @@ func (s *TCPChainSource) Start() error { return err } addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10)) - // IPv4-only. TLS-wrapped when configured; handshake runs explicitly in - // handleConn under tlsx.HandshakeTimeout, pre-hello. - ln, err := net.Listen("tcp4", addr) + // TLS-wrapped when configured; handshake runs explicitly in handleConn + // under tlsx.HandshakeTimeout, pre-hello. + ln, err := core.Listen(context.Background(), &net.ListenConfig{}, s.network, addr) if err != nil { s.auth.Close() return fmt.Errorf("listen %s: %w", addr, err) diff --git a/internal/testutil/fixtures.go b/internal/testutil/fixtures.go index 591d452..6bb41ef 100644 --- a/internal/testutil/fixtures.go +++ b/internal/testutil/fixtures.go @@ -40,8 +40,19 @@ func WriteFile(t testing.TB, path, contents string) { } } -// PKI is a throwaway CA with a server certificate for 127.0.0.1 and one client -// certificate, written as PEM files for plugins that load TLS from disk. +// RequireIPv6 skips a test on a host without an IPv6 loopback +func RequireIPv6(t testing.TB) { + t.Helper() + ln, err := net.Listen("tcp6", "[::1]:0") + if err != nil { + t.Skipf("no IPv6 loopback: %v", err) + } + ln.Close() +} + +// PKI is a throwaway CA with a server certificate for 127.0.0.1 and ::1 and +// one client certificate, written as PEM files for plugins that load TLS from +// disk. type PKI struct { CA, ServerCert, ServerKey, ClientCert, ClientKey string @@ -80,7 +91,7 @@ func NewPKI(t testing.TB, clientCN string) *PKI { } // Leaf issues another certificate from the same CA: a client certificate, or -// a server certificate valid for 127.0.0.1. +// a server certificate valid for 127.0.0.1 and ::1. func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFile string) { t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) @@ -93,7 +104,7 @@ func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFil SerialNumber: big.NewInt(p.serial), Subject: pkix.Name{CommonName: cn}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, } if client { tmpl.ExtKeyUsage, tmpl.IPAddresses = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil diff --git a/internal/tlsx/tlsx.go b/internal/tlsx/tlsx.go index e2f3a58..b29884c 100644 --- a/internal/tlsx/tlsx.go +++ b/internal/tlsx/tlsx.go @@ -63,7 +63,8 @@ func Server(o *config.TLSOptions) (*tls.Config, error) { // Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain // sinks). host seeds ServerName when no override is set; Go verifies IP SANs -// when host is an address. Returns (nil, nil) when disabled. +// when host is an address, without its zone, which names a local interface +// and no certificate carries. Returns (nil, nil) when disabled. func Client(o *config.TLSOptions, host string) (*tls.Config, error) { if o == nil || !o.Enabled { return nil, nil @@ -78,7 +79,7 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) { InsecureSkipVerify: o.InsecureSkipVerify, } if cfg.ServerName == "" { - cfg.ServerName = host + cfg.ServerName, _, _ = strings.Cut(host, "%") } if o.CAFile != "" { pool, err := loadPool(o.CAFile) diff --git a/internal/tlsx/tlsx_test.go b/internal/tlsx/tlsx_test.go index ee2e7c4..d2a6155 100644 --- a/internal/tlsx/tlsx_test.go +++ b/internal/tlsx/tlsx_test.go @@ -6,8 +6,24 @@ import ( "strings" "testing" "time" + + "logwisp/internal/config" ) +// A dialer verifies an IPv6 target by its address: the zone names a local +// interface, which no certificate carries. +func TestClientServerNameDropsTheZone(t *testing.T) { + for host, want := range map[string]string{"fe80::1%eth0": "fe80::1", "::1": "::1", "relay.example": "relay.example"} { + cfg, err := Client(&config.TLSOptions{Enabled: true}, host) + if err != nil { + t.Fatal(err) + } + if cfg.ServerName != want { + t.Errorf("Client(%q).ServerName = %q, want %q", host, cfg.ServerName, want) + } + } +} + // Startup names a certificate outside its validity window or within 30 days // of leaving it; a healthy one stays silent. func TestExpiryWarning(t *testing.T) { diff --git a/internal/version/version.go b/internal/version/version.go index 45a2428..630b683 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -22,4 +22,4 @@ func String() string { // Short returns just the version tag func Short() string { return Version -} \ No newline at end of file +} diff --git a/test/chain-aggregate-test.sh b/test/chain-aggregate-test.sh index 8586c38..7514d23 100755 --- a/test/chain-aggregate-test.sh +++ b/test/chain-aggregate-test.sh @@ -1,95 +1,33 @@ #!/usr/bin/env bash -# logwisp chain topology test -# -# random --> tcp_chain sink --> :15801 tcp_chain src --> :15803 tcp sink -# random --> http_chain sink --> :15802 http_chain src --> :15804 http sink (SSE) -# -# Usage: -# ./chain_aggregate_test.sh manual mode: 2 edge daemons + relay foreground -# ./chain_aggregate_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown -# ./chain_aggregate_test.sh --keep (with --auto) skip teardown on success -# -# The relay aggregates both sinks and fans out aggregated streams into both http and tcp sinks -# +# logwisp chain aggregation test: edge-tcp and edge-http feed one relay +# pipeline that fans both chain sources in and out to a tcp and an http sink. # Requires: bash 5+, coreutils (timeout), curl. Linux dev host only. set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" -RUN="$SCRIPT_DIR/run" -CONF="$RUN/conf" -LOG="$RUN/log" - +RUN=$E2E_DIR/run PORT_TCP_CHAIN=15801 PORT_HTTP_CHAIN=15802 PORT_TCP_SINK=15803 PORT_HTTP_SINK=15804 +PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK" +e2e_init "$@" -AUTO=0; KEEP=0 -for a in "$@"; do case "$a" in - --auto) AUTO=1 ;; - --keep) KEEP=1 ;; - *) echo "unknown arg: $a" >&2; exit 1 ;; -esac; done - -PIDS=() -cleanup() { - local rc=$? - trap - EXIT INT TERM - if (( ${#PIDS[@]} )); then - echo "--- teardown: stopping ${#PIDS[@]} daemon(s)" - kill -TERM "${PIDS[@]}" 2>/dev/null - local deadline=$(( SECONDS + 10 )) - for pid in "${PIDS[@]}"; do - while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done - kill -KILL "$pid" 2>/dev/null - done - fi - exit "$rc" -} -trap cleanup EXIT INT TERM - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; } - -wait_port() { # port timeout_s - local i; for (( i=0; i < $2 * 10; i++ )); do - port_open "$1" && return 0 - sleep 0.1 - done - return 1 -} - -# Reads TCP sink stream for N seconds; connection counts as a sink client -tcp_read() { # port secs - timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true -} - -start_daemon() { # name conf - "$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 & - PIDS+=($!) - echo "started $1 (pid $!)" -} - -# --- Preflight --- -[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; } -for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do - port_open "$p" && { echo "port $p already in use" >&2; exit 1; } -done +section "Setup" +need curl +ports_free $PORTS mkdir -p "$CONF" "$LOG" -# --- Config generation --- -cat > "$CONF/edge_tcp.toml" <"$CONF/$1.toml" < "$CONF/edge_http.toml" < "$CONF/relay.toml" <"$CONF/relay.toml" < live check: nc 127.0.0.1 $PORT_TCP_SINK - $PORT_HTTP_SINK HTTP sink -> live check: browser/curl: - http://127.0.0.1:$PORT_HTTP_SINK/stream (SSE) - http://127.0.0.1:$PORT_HTTP_SINK/status (JSON stats) - Use 127.0.0.1, not localhost — sinks reject IPv6. - Expected: json entries from BOTH nodes ("edge-tcp", "edge-http") - interleaved on :$PORT_TCP_SINK and :$PORT_HTTP_SINK. - Logs: $LOG/ -================================================================ -EOF - -if (( AUTO == 0 )); then - # Manual mode: edges daemonized first (chain sinks backoff-retry), relay foreground - start_daemon edge_tcp edge_tcp.toml - start_daemon edge_http edge_http.toml - echo "--- relay starting in FOREGROUND; Ctrl-C stops relay and both edges" - "$BIN" -c "$CONF/relay.toml" - exit 0 -fi - -# --- Auto mode --- +section "Startup" start_daemon relay relay.toml -for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do - wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; } +for p in $PORTS; do + wait_port "$p" || abort "relay port $p is not listening" relay done -start_daemon edge_tcp edge_tcp.toml +# Chain sinks retry with backoff, so the edges could start first too +start_daemon edge_tcp edge_tcp.toml start_daemon edge_http edge_http.toml +daemons_up -echo "--- settling 3s (connect + first http_chain flush)" +guide "logwisp chain aggregation test" < nc 127.0.0.1 $PORT_TCP_SINK +> curl -N http://127.0.0.1:$PORT_HTTP_SINK/stream +> curl http://127.0.0.1:$PORT_HTTP_SINK/status +Logs: $(short "$LOG")/ +EOF +manual_hold + +section "Checks" +info "settling 3 s (connect, first http_chain flush)" sleep 3 -fail=0 -check() { # label condition_result - if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi -} - -# 1. TCP chain: edge-tcp -> relay -> tcp sink +# 1. tcp sink: both edges, aggregated tcp_out="$(tcp_read "$PORT_TCP_SINK" 4)" -#nt=$(grep -c '"node":"edge-tcp"' <<< "$tcp_out") -#nh=$(grep -c '"node":"edge-http"' <<< "$tcp_out") -nt=$(grep -c 'edge-tcp/' <<< "$tcp_out") -nh=$(grep -c 'edge-http/' <<< "$tcp_out") -check "tcp sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $(( nt >= 1 && nh >= 1 )) +nt=$(grep -c 'edge-tcp/' <<<"$tcp_out") +nh=$(grep -c 'edge-http/' <<<"$tcp_out") +check "tcp sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $((nt >= 1 && nh >= 1)) -# 2. HTTP chain: edge-http -> relay -> SSE sink -sse_out="$(curl -sN --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)" -nt=$(grep -c '^data:.*edge-tcp/' <<< "$sse_out") -nh=$(grep -c '^data:.*edge-http/' <<< "$sse_out") -check "http sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $(( nt >= 1 && nh >= 1 )) +# 2. http sink: both edges, aggregated +sse_out="$(curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)" +nt=$(grep -c '^data:.*edge-tcp/' <<<"$sse_out") +nh=$(grep -c '^data:.*edge-http/' <<<"$sse_out") +check "http sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $((nt >= 1 && nh >= 1)) # 3. HTTP sink status endpoint -status="$(curl -s --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)" -proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<< "$status" | grep -o '[0-9]*' || echo 0) -check "status endpoint: total_processed=$proc > 0" $(( proc > 0 )) - -echo "================================================================" -if (( fail == 0 )); then - echo "RESULT: ALL PASS" - (( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); } -else - echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log" -fi -exit "$fail" +status="$(curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)" +proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<<"$status" | grep -o '[0-9]*' || echo 0) +check "status endpoint: total_processed=$proc > 0" $((proc > 0)) +summary diff --git a/test/chain-test.sh b/test/chain-test.sh index 8a8db11..02d4b36 100755 --- a/test/chain-test.sh +++ b/test/chain-test.sh @@ -1,98 +1,33 @@ #!/usr/bin/env bash -# logwisp chain topology test -# -# random --> tcp_chain sink --> :15801 tcp_chain src --> :15803 tcp sink -# random --> http_chain sink --> :15802 http_chain src --> :15804 http sink (SSE) -# -# Usage: -# ./chain_test.sh manual mode: 2 edge daemons + relay foreground -# ./chain_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown -# ./chain_test.sh --keep (with --auto) skip teardown on success -# -# relay runs TWO independent pipelines; no cross-path aggregation: -# relay_tcp: tcp_chain src :15801 -> tcp sink :15803 (edge-tcp only) -# relay_http: http_chain src :15802 -> http sink :15804 (edge-http only) -# For fan-in aggregation (both edges -> both sinks) see chain_agg_test.sh. -# +# logwisp chain test: edge-tcp and edge-http feed a relay that runs one +# pipeline per transport, without fan-in (chain-aggregate-test.sh fans in). # Requires: bash 5+, coreutils (timeout), curl. Linux dev host only. set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" -RUN="$SCRIPT_DIR/run" -CONF="$RUN/conf" -LOG="$RUN/log" - +RUN=$E2E_DIR/run PORT_TCP_CHAIN=15801 PORT_HTTP_CHAIN=15802 PORT_TCP_SINK=15803 PORT_HTTP_SINK=15804 +PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK" +e2e_init "$@" -AUTO=0; KEEP=0 -for a in "$@"; do case "$a" in - --auto) AUTO=1 ;; - --keep) KEEP=1 ;; - *) echo "unknown arg: $a" >&2; exit 1 ;; -esac; done - -PIDS=() -cleanup() { - local rc=$? - trap - EXIT INT TERM - if (( ${#PIDS[@]} )); then - echo "--- teardown: stopping ${#PIDS[@]} daemon(s)" - kill -TERM "${PIDS[@]}" 2>/dev/null - local deadline=$(( SECONDS + 10 )) - for pid in "${PIDS[@]}"; do - while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done - kill -KILL "$pid" 2>/dev/null - done - fi - exit "$rc" -} -trap cleanup EXIT INT TERM - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; } - -wait_port() { # port timeout_s - local i; for (( i=0; i < $2 * 10; i++ )); do - port_open "$1" && return 0 - sleep 0.1 - done - return 1 -} - -# Reads TCP sink stream for N seconds; connection counts as a sink client -tcp_read() { # port secs - timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true -} - -start_daemon() { # name conf - "$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 & - PIDS+=($!) - echo "started $1 (pid $!)" -} - -# --- Preflight --- -[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; } -for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do - port_open "$p" && { echo "port $p already in use" >&2; exit 1; } -done +section "Setup" +need curl +ports_free $PORTS mkdir -p "$CONF" "$LOG" -# --- Config generation --- -cat > "$CONF/edge_tcp.toml" <"$CONF/$1.toml" < "$CONF/edge_http.toml" < "$CONF/relay.toml" <"$CONF/relay.toml" < live check: nc 127.0.0.1 $PORT_TCP_SINK - $PORT_HTTP_SINK HTTP sink -> live check: browser/curl: - http://127.0.0.1:$PORT_HTTP_SINK/stream (SSE) - http://127.0.0.1:$PORT_HTTP_SINK/status (JSON stats) - Use 127.0.0.1, not localhost — sinks reject IPv6. - Expected: json entries, node "edge-tcp" on :$PORT_TCP_SINK, "edge-http" on :$PORT_HTTP_SINK. - Logs: $LOG/ -================================================================ -EOF - -if (( AUTO == 0 )); then - # Manual mode: edges daemonized first (chain sinks backoff-retry), relay foreground - start_daemon edge_tcp edge_tcp.toml - start_daemon edge_http edge_http.toml - echo "--- relay starting in FOREGROUND; Ctrl-C stops relay and both edges" - "$BIN" -c "$CONF/relay.toml" - exit 0 -fi - -# --- Auto mode --- +section "Startup" start_daemon relay relay.toml -for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do - wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; } +for p in $PORTS; do + wait_port "$p" || abort "relay port $p is not listening" relay done -start_daemon edge_tcp edge_tcp.toml +# Chain sinks retry with backoff, so the edges could start first too +start_daemon edge_tcp edge_tcp.toml start_daemon edge_http edge_http.toml +daemons_up -echo "--- settling 3s (connect + first http_chain flush)" +guide "logwisp chain test" < nc 127.0.0.1 $PORT_TCP_SINK +> curl -N http://127.0.0.1:$PORT_HTTP_SINK/stream +> curl http://127.0.0.1:$PORT_HTTP_SINK/status +Logs: $(short "$LOG")/ +EOF +manual_hold + +section "Checks" +info "settling 3 s (connect, first http_chain flush)" sleep 3 -fail=0 -check() { # label condition_result - if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi -} - # 1. TCP chain: edge-tcp -> relay -> tcp sink tcp_out="$(tcp_read "$PORT_TCP_SINK" 4)" -#n=$(grep -c '"node":"edge-tcp"' <<< "$tcp_out") -n=$(grep -c 'edge-tcp/' <<< "$tcp_out") -check "tcp path: entries on :$PORT_TCP_SINK with node=edge-tcp ($n lines)" $(( n >= 1 )) +n=$(grep -c 'edge-tcp/' <<<"$tcp_out") +check "tcp path: entries on :$PORT_TCP_SINK with node=edge-tcp ($n lines)" $((n >= 1)) # 2. HTTP chain: edge-http -> relay -> SSE sink -sse_out="$(curl -sN --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)" -n=$(grep -c '^data:.*edge-http/' <<< "$sse_out") -check "http path: SSE events on :$PORT_HTTP_SINK with node=edge-http ($n events)" $(( n >= 1 )) +sse_out="$(curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)" +n=$(grep -c '^data:.*edge-http/' <<<"$sse_out") +check "http path: SSE events on :$PORT_HTTP_SINK with node=edge-http ($n events)" $((n >= 1)) # 3. HTTP sink status endpoint -status="$(curl -s --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)" -proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<< "$status" | grep -o '[0-9]*' || echo 0) -check "status endpoint: total_processed=$proc > 0" $(( proc > 0 )) - -echo "================================================================" -if (( fail == 0 )); then - echo "RESULT: ALL PASS" - (( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); } -else - echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log" -fi -exit "$fail" +status="$(curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)" +proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<<"$status" | grep -o '[0-9]*' || echo 0) +check "status endpoint: total_processed=$proc > 0" $((proc > 0)) +summary diff --git a/test/deploy-test.sh b/test/deploy-test.sh new file mode 100755 index 0000000..02b00c0 --- /dev/null +++ b/test/deploy-test.sh @@ -0,0 +1,347 @@ +#!/usr/bin/env bash +# deploy/lw-deploy.sh test: --yes generates edges, aggregators and a standalone +# node for the manual runtime, plain and with TLS and scram, that bin/lw runs; +# --yes fails closed, --dry-run changes nothing (manual, native, docker), and a +# root host with docker and the logwisp:dev image runs the container too. +# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only. + +set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +RUN=$E2E_DIR/run-deploy +DEPLOY=${E2E_DIR%/*}/deploy/lw-deploy.sh +PKI=$RUN/pki +OUT=$RUN/out +APP=$RUN/app # the directory the edges tail +SECRETS=$RUN/secrets +PORT_CHAIN=15871 +PORT_HTTP=15872 +PORT_TCP=15873 +PORT_SA_HTTP=15874 +PORT_S_CHAIN=15875 +PORT_S_HTTP=15876 +PORT_D_CHAIN=15877 +PORT_D_HTTP=15878 +PORT_DRY=15879 # only written by the dry run, never bound +PORTS="$PORT_CHAIN $PORT_HTTP $PORT_TCP $PORT_SA_HTTP $PORT_S_CHAIN $PORT_S_HTTP $PORT_D_CHAIN $PORT_D_HTTP" +IMAGE=logwisp:dev +CONTAINER=lw-e2e-deploy-aggregator +e2e_init "$@" + +deploy() { # NAME ARGS...: lw-deploy.sh --yes, its output in $LOG/deploy-NAME.out + local name=$1 + shift + "$DEPLOY" --yes "$@" >"$LOG/deploy-$name.out" 2>&1 +} +said() { grep -qF -- "$2" "$LOG/deploy-$1.out"; } # NAME TEXT: lw-deploy.sh printed TEXT +declare -A GEN # NAME -> lw-deploy.sh's exit status +generate() { deploy "$1" --runtime manual --config-dir "$RUN/$1" --bin "$BIN" "${@:2}"; GEN[$1]=$?; } + +section "Setup" +need curl openssl +[[ -x $DEPLOY ]] || abort "no $(short "$DEPLOY")" +# A container an earlier run left (--keep) holds its published ports +if [[ -d $RUN/docker ]] && command -v docker >/dev/null; then docker rm -f "$CONTAINER" >/dev/null 2>&1; fi +ports_free $PORTS +rm -rf "$RUN" +mkdir -p "$LOG" "$APP" "$OUT" +pki_ca +pki_leaf aggregator aggregator.internal serverAuth "IP:127.0.0.1" +echo "first line, there before the edges start" >"$APP/app.log" +info "test PKI in $(short "$PKI")/; the edges tail $(short "$APP")/*.log" + +generate aggregator --role aggregator --listen 127.0.0.1 --chain-port $PORT_CHAIN \ + --file-dir "$OUT/aggregator" --http-port $PORT_HTTP --tcp-port $PORT_TCP +generate edge --role edge --log-dir "$APP" --from start --aggregator 127.0.0.1 --chain-port $PORT_CHAIN \ + --node deploy-edge +generate standalone --role standalone --log-dir "$APP" --from start --listen 127.0.0.1 \ + --file-dir "$OUT/standalone" --http-port $PORT_SA_HTTP --format json +SECURE_AGGREGATOR=(--role aggregator --listen 127.0.0.1 --chain http --chain-port $PORT_S_CHAIN + --file-dir "$OUT/secure" --http-port $PORT_S_HTTP --tls --cert-file "$PKI/aggregator.crt" + --key-file "$PKI/aggregator.key" --auth scram --add-user edge-01 --sink-auth scram + --add-viewer viewer-01 --secrets-dir "$SECRETS") +generate secure-aggregator "${SECURE_AGGREGATOR[@]}" +generate secure-edge --role edge --log-dir "$APP" --from start --chain http --aggregator 127.0.0.1 \ + --chain-port $PORT_S_CHAIN --node deploy-secure --tls --ca-file "$PKI/ca.crt" --auth scram \ + --password-file "$SECRETS/edge-01.pass" +info "generated with --yes --runtime manual: aggregator, edge, standalone, secure-aggregator, secure-edge" + +section "Startup" +for name in aggregator standalone secure-aggregator edge secure-edge; do + if [[ ${GEN[$name]} != 0 ]]; then + tail -n 5 "$LOG/deploy-$name.out" | sed 's/^/ /' + abort "lw-deploy.sh did not generate $name (exit ${GEN[$name]})" + fi + spawn "$name" "$BIN" -c "$RUN/$name/logwisp.toml" +done +for p in $PORT_CHAIN $PORT_HTTP $PORT_TCP $PORT_SA_HTTP $PORT_S_CHAIN $PORT_S_HTTP; do + wait_port "$p" || abort "port $p is not listening" +done +daemons_up +write_env LW="$BIN" CA="$PKI/ca.crt" PW="$SECRETS/viewer-01.pass" APP="$APP" + +guide "lw-deploy.sh test" < . $(short "$RUN")/env +Append a line and watch it arrive: +> curl -N http://127.0.0.1:$PORT_HTTP/stream +> echo hello >>\$APP/app.log +The secured sink (bearer keeps the token off argv): +> token=\$(\$LW auth token -url https://127.0.0.1:$PORT_S_HTTP -user viewer-01 -password-file \$PW -ca-file \$CA) +> curl -N --cacert \$CA -H @<(bearer) https://127.0.0.1:$PORT_S_HTTP/stream +Logs: $(short "$LOG")/ +EOF +manual_hold + +total_processed() { # PORT: of a plain http sink + curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$1/status" 2>/dev/null | + grep -o '"total_processed":[0-9]*' | grep -o '[0-9]*$' || echo 0 +} + +section "Generated configurations" +for name in aggregator edge standalone secure-aggregator secure-edge; do + conf="$RUN/$name/logwisp.toml" ok=0 + "$BIN" --check -c "$conf" >>"$LOG/lw-check.out" 2>&1 && ok=1 + check "$name: --yes wrote $(short "$conf") (exit ${GEN[$name]}), lw --check accepts it" $ok +done + +section "Plain chain and outputs" +wait_until 10 grep -q 'deploy-edge/app.log "first line' "$OUT/aggregator/logwisp.log" +n=$(grep -c 'deploy-edge/app.log "first line' "$OUT/aggregator/logwisp.log") +check "edge -> aggregator: the line there at start reached the file output ($n)" $((n == 1)) + +live="appended line $$" +tcp_read $PORT_TCP 4 >"$RUN/tcp.out" & +tcp_pid=$! +curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP/stream" >"$RUN/sse.out" 2>/dev/null & +sse_pid=$! +sleep 1 +echo "$live" >>"$APP/app.log" +wait "$tcp_pid" "$sse_pid" +wait_until 10 grep -q "$live" "$OUT/aggregator/logwisp.log" +n=$(grep -c "deploy-edge/app.log \"$live" "$OUT/aggregator/logwisp.log") +check "edge -> aggregator: a line appended later reached the file output ($n)" $((n == 1)) +n=$(grep -c "^data:.*deploy-edge/app.log \"$live" "$RUN/sse.out") +check "aggregator http sink: /stream delivered it live ($n events)" $((n == 1)) +n=$(grep -c "deploy-edge/app.log \"$live" "$RUN/tcp.out") +check "aggregator tcp sink: delivered it live ($n lines)" $((n == 1)) +n=$(total_processed $PORT_HTTP) +check "aggregator http sink: /status counts both lines (total_processed=$n)" $((n >= 2)) + +wait_until 10 grep -q "$live" "$OUT/standalone/logwisp.log" +n=$(grep -c -e '"fields":\["first line' -e "\"fields\":\[\"$live" "$OUT/standalone/logwisp.log") +check "standalone: both lines in its file output, as json ($n)" $((n == 2)) +n=$(total_processed $PORT_SA_HTTP) +check "standalone http sink: /status counts them (total_processed=$n)" $((n >= 2)) + +section "TLS and scram" +wait_until 20 grep -q "edge-01/app.log \"$live" "$OUT/secure/logwisp.log" +n=$(grep -c 'edge-01/app.log' "$OUT/secure/logwisp.log") +check "secure-edge -> secure-aggregator: entries over http_chain, TLS and scram ($n lines)" $((n >= 2)) +n=$(grep -c 'deploy-secure/' "$OUT/secure/logwisp.log") +check "secure-aggregator: the node label is the username, not the edge's own ($n deploy-secure lines)" $((n == 0)) + +modes="$(stat -c %a "$SECRETS/edge-01.pass" "$SECRETS/viewer-01.pass" 2>/dev/null | paste -sd' ')" +ok=$(is "$modes" "600 600") +grep -qx 'username = "edge-01"' "$RUN/secure-aggregator/users.toml" || ok=0 +grep -qx 'username = "viewer-01"' "$RUN/secure-aggregator/viewers.toml" || ok=0 +grep -q 'viewer-01' "$RUN/secure-aggregator/users.toml" && ok=0 +check "secure-aggregator: passwords generated (modes $modes), edge-01 in users.toml, viewer-01 in viewers.toml" $ok + +ok=1 +cmp -s "$PKI/aggregator.crt" "$RUN/secure-aggregator/tls/node.crt" || ok=0 +cmp -s "$PKI/aggregator.key" "$RUN/secure-aggregator/tls/node.key" || ok=0 +cmp -s "$PKI/ca.crt" "$RUN/secure-edge/tls/ca.crt" || ok=0 +cmp -s "$SECRETS/edge-01.pass" "$RUN/secure-edge/chain.pass" || ok=0 +check "certificates, key and the edge's password copied into the configuration directories" $ok + +token="$("$BIN" auth token -url "https://127.0.0.1:$PORT_S_HTTP" -user viewer-01 \ + -password-file "$SECRETS/viewer-01.pass" -ca-file "$PKI/ca.crt" 2>>"$LOG/auth-cli.out")" +code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt" \ + -H @<(printf 'Authorization: Bearer %s\n' "$token") "https://127.0.0.1:$PORT_S_HTTP/status" 2>/dev/null)" +check "secure-aggregator http sink: viewer-01's token opens /status (HTTP $code)" $(is "$code" 200) +code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt" \ + "https://127.0.0.1:$PORT_S_HTTP/status" 2>/dev/null)" +check "secure-aggregator http sink: /status refused without a token (HTTP $code)" $(is "$code" 401) + +section "Re-running a deployment" +conf="$RUN/secure-aggregator/logwisp.toml" +before="$(cksum <"$conf")" pass_before="$(cksum <"$SECRETS/edge-01.pass")" +deploy rerun "${SECURE_AGGREGATOR[@]}" --runtime manual --config-dir "$RUN/secure-aggregator" --bin "$BIN" +rc=$? +ok=$((rc != 0)) +grep -q 'exists (--force replaces it)' "$LOG/deploy-rerun.out" || ok=0 +[[ $(cksum <"$conf") == "$before" ]] || ok=0 +check "without --force, --yes refuses to replace a configuration (exit $rc)" $ok +deploy rerun-force "${SECURE_AGGREGATOR[@]}" --runtime manual --config-dir "$RUN/secure-aggregator" --bin "$BIN" --force +rc=$? +ok=$((rc == 0)) +[[ -f $conf.bak && $(cksum <"$conf.bak") == "$before" ]] || ok=0 +[[ $(cksum <"$SECRETS/edge-01.pass") == "$pass_before" ]] || ok=0 +check "with --force: the old configuration kept as logwisp.toml.bak, the generated passwords reused (exit $rc)" $ok + +section "--yes fails closed" +deploy fail-edge --runtime manual --role edge --config-dir "$RUN/fail-edge" --bin "$BIN" --log-dir "$APP" +rc=$? +ok=$((rc != 0)) +grep -q 'missing value: --aggregator' "$LOG/deploy-fail-edge.out" || ok=0 +[[ -e $RUN/fail-edge ]] && ok=0 +check "an edge without --aggregator stops before writing anything (exit $rc)" $ok +deploy fail-aggregator --runtime manual --role aggregator --config-dir "$RUN/fail-aggregator" --bin "$BIN" +rc=$? +ok=$((rc != 0)) +grep -q 'no output: give --file-dir, --http-port or --tcp-port' "$LOG/deploy-fail-aggregator.out" || ok=0 +[[ -e $RUN/fail-aggregator ]] && ok=0 +check "an aggregator without an output stops before writing anything (exit $rc)" $ok +deploy implied-auth --dry-run --runtime manual --role aggregator --config-dir "$RUN/dry/conf" --listen 127.0.0.1 \ + --http-port $PORT_DRY --tls --cert-file "$PKI/aggregator.crt" --key-file "$PKI/aggregator.key" \ + --ca-file "$PKI/ca.crt" --add-user edge-01 --sink-allow viewer-01 --secrets-dir "$RUN/dry/secrets" +rc=$? +ok=$((rc == 0)) +said implied-auth "over tcp_chain (TLS, auth scram)" || ok=0 +said implied-auth "/stream /status (txt, TLS, auth mtls)" || ok=0 +check "without --auth and --sink-auth, --add-user implies scram and --sink-allow mtls (exit $rc)" $ok +deploy fail-auth --runtime manual --role aggregator --config-dir "$RUN/fail-auth" --bin "$BIN" \ + --file-dir "$RUN/fail-auth/out" --tls --cert-file "$PKI/aggregator.crt" --key-file "$PKI/aggregator.key" \ + --auth none --add-user edge-01 --secrets-dir "$RUN/fail-auth/secrets" +rc=$? +ok=$((rc != 0)) +said fail-auth '--add-user needs --auth scram' || ok=0 +[[ -e $RUN/fail-auth ]] && ok=0 +check "an --add-user that --auth none contradicts stops before writing anything (exit $rc)" $ok +deploy fail-pattern --runtime manual --role standalone --config-dir "$RUN/fail-pattern" --bin "$BIN" --log-dir "$APP" \ + --file-dir "$RUN/fail-pattern/out" --log-pattern $'*.log\nfrom = "start"' +rc=$? +ok=$((rc != 0)) +said fail-pattern 'invalid --log-pattern' || ok=0 +[[ -e $RUN/fail-pattern ]] && ok=0 +check "a --log-pattern with a newline stops before writing anything (exit $rc)" $ok +(cd "$RUN" && deploy fail-secrets --runtime manual --role aggregator --config-dir "$RUN/fail-secrets" \ + --bin "$BIN" --file-dir "$RUN/fail-secrets/out" --tls --cert-file "$PKI/aggregator.crt" \ + --key-file "$PKI/aggregator.key" --auth scram --add-user edge-01 --secrets-dir '~/secrets') +rc=$? +ok=$((rc != 0)) +said fail-secrets 'not an absolute path: ~/secrets' || ok=0 +[[ -e $RUN/fail-secrets || -e "$RUN/~" ]] && ok=0 +check "a --secrets-dir of ~/secrets stops before writing anything (exit $rc)" $ok +deploy fail-check --runtime manual --role aggregator --config-dir "$RUN/fail-check" --bin "$BIN" \ + --file-dir "$RUN/fail-check/out" --tls --cert-file "$APP/app.log" --key-file "$APP/app.log" +rc=$? +ok=$((rc != 0)) +said fail-check 'lw --check rejects' || ok=0 +check "a configuration lw --check rejects (a log file as the certificate) stops the run (exit $rc)" $ok +deploy fail-tmp --dry-run --runtime native --role standalone --log-dir "$APP" --file-dir /tmp/lw-deploy-test \ + --bin "$BIN" +rc=$? +ok=$((rc != 0)) +said fail-tmp 'PrivateTmp=yes' || ok=0 +check "native Linux: a --file-dir under /tmp, which the unit makes private, stops the run (exit $rc)" $ok +# A failing install stands in for the real one: a regression must not install onto this host +mkdir -p "$RUN/stub" +printf '#!/bin/sh\nexit 1\n' >"$RUN/stub/install" +chmod +x "$RUN/stub/install" +PATH="$RUN/stub:$PATH" deploy fail-os --runtime native --os freebsd --role standalone --log-dir "$APP" \ + --file-dir "$RUN/fail-os" --bin "$BIN" +rc=$? +ok=$((rc != 0)) +said fail-os 'needs a freebsd host' || ok=0 +said fail-os '+ install' && ok=0 +check "--os freebsd --runtime native on this Linux host stops before changing anything (exit $rc)" $ok + +section "--dry-run changes nothing" +# What a dry run could touch: the run directory (minus what the daemons write) +# and what a native install writes +snapshot() { + find "$RUN" \( -path "$LOG" -o -path "$OUT" \) -prune -o -printf '%p %s %T@ %m\n' | sort | cksum + stat -c '%n %s %Y' /usr/local/bin/lw /etc/logwisp /etc/systemd/system/logwisp.service \ + /etc/systemd/system/logwisp.service.d /etc/sysusers.d/logwisp.conf /etc/tmpfiles.d/logwisp.conf 2>&1 +} +before="$(snapshot)" + +deploy dry-manual --dry-run --runtime manual --role aggregator --config-dir "$RUN/dry/conf" --bin "$BIN" \ + --listen 127.0.0.1 --http-port $PORT_DRY --file-dir "$RUN/dry/out" --tls --cert-file "$PKI/aggregator.crt" \ + --key-file "$PKI/aggregator.key" --auth scram --add-user edge-01 --sink-auth scram --add-viewer viewer-01 \ + --secrets-dir "$RUN/dry/secrets" +rc=$? +ok=$((rc == 0)) +said dry-manual "+ write $RUN/dry/conf/logwisp.toml" || ok=0 +said dry-manual "+ generate a password into $RUN/dry/secrets/edge-01.pass" || ok=0 +[[ -e $RUN/dry || $(snapshot) != "$before" ]] && ok=0 +check "manual: printed the configuration and the passwords it would generate, wrote nothing (exit $rc)" $ok + +deploy dry-zone --dry-run --runtime manual --role aggregator --config-dir "$RUN/dry/conf" --listen fe80::1%lo \ + --http-port $PORT_DRY --file-dir "$RUN/dry/out" --tls --cert-file "$PKI/aggregator.crt" \ + --key-file "$PKI/aggregator.key" --sink-auth scram --add-viewer viewer-01 --secrets-dir "$RUN/dry/secrets" +rc=$? +ok=$((rc == 0)) +said dry-zone "-url https://[fe80::1%25lo]:$PORT_DRY " || ok=0 +said dry-zone 'host = "fe80::1%lo"' || ok=0 +check "a link-local listener: its URLs escape the zone as %25, its host keeps it (exit $rc)" $ok + +deploy dry-native --dry-run --runtime native --role edge --log-dir "$APP" --aggregator 127.0.0.1 \ + --chain-port $PORT_CHAIN --node deploy-native --bin "$BIN" +rc=$? +ok=$((rc == 0)) +said dry-native "+ install -m 0755 $BIN /usr/local/bin/lw" || ok=0 +said dry-native "+ write /etc/logwisp/logwisp.toml" || ok=0 +said dry-native "+ systemctl enable logwisp" || ok=0 +[[ $(snapshot) != "$before" ]] && ok=0 +check "native: printed the install, the configuration and systemctl, changed nothing (exit $rc)" $ok + +deploy dry-docker --dry-run --runtime docker --role aggregator --name "$CONTAINER" --image "$IMAGE" --no-build \ + --config-dir "$RUN/docker/conf" --file-dir "$OUT/docker" --listen 127.0.0.1 \ + --chain-port $PORT_D_CHAIN --http-port $PORT_D_HTTP +rc=$? +# The command breaks before each option; joined, it is one line +cmd="$(sed -e ':a' -e '/\\$/N; s/\\\n//; ta' "$LOG/deploy-dry-docker.out" | tr -s ' ' | grep '^+ docker run -d ')" +ok=$((rc == 0)) +for want in "--name $CONTAINER" "--restart unless-stopped" "--read-only" "--cap-drop ALL" \ + "--security-opt no-new-privileges" "-u 65532:65532" "-v $RUN/docker/conf:/etc/logwisp:ro" \ + "-v $OUT/docker:$OUT/docker" "-p 127.0.0.1:$PORT_D_CHAIN:$PORT_D_CHAIN" \ + "-p 127.0.0.1:$PORT_D_HTTP:$PORT_D_HTTP" "$IMAGE -c /etc/logwisp/logwisp.toml"; do + [[ $cmd == *" $want"* ]] || { ok=0; info "the docker run command lacks: $want"; } +done +said dry-docker '| host = "0.0.0.0"' || ok=0 +# The snapshot leaves out $OUT, where its --file-dir lies +[[ -e $OUT/docker || $(snapshot) != "$before" ]] && ok=0 +check "docker: a hardened docker run publishing on --listen, the container binding 0.0.0.0, nothing written (exit $rc)" $ok + +section "Docker runtime" +if ! command -v docker >/dev/null || ! docker info >/dev/null 2>&1; then + skip "docker: no docker daemon here" +elif ((EUID != 0)); then + skip "docker: lw-deploy.sh needs root to give the configuration to the image user" +elif ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + skip "docker: no $IMAGE image here (make image)" +else + at_exit "docker rm -f $CONTAINER" + deploy docker --runtime docker --role aggregator --name "$CONTAINER" --image "$IMAGE" --no-build \ + --config-dir "$RUN/docker/conf" --file-dir "$OUT/docker" --listen 127.0.0.1 \ + --chain-port $PORT_D_CHAIN --http-port $PORT_D_HTTP + rc=$? + check "docker: the aggregator container is running (exit $rc)" $(is "$rc" 0) + generate docker-edge --role edge --log-dir "$APP" --aggregator 127.0.0.1 --chain-port $PORT_D_CHAIN \ + --node deploy-docker + spawn docker-edge "$BIN" -c "$RUN/docker-edge/logwisp.toml" + for i in $(seq 1 15); do + echo "line $i for the container" >>"$APP/app.log" + sleep 1 + grep -q 'deploy-docker/app.log' "$OUT/docker/logwisp.log" 2>/dev/null && break + done + n=$(grep -c 'deploy-docker/app.log' "$OUT/docker/logwisp.log" 2>/dev/null) + check "docker: a native edge's entries reached the container's file output (${n:-0} lines)" $((${n:-0} >= 1)) + n=$(total_processed $PORT_D_HTTP) + check "docker: the published http sink answers /status (total_processed=$n)" $((n >= 1)) +fi + +summary diff --git a/test/ipv6-test.sh b/test/ipv6-test.sh new file mode 100755 index 0000000..b359697 --- /dev/null +++ b/test/ipv6-test.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# logwisp IPv6 test: a relay on ::1 takes both chain transports and serves +# viewers over TLS with an IPv6 SAN, and listeners keep to their address +# family. Skips (exit 77) on a host without an IPv6 loopback. +# Requires: bash 5+, openssl, curl, an IPv6 loopback. Linux dev host only. + +set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +RUN=$E2E_DIR/run-ipv6 +PKI=$RUN/pki +AUTH=$RUN/auth +OUT=$RUN/out +USERS=$AUTH/users.toml +PORT_TCP_CHAIN=15851 +PORT_HTTP_CHAIN=15852 +PORT_TCP_SINK=15853 +PORT_HTTP_SINK=15854 +PORT_FAMILY=15855 +PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_FAMILY" +e2e_init "$@" + +section "Setup" +# /proc/net/if_inet6 lists ::1 as 31 zeros and a 1; it is absent without IPv6 +grep -qs '^0\{31\}1 ' /proc/net/if_inet6 || skip_all "this host has no IPv6 loopback (::1)" +need openssl curl +ports_free $PORTS +rm -rf "$RUN" +mkdir -p "$CONF" "$LOG" "$PKI" "$AUTH" "$OUT" + +# The relay certificate names only ::1: every dialer verifies an IPv6 SAN +pki_ca +pki_leaf relay relay.internal serverAuth "IP:::1" +info "test PKI in $(short "$PKI")/: ca, relay (SAN IP ::1)" +add_users "$USERS" edge-01 viewer-01 +info "users edge-01, viewer-01 in $(short "$USERS")" + +tls_server() { printf '[%s.tls]\nenabled = true\ncert_file = "%s"\nkey_file = "%s"\n' "$1" "$PKI/relay.crt" "$PKI/relay.key"; } +scram_server() { printf '[%s.auth]\ntype = "scram"\ncredentials_file = "%s"\n' "$1" "$USERS"; } + +relay_pipeline() { # name source_type source_port sink_type sink_port + cat <"$CONF/relay.toml" + +edge_conf() { # name sink_type port + cat >"$CONF/$1.toml" <"$CONF/$1.toml" < . $(short "$RUN")/env +Read the tcp sink as viewer-01: +> \$LW auth stream -addr [::1]:$PORT_TCP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA +Read the http sink (bearer keeps the token off argv; -g leaves the brackets to the URL): +> token=\$(\$LW auth token -url https://[::1]:$PORT_HTTP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA) +> curl -gN --cacert \$CA -H @<(bearer) https://[::1]:$PORT_HTTP_SINK/stream +Which listener answers on $PORT_FAMILY: +> curl -s http://127.0.0.1:$PORT_FAMILY/status; curl -gs http://[::1]:$PORT_FAMILY/status +Ingested entries: $(short "$OUT")/ Passwords: $(short "$AUTH")/ +Logs: $(short "$LOG")/ +EOF +manual_hold + +relay_log="$LOG/relay.out" +curl_args=(-g -s --max-time 5 --noproxy '*') +answered_by() { # url -> the instance whose /status answered, empty if refused + curl "${curl_args[@]}" "$1/status" 2>/dev/null | grep -o '"instance_id":"[^"]*"' | cut -d'"' -f4 +} + +section "Scenario 1: chain links and viewers over [::1]" + +# 1. Both chain transports deliver; the http_chain edge logged in at +# https://[::1]:PORT/auth first +wait_until 20 has_entry tcp_chain 'edge-01/' +n=$(ingested tcp_chain | grep -c 'edge-01/') +check "tcp_chain: entries from the edge dialing [::1] reached the relay ($n lines)" $((n >= 1)) +wait_until 20 has_entry http_chain 'edge-01/' +n=$(ingested http_chain | grep -c 'edge-01/') +check "http_chain: entries from the edge dialing [::1] reached the relay ($n lines)" $((n >= 1)) + +# The logger quotes a value holding brackets +n=$(grep -cF "addr \"[::1]:$PORT_TCP_CHAIN\"" "$relay_log") +check "relay logged its listener as [::1]:$PORT_TCP_CHAIN ($n)" $((n >= 1)) + +# 2. HTTP sink: lw auth token at an IPv6 URL, the token carried by curl -g +token="$("$BIN" auth token -url "https://[::1]:$PORT_HTTP_SINK" -user viewer-01 \ + -password-file "$AUTH/viewer-01.pass" -ca-file "$PKI/ca.crt" 2>>"$LOG/auth-cli.out")" +check "http sink: lw auth token -url https://[::1]:$PORT_HTTP_SINK issued a token" $(is_set "$token") + +code="$(curl "${curl_args[@]}" -o /dev/null -w '%{http_code}' --cacert "$PKI/ca.crt" \ + -H @<(printf 'Authorization: Bearer %s\n' "$token") "https://[::1]:$PORT_HTTP_SINK/status" 2>/dev/null || true)" +check "http sink: curl -g https://[::1]:$PORT_HTTP_SINK/status with the token (HTTP $code)" $(is "$code" 200) + +sse="$(timeout 4 curl -gsN --noproxy '*' --cacert "$PKI/ca.crt" \ + -H @<(printf 'Authorization: Bearer %s\n' "$token") \ + "https://[::1]:$PORT_HTTP_SINK/stream" 2>/dev/null || true)" +n=$(grep -c '^data:.*edge-01/' <<<"$sse") +check "http sink: /stream delivered SSE events over [::1] ($n events)" $((n >= 1)) + +# 3. TCP sink: lw auth stream at an IPv6 address +timeout 4 "$BIN" auth stream -addr "[::1]:$PORT_TCP_SINK" -user viewer-01 \ + -password-file "$AUTH/viewer-01.pass" -ca-file "$PKI/ca.crt" \ + >"$RUN/stream.out" 2>"$LOG/stream.err" +n=$(grep -c 'edge-01/' "$RUN/stream.out") +check "tcp sink: lw auth stream -addr [::1]:$PORT_TCP_SINK streamed entries ($n lines)" $((n >= 1)) + +section "Scenario 2: each listener keeps to its address family" + +# 4. :: is IPv6-only: it takes [::1] and leaves IPv4, and its port, alone +who="$(answered_by "http://[::1]:$PORT_FAMILY")" +check ":: listener accepts [::1] (answered by ${who:-nobody})" $(is "$who" any6) +who="$(answered_by "http://127.0.0.1:$PORT_FAMILY")" +check ":: listener refuses 127.0.0.1 (answered by ${who:-nobody})" $(is "$who" "") + +start_daemon any4 any4.toml +wait_port $PORT_FAMILY 127.0.0.1 +who="$(answered_by "http://127.0.0.1:$PORT_FAMILY")" +check "0.0.0.0 binds the same port beside :: and takes 127.0.0.1 (answered by ${who:-nobody})" $(is "$who" any4) + +# 5. 0.0.0.0 is IPv4-only: with :: gone, [::1] finds nobody +kill -TERM "$ANY6_PID" +wait "$ANY6_PID" 2>/dev/null +who="$(answered_by "http://[::1]:$PORT_FAMILY")" +check "0.0.0.0 listener refuses [::1] (answered by ${who:-nobody})" $(is "$who" "") + +summary diff --git a/test/lib.sh b/test/lib.sh new file mode 100755 index 0000000..dc40759 --- /dev/null +++ b/test/lib.sh @@ -0,0 +1,288 @@ +# Shared by the e2e scripts (test/*-test.sh): sourced after the script sets RUN, +# then e2e_init parses the common flags. Daemons log to $LOG/NAME.out only. +# Exit codes: 0 every check passed, 1 a check failed, 77 the script skipped. + +E2E_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +BIN=${LOGWISP_BIN:-${E2E_DIR%/*}/bin/lw} +AUTO=0 KEEP=0 FOLLOW=0 +PIDS=() DAEMONS=() AT_EXIT=() +E2E_PASSED=0 E2E_FAILED=0 E2E_SKIPPED=0 +# What --follow shows besides WARN and ERROR lines; a script may extend it +FOLLOW_KEYS='LogWisp started|hot reload completed|Login accepted|rejected|refused' +PKI_DAYS=45 # past the 30-day expiry warning, which would fill the daemon logs + +# https://no-color.org; FORCE_COLOR=1 keeps colour through a pipe +if [[ ${FORCE_COLOR:-0} != 0 ]] || [[ -t 1 && -z ${NO_COLOR:-} ]]; then + C_BOLD=$'\e[1m' C_DIM=$'\e[2m' C_RED=$'\e[31m' C_GREEN=$'\e[32m' + C_YELLOW=$'\e[33m' C_CYAN=$'\e[36m' C_OFF=$'\e[0m' +else + C_BOLD='' C_DIM='' C_RED='' C_GREEN='' C_YELLOW='' C_CYAN='' C_OFF='' +fi + +usage() { + sed -n '2,/^[^#]/s/^# \{0,1\}//p' "$0" + cat <&2; exit 2 ;; + esac + done + CONF=$RUN/conf LOG=$RUN/log + trap cleanup EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + trap 'exit 129' HUP + trap 'exit 141' PIPE + printf '%s%s%s\n' "$C_BOLD" "${0##*/}$( ((AUTO)) && echo ' --auto')" "$C_OFF" +} + +# --- output --- + +rule() { local s='' n=$1; while ((n-- > 0)); do s+='─'; done; printf '%s' "$s"; } +short() { if [[ $1 == "$PWD"/* ]]; then printf '%s' "${1#"$PWD"/}"; else printf '%s' "$1"; fi; } +section() { printf '\n%s── %s %s%s\n' "$C_BOLD$C_CYAN" "$1" "$(rule $((60 - ${#1})))" "$C_OFF"; } +info() { printf ' %s·%s %s\n' "$C_DIM" "$C_OFF" "$*"; } + +pass() { E2E_PASSED=$((E2E_PASSED + 1)); printf ' %sPASS%s %s\n' "$C_BOLD$C_GREEN" "$C_OFF" "$1"; } +fail() { E2E_FAILED=$((E2E_FAILED + 1)); printf ' %sFAIL%s %s\n' "$C_BOLD$C_RED" "$C_OFF" "$1"; } +skip() { E2E_SKIPPED=$((E2E_SKIPPED + 1)); printf ' %sSKIP%s %s\n' "$C_BOLD$C_YELLOW" "$C_OFF" "$1"; } +check() { if (($2)); then pass "$1"; else fail "$1"; fi; } # LABEL 1|0 +is() { [[ $1 == "$2" ]] && echo 1 || echo 0; } # ACTUAL EXPECTED +is_set() { [[ -n $1 ]] && echo 1 || echo 0; } + +# guide TITLE, the body on stdin: "> " lines are commands, " NNNNN " lines +# ports, lines ending in ":" headings. +guide() { + local line + printf '\n%s┌── %s %s%s\n' "$C_BOLD" "$1" "$(rule $((59 - ${#1})))" "$C_OFF" + while IFS= read -r line; do + case $line in + '> '*) line=" $C_CYAN${line#> }$C_OFF" ;; + ' '[0-9][0-9][0-9][0-9][0-9]' '*) line=" $C_YELLOW${line:2:5}$C_OFF${line:7}" ;; + *:) line="$C_BOLD$line$C_OFF" ;; + esac + printf '%s│%s %s\n' "$C_BOLD" "$C_OFF" "$line" + done + printf '%s└%s%s\n' "$C_BOLD" "$(rule 63)" "$C_OFF" +} + +# write_env NAME=VALUE...: $RUN/env, which the guide's commands source +write_env() { + local kv + { + echo "# . $(short "$RUN")/env: what the guide of ${0##*/} refers to" + echo 'export no_proxy="127.0.0.1,127.0.0.2,::1${no_proxy:+,$no_proxy}"; export NO_PROXY=$no_proxy' + for kv; do printf 'export %s=%q\n' "${kv%%=*}" "${kv#*=}"; done + # keeps a bearer token off argv: curl -H @<(bearer) + echo "bearer() { printf 'Authorization: Bearer %s\\n' \"\$token\"; }" + } >"$RUN/env" +} + +# --- results --- + +# abort MESSAGE [DAEMON]: a failure the checks cannot run past; shows the +# daemon's last warnings and errors +abort() { + fail "$1" + ABORTED=${2:-} + [[ -z $ABORTED ]] || excerpt "$ABORTED" '^([^ ]+ (WARN|ERROR) |Error: )' 5 + summary +} + +summary() { + local name counts="$E2E_PASSED passed, $E2E_FAILED failed, $E2E_SKIPPED skipped" + stop_follow + if ((E2E_FAILED == 0 && KEEP)); then + info "--keep: left running: pids ${PIDS[*]}${AT_EXIT[*]:+; later: ${AT_EXIT[*]}}" + PIDS=() AT_EXIT=() + fi + stop_all + if ((E2E_FAILED)); then + for name in "${DAEMONS[@]}"; do + [[ $name == "${ABORTED:-}" ]] || excerpt "$name" '^([^ ]+ ERROR |Error: )' + done + fi + printf '%s%s%s\n' "$C_DIM" "$(rule 64)" "$C_OFF" + if ((E2E_FAILED)); then + printf '%sRESULT: FAILURES%s (%s); logs in %s/\n' "$C_BOLD$C_RED" "$C_OFF" "$counts" "$(short "$LOG")" + exit 1 + fi + if ((E2E_SKIPPED)); then + printf '%sRESULT: PASS%s (%s)\n' "$C_BOLD$C_GREEN" "$C_OFF" "$counts" + else + printf '%sRESULT: ALL PASS%s (%s)\n' "$C_BOLD$C_GREEN" "$C_OFF" "$counts" + fi + exit 0 +} + +skip_all() { # REASON: nothing in the script applies to this host + skip "$1" + printf '%s%s%s\n%sRESULT: SKIPPED%s\n' "$C_DIM" "$(rule 64)" "$C_OFF" "$C_BOLD$C_YELLOW" "$C_OFF" + exit 77 +} + +# --- processes --- + +spawn() { # NAME COMMAND...: in the background, output to $LOG/NAME.out + local name=$1 + shift + "$@" >"$LOG/$name.out" 2>&1 & + PIDS+=($!) DAEMONS+=("$name") +} +start_daemon() { spawn "$1" "$BIN" -c "$CONF/$2"; } # NAME CONF_FILE +daemons_up() { info "running: ${DAEMONS[*]} (logs: $(short "$LOG")/NAME.out)"; } +at_exit() { AT_EXIT+=("$1"); } # COMMAND, run at teardown + +stop_all() { # TERM every process, KILL what outlives 10 s + local pid deadline=$((SECONDS + 10)) + ((${#PIDS[@]})) || return 0 + kill -TERM "${PIDS[@]}" 2>/dev/null + for pid in "${PIDS[@]}"; do + while kill -0 "$pid" 2>/dev/null && ((SECONDS < deadline)); do sleep 0.1; done + kill -KILL "$pid" 2>/dev/null + done + info "stopped ${#PIDS[@]} process(es)" + PIDS=() +} + +cleanup() { + local rc=$? c + trap '' PIPE + trap - EXIT + stop_follow + stop_all + for c in "${AT_EXIT[@]}"; do eval "$c" >/dev/null 2>&1; done + exit "$rc" +} + +# follow_logs: with --follow, WARN, ERROR and FOLLOW_KEYS lines of every daemon +# started so far, as they are logged +follow_logs() { + local name files=() + ((FOLLOW)) || return 0 + for name in "${DAEMONS[@]}"; do files+=("$LOG/$name.out"); done + tail -v -n +1 -F "${files[@]}" 2>/dev/null > >(follow_filter) & + FOLLOW_PID=$! + info "following warnings, errors and key events of: ${DAEMONS[*]}" +} +follow_filter() { # tail -v output -> "NAME HH:MM:SS LEVEL message"; read, not awk, for line buffering + local line name='' ts lvl rest c + while IFS= read -r line; do + case $line in '==> '*' <==') + name=${line#==> } name=${name% <==} name=${name##*/} name=${name%.out} + continue ;; + esac + read -r ts lvl rest <<<"$line" + # lw's own startup failure, before its logger runs: "Error: ..." + [[ $ts == Error: ]] && rest="$lvl $rest" lvl=ERROR ts='' + case $lvl in + ERROR) c=$C_RED ;; + WARN) c=$C_YELLOW ;; + *) [[ $line =~ $FOLLOW_KEYS ]] || continue; c=$C_CYAN ;; + esac + printf ' %s%-12s %s%s %s%-5s %s%s\n' "$C_DIM" "$name" "${ts:11:8}" "$C_OFF" "$c" "$lvl" "${rest#msg }" "$C_OFF" + done +} +stop_follow() { [[ -z ${FOLLOW_PID:-} ]] || kill "$FOLLOW_PID" 2>/dev/null; FOLLOW_PID=''; } + +# manual_hold: starts --follow; without --auto, keeps the daemons up until Ctrl-C +manual_hold() { + follow_logs + ((AUTO)) && return 0 + info "manual mode: the daemons stay up until Ctrl-C (--auto runs the checks)" + while :; do sleep 1; done +} + +# --- network and files --- + +# listening PORT [HOST]: a listener on PORT in HOST's address family. It reads +# the kernel's table: a probe connection would land in the daemon logs, as a +# failed TLS handshake or hello the checks count. +listening() { + local table=/proc/net/tcp + [[ ${2:-} == *:* ]] && table=/proc/net/tcp6 + awk -v port="$(printf ':%04X' "$1")" '$4 == "0A" && substr($2, length($2) - 4) == port { found = 1 } + END { exit !found }' "$table" 2>/dev/null +} +wait_port() { # PORT [HOST]: up to 10 s + local i + for ((i = 0; i < 100; i++)); do + listening "$@" && return 0 + sleep 0.1 + done + return 1 +} +wait_until() { # SECONDS COMMAND... + local deadline=$((SECONDS + $1)) + shift + until "$@"; do + ((SECONDS >= deadline)) && return 1 + sleep 0.2 + done +} +ports_free() { # PORT...: in either family, as a dual-stack [::] listener holds IPv4 too + local p + for p; do + listening "$p" || listening "$p" :: && abort "port $p is already in use" + done + return 0 +} +need() { # TOOL...: the binary and each tool, or abort + [[ -x $BIN ]] || abort "no lw binary at $(short "$BIN"): make build" + local t + for t; do command -v "$t" >/dev/null || abort "$t not found"; done +} + +# What a plain TCP sink client receives in SECONDS +tcp_read() { timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true; } +ingested() { cat "$OUT/$1"* 2>/dev/null; } # FILE_PREFIX: the relay's file sinks +has_entry() { ingested "$1" | grep -q -- "$2"; } # FILE_PREFIX PATTERN +count() { grep -c -- "$2" "$LOG/$1.out" 2>/dev/null; } # DAEMON PATTERN +logged() { local n; n=$(count "$1" "$2"); ((${n:-0} >= ${3:-1})); } # DAEMON PATTERN [MIN] +excerpt() { # DAEMON ERE [LINES]: its last matching WARN or ERROR lines + { echo "==> $1 <=="; grep -E "$2" "$LOG/$1.out" 2>/dev/null | tail -n "${3:-3}"; } | follow_filter +} + +# --- credentials --- + +pki_key() { openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1" 2>/dev/null; } +pki_ca() { # the test CA in $PKI + mkdir -p "$PKI" + pki_key "$PKI/ca.key" + openssl req -x509 -new -key "$PKI/ca.key" -days "$PKI_DAYS" -out "$PKI/ca.crt" \ + -subj "/CN=LogWisp Test CA" 2>/dev/null || abort "PKI: cannot create the CA" +} +pki_leaf() { # NAME CN EKU [SAN]: $PKI/NAME.crt and NAME.key from the test CA + local name=$1 ext="extendedKeyUsage=$3" + [[ -n ${4:-} ]] && ext+=$'\n'"subjectAltName=$4" + printf '%s\n' "$ext" >"$PKI/$name.ext" + pki_key "$PKI/$name.key" + openssl req -new -key "$PKI/$name.key" -out "$PKI/$name.csr" -subj "/CN=$2" 2>/dev/null + openssl x509 -req -in "$PKI/$name.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" -CAcreateserial \ + -out "$PKI/$name.crt" -days "$PKI_DAYS" -extfile "$PKI/$name.ext" 2>/dev/null + [[ -s $PKI/$name.crt ]] || abort "PKI: cannot issue $name" +} +add_users() { # CREDENTIALS USER...: through the CLI, passwords in $AUTH/USER.pass + local u + for u in "${@:2}"; do + "$BIN" auth add-user -credentials "$1" -user "$u" -password-file "$AUTH/$u.pass" \ + >>"$LOG/auth-cli.out" 2>&1 || abort "lw auth add-user $u failed" auth-cli + done +} diff --git a/test/mtls-chain-test.sh b/test/mtls-chain-test.sh index 52106b7..5ad8745 100755 --- a/test/mtls-chain-test.sh +++ b/test/mtls-chain-test.sh @@ -1,123 +1,43 @@ #!/usr/bin/env bash -# logwisp mTLS authentication test -# -# Scenario 1 — chained instances, client authenticates with mTLS: -# edge-01 cert --> tcp_chain sink --> :15811 tcp_chain src --> file sink -# edge-01 cert --> http_chain sink --> :15812 http_chain src --> file sink -# edge-99 cert --> tcp_chain sink --> :15811 rejected by the allow list -# -# Scenario 2 — a viewer client reads a streaming sink over mTLS: -# viewer-01 cert --> :15813 tcp sink (openssl s_client) -# viewer-01 cert --> :15814 http sink (curl, /stream and /status) -# rogue cert --> both, rejected by the allow list -# -# Also covers: node binding (a peer holding the edge-01 certificate cannot -# label its entries anything else), dialer-side server identity pinning, and -# a peer presenting no certificate at all. -# -# Usage: -# ./mtls-chain-test.sh manual mode: relay + edges up, guide printed -# ./mtls-chain-test.sh --auto automated checks and teardown -# ./mtls-chain-test.sh --keep (with --auto) skip teardown on success -# +# logwisp mTLS test: edges authenticate to a relay with client certificates, +# viewers to its tcp and http sinks; covers the allow lists, node binding (an +# edge-01 certificate labels its entries edge-01), dialer-side server pinning +# and a peer without a certificate. # Requires: bash 5+, coreutils (timeout), openssl, curl. Linux dev host only. set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" -RUN="$SCRIPT_DIR/run-mtls" -CONF="$RUN/conf" -LOG="$RUN/log" -PKI="$RUN/pki" -OUT="$RUN/out" - +RUN=$E2E_DIR/run-mtls +PKI=$RUN/pki +OUT=$RUN/out PORT_TCP_CHAIN=15811 PORT_HTTP_CHAIN=15812 PORT_TCP_SINK=15813 PORT_HTTP_SINK=15814 +PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK" +e2e_init "$@" -AUTO=0; KEEP=0 -for a in "$@"; do case "$a" in - --auto) AUTO=1 ;; - --keep) KEEP=1 ;; - *) echo "unknown arg: $a" >&2; exit 1 ;; -esac; done - -PIDS=() -cleanup() { - local rc=$? - trap - EXIT INT TERM - if (( ${#PIDS[@]} )); then - echo "--- teardown: stopping ${#PIDS[@]} daemon(s)" - kill -TERM "${PIDS[@]}" 2>/dev/null - local deadline=$(( SECONDS + 10 )) - for pid in "${PIDS[@]}"; do - while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done - kill -KILL "$pid" 2>/dev/null - done - fi - exit "$rc" -} -trap cleanup EXIT INT TERM - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; } - -wait_port() { # port timeout_s - local i; for (( i=0; i < $2 * 10; i++ )); do - port_open "$1" && return 0 - sleep 0.1 - done - return 1 -} - -start_daemon() { # name conf - "$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 & - PIDS+=($!) - echo "started $1 (pid $!)" -} - -# --- Preflight --- -[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; } -command -v openssl >/dev/null || { echo "openssl not found" >&2; exit 1; } -command -v curl >/dev/null || { echo "curl not found" >&2; exit 1; } -for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do - port_open "$p" && { echo "port $p already in use" >&2; exit 1; } -done +section "Setup" +need openssl curl +ports_free $PORTS rm -rf "$RUN" mkdir -p "$CONF" "$LOG" "$PKI" "$OUT" -# --- PKI --- # One CA for every peer: the point of the test is that CA membership alone is # no longer sufficient, so the identities must all be issued by the same CA. -gen_key() { openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1" 2>/dev/null; } +pki_ca +pki_leaf relay relay.internal serverAuth "IP:127.0.0.1,DNS:relay.internal" +pki_leaf edge-01 edge-01 clientAuth +pki_leaf edge-02 edge-02 clientAuth +pki_leaf edge-99 edge-99 clientAuth +pki_leaf viewer-01 viewer-01 clientAuth +pki_leaf rogue rogue-viewer clientAuth +info "test PKI in $(short "$PKI")/: ca, relay, edge-01, edge-02, edge-99, viewer-01, rogue" -gen_leaf() { # name CN eku [SAN] - local name=$1 cn=$2 eku=$3 san=${4:-} - gen_key "$PKI/$name.key" - openssl req -new -key "$PKI/$name.key" -out "$PKI/$name.csr" -subj "/CN=$cn" 2>/dev/null - local ext="extendedKeyUsage=$eku" - [[ -n $san ]] && ext+=$'\n'"subjectAltName=$san" - printf '%s\n' "$ext" > "$PKI/$name.ext" - openssl x509 -req -in "$PKI/$name.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" \ - -CAcreateserial -out "$PKI/$name.crt" -days 2 -extfile "$PKI/$name.ext" 2>/dev/null -} - -echo "--- generating test PKI in $PKI" -gen_key "$PKI/ca.key" -openssl req -x509 -new -key "$PKI/ca.key" -days 2 -out "$PKI/ca.crt" \ - -subj "/CN=LogWisp Test CA" 2>/dev/null -gen_leaf relay relay.internal serverAuth "IP:127.0.0.1,DNS:relay.internal" -gen_leaf edge-01 edge-01 clientAuth -gen_leaf edge-99 edge-99 clientAuth -gen_leaf viewer-01 viewer-01 clientAuth -gen_leaf rogue rogue-viewer clientAuth -[[ -s "$PKI/rogue.crt" ]] || { echo "PKI generation failed" >&2; exit 1; } - -# --- Config generation --- # Relay: both ingest ports authorize edge-01 only and bind the node label to # the certificate identity; both streaming sinks authorize viewer-01 only. -cat > "$CONF/relay.toml" <"$CONF/relay.toml" < "$CONF/edge_tcp.toml" < "$CONF/edge_http.toml" < "$CONF/edge_rogue.toml" < "$CONF/edge_pinfail.toml" <"$CONF/$1.toml" <>"$CONF/$1.toml"; } + +# edge_tcp holds the edge-01 certificate but declares node "edge-tcp": +# node_binding = "force" must relabel its entries to "edge-01". +edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN edge-tcp edge-01 info +pin edge_tcp relay.internal +edge_conf edge_http http_chain $PORT_HTTP_CHAIN edge-http edge-01 info "flush_interval_ms = 500" +pin edge_http relay.internal +# edge_rogue holds a CA-issued certificate the relay does not authorize, and +# claims to be edge-01 on top of it. +edge_conf edge_rogue tcp_chain $PORT_TCP_CHAIN edge-01 edge-99 info +# edge_pinfail pins a server identity the relay does not have: the dialer must +# refuse the handshake even though the certificate chains to the trusted CA. +# The relay admits its edge-02, so only the pin can keep its entries out. +edge_conf edge_pinfail tcp_chain $PORT_TCP_CHAIN edge-pinfail edge-02 debug "backoff_max_ms = 1000" +pin edge_pinfail some-other-relay.internal +info "configurations in $(short "$CONF")/" + +section "Startup" +start_daemon relay relay.toml +for p in $PORTS; do + wait_port "$p" || abort "relay port $p is not listening" relay +done +start_daemon edge_tcp edge_tcp.toml +start_daemon edge_http edge_http.toml +start_daemon edge_rogue edge_rogue.toml +start_daemon edge_pinfail edge_pinfail.toml +daemons_up +write_env LW="$BIN" PKI="$PKI" CA="$PKI/ca.crt" CERT="$PKI/viewer-01.crt" KEY="$PKI/viewer-01.key" + +guide "logwisp mTLS test" < . $(short "$RUN")/env +Read the tcp sink, then the http sink, as viewer-01: +> openssl s_client -quiet -connect 127.0.0.1:$PORT_TCP_SINK -CAfile \$CA -cert \$CERT -key \$KEY +> curl -N --cacert \$CA --cert \$CERT --key \$KEY https://127.0.0.1:$PORT_HTTP_SINK/stream +With \$PKI/rogue.crt and rogue.key instead, the policy refuses either. +Ingested entries (node label forced to the certificate CN): $(short "$OUT")/ +Logs: $(short "$LOG")/ +EOF +manual_hold + +info "settling 4 s (connect, first http_chain flush)" +sleep 4 # Viewer helpers tcp_view() { # cert_basename secs @@ -414,7 +230,7 @@ tcp_view() { # cert_basename secs -cert "$PKI/$1.crt" -key "$PKI/$1.key" /dev/null || true } -http_get() { # path cert_basename|"" -> "|" +http_get() { # path cert_basename|"" -> "" local path=$1 name=${2:-} local args=(-s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt") @@ -423,91 +239,83 @@ http_get() { # path cert_basename|"" -> "|" } relay_log="$LOG/relay.out" -ingested() { cat "$OUT"/${1}* 2>/dev/null; } -echo "=== Scenario 1: chained instances over mTLS ===" +section "Scenario 1: chained instances over mTLS" # 1. An authorized edge delivers entries into the relay's file sink tcp_file="$(ingested tcp_chain)" -n=$(grep -c 'edge-01/' <<< "$tcp_file") -check "tcp_chain: authorized edge-01 entries reached the file sink ($n lines)" $(( n >= 1 )) +n=$(grep -c 'edge-01/' <<<"$tcp_file") +check "tcp_chain: authorized edge-01 entries reached the file sink ($n lines)" $((n >= 1)) http_file="$(ingested http_chain)" -n=$(grep -c 'edge-01/' <<< "$http_file") -check "http_chain: authorized edge-01 entries reached the file sink ($n lines)" $(( n >= 1 )) +n=$(grep -c 'edge-01/' <<<"$http_file") +check "http_chain: authorized edge-01 entries reached the file sink ($n lines)" $((n >= 1)) # 2. node_binding = "force" overrode the label the sender configured -n=$(grep -c 'edge-tcp/' <<< "$tcp_file") -check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $(( n == 0 )) -n=$(grep -c 'edge-http/' <<< "$http_file") -check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $(( n == 0 )) +n=$(grep -c 'edge-tcp/' <<<"$tcp_file") +check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $((n == 0)) +n=$(grep -c 'edge-http/' <<<"$http_file") +check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $((n == 0)) # 3. An identity outside the allow list is refused, even claiming to be edge-01 n=$(grep -c 'Connection rejected by auth policy' "$relay_log") -check "allow list: unauthorized edge-99 connection rejected ($n rejections)" $(( n >= 1 )) -n=$(grep -c 'edge-99' <<< "$tcp_file") -check "allow list: no edge-99 entry was ingested" $(( n == 0 )) +check "allow list: unauthorized edge-99 connection rejected ($n rejections)" $((n >= 1)) +n=$(grep -c 'edge-99' <<<"$tcp_file") +check "allow list: no edge-99 entry was ingested" $((n == 0)) -# 4. A peer with no certificate cannot complete the handshake +# 4. A peer with no certificate cannot complete the handshake; edge_pinfail +# fails handshakes too, with "bad certificate" timeout 5 openssl s_client -connect 127.0.0.1:$PORT_TCP_CHAIN \ -CAfile "$PKI/ca.crt" /dev/null 2>&1 sleep 0.5 -n=$(grep -c 'TLS handshake failed' "$relay_log") -check "client_auth: a peer with no certificate was refused ($n handshake errors)" $(( n >= 1 )) +n=$(grep -c "TLS handshake failed.*didn't provide a certificate" "$relay_log") +check "client_auth: a peer with no certificate was refused ($n handshake errors)" $((n >= 1)) # 5. Dialer-side pinning: the relay's identity is not the one edge_pinfail pins n=$(grep -c 'is not allowed' "$LOG/edge_pinfail.out") -check "server pinning: dialer refused a CA-valid server it does not pin ($n refusals)" $(( n >= 1 )) -n=$(grep -c 'edge-pinfail' <<< "$tcp_file") -check "server pinning: pin-failing edge delivered nothing" $(( n == 0 )) +check "server pinning: dialer refused a CA-valid server it does not pin ($n refusals)" $((n >= 1)) +n=$(grep -c 'edge-02/' <<<"$tcp_file") +check "server pinning: pin-failing edge-02 delivered nothing ($n lines)" $((n == 0)) -echo "=== Scenario 2: viewer clients on mTLS-gated sinks ===" +section "Scenario 2: viewer clients on mTLS-gated sinks" # 6. TCP sink: authorized viewer streams, rogue gets nothing out="$(tcp_view viewer-01 4)" -n=$(grep -c 'edge-01/' <<< "$out") -check "tcp sink: viewer-01 streamed entries ($n lines)" $(( n >= 1 )) +n=$(grep -c 'edge-01/' <<<"$out") +check "tcp sink: viewer-01 streamed entries ($n lines)" $((n >= 1)) out="$(tcp_view rogue 4)" -n=$(grep -c '"message"' <<< "$out") -check "tcp sink: rogue viewer received no entries" $(( n == 0 )) +n=$(grep -c 'edge-01/' <<<"$out") +check "tcp sink: rogue viewer received no entries ($n lines)" $((n == 0)) # 7. HTTP sink: stream and status both gated code="$(http_get /status viewer-01)" -check "http sink: /status served to viewer-01 (HTTP $code)" $([[ $code == 200 ]] && echo 1 || echo 0) +check "http sink: /status served to viewer-01 (HTTP $code)" $(is "$code" 200) code="$(http_get /status rogue)" -check "http sink: /status refused to rogue viewer (HTTP $code)" $([[ $code == 403 ]] && echo 1 || echo 0) +check "http sink: /status refused to rogue viewer (HTTP $code)" $(is "$code" 403) code="$(http_get /stream rogue)" -check "http sink: /stream refused to rogue viewer (HTTP $code)" $([[ $code == 403 ]] && echo 1 || echo 0) +check "http sink: /stream refused to rogue viewer (HTTP $code)" $(is "$code" 403) # curl reports 000 when the handshake itself fails, which is what a client # with no certificate must hit code="$(http_get /status)" -check "http sink: client with no certificate failed the handshake (curl $code)" \ - $([[ $code == 000 ]] && echo 1 || echo 0) +check "http sink: client with no certificate failed the handshake (curl $code)" $(is "$code" 000) sse="$(timeout 4 curl -sN --noproxy '*' --cacert "$PKI/ca.crt" \ --cert "$PKI/viewer-01.crt" --key "$PKI/viewer-01.key" \ "https://127.0.0.1:$PORT_HTTP_SINK/stream" 2>/dev/null || true)" -n=$(grep -c '^data:.*edge-01/' <<< "$sse") -check "http sink: viewer-01 received SSE events ($n events)" $(( n >= 1 )) +n=$(grep -c '^data:.*edge-01/' <<<"$sse") +check "http sink: viewer-01 received SSE events ($n events)" $((n >= 1)) # 8. The status endpoint reports the policy and its rejection count status="$(curl -s --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt" \ --cert "$PKI/viewer-01.crt" --key "$PKI/viewer-01.key" \ "https://127.0.0.1:$PORT_HTTP_SINK/status" 2>/dev/null || true)" -n=$(grep -c 'mtls' <<< "$status") -check "http sink: status endpoint reports the auth policy" $(( n >= 1 )) -rej=$(grep -o '"auth_rejected"[ :]*[0-9]*' <<< "$status" | grep -o '[0-9]*$' || echo 0) -check "http sink: status endpoint counts auth rejections (auth_rejected=$rej)" $(( rej >= 1 )) +n=$(grep -c 'mtls' <<<"$status") +check "http sink: status endpoint reports the auth policy" $((n >= 1)) +rej=$(grep -o '"auth_rejected"[ :]*[0-9]*' <<<"$status" | grep -o '[0-9]*$' || echo 0) +check "http sink: status endpoint counts auth rejections (auth_rejected=$rej)" $((rej >= 1)) -echo "================================================================" -if (( fail == 0 )); then - echo "RESULT: ALL PASS" - (( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); } -else - echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log" -fi -exit "$fail" +summary diff --git a/test/passthrough-test.sh b/test/passthrough-test.sh index cceb421..94f8f1a 100755 --- a/test/passthrough-test.sh +++ b/test/passthrough-test.sh @@ -1,27 +1,21 @@ #!/usr/bin/env bash -# logwisp file source pass-through test -# -# file src (raw, from=start) --> raw format --> file sink byte-exact relay -# file src (defaults) --> raw format --> file sink no key dropped -# -# The fixture is a record whose envelope is wider than time/level/msg/fields, -# which is what the narrow JSON branch used to reduce to an empty message. -# -# Usage: ./passthrough-test.sh +# logwisp file pass-through test: a file source relays a record whose envelope +# is wider than time/level/msg/fields byte for byte (raw = true) and, parsed, +# with no key dropped. One-shot: both modes run the checks; no ports. # Requires: bash 5+, coreutils (timeout). Linux dev host only. set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" -RUN="$SCRIPT_DIR/run/passthrough" - -[[ -x $BIN ]] || { echo "no binary at $BIN; run make build" >&2; exit 1; } +RUN=$E2E_DIR/run/passthrough +e2e_init "$@" +section "Setup" +need rm -rf "$RUN" -mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed" +mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed" "$CONF" "$LOG" -cat > "$RUN/in/wide.jsonl" <<'EOF' +cat >"$RUN/in/wide.jsonl" <<'EOF' {"time":"2026-09-08T21:35:44.178372768-04:00","level":"INFO","sub":"app","run":0,"tick":0,"frame":0,"fields":{"msg":"init begin","mode":"play"}} {"time":"2026-09-08T21:35:44.178581366-04:00","level":"PROC","run":0,"tick":0,"frame":0,"fields":{"seq":1,"seed":1788917744178374836}} plain text line, not JSON at all @@ -56,37 +50,29 @@ name = "relay" flush_interval_ms = 100 EOF } +conf "$RUN/out-raw" true >"$CONF/raw.toml" +conf "$RUN/out-parsed" false >"$CONF/parsed.toml" +info "fixture $(short "$RUN/in/wide.jsonl"), configurations in $(short "$CONF")/" -conf "$RUN/out-raw" true > "$RUN/raw.toml" -conf "$RUN/out-parsed" false > "$RUN/parsed.toml" - +section "Relay" for c in raw parsed; do - timeout 5 "$BIN" -c "$RUN/$c.toml" > "$RUN/$c.out" 2>&1 + info "lw -c $(short "$CONF/$c.toml"), 5 s" + timeout 5 "$BIN" -c "$CONF/$c.toml" >"$LOG/$c.out" 2>&1 done -fail=0 -check() { # label condition_result - if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi -} - +section "Checks" # 1. raw = true relays the file byte for byte -if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" > /dev/null; then +if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" >/dev/null; then check "raw = true: output identical to input" 1 else check "raw = true: output identical to input" 0 - diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6 + diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6 | sed 's/^/ /' fi # 2. the default parse keeps every key, JSON branch refused on the wide envelope n=$(grep -c '"sub":"app"' "$RUN/out-parsed/relay.log") -check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $(( n == 1 )) +check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $((n == 1)) n=$(grep -c '1788917744178374836' "$RUN/out-parsed/relay.log") -check "defaults: large integers are not re-encoded through float64 ($n)" $(( n == 1 )) +check "defaults: large integers are not re-encoded through float64 ($n)" $((n == 1)) -echo "================================================================" -if (( fail == 0 )); then - echo "RESULT: ALL PASS" -else - echo "RESULT: FAILURES — inspect $RUN/" -fi -exit "$fail" +summary diff --git a/test/scram-chain-test.sh b/test/scram-chain-test.sh index 923f1da..60de868 100755 --- a/test/scram-chain-test.sh +++ b/test/scram-chain-test.sh @@ -1,122 +1,46 @@ #!/usr/bin/env bash -# logwisp SCRAM authentication test: chain edges and sink viewers log in with -# passwords managed by `logwisp auth`; the guide printed below maps the ports. -# Usage: ./scram-chain-test.sh [--auto [--keep]] manual mode keeps the daemons -# up; --auto runs the checks and tears down, --keep skips that on success. +# logwisp SCRAM test: chain edges and sink viewers log in with passwords +# managed by `lw auth`: wrong and missing credentials, a certificate bound to +# its user, a stream past the exchange deadline, revocation and throttling. # Requires: bash 5+, coreutils (timeout), openssl, curl. Linux dev host only. set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" -RUN="$SCRIPT_DIR/run-scram" -CONF="$RUN/conf" -LOG="$RUN/log" -PKI="$RUN/pki" -AUTH="$RUN/auth" -OUT="$RUN/out" -USERS="$AUTH/users.toml" - +RUN=$E2E_DIR/run-scram +PKI=$RUN/pki +AUTH=$RUN/auth +OUT=$RUN/out +USERS=$AUTH/users.toml PORT_TCP_CHAIN=15821 PORT_HTTP_CHAIN=15822 PORT_TCP_SINK=15823 PORT_HTTP_SINK=15824 PORT_BOUND=15825 +PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_BOUND" +e2e_init "$@" -AUTO=0; KEEP=0 -for a in "$@"; do case "$a" in - --auto) AUTO=1 ;; - --keep) KEEP=1 ;; - *) echo "unknown arg: $a" >&2; exit 1 ;; -esac; done - -PIDS=() -cleanup() { - local rc=$? - trap - EXIT INT TERM - if (( ${#PIDS[@]} )); then - echo "--- teardown: stopping ${#PIDS[@]} process(es)" - kill -TERM "${PIDS[@]}" 2>/dev/null - local deadline=$(( SECONDS + 10 )) - for pid in "${PIDS[@]}"; do - while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done - kill -KILL "$pid" 2>/dev/null - done - fi - exit "$rc" -} -trap cleanup EXIT INT TERM - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; } - -wait_port() { # port timeout_s - local i; for (( i=0; i < $2 * 10; i++ )); do - port_open "$1" && return 0 - sleep 0.1 - done - return 1 -} - -wait_until() { # timeout_s command... - local deadline=$(( SECONDS + $1 )); shift - until "$@"; do - (( SECONDS >= deadline )) && return 1 - sleep 0.2 - done -} - -start_daemon() { # name conf - "$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 & - PIDS+=($!) - echo "started $1 (pid $!)" -} - -# --- Preflight --- -[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; } -command -v openssl >/dev/null || { echo "openssl not found" >&2; exit 1; } -command -v curl >/dev/null || { echo "curl not found" >&2; exit 1; } -for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_BOUND; do - port_open "$p" && { echo "port $p already in use" >&2; exit 1; } -done +section "Setup" +need openssl curl +ports_free $PORTS rm -rf "$RUN" mkdir -p "$CONF" "$LOG" "$PKI" "$AUTH" "$OUT" -# --- PKI --- -gen_key() { openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1" 2>/dev/null; } +pki_ca +pki_leaf relay relay.internal serverAuth "IP:127.0.0.1" +pki_leaf edge-01 edge-01 clientAuth +pki_leaf edge-02 edge-02 clientAuth +info "test PKI in $(short "$PKI")/: ca, relay, edge-01, edge-02" -gen_leaf() { # name CN eku [SAN] - local name=$1 cn=$2 eku=$3 san=${4:-} - gen_key "$PKI/$name.key" - openssl req -new -key "$PKI/$name.key" -out "$PKI/$name.csr" -subj "/CN=$cn" 2>/dev/null - local ext="extendedKeyUsage=$eku" - [[ -n $san ]] && ext+=$'\n'"subjectAltName=$san" - printf '%s\n' "$ext" > "$PKI/$name.ext" - openssl x509 -req -in "$PKI/$name.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" \ - -CAcreateserial -out "$PKI/$name.crt" -days 2 -extfile "$PKI/$name.ext" 2>/dev/null -} +# Credentials through the CLI only, and a password no user has, for the +# wrong-password and unknown-user edges +add_users "$USERS" edge-01 edge-02 viewer-01 +(umask 077; openssl rand -base64 24 >"$AUTH/wrong.pass") +info "users edge-01, edge-02, viewer-01 in $(short "$USERS")" -echo "--- generating test PKI in $PKI" -gen_key "$PKI/ca.key" -openssl req -x509 -new -key "$PKI/ca.key" -days 2 -out "$PKI/ca.crt" \ - -subj "/CN=LogWisp Test CA" 2>/dev/null -gen_leaf relay relay.internal serverAuth "IP:127.0.0.1" -gen_leaf edge-01 edge-01 clientAuth -gen_leaf edge-02 edge-02 clientAuth -[[ -s "$PKI/edge-02.crt" ]] || { echo "PKI generation failed" >&2; exit 1; } - -# --- Credentials, through the CLI only --- -echo "--- adding users in $USERS" -for u in edge-01 edge-02 viewer-01; do - "$BIN" auth add-user -credentials "$USERS" -user "$u" -password-file "$AUTH/$u.pass" \ - 2>>"$LOG/auth-cli.out" || { echo "add-user $u failed (see $LOG/auth-cli.out)" >&2; exit 1; } -done -# A password no user has, for the wrong-password and unknown-user edges -(umask 077; openssl rand -base64 24 > "$AUTH/wrong.pass") - -# --- Config generation --- # One credentials file serves every listener, so revoking viewer-01 leaves the # file with users (the CLI refuses to remove the last one). -cat > "$CONF/relay.toml" <"$CONF/relay.toml" < "$CONF/$name.toml" + } >"$CONF/$name.toml" } # edge_tcp declares node "edge-tcp": node_binding = "force" must relabel it edge-01. # Under force a refused edge that got through would carry its username, so # edge-02 and edge-99 entries in the edge-01 sinks would expose it. -edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN edge-tcp edge-01 "$AUTH/edge-01.pass" -edge_conf edge_http http_chain $PORT_HTTP_CHAIN edge-http edge-01 "$AUTH/edge-01.pass" -edge_conf edge_wrongpw tcp_chain $PORT_TCP_CHAIN edge-wrongpw edge-02 "$AUTH/wrong.pass" -edge_conf edge_unknown tcp_chain $PORT_TCP_CHAIN edge-unknown edge-99 "$AUTH/wrong.pass" -edge_conf edge_noauth tcp_chain $PORT_TCP_CHAIN edge-noauth "" "" -edge_conf edge_bound tcp_chain $PORT_BOUND edge-bound edge-01 "$AUTH/edge-01.pass" edge-01 -edge_conf edge_cross tcp_chain $PORT_BOUND edge-cross edge-02 "$AUTH/edge-02.pass" edge-01 +edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN edge-tcp edge-01 "$AUTH/edge-01.pass" +edge_conf edge_http http_chain $PORT_HTTP_CHAIN edge-http edge-01 "$AUTH/edge-01.pass" +edge_conf edge_wrongpw tcp_chain $PORT_TCP_CHAIN edge-wrongpw edge-02 "$AUTH/wrong.pass" +edge_conf edge_unknown tcp_chain $PORT_TCP_CHAIN edge-unknown edge-99 "$AUTH/wrong.pass" +edge_conf edge_noauth tcp_chain $PORT_TCP_CHAIN edge-noauth "" "" +edge_conf edge_bound tcp_chain $PORT_BOUND edge-bound edge-01 "$AUTH/edge-01.pass" edge-01 +edge_conf edge_cross tcp_chain $PORT_BOUND edge-cross edge-02 "$AUTH/edge-02.pass" edge-01 +info "configurations in $(short "$CONF")/" -# --- Guide --- -cat < . $(short "$RUN")/env +Read the tcp sink as viewer-01: +> \$LW auth stream -addr 127.0.0.1:$PORT_TCP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA +Read the http sink (bearer keeps the token off argv): +> token=\$(\$LW auth token -url https://127.0.0.1:$PORT_HTTP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA) +> curl -N --cacert \$CA -H @<(bearer) https://127.0.0.1:$PORT_HTTP_SINK/stream +Revoke viewer-01: +> \$LW auth remove-user -credentials \$USERS -user viewer-01 && kill -HUP \$RELAY +Ingested entries (node label forced to the username): $(short "$OUT")/ +Passwords: $(short "$AUTH")/ Logs: $(short "$LOG")/ +EOF +manual_hold relay_log="$LOG/relay.out" -ingested() { cat "$OUT"/${1}* 2>/dev/null; } -has_entry() { ingested "$1" | grep -q -- "$2"; } # file_prefix pattern - -logged() { # log_name pattern [min_count] - local n; n=$(grep -c -- "$2" "$LOG/$1.out" 2>/dev/null) - (( ${n:-0} >= ${3:-1} )) -} token_for() { # user -> token on stdout; the exit status is the CLI's "$BIN" auth token -url "https://127.0.0.1:$PORT_HTTP_SINK" -user "$1" \ @@ -360,72 +263,73 @@ http_get() { # path [token] -> http_code; the header goes through a pipe, never fi } -echo "=== Scenario 1: chained instances over SCRAM ===" +section "Scenario 1: chained instances over SCRAM" # 1. An edge with the right password delivers on both transports wait_until 20 has_entry tcp_chain 'edge-01/' tcp_file="$(ingested tcp_chain)" -n=$(grep -c 'edge-01/' <<< "$tcp_file") -check "tcp_chain: edge-01 entries reached the file sink ($n lines)" $(( n >= 1 )) +n=$(grep -c 'edge-01/' <<<"$tcp_file") +check "tcp_chain: edge-01 entries reached the file sink ($n lines)" $((n >= 1)) wait_until 20 has_entry http_chain 'edge-01/' http_file="$(ingested http_chain)" -n=$(grep -c 'edge-01/' <<< "$http_file") -check "http_chain: edge-01 entries reached the file sink ($n lines)" $(( n >= 1 )) +n=$(grep -c 'edge-01/' <<<"$http_file") +check "http_chain: edge-01 entries reached the file sink ($n lines)" $((n >= 1)) # 2. node_binding = "force" replaced the label the sender configured -n=$(grep -c 'edge-tcp/' <<< "$tcp_file") -check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $(( n == 0 )) -n=$(grep -c 'edge-http/' <<< "$http_file") -check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $(( n == 0 )) +n=$(grep -c 'edge-tcp/' <<<"$tcp_file") +check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $((n == 0)) +n=$(grep -c 'edge-http/' <<<"$http_file") +check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $((n == 0)) # 3. Wrong password, unknown user and no credentials are all refused wait_until 20 logged edge_wrongpw 'Chain connect refused' -n=$(grep -c 'Chain connect refused' "$LOG/edge_wrongpw.out") -check "wrong password: edge-02 with a bad password was refused ($n refusals)" $(( n >= 1 )) +n=$(count edge_wrongpw 'Chain connect refused') +check "wrong password: edge-02 with a bad password was refused ($n refusals)" $((n >= 1)) wait_until 20 logged edge_unknown 'Chain connect refused' -n=$(grep -c 'Chain connect refused' "$LOG/edge_unknown.out") -check "unknown user: edge-99 was refused ($n refusals)" $(( n >= 1 )) +n=$(count edge_unknown 'Chain connect refused') +check "unknown user: edge-99 was refused ($n refusals)" $((n >= 1)) wait_until 20 logged relay 'peer offered no credentials' -n=$(grep -c 'peer offered no credentials' "$relay_log") -check "no auth: relay refused an edge that offered no credentials ($n refusals)" $(( n >= 1 )) +n=$(count relay 'peer offered no credentials') +check "no auth: relay refused an edge that offered no credentials ($n refusals)" $((n >= 1)) wait_until 20 logged edge_noauth 'Chain link refused' -n=$(grep -c 'Chain link refused' "$LOG/edge_noauth.out") -check "no auth: the edge read the refusal instead of writing into a closed link ($n)" $(( n >= 1 )) +n=$(count edge_noauth 'Chain link refused') +check "no auth: the edge read the refusal instead of writing into a closed link ($n)" $((n >= 1)) failed_login='Connection rejected by auth policy.*instance_id in_tcp .*invalid credentials' wait_until 20 logged relay "$failed_login" 2 -n=$(grep -c "$failed_login" "$relay_log") -check "relay logged the failed logins on :$PORT_TCP_CHAIN ($n rejections)" $(( n >= 2 )) +n=$(count relay "$failed_login") +check "relay logged the failed logins on :$PORT_TCP_CHAIN ($n rejections)" $((n >= 2)) tcp_file="$(ingested tcp_chain)" -n=$(grep -vc 'edge-01/' <<< "$tcp_file") -check "refused edges: nothing but edge-01 entries was ingested ($n foreign lines)" $(( n == 0 )) +n=$(grep -vc 'edge-01/' <<<"$tcp_file") +check "refused edges: nothing but edge-01 entries was ingested ($n foreign lines)" $((n == 0)) # 4. Certificate bound to the user: edge-01's certificate admits only edge-01 wait_until 20 has_entry bound 'edge-01/' n=$(ingested bound | grep -c 'edge-01/') -check "certificate binding: edge-01 certificate + user edge-01 delivered ($n lines)" $(( n >= 1 )) +check "certificate binding: edge-01 certificate + user edge-01 delivered ($n lines)" $((n >= 1)) wait_until 20 logged edge_cross 'Chain connect refused' -n=$(grep -c 'Chain connect refused' "$LOG/edge_cross.out") -check "certificate binding: edge-01 certificate + user edge-02 was refused ($n refusals)" $(( n >= 1 )) +n=$(count edge_cross 'Chain connect refused') +check "certificate binding: edge-01 certificate + user edge-02 was refused ($n refusals)" $((n >= 1)) wait_until 20 logged relay 'does not match user \\"edge-02\\"' -n=$(grep -c 'does not match user \\"edge-02\\"' "$relay_log") -check "certificate binding: relay logged the certificate/user mismatch ($n)" $(( n >= 1 )) +n=$(count relay 'does not match user \\"edge-02\\"') +check "certificate binding: relay logged the certificate/user mismatch ($n)" $((n >= 1)) n=$(ingested bound | grep -vc 'edge-01/') -check "certificate binding: the cross-bound edge delivered nothing ($n foreign lines)" $(( n == 0 )) +check "certificate binding: the cross-bound edge delivered nothing ($n foreign lines)" $((n == 0)) -echo "=== Scenario 2: viewers on SCRAM-gated sinks ===" +section "Scenario 2: viewers on SCRAM-gated sinks" # 5. TCP sink: a logged-in viewer keeps streaming past the 10 s exchange # deadline; a TLS client that sends no hello gets nothing and is dropped at it +info "streaming as viewer-01 beside a silent TLS client, 13 s" "$BIN" auth stream -addr "127.0.0.1:$PORT_TCP_SINK" -user viewer-01 \ -password-file "$AUTH/viewer-01.pass" -ca-file "$PKI/ca.crt" \ - > "$RUN/stream.out" 2>"$LOG/stream.err" & + >"$RUN/stream.out" 2>"$LOG/stream.err" & stream_pid=$! PIDS+=($stream_pid) timeout 12 openssl s_client -quiet -connect "127.0.0.1:$PORT_TCP_SINK" \ @@ -437,47 +341,47 @@ sleep 2 n2=$(grep -c 'edge-01/' "$RUN/stream.out") kill -TERM "$stream_pid" 2>/dev/null wait "$stream_pid" "$raw_pid" 2>/dev/null -check "tcp sink: viewer-01 streamed entries through logwisp auth stream ($n1 lines at 11 s)" $(( n1 >= 1 )) -check "tcp sink: stream still live after the exchange deadline ($n1 -> $n2 lines, 11 s -> 13 s)" $(( n2 > n1 )) -n=$(wc -c < "$RUN/raw.out") -check "tcp sink: a TLS client with no hello received nothing ($n bytes)" $(( n == 0 )) +check "tcp sink: viewer-01 streamed entries through lw auth stream ($n1 lines at 11 s)" $((n1 >= 1)) +check "tcp sink: stream still live after the exchange deadline ($n1 -> $n2 lines, 11 s -> 13 s)" $((n2 > n1)) +n=$(wc -c <"$RUN/raw.out") +check "tcp sink: a TLS client with no hello received nothing ($n bytes)" $((n == 0)) n=$(grep 'component tcp_sink' "$relay_log" | grep -c 'read hello') -check "tcp sink: relay dropped the silent client at the exchange deadline ($n)" $(( n >= 1 )) +check "tcp sink: relay dropped the silent client at the exchange deadline ($n)" $((n >= 1)) -# 6. HTTP sink: a bearer token from logwisp auth token, carried by curl +# 6. HTTP sink: a bearer token from lw auth token, carried by curl token="$(token_for viewer-01)" -check "http sink: logwisp auth token issued a token for viewer-01" $([[ -n $token ]] && echo 1 || echo 0) +check "http sink: lw auth token issued a token for viewer-01" $(is_set "$token") code="$(http_get /status "$token")" -check "http sink: /status served with the token (HTTP $code)" $([[ $code == 200 ]] && echo 1 || echo 0) +check "http sink: /status served with the token (HTTP $code)" $(is "$code" 200) sse="$(timeout 4 curl -sN --noproxy '*' --cacert "$PKI/ca.crt" \ -H @<(printf 'Authorization: Bearer %s\n' "$token") \ "https://127.0.0.1:$PORT_HTTP_SINK/stream" 2>/dev/null || true)" -n=$(grep -c '^data:.*edge-01/' <<< "$sse") -check "http sink: /stream delivered SSE events with the token ($n events)" $(( n >= 1 )) +n=$(grep -c '^data:.*edge-01/' <<<"$sse") +check "http sink: /stream delivered SSE events with the token ($n events)" $((n >= 1)) code="$(http_get /status)" -check "http sink: /status refused without a token (HTTP $code)" $([[ $code == 401 ]] && echo 1 || echo 0) +check "http sink: /status refused without a token (HTTP $code)" $(is "$code" 401) code="$(http_get /stream not.a.token)" -check "http sink: /stream refused with a garbage token (HTTP $code)" $([[ $code == 401 ]] && echo 1 || echo 0) +check "http sink: /stream refused with a garbage token (HTTP $code)" $(is "$code" 401) # 7. Revocation: remove-user + SIGHUP kills the token and the login -reloads=$(grep -c 'Configuration hot reload completed successfully' "$relay_log") -reloaded() { (( $(grep -c 'Configuration hot reload completed successfully' "$relay_log") > reloads )); } +reloads=$(count relay 'Configuration hot reload completed successfully') +reloaded() { (($(count relay 'Configuration hot reload completed successfully') > reloads)); } "$BIN" auth remove-user -credentials "$USERS" -user viewer-01 2>>"$LOG/auth-cli.out" rc=$? -check "revocation: logwisp auth remove-user removed viewer-01 (exit $rc)" $(( rc == 0 )) +check "revocation: lw auth remove-user removed viewer-01 (exit $rc)" $((rc == 0)) kill -HUP "$RELAY_PID" -wait_until 20 reloaded && wait_port "$PORT_HTTP_SINK" 10 +wait_until 20 reloaded && wait_port "$PORT_HTTP_SINK" rc=$? -check "revocation: relay reloaded on SIGHUP" $(( rc == 0 )) +check "revocation: relay reloaded on SIGHUP" $((rc == 0)) code="$(http_get /status "$token")" -check "revocation: the old token is refused after the reload (HTTP $code)" $([[ $code == 401 ]] && echo 1 || echo 0) +check "revocation: the old token is refused after the reload (HTTP $code)" $(is "$code" 401) token_for viewer-01 >/dev/null rc=$? -check "revocation: logwisp auth token for viewer-01 now fails (exit $rc)" $(( rc == 1 )) +check "revocation: lw auth token for viewer-01 now fails (exit $rc)" $((rc == 1)) # 8. Throttling, last: it leaves this address throttled on the http sink. # At most 4 exchanges may be unfinished per address, so hellos that are @@ -492,13 +396,6 @@ for i in $(seq 1 12); do [[ $code == 429 ]] && break done check "throttling: /auth answered 429 after ${#codes[@]} rapid hellos (${codes[*]})" \ - $([[ $code == 429 ]] && (( ${#codes[@]} <= 6 )) && echo 1 || echo 0) + $([[ $code == 429 ]] && ((${#codes[@]} <= 6)) && echo 1 || echo 0) -echo "================================================================" -if (( fail == 0 )); then - echo "RESULT: ALL PASS" - (( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); } -else - echo "RESULT: FAILURES — inspect $LOG/*.out" -fi -exit "$fail" +summary diff --git a/test/scram-proxy-test.sh b/test/scram-proxy-test.sh index b7bdb64..f93d97d 100755 --- a/test/scram-proxy-test.sh +++ b/test/scram-proxy-test.sh @@ -1,95 +1,50 @@ #!/usr/bin/env bash -# logwisp browser login test: an http sink in proxy mode (auth.trusted_proxies) -# behind a TLS-terminating reverse proxy that mounts it under /logs/. Headless -# Chromium logs in through the shipped pages; the CLI logs in unbound. -# Usage: ./scram-proxy-test.sh [--auto [--keep]] manual mode keeps the daemons -# up; --auto runs the checks and tears down, --keep skips that on success. -# Requires: bash 5+, go, openssl, curl; node with playwright for the browser -# checks (skipped, and reported, without it). Linux dev host only. +# logwisp browser login test: an http sink in proxy mode behind a TLS-terminating +# reverse proxy that mounts it under /logs/; headless Chromium signs in through +# the shipped pages, with cookies and without (token mode); the CLI unbound. +# Requires: bash 5+, go, openssl, curl; node with playwright and both builds +# `playwright install chromium` fetches, or the browser checks skip. set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" -RUN="$SCRIPT_DIR/run-proxy" -CONF="$RUN/conf" -LOG="$RUN/log" -PKI="$RUN/pki" -AUTH="$RUN/auth" -PROXY_SRC="$RUN/proxy" -USERS="$AUTH/users.toml" - +RUN=$E2E_DIR/run-proxy +PKI=$RUN/pki +AUTH=$RUN/auth +PROXY_SRC=$RUN/proxy +USERS=$AUTH/users.toml PORT_PROXY=15831 # the site: TLS ends here, logwisp is under /logs/ PORT_SINK=15832 # logwisp http sink, plaintext, only for 127.0.0.2 PROXY_ADDR=127.0.0.2 SITE="https://127.0.0.1:$PORT_PROXY" +e2e_init "$@" -AUTO=0; KEEP=0 -for a in "$@"; do case "$a" in - --auto) AUTO=1 ;; - --keep) KEEP=1 ;; - *) echo "unknown arg: $a" >&2; exit 1 ;; -esac; done - -PIDS=() -cleanup() { - local rc=$? - trap - EXIT INT TERM - if (( ${#PIDS[@]} )); then - echo "--- teardown: stopping ${#PIDS[@]} process(es)" - kill -TERM "${PIDS[@]}" 2>/dev/null - local deadline=$(( SECONDS + 10 )) - for pid in "${PIDS[@]}"; do - while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done - kill -KILL "$pid" 2>/dev/null - done - fi - exit "$rc" -} -trap cleanup EXIT INT TERM - -port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; } - -wait_port() { # port timeout_s - local i; for (( i=0; i < $2 * 10; i++ )); do - port_open "$1" && return 0 - sleep 0.1 - done - return 1 -} - -# --- Preflight --- -[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; } -for tool in go openssl curl; do - command -v "$tool" >/dev/null || { echo "$tool not found" >&2; exit 1; } -done -for p in $PORT_PROXY $PORT_SINK; do - port_open "$p" && { echo "port $p already in use" >&2; exit 1; } -done -BROWSER=0 -command -v node >/dev/null && node -e 'require.resolve("playwright")' 2>/dev/null && BROWSER=1 +section "Setup" +need go openssl curl +ports_free $PORT_PROXY $PORT_SINK +# The launch each scenario makes: the module resolves without its browsers, and +# chromium.executablePath() names full Chromium, not the default headless shell +launches() { timeout 60 node -e 'require("playwright").chromium.launch(JSON.parse(process.argv[1])).then((b) => b.close())' "$1" >/dev/null 2>&1; } +BROWSER=0 FULL_CHROMIUM=0 +if command -v node >/dev/null; then + launches '{}' && BROWSER=1 + launches '{"channel": "chromium"}' && FULL_CHROMIUM=1 +fi rm -rf "$RUN" mkdir -p "$CONF" "$LOG" "$PKI" "$AUTH" "$PROXY_SRC" -# --- PKI: the site's certificate; logwisp holds none --- -echo "--- generating the site certificate in $PKI" -openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$PKI/ca.key" 2>/dev/null -openssl req -x509 -new -key "$PKI/ca.key" -days 2 -out "$PKI/ca.crt" -subj "/CN=LogWisp Test CA" 2>/dev/null -openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$PKI/site.key" 2>/dev/null -openssl req -new -key "$PKI/site.key" -out "$PKI/site.csr" -subj "/CN=site.internal" 2>/dev/null -printf 'extendedKeyUsage=serverAuth\nsubjectAltName=IP:127.0.0.1\n' > "$PKI/site.ext" -openssl x509 -req -in "$PKI/site.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" -CAcreateserial \ - -out "$PKI/site.crt" -days 2 -extfile "$PKI/site.ext" 2>/dev/null -[[ -s "$PKI/site.crt" ]] || { echo "PKI generation failed" >&2; exit 1; } +# The site's certificate; logwisp holds none +pki_ca +pki_leaf site site.internal serverAuth "IP:127.0.0.1" +info "site certificate in $(short "$PKI")/" -# --- Credentials, through the CLI only (the default 64 MiB profile) --- -"$BIN" auth add-user -credentials "$USERS" -user viewer-01 -password-file "$AUTH/viewer-01.pass" \ - 2>>"$LOG/auth-cli.out" || { echo "add-user failed (see $LOG/auth-cli.out)" >&2; exit 1; } +# The default 64 MiB Argon2 profile, as a deployment would have +add_users "$USERS" viewer-01 +info "user viewer-01 in $(short "$USERS")" -# --- The site's reverse proxy --- # /logs/ forwards to logwisp from $PROXY_ADDR; /plain/ forwards the same way # but says the site was reached over http, which logwisp must refuse. -cat > "$PROXY_SRC/main.go" <<'EOF' +cat >"$PROXY_SRC/main.go" <<'EOF' package main import ( @@ -129,10 +84,13 @@ func main() { log.Fatal(http.ListenAndServeTLS(listen, cert, key, mux)) } EOF -(cd "$PROXY_SRC" && go build -o proxy main.go) || { echo "proxy build failed" >&2; exit 1; } +if ! (cd "$PROXY_SRC" && go build -o proxy main.go) >"$LOG/proxy-build.out" 2>&1; then + tail -n 5 "$LOG/proxy-build.out" | sed 's/^/ /' + abort "the reverse proxy did not build" +fi +info "reverse proxy built in $(short "$PROXY_SRC")/" -# --- Config --- -cat > "$CONF/logwisp.toml" <"$CONF/logwisp.toml" < "$RUN/browser.cjs" <<'EOF' -// Logs in through the shipped pages, as a person would; prints key=value lines +cat >"$RUN/browser.cjs" <<'EOF' +// Signs in through the shipped pages, as a person would; prints key=value lines. +// cookie: a fresh browser. dropped: the session cookie never arrives. blocked: +// a profile that refuses every cookie; it also reloads logwisp (pid) mid-stream. const { chromium } = require("playwright"); -const [base, user, password] = process.argv.slice(2); -(async () => { - const out = { csp_violations: 0 }; - const browser = await chromium.launch(); - try { - const context = await browser.newContext({ ignoreHTTPSErrors: true }); - const page = await context.newPage(); - page.on("console", (m) => { if (/Content Security Policy/i.test(m.text())) out.csp_violations++; }); - const status = () => page.evaluate(() => document.getElementById("status").textContent); +const fs = require("fs"); +const [mode, base, user, passFile, profile, pid] = process.argv.slice(2); +const password = fs.readFileSync(passFile, "utf8").replace(/\n+$/, ""); +const out = { csp_violations: 0 }; +const lines = (page) => page.evaluate(() => document.querySelectorAll("#log > div").length); +const linesAtLeast = (page, n) => + page.waitForFunction((n) => document.querySelectorAll("#log > div").length >= n, n, { timeout: 15000 }); +const statusText = (page) => page.evaluate(() => document.getElementById("status").textContent); +const statusIs = (page, re) => + page.waitForFunction((re) => new RegExp(re).test(document.getElementById("status").textContent), re.source, { timeout: 30000 }); +const signIn = async (page, pass) => { + await page.fill('#login input[name="username"]', user); + await page.fill('#login input[name="password"]', pass); + await page.click('#login button[type="submit"]'); +}; +// The viewer's own form: the login page has a visible #login too, but no #log +const inlineLogin = (page) => page.waitForFunction(() => + document.getElementById("log") && !document.getElementById("login").hidden, null, { timeout: 30000 }); + +const scenarios = { + async cookie(context, page) { await page.goto(base + "auth/view"); - await page.waitForURL(/\/auth\/login\?next=view$/, { timeout: 10000 }); + await page.waitForURL(/\/auth\/login\?next=view%23signed-in$/, { timeout: 10000 }); out.redirected_to_login = 1; - await page.fill('input[name="username"]', user); - await page.fill('input[name="password"]', "not-the-password"); - await page.click('button[type="submit"]'); - await page.waitForFunction(() => /refused/.test(document.getElementById("status").textContent), null, { timeout: 30000 }); + await signIn(page, "not-the-password"); + await statusIs(page, /refused/); out.wrong_password_refused = 1; - await page.fill('input[name="password"]', password); const started = Date.now(); - await page.click('button[type="submit"]'); - await page.waitForURL(/\/auth\/view$/, { timeout: 30000 }).catch(async (e) => { throw new Error(`${e.message}: ${await status()}`); }); + await signIn(page, password); + await page.waitForURL(/\/auth\/view$/, { timeout: 30000 }).catch(async (e) => { throw new Error(`${e.message}: ${await statusText(page)}`); }); out.login_ms = Date.now() - started; - await page.waitForFunction(() => document.querySelectorAll("#log > div").length >= 3, null, { timeout: 15000 }); - out.events = await page.evaluate(() => document.querySelectorAll("#log > div").length); + await linesAtLeast(page, 3); + out.events = await lines(page); + out.inline_login = await page.evaluate(() => !document.getElementById("login").hidden); const c = (await context.cookies()).find((c) => c.name === "logwisp_session"); out.cookie = c ? `path=${c.path};httponly=${c.httpOnly};secure=${c.secure};samesite=${c.sameSite}` : "none"; @@ -200,54 +170,120 @@ const [base, user, password] = process.argv.slice(2); await page.waitForURL(/\/auth\/login$/, { timeout: 10000 }); out.cookie_after_logout = (await context.cookies()).some((c) => c.name === "logwisp_session") ? 1 : 0; out.status_after_logout = await page.evaluate(async (b) => (await fetch(b + "status")).status, base); + }, + + // As if the browser dropped the cookie silently: the viewer must not send + // the person back to the login page in a loop + async dropped(context, page) { + await context.route(/\/auth$/, async (route) => { + const response = await route.fetch(); + await context.clearCookies(); // route.fetch shares the context's cookie jar + const headers = response.headers(); + delete headers["set-cookie"]; + await route.fulfill({ response, headers }); + }); + await page.goto(base + "auth/view"); + await page.waitForURL(/\/auth\/login\?next=/, { timeout: 10000 }); + await signIn(page, password); + await page.waitForURL(/\/auth\/view$/, { timeout: 30000 }); + await inlineLogin(page); + out.inline_login = 1; + const streamed = page.waitForRequest(/\/stream$/, { timeout: 30000 }); + await signIn(page, password); + const token = ((await streamed).headers().authorization ?? "").replace(/^Bearer /, ""); + await page.waitForFunction(() => document.getElementById("state").textContent === "live", null, { timeout: 30000 }); + out.token_in_page_only = await page.evaluate((t) => t !== "" && localStorage.length === 0 && + sessionStorage.length === 0 && !location.href.includes(t) && !document.cookie.includes(t), token) ? 1 : 0; + }, + + async blocked(context, page) { + out.proofs = out.proofs_with_session = 0; + page.on("request", (r) => { + const body = r.method() === "POST" && /\/auth$/.test(r.url()) ? JSON.parse(r.postData() ?? "{}") : {}; + if (body.proof) out.proofs++; + if (body.proof && "session" in body) out.proofs_with_session++; + }); + await page.goto(base + "auth/login"); + out.cookie_enabled = await page.evaluate(() => navigator.cookieEnabled); + out.document_cookie_kept = JSON.stringify(await page.evaluate(() => { document.cookie = "t=1"; return document.cookie; })); + await page.waitForSelector('#status a[href="view"]', { timeout: 10000 }); + out.login_page_disabled = await page.evaluate(() => document.querySelector("#login button").disabled); + await page.click('#status a[href="view"]'); + await inlineLogin(page); + out.inline_login = 1; + + const streamed = page.waitForRequest(/\/stream$/, { timeout: 30000 }); + await signIn(page, password); + const bearer = (await streamed).headers().authorization ?? ""; + out.stream_bearer = /^Bearer \S+$/.test(bearer) ? 1 : 0; + await linesAtLeast(page, 3); + out.events = await lines(page); + const statusWith = () => page.evaluate(async ([b, h]) => + (await fetch(b + "status", { headers: { Authorization: h } })).status, [base, bearer]); + out.token_status = await statusWith(); + + await page.click("#logout"); + await statusIs(page, /^Signed out\.$/); + await inlineLogin(page); + out.token_status_after_logout = await statusWith(); + + // A reload ends the open stream and every token; the reconnect's 401 asks again + await signIn(page, password); + await page.waitForFunction(() => document.getElementById("state").textContent === "live", null, { timeout: 30000 }); + process.kill(Number(pid), "SIGHUP"); + await statusIs(page, /session ended/); + await inlineLogin(page); + out.expired_asks_again = 1; + }, +}; + +(async () => { + let context; + try { + if (mode === "blocked") { + // Full Chromium: the headless shell ignores a profile's Preferences + context = await chromium.launchPersistentContext(profile, { ignoreHTTPSErrors: true, channel: "chromium" }); + } else { + context = await (await chromium.launch()).newContext({ ignoreHTTPSErrors: true }); + } + const page = await context.newPage(); + page.on("console", (m) => { if (/Content Security Policy/i.test(m.text())) out.csp_violations++; }); + await scenarios[mode](context, page); } catch (e) { out.error = JSON.stringify(e.message.split("\n")[0]); } finally { - await browser.close(); + await (context?.browser() ?? context)?.close(); } for (const [k, v] of Object.entries(out)) console.log(`${k}=${v}`); })(); EOF +info "configuration in $(short "$CONF")/, browser driver $(short "$RUN/browser.cjs")" -# --- Guide --- -cat < $SITE/logs/auth/view +Shell setup (LW, CA, PW: viewer-01's password file, SITE): +> . $(short "$RUN")/env +The CLI, unbound through the proxy (bearer keeps the token off argv): +> token=\$(\$LW auth token -unbound -url \$SITE/logs -user viewer-01 -password-file \$PW -ca-file \$CA) +> curl -N --cacert \$CA -H @<(bearer) \$SITE/logs/stream +Logs: $(short "$LOG")/ EOF - -# --- Startup --- -"$BIN" -c "$CONF/logwisp.toml" > "$LOG/logwisp.out" 2>&1 & -PIDS+=($!) -"$PROXY_SRC/proxy" "127.0.0.1:$PORT_PROXY" "http://127.0.0.1:$PORT_SINK" "$PROXY_ADDR" \ - "$PKI/site.crt" "$PKI/site.key" > "$LOG/proxy.out" 2>&1 & -PIDS+=($!) -for p in $PORT_SINK $PORT_PROXY; do - wait_port "$p" 10 || { echo "FAIL: port $p not listening (see $LOG/)"; exit 1; } -done - -if (( AUTO == 0 )); then - echo "--- daemons running; Ctrl-C to stop" - while :; do sleep 1; done -fi - -fail=0 -check() { # label condition_result - if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi -} -is() { [[ $1 == "$2" ]] && echo 1 || echo 0; } # actual expected -> 1 or 0 -is_set() { [[ -n $1 ]] && echo 1 || echo 0; } +manual_hold code_of() { # url [token] -> http_code; the header goes through a pipe, never argv local args=(-s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt") @@ -258,48 +294,87 @@ code_of() { # url [token] -> http_code; the header goes through a pipe, never ar fi } -echo "=== Scenario 1: a person in a browser ===" -if (( BROWSER )); then - browser_out="$(timeout 120 node "$RUN/browser.cjs" "$SITE/logs/" viewer-01 "$(cat "$AUTH/viewer-01.pass")" 2>"$LOG/browser.err")" - echo "$browser_out" > "$LOG/browser.out" - val() { sed -n "s/^$1=//p" <<< "$browser_out"; } - [[ -n $(val error) ]] && echo "browser error: $(val error)" +browser() { # mode: runs browser.cjs, whose key=value lines val reads + info "browser: $1 (output in $(short "$LOG")/browser-$1.out)" + timeout 120 node "$RUN/browser.cjs" "$1" "$SITE/logs/" viewer-01 "$AUTH/viewer-01.pass" \ + "$RUN/profile" "${PIDS[0]}" >"$LOG/browser-$1.out" 2>"$LOG/browser-$1.err" + browser_out="$(cat "$LOG/browser-$1.out")" + [[ -z $(val error) ]] || info "${C_RED}browser error ($1): $(val error)$C_OFF" +} +val() { sed -n "s/^$1=//p" <<<"$browser_out"; } +# The count covers the whole scenario only when it ran to its end without error +csp_clean() { echo $(($(is "$(val error)" "") && $(is "$(val csp_violations)" 0))); } + +section "Scenario 1: a person in a browser" +if ((BROWSER)); then + browser cookie check "browser: an unauthenticated viewer is sent to the login page" $(is "$(val redirected_to_login)" 1) check "browser: a wrong password is refused on the page" $(is "$(val wrong_password_refused)" 1) check "browser: login (Argon2 in the page, $(val login_ms) ms) opened the viewer" $(is_set "$(val login_ms)") n=$(val events) - check "browser: the viewer showed events through EventSource (${n:-0} lines)" $(( ${n:-0} >= 3 )) + check "browser: the viewer showed events through EventSource (${n:-0} lines)" $((${n:-0} >= 3)) + check "browser: with the cookie working the viewer shows no sign-in form" $(is "$(val inline_login)" false) check "browser: session cookie $(val cookie)" \ $(is "$(val cookie)" "path=/logs;httponly=true;secure=true;samesite=Strict") check "browser: page scripts cannot read the session (document.cookie $(val document_cookie))" \ $(is "$(val document_cookie)" '""') check "browser: sign out cleared the cookie" $(is "$(val cookie_after_logout)" 0) check "browser: status refused after sign out (HTTP $(val status_after_logout))" $(is "$(val status_after_logout)" 401) - check "browser: no CSP violations ($(val csp_violations))" $(is "$(val csp_violations)" 0) + check "browser: no CSP violations ($(val csp_violations))" $(csp_clean) n=$(grep 'Login accepted' "$LOG/logwisp.out" | grep -c 'remote_addr 127.0.0.1') - check "logwisp logged the forwarded client, not the proxy ($n logins from 127.0.0.1)" $(( n >= 1 )) + check "logwisp logged the forwarded client, not the proxy ($n logins from 127.0.0.1)" $((n >= 1)) + browser dropped + check "browser: a session cookie the browser dropped brings the viewer's own sign-in form, not a loop" \ + $(is "$(val inline_login)" 1) + check "browser: its token stays out of storage, the URL and cookies" $(is "$(val token_in_page_only)" 1) else - echo "SKIP: browser checks (node with playwright not found)" + skip "browser checks: node, playwright or its Chromium headless shell missing" fi -echo "=== Scenario 2: the CLI through the proxy ===" +section "Scenario 2: a browser that keeps no cookies" +if ((FULL_CHROMIUM)); then + mkdir -p "$RUN/profile/Default" + echo '{"profile": {"default_content_setting_values": {"cookies": 2}}}' >"$RUN/profile/Default/Preferences" + browser blocked + check "no cookies: the profile blocks cookies (document.cookie kept $(val document_cookie_kept); navigator.cookieEnabled $(val cookie_enabled))" \ + $(is "$(val document_cookie_kept)" '""') + check "no cookies: the login page says so, links to the viewer and stays disabled" \ + $(is "$(val login_page_disabled)" true) + check "no cookies: the viewer shows its own sign-in form" $(is "$(val inline_login)" 1) + n=$(val events) + check "no cookies: the viewer streamed events with a bearer token (${n:-0} lines)" \ + $((${n:-0} >= 3 && $(is "$(val stream_bearer)" 1))) + n=$(val proofs) + check "no cookies: the viewer's proofs asked for no cookie (${n:-0} sent, $(val proofs_with_session) with a session)" \ + $((${n:-0} >= 1 && $(is "$(val proofs_with_session)" 0))) + check "no cookies: the page's token opened status (HTTP $(val token_status))" $(is "$(val token_status)" 200) + check "no cookies: sign out revoked it (HTTP $(val token_status_after_logout))" \ + $(is "$(val token_status_after_logout)" 401) + check "no cookies: a reload ended the session and the viewer asked to sign in again" \ + $(is "$(val expired_asks_again)" 1) + check "no cookies: no CSP violations ($(val csp_violations))" $(csp_clean) +else + skip "browser checks: node, playwright or its full Chromium missing" +fi + +section "Scenario 3: the CLI through the proxy" token="$("$BIN" auth token -unbound -url "$SITE/logs" -user viewer-01 -password-file "$AUTH/viewer-01.pass" \ -ca-file "$PKI/ca.crt" 2>>"$LOG/auth-cli.out")" -check "cli: logwisp auth token -unbound logged in through the proxy" $(is_set "$token") +check "cli: lw auth token -unbound logged in through the proxy" $(is_set "$token") code="$(code_of "$SITE/logs/status" "$token")" check "cli: /logs/status served with the token (HTTP $code)" $(is "$code" 200) sse="$(timeout 3 curl -sN --noproxy '*' --cacert "$PKI/ca.crt" \ -H @<(printf 'Authorization: Bearer %s\n' "$token") "$SITE/logs/stream" 2>/dev/null || true)" -n=$(grep -c '^data:' <<< "$sse") -check "cli: /logs/stream delivered SSE events with the token ($n events)" $(( n >= 1 )) +n=$(grep -c '^data:' <<<"$sse") +check "cli: /logs/stream delivered SSE events with the token ($n events)" $((n >= 1)) code="$(code_of "$SITE/logs/status")" check "cli: /logs/status refused without a token (HTTP $code)" $(is "$code" 401) "$BIN" auth token -url "$SITE/logs" -user viewer-01 -password-file "$AUTH/viewer-01.pass" \ -ca-file "$PKI/ca.crt" >/dev/null 2>>"$LOG/auth-cli.out" rc=$? -check "cli: a path without -unbound is a usage error (exit $rc)" $(( rc == 2 )) +check "cli: a path without -unbound is a usage error (exit $rc)" $((rc == 2)) -echo "=== Scenario 3: around the proxy ===" +section "Scenario 4: around the proxy" code="$(code_of "http://127.0.0.1:$PORT_SINK/auth/login")" check "direct: the login page refuses a peer that is not the proxy (HTTP $code)" $(is "$code" 403) code="$(code_of "http://127.0.0.1:$PORT_SINK/status" "$token")" @@ -307,11 +382,4 @@ check "direct: a valid token does not open the sink around the proxy (HTTP $code code="$(code_of "$SITE/plain/status" "$token")" check "plaintext site: a request the proxy received over http is refused (HTTP $code)" $(is "$code" 403) -echo "================================================================" -if (( fail == 0 )); then - echo "RESULT: ALL PASS" - (( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); } -else - echo "RESULT: FAILURES — inspect $LOG/*" -fi -exit "$fail" +summary