lixen/logwisp
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.22.0 several enhancements and fixes for security, network, doc, test, packaging
Test / test (push) Canceled after 0s
Details
Test / e2e (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
lixen committed 2026-10-03 14:52:16 -04:00
1 parent 9ab8c356d2
commit 71d1b82f83
92 files changed
+7044 -2120
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.dockerignore
+21
View File
@@ -0,0 +1,21 @@
# The builder needs the module and its sources only; local state and keys stay out.
.git
.github
.idea
.claude
worktrees
.worktrees
bin
build
cert
data
dev
log
logs
script
test/run*
**/node_modules
*.log
*.pass
*.toml
!config/*.toml
.github/workflows/test.yml
+39 -2
View File
@@ -11,7 +11,44 @@ jobs:
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
- uses: actions/setup-node@v4
with:
node-version: 22
- run: go test -timeout 60s -race ./...
- run: go vet ./...
- run: go build ./cmd/logwisp
- run: GOOS=freebsd GOARCH=amd64 go build -o /tmp/logwisp-freebsd ./cmd/logwisp
- run: go build ./cmd/lw
- run: GOOS=freebsd GOARCH=amd64 go build -o /tmp/lw-freebsd ./cmd/lw
- run: node --test
working-directory: internal/sink/http/web
# The end-to-end scripts on a host with an IPv6 loopback and playwright's
# Chromium; the docker run needs root, so it skips
e2e:
runs-on: ubuntu-latest
timeout-minutes: 25
env:
FORCE_COLOR: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version-file: go.mod
- uses: actions/setup-node@v4
with:
node-version: 22
# "chromium" fetches the headless shell and full Chromium, which the
# cookie-blocking profile needs; the driver's require reads NODE_PATH
- run: |
npm i -g playwright
npx playwright install --with-deps chromium
echo "NODE_PATH=$(npm root -g)" >> "$GITHUB_ENV"
- run: make build
- run: make e2e 2>&1 | tee "$RUNNER_TEMP/e2e.out"
shell: bash # adds pipefail: the step fails with make, not with tee
# ipv6-test.sh skips without ::1, which would leave the job green untested
- name: Require an IPv6 loopback
run: grep -q '^0\{31\}1 ' /proc/net/if_inet6
# scram-proxy-test.sh skips its browser checks without playwright's Chromium
- name: Require the browser checks
run: |
grep -q 'scram-proxy-test.sh --auto' "$RUNNER_TEMP/e2e.out"
if grep 'SKIP.*browser checks' "$RUNNER_TEMP/e2e.out"; then exit 1; fi
.gitignore
+3
View File
@@ -8,6 +8,7 @@ bin/
script/
build/
*.log
*.pass
*.toml
!config/*.toml
build.sh
@@ -17,3 +18,5 @@ test/run/
test/run-mtls/
test/run-scram/
test/run-proxy/
test/run-ipv6/
test/run-deploy/
AGENTS.md
+11 -4
View File
@@ -49,6 +49,9 @@ It is the contract, not a suggestion.
- `doc/` is already long. Condense when you touch it; append only for a new concept
or scope.
- No tables in docs, never a large one: docs are read in terminals and in editors
that do not render Markdown. Use hierarchical bullet lists (item, then its
details indented beneath); fenced blocks for config, commands and port maps.
- A gap you are deferring goes in the "Not Implemented" list of the design doc it
belongs to: extend the item, or add a concise one in the list's pattern.
- The repository describes the architecture, never a machine: docs and commands
@@ -64,7 +67,7 @@ It is the contract, not a suggestion.
`go build ./...`, `gofmt -l` on changed files, and `go test` on the packages the
change touches.
- A change to platform-sensitive code also cross-builds the production target:
`GOOS=freebsd GOARCH=amd64 go build ./cmd/logwisp`.
`GOOS=freebsd GOARCH=amd64 go build ./cmd/lw`.
- Do not run `-race` tests unless investigating a known/suspected race issue; CI
runs them.
@@ -91,8 +94,12 @@ change touches.
dependency-free ES module (sites copy it into their bundles) and the pages
stay free of inline script and style (their CSP); `node --test` there checks
it against Go's Argon2 and `auth`'s known answer.
- Listeners and dialers are IPv4-only (`tcp4`). E2E scripts in `test/` need
`bin/logwisp` and `--auto`, and each owns a port range and a gitignored run
directory.
- Listeners and dialers take their network from `core.Network`, strictly per
family: an IPv4 literal `tcp4`, an IPv6 literal (`::` too) IPv6-only `tcp6`,
a hostname `tcp`. Join addresses with `net.JoinHostPort`, build URLs with
`net/url`. E2E scripts in `test/` need `bin/lw` and `--auto`, and each owns
a port range and a gitignored run directory. They source `test/lib.sh`
(checks, skips, summary; daemons log only to files) and exit 77 when the host
lacks what they test.
- Many older files lack a trailing newline and fail `gofmt -l`; format the files
you change, not the tree.
Dockerfile
+20 -11
View File
@@ -2,23 +2,30 @@
# an older builder reports the mismatch only after downloading the module graph.
ARG GO_VERSION=1.27.1
FROM docker.io/library/golang:${GO_VERSION}-alpine AS build
# The builder runs natively and cross-compiles, so foreign platforms need no emulation.
FROM --platform=$BUILDPLATFORM docker.io/library/golang:${GO_VERSION}-alpine AS build
# Git supplies Go's VCS build information; only /out/logwisp crosses stages.
RUN apk add --no-cache git
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
# The optional build_ca secret adds a CA (an intercepting proxy's) for this step
# only: Go reads SSL_CERT_DIR besides the system bundle, and a secret mount
# never reaches a layer. Proxy variables are Docker's predefined build args.
RUN --mount=type=secret,id=build_ca,target=/run/build-ca/ca.pem \
SSL_CERT_DIR=/run/build-ca go mod download
COPY . .
ARG TARGETOS=linux
ARG TARGETARCH=amd64
# No defaults: a declared default would override the platform BuildKit passes.
ARG TARGETOS
ARG TARGETARCH
ARG VERSION=dev
ARG REVISION=unknown
ARG BUILD_TIME=unknown
# REVISION identifies the commit: the context has no .git, so no VCS stamping.
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
go build -trimpath \
-ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION}" \
-o /out/logwisp ./cmd/logwisp
go build -trimpath -buildvcs=false \
-ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION} -X logwisp/internal/version.BuildTime=${BUILD_TIME}" \
-o /out/lw ./cmd/lw
FROM scratch
@@ -32,9 +39,11 @@ LABEL org.opencontainers.image.title="logwisp" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.licenses="BSD-3-Clause"
COPY --from=build /out/logwisp /logwisp
# Dialers without tls.ca_file, and `lw auth` without -ca-file, verify against system roots.
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
COPY --from=build /out/lw /lw
# Numeric identity is required in scratch and satisfies a restricted pod spec.
USER 65532:65532
ENTRYPOINT ["/logwisp"]
ENTRYPOINT ["/lw"]
Makefile
+211 -36
View File
@@ -1,52 +1,227 @@
# LogWisp Makefile
# Compatible with GNU Make (Linux) and BSD Make (FreeBSD)
# LogWisp. GNU make and BSD make alike: no conditionals or functions specific
# to either; decisions run in the shell. Plain `make` prints the targets.
BINARY_NAME = logwisp
BUILD_DIR = bin
BINARY_PATH = $(BUILD_DIR)/$(BINARY_NAME)
VERSION != git describe --tags --always --dirty 2>/dev/null || echo "dev"
GIT_COMMIT != git rev-parse --short HEAD 2>/dev/null || echo "unknown"
BUILD_TIME != date -u '+%Y-%m-%d_%H:%M:%S'
BINARY := lw
SRC := ./cmd/lw
BIN_DIR := bin
PKG_DIR := deploy/package
WEB_DIR := internal/sink/http/web
# Go build variables
GO = go
GOFLAGS =
LDFLAGS = -X 'logwisp/internal/version.Version=$(VERSION)' \
VERSION_DEFAULT != git describe --tags --always --dirty 2>/dev/null || echo dev
VERSION ?= $(VERSION_DEFAULT)
GIT_COMMIT_DEFAULT != git rev-parse --short HEAD 2>/dev/null || echo unknown
GIT_COMMIT ?= $(GIT_COMMIT_DEFAULT)
# SOURCE_DATE_EPOCH makes package builds reproducible; GNU date takes -d, BSD date -r.
BUILD_TIME_DEFAULT != f='+%Y-%m-%d_%H:%M:%S'; e="$${SOURCE_DATE_EPOCH:-}"; \
if [ -n "$$e" ]; then date -u -d "@$$e" "$$f" 2>/dev/null || date -u -r "$$e" "$$f"; else date -u "$$f"; fi
BUILD_TIME ?= $(BUILD_TIME_DEFAULT)
# The image stamps the commit time, so a rebuild of one commit stays cached
IMAGE_BUILD_TIME_DEFAULT != TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%d_%H:%M:%S 2>/dev/null || echo unknown
IMAGE_BUILD_TIME ?= $(IMAGE_BUILD_TIME_DEFAULT)
VERSION_LDFLAGS = -X 'logwisp/internal/version.Version=$(VERSION)' \
-X 'logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \
-X 'logwisp/internal/version.BuildTime=$(BUILD_TIME)'
# Not LDFLAGS/GOFLAGS: packagers export those for the C linker and go itself.
GO_BUILDFLAGS ?=
GO_LDFLAGS ?=
# Installation directories
PREFIX ?= /usr/local
BINDIR = $(PREFIX)/bin
# FreeBSD packages each Go release under its own name: go.mod's 1.27 is go127.
GO_DEFAULT != uname -s | grep -q FreeBSD && sed -n 's/^go \([0-9]*\)\.\([0-9]*\).*/go\1\2/p' go.mod 2>/dev/null | grep . || echo go
GO ?= $(GO_DEFAULT)
# Default target
all: build
E2E ?= test/*-test.sh
# Build the binary
build:
mkdir -p $(BUILD_DIR)
$(GO) build $(GOFLAGS) -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/logwisp
CONTAINER_ENGINE ?= docker
IMAGE ?= logwisp
IMAGE_TAG ?= dev
IMAGE_REVISION_DEFAULT != git rev-parse HEAD 2>/dev/null || echo unknown
IMAGE_REVISION ?= $(IMAGE_REVISION_DEFAULT)
# A PEM bundle the builder trusts while downloading modules (an intercepting
# proxy); it reaches the build as a BuildKit secret, never a layer.
BUILD_CA ?=
IMAGE_BUILD_FLAGS ?=
IMAGE_CHECK_CONFIG ?= config/logwisp.toml
IMAGE_RUN = $(CONTAINER_ENGINE) run --read-only --user 65532:65532 \
--network none --cap-drop ALL --security-opt no-new-privileges
# Install the binary
install: build
install -m 755 $(BINARY_PATH) $(BINDIR)/
DESTDIR ?=
INSTALL_OS_DEFAULT != uname -s
INSTALL_OS ?= $(INSTALL_OS_DEFAULT)
# FreeBSD keeps everything outside the base system under /usr/local.
PREFIX_DEFAULT != [ '$(INSTALL_OS)' = FreeBSD ] && echo /usr/local || echo /usr
PREFIX ?= $(PREFIX_DEFAULT)
SYSCONFDIR_DEFAULT != [ '$(INSTALL_OS)' = FreeBSD ] && echo /usr/local/etc || echo /etc
SYSCONFDIR ?= $(SYSCONFDIR_DEFAULT)
BINDIR ?= $(PREFIX)/bin
MANDIR ?= $(PREFIX)/share/man
DOCDIR ?= $(PREFIX)/share/doc/logwisp
LICENSEDIR ?= $(PREFIX)/share/licenses/logwisp
.DEFAULT_GOAL := help
.PHONY: help check-go build release dev test verify e2e image image-check install uninstall clean version
help:
@echo "Usage: make <target> [VARIABLE=value ...]"
@echo ""
@echo "Build:"
@echo " build Build $(BIN_DIR)/$(BINARY) with version metadata"
@echo " release Build it static, stripped and trimmed (CGO_ENABLED=0)"
@echo " dev Build it with the race detector"
@echo " version Print the version metadata a build embeds"
@echo " clean Remove $(BIN_DIR)/"
@echo "Check:"
@echo " test Run the Go tests"
@echo " verify vet, gofmt, tests, linux and freebsd cross builds, web client tests"
@echo " e2e Build, then run each of $(E2E) with --auto and report"
@echo "Container:"
@echo " image Build $(IMAGE):$(IMAGE_TAG) with $(CONTAINER_ENGINE) (scratch, static, UID 65532)"
@echo " image-check Run it read-only, offline, without capabilities: --version, then"
@echo " --check on $(IMAGE_CHECK_CONFIG)"
@echo "Install (build or release first):"
@echo " install Binary, manual, sample config, service files, licence and docs"
@echo " uninstall Remove them, keeping $(SYSCONFDIR)/logwisp"
@echo ""
@echo "Variables: GO=$(GO) PREFIX=$(PREFIX) SYSCONFDIR=$(SYSCONFDIR) DESTDIR=$(DESTDIR)"
@echo " INSTALL_OS=$(INSTALL_OS) (the layout install uses: FreeBSD or any other)"
@echo " GO_BUILDFLAGS, GO_LDFLAGS (extra go build and linker flags), E2E (scripts),"
@echo " CONTAINER_ENGINE, IMAGE, IMAGE_TAG, BUILD_CA, IMAGE_BUILD_FLAGS, IMAGE_CHECK_CONFIG"
check-go:
@command -v $(GO) >/dev/null 2>&1 || { \
echo "Go toolchain '$(GO)' not found; go.mod requires go $$(sed -n 's/^go //p' go.mod)."; \
if [ "$$(uname -s)" = FreeBSD ]; then echo " FreeBSD: pkg install $(GO_DEFAULT)"; \
elif [ -f /etc/arch-release ]; then echo " Arch Linux: pacman -S go"; \
else echo " Elsewhere: https://go.dev/dl/ (distribution packages often lag)"; fi; \
echo " Or name a toolchain: make GO=/path/to/go ..."; \
exit 1; }
build: check-go
$(GO) build $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC)
release: check-go
CGO_ENABLED=0 $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "-s -w $(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC)
dev: check-go
$(GO) build -race $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC)
test: check-go
$(GO) test ./...
# gofmt checks the Go files changed since the branch forked from GOFMT_BASE,
# exact renames aside, or the whole tree without git or that ref: older files
# predate the gate, and AGENTS.md formats what a change touches.
GOFMT_BASE ?= main
verify: test
$(GO) vet ./...
@base='$(GOFMT_BASE)'; \
if [ -n "$$base" ] && fork=$$(git merge-base "$$base" HEAD 2>/dev/null); then \
files=$$( { git diff --name-only --find-renames=100% --diff-filter=dr "$$fork" -- cmd internal; \
git ls-files --others --exclude-standard -- cmd internal; } | grep '\.go$$' || true); \
else \
files=$$(find cmd internal -name '*.go'); \
fi; \
bad=$$( [ -z "$$files" ] || "$$($(GO) env GOROOT)/bin/gofmt" -l $$files ); \
if [ -n "$$bad" ]; then echo "gofmt -l reports:"; echo "$$bad"; exit 1; fi
@for target in linux/amd64 linux/arm64 freebsd/amd64 freebsd/arm64; do \
echo "cross build $$target"; \
CGO_ENABLED=0 GOOS=$${target%/*} GOARCH=$${target#*/} $(GO) build -o /dev/null $(SRC) || exit 1; \
done
@if command -v node >/dev/null 2>&1; then \
cd $(WEB_DIR) && node --test; \
else \
echo "node not found: web client tests skipped"; \
fi
# A script exits 0 (passed), 77 (skipped: the host lacks what it tests) or
# anything else (failed); only failures fail the target.
e2e: build
@if [ "$${FORCE_COLOR:-0}" != 0 ] || { [ -t 1 ] && [ -z "$${NO_COLOR:-}" ]; }; then \
g=$$(printf '\033[1;32m'); r=$$(printf '\033[1;31m'); y=$$(printf '\033[1;33m'); \
b=$$(printf '\033[1m'); o=$$(printf '\033[0m'); \
else g=; r=; y=; b=; o=; fi; \
pass=0; fail=0; skip=0; results=; \
for t in $(E2E); do \
echo; "$$t" --auto; rc=$$?; \
case $$rc in \
0) pass=$$((pass + 1)); results="$$results PASS:$$t" ;; \
77) skip=$$((skip + 1)); results="$$results SKIP:$$t" ;; \
*) fail=$$((fail + 1)); results="$$results FAIL:$$t:$$rc" ;; \
esac; \
done; \
printf '\n%se2e summary%s\n' "$$b" "$$o"; \
for x in $$results; do \
st=$${x%%:*}; t=$${x#*:}; c=$$g; note=; \
case $$st in SKIP) c=$$y ;; FAIL) c=$$r; note=" (exit $${t##*:})"; t=$${t%:*} ;; esac; \
printf ' %s%s%s %s%s\n' "$$c" "$$st" "$$o" "$$t" "$$note"; \
done; \
c=$$g; [ "$$fail" -eq 0 ] || c=$$r; \
printf '%se2e: %d passed, %d failed, %d skipped%s\n' "$$c" "$$pass" "$$fail" "$$skip" "$$o"; \
[ "$$fail" -eq 0 ]
image:
@set --; if [ -n '$(BUILD_CA)' ]; then set -- --secret 'id=build_ca,src=$(BUILD_CA)'; fi; \
set -x; \
$(CONTAINER_ENGINE) build "$$@" $(IMAGE_BUILD_FLAGS) \
--build-arg VERSION='$(VERSION)' \
--build-arg REVISION='$(IMAGE_REVISION)' \
--build-arg BUILD_TIME='$(IMAGE_BUILD_TIME)' \
-t '$(IMAGE):$(IMAGE_TAG)' .
# --check builds every pipeline and plugin, as a start would, and binds nothing
image-check:
$(IMAGE_RUN) --rm '$(IMAGE):$(IMAGE_TAG)' --version
@set -eu; cfg='$(IMAGE_CHECK_CONFIG)'; case "$$cfg" in /*) ;; *) cfg="$$(pwd)/$$cfg" ;; esac; \
$(IMAGE_RUN) --rm --mount "type=bind,src=$$cfg,dst=/etc/logwisp/logwisp.toml,readonly" \
'$(IMAGE):$(IMAGE_TAG)' --check -c /etc/logwisp/logwisp.toml
# install stages what a package ships and compiles nothing, so a packager owns
# the build flags. The service files get the installed paths substituted; a
# configuration already in place is kept.
install:
@set -eu; \
[ -x $(BIN_DIR)/$(BINARY) ] || { echo "no $(BIN_DIR)/$(BINARY): run make build or make release first" >&2; exit 1; }; \
put() { install -d -m 0755 "$${3%/*}"; install -m "$$1" "$$2" "$$3"; echo "install $$3"; }; \
sub() { install -d -m 0755 "$${3%/*}"; \
sed -e 's|@BINDIR@|$(BINDIR)|g' -e 's|@SYSCONFDIR@|$(SYSCONFDIR)|g' -e 's|%%BINDIR%%|$(BINDIR)|g' \
-e 's|%%PREFIX%%/etc|$(SYSCONFDIR)|g' -e 's|%%PREFIX%%|$(PREFIX)|g' "$$2" > "$$3"; \
chmod "$$1" "$$3"; echo "install $$3"; }; \
root='$(DESTDIR)'; etc="$$root$(SYSCONFDIR)/logwisp"; \
put 0755 $(BIN_DIR)/$(BINARY) "$$root$(BINDIR)/$(BINARY)"; \
put 0644 doc/lw.1 "$$root$(MANDIR)/man1/lw.1"; \
put 0644 LICENSE "$$root$(LICENSEDIR)/LICENSE"; \
for src in README.md doc/*.md; do put 0644 "$$src" "$$root$(DOCDIR)/$$src"; done; \
case '$(INSTALL_OS)' in \
FreeBSD) \
put 0644 config/logwisp.toml "$$etc/logwisp.toml.sample"; \
if [ -z "$$root" ] && [ ! -e "$$etc/logwisp.toml" ]; then put 0644 config/logwisp.toml "$$etc/logwisp.toml"; fi; \
sub 0755 $(PKG_DIR)/logwisp.rc "$$root$(PREFIX)/etc/rc.d/logwisp" ;; \
*) \
if [ -e "$$etc/logwisp.toml" ]; then echo "keep $$etc/logwisp.toml"; \
else put 0644 config/logwisp.toml "$$etc/logwisp.toml"; fi; \
sub 0644 $(PKG_DIR)/logwisp.service "$$root$(PREFIX)/lib/systemd/system/logwisp.service"; \
sub 0644 $(PKG_DIR)/logwisp.sysusers "$$root$(PREFIX)/lib/sysusers.d/logwisp.conf"; \
sub 0644 $(PKG_DIR)/logwisp.tmpfiles "$$root$(PREFIX)/lib/tmpfiles.d/logwisp.conf" ;; \
esac
# Uninstall the binary
uninstall:
rm -f $(BINDIR)/$(BINARY_NAME)
@set -eu; root='$(DESTDIR)'; \
for f in "$$root$(BINDIR)/$(BINARY)" "$$root$(MANDIR)/man1/lw.1" \
"$$root$(PREFIX)/lib/systemd/system/logwisp.service" \
"$$root$(PREFIX)/lib/sysusers.d/logwisp.conf" "$$root$(PREFIX)/lib/tmpfiles.d/logwisp.conf" \
"$$root$(PREFIX)/etc/rc.d/logwisp" "$$root$(SYSCONFDIR)/logwisp/logwisp.toml.sample"; do \
if [ -e "$$f" ]; then rm -f "$$f"; echo "remove $$f"; fi; \
done; \
for d in "$$root$(DOCDIR)" "$$root$(LICENSEDIR)"; do \
case "$$d" in */logwisp) ;; *) echo "keep $$d (not a logwisp directory)"; continue ;; esac; \
if [ -d "$$d" ]; then rm -rf "$$d"; echo "remove $$d"; fi; \
done; \
echo "keep $$root$(SYSCONFDIR)/logwisp (configuration and credentials)"
# Clean build artifacts
clean:
rm -f $(BINARY_PATH)
rm -rf $(BIN_DIR)
# Development build with race detector
dev:
$(GO) build $(GOFLAGS) -race -ldflags "$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/logwisp
# Show current version
version:
@echo "Version: $(VERSION)"
@echo "Commit: $(GIT_COMMIT)"
@echo "Build Time: $(BUILD_TIME)"
.PHONY: all build install uninstall clean dev version
@echo "Build time: $(BUILD_TIME)"
README.md
+51 -26
View File
@@ -64,11 +64,11 @@ exponential backoff and jitter.
stream and status endpoints, and lets a dialer pin the server it talks to
- Password authentication (Argon2id-SCRAM) on the same block: listeners hold
verifiers, never passwords, and do no KDF work; every login outside proxy
mode is bound to the
listener's certificate, so no relay presenting another one can use it; HTTP
listeners issue short-lived bearer tokens. `logwisp auth` manages credentials
files and logs viewers in; behind a site's TLS-terminating proxy, browsers
log in through a shipped login page and a dependency-free JS client
mode is bound to the listener's certificate, so no relay presenting another
one can use it; HTTP listeners issue short-lived bearer tokens. `lw auth`
manages credentials files and logs viewers in; behind a site's
TLS-terminating proxy, browsers log in through a shipped login page and a
dependency-free JS client
- Node binding: a chain source can label entries from the sender's certificate
identity or username instead of from what the sender claims, so origin
attribution is not forgeable
@@ -82,22 +82,30 @@ authentication designs for the rationale and what is deliberately left out.
## Documentation
| Document | Contents |
|----------|----------|
| [Installation](doc/installation.md) | Building, installing, running as a service |
| [Architecture](doc/architecture.md) | Component model, data flow, concurrency, back-pressure |
| [Configuration](doc/configuration.md) | TOML structure, precedence, environment and CLI overrides |
| [Sources](doc/sources.md) | Every input plugin and its options |
| [Sinks](doc/sinks.md) | Every output plugin and its options |
| [Filters](doc/filters.md) | Pattern-based inclusion and exclusion |
| [Formatters](doc/formatters.md) | Output shaping and sanitization |
| [Chaining](doc/chaining.md) | Multi-node topologies and the chain wire protocol |
| [Networking](doc/networking.md) | Listeners, dialers, timeouts, connection limits |
| [Security](doc/security.md) | TLS, mTLS, and peer authorization; threat model and current limits |
| [mTLS Authentication](doc/mtls-auth-plan.md) | Design and rationale for certificate-based authorization |
| [Password Authentication](doc/scram-auth-plan.md) | Design and rationale for Argon2id-SCRAM authentication; mTLS hardening |
| [CLI](doc/cli.md) | Flags, signals, exit codes, `logwisp auth` |
| [Operations](doc/operations.md) | Running, monitoring, tuning, troubleshooting |
- [Installation](doc/installation.md): building, installing, services,
the container image, packaging
- [Deployment](doc/deployment.md): `deploy/lw-deploy.sh` for edges,
aggregators, containers, services, jails
- [Architecture](doc/architecture.md): component model, data flow,
concurrency, back-pressure
- [Configuration](doc/configuration.md): TOML structure, precedence,
environment and CLI overrides
- [Sources](doc/sources.md) and [Sinks](doc/sinks.md): every plugin and its
options
- [Filters](doc/filters.md) and [Formatters](doc/formatters.md): pattern
matching, output shaping and sanitization
- [Chaining](doc/chaining.md): multi-node topologies and the wire protocol
- [Networking](doc/networking.md): listeners, dialers, timeouts, limits
- [Security](doc/security.md): TLS, mTLS, peer authorization, threat model
- Design notes: [mTLS](doc/mtls-auth-plan.md) and
[password](doc/scram-auth-plan.md) authentication
- [CLI](doc/cli.md): flags, pipeline specifications, signals, exit codes,
`lw auth`; also the `lw(1)` manual, [`doc/lw.1`](doc/lw.1)
- [Operations](doc/operations.md): running, monitoring, tuning,
troubleshooting
- [To Do](doc/todo.md): planned work in priority order: network access
control and the PROXY protocol, a hardening review of the config libraries,
packaging
A fully annotated configuration covering every option ships as
[`config/logwisp.toml`](config/logwisp.toml).
@@ -105,9 +113,13 @@ A fully annotated configuration covering every option ships as
## Quick Start
```bash
make
make build # plain `make` lists the targets
./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \
--format json,sanitizer_policy=json --sink console
```
Or as a configuration file:
```toml
# logwisp.toml
[[pipelines]]
@@ -132,11 +144,23 @@ target = "stdout"
```
```bash
logwisp -c logwisp.toml
./bin/lw -c logwisp.toml
```
Running with no configuration file starts a self-demonstrating pipeline: a
synthetic generator writing JSON to stdout.
Running with neither starts a self-demonstrating pipeline: a synthetic
generator writing JSON to stdout.
In a container, read-only and without capabilities:
```bash
make image
docker run --rm --read-only --cap-drop ALL --security-opt no-new-privileges \
-v /etc/logwisp:/etc/logwisp:ro logwisp:dev -c /etc/logwisp/logwisp.toml
```
`sudo make install` installs the binary, the manual, a sample configuration
and a systemd unit or FreeBSD rc.d script; see
[Installation](doc/installation.md).
## System Requirements
@@ -144,7 +168,8 @@ synthetic generator writing JSON to stdout.
- **Architecture**: amd64
- **Go**: 1.27.1+ to build from source
Network sources and sinks bind and dial over IPv4 only.
Network sources and sinks use IPv4 or IPv6, following the address family of
the configured host.
## License
cmd/logwisp/auth.go → cmd/lw/auth.go
+22 -13
View File
@@ -20,6 +20,7 @@ import (
"logwisp/internal/authz"
"logwisp/internal/config"
"logwisp/internal/core"
"logwisp/internal/tlsx"
"github.com/lixenwraith/auth"
@@ -27,7 +28,7 @@ import (
const reloadHint = "send SIGHUP to logwisp to apply (auto_reload does not watch the credentials file)"
// authCommand is one `logwisp auth` subcommand: define registers its flags
// authCommand is one `lw auth` subcommand: define registers its flags
// and returns the command, which runs once they are parsed and checked.
type authCommand struct {
name, synopsis, summary string
@@ -55,7 +56,7 @@ type usageError string
func (e usageError) Error() string { return string(e) }
// runAuth runs `logwisp auth` and returns the exit status: 0 success or
// runAuth runs `lw auth` and returns the exit status: 0 success or
// help, 1 failure, 2 usage error.
func runAuth(args []string, stdout, stderr io.Writer) int {
if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) {
@@ -67,12 +68,12 @@ func runAuth(args []string, stdout, stderr io.Writer) int {
}
i := slices.IndexFunc(authCommands, func(c authCommand) bool { return c.name == args[0] })
if i < 0 {
fmt.Fprintf(stderr, "logwisp auth: unknown command %q\n\n", args[0])
fmt.Fprintf(stderr, "lw auth: unknown command %q\n\n", args[0])
printAuthUsage(stderr)
return 2
}
c := authCommands[i]
fs := flag.NewFlagSet("logwisp auth "+c.name, flag.ContinueOnError)
fs := flag.NewFlagSet("lw auth "+c.name, flag.ContinueOnError)
fs.SetOutput(stderr)
fs.Usage = func() {
fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), c.synopsis, c.summary)
@@ -101,11 +102,11 @@ func runAuth(args []string, stdout, stderr io.Writer) int {
}
func printAuthUsage(w io.Writer) {
fmt.Fprint(w, "Usage: logwisp auth <command> [flags]\n\n")
fmt.Fprint(w, "Usage: lw auth <command> [flags]\n\n")
for _, c := range authCommands {
fmt.Fprintf(w, " %-12s %s\n", c.name, c.summary)
}
fmt.Fprint(w, "\nRun logwisp auth <command> -h for its flags. Credential changes apply on SIGHUP.\n"+
fmt.Fprint(w, "\nRun lw auth <command> -h for its flags. Credential changes apply on SIGHUP.\n"+
"Exit status: 0 success, 1 failure, 2 usage error.\n")
}
@@ -231,7 +232,7 @@ func defineRemoveUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
}
func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT (with -unbound, a proxy's mount URL may carry a path)")
rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT or https://[IPV6]:PORT (with -unbound, a proxy's mount URL may carry a path)")
user := fs.String("user", "", "user `name`")
passwordFile := fs.String("password-file", "", "`file` holding the password")
unbound := fs.Bool("unbound", false, "log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)")
@@ -245,6 +246,10 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
// TLS passthrough cannot route by path: only a terminating proxy can
return usageError("-url takes a path only with -unbound")
}
network, err := core.Network(u.Hostname())
if err != nil {
return usageError("-url: " + err.Error())
}
tlsCfg, policy, err := dialPolicy(tlsOpts, u.Hostname(), *user, *passwordFile, authz.HTTP)
if err != nil {
return err
@@ -256,7 +261,7 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
Transport: &http.Transport{
DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) {
var d net.Dialer
return d.DialContext(ctx, "tcp4", addr)
return d.DialContext(ctx, network, addr)
},
TLSClientConfig: tlsCfg,
TLSHandshakeTimeout: tlsx.HandshakeTimeout,
@@ -264,7 +269,10 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
// A redirect would carry the login to another endpoint
CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
}
token, err := policy.Token(context.Background(), client, "https://"+u.Host+u.Path)
// String escapes an IPv6 zone, which u.Host holds raw; the parts drop
// an empty "?" that would swallow the /auth path
base := url.URL{Scheme: u.Scheme, Host: u.Host, Path: u.Path, RawPath: u.RawPath}
token, err := policy.Token(context.Background(), client, base.String())
if err != nil {
return err
}
@@ -274,14 +282,15 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
}
func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
addr := fs.String("addr", "", "tcp sink `address`, HOST:PORT")
addr := fs.String("addr", "", "tcp sink `address`, HOST:PORT or [IPV6]:PORT")
user := fs.String("user", "", "user `name`")
passwordFile := fs.String("password-file", "", "`file` holding the password")
tlsOpts := tlsFlags(fs)
return func(stdout, _ io.Writer) error {
host, _, err := net.SplitHostPort(*addr)
if err != nil || host == "" {
return usageError("-addr must be HOST:PORT")
network, nerr := core.Network(host)
if err != nil || nerr != nil || host == "" {
return usageError("-addr must be HOST:PORT, with an IPv6 address in brackets")
}
tlsCfg, policy, err := dialPolicy(tlsOpts, host, *user, *passwordFile, authz.TCP)
if err != nil {
@@ -290,7 +299,7 @@ func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
dctx, cancel := context.WithTimeout(ctx, tlsx.HandshakeTimeout)
conn, err := (&tls.Dialer{Config: tlsCfg}).DialContext(dctx, "tcp4", *addr)
conn, err := (&tls.Dialer{Config: tlsCfg}).DialContext(dctx, network, *addr)
cancel()
if err == nil {
defer conn.Close()
cmd/logwisp/auth_test.go → cmd/lw/auth_test.go
+17 -1
View File
@@ -190,7 +190,23 @@ func TestRemoveUserRefusesTheLastUser(t *testing.T) {
func TestAuthCommandHelpPrintsAuthUsage(t *testing.T) {
code, _, stderr := runAuthTest(t, "add-user", "-h")
if code != 0 || !strings.Contains(stderr, "Usage: logwisp auth add-user -credentials FILE") {
if code != 0 || !strings.Contains(stderr, "Usage: lw auth add-user -credentials FILE") {
t.Fatalf("exit %d, stderr: %s", code, stderr)
}
}
// The login is posted to the -url given plus /auth: a link-local URL keeps
// its zone escaped, not turned into one that no longer parses
func TestTokenLogsInAtTheURLGiven(t *testing.T) {
pass := filepath.Join(t.TempDir(), "edge-01.pass")
if err := os.WriteFile(pass, []byte("correct horse battery\n"), 0o600); err != nil {
t.Fatal(err)
}
// No such interface or listener: the dial fails at once, after the URL is formed
for _, rawURL := range []string{"https://[fe80::1%25nosuchif0]:1", "https://127.0.0.1:1?"} {
code, _, stderr := runAuthTest(t, "token", "-url", rawURL, "-user", "edge-01", "-password-file", pass)
if want := `"` + strings.TrimSuffix(rawURL, "?") + `/auth"`; code != 1 || !strings.Contains(stderr, want) {
t.Errorf("%s: exit %d, stderr: %s", rawURL, code, stderr)
}
}
}
cmd/logwisp/bootstrap.go → cmd/lw/bootstrap.go
+17
View File
@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"os"
_ "logwisp/internal/source/console"
_ "logwisp/internal/source/file"
@@ -108,6 +109,22 @@ func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service
return svc, nil
}
// checkConfig builds every pipeline and plugin as a start would, so TLS files,
// credentials and options are loaded and their warnings print, then exits
// before anything binds or reads: plugins start goroutines only in Start.
func checkConfig(cfg *config.Config) int {
cfg.Logging.Output, cfg.Logging.Level = "stderr", "warn"
if err := initializeLogger(cfg); err == nil && logger.Start() == nil {
defer shutdownLogger()
}
if _, err := service.NewService(context.Background(), cfg, logger); err != nil {
fmt.Fprintf(os.Stderr, "configuration invalid: %v\n", err)
return 1
}
fmt.Printf("configuration ok: %d pipeline(s)\n", len(cfg.Pipelines))
return 0
}
// initializeLogger sets up the global logger based on the application's configuration
func initializeLogger(cfg *config.Config) error {
logger = log.NewLogger()
cmd/logwisp/help.go → cmd/lw/help.go
+21 -8
View File
@@ -6,17 +6,18 @@ import (
"os"
)
// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths.
// helpText is the CLI usage reference. Scalar flags map 1:1 to TOML config paths.
const helpText = `LogWisp %s - log collection, processing, and distribution
Usage:
logwisp [options]
logwisp help | -h | --help
logwisp --version
lw [options]
lw --check [options] Build every pipeline and plugin, report, exit
lw help | -h | --help
lw --version
Subcommands:
logwisp auth <command> SCRAM credentials files, bearer tokens and tcp
sink viewing; logwisp auth -h lists commands
lw auth <command> SCRAM credentials files, bearer tokens and tcp
sink viewing; lw auth -h lists commands
Scalar configuration keys are settable as flags using their TOML paths:
--<path>=<value> e.g. --logging.level=debug
@@ -32,14 +33,26 @@ Logging:
--logging.level=<level> debug|info|warn|error
--logging.file.directory=<path>
Pipelines, plugins and filters are configured in TOML; array-indexed CLI and
environment overrides are unsupported. -- ends option parsing.
Pipelines (replace the file's pipelines; see doc/cli.md):
--pipeline <name> Start a pipeline; earlier specs go to "cli"
--source <spec> TYPE[,key=value...], repeatable
--sink <spec> e.g. http,host=0.0.0.0,port=8080, repeatable
--filter <spec> include|exclude,patterns=RE, repeatable
--format <spec> json|txt|raw[,key=value...]
--rate-limit <spec> rate=N[,burst=N,policy=drop|pass]
--heartbeat <spec> interval_ms=N[,include_stats=true,...]
Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and
'\' escapes ',' '=' '\' in values. -- ends option parsing.
Environment:
LOGWISP_<PATH> Config path, '.' -> '_', uppercase
e.g. LOGWISP_LOGGING_LEVEL=debug
LOGWISP_CONFIG_FILE Configuration file path
LOGWISP_CONFIG_DIR Configuration directory
LOGWISP_PIPELINE, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N], LOGWISP_FILTER[_N],
LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT
One pipeline, specs as the flags (_N adds
more); ignored when a pipeline flag is given
Signals:
SIGINT, SIGTERM Graceful shutdown
cmd/logwisp/main.go → cmd/lw/main.go
+9 -2
View File
@@ -19,7 +19,7 @@ import (
var logger *log.Logger
func main() {
// Before handleHelp, so `logwisp auth <command> -h` prints the auth usage
// Before handleHelp, so `lw auth <command> -h` prints the auth usage
if len(os.Args) > 1 && os.Args[1] == "auth" {
os.Exit(runAuth(os.Args[2:], os.Stdout, os.Stderr))
}
@@ -53,6 +53,9 @@ func main() {
fmt.Println(version.String())
os.Exit(0)
}
if cfg.Check {
os.Exit(checkConfig(cfg))
}
if err := initializeLogger(cfg); err != nil {
FatalError(1, "Failed to initialize logger: %v\n", err)
@@ -63,9 +66,13 @@ func main() {
FatalError(1, "Failed to start logger: %v\n", err)
}
configFile := cfg.ConfigFile
if _, err := os.Stat(configFile); err != nil {
configFile = "none" // flags, environment and built-in defaults only
}
logger.Info("msg", "LogWisp starting",
"version", version.String(),
"config_file", cfg.ConfigFile,
"config_file", configFile,
"log_output", cfg.Logging.Output,
"status_reporter", cfg.StatusReporter,
"auto_reload", cfg.ConfigAutoReload)
cmd/logwisp/output.go → cmd/lw/output.go
View File
File renamed without changes.
cmd/logwisp/reload.go → cmd/lw/reload.go
View File
File renamed without changes.
cmd/logwisp/reload_test.go → cmd/lw/reload_test.go
+44
View File
@@ -2,6 +2,10 @@ package main
import (
"context"
"errors"
"io/fs"
"net"
"os"
"path/filepath"
"strings"
"testing"
@@ -106,3 +110,43 @@ func TestEveryPluginRejectsUnknownKeys(t *testing.T) {
check("sink", name, err)
}
}
// A misspelled spec key reaches the plugin as the nested key it names, so the
// sink refuses to build instead of starting without the TLS it was asked for.
func TestMisspelledSpecKeyFailsPluginConstruction(t *testing.T) {
testLogger(t)
path := filepath.Join(t.TempDir(), "empty.toml")
testutil.WriteFile(t, path, "")
cfg := loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15861,tls.enabeld=true")
svc, err := bootstrapService(context.Background(), cfg)
if err == nil {
svc.Shutdown()
}
if err == nil || !strings.Contains(err.Error(), `unknown key "tls.enabeld"`) {
t.Fatalf("misspelled spec key accepted: %v", err)
}
}
// lw --check builds every plugin, so a bad option fails it, but starts none:
// the port a valid sink names stays free and its log directory uncreated.
func TestCheckBuildsWithoutStarting(t *testing.T) {
testLogger(t)
dir := t.TempDir()
path, out := filepath.Join(dir, "empty.toml"), filepath.Join(dir, "out")
testutil.WriteFile(t, path, "")
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15862",
"--sink", "file,name=check,directory="+out)); code != 0 {
t.Fatalf("valid configuration: exit %d", code)
}
ln, err := net.Listen("tcp4", "127.0.0.1:15862")
if err != nil {
t.Fatalf("check bound the sink's port: %v", err)
}
ln.Close()
if _, err := os.Stat(out); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("check created the file sink's directory: %v", err)
}
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,port=15862,tls.enabled=true")); code != 1 {
t.Fatalf("listener TLS without a certificate: exit %d, want 1", code)
}
}
cmd/logwisp/status.go → cmd/lw/status.go
View File
File renamed without changes.
cmd/logwisp/status_test.go → cmd/lw/status_test.go
View File
File renamed without changes.
cmd/logwisp/test_helpers_test.go → cmd/lw/test_helpers_test.go
+2 -2
View File
@@ -16,10 +16,10 @@ func testLogger(t *testing.T) {
t.Cleanup(func() { logger = previous })
}
func loadTestConfig(t *testing.T, path string) *config.Config {
func loadTestConfig(t *testing.T, path string, args ...string) *config.Config {
t.Helper()
testutil.ClearEnvPrefix(t, "LOGWISP_")
m, err := config.Load([]string{"-c", path, "--quiet", "--status_reporter=false"})
m, err := config.Load(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...))
if err != nil {
t.Fatal(err)
}
config/logwisp.toml
+15 -15
View File
@@ -10,7 +10,7 @@
### http_client/tcp_client sinks are not available in this version. TLS is
### available on every network source and sink (the [...tls] blocks below), and
### the [...auth] blocks authenticate peers by certificate (mtls) or password
### (scram; credentials via `logwisp auth`). Unknown keys anywhere in this file
### (scram; credentials via `lw auth`). Unknown keys anywhere in this file
### or in any plugin config are rejected at startup.
###
### Environment overrides use the LOGWISP_ prefix
@@ -87,7 +87,7 @@ name = "default"
# interval_ms = 1000 # Minimum 100
# include_timestamp = false
# include_stats = false
# format = "txt" # txt|json|raw ("comment" is rejected)
# format = "txt" # accepted, no effect: heartbeats use flow.format
###============================================================================
### TLS (shared shape; applies to every network source and sink)
@@ -119,7 +119,7 @@ name = "default"
### On the http sink the block gates stream_path AND status_path.
### Listeners (tcp/http sinks, tcp_chain/http_chain sources):
### mtls authorizes the client certificate; REQUIRES tls.client_auth.
### scram admits the users in credentials_file (`logwisp auth add-user`).
### scram admits the users in credentials_file (`lw auth add-user`).
### Dialers (tcp_chain/http_chain sinks):
### mtls pins the server identity.
### scram logs in as username with the password in password_file.
@@ -192,7 +192,7 @@ from = "end" # "end" or "start" of a newly discovered
# id = "chain_in"
# type = "tcp_chain"
# [pipelines.plugin_sources.config]
# host = "0.0.0.0" # IPv4 only
# host = "0.0.0.0" # IPv4 only; "::" or "::1": IPv6 only
# port = 9440 # Required
# buffer_size = 1000
# max_connections = 0 # 0 = unlimited
@@ -212,14 +212,14 @@ from = "end" # "end" or "start" of a newly discovered
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# node_binding = "force" # none | assert | force; overrides trust_node
# credentials_file = "" # scram: written by `logwisp auth add-user`
# credentials_file = "" # scram: written by `lw auth add-user`
## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks)
# [[pipelines.plugin_sources]]
# id = "hchain_in"
# type = "http_chain"
# [pipelines.plugin_sources.config]
# host = "0.0.0.0"
# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only)
# port = 9441 # Required
# ingest_path = "/ingest" # Must start with "/"
# buffer_size = 1000
@@ -238,7 +238,7 @@ from = "end" # "end" or "start" of a newly discovered
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# node_binding = "force" # none | assert | force; overrides trust_node
# credentials_file = "" # scram: written by `logwisp auth add-user`
# credentials_file = "" # scram: written by `lw auth add-user`
# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes
###============================================================================
@@ -271,14 +271,14 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# buffer_size = 1000
# flush_interval_ms = 100
## HTTP sink (SSE streaming server + JSON status endpoint; IPv4 clients only)
## HTTP sink (SSE streaming server + JSON status endpoint)
## Without an auth block both endpoints are UNAUTHENTICATED and the stream sends
## Access-Control-Allow-Origin: *. Bind to a trusted interface.
# [[pipelines.plugin_sinks]]
# id = "http_out"
# type = "http"
# [pipelines.plugin_sinks.config]
# host = "0.0.0.0"
# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only)
# port = 8081 # Required
# stream_path = "/stream" # Must start with "/"
# status_path = "/status" # Must differ from stream_path
@@ -299,16 +299,16 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# identity = "cn" # mtls default; under scram unset, set binds the cert to the user
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# credentials_file = "" # scram: log in with `logwisp auth token`
# credentials_file = "" # scram: log in with `lw auth token`
# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes
# trusted_proxies = [] # scram: proxies ending the browsers' TLS; tls may then be off
## TCP sink (streaming server, IPv4 clients only)
## TCP sink (streaming server)
# [[pipelines.plugin_sinks]]
# id = "tcp_out"
# type = "tcp"
# [pipelines.plugin_sinks.config]
# host = "0.0.0.0"
# host = "0.0.0.0" # or "::" for IPv6 (IPv6 only)
# port = 9090 # Required
# buffer_size = 1000
# client_buffer_size = 256
@@ -327,7 +327,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# identity = "cn" # mtls default; under scram unset, set binds the cert to the user
# allow = [] # mtls: exact identities; empty = any the CA issued
# allow_patterns = [] # mtls: RE2, anchor them yourself
# credentials_file = "" # scram: readers use `logwisp auth stream`
# credentials_file = "" # scram: readers use `lw auth stream`
## TCP chain sink (stdlib client; forwards to downstream tcp_chain source)
## Do NOT point a chain sink at a chain source in the SAME pipeline: entries
@@ -336,7 +336,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# id = "chain_out"
# type = "tcp_chain"
# [pipelines.plugin_sinks.config]
# host = "10.0.0.1" # Required
# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1"
# port = 9440 # Required
# node = "" # origin label, default hostname; preserved across hops
# buffer_size = 1000
@@ -367,7 +367,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# id = "hchain_out"
# type = "http_chain"
# [pipelines.plugin_sinks.config]
# host = "10.0.0.1" # Required
# host = "10.0.0.1" # Required; IPv6 bare, e.g. "fd00::1"
# port = 9441 # Required
# ingest_path = "/ingest"
# node = "" # origin label, default hostname; preserved across hops
deploy/README.md
+31
View File
@@ -0,0 +1,31 @@
# deploy
`lw-deploy.sh` writes a LogWisp configuration for one node and deploys it: in a
docker or podman container, as a systemd or FreeBSD rc.d service (on the host
or inside a jail), or as a configuration you run yourself. It is POSIX sh,
written for FreeBSD's `/bin/sh` as much as for dash.
```sh
deploy/lw-deploy.sh # asks, with inline menus
deploy/lw-deploy.sh -h # every flag
deploy/lw-deploy.sh --dry-run ... # print the configuration and commands only
deploy/lw-deploy.sh --yes ... # no questions: defaults, or fail on a missing value
```
- Roles
- `edge`: tails log directories, forwards the entries to an aggregator
- `aggregator`: receives from edges; writes files, serves HTTP and TCP streams
- `standalone`: tails log directories; writes files, serves streams
- Runtimes
- `docker`, `podman`: a hardened container of the `logwisp` image (Linux)
- `native`: systemd on Linux, rc.d on FreeBSD, `--jail NAME` for a jail
- `manual`: the configuration only, for `lw -c`
- Every question has a flag. Before changing anything the script prints its
plan and the command line that repeats it without questions, ready for the
next node. It starts nothing that `lw --check` rejects.
- The native runtimes install the service files of `deploy/package/`.
- Not covered, and printed by the script: filters, rate limits, heartbeat,
several pipelines, proxy mode, ACLs, certificate creation. Edit the generated
`logwisp.toml` for those.
Scenarios and what each runtime writes: [doc/deployment.md](../doc/deployment.md).
deploy/lw-deploy.sh
Executable
+1331
View File
File diff suppressed because it is too large. Load diff
deploy/package/arch/PKGBUILD
+46
View File
@@ -0,0 +1,46 @@
# Maintainer: <name> <email>
# Skeleton: source and checksum wait for a tagged release; run namcap and
# makepkg --printsrcinfo > .SRCINFO before publishing to the AUR.
pkgname=logwisp
pkgver=0.21.0
pkgrel=1
pkgdesc='Pipeline-based log transport with TLS and SCRAM authentication'
arch=('x86_64' 'aarch64')
url='https://github.com/lixenwraith/logwisp'
license=('BSD-3-Clause')
depends=('glibc')
makedepends=('go')
backup=('etc/logwisp/logwisp.toml')
source=("$pkgname-$pkgver.tar.gz::$url/archive/v$pkgver.tar.gz")
sha256sums=('SKIP')
prepare() {
cd "$pkgname-$pkgver"
export GOPATH="$srcdir/gopath"
go mod download -modcacherw
}
# The Go package guidelines' flags; -linkmode=external goes through GO_LDFLAGS
# because the Makefile's own -ldflags (the version) would override GOFLAGS'.
build() {
cd "$pkgname-$pkgver"
export GOPATH="$srcdir/gopath"
export CGO_CPPFLAGS="$CPPFLAGS"
export CGO_CFLAGS="$CFLAGS"
export CGO_CXXFLAGS="$CXXFLAGS"
export CGO_LDFLAGS="$LDFLAGS"
export GOFLAGS='-buildmode=pie -trimpath -mod=readonly -modcacherw'
make build VERSION="$pkgver" GIT_COMMIT="v$pkgver" GO_LDFLAGS='-linkmode=external'
}
check() {
cd "$pkgname-$pkgver"
export GOPATH="$srcdir/gopath"
go test ./...
}
package() {
cd "$pkgname-$pkgver"
make install DESTDIR="$pkgdir" PREFIX=/usr SYSCONFDIR=/etc
}
deploy/package/freebsd/Makefile
+44
View File
@@ -0,0 +1,44 @@
# Skeleton for sysutils/logwisp. GO_MODULE fetches the module by its canonical
# path, so it waits for the module rename to github.com/lixenwraith/logwisp
# (the -X paths below follow it), a tagged release, a maintainer, and a
# UIDs/GIDs entry for logwisp. files/logwisp.in links to ../logwisp.rc: cp -L.
PORTNAME= logwisp
DISTVERSIONPREFIX= v
DISTVERSION= 0.21.0
CATEGORIES= sysutils
MAINTAINER= maintainer@example.org
COMMENT= Pipeline-based log transport with TLS and SCRAM authentication
WWW= https://github.com/lixenwraith/logwisp
LICENSE= BSD3CLAUSE
LICENSE_FILE= ${WRKSRC}/LICENSE
USES= go:1.27,modules
USE_RC_SUBR= logwisp
SUB_LIST= BINDIR=${PREFIX}/bin
GO_MODULE= github.com/lixenwraith/logwisp
GO_TARGET= ./cmd/lw
GO_BUILDFLAGS= -ldflags="-s -w \
-X logwisp/internal/version.Version=${DISTVERSION} \
-X logwisp/internal/version.GitCommit=${DISTVERSIONFULL}"
USERS= logwisp
GROUPS= logwisp
OPTIONS_DEFINE= DOCS
PORTDOCS= *
post-install:
${INSTALL_MAN} ${WRKSRC}/doc/lw.1 ${STAGEDIR}${PREFIX}/share/man/man1
${MKDIR} ${STAGEDIR}${ETCDIR}
${INSTALL_DATA} ${WRKSRC}/config/logwisp.toml ${STAGEDIR}${ETCDIR}/logwisp.toml.sample
post-install-DOCS-on:
${MKDIR} ${STAGEDIR}${DOCSDIR}/doc
${INSTALL_DATA} ${WRKSRC}/README.md ${STAGEDIR}${DOCSDIR}
${INSTALL_DATA} ${WRKSRC}/doc/*.md ${STAGEDIR}${DOCSDIR}/doc
.include <bsd.port.mk>
deploy/package/freebsd/files/logwisp.in
Symlink
+1
View File
@@ -0,0 +1 @@
../../logwisp.rc
deploy/package/freebsd/pkg-descr
+5
View File
@@ -0,0 +1,5 @@
LogWisp collects log entries from files, standard input or other LogWisp
nodes; rate-limits, filters and formats them; and distributes them to files,
the console, live HTTP (Server-Sent Events) and TCP streams, or downstream
nodes. Every network source and sink supports TLS, mutual TLS and
Argon2id-SCRAM password authentication, and reloads on SIGHUP.
deploy/package/freebsd/pkg-plist
+3
View File
@@ -0,0 +1,3 @@
bin/lw
@sample %%ETCDIR%%/logwisp.toml.sample
share/man/man1/lw.1.gz
deploy/package/logwisp.rc
+47
View File
@@ -0,0 +1,47 @@
#!/bin/sh
# PROVIDE: logwisp
# REQUIRE: LOGIN NETWORKING
# KEYWORD: shutdown
# Add to /etc/rc.conf (sysrc logwisp_enable=YES):
# logwisp_enable "YES" starts LogWisp at boot (default "NO")
# logwisp_config configuration file (default %%PREFIX%%/etc/logwisp/logwisp.toml)
# logwisp_user account lw runs as (default "logwisp")
# logwisp_chdir working directory (default /var/db/logwisp)
# logwisp_args extra lw options, e.g. --logging.level=debug (default empty);
# rc.subr hands logwisp_flags to daemon(8), not to lw
. /etc/rc.subr
name=logwisp
rcvar=logwisp_enable
desc="LogWisp log transport"
load_rc_config $name
: ${logwisp_enable:="NO"}
: ${logwisp_config:="%%PREFIX%%/etc/logwisp/logwisp.toml"}
: ${logwisp_user:="logwisp"}
: ${logwisp_chdir:="/var/db/logwisp"}
: ${logwisp_args:=""}
# daemon(8) records the child's pid, so stop and reload (SIGHUP) signal lw
# itself, and sends its output to syslog under the tag logwisp.
pidfile="/var/run/${name}/${name}.pid"
procname="%%BINDIR%%/lw"
command="/usr/sbin/daemon"
command_args="-f -S -T ${name} -p ${pidfile} ${procname} -c ${logwisp_config} ${logwisp_args}"
required_files="${logwisp_config}"
extra_commands="reload"
start_precmd="${name}_prestart"
# rc.subr starts daemon(8) as logwisp_user, which must own the pidfile directory.
# An existing logwisp_chdir keeps its owner: it may be a shared directory.
logwisp_prestart()
{
install -d -o "${logwisp_user}" -m 0755 "/var/run/${name}"
[ -d "${logwisp_chdir}" ] || install -d -o "${logwisp_user}" -m 0750 "${logwisp_chdir}"
}
run_rc_command "$1"
deploy/package/logwisp.service
+54
View File
@@ -0,0 +1,54 @@
[Unit]
Description=LogWisp log transport
Documentation=man:lw(1) https://github.com/lixenwraith/logwisp
Wants=network-online.target
After=network-online.target
# A broken configuration stops after five tries instead of looping forever.
StartLimitIntervalSec=60
StartLimitBurst=5
[Service]
Type=exec
User=logwisp
Group=logwisp
ExecStart=@BINDIR@/lw -c @SYSCONFDIR@/logwisp/logwisp.toml
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=5s
TimeoutStopSec=15s
# logwisp.tmpfiles creates both directories. A file sink elsewhere needs a
# drop-in with ReadWritePaths=, a source under /home one with ProtectHome=read-only,
# and a port below 1024 CapabilityBoundingSet= and AmbientCapabilities= both
# set to CAP_NET_BIND_SERVICE.
WorkingDirectory=/var/lib/logwisp
ReadWritePaths=/var/lib/logwisp /var/log/logwisp
UMask=0027
NoNewPrivileges=yes
CapabilityBoundingSet=
AmbientCapabilities=
PrivateDevices=yes
PrivateTmp=yes
ProtectClock=yes
ProtectControlGroups=yes
ProtectHome=yes
ProtectHostname=yes
ProtectKernelLogs=yes
ProtectKernelModules=yes
ProtectKernelTunables=yes
ProtectProc=invisible
ProcSubset=pid
ProtectSystem=strict
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
RestrictNamespaces=yes
RestrictRealtime=yes
RestrictSUIDSGID=yes
RemoveIPC=yes
SystemCallArchitectures=native
SystemCallFilter=@system-service
LockPersonality=yes
MemoryDenyWriteExecute=yes
[Install]
WantedBy=multi-user.target
deploy/package/logwisp.sysusers
+2
View File
@@ -0,0 +1,2 @@
# Type Name ID Description Home Shell
u logwisp - "LogWisp log transport" /var/lib/logwisp -
deploy/package/logwisp.tmpfiles
+3
View File
@@ -0,0 +1,3 @@
# Type Path Mode User Group Age Argument
d /var/lib/logwisp 0750 logwisp logwisp - -
d /var/log/logwisp 0750 logwisp logwisp - -
doc/README.md
+31 -20
View File
@@ -7,22 +7,32 @@ streams, or downstream LogWisp nodes.
## Documentation Map
| Document | Contents |
|----------|----------|
| [Installation](installation.md) | Building, installing, and running as a service |
| [Architecture](architecture.md) | Component model, data flow, concurrency, back-pressure |
| [Configuration](configuration.md) | TOML structure, precedence, environment and CLI overrides |
| [Sources](sources.md) | Every input plugin and its options |
| [Sinks](sinks.md) | Every output plugin and its options |
| [Filters](filters.md) | Pattern-based inclusion and exclusion |
| [Formatters](formatters.md) | Output shaping and sanitization |
| [Chaining](chaining.md) | Multi-node topologies and the chain wire protocol |
| [Networking](networking.md) | Listeners, dialers, timeouts, connection limits |
| [Security](security.md) | TLS, mTLS, and peer authorization; threat model and current limits |
| [mTLS Authentication](mtls-auth-plan.md) | Design and rationale for certificate-based authorization |
| [Password Authentication](scram-auth-plan.md) | Design and rationale for Argon2id-SCRAM authentication; mTLS hardening |
| [CLI](cli.md) | Flags, signals, exit codes, `logwisp auth` |
| [Operations](operations.md) | Running, monitoring, tuning, troubleshooting |
- [Installation](installation.md): building, installing, and running as a
service
- [Deployment](deployment.md): `deploy/lw-deploy.sh` for edges, aggregators,
containers, services, jails
- [Architecture](architecture.md): component model, data flow, concurrency,
back-pressure
- [Configuration](configuration.md): TOML structure, precedence, environment
and CLI overrides
- [Sources](sources.md): every input plugin and its options
- [Sinks](sinks.md): every output plugin and its options
- [Filters](filters.md): pattern-based inclusion and exclusion
- [Formatters](formatters.md): output shaping and sanitization
- [Chaining](chaining.md): multi-node topologies and the chain wire protocol
- [Networking](networking.md): listeners, dialers, timeouts, connection limits
- [Security](security.md): TLS, mTLS, and peer authorization; threat model and
current limits
- [mTLS Authentication](mtls-auth-plan.md): design and rationale for
certificate-based authorization
- [Password Authentication](scram-auth-plan.md): design and rationale for
Argon2id-SCRAM authentication; mTLS hardening
- [CLI](cli.md): flags, signals, exit codes, `lw auth`; also the `lw(1)`
manual, [`lw.1`](lw.1)
- [Operations](operations.md): running, monitoring, tuning, troubleshooting
- [To Do](todo.md): planned work in priority order: network access control
and the PROXY protocol, a hardening review of the config libraries,
packaging
A fully annotated configuration covering every option lives at
[`config/logwisp.toml`](../config/logwisp.toml).
@@ -64,7 +74,7 @@ endpoint), `tcp` (broadcast server), `null`, and the chain forwarders
- Authorization by certificate identity, per listener: named peers rather than
everything the CA issued, with the `http` sink's endpoints gated too
- Password authentication (Argon2id-SCRAM) bound to the listener's certificate,
with bearer tokens on HTTP listeners and a `logwisp auth` CLI, and browser
with bearer tokens on HTTP listeners and a `lw auth` CLI, and browser
logins behind a TLS-terminating proxy
- Node binding, so a chain source labels entries from the sender's certificate
identity or username rather than from what the sender claims
@@ -96,16 +106,17 @@ target = "stdout"
```
```bash
logwisp -c config.toml
lw -c config.toml
```
## System Requirements
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
- **Architecture**: amd64
- **Go**: 1.26+ to build from source
- **Go**: 1.27.1+ to build from source
Network sources and sinks bind and dial over IPv4 only.
Network sources and sinks bind and dial IPv4 or IPv6, each keeping strictly to
the family of its host ([Networking](networking.md#address-family)).
## License
doc/architecture.md
+62 -50
View File
@@ -25,29 +25,33 @@ main
Package map:
| Package | Responsibility |
|---------|----------------|
| `cmd/logwisp` | Entry point, help, logger bootstrap, signal loop, status reporter |
| `internal/config` | Typed config schema, loading, top-level validation |
| `internal/service` | Owns the pipeline set; start, stop, shutdown, global stats |
| `internal/pipeline` | Pipeline runtime and per-pipeline plugin registry |
| `internal/flow` | Rate limiter, filter chain invocation, formatting, heartbeat |
| `internal/filter` | Regex filter and filter chain |
| `internal/format` | Adapter over `lixenwraith/log` formatter + sanitizer |
| `internal/source/*` | Source plugins |
| `internal/sink/*` | Sink plugins |
| `internal/plugin` | Global factory registry populated by plugin `init()` |
| `internal/chain` | Chain wire protocol: hello preamble, entry codec, backoff |
| `internal/tlsx` | The single seam between `TLSOptions` and `crypto/tls` |
| `internal/session` | Session manager and per-instance proxy |
| `internal/core` | Shared types (`LogEntry`, `TransportEvent`), capabilities, constants |
| `internal/tokenbucket` | Rate limiter primitive |
| `internal/sanitize` | Standalone hex-escaping helpers |
- `cmd/lw`: entry point, help, logger bootstrap, signal loop, status
reporter.
- `internal/config`: typed config schema, loading, top-level validation.
- `internal/service`: owns the pipeline set; start, stop, shutdown, global
stats.
- `internal/pipeline`: pipeline runtime and per-pipeline plugin registry.
- `internal/flow`: rate limiter, filter chain invocation, formatting,
heartbeat.
- `internal/filter`: regex filter and filter chain.
- `internal/format`: adapter over the `lixenwraith/log` formatter and
sanitizer.
- `internal/source/*`, `internal/sink/*`: the source and sink plugins.
- `internal/plugin`: global factory registry populated by plugin `init()`.
- `internal/chain`: chain wire protocol: hello preamble, entry codec, backoff.
- `internal/tlsx`: the single seam between `TLSOptions` and `crypto/tls`.
- `internal/authz`: the single seam between `AuthOptions` and the network
plugins: `Admit`, `AuthorizeRequest`, `Greet`, `Prepare`.
- `internal/session`: session manager and per-instance proxy.
- `internal/core`: shared types (`LogEntry`, `TransportEvent`), capabilities,
constants.
- `internal/tokenbucket`: rate limiter primitive.
- `internal/sanitize`: standalone hex-escaping helpers.
## Plugin Registration
Every plugin registers itself in an `init()` function, and
`cmd/logwisp/bootstrap.go` blank-imports each package to trigger those
`cmd/lw/bootstrap.go` blank-imports each package to trigger those
`init()`s. Adding a plugin therefore means writing the package, calling
`plugin.RegisterSource` / `plugin.RegisterSink`, and adding one blank import.
@@ -71,15 +75,14 @@ registry rejects a second instance of any such type.
`LogEntry` fields:
| Field | Purpose |
|-------|---------|
| `Time` | Entry timestamp |
| `Node` | Origin node label for chained topologies; stamped at the first hop, preserved by relays |
| `Source` | Origin identifier within the node (filename, plugin id, …) |
| `Level` | `DEBUG`/`INFO`/`WARN`/`ERROR`/`TRACE`, when detected |
| `Message` | Log content |
| `Fields` | Optional structured metadata as raw JSON |
| `RawSize` | Original byte size, used by the entry-size cap |
- `Time`: entry timestamp.
- `Node`: origin node label for chained topologies; stamped at the first hop,
preserved by relays.
- `Source`: origin identifier within the node (filename, plugin id, …).
- `Level`: `DEBUG`, `INFO`, `WARN`, `ERROR` or `TRACE`, when detected.
- `Message`: log content.
- `Fields`: optional structured metadata as raw JSON.
- `RawSize`: original byte size, used by the entry-size cap.
Carrying `Entry` alongside `Payload` is what makes chain sinks
format-independent: a `tcp_chain` or `http_chain` sink re-serializes the
@@ -88,13 +91,16 @@ structured entry rather than shipping whatever text the local formatter chose.
### Back-pressure and drops
There is exactly one drop policy and it is not configurable: **never block**.
Each stage drops and counts instead of waiting:
| Stage | Full-buffer behaviour | Counter |
|-------|----------------------|---------|
| Source → subscriber | Drop the entry | source `dropped_entries` |
| Flow | Drop on rate limit, filter, or format error | `flow.total_dropped` |
| Pipeline → sink | Drop for that sink only | pipeline `total_dropped_by_sink` |
| TCP/HTTP sink → client queue | Drop for that client only | sink `dropped_writes` |
- Source → subscriber: a full channel drops the entry; source
`dropped_entries`.
- Flow: the rate limiter, a filter rejection or a format error drops it;
`flow.total_dropped`.
- Pipeline → sink: a full sink input drops it for that sink only; pipeline
`total_dropped_by_sink`.
- TCP/HTTP sink → client queue: a full queue drops it for that client only;
sink `dropped_writes`.
The `tcp_chain` sink is the one deliberate exception. It holds a line across
reconnects until it is written or the process shuts down, so a downstream
@@ -126,17 +132,20 @@ on a non-retryable response or on shutdown (`dropped_batches`).
## Network Architecture
All listeners bind `tcp4` and all dialers dial `tcp4`. IPv6 clients cannot
connect; this is deliberate, not an oversight.
Each listener and dialer keeps strictly to the family of its host literal:
`tcp4` for IPv4, IPv6-only `tcp6` for IPv6 (`::` too); a hostname resolves.
See [Networking](networking.md#address-family).
| Plugin | Role | Protocol |
|--------|------|----------|
| `tcp` sink | Listener | Raw broadcast of formatted payloads |
| `http` sink | Listener | HTTP/1.1 SSE; HTTP/2 negotiated via ALPN when TLS is on |
| `tcp_chain` source | Listener | Chain protocol, persistent NDJSON stream |
| `http_chain` source | Listener | Chain protocol, NDJSON batches over POST |
| `tcp_chain` sink | Dialer | Chain protocol, persistent stream, auto-reconnect |
| `http_chain` sink | Dialer | Chain protocol, batched POST with retry |
The network plugins, by role:
- Listeners
- `tcp` sink: raw broadcast of formatted payloads.
- `http` sink: HTTP/1.1 SSE; HTTP/2 negotiated via ALPN when TLS is on.
- `tcp_chain` source: chain protocol, persistent NDJSON stream.
- `http_chain` source: chain protocol, NDJSON batches over POST.
- Dialers
- `tcp_chain` sink: chain protocol, persistent stream, auto-reconnect.
- `http_chain` sink: chain protocol, batched POST with retry.
TLS is built in exactly one place, `internal/tlsx`, which exposes
`Server(opts)` for listeners and `Client(opts, host)` for dialers. See
@@ -176,17 +185,20 @@ Reload (signal or file watch) rebuilds the entire service:
Because this is a full rebuild, listening sockets close and reopen and all
clients are disconnected. Application logging is configured once at startup and
is **not** re-applied on reload.
Watch errors do not rebuild services. Queued changes are combined, and an unchanged
pipeline/status snapshot is skipped. Signals always rebuild for certificate rotation.
Failed candidates release their session workers. Once the old service has stopped,
a bind/start failure requires another corrected reload or process restart.
Watch errors do not rebuild services. Queued changes are combined, and an
unchanged pipeline/status snapshot is skipped. Signals always rebuild for
certificate rotation. Failed candidates release their session workers. Once the
old service has stopped, a bind/start failure requires another corrected reload
or process restart.
## Resource Management
- Every buffer is bounded; the drop-not-block policy keeps memory flat under
load.
- Network sinks and chain sources accept a `max_connections` cap. Admission is
a load-then-check, so a burst can over-admit by roughly one connection.
- The `tcp` and `http` sinks and the `tcp_chain` source accept a
`max_connections` cap. Admission is a load-then-check, so a burst can
over-admit by roughly one connection.
- Chain listeners bound a single line at `core.MaxLogEntryBytes` (1 MiB); an
oversized line is a protocol violation and terminates the connection.
- The `http_chain` source caps each request body at `max_body_bytes`.
doc/chaining.md
+29 -24
View File
@@ -36,24 +36,25 @@ Chained entries carry a `node` label identifying where they originated.
The label comes from the `node` option, defaulting to `os.Hostname()`.
- A chain **source** either honours the sender's label or overrides it,
according to `trust_node`:
| `trust_node` | Behaviour |
|--------------|-----------|
| `true` (default) | Keep the label the sender declared; fall back to the remote address when absent |
| `false` | Always overwrite with the sender's remote address |
- `true` (default): keep the label the sender declared; fall back to the
remote address when absent.
- `false`: always overwrite with the sender's remote address.
Relays preserve `node`, so a label survives any number of hops and identifies
the original producer rather than the last relay.
With an `auth` block the source can instead bind the label to the sender's
authenticated identity — its certificate identity under `mtls`, its username
under `scram` — which overrides `trust_node` entirely:
under `scram`. `assert` and `force` take the connection label over from
`trust_node`; only `force` also overrides the per-entry labels.
`auth.node_binding`:
| `auth.node_binding` | Connection label | Per-entry `node` field |
|---------------------|------------------|------------------------|
| `none` | `trust_node` governs | `trust_node` governs |
| `assert` | Must equal the identity, or the peer is rejected | `trust_node` governs |
| `force` (default with an `auth` block) | The identity | Overwritten with the identity |
- `none`: `trust_node` governs the connection label and the per-entry `node`
field.
- `assert`: the connection label must equal the identity, or the peer is
rejected; `trust_node` governs the per-entry `node` field.
- `force` (default with an `auth` block): the identity is the connection label
and overwrites every per-entry `node` field.
Pick `force` at an ingest boundary you do not trust — it is the only setting
where a compromised edge cannot mislabel its entries, including through the
@@ -116,13 +117,13 @@ binary ignores it and the dialer gives up after 10 s.
Batches of NDJSON delivered by `POST`, with the preamble expressed as headers.
| Header | Direction | Meaning |
|--------|-----------|---------|
| `X-Logwisp-Protocol` | request | Protocol version; must be `1` |
| `X-Logwisp-Node` | request | Origin node label |
| `Content-Type` | request | `application/x-ndjson` |
| `Authorization` | request | `Bearer <token>`, under `scram` |
| `X-Logwisp-Accepted` | response | Number of entries ingested |
- Request headers
- `X-Logwisp-Protocol`: the protocol version; must be `1`.
- `X-Logwisp-Node`: the origin node label.
- `Content-Type`: `application/x-ndjson`.
- `Authorization`: `Bearer <token>`, under `scram`.
- Response header
- `X-Logwisp-Accepted`: the number of entries ingested.
Responses: `204` on success, `400` for a bad protocol version or a malformed
body, `413` when the body cap is exceeded, `405` for a non-`POST` method, `401`
@@ -152,10 +153,14 @@ in at ingest.
## Delivery Semantics
| Transport | Guarantee | Failure behaviour |
|-----------|-----------|-------------------|
| `tcp_chain` | Per-line, held across reconnects | Retries with exponential backoff plus ±20 % jitter until written or shutdown; back-pressure appears upstream as `total_dropped_by_sink` |
| `http_chain` | At-least-once per batch | Retries transport errors, `408`, `429`, `5xx` and, under `scram`, failed logins and `401`; drops on any other non-2xx (`dropped_batches`) |
- `tcp_chain`: per line, each held across reconnects.
- On failure it retries with exponential backoff plus ±20 % jitter until
the line is written or the sink shuts down.
- Back-pressure appears upstream as `total_dropped_by_sink`.
- `http_chain`: at least once per batch.
- It retries transport errors, `408`, `429`, `5xx` and, under `scram`, failed
logins and `401`.
- It drops the batch on any other non-2xx, counted in `dropped_batches`.
`http_chain` batches can be delivered twice when a successful request's response
is lost. There is no de-duplication downstream; design your consumers to
@@ -266,5 +271,5 @@ optional, and entries are labelled with the username. See
otherwise-idle links and their sessions warm.
- **Ports** used by the bundled test scripts: `15801` tcp_chain ingest, `15802`
http_chain ingest, `15803` tcp sink, `15804` http sink.
- **Use `127.0.0.1`, not `localhost`**, when testing locally: all listeners and
dialers are IPv4-only, and `localhost` may resolve to `::1`.
- **Dial the listener's own literal** when testing locally: a listener on
`127.0.0.1` takes IPv4 only, and `localhost` may resolve to `::1`.
doc/cli.md
+177 -86
View File
@@ -1,19 +1,21 @@
# Command Line Interface
```
logwisp [options]
logwisp auth <command> [flags]
logwisp help | -h | --help
logwisp --version
lw [options]
lw --check [options]
lw auth <command> [flags]
lw help | -h | --help
lw --version
```
`logwisp auth` manages SCRAM credentials and logs in to `scram` listeners; see
[below](#logwisp-auth). There is no certificate-generation command: use
`lw auth` manages SCRAM credentials and logs in to `scram` listeners; see
[below](#lw-auth). There is no certificate-generation command: use
`openssl` or your PKI tooling — see [Security](security.md#enabling-mtls).
## Options
Any scalar configuration key is settable as a flag using its TOML path:
Any scalar configuration key is settable as a flag using its TOML path;
pipelines have [their own flags](#pipelines):
```
--<path>=<value> e.g. --logging.level=debug
@@ -23,49 +25,92 @@ Any scalar configuration key is settable as a flag using its TOML path:
### Common
| Flag | Description | Default |
|------|-------------|---------|
| `-c <path>` | Configuration file | `./logwisp.toml` |
| `--config <path>` / `--config=<path>` | Configuration file | `./logwisp.toml` |
| `--quiet` | Suppress all application output | `false` |
| `--status_reporter=<bool>` | Periodic status logging | `true` |
| `--auto_reload=<bool>` | Reload config when the file changes | `false` |
| `--version` | Print version and exit | — |
| `-h`, `--help`, `help` | Print usage and exit | — |
- `-c <path>`, `--config <path>`, `--config=<path>`: the configuration file,
default `./logwisp.toml`
- `--check`: build every pipeline and plugin, bind nothing, report and exit;
see [Usage Patterns](#usage-patterns)
- `--quiet`: suppress all application output, default `false`
- `--status_reporter=<bool>`: periodic status logging, default `true`
- `--auto_reload=<bool>`: reload when the configuration file changes, default
`false`
- `--version`: print the version and exit
- `-h`, `--help`, `help`: print usage and exit
The last file-selection flag wins. `-c=<path>` also works. A missing or empty
path returns an error, and `--` ends option parsing.
### Logging
| Flag | Values |
|------|--------|
| `--logging.output` | `file`, `stdout`, `stderr`, `split`, `all`, `none` |
| `--logging.level` | `debug`, `info`, `warn`, `error` |
| `--logging.format` | `raw`, `txt`, `json` |
| `--logging.sanitization` | `raw`, `json`, `txt`, `shell` |
| `--logging.file.directory` | path |
| `--logging.file.name` | string |
| `--logging.file.max_size_mb` | integer |
| `--logging.file.max_total_size_mb` | integer |
| `--logging.file.retention_hours` | float |
- `--logging.output`: `file`, `stdout`, `stderr`, `split`, `all` or `none`
- `--logging.level`: `debug`, `info`, `warn` or `error`
- `--logging.format`: `raw`, `txt` or `json`
- `--logging.sanitization`: `raw`, `json`, `txt` or `shell`
- `--logging.file.directory`: a path
- `--logging.file.name`: a string
- `--logging.file.max_size_mb`, `--logging.file.max_total_size_mb`: integers
- `--logging.file.retention_hours`: a float
`--logging.console.target` is accepted but has no effect; the console
destination is derived from `--logging.output`.
### Pipelines
Pipelines, sources, sinks, and filters **cannot** be configured from the command
line. Array-indexed paths such as `--pipelines.0.name=app` or
`--pipelines.0.plugin_sinks.0.type=null` are reported as unrecognized and
ignored:
Pipeline flags define whole pipelines. They replace the configuration file's
pipelines and the built-in default; every other key keeps its precedence.
- `--pipeline NAME` starts a pipeline; specs before the first one go to a
pipeline named `cli`
- `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable
- `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline
- both `--flag SPEC` and `--flag=SPEC` work; `--` ends options
A SPEC is a comma-separated list:
- source, sink, filter and format specs start with a TYPE
- the plugin type: [Sources](sources.md), [Sinks](sinks.md)
- `include` or `exclude`: [Filters](filters.md)
- `json`, `txt` or `raw`: [Formatters](formatters.md)
- rate-limit and heartbeat specs have no TYPE
- the rest are `key=value` pairs with the TOML keys of the plugin's `config`
table or of the [flow stage](configuration.md#flow-stages)
- a dotted key reaches a nested table: `tls.cert_file=...`, `auth.type=scram`
- a repeated key makes a list: `patterns=ERROR,patterns=WARN`
- each value is one list entry, commas included: `auth.allow_patterns=^a{1\,3}$`
is one regex, `auth.allow=a,auth.allow=b` two entries
- values convert to the option's type: `port=8080`, `raw=true`
- `\` escapes `,`, `=` and `\` in a value; any other backslash stays, so regex
escapes such as `\d` pass unchanged
- `id=NAME` names a source or sink; the default is its TYPE, then `TYPE_2`,
`TYPE_3`, ...
- naming a stage turns it on: `--rate-limit` defaults `policy` to `drop`,
`--heartbeat` defaults `enabled` to `true`
Errors name the flag and the offending part, and nothing starts:
```
Warning: unrecognized flags ignored: [pipelines.0.name]
--sink http,port: missing "="
--rate-limit rate=100,polcy=drop: unknown key "polcy"
```
Use a configuration file. Older documentation described CLI pipeline overrides
that the current loader does not implement.
A misspelled plugin key fails when the plugin is built, before any listener
opens: `failed to create sink http: ... unknown key "tls.enabeld"`.
```bash
# tail a directory, serve it over SSE
lw --source 'file,directory=/var/log/app,pattern=*.log' \
--sink http,host=0.0.0.0,port=8080
# errors and warnings as JSON, over TLS with SCRAM logins
lw --source file,directory=/var/log/app \
--filter include,patterns=ERROR,patterns=WARN --format json \
--sink "http,port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\
tls.enabled=true,tls.cert_file=/etc/logwisp/server.crt,tls.key_file=/etc/logwisp/server.key"
# two pipelines
lw --pipeline app --source file,directory=/var/log/app --sink console \
--pipeline relay --source tcp_chain,port=9000 \
--sink file,directory=/var/log/relay,name=relay
```
## Environment Variables
@@ -83,28 +128,54 @@ forms when upgrading; see [Configuration](configuration.md#environment-variables
The path resolver reads these variables directly:
| Variable | Effect |
|----------|--------|
| `LOGWISP_CONFIG_FILE` | Configuration file path; joined onto `LOGWISP_CONFIG_DIR` when both are set |
| `LOGWISP_CONFIG_DIR` | Configuration directory; alone, implies `<dir>/logwisp.toml` |
- `LOGWISP_CONFIG_FILE`: the configuration file path, joined onto
`LOGWISP_CONFIG_DIR` when both are set
- `LOGWISP_CONFIG_DIR`: the configuration directory; alone, it implies
`<dir>/logwisp.toml`
As with flags, array elements cannot be set this way.
### Pipeline Variables
One pipeline can come from the environment, with the [SPEC](#pipelines) syntax.
Any pipeline flag on the command line makes LogWisp ignore all of them.
- `LOGWISP_PIPELINE`: the pipeline's name, default `cli`
- `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each
- `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered
one in numeric order; likewise `LOGWISP_SINK_N` and `LOGWISP_FILTER_N`
- `LOGWISP_FORMAT`, `LOGWISP_RATE_LIMIT`, `LOGWISP_HEARTBEAT`
- an empty variable counts as unset
A container needs no configuration file:
```bash
docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \
-e LOGWISP_PIPELINE=app \
-e LOGWISP_SOURCE='file,directory=/logs,pattern=*.log' \
-e LOGWISP_FILTER='exclude,patterns=DEBUG' \
-e LOGWISP_SINK='http,host=0.0.0.0,port=8080' \
-e LOGWISP_SINK_1=console \
logwisp
```
## Precedence
1. Command-line flags
2. Environment variables
3. Configuration file
4. Built-in defaults
- Pipelines
- pipeline flags, if any are given
- else pipeline variables, if any are set
- else the configuration file's `[[pipelines]]`
- else the built-in default
- Every other key
1. command-line flags
2. environment variables
3. configuration file
4. built-in defaults
A reload rereads the file and keeps the command-line or environment pipelines.
## Signals
| Signal | Action |
|--------|--------|
| `SIGINT` | Graceful shutdown |
| `SIGTERM` | Graceful shutdown |
| `SIGHUP` | Reload configuration |
| `SIGUSR1` | Reload configuration |
- `SIGINT`, `SIGTERM`: graceful shutdown
- `SIGHUP`, `SIGUSR1`: reload the configuration
`SIGHUP` is ignored during startup, before the signal handler is installed, so
LogWisp survives a terminal hang-up like `nohup`. Once running, it triggers a
@@ -115,11 +186,10 @@ service untouched; see [Configuration](configuration.md#hot-reload).
## Exit Codes
| Code | Meaning |
|------|---------|
| `0` | Clean shutdown, or `--version` / `--help` |
| `1` | General error: config load or validation failure, logger init failure, service bootstrap failure |
| `2` | Explicitly requested configuration file not found |
- `0`: clean shutdown, `--version` / `--help`, or a valid `--check`
- `1`: general error: a configuration load or validation failure (`--check`
included), a logger init failure, a service bootstrap failure
- `2`: an explicitly requested configuration file is not found
Exit code 2 applies only when the file was named explicitly (`-c`,
`--config=`, or the `LOGWISP_CONFIG_*` variables). A missing discovered default
@@ -133,8 +203,9 @@ a rate limit of 5 entries/second with a burst of 10 and `policy = "drop"`, and a
`console` sink on stdout. It is a self-demonstrating idle mode, not a useful
production configuration.
Note that as soon as your file defines `[[pipelines]]`, that entire default
pipeline — rate limit included — is replaced rather than merged.
As soon as the file defines `[[pipelines]]`, or pipeline flags or variables are
given, that entire default pipeline — rate limit included — is replaced rather
than merged.
## Usage Patterns
@@ -142,26 +213,29 @@ pipeline — rate limit included — is replaced rather than merged.
```bash
# verbose, everything to stderr
logwisp -c dev.toml --logging.output=stderr --logging.level=debug
lw -c dev.toml --logging.output=stderr --logging.level=debug
# no config at all: synthetic generator to stdout
logwisp
lw
```
**Configuration check**
```bash
# starts the service; a config error exits non-zero before any pipeline runs
logwisp -c /etc/logwisp/logwisp.toml --logging.level=debug
```
`lw --check` validates without running: it builds every pipeline and plugin as
a start would (options, TLS files, credentials files, startup warnings), but
binds no port and opens, reads or creates no log file; it prints
`configuration ok` and exits 0, or the error and exits 1 (2 when a named file
is missing, see [Exit Codes](#exit-codes)):
There is no dry-run or validate-only mode. The closest approximation is starting
with debug logging and stopping once the pipelines report as started.
```bash
lw --check -c /etc/logwisp/logwisp.toml
lw --check --source file,directory=/var/log/app --sink http,port=8080
```
**Production**
```bash
logwisp -c /etc/logwisp/logwisp.toml --logging.output=file
lw -c /etc/logwisp/logwisp.toml --logging.output=file
```
Run under a supervisor (systemd, rc.d) rather than backgrounding it — there is
@@ -171,29 +245,38 @@ no `--background` flag; earlier releases had one and it was removed. See
**Reload**
```bash
kill -HUP $(pidof logwisp)
kill -USR1 $(pidof logwisp)
kill -HUP $(pidof lw)
kill -USR1 $(pidof lw)
```
## `logwisp auth`
## `lw auth`
Manages the credentials files of `auth.type = "scram"` listeners and logs in to
them as a viewer. See
[Password Authentication](security.md#password-authentication-scram).
| Command | Does |
|---------|------|
| `add-user -credentials FILE -user NAME [-password-file FILE] [-generate]` | Adds a user, or replaces its password |
| `remove-user -credentials FILE -user NAME` | Removes a user; refuses the last one |
| `token -url https://HOST:PORT[/PATH] -user NAME -password-file FILE [-unbound] [TLS flags]` | Logs in to an `http` sink or `http_chain` source and prints a bearer token |
| `stream -addr HOST:PORT -user NAME -password-file FILE [TLS flags]` | Logs in to a `tcp` sink and copies its stream to stdout until interrupted |
```
lw auth add-user -credentials FILE -user NAME [-password-file FILE] [-generate]
lw auth remove-user -credentials FILE -user NAME
lw auth token -url https://HOST:PORT[/PATH] -user NAME -password-file FILE
[-unbound] [TLS flags]
lw auth stream -addr HOST:PORT -user NAME -password-file FILE [TLS flags]
```
`logwisp auth <command> -h` lists a command's flags. The exit status is `0` on
- `add-user`: adds a user, or replaces its password
- `remove-user`: removes a user; refuses the last one
- `token`: logs in to an `http` sink or `http_chain` source and prints a
bearer token
- `stream`: logs in to a `tcp` sink and copies its stream to stdout until
interrupted
`lw auth <command> -h` lists a command's flags. The exit status is `0` on
success, `1` on failure and `2` on a usage error.
**`add-user`** creates the credentials file, with a fresh `decoy_key`, when it
does not exist or is empty (create it empty first to choose its owner). The password comes from `-password-file` when that file exists
(at least 8 bytes; one trailing line break is trimmed). Otherwise a random
does not exist or is empty (create it empty first to choose its owner). The
password comes from `-password-file` when that file exists (at least 8 bytes;
one trailing line break is trimmed). Otherwise a random
26-character password (130 bits) is generated and written to `-password-file`,
or printed once to stdout when there is none. Replacing an existing user's
password needs an existing `-password-file` or `-generate`, so a mistyped path
@@ -215,29 +298,37 @@ There is no flag to skip verification: an unverified server could relay the
login. Redirects are not followed. `-unbound` logs in to an `http` sink behind
a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's
certificate across the two requests; only then may `-url` carry the path the
proxy mounts LogWisp at. `stream` exits `0`
on `SIGINT` or `SIGTERM`, and `1` when the server ends the stream (a reload or
shutdown).
proxy mounts LogWisp at. `stream` exits `0` on `SIGINT` or `SIGTERM`, and `1`
when the server ends the stream (a reload or shutdown).
Addresses, as for the plugins ([Networking](networking.md#address-family)):
- An IPv6 address goes in brackets: `-addr [::1]:PORT`,
`-url https://[::1]:PORT`; in a URL a link-local zone is escaped,
`https://[fe80::1%25eth0]:PORT`.
- The dial keeps to the address's family; a hostname resolves.
- `-server-name` defaults to the host; an address, without brackets or zone,
must then be among the certificate's IP SANs.
```bash
# listener host: a file the service user can read, users, then apply
install -m 0640 -o root -g logwisp /dev/null /etc/logwisp/users.toml
logwisp auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \
lw auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \
-password-file /etc/logwisp/edge-01.pass
logwisp auth add-user -credentials /etc/logwisp/users.toml -user viewer \
lw auth add-user -credentials /etc/logwisp/users.toml -user viewer \
-password-file viewer.pass
kill -HUP $(pidof logwisp)
kill -HUP $(pidof lw)
# rotate: new password into the file; SIGHUP, deploy the file, SIGHUP the edge
logwisp auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \
lw auth add-user -credentials /etc/logwisp/users.toml -user edge-01 \
-password-file /etc/logwisp/edge-01.pass -generate
# http sink status, with the token kept out of curl's argv
curl --cacert ca.crt -H @<(printf 'Authorization: Bearer %s\n' \
"$(logwisp auth token -url https://HOST:PORT -user viewer \
"$(lw auth token -url https://HOST:PORT -user viewer \
-password-file viewer.pass -ca-file ca.crt)") https://HOST:PORT/status
# follow a tcp sink
logwisp auth stream -addr HOST:PORT -user viewer -password-file viewer.pass \
lw auth stream -addr HOST:PORT -user viewer -password-file viewer.pass \
-ca-file ca.crt
```
doc/configuration.md
+94 -77
View File
@@ -5,16 +5,18 @@ and its default ships as [`config/logwisp.toml`](../config/logwisp.toml).
## Configuration Precedence
Sources are merged in this order, highest priority first:
1. Command-line flags
2. Environment variables
3. Configuration file
4. Built-in defaults
The `pipelines` array is replaced wholesale, not merged: as soon as your file
defines `[[pipelines]]`, the built-in default pipeline (and its default rate
limit and formatter) disappears entirely.
- Pipelines come from the first of these that defines any, replacing the rest
wholesale (the default pipeline's rate limit and formatter included):
1. pipeline flags: `--source`, `--sink`, ... ([CLI](cli.md#pipelines))
2. pipeline variables: `LOGWISP_SOURCE`, ...
([CLI](cli.md#pipeline-variables))
3. the file's `[[pipelines]]`
4. the built-in default pipeline
- Every other key merges, highest priority first:
1. command-line flags
2. environment variables
3. configuration file
4. built-in defaults
## File Location
@@ -36,11 +38,14 @@ runtime metadata and cannot be redirected by a `config_file` key inside the file
## Global Settings
| Setting | Type | Default | Description |
|---------|------|---------|-------------|
| `quiet` | bool | `false` | Disable all application logging and console diagnostics |
| `status_reporter` | bool | `true` | Emit a periodic status report every 30 s at DEBUG level |
| `auto_reload` | bool | `false` | Watch the config file and reload pipelines on change |
Top-level keys, each as type and default:
- `quiet` (bool, `false`): disable all application logging and console
diagnostics.
- `status_reporter` (bool, `true`): emit a status report every 30 s at DEBUG
level.
- `auto_reload` (bool, `false`): watch the config file and reload pipelines on
change.
`--version` prints version information and exits; it is not a persistent
setting.
@@ -69,14 +74,12 @@ retention_hours = 168.0
### Output modes
| Mode | Behaviour |
|------|-----------|
| `file` | Files only |
| `stdout` | Standard output only |
| `stderr` | Standard error only |
| `split` | DEBUG/INFO to stdout, WARN/ERROR to stderr |
| `all` | Files plus split console |
| `none` | No application logging |
- `file`: files only.
- `stdout`: standard output only.
- `stderr`: standard error only.
- `split`: DEBUG/INFO to stdout, WARN/ERROR to stderr.
- `all`: files plus split console.
- `none`: no application logging.
`[logging.file]` applies only to the `file` and `all` modes.
@@ -131,32 +134,34 @@ port = 8080
Every source and sink is a plugin instance with three keys:
| Key | Meaning |
|-----|---------|
| `id` | Instance identifier, unique within the pipeline; appears in logs and stats |
| `type` | Registered plugin type |
| `config` | Plugin-specific table; see [Sources](sources.md) and [Sinks](sinks.md) |
- `id`: instance identifier, unique within the pipeline; appears in logs and
stats.
- `type`: registered plugin type.
- `config`: plugin-specific table; see [Sources](sources.md) and
[Sinks](sinks.md).
`config_file` is reserved on both structures for a future include mechanism and
is not implemented.
### Flow stages
| Block | Optional | Reference |
|-------|----------|-----------|
| `flow.rate_limit` | yes | below |
| `flow.filters` | yes | [Filters](filters.md) |
| `flow.format` | yes (defaults to `raw`) | [Formatters](formatters.md) |
| `flow.heartbeat` | yes | below |
Every stage is optional:
- `flow.rate_limit`: see [Rate limiting](#rate-limiting).
- `flow.filters`: see [Filters](filters.md).
- `flow.format`: defaults to `raw`; see [Formatters](formatters.md).
- `flow.heartbeat`: see [Heartbeat](#heartbeat).
#### Rate limiting
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `rate` | float | `0` | Entries per second; `<= 0` disables the limiter entirely |
| `burst` | float | `rate` | Token bucket capacity |
| `policy` | string | `pass` | `pass` allows everything through, `drop` discards over-limit entries |
| `max_entry_size_bytes` | int | `0` | Per-entry byte cap; `0` = unlimited |
Options, each as type and default:
- `rate` (float, `0`): entries per second; `<= 0` disables the limiter
entirely.
- `burst` (float, `rate`): token bucket capacity.
- `policy` (string, `pass`): `pass` allows everything through, `drop` discards
over-limit entries.
- `max_entry_size_bytes` (int, `0`): per-entry byte cap; `0` = unlimited.
Two behaviours are easy to trip over:
@@ -167,13 +172,16 @@ Two behaviours are easy to trip over:
#### Heartbeat
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `enabled` | bool | `false` | Enable heartbeat generation |
| `interval_ms` | int | `1000` | Interval; minimum `100` |
| `include_timestamp` | bool | `false` | `false` formats with level only, no timestamp |
| `include_stats` | bool | `false` | Attach `beat_count` and measured `interval_ms` as fields |
| `format` | string | `txt` | `txt`, `json`, or `raw` |
Options, each as type and default:
- `enabled` (bool, `false`): enable heartbeat generation.
- `interval_ms` (int, `1000`): interval; minimum `100`.
- `include_timestamp` (bool, `false`): `false` formats with level only, no
timestamp.
- `include_stats` (bool, `false`): attach `beat_count` and measured
`interval_ms` as fields.
- `format` (string, `txt`): `txt`, `json` or `raw` are accepted but have no
effect: heartbeats go through the flow's `format`, like every entry.
Heartbeats are ordinary entries with source `heartbeat` and level `INFO`. They
are generated after the flow's filter and rate-limit stages, so filters do not
@@ -186,14 +194,14 @@ suppress them, and they reach every sink in the pipeline.
## Environment Variables
Environment overrides are derived from the TOML path: `.` becomes `_`, the
result is uppercased, and `LOGWISP_` is prepended.
result is uppercased, and `LOGWISP_` is prepended:
| TOML path | Environment variable |
|-----------|---------------------|
| `quiet` | `LOGWISP_QUIET` |
| `status_reporter` | `LOGWISP_STATUS_REPORTER` |
| `logging.level` | `LOGWISP_LOGGING_LEVEL` |
| `logging.file.directory` | `LOGWISP_LOGGING_FILE_DIRECTORY` |
```
quiet LOGWISP_QUIET
status_reporter LOGWISP_STATUS_REPORTER
logging.level LOGWISP_LOGGING_LEVEL
logging.file.directory LOGWISP_LOGGING_FILE_DIRECTORY
```
Migration: the old custom transform accidentally read bare names such as `QUIET`
and `LOGGING_LEVEL`. Rename those variables to their prefixed forms; bare names
@@ -201,16 +209,17 @@ are now ignored. `LOGWISP_CONFIG_FILE` and `LOGWISP_CONFIG_DIR` still select the
file directly.
Only scalar paths that exist in the configuration schema can be set this way.
Array elements cannot: `LOGWISP_PIPELINES_0_NAME` has no effect.
Array elements cannot: `LOGWISP_PIPELINES_0_NAME` has no effect. A whole
pipeline can: see [pipeline variables](cli.md#pipeline-variables).
## Command-Line Overrides
Any scalar configuration path is settable as a flag using its TOML path:
```bash
logwisp --logging.level=debug --status_reporter=false
logwisp --logging.level debug # space form also works
logwisp --quiet # bare flag means true
lw --logging.level=debug --status_reporter=false
lw --logging.level debug # space form also works
lw --quiet # bare flag means true
```
Unrecognized flags are reported on stderr before the logger exists and are then
@@ -220,11 +229,8 @@ ignored:
Warning: unrecognized flags ignored: [pipelines.0.name]
```
> Array-indexed paths are **not** settable from the command line.
> `--pipelines.0.name=x`, `--pipelines.0.plugin_sinks.0.type=null`, and similar
> flags are reported as unrecognized and ignored. Pipelines, sources, sinks, and
> filters can only be defined in the configuration file. Older documentation
> claimed otherwise.
Array-indexed paths such as `--pipelines.0.name=x` are unrecognized; whole
pipelines have [their own flags](cli.md#pipelines).
## Validation
@@ -236,6 +242,8 @@ Startup validation is intentionally split.
`config` table: `config file "…": unknown key
"pipelines[0].plugin_sinks[0].confg"`. A misspelled table path would otherwise
drop the whole table it heads. A top-level `config_file` is ignored
- pipeline [specs](cli.md#pipelines): syntax, and unknown flow-stage keys such
as `--rate-limit rate=1,polcy=drop`
- at least one pipeline
- unique, non-empty pipeline names
- at least one source and one sink per pipeline
@@ -265,7 +273,11 @@ auto_reload = true
or send `SIGHUP` / `SIGUSR1`. Signals reread the selected file even when watching
is disabled and always rebuild, allowing certificate and credentials rotation
without TOML edits; `auto_reload` watches only the configuration file.
CLI and environment overrides are captured at startup and keep their precedence.
CLI and environment overrides, pipelines included, are captured at startup and
keep their precedence. As at startup, a discovered default file that does not
exist is not an error, so an instance configured by flags or variables alone
still rebuilds. Once read, though, its removal fails the reload and the running
service stays as it is; restart to run without the file.
Reload rebuilds the whole service: a new service is constructed from the new
configuration first, and only if that succeeds is the old one shut down. Each
@@ -279,25 +291,30 @@ without rebuilding. Queued path changes are combined, and unchanged pipelines
and status settings do not trigger another rebuild. Removed file keys fall back
to the remaining sources on the next successful load.
| Reloaded | Not reloaded |
|----------|--------------|
| Pipelines, sources, sinks | `logging.*` (applied once at startup) |
| Filters, formatters, rate limits, heartbeats | `quiet` |
| `status_reporter` | `auto_reload` (the watcher is not restarted) |
What a reload applies:
- Reloaded:
- pipelines, sources, sinks
- filters, formatters, rate limits, heartbeats
- `status_reporter`
- Not reloaded:
- `logging.*`: applied once at startup
- `quiet`
- `auto_reload`: the watcher is not restarted
Because the rebuild is total, listeners close and reopen and every connected
client is disconnected. Chain sinks reconnect on their own backoff schedule.
## Type Reference
| TOML type | Go type | Command-line / environment form |
|-----------|---------|-------------------------------|
| String | `string` | Plain text |
| Integer | `int64` | Decimal string |
| Float | `float64` | Decimal string |
| Boolean | `bool` | `true` / `false`, or a bare flag for `true` |
| Array | `[]T` | Not settable outside the file |
| Table | struct | Nested path with `.` (flags) or `_` (environment) |
Each TOML type, its Go type, and its command-line / environment form:
- String (`string`): plain text.
- Integer (`int64`): decimal string.
- Float (`float64`): decimal string.
- Boolean (`bool`): `true` / `false`, or a bare flag for `true`.
- Array (`[]T`): only in [pipeline specs](cli.md#pipelines), as a repeated key.
- Table (struct): nested path with `.` (flags) or `_` (environment).
Integer fields reject fractions, overflow and negative unsigned values. Non-finite
floats and integer-to-float precision loss are rejected too. This applies to plugin
doc/deployment.md
+198
View File
@@ -0,0 +1,198 @@
# Deployment
`deploy/lw-deploy.sh` configures and deploys one LogWisp node per run. It asks
for what its flags leave open, prints a plan with the equivalent non-interactive
command, then writes the files, runs `lw --check` on them (the node's binary or
image, with its mounts) and starts the node. `--dry-run` stops at printing;
`--yes` never asks and fails on a missing required value. It creates no
certificates: see [Security](security.md#enabling-mtls).
## Roles
- `edge`
- tails each `--log-dir` (not recursive) for `--log-pattern` (default `*.log`)
- forwards to `--aggregator` over `tcp_chain` (port 9440) or, with
`--chain http`, `http_chain` (port 9441)
- labels its entries `--node`: the short host name, or the jail name
- `aggregator`
- one chain listener on `--listen` (default `0.0.0.0`) and `--chain-port`
- outputs, at least one: `--file-dir` (rotating `--file-name`.log),
`--http-port` (SSE `/stream`, JSON `/status`), `--tcp-port`
- `--format` `txt` (default), `json` or `raw`
- `standalone`
- the edge's sources and the aggregator's outputs, no chain
## Runtimes
- `docker`, `podman` (Linux)
- the container runs `--read-only --cap-drop ALL --security-opt
no-new-privileges -u 65532:65532 --restart unless-stopped`
- `--config-dir` (default `/etc/logwisp/NAME`) holds `logwisp.toml`, `tls/`,
and the credentials files, owned by 65532 and mounted read-only at
`/etc/logwisp`
- log directories mount read-only and the output directory writable, each at
its host path; `--log-group` adds that group's id to the container
- listener ports publish on `--listen`, an IP address; the process inside
binds `0.0.0.0`
- `--network NET` joins a user network, created when missing, where edges
reach an aggregator by container name; `host` publishes nothing
- `--image` defaults to `logwisp:dev`, the `make image` tag; `--build` builds
it from the repository. Behind an HTTPS-intercepting proxy add
`--build-ca FILE` and, for a proxy on loopback,
`--build-flags "--network host"`; the environment's proxy variables pass
through as build arguments
- rootful engines need root to give the files to 65532; rootless podman maps
the caller to 65532 with `--userns keep-id`
- SELinux hosts must label the mounted directories for containers; the
script does not relabel, since `:z` on a shared directory such as
`/var/log` would relabel it for every other service too
- `native`, Linux
- needs systemd; installs `lw` to `/usr/local/bin` unless one is installed
(`--bin FILE`, or `--build` for `make build`), and
`deploy/package/logwisp.service`, `logwisp.sysusers` and
`logwisp.tmpfiles` under `/etc`
- a package's unit (`/usr/lib/systemd/system/logwisp.service`) is used as it
is, with the `lw` its `ExecStart` runs; `--bin` and `--build` then stop the
run, since the service would never run that binary
- writes `/etc/logwisp/logwisp.toml`, `root:logwisp`, mode 0640
- refuses log and output directories under `/tmp` and `/var/tmp`: the unit's
`PrivateTmp=yes` gives the service its own, empty ones
- adds `logwisp.service.d/deploy.conf` when the node needs it:
`ReadWritePaths` for an output outside `/var/log/logwisp`,
`ProtectHome=read-only` for logs or output under `/home`,
`SupplementaryGroups` for `--log-group`, `CAP_NET_BIND_SERVICE` for a port
below 1024
- `native`, FreeBSD
- installs `deploy/package/logwisp.rc` as `/usr/local/etc/rc.d/logwisp`,
creates the `logwisp` user with `pw`, enables it with `sysrc`; LogWisp logs
to syslog under the tag `logwisp`
- writes `/usr/local/etc/logwisp/logwisp.toml`
- `--jail NAME` writes the files and runs the commands through `jexec`, so
the jail's own symlinks cannot redirect a write onto the host; the jail
must be running, and the script prints a `jail.conf` example when it is not
- a non-root user binds a port below 1024 only with
`sysctl net.inet.ip.portrange.reservedhigh` lowered or `mac_portacl`
- `manual`
- writes into `--config-dir`, an absolute path (default `~/logwisp-ROLE`),
adds users with the local `lw`, and prints the `lw -c` command
FreeBSD runs no Docker, and podman there runs FreeBSD images only while the
LogWisp image is Linux: on FreeBSD the script offers `native` instead, on the
host or in a jail. A `native` install needs a host of its `--os`; for another
one, `--dry-run` shows the install and `--runtime manual` writes the
configuration.
Re-running the same command updates the node: an existing `logwisp.toml` is
kept as `logwisp.toml.bak`, the container is replaced, generated passwords are
reused. A user already in a credentials file whose password file is missing
stops the run before any change, instead of getting a new password that would
lock its edge out. Under `--yes`, replacing a configuration or a container
needs `--force`.
## Security options
- `--tls` secures every network link of the node; the files are copied into
the configuration directory as `tls/node.crt`, `tls/node.key`, `tls/ca.crt`
- listeners present `--cert-file` and `--key-file`
- an edge verifies the aggregator with `--ca-file`, or the system roots; the
certificate must name the address dialed (an IP SAN for an IP), or the
name given as `--server-name`
- `--auth` on the chain link
- `mtls`: edges present `--cert-file`; the aggregator admits the `--allow`
CNs (none: any certificate the CA issued); an edge's `--allow` pins the
aggregator's CN
- `scram`: the aggregator creates each `--add-user`, its password generated
into `--secrets-dir`, an absolute path (default `~/logwisp-secrets`), and
handed to `lw auth add-user`, natively or through the image; copy each
password file to its edge as `--password-file`, whose name without
`.pass` is the default `--username`
- either binds the node label to the authenticated identity
- `--sink-auth` on the `http` and `tcp` outputs
- `mtls` with `--sink-allow`, or `scram` with `--add-viewer` (a separate
`viewers.toml`); viewers log in with `lw auth token` and `lw auth stream`
- without it the outputs admit anyone who reaches them: bind `--listen` to a
trusted address
- without `--auth` or `--sink-auth`, the auth flags imply their mode:
`--allow` and `--sink-allow` mtls; `--add-user`, `--username`,
`--password-file` and `--add-viewer` scram. Under `--yes` one that does not
apply, such as `--add-user` with `--auth none` or without `--tls`, stops the
run rather than deploy the node open
## FreeBSD host, jails and one aggregator
Application jails run an edge each; a `logs` jail aggregates. Every jail runs
already, and the `logs` jail has an address the others reach (`AGG_ADDR`).
1. Certificates: a CA, and a server certificate for `logs` naming `AGG_ADDR`
([Security](security.md#enabling-mtls)).
2. The aggregator, from the host:
```sh
deploy/lw-deploy.sh --yes --runtime native --jail logs --role aggregator \
--file-dir /var/log/logwisp --http-port 8080 \
--tls --cert-file logs.crt --key-file logs.key \
--auth scram --add-user web1 --add-user db1 \
--sink-auth scram --add-viewer ops
```
The passwords land in `~/logwisp-secrets/`: `web1.pass`, `db1.pass` and
`ops.pass`.
3. An edge per application jail:
```sh
deploy/lw-deploy.sh --yes --runtime native --jail web1 --role edge \
--log-dir /var/log/nginx --aggregator AGG_ADDR \
--tls --ca-file ca.crt --auth scram --password-file ~/logwisp-secrets/web1.pass
```
4. Check: `jexec logs service logwisp status`, the files in the `logs` jail's
`/var/log/logwisp`, and `https://AGG_ADDR:8080/status` with a token from
`lw auth token -url https://AGG_ADDR:8080 -user ops -password-file
~/logwisp-secrets/ops.pass -ca-file ca.crt` ([CLI](cli.md#lw-auth)).
Variants:
- the aggregator on the host itself: drop `--jail logs`
- no daemon per jail: one `standalone` or `edge` on the host tails each jail's
logs through the host path (`--log-dir JAIL_PATH/var/log`); the entries then
carry the host's label and the bare file name, so files of the same name in
two jails cannot be told apart
- a Linux aggregator: run step 2 with `--runtime docker` on that host and point
the edges at it; the chain protocol is the same
## Linux host with containers
An aggregator container and an edge container per application, on one user
network:
```sh
deploy/lw-deploy.sh --yes --runtime docker --role aggregator --network logwisp \
--file-dir /srv/logwisp --http-port 8080 --listen 127.0.0.1
deploy/lw-deploy.sh --yes --runtime docker --role edge --name logwisp-edge-app \
--network logwisp --aggregator logwisp-aggregator --log-dir /var/log/app
```
`--listen 127.0.0.1` keeps the published ports on loopback; edges on other
hosts need it on an address they reach. `docker kill -s HUP NAME` reloads an
edited configuration or credentials file.
## Not covered
Edit the generated `logwisp.toml` for these; `config/logwisp.toml` documents
every key.
- filters, rate limits, heartbeat, formatter flags and timestamp layouts
- more than one pipeline; both chain transports, or a further hop, on one node
- proxy mode: browser logins through a TLS-terminating proxy
- network ACLs: not implemented in LogWisp yet
- mtls identities other than the certificate CN; `node_binding` other than
`force`
- creating certificates
- several native instances on one host: use one per host or jail
## Testing the script
`test/deploy-test.sh --auto` drives the script with `--yes` and `--dry-run` for
the manual, native and docker runtimes, runs the generated edge, aggregator and
standalone nodes end to end, and checks that `--yes` fails closed and that
`--dry-run` changes nothing.
doc/filters.md
+15 -19
View File
@@ -12,11 +12,11 @@ patterns = ["ERROR", "WARN"]
## Options
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `type` | string | `include` | `include` (only matches pass) or `exclude` (matches are dropped) |
| `logic` | string | `or` | `or` (any pattern matches) or `and` (every pattern matches) |
| `patterns` | []string | `[]` | Go RE2 regular expressions |
- `type`, string, default `include`: `include` (only matches pass) or
`exclude` (matches are dropped).
- `logic`, string, default `or`: `or` (any pattern matches) or `and` (every
pattern matches).
- `patterns`, []string, default `[]`: Go RE2 regular expressions.
A filter with no patterns passes everything. Invalid patterns fail at startup
with the filter index and the offending pattern in the message.
@@ -33,10 +33,8 @@ Empty parts are omitted, so an entry with no detected level matches
`"<source> <message>"`. This means a pattern can target the source name or the
level as easily as the message body:
| Pattern | Matches |
|---------|---------|
| `"^app\\.log "` | Entries whose source is `app.log` |
| `"ERROR"` | Level `ERROR`, or the word `ERROR` anywhere in the message |
- `"^app\\.log "` matches entries whose source is `app.log`.
- `"ERROR"` matches level `ERROR`, or the word `ERROR` anywhere in the message.
The structured `fields` payload is **not** part of the match text.
@@ -121,15 +119,13 @@ so later ones evaluate fewer entries.
Go's RE2 syntax. No backreferences and no lookaround — RE2 guarantees linear
time, which is exactly what you want in a log hot path.
| Need | Pattern |
|------|---------|
| Literal substring | `ERROR` |
| Case-insensitive | `(?i)error` |
| Whole word | `\\berror\\b` |
| Alternation | `ERROR\|WARN\|FATAL` |
| Character class | `[0-9]{3}` |
| Anchors | `^ERROR`, `ERROR$` |
| Any characters | `.*exception.*` |
- Literal substring: `ERROR`
- Case-insensitive: `(?i)error`
- Whole word: `\berror\b`
- Alternation: `ERROR|WARN|FATAL`
- Character class: `[0-9]{3}`
- Anchors: `^ERROR`, `ERROR$`
- Any characters: `.*exception.*`
Remember that TOML basic strings process escapes, so a regex backslash needs
doubling: `"\\berror\\b"`. TOML literal strings avoid the issue:
@@ -137,7 +133,7 @@ doubling: `"\\berror\\b"`. TOML literal strings avoid the issue:
Anchors apply to the assembled match text, which begins with the source name —
so `^ERROR` will not match an entry whose source is non-empty. Use
`\\bERROR\\b` instead unless you mean to anchor on the source.
`\bERROR\b` instead unless you mean to anchor on the source.
## Common Recipes
doc/formatters.md
+40 -30
View File
@@ -19,12 +19,14 @@ Omitting `[pipelines.flow.format]` entirely selects `raw`.
## Options
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `type` | string | `raw` | `raw`, `txt` (alias `text`), or `json` |
| `sanitizer_policy` | string | derived from `type` | `raw`, `txt`, `json`, or `shell` |
| `flags` | int64 | `0` | Bitmask override; `0` selects a per-type default |
| `timestamp_format` | string | formatter default | Go reference layout, e.g. `"2006-01-02T15:04:05Z07:00"` |
Each as type and default:
- `type` (string, `raw`): `raw`, `txt` (alias `text`), or `json`.
- `sanitizer_policy` (string, derived from `type`): `raw`, `txt`, `json`, or
`shell`.
- `flags` (int64, `0`): bitmask override; `0` selects a per-type default.
- `timestamp_format` (string, `time.RFC3339Nano`): Go reference layout, e.g.
`"2006-01-02T15:04:05Z07:00"`.
## Types
@@ -86,33 +88,37 @@ LogWisp's own configuration surface.
`flags` is a bitmask passed to the underlying formatter. Leave it at `0` unless
you need to override the defaults.
| Value | Name | Effect |
|-------|------|--------|
| `1` | Raw | Bypass formatting and sanitization entirely |
| `2` | ShowTimestamp | Emit the timestamp |
| `4` | ShowLevel | Emit the level |
| `8` | StructuredJSON | Render attached fields as a JSON object |
| `16` | NoTimestamp | Suppress the timestamp |
| `32` | NoLevel | Suppress the level |
- `1` Raw: bypass formatting and sanitization entirely.
- `2` ShowTimestamp: emit the timestamp.
- `4` ShowLevel: emit the level.
- `8` StructuredJSON: render attached fields as a JSON object.
- `16` NoTimestamp: suppress the timestamp.
- `32` NoLevel: suppress the level.
With `flags = 0` the formatter selects `1` for `type = "raw"` and `6`
(timestamp + level) for every other type. `8` is added automatically whenever an
entry carries parseable `fields` and `1` is not set; `1` always wins.
Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp
only, no level.
`txt` and `json` show timestamp and level unless a suppress bit says otherwise
(suppress wins over show), so `2` and `4` alone change nothing. Examples:
`flags = 16` for level only, no timestamp; `flags = 32` for timestamp only, no
level.
## Sanitizer Policies
The sanitizer runs before serialization and neutralizes control characters that
would otherwise break framing or reach a terminal.
| Policy | Behaviour | Use with |
|--------|-----------|----------|
| `raw` | No-op passthrough | `type = "raw"` where you control the data |
| `txt` | Escapes non-printable characters | File and console sinks |
| `json` | Escapes control characters for safe JSON embedding | `type = "json"`, chain links |
| `shell` | Strips shell metacharacters, whitespace, and control characters | Data that will be passed to a command |
Each policy, then what to use it with:
- `raw`: no-op passthrough.
- `type = "raw"` where you control the data.
- `txt`: escapes non-printable characters.
- File and console sinks.
- `json`: escapes control characters for safe JSON embedding.
- `type = "json"`.
- `shell`: strips shell metacharacters, whitespace, and control characters.
- Data that will be passed to a command.
When `sanitizer_policy` is omitted, the policy is derived from `type`: `json`
for `json`, `txt` for `txt`/`text`, and `raw` for anything else — so the safe
@@ -129,7 +135,8 @@ message.
## Node Identity in Output
Entries that arrived over a chain link carry a `Node` label. The formatter
renders it as a syslog-style prefix on the source field:
renders it as a syslog-style prefix on the source field (`trace` in `json`
output):
```
edge-01/app.log
@@ -149,13 +156,16 @@ from the heartbeat generator when `include_stats = true`.
## Choosing a Configuration
| Goal | Configuration |
|------|---------------|
| Maximum throughput, data already formatted | `type = "raw"` |
| Human reading in a terminal or file | `type = "txt"`, `sanitizer_policy = "txt"` |
| Downstream ingestion (Loki, Elasticsearch, jq) | `type = "json"`, `sanitizer_policy = "json"` |
| Compact console output | `type = "txt"`, `flags = 4` |
| Untrusted log content | never `raw`; pick `txt` or `json` and set the matching policy |
By goal:
- Maximum throughput, data already formatted: `type = "raw"`.
- Human reading in a terminal or file: `type = "txt"`,
`sanitizer_policy = "txt"`.
- Downstream ingestion (Loki, Elasticsearch, jq): `type = "json"`,
`sanitizer_policy = "json"`.
- Compact console output: `type = "txt"`, `flags = 16`.
- Untrusted log content: never `raw`; pick `txt` or `json` and set the
matching policy.
## Formatting and Chain Links
doc/installation.md
+253 -180
View File
@@ -2,231 +2,304 @@
## Requirements
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
- **Architecture**: amd64
- **Go**: 1.27.1 or newer, to build from source
- Operating systems: Linux (kernel 6.10+), FreeBSD (14.0+)
- Architecture: amd64; arm64 builds but is untested
- To build: Go 1.27.1 or newer (FreeBSD: the `go127` package), and GNU make
or BSD make
- For `make e2e`: bash 5+, coreutils, curl, openssl and go (the proxy test
builds its reverse proxy). Optional, else those checks SKIP: node with
playwright and the Chromium builds `playwright install chromium` fetches
(browser checks), an IPv6 loopback (`ipv6-test.sh`), docker as root with the
`logwisp:dev` image (`deploy-test.sh`'s container run)
## Building from Source
```bash
git clone https://github.com/lixenwraith/logwisp.git
cd logwisp
make
sudo make install # installs to $PREFIX/bin, default /usr/local/bin
make build # bin/lw; plain `make` only lists the targets
```
The Makefile works with both GNU make and BSD make. Targets:
The Makefile works with GNU make and BSD make alike. Targets:
| Target | Effect |
|--------|--------|
| `make` / `make build` | Build `bin/logwisp` with version metadata |
| `make dev` | Build with the race detector enabled |
| `make install` | Install the binary to `$(PREFIX)/bin` (default `/usr/local`) |
| `make uninstall` | Remove `$(BINDIR)/logwisp` |
| `make clean` | Remove the built binary |
| `make version` | Print the version, commit, and build time that would be embedded |
- Build
- `make build`: `bin/lw`, with version, commit and build time from git
- `make release`: the same, static (`CGO_ENABLED=0`), `-trimpath`, stripped
- `make dev`: built with the race detector
- `make version`: the metadata a build would embed
- `make clean`: remove `bin/`
- Check
- `make test`: the Go tests
- `make verify`: tests, `go vet`, `gofmt -l` on the Go files changed since
`main`, linux and freebsd cross builds for amd64 and arm64, and the web
client's `node --test` when node is installed
- `make e2e`: builds, then runs every `test/*-test.sh --auto` in turn and
reports passes, failures and skips; `E2E='test/scram-*-test.sh'` runs a
subset; see [Test Scripts](#test-scripts)
- Container
- `make image`, `make image-check`: see [Container Image](#container-image)
- Install
- `make install`, `make uninstall`: see [Installing](#installing)
Version, commit hash, and build time are injected via `-ldflags` from `git
describe` and `git rev-parse`. A plain `go build` produces a working binary that
reports `dev` for all three:
Variables:
- `GO`: the toolchain. On FreeBSD the default is the versioned package that
`go.mod` asks for (`go127`); `make GO=/path/to/go` overrides it, and
`check-go` names the package to install when it is missing.
- `GO_BUILDFLAGS`, `GO_LDFLAGS`: extra `go build` and linker flags for
packagers. They are distinct from `GOFLAGS` and `LDFLAGS`, which the go
command and the C linker read themselves.
- `SOURCE_DATE_EPOCH`: when set, the embedded build time comes from it, so
package builds are reproducible.
A plain `go build -o bin/lw ./cmd/lw` works too, and reports `dev` for the
version metadata. `go install github.com/lixenwraith/logwisp/cmd/lw@latest`
does not work until the module is renamed to its canonical path.
## Installing
`make install` copies what a package ships and compiles nothing, so build
first. It honours `DESTDIR`, `PREFIX`, `SYSCONFDIR` and `BINDIR` (default
`$PREFIX/bin`, the path the systemd unit and the rc.d script run `lw` from):
- Linux defaults: `PREFIX=/usr`, `SYSCONFDIR=/etc`. Outside a package
manager, prefer `PREFIX=/usr/local`; systemd reads units, sysusers and
tmpfiles from there too.
- FreeBSD defaults: `PREFIX=/usr/local`, `SYSCONFDIR=/usr/local/etc`.
- `INSTALL_OS` (default: `uname -s`) picks the layout, so
`INSTALL_OS=FreeBSD DESTDIR=...` stages the FreeBSD one on another system.
```bash
go build -o bin/logwisp ./cmd/logwisp
make release
sudo make install PREFIX=/usr/local
```
`go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with
the same loss of version metadata.
Every system gets:
## Container Image
The root `Dockerfile` builds the same package into `scratch` under UID 65532,
static and stripped. There is no shell and no config in the image: mount one and
name it, as the binary has no daemon mode and no built-in defaults worth running.
```bash
REV=$(git rev-parse HEAD)
docker build -t "logwisp:$(git rev-parse --short HEAD)" \
--build-arg VERSION="$(git describe --tags --always)" \
--build-arg REVISION="$REV" .
docker run --rm -v /etc/logwisp:/etc/logwisp:ro logwisp:... -c /etc/logwisp/logwisp.toml
```
Sinks that listen (`http`, `tcp`) need their ports published; the read-only
root filesystem and dropped capabilities a restricted runtime imposes are all
compatible with it, provided a `file` sink's directory is writable by 65532.
## Configuration
Copy the annotated reference configuration and edit it:
```bash
sudo mkdir -p /etc/logwisp
sudo cp config/logwisp.toml /etc/logwisp/logwisp.toml
```
LogWisp searches, in order: `-c <path>` / `--config <path>` (also `=<path>`),
`$LOGWISP_CONFIG_DIR`/`$LOGWISP_CONFIG_FILE`, `~/.config/logwisp/logwisp.toml`,
`./logwisp.toml`. See [Configuration](configuration.md).
## Running as a Service
LogWisp has no daemon mode; run it in the foreground under a supervisor.
- `$BINDIR/lw` and the manual `$PREFIX/share/man/man1/lw.1`
- `$PREFIX/share/doc/logwisp/` (this documentation) and
`$PREFIX/share/licenses/logwisp/LICENSE`
### Linux (systemd)
`/etc/systemd/system/logwisp.service`:
```ini
[Unit]
Description=LogWisp Log Transport Service
After=network.target
[Service]
Type=simple
User=logwisp
Group=logwisp
ExecStart=/usr/local/bin/logwisp -c /etc/logwisp/logwisp.toml
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=10
WorkingDirectory=/var/lib/logwisp
StandardOutput=journal
StandardError=journal
# Hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/logwisp /var/lib/logwisp
[Install]
WantedBy=multi-user.target
```
`ExecReload` gives you `systemctl reload logwisp` for configuration and
certificate rotation without dropping the process.
If a pipeline binds a port below 1024, add
`AmbientCapabilities=CAP_NET_BIND_SERVICE` rather than running as root.
Setup:
- `$SYSCONFDIR/logwisp/logwisp.toml`: the annotated sample configuration,
kept when one is already there. Edit it before starting the service.
- `$PREFIX/lib/systemd/system/logwisp.service`: runs
`lw -c $SYSCONFDIR/logwisp/logwisp.toml` as the `logwisp` user, from
`/var/lib/logwisp`; `systemctl reload` sends `SIGHUP`.
- `$PREFIX/lib/sysusers.d/logwisp.conf`: the `logwisp` account.
- `$PREFIX/lib/tmpfiles.d/logwisp.conf`: `/var/lib/logwisp` and
`/var/log/logwisp`, owned by it.
```bash
sudo useradd -r -s /usr/sbin/nologin logwisp
sudo mkdir -p /etc/logwisp /var/lib/logwisp /var/log/logwisp
sudo chown logwisp:logwisp /var/lib/logwisp /var/log/logwisp
sudo systemd-sysusers logwisp.conf
sudo systemd-tmpfiles --create logwisp.conf
sudo systemctl daemon-reload
sudo systemctl enable --now logwisp
```
The service account needs **read** access to every directory a `file` source
watches and **write** access to every directory a `file` sink or
`logging.file` writes to.
The unit is hardened: no capabilities, a read-only view of the system with
only its two directories writable, `/home` hidden, and IPv4, IPv6 and Unix
sockets only. Widen it with a drop-in (`systemctl edit logwisp`) rather than
editing the unit:
- A `file` sink or `logging.file` elsewhere: `ReadWritePaths=/srv/logs`
- A `file` source under `/home`: `ProtectHome=read-only`
- A port below 1024: `CapabilityBoundingSet=CAP_NET_BIND_SERVICE` and
`AmbientCapabilities=CAP_NET_BIND_SERVICE`
The service must read every key, credentials and password file it names.
Create a credentials file with its owner and mode first; `lw auth` keeps
both:
```bash
sudo install -m 0600 -o logwisp -g logwisp /dev/null /etc/logwisp/users.toml
sudo lw auth add-user -credentials /etc/logwisp/users.toml -user <name> \
-password-file <file>
sudo systemctl reload logwisp
```
### FreeBSD (rc.d)
`/usr/local/etc/rc.d/logwisp`:
```sh
#!/bin/sh
# PROVIDE: logwisp
# REQUIRE: DAEMON NETWORKING
# KEYWORD: shutdown
. /etc/rc.subr
name="logwisp"
rcvar="${name}_enable"
pidfile="/var/run/${name}.pid"
procname="/usr/local/bin/logwisp"
command="/usr/sbin/daemon"
command_args="-p ${pidfile} -f ${procname} -c /usr/local/etc/logwisp/logwisp.toml"
load_rc_config $name
: ${logwisp_enable:="NO"}
run_rc_command "$1"
```
Setup:
- `$SYSCONFDIR/logwisp/logwisp.toml.sample`, copied to `logwisp.toml` when
none exists and `DESTDIR` is empty
- `$PREFIX/etc/rc.d/logwisp`: runs `lw` through `daemon(8)` as
`logwisp_user`, with its output in syslog under the tag `logwisp`;
`service logwisp reload` sends `SIGHUP`
```bash
sudo chmod +x /usr/local/etc/rc.d/logwisp
sudo pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin
sudo mkdir -p /usr/local/etc/logwisp /var/log/logwisp
sudo chown logwisp:logwisp /var/log/logwisp
sudo sysrc logwisp_enable="YES"
sudo pw useradd logwisp -d /nonexistent -s /usr/sbin/nologin -c "LogWisp log transport"
sudo sysrc logwisp_enable=YES
sudo service logwisp start
```
## Directory Layout
rc.conf variables:
| Purpose | Linux | FreeBSD |
|---------|-------|---------|
| Binary | `/usr/local/bin/logwisp` | `/usr/local/bin/logwisp` |
| Configuration | `/etc/logwisp/` | `/usr/local/etc/logwisp/` |
| TLS material | `/etc/logwisp/tls/` | `/usr/local/etc/logwisp/tls/` |
| Working directory | `/var/lib/logwisp/` | `/var/db/logwisp/` |
| Application logs | `/var/log/logwisp/` | `/var/log/logwisp/` |
- `logwisp_enable`: `YES` to start at boot (default `NO`)
- `logwisp_config`: the configuration (default
`/usr/local/etc/logwisp/logwisp.toml`)
- `logwisp_user`: the account (default `logwisp`)
- `logwisp_chdir`: the working directory (default `/var/db/logwisp`, created
for `logwisp_user` when missing)
- `logwisp_args`: extra `lw` options, e.g. `--logging.level=debug` (default
empty); `logwisp_flags` goes to `daemon(8)`, not to `lw`
Key files should be mode `0600` and owned by the service account.
### Uninstall
`sudo make uninstall` (with the `PREFIX` used to install) removes everything
`make install` wrote except `$SYSCONFDIR/logwisp`, which holds configuration
and credentials. Remove that, the working directories and the account by
hand.
## Container Image
The `Dockerfile` builds `lw` in a Go builder and copies it alone, static and
stripped, into `scratch`: `/lw` running as UID 65532, plus the CA bundle that
TLS dialers without a `ca_file` verify against. There is no shell and no
configuration in the image.
```bash
make image # logwisp:dev; IMAGE=, IMAGE_TAG= rename it
make image-check
```
`image-check` runs the image the way it should run in production: read-only,
`--network none`, `--cap-drop ALL`, `no-new-privileges`, as 65532. It prints
`--version`, then runs `--check` on `config/logwisp.toml` (or
`IMAGE_CHECK_CONFIG`) mounted read-only.
Building behind a proxy:
- A TLS-intercepting proxy: `make image BUILD_CA=/path/to/ca-bundle.pem`.
The bundle reaches only the module download, as a BuildKit secret.
- An explicit proxy: pass Docker's predefined proxy build arguments, which
never reach an image layer, e.g.
`IMAGE_BUILD_FLAGS='--network host --build-arg HTTPS_PROXY=http://127.0.0.1:<port>'`.
- `CONTAINER_ENGINE=podman` works too.
### Running it securely
```bash
docker run -d --name logwisp \
--read-only --cap-drop ALL --security-opt no-new-privileges \
-v /etc/logwisp:/etc/logwisp:ro \
-v /srv/logwisp/secrets:/run/secrets:ro \
-p 8443:8443 \
-e LOGWISP_LOGGING_LEVEL=info \
logwisp:<tag> -c /etc/logwisp/logwisp.toml
```
- Configuration: mount it read-only at `/etc/logwisp` and name it with `-c`.
The files must be readable by UID 65532.
- Scalar settings: environment variables, `LOGWISP_<PATH>`.
- Pipelines without a file: the pipeline variables, in the `--source` and
`--sink` syntax (see [CLI](cli.md)); they replace the file's pipelines:
```bash
-e LOGWISP_PIPELINE=relay \
-e LOGWISP_SOURCE=tcp_chain,port=9000,tls.enabled=true,tls.cert_file=/run/secrets/tls.crt,tls.key_file=/run/secrets/tls.key,auth.type=scram,auth.credentials_file=/run/secrets/users.toml \
-e LOGWISP_SINK=console
```
- Secrets: only as mounted files, at `/run/secrets` (Docker or Compose
secrets, a Kubernetes secret volume), named by `tls.key_file`,
`auth.credentials_file` and `auth.password_file`. Never put a password in
the environment: `docker inspect` and `/proc` show it. Make each file
readable by 65532 only (`chown 65532` and mode `0400`).
- Privileges: keep the image's non-root user, a read-only root filesystem,
`--cap-drop ALL` and `no-new-privileges`. Without capabilities a listener
needs a port of 1024 or above; publish it on any host port.
- Listeners bind `0.0.0.0` inside the container; `127.0.0.1` would be
unreachable through `-p`.
- Writable data: a `file` sink or `logging.file` needs a volume writable by
65532; nothing else is written.
- Reload: `docker kill -s HUP logwisp`.
- `lw auth` runs from the image too, with directories writable by 65532.
Keep the generated passwords out of the directory the server mounts: it
needs only `users.toml` (verifiers) and the TLS files, while a `.pass` file
is the client's plaintext password.
```bash
docker run --rm --read-only --cap-drop ALL --network none \
-v /srv/logwisp/secrets:/work -v /srv/logwisp/passwords:/pw logwisp:<tag> \
auth add-user -credentials /work/users.toml -user <name> -password-file /pw/<name>.pass
```
## Packaging Status
The foundation for distribution packages is in place:
- the `lw` binary name, and `make install` with `DESTDIR`, `PREFIX` and
`SYSCONFDIR`
- the `doc/lw.1` manual
- service files in `deploy/package/`: `logwisp.service`, `logwisp.sysusers`,
`logwisp.tmpfiles` and the FreeBSD `logwisp.rc`
- skeletons: `deploy/package/arch/PKGBUILD` and a `sysutils/logwisp` port in
`deploy/package/freebsd/`
Still missing: tagged, signed release tarballs; the module rename to
`github.com/lixenwraith/logwisp`, which `go install` and FreeBSD's
`go:modules` need; finishing and submitting the AUR package and the port;
Debian packaging. [To Do](todo.md) has the steps.
## Verification
```bash
logwisp --version
# start in the foreground with debug logging and watch pipelines come up
logwisp -c /etc/logwisp/logwisp.toml --logging.level=debug --logging.output=stderr
lw --version
lw -c /etc/logwisp/logwisp.toml --logging.level=debug --logging.output=stderr
sudo systemctl status logwisp # Linux
sudo service logwisp status # FreeBSD
```
Expect `Created source instance`, `Created sink instance`, and
`Starting pipeline` for each configured pipeline. There is no validate-only
mode; see [Operations](operations.md#checking-a-configuration).
Expect `Created source instance`, `Created sink instance` and
`Starting pipeline` for each pipeline. To validate a configuration without
starting it, run `lw --check -c FILE`; see
[Operations](operations.md#checking-a-configuration).
## Test Scripts
End-to-end scripts under `test/` run against a local build:
The end-to-end scripts in `test/` run against `bin/lw` and share
`test/lib.sh`. `make e2e` runs each with `--auto` and reports it as PASS, FAIL
or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target.
```bash
make
./test/chain-test.sh --auto # two independent relay pipelines
./test/chain-aggregate-test.sh --auto # fan-in: both edges into one pipeline
./test/mtls-chain-test.sh --auto # the same fan-in under mTLS
./test/scram-chain-test.sh --auto # SCRAM logins on every network plugin
./test/passthrough-test.sh # file source relays a wide envelope intact
- Modes:
- without flags: start the daemons, print a guide of the ports and one-line
commands (run `. test/run-X/env` first for `$LW`, `$CA` and the token
helper), and wait for Ctrl-C
- `--auto`: run the checks and tear down; `--keep` leaves the daemons up
after a pass
- `--follow`: show only WARN, ERROR and key events from the daemons
- Daemons log only to `test/run*/log/NAME.out`, never to the terminal.
- Colour only on a terminal; `NO_COLOR` turns it off, `FORCE_COLOR=1` on.
- Exit status: 0 all passed, 1 a failure, 77 skipped.
- Each script owns a port range and a gitignored run directory:
```
15801-15804 chain-test.sh, chain-aggregate-test.sh test/run/
(no ports) passthrough-test.sh test/run/passthrough/
15811-15814 mtls-chain-test.sh test/run-mtls/
15821-15825 scram-chain-test.sh test/run-scram/
15831-15832 scram-proxy-test.sh test/run-proxy/
15851-15855 ipv6-test.sh test/run-ipv6/
15871-15879 deploy-test.sh test/run-deploy/
```
Without `--auto` the chain scripts run the relay in the foreground for
interactive inspection. They need bash 5+, coreutils, and curl (the mTLS and
SCRAM scripts also openssl). The plain chain scripts bind ports 15801–15804 and
write to `test/run/`; the mTLS script uses 15811–15814 and `test/run-mtls/`, the
SCRAM script 15821–15825 and `test/run-scram/`. The pass-through test binds
nothing.
## Uninstall
### Linux
```bash
sudo systemctl disable --now logwisp
sudo rm /usr/local/bin/logwisp /etc/systemd/system/logwisp.service
sudo systemctl daemon-reload
sudo rm -rf /etc/logwisp /var/lib/logwisp /var/log/logwisp
sudo userdel logwisp
```
### FreeBSD
```bash
sudo service logwisp stop
sudo sysrc -x logwisp_enable
sudo rm /usr/local/bin/logwisp /usr/local/etc/rc.d/logwisp
sudo rm -rf /usr/local/etc/logwisp /var/db/logwisp /var/log/logwisp
sudo pw userdel logwisp
```
- `chain-test.sh`: two edges into a relay, one pipeline per chain transport;
ports 15801-15804
- `chain-aggregate-test.sh`: the same edges, with the relay fanning both into
both sinks; ports 15801-15804
- `passthrough-test.sh`: `file` source to `file` sink, a wide envelope byte
for byte (`raw = true`) and, parsed, with no key dropped; no ports
- `mtls-chain-test.sh`: mTLS chain edges and viewers, allow lists, node
binding; ports 15811-15814
- `scram-chain-test.sh`: SCRAM chain edges and viewers with `lw auth`
credentials; ports 15821-15825
- `scram-proxy-test.sh`: browser and CLI logins to an `http` sink behind a
TLS-terminating proxy; ports 15831-15832; the browser checks skip without
node and playwright's Chromium: the headless shell for the scenarios with
cookies, full Chromium for the profile that blocks them
- `ipv6-test.sh`: a relay on `::1` over both chain transports, IPv6 SANs,
per-family listeners; ports 15851-15855; skips (exit 77) without an IPv6
loopback
- `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed
and checked; ports 15871-15879
doc/lw.1
+328
View File
@@ -0,0 +1,328 @@
.Dd October 2, 2026
.Dt LW 1
.Os
.Sh NAME
.Nm lw
.Nd LogWisp log transport
.Sh SYNOPSIS
.Nm
.Op Fl \-check
.Op Fl c Ar file
.Op Fl \-quiet
.Op Fl \- Ns Ar key Ns = Ns Ar value ...
.Op Fl \-pipeline Ar name
.Op Fl \-source Ar spec ...
.Op Fl \-sink Ar spec ...
.Op Fl \-filter Ar spec ...
.Op Fl \-format Ar spec
.Op Fl \-rate\-limit Ar spec
.Op Fl \-heartbeat Ar spec
.Nm
.Cm auth
.Ar command
.Op Ar flags
.Nm
.Fl \-version
.Nm
.Cm help | Fl h | Fl \-help
.Sh DESCRIPTION
.Nm
runs LogWisp: independent pipelines, each reading entries from sources,
passing them through a flow of rate limiting, filters and formatting, and
writing them to sinks.
Sources tail files, read standard input or accept other LogWisp nodes;
sinks write to the console or files, serve live streams over HTTP or TCP, or
forward to other nodes.
Every network source and sink can use TLS, mutual TLS and
Argon2id-SCRAM password authentication.
.Pp
.Nm
stays in the foreground and logs to standard output; run it under a
supervisor such as
.Xr systemd 1
or
.Xr daemon 8 .
With no configuration file it runs a demonstration pipeline: a synthetic
generator writing JSON to standard output.
.Pp
The options are as follows:
.Bl -tag -width Ds
.It Fl c Ar file , Fl \-config Ar file , Fl \-config= Ns Ar file
Read the configuration from
.Ar file ,
which must exist.
The last one given wins.
.It Fl \- Ns Ar key Ns = Ns Ar value , Fl \- Ns Ar key Ar value
Set any scalar configuration key by its TOML path, such as
.Fl \-logging.level=debug .
A bare
.Fl \- Ns Ar key
sets a boolean to true.
.It Fl \-quiet
Suppress console output.
.It Fl \-status_reporter= Ns Ar bool
Log pipeline statistics every 30 seconds at the debug level
.Pq default true .
.It Fl \-auto_reload= Ns Ar bool
Reload when the configuration file changes
.Pq default false .
.It Fl \-logging.output= Ns Ar mode
One of
.Cm file , stdout , stderr , split , all
or
.Cm none .
.It Fl \-logging.level= Ns Ar level
One of
.Cm debug , info , warn
or
.Cm error .
.It Fl \-check
Build every pipeline and plugin as a start would, loading options, TLS files
and credentials files and printing startup warnings, without binding a port
or opening, reading or creating a log file; print
.Dq configuration ok
and exit 0, or the error and exit 1, or 2 when a named configuration file is
missing
.Pq see Sx EXIT STATUS .
.It Fl \-version
Print the version, commit and build time, and exit.
.It Fl h , Fl \-help , Cm help
Print usage and exit.
.It Fl \-
End option processing.
.El
.Ss Pipeline specifications
Pipelines can be given on the command line or in the environment instead of
the configuration file.
They then replace the file's pipelines and the built-in default; every other
key keeps its usual precedence.
A reload rereads the file but keeps these pipelines.
.Bl -tag -width Ds
.It Fl \-pipeline Ar name
Start a new pipeline.
Specifications before the first
.Fl \-pipeline
belong to a pipeline named
.Dq cli .
.It Fl \-source Ar spec , Fl \-sink Ar spec , Fl \-filter Ar spec
Add a source, sink or filter; each is repeatable.
.It Fl \-format Ar spec , Fl \-rate\-limit Ar spec , Fl \-heartbeat Ar spec
Set the pipeline's formatter, rate limit and heartbeat.
.El
.Pp
A source, sink, filter or format
.Ar spec
is a type followed by comma-separated
.Ar key Ns = Ns Ar value
pairs; a rate limit or heartbeat
.Ar spec
has the pairs only:
.Bd -literal -offset indent
\-\-source file,directory=/var/log/app,pattern=*.log
\-\-sink http,host=127.0.0.1,port=8080,id=web
\-\-filter include,patterns=ERROR,patterns=WARN
\-\-format json
\-\-rate\-limit rate=100,burst=200,policy=drop
.Ed
.Pp
Keys are dotted paths into the plugin's options, such as
.Cm tls.cert_file .
A repeated key makes a list.
A backslash escapes
.Ql \&, ,
.Ql =
and
.Ql \e
inside a value; quote such a
.Ar spec
so the shell keeps the backslash.
.Cm id= Ns Ar name
names a source or sink; the default is its type, then
.Ar type Ns _2 ,
.Ar type Ns _3
and so on.
Any pipeline flag on the command line makes
.Nm
ignore the pipeline variables in
.Sx ENVIRONMENT .
.Ss Authentication commands
.Nm
.Cm auth
manages SCRAM credentials files and logs in to SCRAM listeners.
Changes to a credentials file apply when the server receives
.Dv SIGHUP .
.Bl -tag -width Ds
.It Cm add-user Fl credentials Ar file Fl user Ar name Oo Fl password-file Ar file Oc Op Fl generate
Add a user, or replace its password.
The password is read from the
.Fl password-file
when it exists; otherwise one is generated and written there, or printed
once to standard output.
.Fl generate
replaces an existing password without reading one.
A missing or empty credentials file is created with mode 0600; an existing
one keeps its owner and mode.
.It Cm remove-user Fl credentials Ar file Fl user Ar name
Remove a user.
The last user cannot be removed.
.It Cm token Fl url Ar https://host:port Fl user Ar name Fl password-file Ar file Oo Ar tls-flags Oc Op Fl unbound
Log in to an
.Cm http
sink or
.Cm http_chain
source and print a bearer token.
.Fl unbound
logs in without channel binding, to an
.Cm http
sink behind a TLS-terminating proxy; the URL may then carry the proxy's path.
.It Cm stream Fl addr Ar host:port Fl user Ar name Fl password-file Ar file Op Ar tls-flags
Log in to a
.Cm tcp
sink and copy its stream to standard output until interrupted.
.El
.Pp
The
.Ar tls-flags
are
.Fl ca-file Ar file
.Pq default: the system roots ,
.Fl server-name Ar name ,
and
.Fl cert-file Ar file
with
.Fl key-file Ar file
for a listener that requires client certificates.
There is no flag to skip server verification.
An IPv6
.Ar host
is written in brackets, as in
.Li [2001:db8::1]:8443 .
.Sh ENVIRONMENT
.Bl -tag -width Ds
.It Ev LOGWISP_ Ns Ar KEY
Any scalar configuration key: its TOML path in upper case with
.Ql \&.
replaced by
.Ql _ ,
as in
.Ev LOGWISP_LOGGING_LEVEL=debug .
Command-line options take precedence, and both take precedence over the
configuration file.
.It Ev LOGWISP_CONFIG_FILE
The configuration file, relative to
.Ev LOGWISP_CONFIG_DIR
when both are set.
.It Ev LOGWISP_CONFIG_DIR
The directory holding
.Pa logwisp.toml .
.It Ev LOGWISP_PIPELINE
The name of the one pipeline the variables below define.
All of these are ignored when a pipeline option is on the command line.
.It Ev LOGWISP_SOURCE , LOGWISP_SINK , LOGWISP_FILTER
Source, sink and filter specifications, in the syntax of the matching
options; a suffix
.Ar _N ,
counting from 1, adds more, as in
.Ev LOGWISP_SINK_1 .
.It Ev LOGWISP_FORMAT , LOGWISP_RATE_LIMIT , LOGWISP_HEARTBEAT
The pipeline's format, rate limit and heartbeat specifications.
.El
.Pp
Secrets never go in the environment: TLS keys, credentials files and
password files are named by path
.Pq Cm tls.key_file , auth.credentials_file , auth.password_file .
.Sh FILES
Without
.Fl c
or the
.Ev LOGWISP_CONFIG_*
variables,
.Nm
reads the first of these that exists:
.Bl -enum -compact
.It
.Pa ~/.config/logwisp/logwisp.toml
.It
.Pa ./logwisp.toml
.El
.Pp
Packages install a sample configuration and service files that pass it with
.Fl c :
.Bl -tag -width Ds
.It Pa /etc/logwisp/logwisp.toml
The configuration on Linux, used by
.Pa logwisp.service .
.It Pa /usr/local/etc/logwisp/logwisp.toml
The configuration on
.Fx ,
installed as
.Pa logwisp.toml.sample
and used by
.Pa /usr/local/etc/rc.d/logwisp .
.It Pa /var/lib/logwisp , /var/db/logwisp
The service's working directory on Linux and on
.Fx .
.El
.Sh EXIT STATUS
.Nm
exits 0 after a clean shutdown,
.Fl \-version
or help; 1 on a configuration, logger or startup error; and 2 when a
configuration file named by
.Fl c
or
.Ev LOGWISP_CONFIG_*
does not exist.
.Pp
.Nm
.Cm auth
exits 0 on success or help, 1 on failure, and 2 on a usage error.
.Sh SIGNALS
.Bl -tag -width "SIGINT, SIGTERM"
.It Dv SIGINT , SIGTERM
Shut down gracefully.
.It Dv SIGHUP , SIGUSR1
Reread the configuration file, certificates and credentials, and rebuild
every pipeline.
An invalid configuration leaves the running service untouched.
.El
.Sh EXAMPLES
Run a configuration in the foreground with debug logging:
.Bd -literal -offset indent
$ lw \-c /etc/logwisp/logwisp.toml \-\-logging.level=debug
.Ed
.Pp
Tail a directory and serve it as a live stream, without a file:
.Bd -literal -offset indent
$ lw \-\-source 'file,directory=/var/log/app,pattern=*.log' \e
\-\-sink http,host=127.0.0.1,port=8080
.Ed
.Pp
Create a credentials file the service account can read, add a user,
letting
.Nm
generate the password into a file, then reload the server:
.Bd -literal -offset indent
# install \-m 0600 \-o logwisp \-g logwisp /dev/null /etc/logwisp/users.toml
# lw auth add-user \-credentials /etc/logwisp/users.toml \e
\-user alice \-password-file /root/alice.pass
# systemctl reload logwisp
.Ed
.Sh SEE ALSO
.Xr rc.conf 5 ,
.Xr systemd.exec 5 ,
.Xr sysusers.d 5 ,
.Xr tmpfiles.d 5 ,
.Xr daemon 8
.Pp
The documentation installed in
.Pa share/doc/logwisp
under the installation prefix, and
.Lk https://github.com/lixenwraith/logwisp
.Sh SECURITY CONSIDERATIONS
A listener without an
.Cm auth
block serves or accepts logs from anyone who can reach it.
Keep key, credentials and password files at mode 0600, owned by the account
.Nm
runs as; startup warns about world-readable ones.
doc/mtls-auth-plan.md
+37 -32
View File
@@ -65,14 +65,13 @@ back to authenticated peers.
The authenticated identity is a single string derived from the peer's verified
leaf certificate. Because `tls.RequireAndVerifyClientCert` has already validated
the chain, signature, and validity window by the time we look, extraction is
pure field selection — `tlsx.PeerIdentity`.
pure field selection — `tlsx.PeerIdentity`. The `identity` modes:
| `identity` mode | Source | Notes |
|-----------------|--------|-------|
| `cn` (default) | `Subject.CommonName` | Matches the existing `tls_peer_cn` metadata |
| `san_dns` | first `DNSNames` entry | Preferred for host identities |
| `san_uri` | first `URIs` entry | SPIFFE-style IDs |
| `san_email` | first `EmailAddresses` entry | Operator identities |
- `cn` (default): `Subject.CommonName`, matching the existing `tls_peer_cn`
metadata.
- `san_dns`: the first `DNSNames` entry; preferred for host identities.
- `san_uri`: the first `URIs` entry; SPIFFE-style IDs.
- `san_email`: the first `EmailAddresses` entry; operator identities.
An empty identity is a rejection, not an empty match: a certificate with no
usable identity field cannot satisfy any policy.
@@ -96,13 +95,14 @@ allow_patterns = ["^edge-\\d{2}$"] # RE2, anchored by the author
node_binding = "force" # none | assert | force
```
| Option | Type | Default | Meaning |
|--------|------|---------|---------|
| `type` | string | `none` | `none` preserves pre-auth behaviour exactly; `mtls` enables the policy |
| `identity` | string | `cn` | Which certificate field is the identity |
| `allow` | []string | `[]` | Exact identity matches |
| `allow_patterns` | []string | `[]` | RE2 patterns matched against the identity |
| `node_binding` | string | `force` when `type = "mtls"` | Chain sources only; see below |
- `type`, string, default `none`: `none` preserves pre-auth behaviour
exactly; `mtls` enables the policy.
- `identity`, string, default `cn`: which certificate field is the identity.
- `allow`, []string, default `[]`: exact identity matches.
- `allow_patterns`, []string, default `[]`: RE2 patterns matched against the
identity.
- `node_binding`, string, default `force` when `type = "mtls"`: chain sources
only; see below.
Empty `allow` **and** empty `allow_patterns` under `type = "mtls"` means "any
identity the CA vouches for" — that is, the pre-auth behaviour, but with the
@@ -111,13 +111,15 @@ default rather than a silent deny-all, and the plugin logs a WARN at startup
saying so.
`node_binding` applies only to the chain sources, where a `node` label is
declared. Setting it on any other plugin is a configuration error.
declared. Setting it to anything but `none` on any other plugin is a
configuration error. Its values:
| Value | Connection label | Per-entry `node` field |
|-------|------------------|------------------------|
| `none` | `trust_node` governs, as before | `trust_node` governs |
| `assert` | Must equal the identity; a mismatch or an omission is rejected | `trust_node` governs |
| `force` | The declared label is ignored and the identity is used | Overwritten with the identity |
- `none`: `trust_node` governs the connection label, as before, and the
per-entry `node` field.
- `assert`: the connection label must equal the identity, and a mismatch or an
omission is rejected; `trust_node` governs the per-entry `node` field.
- `force`: the declared label is ignored and the identity is used; every
per-entry `node` field is overwritten with the identity.
The split between `assert` and `force` is what makes both worth having:
@@ -131,8 +133,9 @@ The split between `assert` and `force` is what makes both worth having:
it forwards keep the origin labels stamped at the first hop, so multi-hop
attribution survives.
`node_binding` overrides `trust_node`; when binding is active the constructor
logs that `trust_node` is being ignored.
`node_binding` overrides `trust_node` for the connection label. Under `force`
the constructor logs that `trust_node` is ignored; under `assert`, that it still
governs the per-entry labels.
Dialer-side plugins (`tcp_chain` and `http_chain` sinks) accept the same block
to pin the *server's* identity beyond hostname verification. There
@@ -161,7 +164,8 @@ One small package, mirroring `internal/tlsx`, is the single seam between the
against the sibling TLS config and returns `(nil, nil)` when auth is disabled;
every method tolerates a nil receiver, so call sites never branch on config.
Listeners admit only through `Admit` (TCP) and `AuthorizeRequest` (HTTP); the
current API, including SCRAM, is in [scram-auth-plan.md](scram-auth-plan.md#the-authz-seam).
current API, including SCRAM, is in
[scram-auth-plan.md](scram-auth-plan.md#the-authz-seam).
### Enforcement Points
@@ -169,18 +173,19 @@ current API, including SCRAM, is in [scram-auth-plan.md](scram-auth-plan.md#the-
Handshake → `Admit` (authorize, then read hello) → `ResolveNode` → create
session. Under `mtls` the certificate is checked before the hello is read, so
an unauthorized peer never gets a preamble parsed on its behalf. `chain.DecodeEntry` is then
called with `auth.TrustsEntryNode(trust_node)` rather than `trust_node` itself.
an unauthorized peer never gets a preamble parsed on its behalf.
`chain.DecodeEntry` is then called with `auth.TrustsEntryNode(trust_node)`
rather than `trust_node` itself.
**`http_chain` source** (`internal/source/httpchain/httpchain.go`, `handleIngest`)
Per request, through `AuthorizeRequest`, before the body is read — an unauthorized sender does
not get to stream `max_body_bytes` into the process. Rejection is `403`,
distinct from the `400` used for protocol errors, so a sender can tell "you are
not allowed" from "your batch was malformed". `ResolveNode` then governs the
`X-Logwisp-Node` header exactly as it governs the TCP hello. The session cache
key includes the identity, so two peers sharing a remote address never share a
session.
Per request, through `AuthorizeRequest`, before the body is read — an
unauthorized sender does not get to stream `max_body_bytes` into the process.
Rejection is `403`, distinct from the `400` used for protocol errors, so a
sender can tell "you are not allowed" from "your batch was malformed".
`ResolveNode` then governs the `X-Logwisp-Node` header exactly as it governs the
TCP hello. The session cache key includes the identity, so two peers sharing a
remote address never share a session.
**`tcp` sink** (`internal/sink/tcp/tcp.go`, `handleConn`)
doc/networking.md
+96 -39
View File
@@ -6,23 +6,54 @@ see [Chaining](chaining.md).
## Address Family
**All listeners bind `tcp4` and all dialers dial `tcp4`.** IPv6 is not
supported, deliberately. An IPv6 client cannot connect and will simply see a
connection failure.
Every listener and dialer keeps strictly to the family of its `host`:
When testing locally use `127.0.0.1`, not `localhost` — the latter may resolve
to `::1` and appear as an unexplained connection refusal.
- IPv4 literal (`127.0.0.1`, the default `0.0.0.0`): IPv4 only (`tcp4`). An
empty `host` is the IPv4 wildcard, as `0.0.0.0`.
- IPv6 literal (`::1`, `::`, `fe80::1%eth0`): IPv6 only (`tcp6`).
- `::` takes no IPv4 connections, on Linux and FreeBSD alike, whatever the
system's `bindv6only` default.
- To serve both families, run two listeners on the same port, one on
`0.0.0.0` and one on `::`: neither holds the other's port.
- Hostname: resolved. A listener binds one address (IPv4 when there is one)
in that address's family alone, a name for `0.0.0.0` or `::` too, so name
the address to be sure; a dialer tries each address in turn.
### IPv6
- Notation
- `host` takes the bare address: `host = "::1"`. A bracketed host, one
with a port, or an IPv4-mapped one (`::ffff:10.0.0.1`) fails at load.
- Addresses and URLs bracket it: logs print `[::1]:8443`, and `lw auth`
takes `-addr [::1]:8443` and `-url https://[::1]:8443`. Older `curl`
releases read the brackets as a glob; `-g` stops that and is harmless.
- A link-local address carries its zone, `fe80::1%eth0`, escaped in URLs as
`https://[fe80::1%25eth0]:8443`.
- TLS: a dialer verifies an IPv6 target against the certificate's IP SANs
(`subjectAltName = IP:::1`), as it does an IPv4 one. The zone is not part of
the name; `server_name` overrides as usual.
- Peers
- Logs and sessions name an IPv6 peer by its address, a link-local one with
its zone.
- SCRAM throttling counts an IPv6 client by its /64, so the hosts on a link
share one link-local budget per interface (zone): each can pick any
`fe80::/64` address.
- A peer with a zone never matches `auth.trusted_proxies`: put the proxy on
loopback or a routed address.
## Network Plugins
| Plugin | Role | Protocol | Purpose |
|--------|------|----------|---------|
| `tcp` sink | Listener | Raw stream | Broadcast formatted payloads to clients |
| `http` sink | Listener | HTTP SSE | Browser-friendly live stream plus status JSON |
| `tcp_chain` source | Listener | Chain v1 | Ingest a persistent NDJSON stream |
| `http_chain` source | Listener | Chain v1 | Ingest NDJSON batches over POST |
| `tcp_chain` sink | Dialer | Chain v1 | Forward entries over a persistent connection |
| `http_chain` sink | Dialer | Chain v1 | Forward entries as batched POSTs |
Listeners:
- `tcp` sink: raw stream; broadcasts formatted payloads to clients.
- `http` sink: HTTP SSE; a browser-friendly live stream plus status JSON.
- `tcp_chain` source: chain v1; ingests a persistent NDJSON stream.
- `http_chain` source: chain v1; ingests NDJSON batches over POST.
Dialers:
- `tcp_chain` sink: chain v1; forwards entries over a persistent connection.
- `http_chain` sink: chain v1; forwards entries as batched POSTs.
There is no port registry and no default port: `port` is required on every
network plugin. There is also no cross-pipeline conflict detection — two sinks
@@ -34,30 +65,36 @@ ERROR msg="Failed to start sink" error="tcp sink bind 0.0.0.0:9090: listen tcp4
## Timeouts
Every network plugin exposes the deadlines relevant to its role. Zero means "no
deadline" wherever the table says so.
Every network plugin exposes the deadlines relevant to its role. `0` means no
deadline where the default is `0` (none); elsewhere it selects the default.
| Plugin | Option | Default | Bounds |
|--------|--------|---------|--------|
| `tcp` sink | `write_timeout_ms` | `5000` | One write to one client; a miss disconnects that client |
| `http` sink | `write_timeout_ms` | `0` (none) | One SSE event write |
| `tcp_chain` source | `hello_timeout_ms` | `10000` | Reading the protocol preamble, and under `scram` the whole login |
| `tcp_chain` source | `read_timeout_ms` | `0` (none) | Idle time between entries |
| `http_chain` source | `read_timeout_ms` | `30000` | Reading a whole request body |
| `tcp_chain` sink | `dial_timeout_ms` | `5000` | TCP connect |
| `tcp_chain` sink | `write_timeout_ms` | `5000` | One line write |
| `http_chain` sink | `request_timeout_ms` | `10000` | Dial plus write plus response, and a SCRAM login when one is due |
- `tcp` sink
- `write_timeout_ms`, default `5000`: one write to one client; a miss
disconnects that client.
- `http` sink
- `write_timeout_ms`, default `0` (none): one SSE event write.
- `tcp_chain` source
- `hello_timeout_ms`, default `10000`: reading the protocol preamble, and
under `scram` the whole login.
- `read_timeout_ms`, default `0` (none): idle time between entries.
- `http_chain` source
- `read_timeout_ms`, default `30000`: reading a whole request body.
- `tcp_chain` sink
- `dial_timeout_ms`, default `5000`: TCP connect.
- `write_timeout_ms`, default `5000`: one line write.
- `http_chain` sink
- `request_timeout_ms`, default `10000`: dial plus write plus response, and
a SCRAM login when one is due.
Fixed, non-configurable bounds:
| Bound | Value | Applies to |
|-------|-------|------------|
| TLS handshake | 10 s | All TLS listeners and dialers |
| HTTP read-header timeout | 10 s | `http` sink, `http_chain` source |
| HTTP server shutdown grace | 2 s | `http` sink, `http_chain` source |
| Max single entry line | 1 MiB | Chain listeners |
| SCRAM login | 10 s | `tcp` sink, chain sinks, `logwisp auth`, each `/auth` request |
| SCRAM line or `/auth` body | 4 KiB | `scram` listeners and dialers |
- TLS handshake: 10 s, on all TLS listeners and dialers.
- HTTP read-header timeout: 10 s, on the `http` sink and `http_chain` source.
- HTTP server shutdown grace: 2 s, on the `http` sink and `http_chain` source.
- Single entry line: at most 1 MiB, on chain listeners.
- SCRAM login: 10 s, on the `tcp` sink, chain sinks, `lw auth`, and each
`/auth` request.
- SCRAM line or `/auth` body: at most 4 KiB, on `scram` listeners and dialers.
The `http` sink deliberately leaves the server's `WriteTimeout` unset, since it
would terminate long-lived SSE streams; per-event deadlines come from
@@ -163,7 +200,9 @@ headers, and entry encoding.
**Connection refused**
- Confirm the pipeline started; a bind failure is logged at ERROR.
- Confirm you are dialing IPv4. `localhost` may resolve to `::1`.
- Confirm the dialer uses the listener's family: `[::1]` does not reach a
`127.0.0.1` or `0.0.0.0` listener, nor `127.0.0.1` a `::` one. `localhost`
may resolve to either.
- Check the port is not already bound by another pipeline in the same process.
**TLS handshake failure**
@@ -176,7 +215,8 @@ headers, and entry encoding.
is not covered by the server certificate's SANs; set `server_name`.
- `protocol version not supported` — one side is pinned to `min_version = "1.3"`
and the other cannot negotiate it.
- Handshake failures appear as WARN with the remote address, and increment
- Listeners log handshake failures at WARN with the remote address (the `tcp`
sink at DEBUG); the `tcp` sink and `tcp_chain` source count them in
`tls_handshake_errors`.
**Rejected after a successful handshake**
@@ -206,7 +246,7 @@ headers, and entry encoding.
between the peers terminates TLS. Pass TLS through to LogWisp; the login
cannot work otherwise, except on an `http` sink in proxy mode
(`auth.trusted_proxies`), where `the client bound its proof to the proxy's
certificate` means a client that needs `logwisp auth token -unbound`, and
certificate` means a client that needs `lw auth token -unbound`, and
`the client sent an unbound proof` the reverse. On an `http_chain` sink,
`server certificate differs from the one the SCRAM login was bound to` means
the certificate changed after the login: a rotation (the retry binds anew),
@@ -217,15 +257,16 @@ headers, and entry encoding.
`X-Forwarded-For`.
- `too many attempts` — the address failed or abandoned logins faster than one
per second beyond a burst of 10, or has 4 unfinished; it clears within seconds
once the failing peer stops. Peers behind one NAT share the budget. `busy` —
4,096 logins in flight, or the listener is stopping.
once the failing peer stops. Peers behind one NAT or one passthrough proxy
share the budget. `busy` — 4,096 logins in flight, or the listener is
stopping.
- `authentication not enabled` — the dialer has `scram`; the listener's
`auth.type` is `none` or `mtls`.
- `no challenge within 10s (older logwisp, or not a scram listener)` — the
server read the hello and said nothing. Over HTTP the same case reads
`no auth endpoint at …/auth`.
- `Argon2 parameters below client minimum` — the credentials file holds a
cheaper Argon2 profile than the defaults; recreate it with `logwisp auth`.
cheaper Argon2 profile than the defaults; recreate it with `lw auth`.
- `peer offered no credentials` (`authentication required` on the wire) in the
listener's log — a peer without `scram` reached a `scram` listener; a client
that sends nothing logs `read hello: … i/o timeout` after 10 s instead. A
@@ -245,6 +286,22 @@ headers, and entry encoding.
- `403` is not the token: it is the `identity` binding, node binding or, in
proxy mode, the proxy gate.
**Behind an L4 proxy**
- Every connection through the proxy fails: it sends a PROXY header (nginx
`stream` with `proxy_protocol on`, HAProxy `send-proxy`) to a LogWisp TLS
listener, which cannot read one yet. The header lands in front of the TLS
handshake, which fails; the listener logs the proxy's address and
`first record does not look like a TLS handshake` (the `tcp` sink at DEBUG).
Route LogWisp without it: see
[Behind nginx or another proxy](security.md#behind-nginx-or-another-proxy).
- All peers share one throttling budget behind a passthrough proxy: without
PROXY every `scram` peer arrives from the proxy's address. One client failing
logins exhausts everyone's, and a fifth login while 4 are unfinished (edges
reconnecting together after a restart) is refused; both answer
`too many attempts` ([Throttling](security.md#throttling)).
Expose the LogWisp ports directly, or keep the proxy hop on a trusted
network so only trusted clients share the budget.
**Entries not arriving over a chain link**
- Check the sink's `connected` statistic and its `reconnects` count.
- Check the source's `auth_rejected` — an allow-list miss or a failed login
doc/operations.md
+40 -29
View File
@@ -6,10 +6,10 @@ Running, monitoring, and maintaining LogWisp.
```bash
# foreground, explicit config
logwisp -c /etc/logwisp/logwisp.toml
lw -c /etc/logwisp/logwisp.toml
# no config: built-in demo pipeline (random source -> stdout)
logwisp
lw
```
There is no built-in daemon mode. Run LogWisp in the foreground under a
@@ -42,7 +42,7 @@ auto_reload = true
or send a signal:
```bash
kill -HUP $(pidof logwisp)
kill -HUP $(pidof lw)
```
Signals reread the selected TOML file even when `auto_reload` is disabled.
@@ -60,8 +60,9 @@ What reload does *not* do:
- Preserve connections. Listeners close and reopen, and every SSE, TCP, and
chain client is disconnected. Chain sinks reconnect on their own backoff;
browsers reconnect SSE automatically; raw TCP consumers must retry themselves.
- Reload certificates without a reload — certificate files are read at plugin
construction, so rotation requires `SIGHUP`.
- Notice rotated certificates on its own: certificate files are read at plugin
construction and `auto_reload` watches only the configuration file, so
rotation requires `SIGHUP`.
Plan reloads on a busy relay the way you would plan a restart.
Listener bind/start failures happen after the old service stops; these can leave
@@ -70,16 +71,17 @@ File-watch errors do not restart services, and queued changes are combined.
### Checking a configuration
There is no validate-only mode. To check a file, start it with debug logging and
watch for pipeline startup:
`lw --check` builds every pipeline and plugin as a start would, without binding
a port or opening, reading or creating a log file, prints the startup warnings,
and exits 0, 1 on an invalid configuration, or 2 when a named file is missing:
```bash
logwisp -c candidate.toml --logging.level=debug --logging.output=stderr
lw --check -c candidate.toml
```
Success looks like `Created source instance`, `Created sink instance`, and
`Starting pipeline` for each pipeline. Failures name the pipeline and the
offending key:
Before a reload, check the edited file this way: a failed reload keeps the
running service, but `--check` says why without touching it. Failures name the
pipeline and the offending key:
```
ERROR msg="Failed to create pipeline" pipeline=app error="failed to create sink out: port: must be 1-65535, got 0"
@@ -145,22 +147,28 @@ curl -s http://127.0.0.1:8080/status | jq .
This endpoint is scoped to one sink, not to the whole process, and without an
`auth` block it is **unauthenticated**. Bind it to a trusted interface, or
query it with a client certificate or a token from `logwisp auth token`.
query it with a client certificate or a token from `lw auth token`.
### Metrics worth watching
| Metric | Where | Meaning if rising |
|--------|-------|-------------------|
| `dropped_entries` | source | Downstream cannot keep up with the source |
| `total_dropped` | flow | Rate limit or filters are discarding entries (often intended) |
| `total_dropped_by_sink` | pipeline | A sink's input queue is full |
| `dropped_writes` | tcp/http sink | A client's queue overflowed: either it is too slow, or one burst exceeded `client_buffer_size` |
| `rejected_conns` / `rejected_clients` | tcp/http sink, tcp_chain source | `max_connections` is being hit |
| `tls_handshake_errors` | tcp sink, tcp_chain source | Certificate or version mismatch, or scanning |
| `parse_errors` | chain source | Protocol or version skew upstream |
| `reconnects` | chain sink | Unstable link or a flapping downstream |
| `dropped_batches` | http_chain sink | Downstream rejecting batches permanently |
| `synthesized` | chain sink | Events reaching the sink without structure |
Each counter, where it is reported, and what a rise means:
- `dropped_entries` (source): downstream cannot keep up with the source
- `total_dropped` (flow): the rate limit or filters are discarding entries,
often by intent
- `total_dropped_by_sink` (pipeline): a sink's input queue is full
- `dropped_writes` (`tcp` and `http` sinks): a client's queue overflowed,
because it is too slow or one burst exceeded `client_buffer_size`
- `rejected_conns` (`tcp` sink, `tcp_chain` source), `rejected_clients`
(`http` sink): `max_connections` is being hit; `rejected_conns` also counts
auth refusals
- `tls_handshake_errors` (`tcp` sink, `tcp_chain` source): a certificate or
version mismatch, or scanning
- `parse_errors` (chain sources): protocol or version skew upstream
- `reconnects` (`tcp_chain` sink): an unstable link or a flapping downstream
- `dropped_batches` (`http_chain` sink): downstream is rejecting batches
permanently
- `synthesized` (chain sinks): events reach the sink without structure
## Log Management
@@ -264,15 +272,18 @@ sink blocked on an unreachable downstream also holds its full input queue.
**Chain link not delivering**
Check `connected` and `reconnects` on the sink, `parse_errors` on the source,
and remember `http_chain` waits up to `flush_interval_ms`. For TLS problems see
[Networking](networking.md#troubleshooting).
Check `connected` and `reconnects` on a `tcp_chain` sink, `parse_errors` on the
source, and remember `http_chain` waits up to `flush_interval_ms`. For TLS
problems see [Networking](networking.md#troubleshooting).
**Environment variable override has no effect**
LogWisp reads `LOGWISP_QUIET`, `LOGWISP_LOGGING_LEVEL`, and other prefixed
names. Bare names used by older versions must be renamed. Array-indexed paths cannot be set from the
environment or the command line at all.
names. Bare names used by older versions must be renamed. Array-indexed paths,
such as a key inside `[[pipelines]]`, cannot be set from the environment or the
command line; define whole pipelines there with the
[pipeline flags](cli.md#pipelines) or
[pipeline variables](cli.md#pipeline-variables).
## Security Operations
doc/scram-auth-plan.md
+79 -64
View File
@@ -6,7 +6,7 @@ channel binding and the mTLS layer was hardened (see
on the `auth` release that carries channel binding; then browser logins behind
a TLS-terminating proxy. Operator documentation lives in
[Security](security.md#password-authentication-scram) and
[`logwisp auth`](cli.md#logwisp-auth).
[`lw auth`](cli.md#lw-auth).
**Scope:** optional username/password authentication on every network plugin,
next to the existing certificate method, without weakening what mTLS gives.
@@ -18,7 +18,7 @@ next to the existing certificate method, without weakening what mTLS gives.
- No password or password-equivalent on the wire, no KDF work on listeners, and
no credential relay through a TLS-terminating MITM.
- Default stays `auth.type = "none"`; existing configurations are unaffected.
- Config in TOML, credential management and viewer clients in a `logwisp auth`
- Config in TOML, credential management and viewer clients in a `lw auth`
CLI.
## Non-Goals
@@ -94,8 +94,9 @@ Validation at construction:
CA-issued certificate plus any valid password (`clientcert=verify-ca`).
- `token_lifetime_ms` only on HTTP listeners; default 15 minutes.
- `/auth` and every path under it are reserved.
- `mtls` rejects the four SCRAM keys. A block that names peers or credentials
with `type = "none"` is refused: the type was forgotten, not the block.
- `mtls` rejects every SCRAM key, `trusted_proxies` included. A block that
names peers or credentials with `type = "none"` is refused: the type was
forgotten, not the block.
**Credentials file** (TOML, written by the CLI, mode 0600):
@@ -113,12 +114,12 @@ username = "edge-01"
`decoy_key` keeps unknown-user challenges stable across restarts and edits, so
a single probe cannot tell a real user from an unknown one (a real user's salt
changes when it is added, rotated or removed). It is created by the first `add-user` and
preserved by every rewrite. An empty user set, a duplicate user, a missing or
short `decoy_key`, an unknown key, or users differing in KDF profile or salt
length are configuration errors. The file is read at construction; the SCRAM
server starts in the plugin's `Start` and stops in `Stop`, so a rejected reload
leaks nothing.
changes when it is added, rotated or removed). It is created by the first
`add-user` and preserved by every rewrite. An empty user set, a duplicate
user, a missing or short `decoy_key`, an unknown key, or users differing in KDF
profile or salt length are configuration errors. The file is read at
construction; the SCRAM server starts in the plugin's `Start` and stops in
`Stop`, so a rejected reload leaks nothing.
## Wire Protocol
@@ -140,12 +141,15 @@ silent, and the dialer gives up after its deadline with "no challenge within
**HTTP** (`http_chain` source, `http` sink): `POST /auth`, outside the auth
middleware, with its own 10 s read/write deadline. It answers `404` unless the
policy is `scram`.
policy is `scram`. Each step, by its request body:
| Step | Request body | Success | Failure |
|------|--------------|---------|---------|
| 1 | hello | `200` challenge | `429` throttled, `503` busy, `400` malformed, `413` oversized |
| 2 | proof | `200` final, token, expires_in | `401 {"error":"authentication failed"}`, `503` if the server stopped |
- Step 1, the hello
- Success: `200` with the challenge.
- Failure: `429` throttled, `503` busy, `400` malformed, `413` oversized.
- Step 2, the proof
- Success: `200` with the final, `token` and `expires_in`.
- Failure: `401 {"error":"authentication failed"}`, or `503` if the server
stopped.
Protected endpoints take `Authorization: Bearer <token>`: missing or invalid is
`401` with `WWW-Authenticate: Bearer realm="logwisp"`, a certificate-binding miss
@@ -157,23 +161,22 @@ checked at connect and outlives its token; a reload ends it.
batch is held under backoff); `404`/`405` from `/auth` reads "no auth endpoint
at <url> (older logwisp, or not a scram listener)"; an ingest `401` clears the
token and is retried (tokens are renewed ahead of expiry, so this follows a
listener reload); `403` drops
the batch; a pin mismatch clears token and pin. Refusals are logged at WARN on
every attempt, paced by the backoff (`Chain connect refused` on `tcp_chain`,
`Chain batch delivery failed` on `http_chain`), and failed logins are counted
in `auth_failures` / `last_auth_error`.
listener reload); `403` drops the batch; a pin mismatch clears token and pin.
Refusals are logged at WARN on every attempt, paced by the backoff (`Chain
connect refused` on `tcp_chain`, `Chain batch delivery failed` on
`http_chain`), and failed logins are counted in `auth_failures` /
`last_auth_error`.
## Throttling
Per remote IP (the socket address; in proxy mode the forwarded client, an IPv6
one per /64), on handshake
starts: a token bucket (burst 10, 1/s) refunded on success, and at most 4
unfinished exchanges. An unanswered HTTP challenge holds its slot for the
`auth` handshake timeout (30 s); abandoned TCP exchanges release their slot,
and their entry in the `auth` handshake table, immediately. The address table
holds 65,536 entries, drops one idle for a minute once its challenges have
expired, and fails closed when full; the SCRAM server itself caps in-flight
handshakes at 4,096 (`busy`).
one per /64), on handshake starts: a token bucket (burst 10, 1/s) refunded on
success, and at most 4 unfinished exchanges. An unanswered HTTP challenge holds
its slot for the `auth` handshake timeout (30 s); abandoned TCP exchanges
release their slot, and their entry in the `auth` handshake table,
immediately. The address table holds 65,536 entries, drops one idle for a
minute once its challenges have expired, and fails closed when full; the SCRAM
server itself caps in-flight handshakes at 4,096 (`busy`).
Counters: `auth_allowed` counts logins; `auth_rejected` every refusal — failed
proofs, malformed requests, missing credentials and, on HTTP, refused tokens;
@@ -201,10 +204,10 @@ closed.
## CLI
```
logwisp auth add-user -credentials F -user U [-password-file P] [-generate]
logwisp auth remove-user -credentials F -user U
logwisp auth token -url https://host:port[/path] -user U -password-file P [-unbound] [TLS flags]
logwisp auth stream -addr host:port -user U -password-file P [TLS flags]
lw auth add-user -credentials F -user U [-password-file P] [-generate]
lw auth remove-user -credentials F -user U
lw auth token -url https://host:port[/path] -user U -password-file P [-unbound] [TLS flags]
lw auth stream -addr host:port -user U -password-file P [TLS flags]
```
- `add-user` takes the password from `-password-file` when it exists (at least
@@ -266,44 +269,54 @@ SameSite=Strict; Max-Age=<lifetime>`) and carries no token in the body. It has
no `Path`: RFC 6265 defaults it to the directory of `POST <mount>/auth`, which
is the mount behind any proxy prefix. Stream and status accept the cookie or
`Authorization: Bearer`; another `Authorization` scheme, such as the site's own
Basic auth, leaves the cookie in charge. Logout is `{"logout": true}` on `POST /auth` itself,
because a clearing cookie set from `/auth/logout` would default to another path
and miss the login cookie. It clears the cookie and revokes a valid token until
it expires (a set of at most 65,536 hashes, gone on reload like every token). An
open stream outlives its token; a reconnect after expiry gets `401`. `/auth`
takes only `application/json`, which a cross-origin page cannot send without a
preflight nobody answers.
Basic auth, leaves the cookie in charge. Logout is `{"logout": true}` on
`POST /auth` itself, because a clearing cookie set from `/auth/logout` would
default to another path and miss the login cookie. It clears the cookie and
revokes a valid token until it expires (a set of at most 65,536 hashes, gone on
reload like every token). An open stream outlives its token; a reconnect after
expiry gets `401`. `/auth` takes only `application/json`, which a cross-origin
page cannot send without a preflight nobody answers.
**Token mode.** A proof step without `"session"` gets the token in the answer
body, in proxy mode as everywhere else, so a browser that keeps no cookies needs
no server change. `scram.js` asks for it with `{session: "token"}`, reads the
stream through `fetch` with the bearer (`EventSource` cannot send one), and
revokes it with `logout(base, {token})`. The viewer falls back to it when a
cookie would not stick or a fresh cookie login still gets `401`, and keeps the
token in a page variable only: it lasts no longer than the page or its lifetime.
**Endpoints.** `/auth` and every path under it are reserved; every URL in the
pages is relative, so a proxy prefix works unchanged.
| Path | Purpose |
|------|---------|
| `POST /auth` | SCRAM hello, proof, or logout; unbound in proxy mode |
| `GET /auth/scram.js` | the client library, always in proxy mode |
| `GET /auth/login`, `login.js`, `style.css` | login page (`login_page`) |
| `GET /auth/view`, `view.js` | minimal live viewer (`viewer_page`, needs `login_page`) |
- `POST /auth`: SCRAM hello, proof or logout; unbound in proxy mode.
- `GET /auth/scram.js`: the client library, always served in proxy mode.
- `GET /auth/login`, `login.js`, `style.css`: the login page (`login_page`).
- `GET /auth/view`, `view.js`: a minimal live viewer (`viewer_page`, needs
`login_page`).
Files carry `default-src 'none'; script-src 'self'; connect-src 'self';
style-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'`,
`nosniff` and `no-referrer`; nothing is inline. The viewer learns a custom
`status_path` from a meta tag the sink fills in, renders entries as text, and
sends a signed-out visitor to `login?next=view`; the login page follows `next`
only within its origin. The `proxy_tls` capability satisfies the pipeline's
"auth needs TLS" check.
sends a signed-out visitor to `login?next=view%23signed-in` (the fragment marks
a fresh cookie login; without cookies it shows its own sign-in form); the login
page follows `next` only within its origin. The `proxy_tls` capability
satisfies the pipeline's "auth needs TLS" check.
**Client library.** `internal/sink/http/web/scram.js`, embedded with `go:embed`,
is one dependency-free ES module, so a site with its own CSP can copy it into
its bundle. It exports `login(base, username, password, {onProgress})` and
`logout(base)`. BLAKE2b and Argon2id are plain JS (no WebAssembly, which would
need `'wasm-unsafe-eval'`), yielding to the page every 50 ms; SHA-256 and HMAC
come from WebCrypto, so it needs a secure context. It checks the challenge as
the Go client does (nonce, salt, cost limits and the dialer's Argon2 floor),
its bundle. It exports `login(base, username, password, {onProgress, session})`,
`logout(base, {token})`, `stream(url, {token, signal, onEvent})` (an event
stream parsed as the HTML standard does), `cookiesUsable()` and
`loginUnavailable()`. BLAKE2b and Argon2id are plain JS (no WebAssembly, which
would need `'wasm-unsafe-eval'`), yielding to the page every 50 ms; SHA-256 and
HMAC come from WebCrypto, so it needs a secure context. It checks the challenge
as the Go client does (nonce, salt, cost limits and the dialer's Argon2 floor),
refuses redirects, and reports success only after verifying the server's
signature. A 64 MiB login takes about 2 s on a desktop. The wire protocol above
is the contract: a site may implement its own client.
**CLI.** `logwisp auth token -unbound` logs in through the proxy, still pinning
**CLI.** `lw auth token -unbound` logs in through the proxy, still pinning
its certificate across the two requests; only then may `-url` carry the mount
path.
@@ -336,29 +349,31 @@ their types and CSP, direct peers refused, pages only in proxy mode, and the
CLI: private files with a matching verifier, no password replaced without a
source, mode, decoy key and symlink kept across rewrites, an empty file filled,
last-user refusal. Under node, `scram.js` against the RFC 9106 vector, Go's
`argon2.IDKey`, and `auth`'s known-answer proof.
`argon2.IDKey`, and `auth`'s known-answer proof; the event-stream line breaks
and fields, the bearer, refusals and abort; a token-mode login and logout.
`test/scram-chain-test.sh --auto` (ports 15821-15825): a relay with SCRAM on all
four listener types plus one `client_auth` + SCRAM port; authorized edges over
both chain transports; node labels forced to usernames; wrong password, unknown
user and no-auth edges deliver nothing; a `tcp` sink viewer through `logwisp auth
user and no-auth edges deliver nothing; a `tcp` sink viewer through `lw auth
stream` held past the exchange deadline; a raw TLS client without a hello gets
nothing; the `http` sink through `logwisp auth token` and curl; missing and bad
nothing; the `http` sink through `lw auth token` and curl; missing and bad
tokens get `401`; `remove-user` + `SIGHUP` revokes token and login; throttling
last. `test/scram-proxy-test.sh --auto` (ports 15831-15832): a Go reverse proxy
ending TLS in front of a plaintext `http` sink mounted at `/logs/`; headless
Chromium sent to the login page, refused a wrong password, streaming events
after login, with the cookie scoped to `/logs` and hidden from scripts, cleared
by sign-out, and no CSP violations; `token -unbound` and curl through the
proxy; direct peers and plaintext-forwarded requests `403`. The existing
scripts keep passing.
by sign-out, and no CSP violations; a cookie the browser drops leading to the
viewer's own form, not a loop, its token kept out of storage and the URL; a
profile blocking every cookie signing in on the viewer, its proofs asking for
no cookie, streaming with a bearer, sign-out revoking the token, and a reload's
`401` asking again; `token -unbound` and curl through the proxy; direct peers
and plaintext-forwarded requests `403`. The existing scripts keep passing.
## Not Implemented
1. **PROXY protocol behind TLS passthrough.** Every client then shares the
proxy's address, and so one throttling budget that a single client can
exhaust. Accepting PROXY v2 from listed proxies would restore per-client
throttling on the TCP listeners and on HTTP listeners outside proxy mode.
1. **PROXY protocol behind TLS passthrough**: planned in
[To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd).
## mTLS Hardening
doc/security.md
+297 -102
View File
@@ -5,21 +5,37 @@ configure it, and — equally important — what it does not yet do.
## Current State
| Capability | Status |
|------------|--------|
| TLS 1.2 / 1.3 on all network sources and sinks | Implemented |
| Server certificate verification by dialers | Implemented |
| Mutual TLS (client certificate required and verified) | Implemented at the transport layer |
| Peer identity recorded per session | Implemented |
| Authorization from certificate identity (allow-lists, node binding) | Implemented — see [The Auth Block](#the-auth-block) |
| Password (Argon2id-SCRAM) authentication, bound to the TLS channel | Implemented — see [Password Authentication](#password-authentication-scram) |
| Authentication on the `http` sink's stream and status endpoints | Implemented: client certificate or bearer token |
| Browser logins behind a site's TLS-terminating proxy | Implemented — see [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy) |
| Server pinning by dialers | Implemented: certificate identity (`mtls`), bound certificate (`scram`) |
| Startup warnings for expiring certificates and risky settings | Implemented — see [Startup Warnings](#startup-warnings) |
| Unknown configuration keys rejected | Implemented — a typo in `tls`, `auth` or a table path fails startup |
| Certificate revocation lists (CRL) or OCSP | **Not implemented** — revoke by editing the allow-list or credentials file |
| IP allow/deny lists, per-IP connection or request limits | **Not implemented** — only SCRAM logins are throttled per address |
**Implemented:**
- TLS 1.2 / 1.3 on all network sources and sinks.
- Server certificate verification by dialers.
- Mutual TLS (client certificate required and verified), at the transport
layer.
- Peer identity recorded per session.
- Authorization from certificate identity (allow-lists, node binding): see
[The Auth Block](#the-auth-block).
- Password (Argon2id-SCRAM) authentication, bound to the TLS channel: see
[Password Authentication](#password-authentication-scram).
- Authentication on the `http` sink's stream and status endpoints: client
certificate or bearer token.
- Browser logins behind a site's TLS-terminating proxy: see
[Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy).
- Server pinning by dialers: certificate identity (`mtls`), bound certificate
(`scram`).
- Startup warnings for expiring certificates and risky settings: see
[Startup Warnings](#startup-warnings).
- Unknown configuration keys rejected: a typo in `tls`, `auth` or a table path
fails startup.
**Not implemented:**
- Certificate revocation lists (CRL) or OCSP: revoke by editing the allow-list
or credentials file.
- IP allow/deny lists, per-IP connection or request limits: only SCRAM logins
are throttled per address.
- PROXY protocol: behind a proxy that passes TLS through, every peer shares the
proxy's address; see
[Behind nginx or another proxy](#behind-nginx-or-another-proxy).
Two credentials are supported. Certificates (`mtls`) are the one the transport
already carries: the `tls` block establishes that a peer chains to your CA, and
@@ -46,17 +62,23 @@ insecure_skip_verify = false
min_version = "1.3"
```
| Option | Role | Default | Description |
|--------|------|---------|-------------|
| `enabled` | both | `false` | Master switch; when false the whole block is ignored |
| `cert_file` | both | — | Local certificate. **Required** for listeners; optional client identity for dialers |
| `key_file` | both | — | Private key for `cert_file`. Must be set together with it |
| `client_auth` | listener | `false` | Require and verify a client certificate (mTLS) |
| `client_ca_file` | listener | — | CA bundle used to verify client certificates. **Required** when `client_auth` is true |
| `ca_file` | dialer | system store | CA bundle used to verify the server certificate |
| `server_name` | dialer | the configured `host` | SNI and certificate name to verify against |
| `insecure_skip_verify` | dialer | `false` | Disable server verification |
| `min_version` | both | `"1.3"` | `"1.2"` or `"1.3"` |
Options by role, each with its default:
- Both roles
- `enabled` (`false`): master switch; when false the whole block is ignored.
- `cert_file`: local certificate; **required** for listeners, an optional
client identity for dialers.
- `key_file`: private key for `cert_file`; set the two together.
- `min_version` (`"1.3"`): `"1.2"` or `"1.3"`.
- Listeners
- `client_auth` (`false`): require and verify a client certificate (mTLS).
- `client_ca_file`: CA bundle that verifies client certificates; **required**
when `client_auth` is true.
- Dialers
- `ca_file` (system store): CA bundle that verifies the server certificate.
- `server_name` (the configured `host`): SNI and certificate name to verify
against.
- `insecure_skip_verify` (`false`): disable server verification.
Listeners are the `tcp` and `http` sinks and the `tcp_chain` and `http_chain`
sources; dialers are the `tcp_chain` and `http_chain` sinks.
@@ -99,26 +121,44 @@ password_file = "" # scram dialers
trusted_proxies = [] # scram http sink behind a TLS-terminating proxy
```
| Option | Applies to | Default | Description |
|--------|------------|---------|-------------|
| `type` | all | `none` | `none` ignores the block; `mtls` authorizes by certificate identity; `scram` by password |
| `identity` | `mtls`; `scram` listeners | `cn` (`mtls`), unset (`scram`) | Certificate field carrying the identity. Under `scram` it binds the client certificate to the user |
| `allow` | `mtls` | `[]` | Exact identities to admit |
| `allow_patterns` | `mtls` | `[]` | RE2 patterns matched against the identity; anchor them yourself |
| `node_binding` | chain sources | `force` | `none`, `assert`, or `force` |
| `credentials_file` | `scram` listeners | — | Verifiers written by [`logwisp auth add-user`](cli.md#logwisp-auth) |
| `token_lifetime_ms` | `scram` on the `http` sink and `http_chain` source | 15 minutes | Bearer token lifetime, 10 s to 24 h |
| `username` | `scram` dialers | — | User to log in as |
| `password_file` | `scram` dialers | — | File holding the password; one trailing line break is trimmed |
| `trusted_proxies` | `scram` on the `http` sink | `[]` | Addresses or CIDRs of the reverse proxies that end the browsers' TLS; see [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy) |
Options by where they apply, each with its default:
**Roles by plugin:**
- Every plugin
- `type` (`none`): `none` ignores the block; `mtls` authorizes by
certificate identity, `scram` by password.
- Chain sources
- `node_binding` (`force`): `none`, `assert` or `force`.
- `mtls`, and `scram` listeners
- `identity`: the certificate field carrying the identity; `cn` under
`mtls`, unset under `scram`, where it binds the client certificate to the
user.
- `mtls`
- `allow` (`[]`): exact identities to admit.
- `allow_patterns` (`[]`): RE2 patterns matched against the identity; anchor
them yourself.
- `scram` listeners
- `credentials_file`: verifiers written by
[`lw auth add-user`](cli.md#lw-auth).
- `token_lifetime_ms` (15 minutes): bearer token lifetime, 10 s to 24 h; the
`http` sink and `http_chain` source only.
- `trusted_proxies` (`[]`): addresses or CIDRs of the reverse proxies that
end the browsers' TLS; the `http` sink only, see
[Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy).
- `scram` dialers
- `username`: user to log in as.
- `password_file`: file holding the password; one trailing line break is
trimmed.
| Plugin | Role | Decides |
|--------|------|---------|
| `tcp_chain` source, `http_chain` source | Listener | Which senders may ingest, and what node label their entries carry |
| `tcp` sink, `http` sink | Listener | Which clients may read the stream (and, on `http`, the status endpoint) |
| `tcp_chain` sink, `http_chain` sink | Dialer | `mtls`: which server identity to accept, beyond hostname verification. `scram`: which user to log in as |
**Roles by plugin**, and what the block decides:
- Listeners
- `tcp_chain` and `http_chain` sources: which senders may ingest, and what
node label their entries carry.
- `tcp` and `http` sinks: which clients may read the stream (and, on
`http`, the status endpoint).
- Dialers: the `tcp_chain` and `http_chain` sinks.
- `mtls`: which server identity to accept, beyond hostname verification.
- `scram`: which user to log in as.
### Identity
@@ -127,12 +167,13 @@ certificate; under `scram` it is the username. The handshake has already
checked the chain, signature, and validity window, so this is pure field
selection.
| Mode | Source | Typical use |
|------|--------|-------------|
| `cn` (default) | `Subject.CommonName` | Matches the existing `tls_peer_cn` metadata |
| `san_dns` | first DNS SAN | Host identities |
| `san_uri` | first URI SAN | SPIFFE-style IDs |
| `san_email` | first email SAN | Operator identities |
Modes, each with the field it reads and its typical use:
- `cn` (default): `Subject.CommonName`; matches the existing `tls_peer_cn`
metadata.
- `san_dns`: the first DNS SAN; host identities.
- `san_uri`: the first URI SAN; SPIFFE-style IDs.
- `san_email`: the first email SAN; operator identities.
A certificate with no usable value in the chosen field is rejected. An empty
identity is a refusal, not an empty match.
@@ -159,13 +200,19 @@ reported at startup.
### Node binding
`node_binding` applies only to the chain sources, and it overrides `trust_node`.
`node_binding` applies only to the chain sources. `assert` and `force` take the
connection label over from `trust_node`; only `force` also overrides the
per-entry labels.
| Value | Connection label | Per-entry `node` field |
|-------|------------------|------------------------|
| `none` | `trust_node` governs | `trust_node` governs |
| `assert` | Must equal the identity; a mismatch or an omission is rejected | `trust_node` governs |
| `force` | Ignored; the identity is used | Overwritten with the identity |
- `none`: `trust_node` governs both the connection label and the per-entry
`node` field.
- `assert`
- Connection label: must equal the identity; a mismatch or an omission is
rejected.
- Per-entry `node` field: `trust_node` governs.
- `force`
- Connection label: ignored; the identity is used.
- Per-entry `node` field: overwritten with the identity.
Use **`force`** on an ingest boundary you do not trust. Every entry is
relabelled, so a compromised edge cannot smuggle a foreign origin through the
@@ -206,14 +253,14 @@ was shown instead; see [Channel binding](#channel-binding).
Misconfiguration fails at plugin construction, before the pipeline starts:
- `type` `mtls` or `scram` without `tls.enabled` (proxy mode aside), or on a dialer with
`tls.insecure_skip_verify`: an identity read from an unverified chain is a
claim, and an unverified server could relay a login
- `type` `mtls` or `scram` without `tls.enabled` (proxy mode aside), or on a
dialer with `tls.insecure_skip_verify`: an identity read from an unverified
chain is a claim, and an unverified server could relay a login
- `type = "mtls"` on a listener without `tls.client_auth`
- a block naming peers or credentials (`allow`, `allow_patterns`,
`credentials_file`, `token_lifetime_ms`, `username`, `password_file`,
`trusted_proxies`) whose
`type` is `none` or unset: auth was intended and the type forgotten
`trusted_proxies`) whose `type` is `none` or unset: auth was intended and the
type forgotten
- a key of the other method: `allow` or `allow_patterns` under `scram`, a
`scram` key under `mtls`
- a `scram` listener without a loadable `credentials_file` (see
@@ -350,7 +397,7 @@ end to end — run it with `--auto` to see each guarantee asserted.
`type = "scram"` authenticates peers by username and password with
Argon2id-SCRAM from `lixenwraith/auth`. It needs TLS but no client
certificates, works on all six network plugins, and is LogWisp's own protocol,
not standard SASL: viewers use the [`logwisp auth`](cli.md#logwisp-auth) CLI or,
not standard SASL: viewers use the [`lw auth`](cli.md#lw-auth) CLI or,
behind a TLS-terminating proxy, the shipped browser client.
```toml
@@ -405,7 +452,7 @@ connection. After the challenge the dialer pins the certificate it was bound to:
the proof and every ingest request must meet the same certificate, or the TLS
handshake fails before anything is sent. The `http_chain` sink then drops token
and pin and logs in again, which also covers a rotated server certificate. A
token printed by `logwisp auth token` is not pinned.
token printed by `lw auth token` is not pinned.
### Certificates and users
@@ -433,21 +480,22 @@ exchange.
Logins are throttled per remote socket address or, on an `http` sink in proxy
mode, per forwarded client (an IPv6 client per /64); forwarded headers are read
only from `trusted_proxies`. Each exchange takes a token from a bucket of 10 that refills at one per
second, and a successful login gives it back, so only failed or abandoned
attempts drain it. At most 4 exchanges per address may be unfinished: an HTTP
challenge never answered holds its slot for up to 30 s, a TCP connection that
ends mid-exchange frees it at once. A refused start answers `too many attempts`
(HTTP `429`) and counts `auth_throttled`; the table holds 65,536 addresses and
refuses new ones when full. Separately, at most 4,096 exchanges may be in flight
per listener; beyond that, or while the plugin stops, a login gets `busy` (HTTP
`503`) and counts `auth_busy`. Peers behind one NAT or one passthrough proxy
share a bucket: there, one client can exhaust every other client's logins.
only from `trusted_proxies`. Each exchange takes a token from a bucket of 10
that refills at one per second, and a successful login gives it back, so only
failed or abandoned attempts drain it. At most 4 exchanges per address may be
unfinished: an HTTP challenge never answered holds its slot for up to 30 s, a
TCP connection that ends mid-exchange frees it at once. A refused start answers
`too many attempts` (HTTP `429`) and counts `auth_throttled`; the table holds
65,536 addresses and refuses new ones when full. Separately, at most 4,096
exchanges may be in flight per listener; beyond that, or while the plugin
stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`. Peers behind
one NAT or one passthrough proxy share a bucket: there, one client can exhaust
every other client's logins.
### Credentials file
```toml
# logwisp SCRAM verifiers, written by `logwisp auth add-user`. Keep it private.
# logwisp SCRAM verifiers, written by `lw auth add-user`. Keep it private.
decoy_key = "<base64, 32 random bytes>"
[[users]]
@@ -460,7 +508,7 @@ stored_key = "<base64>"
username = "edge-01"
```
Write it with [`logwisp auth add-user`](cli.md#logwisp-auth) rather than by
Write it with [`lw auth add-user`](cli.md#lw-auth) rather than by
hand. The whole file is validated when the plugin is built: a `decoy_key` of at
least 32 bytes, at least one user, unique names, no unknown keys, and one Argon2
profile and salt length for every user, since mixed profiles would tell a prober
@@ -477,14 +525,14 @@ different users.
- **Rollout.** Turning `scram` on cuts off every dialer of that listener that
has no credentials. Upgrade every binary first, then add a second listener
with `scram` on another port, move the dialers to it, and remove the old one.
- **Password rotation.** `logwisp auth add-user -generate` with the user's
- **Password rotation.** `lw auth add-user -generate` with the user's
`-password-file` replaces verifier and password together. `SIGHUP` the
listener, deploy the password file, `SIGHUP` the dialer. Logins fail in
between and the dialer retries under backoff, holding its current entry or
batch while its input queue fills. To avoid the gap, add a second user, move
the dialer to it, then remove the first; under `node_binding = "force"` the
node label follows the username.
- **Revocation.** `logwisp auth remove-user`, then `SIGHUP`. The reload drops
- **Revocation.** `lw auth remove-user`, then `SIGHUP`. The reload drops
every connection and revokes every token; other dialers log in again on
their own.
@@ -493,14 +541,15 @@ different users.
- TLS must terminate at LogWisp, except on an `http` sink in proxy mode. Behind
any other terminating proxy or load balancer every login fails by design;
pass TLS through instead (TCP or SNI routing). All clients then share the
proxy's address, and so one throttling budget.
proxy's address, and so one throttling budget: see
[Behind nginx or another proxy](#behind-nginx-or-another-proxy).
- One process per HTTP address. Handshake state and the token key live in one
instance, so behind a balancer the proof or the token can reach an instance
that never saw the login.
- Browsers log in only to an `http` sink in proxy mode, below; elsewhere use
`mtls` for browsers.
- `nc` and `openssl s_client` cannot read a `scram` `tcp` sink; use
`logwisp auth stream`.
`lw auth stream`.
`test/scram-chain-test.sh --auto` exercises these guarantees end to end.
@@ -541,28 +590,169 @@ location /logs/ {
could relay a login. Keep that hop on loopback or a trusted network, or
enable `tls` on the sink; a plaintext hop to a proxy off this host is
warned about at startup. `identity` cannot be combined with proxy mode.
- **Sessions.** The page asks for a cookie: `logwisp_session`, `HttpOnly`,
`Secure`, `SameSite=Strict`, `Max-Age` the token lifetime, and no `Path`, so
it scopes itself to the mount (`/logs` above). Stream and status accept it or
a bearer token, so `new EventSource("/logs/stream")` works on any page of the
site, beside the site's own Basic auth too, and `logwisp auth token -unbound`
keeps working. `POST /auth` with
`{"logout": true}` clears the cookie and revokes the token until it expires.
`/auth` takes only `application/json`, which a cross-origin page cannot send
without a preflight.
- **Sessions.** The login page asks for a cookie: `logwisp_session`,
`HttpOnly`, `Secure`, `SameSite=Strict`, `Max-Age` the token lifetime, and no
`Path`, so it scopes itself to the mount (`/logs` above). Stream and status
accept it or a bearer token, so `new EventSource("/logs/stream")` works on any
page of the site, beside the site's own Basic auth too, and `lw auth token
-unbound` keeps working. `POST /auth` with `{"logout": true}` clears the
cookie and revokes the token until it expires. `/auth` takes only
`application/json`, which a cross-origin page cannot send without a preflight.
- **Private windows.** Private or incognito windows keep cookies in memory,
apart from normal windows: they sign in on their own, and the session ends at
the token lifetime or when the last private window closes.
- **Cookies disabled.** The viewer runs in token mode when the browser keeps no
cookie for the site.
- It notices from `navigator.cookieEnabled`, from a throwaway cookie with the
session's attributes that does not come back (Chromium blocking every cookie
still reports `cookieEnabled`), or from a `401` right after a cookie login,
and shows its own sign-in form.
- The token lives in a page variable only, never in storage or the URL. It
lasts as long as the page and at most the token lifetime: a reload, another
tab, or a reconnect after expiry asks again; sign out revokes it.
- The login page says that cookies are unavailable and links to the viewer.
A page not served over HTTPS keeps no `Secure` cookie either: both pages
name HTTPS instead and offer no sign-in.
- **Integrating `scram.js`.** A site with its own CSP can copy `scram.js` from
`internal/sink/http/web/` into its bundle: one dependency-free ES module,
where `base` is the mount URL ending in `/`. It needs a secure context for
WebCrypto and takes about 2 s of Argon2 per login on a desktop.
- Cookie mode, the default: `login(base, username, password, {onProgress})`,
then `EventSource` and `fetch` as for any same-origin resource;
`logout(base)`.
- Token mode: `login(..., {session: "token"})` resolves to `{username,
expiresIn, token}`. Keep the token in memory; `stream(url, {token, signal,
onEvent})` reads the event stream through `fetch` with the bearer, status
takes `Authorization: Bearer`, and `logout(base, {token})` revokes it. The
bearer replaces the Basic credentials a browser would send, so token mode
cannot pass a proxy that asks for its own Basic auth.
- `cookiesUsable()` tells which applies: false when the session cookie would
not stick. `loginUnavailable()`, asked first, names why neither can run
(no secure context), or is empty.
- **Pages.** Under `/auth/` the sink serves `scram.js` always, and with
`login_page` / `viewer_page` the login page and a minimal live viewer with
their script and style, under `default-src 'none'; script-src 'self';
connect-src 'self'; style-src 'self'; form-action 'self'; frame-ancestors
'none'; base-uri 'none'`. A site with its own CSP can copy `scram.js` from
`internal/sink/http/web/` into its bundle: one dependency-free ES module
exporting `login(base, username, password, {onProgress})` and
`logout(base)`, where `base` is the mount URL ending in `/`. It needs a
secure context for WebCrypto and takes about 2 s of Argon2 per login on a
desktop.
'none'; base-uri 'none'`.
`test/scram-proxy-test.sh --auto` logs in from headless Chromium through such a
proxy.
proxy, with cookies and with every cookie blocked.
### Behind nginx or another proxy
A proxy in front of LogWisp works at one of two layers:
- **L7, ending TLS** (nginx `http` block, HAProxy `mode http`): the `http` sink
in [proxy mode](#browsers-behind-a-tls-terminating-proxy).
- `trusted_proxies` lists the proxy's address.
- The proxy sends `X-Forwarded-Proto: https` and `X-Forwarded-For`. It may
overwrite `X-Forwarded-For` with the real client: the rightmost hop that is
not a proxy is the client either way.
- **L4, passing TLS through to LogWisp's own TLS** (nginx `stream` with
`ssl_preread`, HAProxy `mode tcp`): whatever needs TLS to end at LogWisp.
- Chain links (`tcp_chain`, `http_chain`).
- `lw auth stream` viewers of a `tcp` sink.
- An `http` sink that keeps its own TLS, and with it SCRAM channel binding
or client certificates.
A common shape serves both from port 443:
```
:443 nginx stream: ssl_preread routes by SNI, proxy_protocol on
logs.example.org -> 127.0.0.1:8443 nginx http
relay.example.org -> a LogWisp TLS listener, without PROXY (below)
127.0.0.1:8443 nginx http: listens with proxy_protocol, ends TLS
/logs/ -> 127.0.0.1:8081 LogWisp http sink, proxy mode
```
**The `http` sink behind the `http` block** works in proxy mode as is:
`trusted_proxies` is the `http` block's address, usually `127.0.0.1`, and the
block names the client from the PROXY header.
```nginx
server {
listen 127.0.0.1:8443 ssl proxy_protocol;
server_name logs.example.org;
# ssl_certificate, ssl_certificate_key: the site's own
location /logs/ {
proxy_pass http://127.0.0.1:8081/;
proxy_set_header X-Real-IP $proxy_protocol_addr;
proxy_set_header X-Forwarded-For $proxy_protocol_addr;
proxy_set_header X-Forwarded-Proto https;
}
}
```
`$proxy_add_x_forwarded_for` does not fit this shape: it appends the stream
server's address, not the client's. LogWisp would name that address as every
client or, when it is trusted (both on `127.0.0.1`), the hop to its left: one
the client wrote, so the client picks its own throttling address. LogWisp
ignores `X-Real-IP`.
**LogWisp's own TLS listeners take no PROXY header.** LogWisp cannot read the
PROXY protocol yet; it is planned first in the ACL work
([To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd)).
A stream route that sends `proxy_protocol` straight to a LogWisp TLS listener
breaks every connection: the header lands in front of the TLS handshake. nginx
sets `proxy_protocol` per stream `server`, so route LogWisp SNIs without it:
- Route 1: LogWisp gets its own stream `server` and port, without
`proxy_protocol`.
- Route 2: port 443 sends LogWisp SNIs to an internal stream `server` that
`listen`s with `proxy_protocol` and proxy_passes to LogWisp without it; that
hop consumes the header.
- Either way dialers must send the name: a `host` that is one, or
`tls.server_name` (`-server-name` for `lw auth`); an IP literal sends no SNI.
- With HAProxy, leave `send-proxy` off the LogWisp backends.
```nginx
stream {
map $ssl_preread_server_name $route {
relay.example.org 127.0.0.1:10443; # route 2: the internal hop
tail.example.org 127.0.0.1:10443;
default 127.0.0.1:8443; # the http block, logs.example.org
}
map $ssl_preread_server_name $logwisp {
relay.example.org 127.0.0.1:9001; # tcp_chain source
tail.example.org 127.0.0.1:9002; # tcp sink, for lw auth stream
}
server {
listen 443;
ssl_preread on;
proxy_protocol on; # on every route of this server
proxy_pass $route;
}
server { # route 1: its own port, no header
listen 9443;
ssl_preread on;
proxy_pass $logwisp;
}
server { # route 2: consumes the header
listen 127.0.0.1:10443 proxy_protocol;
ssl_preread on;
proxy_pass $logwisp;
}
}
```
**Passthrough without PROXY** brings every peer from the proxy's address:
- All peers share one SCRAM [throttling](#throttling) budget: 10 failed or
abandoned logins, then one per second, and 4 unfinished at once. One client
that keeps failing logins makes every other client's logins answer
`too many attempts` until it stops.
- Established `tcp_chain` links and open streams keep flowing; a reconnect
waits for the budget. An `http_chain` sink whose early renewal is put off
(`too many attempts` or `busy`) keeps sending on its token to the server it
pinned, retrying every few seconds, and holds its batches only once that
token has expired; any other refusal ends the token at once.
- Logs and sessions name the proxy, not the peer.
- Proxy mode is unaffected: it throttles on the forwarded client.
- Until LogWisp reads PROXY, mitigate:
- expose the chain ports directly, without the proxy, where you can;
- keep the proxy hop on a trusted network, so only trusted clients share the
budget;
- or accept the shared budget.
## What Each Layer Enforces
@@ -598,12 +788,17 @@ would add.
An `auth` block (`mtls` or `scram`) closes each of these. Without one, bind them
to a trusted interface or front them with an authenticating proxy.
| Surface | Exposure when `auth.type = "none"` |
|---------|-----------------------------------|
| `http` sink `stream_path` | Full log stream, with `Access-Control-Allow-Origin: *`, so any browser origin can read it (the header is omitted once an auth policy is set) |
| `http` sink `status_path` | Host, port, TLS flag, uptime, client counts, throughput counters |
| `tcp` sink | Full log stream to any client that connects |
| `tcp_chain` / `http_chain` source | Ingest from any peer that can connect (with `client_auth`, any the CA vouches for), under any node label it claims |
What each exposes when `auth.type = "none"`:
- `http` sink `stream_path`: the full log stream, with
`Access-Control-Allow-Origin: *`, so any browser origin can read it (the
header is omitted once an auth policy is set).
- `http` sink `status_path`: host, port, TLS flag, uptime, client counts,
throughput counters.
- `tcp` sink: the full log stream to any client that connects.
- `tcp_chain` and `http_chain` sources: ingest from any peer that can connect
(with `client_auth`, any the CA vouches for), under any node label it
claims.
`max_connections` bounds concurrency on all of them but does not distinguish
callers.
@@ -618,7 +813,7 @@ where TLS ends at the trusted proxy; `mtls` also requires `client_auth`.
- Use a dedicated CA for LogWisp so its trust decisions stay independent.
- Keep leaf lifetimes short (90–825 days) and automate renewal.
- Key, credentials and password files should be `0600` and owned by the service
account; a world-readable one is reported at startup. `logwisp auth` creates
account; a world-readable one is reported at startup. `lw auth` creates
them `0600` and keeps the mode of an existing file.
- Rotation requires a reload (`SIGHUP`), because certificates and credentials
are loaded once at plugin construction; there is no on-disk watch for them.
doc/sinks.md
+86 -75
View File
@@ -27,10 +27,10 @@ target = "stdout"
buffer_size = 1000
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `target` | string | `stdout` | `stdout` or `stderr` |
| `buffer_size` | int | `1000` | Sink input queue depth |
Options, each as type and default:
- `target` (string, `stdout`): `stdout` or `stderr`.
- `buffer_size` (int, `1000`): sink input queue depth.
> `split` is **not** a valid target for this sink and is rejected at startup.
> Level-based splitting exists only for LogWisp's own application log
@@ -60,16 +60,16 @@ buffer_size = 1000
flush_interval_ms = 100
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `directory` | string | **required** | Output directory |
| `name` | string | **required** | Base filename |
| `max_size_mb` | int | `100` | Rotate when the active file reaches this size |
| `max_total_size_mb` | int | `1000` | Cap across all rotated files |
| `min_disk_free_mb` | int | `0` | Free-space floor before writing; `0` = no floor |
| `retention_hours` | float | `168.0` | Delete rotated files older than this |
| `buffer_size` | int | `1000` | Sink input queue depth |
| `flush_interval_ms` | int | `100` | Forced flush interval |
Options, each as type and default:
- `directory` (string, **required**): output directory.
- `name` (string, **required**): base filename.
- `max_size_mb` (int, `100`): rotate when the active file reaches this size.
- `max_total_size_mb` (int, `1000`): cap across all rotated files.
- `min_disk_free_mb` (int, `0`): free-space floor before writing; `0` = none.
- `retention_hours` (float, `168.0`): delete rotated files older than this.
- `buffer_size` (int, `1000`): sink input queue depth.
- `flush_interval_ms` (int, `100`): forced flush interval.
> `min_disk_free_mb` has an unusual default. The constructor replaces only
> *negative* values with `100`; leaving the key unset yields `0`, which means no
@@ -125,20 +125,25 @@ type = "mtls"
allow = ["viewer-01"]
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
| `port` | int | **required** | Listen port |
| `stream_path` | string | `/stream` | SSE endpoint; must start with `/` |
| `status_path` | string | `/status` | Status endpoint; must start with `/` and differ from `stream_path` |
| `buffer_size` | int | `1000` | Sink input queue depth |
| `client_buffer_size` | int | `256` | Per-client send queue depth |
| `write_timeout_ms` | int | `0` | Per-event write deadline; `0` = none |
| `max_connections` | int | `0` | Concurrent stream cap; `0` = unlimited |
| `login_page` | bool | `false` | Serve the browser login page at `/auth/login`; needs `auth.trusted_proxies` |
| `viewer_page` | bool | `false` | Serve a minimal live viewer at `/auth/view`; needs `login_page` |
| `tls` | table | — | Listener TLS; see [Security](security.md) |
| `auth` | table | — | Client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block) |
Options, each as type and default:
- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the listener
keeps to its family ([Networking](networking.md#address-family)).
- `port` (int, **required**): listen port.
- `stream_path` (string, `/stream`): SSE endpoint; must start with `/`.
- `status_path` (string, `/status`): status endpoint; must start with `/` and
differ from `stream_path`.
- `buffer_size` (int, `1000`): sink input queue depth.
- `client_buffer_size` (int, `256`): per-client send queue depth.
- `write_timeout_ms` (int, `0`): per-event write deadline; `0` = none.
- `max_connections` (int, `0`): concurrent stream cap; `0` = unlimited.
- `login_page` (bool, `false`): serve the browser login page at `/auth/login`;
needs `auth.trusted_proxies`.
- `viewer_page` (bool, `false`): serve a minimal live viewer at `/auth/view`;
needs `login_page`.
- `tls` (table): listener TLS; see [Security](security.md).
- `auth` (table): client authentication (`mtls` or `scram`); see
[Security](security.md#the-auth-block).
**Behaviour**
@@ -148,7 +153,7 @@ allow = ["viewer-01"]
- With an `auth` block, one middleware gates **both** endpoints, with no body
detail in a refusal. Under `mtls` a refused certificate gets `403`. Under
`scram` a client logs in at `POST /auth`, which sits outside the gate (see
[`logwisp auth token`](cli.md#logwisp-auth)), and sends
[`lw auth token`](cli.md#lw-auth)), and sends
`Authorization: Bearer <token>`; a missing, invalid or expired token gets
`401` with `WWW-Authenticate`, a certificate that does not match the token's
user `403`. A stream is checked when it connects and outlives its token; a
@@ -233,18 +238,20 @@ type = "mtls"
allow = ["viewer-01"]
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
| `port` | int | **required** | Listen port |
| `buffer_size` | int | `1000` | Sink input queue depth |
| `client_buffer_size` | int | `256` | Per-client send queue depth |
| `write_timeout_ms` | int | `5000` | Per-write deadline |
| `keep_alive` | bool | `true` | Enable TCP keep-alive on accepted connections |
| `keep_alive_period_ms` | int | `30000` | Keep-alive idle period |
| `max_connections` | int | `0` | Concurrent connection cap; `0` = unlimited |
| `tls` | table | — | Listener TLS |
| `auth` | table | — | Client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block) |
Options, each as type and default:
- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the listener
keeps to its family ([Networking](networking.md#address-family)).
- `port` (int, **required**): listen port.
- `buffer_size` (int, `1000`): sink input queue depth.
- `client_buffer_size` (int, `256`): per-client send queue depth.
- `write_timeout_ms` (int, `5000`): per-write deadline.
- `keep_alive` (bool, `true`): TCP keep-alive on accepted connections.
- `keep_alive_period_ms` (int, `30000`): keep-alive idle period.
- `max_connections` (int, `0`): concurrent connection cap; `0` = unlimited.
- `tls` (table): listener TLS.
- `auth` (table): client authentication (`mtls` or `scram`); see
[Security](security.md#the-auth-block).
**Behaviour**
@@ -254,7 +261,7 @@ allow = ["viewer-01"]
first: a hello carrying the login, then challenge, proof and final, all
within 10 s, after which the stream follows on the same connection. `nc` and
`openssl s_client` cannot do this; use
[`logwisp auth stream`](cli.md#logwisp-auth).
[`lw auth stream`](cli.md#lw-auth).
- A write that misses its deadline means the kernel buffer stayed full for the
whole timeout, so the client is disconnected immediately rather than retried.
- A client whose send queue is full has that event dropped (`dropped_writes`)
@@ -300,20 +307,22 @@ cert_file = "/etc/logwisp/tls/client.crt"
key_file = "/etc/logwisp/tls/client.key"
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `host` | string | **required** | Downstream host |
| `port` | int | **required** | Downstream port |
| `node` | string | `os.Hostname()` | Origin label stamped on first-hop entries |
| `buffer_size` | int | `1000` | Sink input queue depth |
| `dial_timeout_ms` | int | `5000` | TCP connect timeout |
| `write_timeout_ms` | int | `5000` | Per-write deadline |
| `backoff_min_ms` | int | `500` | Reconnect backoff floor |
| `backoff_max_ms` | int | `30000` | Reconnect backoff ceiling |
| `keep_alive` | bool | `true` | Enable TCP keep-alive |
| `keep_alive_period_ms` | int | `30000` | Keep-alive idle period |
| `tls` | table | — | Dialer TLS; `cert_file`/`key_file` present a client identity |
| `auth` | table | — | Server identity pinning (`mtls`) or a login (`scram`); see [Security](security.md#the-auth-block) |
Options, each as type and default:
- `host` (string, **required**): downstream host or address; an IPv6 one goes
bare (`::1`).
- `port` (int, **required**): downstream port.
- `node` (string, `os.Hostname()`): origin label stamped on first-hop entries.
- `buffer_size` (int, `1000`): sink input queue depth.
- `dial_timeout_ms` (int, `5000`): TCP connect timeout.
- `write_timeout_ms` (int, `5000`): per-write deadline.
- `backoff_min_ms` (int, `500`): reconnect backoff floor.
- `backoff_max_ms` (int, `30000`): reconnect backoff ceiling.
- `keep_alive` (bool, `true`): TCP keep-alive.
- `keep_alive_period_ms` (int, `30000`): keep-alive idle period.
- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity.
- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see
[Security](security.md#the-auth-block).
**Behaviour**
@@ -380,21 +389,23 @@ cert_file = "/etc/logwisp/tls/client.crt"
key_file = "/etc/logwisp/tls/client.key"
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `host` | string | **required** | Downstream host |
| `port` | int | **required** | Downstream port |
| `ingest_path` | string | `/ingest` | Endpoint path; must start with `/` |
| `node` | string | `os.Hostname()` | Origin label stamped on first-hop entries |
| `buffer_size` | int | `1000` | Sink input queue depth |
| `max_batch_count` | int | `100` | Flush after this many entries |
| `max_batch_bytes` | int | `1048576` | Flush after this many bytes (1 MiB) |
| `flush_interval_ms` | int | `1000` | Flush after this long |
| `request_timeout_ms` | int | `10000` | Covers dial, write, and response |
| `backoff_min_ms` | int | `500` | Retry backoff floor |
| `backoff_max_ms` | int | `30000` | Retry backoff ceiling |
| `tls` | table | — | Dialer TLS; `cert_file`/`key_file` present a client identity |
| `auth` | table | — | Server identity pinning (`mtls`) or a login (`scram`); see [Security](security.md#the-auth-block) |
Options, each as type and default:
- `host` (string, **required**): downstream host or address; an IPv6 one goes
bare (`::1`).
- `port` (int, **required**): downstream port.
- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`.
- `node` (string, `os.Hostname()`): origin label stamped on first-hop entries.
- `buffer_size` (int, `1000`): sink input queue depth.
- `max_batch_count` (int, `100`): flush after this many entries.
- `max_batch_bytes` (int, `1048576`): flush after this many bytes (1 MiB).
- `flush_interval_ms` (int, `1000`): flush after this long.
- `request_timeout_ms` (int, `10000`): covers dial, write, and response.
- `backoff_min_ms` (int, `500`): retry backoff floor.
- `backoff_max_ms` (int, `30000`): retry backoff ceiling.
- `tls` (table): dialer TLS; `cert_file`/`key_file` present a client identity.
- `auth` (table): server identity pinning (`mtls`) or a login (`scram`); see
[Security](security.md#the-auth-block).
**Behaviour**
@@ -407,10 +418,10 @@ key_file = "/etc/logwisp/tls/client.key"
sends `Authorization: Bearer <token>`. A failed login is retried like a
transport error, holding the batch. The token is renewed ahead of its expiry;
a `401` on ingest (the source reloaded) drops it and the retry logs in again.
A `403` is permanent. A
connection presenting a certificate other than the one the login was bound
to drops token and pin, and the retry logs in anew. A sink without `scram`
facing a `scram` source gets `401` and drops every batch.
A `403` is permanent. A connection presenting a certificate other than the
one the login was bound to drops token and pin, and the retry logs in anew.
A sink without `scram` facing a `scram` source gets `401` and drops every
batch.
- Redirects are never followed; a `3xx` is permanent too. Following one would
resend the batch wherever the response points, plaintext `http` included.
- HTTP/2 is off by design; batched NDJSON POSTs gain nothing from it.
doc/sources.md
+53 -39
View File
@@ -36,13 +36,14 @@ raw = false
from = "end"
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `directory` | string | **required** | Directory to scan; not recursive |
| `pattern` | string | `*` | Glob over filenames; `*` and `?` only |
| `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` |
| `raw` | bool | `false` | Never parse a line: the whole line is the message |
| `from` | string | `end` | Where a new watcher starts: `end` or `start` of the file |
Options, each as type and default:
- `directory` (string, required): directory to scan; not recursive.
- `pattern` (string, `*`): glob over filenames; `*` and `?` only.
- `check_interval_ms` (int, `100`): directory rescan interval; minimum `10`.
- `raw` (bool, `false`): never parse a line: the whole line is the message.
- `from` (string, `end`): where a new watcher starts, `end` or `start` of the
file.
**Behaviour**
@@ -93,9 +94,9 @@ type = "console"
buffer_size = 1000
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `buffer_size` | int | `1000` | Subscriber channel depth |
Option, as type and default:
- `buffer_size` (int, `1000`): subscriber channel depth.
At most **one** instance per pipeline: the type is registered with
`MaxInstances: 1`, and a second instance is rejected at pipeline construction.
@@ -119,13 +120,15 @@ length = 20
special = false
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `interval_ms` | int | `500` | Emission period |
| `jitter_ms` | int | `0` | Symmetric jitter; clamped to `interval_ms`, must be non-negative |
| `format` | string | `txt` | `raw` (message only), `txt` (bracketed line), `json` (JSON object as the message) |
| `length` | int | `20` | Message length in characters |
| `special` | bool | `false` | Inject control and non-ASCII characters |
Options, each as type and default:
- `interval_ms` (int, `500`): emission period.
- `jitter_ms` (int, `0`): symmetric jitter; must be non-negative, clamped to
`interval_ms`.
- `format` (string, `txt`): `raw` (message only), `txt` (bracketed line) or
`json` (JSON object as the message).
- `length` (int, `20`): message length in characters.
- `special` (bool, `false`): inject control and non-ASCII characters.
`special = true` is the intended way to exercise sanitizer policies: it inserts
control bytes and multi-byte Unicode into otherwise ordinary messages. Levels
@@ -180,17 +183,22 @@ allow = ["edge-01", "edge-02"]
node_binding = "force"
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
| `port` | int | **required** | Listen port, 1–65535 |
| `buffer_size` | int | `1000` | Subscriber channel depth |
| `max_connections` | int | `0` | Concurrent connection cap; `0` = unlimited |
| `read_timeout_ms` | int | `0` | Per-connection idle read deadline; `0` = none |
| `hello_timeout_ms` | int | `10000` | Deadline for the hello preamble |
| `trust_node` | bool | `true` | `false` overrides the sender's node label with its remote address. Ignored when `auth.node_binding` is active |
| `tls` | table | — | Listener TLS; see [Security](security.md) |
| `auth` | table | — | Peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block) |
Options, each as type and default:
- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the
listener keeps to its family ([Networking](networking.md#address-family)).
- `port` (int, required): listen port, 1–65535.
- `buffer_size` (int, `1000`): subscriber channel depth.
- `max_connections` (int, `0`): concurrent connection cap; `0` = unlimited.
- `read_timeout_ms` (int, `0`): per-connection idle read deadline; `0` = none.
- `hello_timeout_ms` (int, `10000`): deadline for the hello preamble.
- `trust_node` (bool, `true`): `false` overrides the sender's node label with
its remote address. `auth.node_binding` `assert` or `force` takes over the
connection label; only `force` also overrides the per-entry labels; `none`
leaves both to `trust_node`.
- `tls` (table): listener TLS; see [Security](security.md).
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
see [Security](security.md#the-auth-block).
For passwords instead of certificates, the `auth` block takes `type = "scram"`
and a `credentials_file`, and `client_auth` becomes optional; see
@@ -258,17 +266,23 @@ allow = ["edge-01", "edge-02"]
node_binding = "force"
```
| Option | Type | Default | Description |
|--------|------|---------|-------------|
| `host` | string | `0.0.0.0` | Bind address; IPv4 only |
| `port` | int | **required** | Listen port |
| `ingest_path` | string | `/ingest` | Endpoint path; must start with `/` |
| `buffer_size` | int | `1000` | Subscriber channel depth |
| `max_body_bytes` | int | `8388608` | Per-request body cap (8 MiB) |
| `read_timeout_ms` | int | `30000` | Full request read deadline |
| `trust_node` | bool | `true` | `false` overrides the sender's node label with its remote address. Ignored when `auth.node_binding` is active |
| `tls` | table | — | Listener TLS |
| `auth` | table | — | Peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block) |
Options, each as type and default:
- `host` (string, `0.0.0.0`): bind address, IPv4 or IPv6 (`::`); the
listener keeps to its family ([Networking](networking.md#address-family)).
- `port` (int, required): listen port, 1–65535.
- `ingest_path` (string, `/ingest`): endpoint path; must start with `/`, and
`/auth` is reserved for the login.
- `buffer_size` (int, `1000`): subscriber channel depth.
- `max_body_bytes` (int, `8388608`): per-request body cap (8 MiB).
- `read_timeout_ms` (int, `30000`): full request read deadline.
- `trust_node` (bool, `true`): `false` overrides the sender's node label with
its remote address. `auth.node_binding` `assert` or `force` takes over the
connection label; only `force` also overrides the per-entry labels; `none`
leaves both to `trust_node`.
- `tls` (table): listener TLS; see [Security](security.md).
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
see [Security](security.md#the-auth-block).
**Behaviour**
doc/todo.md
+202
View File
@@ -0,0 +1,202 @@
# To Do
Planned work, in order of priority. Each item names what to build, how it fits
the existing seams, and how to verify it. Finished items move to the design doc
they belong to.
## 1. Network access control (ACL)
The next feature push. Today a listener decides *who* a peer is (`tls`, `auth`);
nothing decides *where* a peer may connect from, and every limit except SCRAM
throttling is global. The ACL adds address rules and per-client limits at the
connection level, below TLS, so a refused peer costs no handshake.
### 1.1 One seam: a filtering listener
- New package `internal/netacl`, the third network seam beside `tlsx` and
`authz` (AGENTS.md lists the seams; add it there).
- `netacl.New(opts *config.ACLOptions, role) (*Policy, error)` compiles the
block; a nil policy is valid and transparent, like `authz`.
- `(*Policy).Listener(ln net.Listener) net.Listener` wraps the raw listener
before TLS:
- TCP plugins wrap it before `tls.NewListener`.
- HTTP plugins wrap it before `Serve`/`ServeTLS`.
- `Accept` reads an optional PROXY header (1.2), decides allow or deny (1.3),
applies per-address connection caps (1.4), and returns a `net.Conn` whose
`RemoteAddr` is the real client.
- Everything downstream then sees the real client with no further change:
`authz.remoteIP`, SCRAM throttling, sessions, logs and `http.Request.RemoteAddr`.
- Keep the proxy's own address as `peer_addr` in session metadata for audit.
- A denied connection is closed before any byte is read or written, with a
counter (`acl_denied`) and a rate-limited WARN.
### 1.2 PROXY protocol (deferred gap of SCRAM, see scram-auth-plan.md)
- Why: behind an L4 proxy that passes TLS through (nginx `stream` with
`ssl_preread`, HAProxy `mode tcp`), every peer arrives from the proxy's
address.
- All peers share one SCRAM throttling budget, so one client can lock out
every login.
- Logs, sessions and the ACL itself see only the proxy.
- What: accept PROXY v1 (text) and v2 (binary) headers, but only from
`proxy_from` addresses or CIDRs. Two config keys in the ACL block:
- `proxy_protocol = "off" | "optional" | "required"`, default off.
- `proxy_from = [...]`.
- Rules:
- A listed peer must send the header when `required`.
- An unlisted peer that sends a header is refused, since the header is
spoofable.
- The header must arrive within a short deadline (reuse
`tlsx.HandshakeTimeout`), with a size cap: v1 at most 107 bytes, v2 with a
bounded TLV length.
- `LOCAL`/`UNKNOWN` commands (proxy health checks) keep the socket address.
- TLVs are ignored.
- The header precedes the TLS ClientHello, so SCRAM channel binding is
unaffected: LogWisp still terminates TLS.
- nginx side:
- In the `stream` server, `proxy_protocol on` on the route to LogWisp (it
sends v1).
- HAProxy: `send-proxy` or `send-proxy-v2`.
- Relation to `auth.trusted_proxies` (HTTP proxy mode):
- That list trusts `X-Forwarded-For` from a TLS-terminating L7 proxy;
`proxy_from` trusts a PROXY header from an L4 proxy.
- Keep both, with distinct names and docs. They answer different layers, and
one deployment can use both: nginx `stream` with `proxy_protocol` into an
nginx `http` block that sets `X-Forwarded-For $proxy_protocol_addr`.
### 1.3 Address rules
- `allow = [CIDR...]`, `deny = [CIDR...]`, IPv4 and IPv6 (strict per family,
as the listeners are).
- Order: deny wins, then allow; an empty allow list admits everyone not denied.
- Rules match the real client (after PROXY), never the proxy.
- In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after
the connection-level decision.
- Address rules for forwarded HTTP clients therefore belong in a second,
request-level check inside `authz.ClientAddr`: reuse the compiled `netacl`
rules there rather than duplicating them.
- Startup warnings (in `LogStartup` style): an allow list containing
`0.0.0.0/0` or `::/0`, `proxy_from` covering public ranges, a listener on a
wildcard host with no `allow`.
### 1.4 Per-client limits
- Generalize the SCRAM limiter in `internal/authz/scram.go` (token bucket per
key, bounded table that fails closed, idle sweep, IPv6 per /64 via
`throttleKey`) into the shared per-key limiter. One mechanism (AGENTS.md):
SCRAM throttling then uses it too.
- Keys:
- `max_connections_per_client` on every listener (concurrent connections per
address).
- `requests_per_second_per_client` on HTTP listeners (ingest, stream and
status requests).
- Later: per-identity limits once a peer is authenticated (the mtls plan's
deferred item 2), keyed by identity instead of address.
### 1.5 Config, validation, docs
- Config shape: `[...config.acl]` beside `tls` and `auth` on the four listener
plugins.
- Decode through `config.Scan`, so unknown keys fail.
- Dialers have no ACL block.
- Intent rule as in `authz`: an ACL key on a dialer, or `proxy_from` without
`proxy_protocol`, is an error.
- Stats: `acl_denied`, `acl_proxy_headers`, `acl_limited`, merged into the
plugin stats like `authz.Stats`.
- Docs: a security.md section, plus networking.md troubleshooting lines for
each refusal.
### 1.6 Verification
- Go tests, one per rule:
- PROXY v1 and v2 parse, including malformed, oversized and slow headers.
- A header from an unlisted peer refused; `required` without a header
refused; `LOCAL` kept.
- Deny over allow; IPv6 rules.
- Per-client connection cap; the limiter table failing closed.
- The real client reaching SCRAM throttling and session metadata.
- `test/acl-test.sh`:
- A Go stub proxy that sends v1 and v2 headers in front of each listener
type.
- nginx `stream` with `proxy_protocol on` when nginx is installed.
- Denied and allowed clients; per-client SCRAM budgets behind one proxy
address.
## 2. Hardening review of `lixenwraith/config` and `lixenwraith/toml`
Both parse untrusted-shaped input (config files, environment, command line)
and were not reviewed with the auth work. Review them as `auth` was reviewed:
adversarial lenses, reproduction before belief, a fix per confirmed finding in
the library repository, then a dependency bump here.
- toml parser:
- Limits on nesting depth, key count, string and array length, and number
sizes.
- Behaviour on invalid UTF-8, duplicate keys and tables, dotted-key and
inline-table redefinition.
- Fuzz targets (`go test -fuzz`) seeded with the TOML test suite.
- config:
- The weakly typed decoder: string-to-number overflow, duration and size
parsing, the comma split into slices.
- `maxValueDepth` and `MaxValueSize`, environment and CLI precedence, and
unknown-key reporting.
- Files:
- `PreventPathTraversal`, symlink handling and the file-size cap.
- The watcher: polling, debounce, a file replaced by rename or symlink swap,
TOCTOU between stat and read.
- Errors: values must never echo secrets (a misplaced password in a config
value reported back in an error).
- Deliverables:
- Findings per library.
- Fuzz targets committed in the library repositories.
- A note in this repository's security.md once done.
## 3. Packaging: AUR, FreeBSD ports, Debian
The foundation exists:
- The `lw` binary name.
- `make install` with `DESTDIR`, `PREFIX` and `SYSCONFDIR`.
- The `doc/lw.1` manual.
- Service files in `deploy/package/`: systemd unit, sysusers, tmpfiles and the
FreeBSD rc.d script.
- Skeletons in `deploy/package/arch/` and `deploy/package/freebsd/`.
What remains:
- Releases:
- The repository has no tags, so versions come from `git describe` and fall
back to a commit hash. Tag releases `vX.Y.Z`.
- Publish source tarballs with checksums, and sign them.
- Module path:
- The module is `logwisp`, so `go install github.com/lixenwraith/logwisp/cmd/lw@latest`
does not work.
- FreeBSD's `USES=go:modules` and Debian's dh-golang expect the canonical
path.
- Rename the module to `github.com/lixenwraith/logwisp`: a mechanical
import rewrite in one commit.
- Name check before submission:
- Arch: `pkgfile -s lw` against the official repositories, and the AUR
package list for `lw` and `logwisp`.
- Already checked: Ubuntu 26.04 and 24.04 contents and the FreeBSD 15.1
ports tree ship no `bin/lw`.
- Arch (AUR): `deploy/package/arch/PKGBUILD` already has the source URL, the
Arch Go flags (`-trimpath -buildmode=pie -mod=readonly -modcacherw`),
`backup` for the config and `check()`. Remaining:
- the checksum of a tagged tarball in place of `SKIP`, and a maintainer;
- namcap, `.SRCINFO`, and publishing `logwisp` (optionally `logwisp-git`).
- FreeBSD: `deploy/package/freebsd/` already has `USES=go:1.27,modules`,
`USE_RC_SUBR=logwisp`, `USERS`/`GROUPS`, and a pkg-plist with the `.sample`
config and the manual. Remaining:
- `distinfo` (or `GH_TUPLE`), which needs the module rename and a tag;
- a real `MAINTAINER` and a `UIDs`/`GIDs` entry for `logwisp`;
- poudriere on 14.x and 15.x, then submission through Bugzilla
(`sysutils/logwisp`).
- Debian:
- Debian policy wants every Go dependency packaged; the four `lixenwraith`
libraries are not.
- Either package them too (dh-golang), or start with an `.deb` built by the
Makefile `install` target (nfpm or `dpkg-deb`) and a PPA, and move to the
archive later.
- Shell completion for bash, zsh and fish, installed by `make install`.
- A packaging CI job: build the AUR package in an Arch container and the port
in a FreeBSD VM, run `lw --version` and `make image-check`.
internal/authz/authz.go
+22 -5
View File
@@ -414,14 +414,27 @@ func (p *Policy) Prepare(ctx context.Context, client *http.Client, baseURL strin
if p == nil || p.dialer == nil {
return nil
}
token := p.dialer.token.Load()
if token == nil || time.Now().UnixNano() >= p.dialer.renewAt.Load() {
d := p.dialer
token := d.token.Load()
if token == nil || time.Now().UnixNano() >= d.renewAt.Load() {
pin, expires := d.pin.Load(), d.expires.Load()
t, err := p.Token(ctx, client, baseURL)
if err != nil {
// A renewal put off while the old token holds (a throttled login behind
// a shared address) keeps sending on it and retries in a few seconds.
// Idle connections close first: Token dialed them with no pin, and the
// token may ride only connections the old pin verifies
var later putOff
if left := time.Duration(expires - time.Now().UnixNano()); errors.As(err, &later) && token != nil && left > 0 {
client.CloseIdleConnections()
d.pin.Store(pin)
d.token.Store(token)
d.renewAt.Store(time.Now().Add(min(5*time.Second, left/2)).UnixNano())
} else if err != nil {
return err
}
} else {
token = &t
}
}
req.Header.Set("Authorization", "Bearer "+*token)
return nil
}
@@ -561,7 +574,11 @@ func (p *Policy) LogStartup(l *log.Logger, component, id string, trustNode bool)
"hint", "set auth.allow or auth.allow_patterns to authorize named peers")
}
if p.BindsNode() {
l.Info("msg", "Node labels bound to peer identity; trust_node is ignored",
msg := "Connection node label bound to peer identity; trust_node still governs per-entry labels"
if p.binding == BindingForce {
msg = "Node labels bound to peer identity; trust_node is ignored"
}
l.Info("msg", msg,
"component", component,
"instance_id", id,
"node_binding", p.binding,
internal/authz/proxy_test.go
+1 -1
View File
@@ -177,7 +177,7 @@ func TestBrowserSessionBehindProxy(t *testing.T) {
}
// Behind a proxy logwisp never sees the certificate a client binds to, so
// only unbound proofs (logwisp auth token -unbound) log in.
// only unbound proofs (lw auth token -unbound) log in.
func TestOnlyUnboundLoginsBehindProxy(t *testing.T) {
f := newFixture(t)
l := f.listener(t, config.AuthOptions{TrustedProxies: []string{"127.0.0.1"}}, f.serverTLS, RoleListener, HTTP)
internal/authz/scram.go
+28 -7
View File
@@ -167,12 +167,12 @@ func (c *Credentials) Marshal() ([]byte, error) {
if err != nil {
return nil, err
}
header := "# logwisp SCRAM verifiers, written by `logwisp auth add-user`. Keep it private.\n"
header := "# logwisp SCRAM verifiers, written by `lw auth add-user`. Keep it private.\n"
return append([]byte(header), body...), nil
}
// ReadPassword reads a password file, trimming one trailing line break so a
// file from an editor and one from `logwisp auth add-user` agree.
// file from an editor and one from `lw auth add-user` agree.
func ReadPassword(path string) (string, error) {
data, err := os.ReadFile(path)
if err != nil {
@@ -355,7 +355,7 @@ func (p *Policy) finish(ip, nonce string, step authStep, cs *tls.ConnectionState
p.rejected.Add(1)
switch {
case step.Binding != "" && l.proxy != nil:
err = fmt.Errorf("%w; the client bound its proof to the proxy's certificate: behind trusted_proxies, log in unbound (logwisp auth token -unbound)", err)
err = fmt.Errorf("%w; the client bound its proof to the proxy's certificate: behind trusted_proxies, log in unbound (lw auth token -unbound)", err)
case step.Binding == "" && l.cb != nil:
err = fmt.Errorf("%w; the client sent an unbound proof (-unbound), but this listener binds logins to its certificate", err)
case step.Binding != "" && step.Binding != base64.StdEncoding.EncodeToString(l.cb):
@@ -566,6 +566,7 @@ type scramDialer struct {
unbound bool // HTTP logins to a listener behind a TLS-terminating proxy
token atomic.Pointer[string]
renewAt atomic.Int64 // unix nanoseconds; Prepare logs in again from then
expires atomic.Int64 // unix nanoseconds; 0 = lifetime unknown
pin atomic.Pointer[[]byte] // HTTP: certificate the token's login was bound to
failures atomic.Uint64
lastErr atomic.Pointer[string]
@@ -722,7 +723,12 @@ func (p *Policy) Token(ctx context.Context, client *http.Client, baseURL string)
if step.Token == "" {
return "", errors.New("auth: server issued no token")
}
d.renewAt.Store(renewAt(time.Now(), time.Duration(step.ExpiresIn)*time.Second).UnixNano())
now, lifetime := time.Now(), time.Duration(step.ExpiresIn)*time.Second
d.renewAt.Store(renewAt(now, lifetime).UnixNano())
d.expires.Store(0)
if lifetime > 0 {
d.expires.Store(now.Add(lifetime).UnixNano())
}
d.token.Store(&step.Token)
return step.Token, nil
}
@@ -770,6 +776,12 @@ func (d *scramDialer) stats(m map[string]any) {
}
}
// putOff marks a login the server deferred (throttled, busy) rather than
// refused: a token still held stays usable meanwhile.
type putOff struct{ error }
func (e putOff) Unwrap() error { return e.error }
// postStep sends one /auth request; any answer but 200 is a refusal
func postStep(ctx context.Context, client *http.Client, url string, body []byte) (authStep, *http.Response, error) {
var step authStep
@@ -796,7 +808,11 @@ func postStep(ctx context.Context, client *http.Client, url string, body []byte)
if msg == "" {
msg = resp.Status
}
return step, resp, fmt.Errorf("%w: server: %s", ErrRefused, msg)
err := fmt.Errorf("%w: server: %s", ErrRefused, msg)
if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode == http.StatusServiceUnavailable {
err = putOff{err}
}
return step, resp, err
case decodeErr != nil:
return step, resp, fmt.Errorf("auth: malformed answer: %w", decodeErr)
}
@@ -888,13 +904,18 @@ func remoteIP(addr string) string {
// --- Throttling ---
// throttleKey is what the limiter counts: an address, or for IPv6 its /64,
// which one host usually holds whole
// which one host usually holds whole. Link-local too, per zone: a peer picks
// any fe80::/64 address, so per address it would escape its budget.
func throttleKey(ip string) string {
addr, err := netip.ParseAddr(ip)
if err != nil || addr.Is4() {
return ip
}
return netip.PrefixFrom(addr, 64).Masked().String()
key := netip.PrefixFrom(addr, 64).Masked().String()
if zone := addr.Zone(); zone != "" {
key += "%" + zone
}
return key
}
// limiter bounds SCRAM attempts per remote address: failed or abandoned
internal/authz/scram_test.go
+86
View File
@@ -417,6 +417,15 @@ func TestDialerRefusesACheapChallenge(t *testing.T) {
}
}
// Link-local peers share one budget per link, not one across every link: the
// zone names the link and stays in the key.
func TestLinkLocalBudgetIsPerLink(t *testing.T) {
a, b, other := throttleKey("fe80::1%eth0"), throttleKey("fe80::2%eth0"), throttleKey("fe80::1%eth1")
if a != b || a == other {
t.Fatalf("keys %q, %q on one link, %q on another", a, b, other)
}
}
// Only failed attempts drain an address's budget; beyond it the listener
// answers "too many attempts" without running the exchange.
func TestOnlyFailedAttemptsAreThrottled(t *testing.T) {
@@ -734,3 +743,80 @@ func TestTokenIsRenewedBeforeExpiry(t *testing.T) {
t.Fatalf("logins after the renewal point = %d, want 2", n)
}
}
// A renewal put off (throttled, or the server busy) keeps the link sending on
// the token it still holds, and retries soon, but only to the pinned server;
// any other refusal, or the token's expiry, ends it.
func TestPutOffRenewalKeepsTheValidToken(t *testing.T) {
f := newFixture(t)
l := f.listener(t, config.AuthOptions{TokenLifetimeMS: MinTokenLifetime.Milliseconds()}, f.serverTLS, RoleListener, HTTP)
srv, _ := f.httpListener(t, l, f.serverTLS)
// An impostor with another certificate of the same CA
var answer, stolen atomic.Int64
mux := http.NewServeMux()
mux.HandleFunc("POST "+chain.AuthPath, func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, int(answer.Load()), authStep{Error: "no"})
})
mux.HandleFunc("GET /protected", func(w http.ResponseWriter, r *http.Request) { stolen.Add(1) })
imp := httptest.NewUnstartedServer(mux)
imp.TLS = f.serverTLS.Clone()
imp.TLS.Certificates = []tls.Certificate{f.serverLeaf(t, "impostor")}
imp.StartTLS()
t.Cleanup(imp.Close)
d := f.dialer(t, "edge-01", "edge-01-secret")
client := f.httpClient(d)
var reroute atomic.Pointer[string]
client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
if to := reroute.Load(); to != nil {
addr = *to
}
return (&net.Dialer{}).DialContext(ctx, network, addr)
}
prepare := func(renew bool) (*http.Request, error) {
if renew {
d.dialer.renewAt.Store(time.Now().UnixNano())
}
req, _ := http.NewRequest(http.MethodGet, srv.URL+"/protected", nil)
return req, d.Prepare(t.Context(), client, srv.URL, req)
}
held := func() string {
req, err := prepare(false)
if err != nil {
t.Fatal(err)
}
return req.Header.Get("Authorization")
}
first := held()
client.CloseIdleConnections()
to := imp.Listener.Addr().String()
reroute.Store(&to)
answer.Store(http.StatusTooManyRequests)
req, err := prepare(true)
if err != nil || req.Header.Get("Authorization") != first {
t.Fatalf("put-off renewal: header %q, %v; want the held token", req.Header.Get("Authorization"), err)
}
if ahead := time.Until(time.Unix(0, d.dialer.renewAt.Load())); ahead <= 0 || ahead > 5*time.Second {
t.Fatalf("next renewal %v ahead, want within 5 s", ahead)
}
if _, err := client.Do(req); !errors.Is(err, errPinMismatch) || stolen.Load() != 0 {
t.Fatalf("the held token went to a server the pin never checked: %v, %d requests", err, stolen.Load())
}
answer.Store(http.StatusUnauthorized)
if _, err := prepare(true); err == nil || d.dialer.token.Load() != nil {
t.Fatalf("a refused renewal kept the token: %v", err)
}
client.CloseIdleConnections()
reroute.Store(nil)
first = held()
l.Close() // every further login answers busy
if req, err := prepare(true); err != nil || req.Header.Get("Authorization") != first {
t.Fatalf("busy renewal: %v; want the held token", err)
}
d.dialer.expires.Store(time.Now().UnixNano())
if _, err := prepare(true); err == nil {
t.Fatal("an expired token was kept after a put-off renewal")
}
}
internal/config/config.go
+2 -1
View File
@@ -6,6 +6,7 @@ package config
type Config struct {
// Top-level flags for application control
ShowVersion bool `toml:"version"`
Check bool `toml:"check"` // build every plugin, report, exit: lw --check
Quiet bool `toml:"quiet"`
// Runtime behavior flags
@@ -347,7 +348,7 @@ type AuthOptions struct {
// Overrides trust_node.
NodeBinding string `toml:"node_binding"`
// scram listeners: verifier file written by `logwisp auth add-user`, and
// scram listeners: verifier file written by `lw auth add-user`, and
// the bearer token lifetime on HTTP listeners (default 15 minutes)
CredentialsFile string `toml:"credentials_file"`
TokenLifetimeMS int64 `toml:"token_lifetime_ms"`
internal/config/loader.go
+43 -3
View File
@@ -20,6 +20,9 @@ var ErrConfigNotFound = lconfig.ErrConfigNotFound
type Manager struct {
config *lconfig.Config
path string
explicit bool
read bool // the file was loaded once
specs []pipelineSpec // command-line or environment pipelines, kept across reloads
}
// Load reads the startup sources and validates the initial configuration.
@@ -29,13 +32,30 @@ func Load(args []string) (*Manager, error) {
if err != nil {
return nil, err
}
specs, configArgs, err := takePipelineSpecs(configArgs)
if err != nil {
return nil, err
}
if len(specs) == 0 {
specs = envPipelineSpecs()
}
if _, err := buildPipelines(specs); err != nil {
return nil, err
}
m := &Manager{path: configPath, explicit: isExplicit, specs: specs}
initial := defaults()
cfg, err := lconfig.NewBuilder().
WithTarget(initial).
WithEnvPrefix("LOGWISP_").
WithArgs(configArgs).
WithFile(configPath).
WithTypedValidator(ValidateConfig).
WithTypedValidator(func(cfg *Config) error {
effective := *cfg // the builder keeps cfg; only the copy is replaced
if err := m.usePipelines(&effective); err != nil {
return err
}
return ValidateConfig(&effective)
}).
WithSecurityOptions(lconfig.SecurityOptions{
PreventPathTraversal: true,
MaxFileSize: 10 * 1024 * 1024,
@@ -50,10 +70,12 @@ func Load(args []string) (*Manager, error) {
}
// A missing discovered default still permits valid CLI/env/default values.
}
m.read = err == nil
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !initial.Quiet {
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
}
return &Manager{config: cfg, path: configPath}, nil
m.config = cfg
return m, nil
}
// Snapshot validates a detached candidate before it is used to build a service.
@@ -65,6 +87,9 @@ func (m *Manager) Snapshot() (*Config, error) {
}
cfg := value.(*Config)
cfg.ConfigFile = m.path
if err := m.usePipelines(cfg); err != nil {
return nil, err
}
if err := checkFileKeys(m.path); err != nil {
return nil, fmt.Errorf("config file %q: %w", m.path, err)
}
@@ -74,12 +99,27 @@ func (m *Manager) Snapshot() (*Config, error) {
return cfg, nil
}
// usePipelines replaces the file's or default pipelines with the spec ones.
func (m *Manager) usePipelines(cfg *Config) error {
if len(m.specs) == 0 {
return nil
}
pipelines, err := buildPipelines(m.specs)
cfg.Pipelines = pipelines
return err
}
// Reload rereads the selected file, retaining the startup CLI/environment sources.
// Signals must call this even when automatic watching is disabled.
func (m *Manager) Reload() (*Config, error) {
if err := m.config.LoadFile(m.path); err != nil {
// As at startup, a discovered default that does not exist is no error, so a
// file-less instance still rebuilds and rotates its certificates. Once read,
// its removal is one: the loader would keep the removed file's values.
err := m.config.LoadFile(m.path)
if err != nil && (m.explicit || m.read || !errors.Is(err, ErrConfigNotFound)) {
return nil, fmt.Errorf("reload %q: %w", m.path, err)
}
m.read = m.read || err == nil
return m.Snapshot()
}
internal/config/loader_test.go
+8
View File
@@ -2,6 +2,7 @@ package config
import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
@@ -124,6 +125,13 @@ func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
if err != nil || !recovered.StatusReporter {
t.Fatalf("override removal/recovery: %+v %v", recovered, err)
}
// Only a missing discovered default is tolerated; a named file must exist.
if err := os.Remove("reload.toml"); err != nil {
t.Fatal(err)
}
if _, err := m.Reload(); !errors.Is(err, ErrConfigNotFound) {
t.Fatalf("reload without the named file: %v", err)
}
}
func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) {
internal/config/spec.go
+272
View File
@@ -0,0 +1,272 @@
package config
import (
"cmp"
"errors"
"fmt"
"os"
"slices"
"strconv"
"strings"
)
// specKind is one pipeline spec flag. Its environment form is LOGWISP_ plus
// the flag uppercased, '-' as '_'.
type specKind struct {
flag string
typed bool // SPEC starts with TYPE
many bool // repeatable within a pipeline; the environment adds _1.._N
apply func(p *PipelineConfig, typ string, opts map[string]any) error
}
var specKinds = []specKind{
{"source", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
ids := make([]string, len(p.PluginSources))
for i, s := range p.PluginSources {
ids[i] = s.ID
}
id, err := instanceID(typ, opts, ids)
p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
return err
}},
{"sink", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
ids := make([]string, len(p.PluginSinks))
for i, s := range p.PluginSinks {
ids[i] = s.ID
}
id, err := instanceID(typ, opts, ids)
p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
return err
}},
{"filter", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
opts["type"] = typ
f := FilterConfig{}
err := Scan(opts, &f)
p.Flow.Filters = append(p.Flow.Filters, f)
return err
}},
{"format", true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
opts["type"] = typ
p.Flow.Format = &FormatConfig{}
return Scan(opts, p.Flow.Format)
}},
// Naming a stage turns it on: the file defaults (pass, disabled) would not.
{"rate-limit", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
return Scan(opts, p.Flow.RateLimit)
}},
{"heartbeat", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
return Scan(opts, p.Flow.Heartbeat)
}},
}
// pipelineSpec is one pipeline flag or variable; a nil kind starts a pipeline.
type pipelineSpec struct {
name string // "--sink" or "LOGWISP_SINK_2", for errors
kind *specKind
value string
}
// takePipelineSpecs consumes the pipeline flags, which lixenwraith/config
// would report as unrecognized.
func takePipelineSpecs(args []string) (specs []pipelineSpec, rest []string, err error) {
for i := 0; i < len(args); i++ {
if args[i] == "--" {
return specs, append(rest, args[i:]...), nil
}
name, value, inline := strings.Cut(args[i], "=")
var kind *specKind
if name != "--pipeline" {
k := slices.IndexFunc(specKinds, func(k specKind) bool { return "--"+k.flag == name })
if k < 0 {
rest = append(rest, args[i])
continue
}
kind = &specKinds[k]
}
if !inline && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") {
i++
value = args[i]
}
if value == "" {
return nil, nil, fmt.Errorf("%s requires a value", name)
}
specs = append(specs, pipelineSpec{name, kind, value})
}
return specs, rest, nil
}
// envPipelineSpecs reads the one-pipeline environment form. An empty variable
// counts as unset, so container templates may leave optional ones blank.
func envPipelineSpecs() []pipelineSpec {
var specs []pipelineSpec
if name := os.Getenv("LOGWISP_PIPELINE"); name != "" {
specs = append(specs, pipelineSpec{"LOGWISP_PIPELINE", nil, name})
}
for i := range specKinds {
kind := &specKinds[i]
base := "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(kind.flag, "-", "_"))
type numbered struct {
n uint64
name, spec string
}
var found []numbered
for _, kv := range os.Environ() {
name, spec, _ := strings.Cut(kv, "=")
suffix, ok := strings.CutPrefix(name, base+"_")
n, err := strconv.ParseUint(suffix, 10, 64)
if name == base || ok && kind.many && err == nil {
found = append(found, numbered{n, name, spec})
}
}
slices.SortFunc(found, func(a, b numbered) int { return cmp.Or(cmp.Compare(a.n, b.n), cmp.Compare(a.name, b.name)) })
for _, f := range found {
if f.spec != "" {
specs = append(specs, pipelineSpec{f.name, kind, f.spec})
}
}
}
return specs
}
// buildPipelines turns specs, in order, into pipelines; specs before the first
// --pipeline go to one named "cli". Every call returns fresh maps.
func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
var pipelines []PipelineConfig
var seen map[*specKind]bool
for _, s := range specs {
if s.kind == nil || pipelines == nil {
name := "cli"
if s.kind == nil {
name = s.value
}
pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
seen = map[*specKind]bool{}
if s.kind == nil {
continue
}
}
p := &pipelines[len(pipelines)-1]
if seen[s.kind] && !s.kind.many {
return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
}
seen[s.kind] = true
typ, opts, err := parseSpec(s.value, s.kind.typed)
if err != nil {
return nil, fmt.Errorf("%s %w", s.name, err)
}
if err := s.kind.apply(p, typ, opts); err != nil {
return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
}
}
return pipelines, nil
}
// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
// a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {
rest, more := spec, true
if typed {
typ, rest, more = cutUnescaped(spec, ',')
if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq {
return "", nil, fmt.Errorf("%s: missing TYPE", spec)
}
typ = unescape(typ)
}
opts = map[string]any{}
for more {
var part string
part, rest, more = cutUnescaped(rest, ',')
key, value, eq := cutUnescaped(part, '=')
switch key = unescape(key); {
case !eq:
err = errors.New(`missing "="`)
case typed && key == "type":
err = errors.New(`TYPE already sets "type"`)
default:
err = setOption(opts, key, unescape(value))
}
if err != nil {
return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
}
}
return typ, opts, nil
}
// cutUnescaped cuts s at its first unescaped sep, leaving escapes in place.
func cutUnescaped(s string, sep byte) (before, after string, found bool) {
for i := 0; i < len(s); i++ {
if escapes(s, i) {
i++
} else if s[i] == sep {
return s[:i], s[i+1:], true
}
}
return s, "", false
}
func unescape(s string) string {
var b strings.Builder
for i := 0; i < len(s); i++ {
if escapes(s, i) {
i++
}
b.WriteByte(s[i])
}
return b.String()
}
// escapes keeps any other backslash literal, so regex escapes such as \d pass.
func escapes(s string, i int) bool {
return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0
}
// setOption stores value at a dotted key; a second value turns it into a list.
func setOption(opts map[string]any, key, value string) error {
segments := strings.Split(key, ".")
if slices.Contains(segments, "") {
return fmt.Errorf("empty key in %q", key)
}
table, last := opts, len(segments)-1
for i, segment := range segments[:last] {
switch existing := table[segment].(type) {
case nil:
next := map[string]any{}
table[segment], table = next, next
case map[string]any:
table = existing
default:
return fmt.Errorf("%q is both a value and a table", strings.Join(segments[:i+1], "."))
}
}
switch existing := table[segments[last]].(type) {
case nil:
table[segments[last]] = value
case string:
table[segments[last]] = []any{existing, value}
case []any:
table[segments[last]] = append(existing, value)
default:
return fmt.Errorf("%q is both a table and a value", key)
}
return nil
}
// instanceID takes the id option; an unnamed instance gets the first of TYPE,
// TYPE_2, TYPE_3... that no earlier instance of its role holds.
func instanceID(typ string, opts map[string]any, taken []string) (string, error) {
id, _ := opts["id"].(string)
if _, set := opts["id"]; set && id == "" {
return "", errors.New(`"id" takes one non-empty value`)
}
delete(opts, "id")
if id != "" {
return id, nil
}
id = typ
for n := 2; slices.Contains(taken, id); n++ {
id = typ + "_" + strconv.Itoa(n)
}
return id, nil
}
internal/config/spec_test.go
+190
View File
@@ -0,0 +1,190 @@
package config
import (
"errors"
"os"
"reflect"
"testing"
"logwisp/internal/testutil"
)
func loadPipelines(t *testing.T, args ...string) []PipelineConfig {
t.Helper()
m, err := Load(args)
if err != nil {
t.Fatal(err)
}
defer m.Close()
cfg, err := m.Snapshot()
if err != nil {
t.Fatal(err)
}
return cfg.Pipelines
}
func TestPipelineSpecGrammar(t *testing.T) {
isolateConfig(t)
got := loadPipelines(t,
"--source", `file,directory=/var/log/a\,b,pattern=*.log`,
`--source=file,id=app,directory=x\=y`,
"--source", `file,directory=c:\\logs\\`,
"--sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c",
"--filter", "include,patterns=ERROR,patterns=WARN",
"--filter", `exclude,patterns=\d{1\,3}`,
"--format", `txt,timestamp_format=Jan 2\, 2006`,
"--pipeline", "b", "--source", "null", "--sink", "null", "--sink", "null", "--heartbeat", "interval_ms=1000",
"--", "--source", "random")
want := []PipelineConfig{{
Name: "cli",
Flow: &FlowConfig{
Filters: []FilterConfig{
{Type: "include", Patterns: []string{"ERROR", "WARN"}},
{Type: "exclude", Patterns: []string{`\d{1,3}`}},
},
Format: &FormatConfig{Type: "txt", TimestampFormat: "Jan 2, 2006"},
},
PluginSources: []PluginSourceConfig{
{ID: "file", Type: "file", Config: map[string]any{"directory": "/var/log/a,b", "pattern": "*.log"}},
{ID: "app", Type: "file", Config: map[string]any{"directory": "x=y"}},
{ID: "file_2", Type: "file", Config: map[string]any{"directory": `c:\logs\`}},
},
PluginSinks: []PluginSinkConfig{{ID: "http", Type: "http", Config: map[string]any{
"port": "8080",
"auth": map[string]any{"type": "scram", "credentials_file": "users.toml"},
"tls": map[string]any{"cert_file": "c"},
}}},
}, {
Name: "b",
Flow: &FlowConfig{Heartbeat: &HeartbeatConfig{Enabled: true, IntervalMS: 1000}},
PluginSources: []PluginSourceConfig{{ID: "null", Type: "null", Config: map[string]any{}}},
PluginSinks: []PluginSinkConfig{
{ID: "null", Type: "null", Config: map[string]any{}},
{ID: "null_2", Type: "null", Config: map[string]any{}},
},
}}
if !reflect.DeepEqual(got, want) {
t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
}
for want, args := range map[string][]string{
`--sink http,port: missing "="`: {"--sink", "http,port"},
`--source directory=/x: missing TYPE`: {"--source", "directory=/x"},
`--source file,type=x: TYPE already sets "type"`: {"--source", "file,type=x"},
`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"--sink", "http,tls=1,tls.cert_file=c"},
`--sink requires a value`: {"--sink="},
`--format txt: pipeline "cli" already has one`: {"--format", "json", "--format", "txt"},
`--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"--rate-limit", "rate=1,polcy=drop"},
} {
if _, err := Load(args); err == nil || err.Error() != want {
t.Errorf("%q: err = %v, want %s", args, err, want)
}
}
}
// A spec value is one list entry, commas included: a regex quantifier or a DN
// must not split into several, wider entries.
func TestSpecValueIsOneListEntry(t *testing.T) {
isolateConfig(t)
sink := loadPipelines(t, "--source", "null", "--sink",
`tcp,auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0]
var opts TCPSinkOptions
if err := Scan(sink.Config, &opts); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(opts.Auth.AllowPatterns, []string{`^edge-\d{1,3}$`}) ||
!reflect.DeepEqual(opts.Auth.Allow, []string{"CN=a,O=b", "CN=c"}) {
t.Fatalf("allow_patterns %q, allow %q", opts.Auth.AllowPatterns, opts.Auth.Allow)
}
}
// One pipeline from LOGWISP_*: numbered variables follow the bare one in
// numeric order, only repeatable kinds are numbered, and empty means unset.
func TestEnvironmentPipeline(t *testing.T) {
isolateConfig(t)
for name, value := range map[string]string{
"LOGWISP_PIPELINE": "edge",
"LOGWISP_SOURCE": "random",
"LOGWISP_SOURCE_10": "null,id=ten",
"LOGWISP_SOURCE_2": "null,id=two",
"LOGWISP_SINK_1": "console",
"LOGWISP_FILTER": "",
"LOGWISP_FORMAT_1": "raw",
"LOGWISP_RATE_LIMIT": "rate=10",
"LOGWISP_LOGGING_LEVEL": "debug",
} {
t.Setenv(name, value)
}
m, err := Load(nil)
if err != nil {
t.Fatal(err)
}
defer m.Close()
cfg, err := m.Snapshot()
if err != nil {
t.Fatal(err)
}
p := cfg.Pipelines
if len(p) != 1 || p[0].Name != "edge" || len(p[0].PluginSources) != 3 || p[0].PluginSources[1].ID != "two" ||
p[0].PluginSources[2].ID != "ten" || len(p[0].PluginSinks) != 1 || p[0].Flow.Filters != nil || p[0].Flow.Format != nil ||
*p[0].Flow.RateLimit != (RateLimitConfig{Rate: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" {
t.Fatalf("environment pipeline: %+v %+v", p, cfg.Logging)
}
}
func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) {
isolateConfig(t)
t.Setenv("LOGWISP_SOURCE", "random")
t.Setenv("LOGWISP_SINK", "console")
got := loadPipelines(t, "--source", "null", "--sink", "null")
if len(got) != 1 || len(got[0].PluginSources) != 1 || got[0].PluginSources[0].Type != "null" ||
len(got[0].PluginSinks) != 1 || got[0].PluginSinks[0].Type != "null" {
t.Fatalf("environment specs mixed into command-line pipelines: %+v", got)
}
}
// The file's pipelines are dropped unvalidated; its other keys still apply.
func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
isolateConfig(t)
testutil.WriteFile(t, "file.toml", "[logging]\nlevel = \"warn\"\n[[pipelines]]\nname = \"file\"\n")
m, err := Load([]string{"-c", "file.toml", "--source", "null", "--sink", "null", "--status_reporter=false"})
if err != nil {
t.Fatal(err)
}
defer m.Close()
if unknown := m.config.UnknownCLIKeys(); len(unknown) != 0 {
t.Fatalf("spec flags reached the schema parser: %v", unknown)
}
cfg, err := m.Snapshot()
if err != nil || len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "cli" || cfg.Logging.Level != "warn" || cfg.StatusReporter {
t.Fatalf("snapshot: %+v %v", cfg, err)
}
}
// Without a file a reload still rebuilds; once the discovered default
// appears it is read, and its pipelines still yield to the specs. Its later
// removal fails the reload instead of keeping the removed file's values.
func TestReloadKeepsSpecPipelines(t *testing.T) {
isolateConfig(t)
m, err := Load([]string{"--source", "random,special=true", "--sink", "null"})
if err != nil {
t.Fatal(err)
}
defer m.Close()
first, err := m.Reload()
if err != nil || !first.StatusReporter {
t.Fatalf("reload without a file: %+v %v", first, err)
}
first.Pipelines[0].PluginSources[0].Config["special"] = "mutated"
testutil.WriteFile(t, "logwisp.toml", "status_reporter = false\n[[pipelines]]\nname = \"file\"\n")
next, err := m.Reload()
if err != nil || next.StatusReporter || len(next.Pipelines) != 1 || next.Pipelines[0].Name != "cli" ||
next.Pipelines[0].PluginSources[0].Config["special"] != "true" {
t.Fatalf("reload: %+v %v", next, err)
}
if err := os.Remove("logwisp.toml"); err != nil {
t.Fatal(err)
}
if _, err := m.Reload(); !errors.Is(err, ErrConfigNotFound) {
t.Fatalf("reload after the file was removed: %v", err)
}
}
internal/config/validate.go
+13 -7
View File
@@ -94,15 +94,17 @@ func validateLogConfig(cfg *LogConfig) error {
// not declare: a misspelled tls or auth key must fail, not silently disable
// the protection it was meant to configure.
func Scan(configMap map[string]any, target any) error {
if err := unknownKeys(configMap, reflect.TypeOf(target), ""); err != nil {
if err := checkKeys(configMap, reflect.TypeOf(target), ""); err != nil {
return err
}
return lconfig.ScanMap(configMap, target)
}
// unknownKeys walks nested tables against the toml tags of t. Map-typed fields
// hold free-form keys and are not descended into.
func unknownKeys(m map[string]any, t reflect.Type, prefix string) error {
// checkKeys walks nested tables against the toml tags of t, refusing unknown
// keys and making a lone string for a list field a one-entry list: the weakly
// typed decoder would split it at commas, which a pattern or a DN may hold.
// Map-typed fields hold free-form keys and are not descended into.
func checkKeys(m map[string]any, t reflect.Type, prefix string) error {
for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice {
t = t.Elem()
}
@@ -122,8 +124,12 @@ func unknownKeys(m map[string]any, t reflect.Type, prefix string) error {
}
var tables []map[string]any
switch v := m[key].(type) {
case string:
if ft.Kind() == reflect.Slice && ft.Elem().Kind() == reflect.String {
m[key] = []string{v}
}
case map[string]any:
if err := unknownKeys(v, ft, prefix+key+"."); err != nil {
if err := checkKeys(v, ft, prefix+key+"."); err != nil {
return err
}
case []map[string]any:
@@ -136,7 +142,7 @@ func unknownKeys(m map[string]any, t reflect.Type, prefix string) error {
}
}
for i, table := range tables {
if err := unknownKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil {
if err := checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil {
return err
}
}
@@ -160,5 +166,5 @@ func checkFileKeys(path string) error {
return err
}
delete(root, "config_file") // runtime metadata, documented as ignored in the file
return unknownKeys(root, reflect.TypeOf(Config{}), "")
return checkKeys(root, reflect.TypeOf(Config{}), "")
}
internal/core/const.go
+22
View File
@@ -1,9 +1,31 @@
package core
import (
"fmt"
"net/netip"
"strings"
"time"
)
// Network is what a listener on, or a dialer to, host uses: tcp4 for an IPv4
// literal or "" (the IPv4 wildcard, as the default 0.0.0.0), tcp6 for an IPv6
// literal, which Go binds IPv6-only, and tcp for a hostname, which resolves.
func Network(host string) (string, error) {
addr, err := netip.ParseAddr(host)
switch {
case host == "" || err == nil && addr.Is4():
return "tcp4", nil
case err == nil && addr.Is4In6():
// tcp6 refuses it at bind or dial, long after the config loaded
return "", fmt.Errorf("%q is an IPv4-mapped address; write it as IPv4", host)
case err == nil:
return "tcp6", nil
case strings.ContainsAny(host, "[]:"):
return "", fmt.Errorf("%q is neither an address nor a hostname (write an IPv6 address without brackets or port)", host)
}
return "tcp", nil
}
const (
MaxLogEntryBytes = 1024 * 1024
internal/core/const_test.go
+53
View File
@@ -0,0 +1,53 @@
package core
import (
"net"
"testing"
"logwisp/internal/testutil"
)
// The host literal picks the family, strictly: an IPv4 literal or the empty
// host is IPv4 only, an IPv6 literal (the wildcard :: too) IPv6 only, and a
// hostname resolves. Brackets and ports belong to addresses, not hosts, and
// an IPv4-mapped address is IPv4 written as IPv6.
func TestNetworkFollowsTheHostLiteral(t *testing.T) {
for _, tc := range []struct{ host, want string }{
{"127.0.0.1", "tcp4"}, {"0.0.0.0", "tcp4"}, {"", "tcp4"},
{"::1", "tcp6"}, {"::", "tcp6"}, {"fe80::1%eth0", "tcp6"},
{"relay.example", "tcp"}, {"localhost", "tcp"},
{"[::1]", ""}, {"::1]", ""}, {"10.0.0.1:80", ""}, {"::ffff:10.0.0.1", ""},
} {
got, err := Network(tc.host)
if got != tc.want || (err != nil) != (tc.want == "") {
t.Errorf("Network(%q) = %q, %v; want %q", tc.host, got, err, tc.want)
}
}
testutil.RequireIPv6(t)
network, _ := Network("::")
ln, err := net.Listen(network, "[::]:0")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
c.Close()
}
}()
_, port, _ := net.SplitHostPort(ln.Addr().String())
if c, err := net.Dial("tcp4", net.JoinHostPort("127.0.0.1", port)); err == nil {
c.Close()
t.Error(":: accepted an IPv4 connection")
}
c, err := net.Dial("tcp6", net.JoinHostPort("::1", port))
if err != nil {
t.Fatalf(":: refused [::1]: %v", err)
}
c.Close()
}
internal/core/flow.go
-1
View File
@@ -24,4 +24,3 @@ type TransportEvent struct {
// Structured entry for re-serializing sinks (chain links). Zero Time => absent
Entry LogEntry
}
internal/core/listen.go
+49
View File
@@ -0,0 +1,49 @@
package core
import (
"context"
"fmt"
"net"
)
// lookupNetIP resolves listener hostnames; tests replace it
var lookupNetIP = net.DefaultResolver.LookupNetIP
// Listen is lc.Listen, but a hostname first resolves to one address, IPv4 when
// it has one (as Go picks), listened on in that address's family alone: Go
// would bind a name for a wildcard address in both families.
func Listen(ctx context.Context, lc *net.ListenConfig, network, addr string) (net.Listener, error) {
network, addr, err := pinFamily(ctx, network, addr)
if err != nil {
return nil, err
}
return lc.Listen(ctx, network, addr)
}
func pinFamily(ctx context.Context, network, addr string) (string, string, error) {
if network != "tcp" {
return network, addr, nil
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return "", "", err
}
ips, err := lookupNetIP(ctx, "ip", host)
if err != nil {
return "", "", err
}
if len(ips) == 0 {
return "", "", fmt.Errorf("%s: no address", host)
}
ip := ips[0].Unmap()
for _, a := range ips {
if a.Unmap().Is4() {
ip = a.Unmap()
break
}
}
if network, err = Network(ip.String()); err != nil {
return "", "", err
}
return network, net.JoinHostPort(ip.String(), port), nil
}
internal/core/listen_test.go
+34
View File
@@ -0,0 +1,34 @@
package core
import (
"context"
"net/netip"
"testing"
)
// A hostname listener binds one of its addresses, IPv4 when it has one, in
// that address's family alone, a name for a wildcard address too.
func TestHostnameListenerKeepsToOneFamily(t *testing.T) {
names := map[string][]netip.Addr{
"wild4": {netip.MustParseAddr("0.0.0.0")},
"wild6": {netip.MustParseAddr("::")},
"both": {netip.MustParseAddr("::1"), netip.MustParseAddr("::ffff:127.0.0.1")},
}
saved := lookupNetIP
t.Cleanup(func() { lookupNetIP = saved })
lookupNetIP = func(_ context.Context, _, host string) ([]netip.Addr, error) { return names[host], nil }
for _, tc := range []struct{ network, addr, wantNetwork, wantAddr string }{
{"tcp", "wild4:80", "tcp4", "0.0.0.0:80"},
{"tcp", "wild6:80", "tcp6", "[::]:80"},
{"tcp", "both:80", "tcp4", "127.0.0.1:80"},
{"tcp6", "[::1]:80", "tcp6", "[::1]:80"},
} {
network, addr, err := pinFamily(t.Context(), tc.network, tc.addr)
if err != nil || network != tc.wantNetwork || addr != tc.wantAddr {
t.Errorf("%s %s: %s %s, %v; want %s %s", tc.network, tc.addr, network, addr, err, tc.wantNetwork, tc.wantAddr)
}
}
if _, _, err := pinFamily(t.Context(), "tcp", "none:80"); err == nil {
t.Error("a name without addresses was accepted")
}
}
internal/flow/flow.go
-1
View File
@@ -161,4 +161,3 @@ func (f *Flow) GetStats() map[string]any {
return stats
}
internal/flow/heartbeat.go
+3 -5
View File
@@ -137,12 +137,10 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent
// Use flow's formatter for consistent formatting
if adapter, ok := hg.formatter.(*format.FormatterAdapter); ok {
// Customize flags for heartbeat if needed
customFlags := int64(0)
// The formatter shows a timestamp unless told otherwise
customFlags := formatter.FlagDefault
if !hg.config.IncludeTimestamp {
// Remove timestamp flag if not wanted
customFlags = formatter.FlagShowLevel
} else {
customFlags = formatter.FlagDefault
customFlags = formatter.FlagShowLevel | formatter.FlagNoTimestamp
}
payload, err = adapter.FormatWithFlags(entry, customFlags)
} else {
internal/flow/heartbeat_test.go
+29
View File
@@ -0,0 +1,29 @@
package flow
import (
"bytes"
"testing"
"time"
"logwisp/internal/config"
"logwisp/internal/format"
)
// include_timestamp = false drops the timestamp the formatter shows by default
func TestHeartbeatTimestampFollowsIncludeTimestamp(t *testing.T) {
at := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
for _, include := range []bool{false, true} {
f, err := format.NewFormatter(&config.FormatConfig{Type: "txt"})
if err != nil {
t.Fatal(err)
}
hg, err := NewHeartbeatGenerator(&config.HeartbeatConfig{Enabled: true, IncludeTimestamp: include}, f, nil)
if err != nil {
t.Fatal(err)
}
payload := hg.generateHeartbeat(at).Payload
if got := bytes.Contains(payload, []byte("2026-01-02")); got != include {
t.Errorf("include_timestamp=%v: payload %q", include, payload)
}
}
}
internal/sink/file/file.go
+9 -5
View File
@@ -36,6 +36,7 @@ type FileSink struct {
// Application
input chan core.TransportEvent
writer *log.Logger // internal logger for file writing
writerConfig *log.Config
logger *log.Logger // application logger
// Runtime
@@ -117,9 +118,9 @@ func NewFileSinkPlugin(
writerConfig.ShowLevel = false
writerConfig.Format = "raw"
// Create internal logger for file writing
writer := log.NewLogger()
if err := writer.ApplyConfig(writerConfig); err != nil {
// Validated here, applied in Start: applying creates the directory and
// file, which lw --check and a rejected reload must not do
if err := writerConfig.Validate(); err != nil {
return nil, fmt.Errorf("failed to initialize file writer: %w", err)
}
@@ -128,7 +129,8 @@ func NewFileSinkPlugin(
proxy: proxy,
config: opts,
input: make(chan core.TransportEvent, opts.BufferSize),
writer: writer,
writer: log.NewLogger(),
writerConfig: writerConfig,
done: make(chan struct{}),
logger: logger,
}
@@ -168,7 +170,9 @@ func (fs *FileSink) Input() chan<- core.TransportEvent {
// Start begins the processing loop for the sink
func (fs *FileSink) Start(ctx context.Context) error {
// Start the internal file writer
if err := fs.writer.ApplyConfig(fs.writerConfig); err != nil {
return fmt.Errorf("failed to initialize file writer: %w", err)
}
if err := fs.writer.Start(); err != nil {
return fmt.Errorf("failed to start file writer: %w", err)
}
internal/sink/http/http.go
+7 -2
View File
@@ -57,6 +57,7 @@ type HTTPSink struct {
// Configuration
config *config.HTTPSinkOptions
addr string
network string
// Network
server *http.Server
@@ -116,6 +117,10 @@ func NewHTTPSinkPlugin(
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
network, err := core.Network(opts.Host)
if err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if opts.StreamPath == "" {
opts.StreamPath = DefaultHTTPStreamPath
} else if !strings.HasPrefix(opts.StreamPath, "/") {
@@ -166,6 +171,7 @@ func NewHTTPSinkPlugin(
proxy: proxy,
config: opts,
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
network: network,
input: make(chan core.TransportEvent, opts.BufferSize),
done: make(chan struct{}),
logger: logger,
@@ -217,10 +223,9 @@ func (h *HTTPSink) Input() chan<- core.TransportEvent {
// Start binds the listener and serves stream/status endpoints
func (h *HTTPSink) Start(ctx context.Context) error {
// IPv4-only, parity with existing network sinks.
// TLS is applied via server.TLSConfig + ServeTLS below, not by wrapping
// ln; net/http then owns handshake, ALPN (h2), and per-conn errors.
ln, err := net.Listen("tcp4", h.addr)
ln, err := core.Listen(ctx, &net.ListenConfig{}, h.network, h.addr)
if err != nil {
return fmt.Errorf("http sink bind %s: %w", h.addr, err)
}
internal/sink/http/web/login.js
+17 -2
View File
@@ -1,4 +1,4 @@
import { login } from "./scram.js";
import { cookiesUsable, login, loginUnavailable } from "./scram.js";
const base = new URL("..", location.href);
const form = document.getElementById("login");
@@ -44,5 +44,20 @@ form.addEventListener("submit", async (event) => {
}
});
// A sign-in here only sets a cookie; the viewer, when the sink serves it,
// signs in by itself and keeps a token in memory instead
async function cookiesUnavailable() {
status.textContent = "This browser keeps no cookie for this site, so a sign-in here would not last.";
const viewer = await fetch("view", { method: "HEAD", cache: "no-store", redirect: "error" }).catch(() => null);
if (!viewer?.ok) return;
const link = document.createElement("a");
link.href = "view";
link.textContent = "Sign in on the viewer instead.";
status.append(" ", link);
}
// Enabled only once this script runs, so a failed load cannot submit the form natively
button.disabled = false;
const unavailable = loginUnavailable();
if (unavailable) status.textContent = unavailable;
else if (cookiesUsable()) button.disabled = false;
else cookiesUnavailable();
internal/sink/http/web/scram.js
+122 -14
View File
@@ -16,11 +16,14 @@ const encoder = new TextEncoder();
/**
* Logs in to the logwisp http sink mounted at base (a URL ending in "/"), per
* the wire protocol in logwisp's doc/scram-auth-plan.md. The session arrives as
* an HttpOnly cookie; the promise resolves only once the server has proved it
* holds this user's verifier. onProgress(fraction) follows the Argon2 pass.
* logwisp's doc/scram-auth-plan.md; it resolves only once the server has proved
* it holds this user's verifier. The session is an HttpOnly cookie, or with
* session "token" the returned token. onProgress(fraction) follows Argon2.
*/
export async function login(base, username, password, { onProgress } = {}) {
export async function login(base, username, password, { onProgress, session = "cookie" } = {}) {
if (session !== "cookie" && session !== "token") {
throw new Error('session must be "cookie" or "token"');
}
const pw = checkCredentials(username, password);
subtle(); // an insecure page fails before anything is sent
const url = new URL("auth", base);
@@ -29,7 +32,8 @@ export async function login(base, username, password, { onProgress } = {}) {
const first = await post(url, { logwisp: 1, scram: { username, client_nonce: clientNonce } });
const { proof, serverSignature } = await prove(username, pw, clientNonce, first.challenge,
MIN_TIME, MIN_MEMORY, onProgress);
const step = await post(url, { proof, session: "cookie" });
// Without "session" the server answers with the token in the body
const step = await post(url, session === "cookie" ? { proof, session } : { proof });
const final = step.final;
if (!final || typeof final !== "object") {
throw new Error("server sent no final message");
@@ -40,15 +44,19 @@ export async function login(base, username, password, { onProgress } = {}) {
throw new Error("server could not prove it holds this user's verifier");
}
const expiresIn = Number.isSafeInteger(step.expires_in) && step.expires_in > 0 ? step.expires_in : 0;
return { username, expiresIn };
if (session === "cookie") return { username, expiresIn };
if (typeof step.token !== "string" || step.token === "") {
throw new Error("server issued no token");
}
return { username, expiresIn, token: step.token };
}
// Logout posts to auth itself: a clearing cookie from any other path would
// get another default path and miss the session cookie
export async function logout(base) {
// get another default path and miss the session cookie. A token is revoked.
export async function logout(base, { token } = {}) {
const res = await send(new URL("auth", base), {
method: "POST",
headers: { "Content-Type": "application/json" },
headers: { "Content-Type": "application/json", ...bearer(token) },
body: JSON.stringify({ logout: true }),
});
if (res.status !== 200 && res.status !== 204) {
@@ -56,6 +64,101 @@ export async function logout(base) {
}
}
// Why no login can run on this page, or "": an insecure page refuses the probe's
// Secure cookie too, so a page asks this before cookiesUsable().
export function loginUnavailable() {
return globalThis.crypto?.subtle ? "" : "WebCrypto unavailable: the page must be served over HTTPS";
}
// Whether the session cookie would stick. Chromium blocking every cookie still
// reports navigator.cookieEnabled, so a cookie with the session's attributes
// must also come back.
export function cookiesUsable() {
if (!globalThis.navigator?.cookieEnabled) return false;
document.cookie = "logwisp_probe=1; Secure; SameSite=Strict";
const usable = document.cookie.split("; ").includes("logwisp_probe=1");
document.cookie = "logwisp_probe=; Max-Age=0; Secure; SameSite=Strict";
return usable;
}
/**
* Reads an event stream through fetch, which can send the bearer EventSource
* cannot. onEvent({type, data, lastEventId}) gets each event EventSource would
* dispatch. Resolves to {lastEventId, retry} when the server ends the stream;
* rejects with err.status on an answer other than 200, or on signal's abort.
*/
export async function stream(url, { token, signal, onEvent } = {}) {
const res = await send(new URL(url, globalThis.location?.href), {
headers: { Accept: "text/event-stream", ...bearer(token) },
signal,
});
const type = res.headers.get("Content-Type")?.split(";")[0].trim().toLowerCase();
if (res.status !== 200 || type !== "text/event-stream") {
res.body?.cancel().catch(() => {});
const err = new Error(res.status !== 200 ? `stream refused: ${res.status}` : "not an event stream");
err.status = res.status;
throw err;
}
const parser = eventParser(onEvent);
const reader = res.body.getReader();
const decoder = new TextDecoder(); // UTF-8, dropping a leading BOM as the spec asks
try {
for (let r = await reader.read(); !r.done; r = await reader.read()) {
parser.feed(decoder.decode(r.value, { stream: true }));
}
} finally {
reader.cancel().catch(() => {});
}
return parser.result();
}
// The event stream interpretation of the HTML standard; a CR that ends one
// chunk may be the first half of a CRLF the next chunk completes.
function eventParser(onEvent) {
let line = "", afterCR = false, data = "", type = "", idBuffer = "", lastEventId = "", retry = null;
const field = (name, value) => {
switch (name) {
case "event": type = value; break;
case "data": data += value + "\n"; break;
case "id": if (!value.includes("\0")) idBuffer = value; break;
case "retry": if (/^[0-9]+$/.test(value)) retry = Number(value); break;
}
};
const dispatch = () => {
lastEventId = idBuffer;
if (data !== "") onEvent?.({ type: type || "message", data: data.slice(0, -1), lastEventId });
data = type = "";
};
const interpret = (l) => {
if (l === "") return dispatch();
if (l.startsWith(":")) return;
const colon = l.indexOf(":");
if (colon < 0) return field(l, "");
const value = l.slice(colon + 1);
field(l.slice(0, colon), value.startsWith(" ") ? value.slice(1) : value);
};
const breaks = /[\r\n]/g;
return {
feed(text) {
let i = 0;
if (afterCR && text.startsWith("\n")) i = 1;
if (text !== "") afterCR = false;
for (breaks.lastIndex = i; breaks.exec(text); i = breaks.lastIndex) {
const end = breaks.lastIndex - 1;
interpret(line + text.slice(i, end));
line = "";
if (text[end] === "\r") {
if (end + 1 === text.length) afterCR = true;
else if (text[end + 1] === "\n") breaks.lastIndex++;
}
}
line += text.slice(i);
},
// An event the stream ended before its blank line is discarded
result: () => ({ lastEventId, retry }),
};
}
// Test hook: the proof for a fixed client nonce, under an explicit Argon2 floor
export async function _proofForTest(username, password, clientNonce, challenge, minTime, minMemory) {
const { proof, serverSignature } = await prove(username, checkCredentials(username, password),
@@ -135,19 +238,24 @@ async function post(url, body) {
return step;
}
// A redirect would replay the proof to wherever it points
// A redirect would replay the proof or the token to wherever it points
async function send(url, init) {
try {
return await fetch(url, { ...init, credentials: "same-origin", cache: "no-store", redirect: "error" });
} catch {
} catch (err) {
if (err?.name === "AbortError") throw err;
throw new Error(`cannot reach ${url.pathname}`);
}
}
function bearer(token) {
return token ? { Authorization: `Bearer ${token}` } : {};
}
function subtle() {
const s = globalThis.crypto?.subtle;
if (!s) throw new Error("WebCrypto unavailable: the page must be served over HTTPS");
return s;
const reason = loginUnavailable();
if (reason) throw new Error(reason);
return crypto.subtle;
}
async function hmac(key, message) {
internal/sink/http/web/scram.test.mjs
+99 -1
View File
@@ -1,6 +1,6 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { argon2id, _proofForTest } from "./scram.js";
import { argon2id, _proofForTest, login, logout, stream } from "./scram.js";
const hex = (b) => Buffer.from(b).toString("hex");
const unhex = (s) => new Uint8Array(Buffer.from(s, "hex"));
@@ -42,3 +42,101 @@ test("proof and server signature match lixenwraith/auth", async () => {
server_signature: "0HKxosIZvdoAzstS4dEkNgoYvcDpFnOgwBjyhiSsK00=",
});
});
const STREAM = "https://site.test/logs/stream";
// A fetch answering with an event stream sent in the given chunks
function sseFetch(t, chunks, { status = 200, type = "text/event-stream" } = {}) {
return t.mock.method(globalThis, "fetch", async () => new Response(new ReadableStream({
start(c) {
for (const chunk of chunks) c.enqueue(new TextEncoder().encode(chunk));
c.close();
},
}), { status, headers: { "Content-Type": type } }));
}
async function events(t, chunks) {
sseFetch(t, chunks);
const got = [];
const end = await stream(STREAM, { onEvent: (e) => got.push(e) });
return { got, end };
}
// A split CRLF is one break, so "d" and "e" stay one event; a kept BOM would
// rename the first field and drop "a".
test("stream splits lines at CRLF, LF and a lone CR, across chunks", async (t) => {
const { got } = await events(t, ["\uFEFFdata: a\r\ndata: b\rdata: c\n\r", "\n", "data: d\r", "\ndata: e\r", "\r\n"]);
assert.deepEqual(got, [
{ type: "message", data: "a\nb\nc", lastEventId: "" },
{ type: "message", data: "d\ne", lastEventId: "" },
]);
});
test("stream interprets fields as the HTML standard does", async (t) => {
const { got, end } = await events(t, [
": comment\nevent: connected\ndata:{\"a\":1}\nid: 7\n\n",
"data\n\n",
"data: one space kept\nunknown: x\nid: 8\0\nretry: 3000\nretry: 15x\n\n",
"id: 8\nevent: no-data\n\n",
"id\ndata: never ended\n",
]);
assert.deepEqual(got, [
{ type: "connected", data: "{\"a\":1}", lastEventId: "7" },
{ type: "message", data: "", lastEventId: "7" },
{ type: "message", data: " one space kept", lastEventId: "7" },
]);
assert.deepEqual(end, { lastEventId: "8", retry: 3000 });
});
test("stream presents the bearer and refuses anything but a 200 event stream", async (t) => {
const fetch = sseFetch(t, []);
await stream(STREAM, { token: "tok" });
const init = fetch.mock.calls[0].arguments[1];
assert.equal(init.headers.Authorization, "Bearer tok");
assert.equal(init.redirect, "error");
fetch.mock.restore();
sseFetch(t, [], { status: 401 });
await assert.rejects(stream(STREAM, { token: "tok" }), { status: 401 });
sseFetch(t, [], { type: "text/html" });
await assert.rejects(stream(STREAM), { message: "not an event stream" });
});
test("stream rejects with the AbortError once its signal aborts", async (t) => {
t.mock.method(globalThis, "fetch", async (url, { signal }) => signal.throwIfAborted() ?? new Response(new ReadableStream({
start(c) {
c.enqueue(new TextEncoder().encode("data: first\n\n"));
signal.addEventListener("abort", () => c.error(signal.reason));
},
}), { headers: { "Content-Type": "text/event-stream" } }));
const ac = new AbortController();
await assert.rejects(stream(STREAM, { signal: ac.signal, onEvent: () => ac.abort() }), { name: "AbortError" });
await assert.rejects(stream(STREAM, { signal: AbortSignal.abort() }), { name: "AbortError" });
});
// The mock server signs with the client's own derivation at the Argon2 floor
test("token mode returns the token, and logout presents it", async (t) => {
const challenge = { salt: Buffer.alloc(16, 9).toString("base64"), argon_time: 3, argon_memory: 65536, argon_threads: 1 };
const sent = [];
t.mock.method(globalThis, "fetch", async (url, init) => {
const body = JSON.parse(init.body);
sent.push({ body, headers: init.headers });
if (body.scram) {
challenge.client_nonce = body.scram.client_nonce;
challenge.full_nonce = body.scram.client_nonce + "server-nonce";
return Response.json({ challenge });
}
if (body.proof) {
const want = await _proofForTest("alice", "password123", challenge.client_nonce, challenge, 3, 65536);
challenge.proof_ok = body.proof.client_proof === want.client_proof;
return Response.json({ final: { username: "alice", server_signature: want.server_signature }, token: "tok", expires_in: 900 });
}
return new Response(null, { status: 204 });
});
const base = "https://site.test/logs/";
assert.deepEqual(await login(base, "alice", "password123", { session: "token" }),
{ username: "alice", expiresIn: 900, token: "tok" });
assert.ok(challenge.proof_ok);
assert.equal("session" in sent[1].body, false);
await logout(base, { token: "tok" });
assert.equal(sent[2].headers.Authorization, "Bearer tok");
});
internal/sink/http/web/style.css
+9
View File
@@ -115,3 +115,12 @@ progress {
white-space: pre-wrap;
overflow-wrap: anywhere;
}
[hidden] {
display: none !important;
}
.note {
margin: 0;
color: var(--muted);
}
internal/sink/http/web/view.html
+8
View File
@@ -14,6 +14,14 @@
<span id="state" role="status">connecting…</span>
<button id="logout" type="button">Sign out</button>
</header>
<form id="login" class="card" method="post" hidden>
<p class="note">This browser keeps no cookie for this site, so the session lives in this page and ends when it closes.</p>
<label>Username <input name="username" autocomplete="username" required></label>
<label>Password <input name="password" type="password" autocomplete="current-password" required></label>
<button type="submit">Sign in</button>
<progress id="progress" max="1" hidden></progress>
<p id="status" role="status"></p>
</form>
<main id="log"></main>
</body>
</html>
internal/sink/http/web/view.js
+126 -21
View File
@@ -1,4 +1,4 @@
import { logout } from "./scram.js";
import { cookiesUsable, login, loginUnavailable, logout, stream } from "./scram.js";
const base = new URL("..", location.href);
// The server writes its status path here, relative to the mount
@@ -6,7 +6,22 @@ const statusPath = document.querySelector('meta[name="logwisp-status"]').content
const MAX_LINES = 5000;
const log = document.getElementById("log");
const state = document.getElementById("state");
let source = null;
const signOut = document.getElementById("logout");
const form = document.getElementById("login");
const progress = document.getElementById("progress");
const status = document.getElementById("status");
const unavailable = loginUnavailable();
// Without cookies the session is a token in this variable only: never stored,
// so it ends with the page, or earlier with its lifetime or sign-out
let tokenMode = !unavailable && !cookiesUsable();
let token = null;
// The login page comes back with this fragment: a 401 now means the browser
// dropped the session cookie
let fresh = location.hash === "#signed-in";
if (fresh) history.replaceState(null, "", location.pathname + location.search);
let stopStream = null;
let timer = null;
function append(line) {
const atBottom = log.scrollTop + log.clientHeight >= log.scrollHeight - 4;
@@ -18,50 +33,140 @@ function append(line) {
}
// The status answer names the stream path; its 401 is the only way to tell an
// expired session apart, since EventSource hides the status of a failure.
// ended session apart, since EventSource hides the status of a failure.
async function connect() {
if (tokenMode && !token) return showLogin("");
let res;
try {
res = await fetch(new URL("./" + statusPath, base), { credentials: "same-origin", cache: "no-store", redirect: "error" });
res = await fetch(new URL("./" + statusPath, base), {
headers: token ? { Authorization: `Bearer ${token}` } : {},
credentials: "same-origin", cache: "no-store", redirect: "error",
});
} catch {
return retry("server unreachable");
}
if (res.status === 401) {
location.replace(new URL("login?next=view", location.href).href);
return;
}
if (res.status === 401) return sessionEnded();
fresh = false;
const path = res.ok ? (await res.json().catch(() => ({})))?.endpoints?.stream : null;
if (typeof path !== "string" || !path.startsWith("/")) {
return retry(`status unavailable (${res.status})`);
}
// Stream paths are absolute in logwisp; the proxy may mount it under a prefix
source = new EventSource(new URL("./" + path.replace(/^\/+/, ""), base));
source.addEventListener("connected", () => { state.textContent = "live"; });
source.addEventListener("disconnect", () => {
source.close();
const url = new URL("./" + path.replace(/^\/+/, ""), base);
if (tokenMode) streamWithToken(url);
else streamWithCookie(url);
}
function onEvent(type, data) {
if (type === "connected") state.textContent = "live";
if (type === "disconnect") {
stopStream();
retry("server shut down");
});
source.onmessage = (event) => event.data.split("\n").forEach(append);
}
if (type === "message") data.split("\n").forEach(append);
}
function streamWithCookie(url) {
const source = new EventSource(url);
stopStream = () => source.close();
for (const type of ["connected", "disconnect", "message"]) {
source.addEventListener(type, (event) => onEvent(type, event.data));
}
source.onerror = () => {
if (source.readyState === EventSource.CLOSED) retry("disconnected");
else state.textContent = "reconnecting…";
};
}
function retry(reason) {
state.textContent = `${reason}, retrying…`;
setTimeout(connect, 5000);
async function streamWithToken(url) {
const abort = new AbortController();
stopStream = () => abort.abort();
let refused = 0;
try {
await stream(url, { token, signal: abort.signal, onEvent: (event) => onEvent(event.type, event.data) });
} catch (err) {
refused = err.status ?? 0;
}
if (abort.signal.aborted) return;
if (refused === 401) sessionEnded();
else retry(refused ? `stream refused (${refused})` : "disconnected");
}
document.getElementById("logout").addEventListener("click", async () => {
source?.close();
function sessionEnded() {
if (!tokenMode && !fresh) {
location.replace(new URL("login?next=" + encodeURIComponent("view#signed-in"), location.href).href);
return;
}
const expired = token !== null;
tokenMode = true;
token = null;
showLogin(expired ? "The session ended; sign in again." : "");
}
function showLogin(message) {
stopStream?.();
clearTimeout(timer);
state.textContent = "signed out";
signOut.hidden = true;
status.textContent = message;
form.hidden = false;
form.elements.username.focus();
}
function retry(reason) {
state.textContent = `${reason}, retrying…`;
clearTimeout(timer);
timer = setTimeout(connect, 5000);
}
form.addEventListener("submit", async (event) => {
event.preventDefault();
const fields = new FormData(form);
const button = form.querySelector("button");
button.disabled = true;
progress.removeAttribute("value");
progress.hidden = false;
status.textContent = "Deriving the key, this takes a few seconds…";
try {
await logout(base);
({ token } = await login(base, fields.get("username"), fields.get("password"), {
session: "token",
onProgress: (fraction) => { progress.value = fraction; },
}));
} catch (err) {
status.textContent = err.message;
return;
} finally {
progress.hidden = true;
button.disabled = false;
}
form.elements.password.value = "";
form.hidden = true;
signOut.hidden = false;
state.textContent = "connecting…";
connect();
});
signOut.addEventListener("click", async () => {
stopStream?.();
clearTimeout(timer);
try {
await logout(base, { token });
} catch (err) {
state.textContent = err.message;
return;
}
if (!tokenMode) {
location.replace(new URL("login", location.href).href);
return;
}
token = null;
showLogin("Signed out.");
});
connect();
// No session of either kind can start on this page, so it offers no form
if (unavailable) {
state.textContent = unavailable;
signOut.hidden = true;
} else {
connect();
}
internal/sink/httpchain/httpchain.go
+11 -7
View File
@@ -9,6 +9,7 @@ import (
"maps"
"net"
"net/http"
"net/url"
"os"
"strconv"
"strings"
@@ -103,6 +104,10 @@ func NewHTTPChainSinkPlugin(
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
network, err := core.Network(opts.Host)
if err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if opts.IngestPath == "" {
opts.IngestPath = DefaultHTTPChainSinkIngestPath
} else if !strings.HasPrefix(opts.IngestPath, "/") {
@@ -158,10 +163,9 @@ func NewHTTPChainSinkPlugin(
addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
transport := &http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
// IPv4-only, aligns with tcp/http sinks
DialContext: func(ctx context.Context, _, address string) (net.Conn, error) {
d := net.Dialer{}
return d.DialContext(ctx, "tcp4", address)
return d.DialContext(ctx, network, address)
},
MaxIdleConnsPerHost: 2,
IdleConnTimeout: 90 * time.Second,
@@ -172,9 +176,9 @@ func NewHTTPChainSinkPlugin(
// NDJSON POSTs gain nothing from it
}
scheme := "http"
base := url.URL{Scheme: "http", Host: addr} // escapes an IPv6 zone
if tlsCfg != nil {
scheme = "https"
base.Scheme = "https"
}
t := &HTTPChainSink{
@@ -185,8 +189,8 @@ func NewHTTPChainSinkPlugin(
tlsEnabled: tlsCfg != nil,
mtls: tlsCfg != nil && len(tlsCfg.Certificates) > 0,
auth: authPolicy,
baseURL: scheme + "://" + addr,
url: scheme + "://" + addr + opts.IngestPath,
baseURL: base.String(),
url: base.String() + opts.IngestPath,
// A redirect would resend the batch, even from https to plaintext http
client: &http.Client{
Transport: transport,
internal/sink/httpchain/httpchain_test.go
+74
View File
@@ -1,6 +1,7 @@
package httpchain
import (
"context"
"crypto/tls"
"net"
"net/http"
@@ -16,6 +17,7 @@ import (
"logwisp/internal/config"
"logwisp/internal/core"
"logwisp/internal/session"
ingest "logwisp/internal/source/httpchain"
"logwisp/internal/testutil"
"github.com/lixenwraith/auth"
@@ -126,3 +128,75 @@ func TestRefusedTokenLogsInAgainAndDeliversOnce(t *testing.T) {
t.Fatalf("logins = %v, want 2", logins)
}
}
// The target URL brackets an IPv6 host and escapes its zone: a URL built
// from the raw host could not even form a request.
func TestTargetURLBracketsAnIPv6Host(t *testing.T) {
manager := session.NewManager(time.Hour)
t.Cleanup(manager.Stop)
for host, want := range map[string]string{
"::1": "http://[::1]:9000/ingest",
"fe80::1%eth0": "http://[fe80::1%25eth0]:9000/ingest",
} {
created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{"host": host, "port": int64(9000)},
log.NewLogger(), session.NewProxy(manager, "fwd"))
if err != nil {
t.Fatal(err)
}
if got := created.(*HTTPChainSink).url; got != want {
t.Errorf("host %q: target %q, want %q", host, got, want)
}
}
}
// A link runs over the IPv6 loopback with TLS: the source binds ::1, and the
// sink dials it and verifies the certificate's ::1 address.
func TestLinkOverIPv6LoopbackWithTLS(t *testing.T) {
testutil.RequireIPv6(t)
pki := testutil.NewPKI(t, "edge-01")
probe, err := net.Listen("tcp6", "[::1]:0")
if err != nil {
t.Fatal(err)
}
_, port, _ := net.SplitHostPort(probe.Addr().String())
probe.Close()
p, _ := strconv.ParseInt(port, 10, 64)
manager := session.NewManager(time.Hour)
t.Cleanup(manager.Stop)
src, err := ingest.NewHTTPChainSourcePlugin("in", map[string]any{
"host": "::1", "port": p,
"tls": map[string]any{"enabled": true, "cert_file": pki.ServerCert, "key_file": pki.ServerKey},
}, log.NewLogger(), session.NewProxy(manager, "in"))
if err != nil {
t.Fatal(err)
}
entries := src.Subscribe()
if err := src.Start(); err != nil {
t.Fatal(err)
}
t.Cleanup(src.Stop)
created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{
"host": "::1", "port": p,
"tls": map[string]any{"enabled": true, "ca_file": pki.CA},
}, log.NewLogger(), session.NewProxy(manager, "fwd"))
if err != nil {
t.Fatal(err)
}
sink := created.(*HTTPChainSink)
sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("over IPv6")})
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
defer cancel()
if !sink.flush(ctx) || sink.droppedBatches.Load() != 0 {
t.Fatalf("batch not delivered: %d request errors", sink.requestErrors.Load())
}
select {
case e := <-entries:
if e.Message != "over IPv6" {
t.Fatalf("message %q", e.Message)
}
case <-time.After(2 * time.Second):
t.Fatal("the source published nothing")
}
}
internal/sink/tcp/tcp.go
+7 -2
View File
@@ -50,6 +50,7 @@ type TCPSink struct {
// Configuration
config *config.TCPSinkOptions
addr string
network string
// Network
listener net.Listener
@@ -114,6 +115,10 @@ func NewTCPSinkPlugin(
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
network, err := core.Network(opts.Host)
if err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultTCPBufferSize
}
@@ -140,6 +145,7 @@ func NewTCPSinkPlugin(
proxy: proxy,
config: opts,
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
network: network,
input: make(chan core.TransportEvent, opts.BufferSize),
done: make(chan struct{}),
logger: logger,
@@ -192,8 +198,7 @@ func (t *TCPSink) listen() (net.Listener, error) {
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
}
}
// IPv4-only, parity with existing network sinks
ln, err := lc.Listen(context.Background(), "tcp4", t.addr)
ln, err := core.Listen(context.Background(), &lc, t.network, t.addr)
if err != nil {
return nil, err
}
internal/sink/tcpchain/tcpchain.go
+8 -2
View File
@@ -51,6 +51,7 @@ type TCPChainSink struct {
node string
addr string
network string
tlsConfig *tls.Config
// Authorization: pins the downstream server's identity
@@ -100,6 +101,10 @@ func NewTCPChainSinkPlugin(
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
network, err := core.Network(opts.Host)
if err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultChainSinkBufferSize
@@ -149,6 +154,7 @@ func NewTCPChainSinkPlugin(
config: opts,
node: node,
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
network: network,
tlsConfig: tlsCfg,
auth: authPolicy,
input: make(chan core.TransportEvent, opts.BufferSize),
@@ -377,10 +383,10 @@ func (t *TCPChainSink) connect(ctx context.Context) error {
// handshake under ctx, so bound dial + handshake together here
dctx, cancel := context.WithTimeout(ctx, t.dialTimeout+tlsx.HandshakeTimeout)
td := tls.Dialer{NetDialer: &nd, Config: t.tlsConfig}
conn, err = td.DialContext(dctx, "tcp4", t.addr) // IPv4-only
conn, err = td.DialContext(dctx, t.network, t.addr)
cancel()
} else {
conn, err = nd.DialContext(ctx, "tcp4", t.addr) // IPv4-only
conn, err = nd.DialContext(ctx, t.network, t.addr)
}
if err != nil {
return err
internal/source/httpchain/httpchain.go
+7 -2
View File
@@ -48,6 +48,7 @@ type HTTPChainSource struct {
id string
proxy *session.Proxy
config *config.HTTPChainSourceOptions
network string
subscribers []chan core.LogEntry
server *http.Server
@@ -91,6 +92,10 @@ func NewHTTPChainSourcePlugin(
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
network, err := core.Network(opts.Host)
if err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if opts.IngestPath == "" {
opts.IngestPath = DefaultHTTPChainSourceIngestPath
} else if !strings.HasPrefix(opts.IngestPath, "/") {
@@ -120,6 +125,7 @@ func NewHTTPChainSourcePlugin(
id: id,
proxy: proxy,
config: opts,
network: network,
subscribers: make([]chan core.LogEntry, 0),
sessions: make(map[string]string),
logger: logger,
@@ -169,8 +175,7 @@ func (s *HTTPChainSource) Start() error {
return err
}
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
// IPv4-only, aligns with tcp/http sinks
ln, err := net.Listen("tcp4", addr)
ln, err := core.Listen(context.Background(), &net.ListenConfig{}, s.network, addr)
if err != nil {
s.auth.Close()
return fmt.Errorf("listen %s: %w", addr, err)
internal/source/tcpchain/tcpchain.go
+9 -3
View File
@@ -42,6 +42,7 @@ type TCPChainSource struct {
id string
proxy *session.Proxy
config *config.TCPChainSourceOptions
network string
subscribers []chan core.LogEntry
listener net.Listener
@@ -86,6 +87,10 @@ func NewTCPChainSourcePlugin(
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
network, err := core.Network(opts.Host)
if err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultChainSourceBufferSize
}
@@ -105,6 +110,7 @@ func NewTCPChainSourcePlugin(
id: id,
proxy: proxy,
config: opts,
network: network,
subscribers: make([]chan core.LogEntry, 0),
conns: make(map[net.Conn]struct{}),
logger: logger,
@@ -153,9 +159,9 @@ func (s *TCPChainSource) Start() error {
return err
}
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
// IPv4-only. TLS-wrapped when configured; handshake runs explicitly in
// handleConn under tlsx.HandshakeTimeout, pre-hello.
ln, err := net.Listen("tcp4", addr)
// TLS-wrapped when configured; handshake runs explicitly in handleConn
// under tlsx.HandshakeTimeout, pre-hello.
ln, err := core.Listen(context.Background(), &net.ListenConfig{}, s.network, addr)
if err != nil {
s.auth.Close()
return fmt.Errorf("listen %s: %w", addr, err)
internal/testutil/fixtures.go
+15 -4
View File
@@ -40,8 +40,19 @@ func WriteFile(t testing.TB, path, contents string) {
}
}
// PKI is a throwaway CA with a server certificate for 127.0.0.1 and one client
// certificate, written as PEM files for plugins that load TLS from disk.
// RequireIPv6 skips a test on a host without an IPv6 loopback
func RequireIPv6(t testing.TB) {
t.Helper()
ln, err := net.Listen("tcp6", "[::1]:0")
if err != nil {
t.Skipf("no IPv6 loopback: %v", err)
}
ln.Close()
}
// PKI is a throwaway CA with a server certificate for 127.0.0.1 and ::1 and
// one client certificate, written as PEM files for plugins that load TLS from
// disk.
type PKI struct {
CA, ServerCert, ServerKey, ClientCert, ClientKey string
@@ -80,7 +91,7 @@ func NewPKI(t testing.TB, clientCN string) *PKI {
}
// Leaf issues another certificate from the same CA: a client certificate, or
// a server certificate valid for 127.0.0.1.
// a server certificate valid for 127.0.0.1 and ::1.
func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFile string) {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
@@ -93,7 +104,7 @@ func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFil
SerialNumber: big.NewInt(p.serial), Subject: pkix.Name{CommonName: cn},
NotBefore: now.Add(-time.Hour), NotAfter: now.Add(24 * time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1)},
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
}
if client {
tmpl.ExtKeyUsage, tmpl.IPAddresses = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil
internal/tlsx/tlsx.go
+3 -2
View File
@@ -63,7 +63,8 @@ func Server(o *config.TLSOptions) (*tls.Config, error) {
// Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain
// sinks). host seeds ServerName when no override is set; Go verifies IP SANs
// when host is an address. Returns (nil, nil) when disabled.
// when host is an address, without its zone, which names a local interface
// and no certificate carries. Returns (nil, nil) when disabled.
func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
if o == nil || !o.Enabled {
return nil, nil
@@ -78,7 +79,7 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
InsecureSkipVerify: o.InsecureSkipVerify,
}
if cfg.ServerName == "" {
cfg.ServerName = host
cfg.ServerName, _, _ = strings.Cut(host, "%")
}
if o.CAFile != "" {
pool, err := loadPool(o.CAFile)
internal/tlsx/tlsx_test.go
+16
View File
@@ -6,8 +6,24 @@ import (
"strings"
"testing"
"time"
"logwisp/internal/config"
)
// A dialer verifies an IPv6 target by its address: the zone names a local
// interface, which no certificate carries.
func TestClientServerNameDropsTheZone(t *testing.T) {
for host, want := range map[string]string{"fe80::1%eth0": "fe80::1", "::1": "::1", "relay.example": "relay.example"} {
cfg, err := Client(&config.TLSOptions{Enabled: true}, host)
if err != nil {
t.Fatal(err)
}
if cfg.ServerName != want {
t.Errorf("Client(%q).ServerName = %q, want %q", host, cfg.ServerName, want)
}
}
}
// Startup names a certificate outside its validity window or within 30 days
// of leaving it; a healthy one stays silent.
func TestExpiryWarning(t *testing.T) {
test/chain-aggregate-test.sh
+56 -165
View File
@@ -1,95 +1,33 @@
#!/usr/bin/env bash
# logwisp chain topology test
#
# random --> tcp_chain sink --> :15801 tcp_chain src --> :15803 tcp sink
# random --> http_chain sink --> :15802 http_chain src --> :15804 http sink (SSE)
#
# Usage:
# ./chain_aggregate_test.sh manual mode: 2 edge daemons + relay foreground
# ./chain_aggregate_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown
# ./chain_aggregate_test.sh --keep (with --auto) skip teardown on success
#
# The relay aggregates both sinks and fans out aggregated streams into both http and tcp sinks
#
# logwisp chain aggregation test: edge-tcp and edge-http feed one relay
# pipeline that fans both chain sources in and out to a tcp and an http sink.
# Requires: bash 5+, coreutils (timeout), curl. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run"
CONF="$RUN/conf"
LOG="$RUN/log"
RUN=$E2E_DIR/run
PORT_TCP_CHAIN=15801
PORT_HTTP_CHAIN=15802
PORT_TCP_SINK=15803
PORT_HTTP_SINK=15804
PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK"
e2e_init "$@"
AUTO=0; KEEP=0
for a in "$@"; do case "$a" in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
*) echo "unknown arg: $a" >&2; exit 1 ;;
esac; done
PIDS=()
cleanup() {
local rc=$?
trap - EXIT INT TERM
if (( ${#PIDS[@]} )); then
echo "--- teardown: stopping ${#PIDS[@]} daemon(s)"
kill -TERM "${PIDS[@]}" 2>/dev/null
local deadline=$(( SECONDS + 10 ))
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
kill -KILL "$pid" 2>/dev/null
done
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
wait_port() { # port timeout_s
local i; for (( i=0; i < $2 * 10; i++ )); do
port_open "$1" && return 0
sleep 0.1
done
return 1
}
# Reads TCP sink stream for N seconds; connection counts as a sink client
tcp_read() { # port secs
timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true
}
start_daemon() { # name conf
"$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 &
PIDS+=($!)
echo "started $1 (pid $!)"
}
# --- Preflight ---
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
done
section "Setup"
need curl
ports_free $PORTS
mkdir -p "$CONF" "$LOG"
# --- Config generation ---
cat > "$CONF/edge_tcp.toml" <<EOF
edge_conf() { # name sink_type port [sink options]
cat >"$CONF/$1.toml" <<EOF
status_reporter = false
[logging]
output = "file"
output = "stdout"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_tcp"
[[pipelines]]
name = "edge_tcp"
name = "$1"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
@@ -99,42 +37,18 @@ format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "tcp_chain"
type = "$2"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_TCP_CHAIN
node = "edge-tcp"
port = $3
node = "${1/_/-}"
${4:-}
EOF
}
edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN
edge_conf edge_http http_chain $PORT_HTTP_CHAIN "flush_interval_ms = 500"
cat > "$CONF/edge_http.toml" <<EOF
status_reporter = false
[logging]
output = "file"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_http"
[[pipelines]]
name = "edge_http"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "http_chain"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_HTTP_CHAIN
node = "edge-http"
flush_interval_ms = 500
EOF
cat > "$CONF/relay.toml" <<EOF
cat >"$CONF/relay.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
@@ -175,74 +89,51 @@ type = "http"
host = "127.0.0.1"
port = $PORT_HTTP_SINK
EOF
info "configurations in $(short "$CONF")/"
# --- Guide ---
cat <<EOF
================================================================
logwisp chain test — port map
$PORT_TCP_CHAIN relay ingest (tcp_chain, from edge-tcp)
$PORT_HTTP_CHAIN relay ingest (http_chain, from edge-http, POST /ingest)
$PORT_TCP_SINK TCP sink -> live check: nc 127.0.0.1 $PORT_TCP_SINK
$PORT_HTTP_SINK HTTP sink -> live check: browser/curl:
http://127.0.0.1:$PORT_HTTP_SINK/stream (SSE)
http://127.0.0.1:$PORT_HTTP_SINK/status (JSON stats)
Use 127.0.0.1, not localhost — sinks reject IPv6.
Expected: json entries from BOTH nodes ("edge-tcp", "edge-http")
interleaved on :$PORT_TCP_SINK and :$PORT_HTTP_SINK.
Logs: $LOG/
================================================================
EOF
if (( AUTO == 0 )); then
# Manual mode: edges daemonized first (chain sinks backoff-retry), relay foreground
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
echo "--- relay starting in FOREGROUND; Ctrl-C stops relay and both edges"
"$BIN" -c "$CONF/relay.toml"
exit 0
fi
# --- Auto mode ---
section "Startup"
start_daemon relay relay.toml
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; }
for p in $PORTS; do
wait_port "$p" || abort "relay port $p is not listening" relay
done
# Chain sinks retry with backoff, so the edges could start first too
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
daemons_up
echo "--- settling 3s (connect + first http_chain flush)"
guide "logwisp chain aggregation test" <<EOF
Ports (IPv4 only: 127.0.0.1, not localhost):
$PORT_TCP_CHAIN relay tcp_chain ingest, from edge-tcp
$PORT_HTTP_CHAIN relay http_chain ingest (POST /ingest), from edge-http
$PORT_TCP_SINK tcp sink: both edges' entries interleaved, as JSON
$PORT_HTTP_SINK http sink: the same, SSE /stream, JSON /status
Watch:
> nc 127.0.0.1 $PORT_TCP_SINK
> curl -N http://127.0.0.1:$PORT_HTTP_SINK/stream
> curl http://127.0.0.1:$PORT_HTTP_SINK/status
Logs: $(short "$LOG")/
EOF
manual_hold
section "Checks"
info "settling 3 s (connect, first http_chain flush)"
sleep 3
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
# 1. TCP chain: edge-tcp -> relay -> tcp sink
# 1. tcp sink: both edges, aggregated
tcp_out="$(tcp_read "$PORT_TCP_SINK" 4)"
#nt=$(grep -c '"node":"edge-tcp"' <<< "$tcp_out")
#nh=$(grep -c '"node":"edge-http"' <<< "$tcp_out")
nt=$(grep -c 'edge-tcp/' <<< "$tcp_out")
nh=$(grep -c 'edge-http/' <<< "$tcp_out")
check "tcp sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $(( nt >= 1 && nh >= 1 ))
nt=$(grep -c 'edge-tcp/' <<<"$tcp_out")
nh=$(grep -c 'edge-http/' <<<"$tcp_out")
check "tcp sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $((nt >= 1 && nh >= 1))
# 2. HTTP chain: edge-http -> relay -> SSE sink
sse_out="$(curl -sN --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
nt=$(grep -c '^data:.*edge-tcp/' <<< "$sse_out")
nh=$(grep -c '^data:.*edge-http/' <<< "$sse_out")
check "http sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $(( nt >= 1 && nh >= 1 ))
# 2. http sink: both edges, aggregated
sse_out="$(curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
nt=$(grep -c '^data:.*edge-tcp/' <<<"$sse_out")
nh=$(grep -c '^data:.*edge-http/' <<<"$sse_out")
check "http sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $((nt >= 1 && nh >= 1))
# 3. HTTP sink status endpoint
status="$(curl -s --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<< "$status" | grep -o '[0-9]*' || echo 0)
check "status endpoint: total_processed=$proc > 0" $(( proc > 0 ))
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
else
echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log"
fi
exit "$fail"
status="$(curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<<"$status" | grep -o '[0-9]*' || echo 0)
check "status endpoint: total_processed=$proc > 0" $((proc > 0))
summary
test/chain-test.sh
+52 -162
View File
@@ -1,98 +1,33 @@
#!/usr/bin/env bash
# logwisp chain topology test
#
# random --> tcp_chain sink --> :15801 tcp_chain src --> :15803 tcp sink
# random --> http_chain sink --> :15802 http_chain src --> :15804 http sink (SSE)
#
# Usage:
# ./chain_test.sh manual mode: 2 edge daemons + relay foreground
# ./chain_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown
# ./chain_test.sh --keep (with --auto) skip teardown on success
#
# relay runs TWO independent pipelines; no cross-path aggregation:
# relay_tcp: tcp_chain src :15801 -> tcp sink :15803 (edge-tcp only)
# relay_http: http_chain src :15802 -> http sink :15804 (edge-http only)
# For fan-in aggregation (both edges -> both sinks) see chain_agg_test.sh.
#
# logwisp chain test: edge-tcp and edge-http feed a relay that runs one
# pipeline per transport, without fan-in (chain-aggregate-test.sh fans in).
# Requires: bash 5+, coreutils (timeout), curl. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run"
CONF="$RUN/conf"
LOG="$RUN/log"
RUN=$E2E_DIR/run
PORT_TCP_CHAIN=15801
PORT_HTTP_CHAIN=15802
PORT_TCP_SINK=15803
PORT_HTTP_SINK=15804
PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK"
e2e_init "$@"
AUTO=0; KEEP=0
for a in "$@"; do case "$a" in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
*) echo "unknown arg: $a" >&2; exit 1 ;;
esac; done
PIDS=()
cleanup() {
local rc=$?
trap - EXIT INT TERM
if (( ${#PIDS[@]} )); then
echo "--- teardown: stopping ${#PIDS[@]} daemon(s)"
kill -TERM "${PIDS[@]}" 2>/dev/null
local deadline=$(( SECONDS + 10 ))
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
kill -KILL "$pid" 2>/dev/null
done
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
wait_port() { # port timeout_s
local i; for (( i=0; i < $2 * 10; i++ )); do
port_open "$1" && return 0
sleep 0.1
done
return 1
}
# Reads TCP sink stream for N seconds; connection counts as a sink client
tcp_read() { # port secs
timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true
}
start_daemon() { # name conf
"$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 &
PIDS+=($!)
echo "started $1 (pid $!)"
}
# --- Preflight ---
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
done
section "Setup"
need curl
ports_free $PORTS
mkdir -p "$CONF" "$LOG"
# --- Config generation ---
cat > "$CONF/edge_tcp.toml" <<EOF
edge_conf() { # name sink_type port [sink options]
cat >"$CONF/$1.toml" <<EOF
status_reporter = false
[logging]
output = "file"
output = "stdout"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_tcp"
[[pipelines]]
name = "edge_tcp"
name = "$1"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
@@ -102,42 +37,18 @@ format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "tcp_chain"
type = "$2"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_TCP_CHAIN
node = "edge-tcp"
port = $3
node = "${1/_/-}"
${4:-}
EOF
}
edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN
edge_conf edge_http http_chain $PORT_HTTP_CHAIN "flush_interval_ms = 500"
cat > "$CONF/edge_http.toml" <<EOF
status_reporter = false
[logging]
output = "file"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_http"
[[pipelines]]
name = "edge_http"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "http_chain"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_HTTP_CHAIN
node = "edge-http"
flush_interval_ms = 500
EOF
cat > "$CONF/relay.toml" <<EOF
cat >"$CONF/relay.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
@@ -179,70 +90,49 @@ type = "http"
host = "127.0.0.1"
port = $PORT_HTTP_SINK
EOF
info "configurations in $(short "$CONF")/"
# --- Guide ---
cat <<EOF
================================================================
logwisp chain test — port map
$PORT_TCP_CHAIN relay ingest (tcp_chain, from edge-tcp)
$PORT_HTTP_CHAIN relay ingest (http_chain, from edge-http, POST /ingest)
$PORT_TCP_SINK TCP sink -> live check: nc 127.0.0.1 $PORT_TCP_SINK
$PORT_HTTP_SINK HTTP sink -> live check: browser/curl:
http://127.0.0.1:$PORT_HTTP_SINK/stream (SSE)
http://127.0.0.1:$PORT_HTTP_SINK/status (JSON stats)
Use 127.0.0.1, not localhost — sinks reject IPv6.
Expected: json entries, node "edge-tcp" on :$PORT_TCP_SINK, "edge-http" on :$PORT_HTTP_SINK.
Logs: $LOG/
================================================================
EOF
if (( AUTO == 0 )); then
# Manual mode: edges daemonized first (chain sinks backoff-retry), relay foreground
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
echo "--- relay starting in FOREGROUND; Ctrl-C stops relay and both edges"
"$BIN" -c "$CONF/relay.toml"
exit 0
fi
# --- Auto mode ---
section "Startup"
start_daemon relay relay.toml
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; }
for p in $PORTS; do
wait_port "$p" || abort "relay port $p is not listening" relay
done
# Chain sinks retry with backoff, so the edges could start first too
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
daemons_up
echo "--- settling 3s (connect + first http_chain flush)"
guide "logwisp chain test" <<EOF
Ports (IPv4 only: 127.0.0.1, not localhost):
$PORT_TCP_CHAIN relay tcp_chain ingest, from edge-tcp
$PORT_HTTP_CHAIN relay http_chain ingest (POST /ingest), from edge-http
$PORT_TCP_SINK tcp sink: edge-tcp's entries as JSON
$PORT_HTTP_SINK http sink: edge-http's entries, SSE /stream, JSON /status
Watch:
> nc 127.0.0.1 $PORT_TCP_SINK
> curl -N http://127.0.0.1:$PORT_HTTP_SINK/stream
> curl http://127.0.0.1:$PORT_HTTP_SINK/status
Logs: $(short "$LOG")/
EOF
manual_hold
section "Checks"
info "settling 3 s (connect, first http_chain flush)"
sleep 3
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
# 1. TCP chain: edge-tcp -> relay -> tcp sink
tcp_out="$(tcp_read "$PORT_TCP_SINK" 4)"
#n=$(grep -c '"node":"edge-tcp"' <<< "$tcp_out")
n=$(grep -c 'edge-tcp/' <<< "$tcp_out")
check "tcp path: entries on :$PORT_TCP_SINK with node=edge-tcp ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-tcp/' <<<"$tcp_out")
check "tcp path: entries on :$PORT_TCP_SINK with node=edge-tcp ($n lines)" $((n >= 1))
# 2. HTTP chain: edge-http -> relay -> SSE sink
sse_out="$(curl -sN --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
n=$(grep -c '^data:.*edge-http/' <<< "$sse_out")
check "http path: SSE events on :$PORT_HTTP_SINK with node=edge-http ($n events)" $(( n >= 1 ))
sse_out="$(curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
n=$(grep -c '^data:.*edge-http/' <<<"$sse_out")
check "http path: SSE events on :$PORT_HTTP_SINK with node=edge-http ($n events)" $((n >= 1))
# 3. HTTP sink status endpoint
status="$(curl -s --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<< "$status" | grep -o '[0-9]*' || echo 0)
check "status endpoint: total_processed=$proc > 0" $(( proc > 0 ))
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
else
echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log"
fi
exit "$fail"
status="$(curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<<"$status" | grep -o '[0-9]*' || echo 0)
check "status endpoint: total_processed=$proc > 0" $((proc > 0))
summary
test/deploy-test.sh
Executable
+347
View File
@@ -0,0 +1,347 @@
#!/usr/bin/env bash
# deploy/lw-deploy.sh test: --yes generates edges, aggregators and a standalone
# node for the manual runtime, plain and with TLS and scram, that bin/lw runs;
# --yes fails closed, --dry-run changes nothing (manual, native, docker), and a
# root host with docker and the logwisp:dev image runs the container too.
# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
RUN=$E2E_DIR/run-deploy
DEPLOY=${E2E_DIR%/*}/deploy/lw-deploy.sh
PKI=$RUN/pki
OUT=$RUN/out
APP=$RUN/app # the directory the edges tail
SECRETS=$RUN/secrets
PORT_CHAIN=15871
PORT_HTTP=15872
PORT_TCP=15873
PORT_SA_HTTP=15874
PORT_S_CHAIN=15875
PORT_S_HTTP=15876
PORT_D_CHAIN=15877
PORT_D_HTTP=15878
PORT_DRY=15879 # only written by the dry run, never bound
PORTS="$PORT_CHAIN $PORT_HTTP $PORT_TCP $PORT_SA_HTTP $PORT_S_CHAIN $PORT_S_HTTP $PORT_D_CHAIN $PORT_D_HTTP"
IMAGE=logwisp:dev
CONTAINER=lw-e2e-deploy-aggregator
e2e_init "$@"
deploy() { # NAME ARGS...: lw-deploy.sh --yes, its output in $LOG/deploy-NAME.out
local name=$1
shift
"$DEPLOY" --yes "$@" >"$LOG/deploy-$name.out" 2>&1
}
said() { grep -qF -- "$2" "$LOG/deploy-$1.out"; } # NAME TEXT: lw-deploy.sh printed TEXT
declare -A GEN # NAME -> lw-deploy.sh's exit status
generate() { deploy "$1" --runtime manual --config-dir "$RUN/$1" --bin "$BIN" "${@:2}"; GEN[$1]=$?; }
section "Setup"
need curl openssl
[[ -x $DEPLOY ]] || abort "no $(short "$DEPLOY")"
# A container an earlier run left (--keep) holds its published ports
if [[ -d $RUN/docker ]] && command -v docker >/dev/null; then docker rm -f "$CONTAINER" >/dev/null 2>&1; fi
ports_free $PORTS
rm -rf "$RUN"
mkdir -p "$LOG" "$APP" "$OUT"
pki_ca
pki_leaf aggregator aggregator.internal serverAuth "IP:127.0.0.1"
echo "first line, there before the edges start" >"$APP/app.log"
info "test PKI in $(short "$PKI")/; the edges tail $(short "$APP")/*.log"
generate aggregator --role aggregator --listen 127.0.0.1 --chain-port $PORT_CHAIN \
--file-dir "$OUT/aggregator" --http-port $PORT_HTTP --tcp-port $PORT_TCP
generate edge --role edge --log-dir "$APP" --from start --aggregator 127.0.0.1 --chain-port $PORT_CHAIN \
--node deploy-edge
generate standalone --role standalone --log-dir "$APP" --from start --listen 127.0.0.1 \
--file-dir "$OUT/standalone" --http-port $PORT_SA_HTTP --format json
SECURE_AGGREGATOR=(--role aggregator --listen 127.0.0.1 --chain http --chain-port $PORT_S_CHAIN
--file-dir "$OUT/secure" --http-port $PORT_S_HTTP --tls --cert-file "$PKI/aggregator.crt"
--key-file "$PKI/aggregator.key" --auth scram --add-user edge-01 --sink-auth scram
--add-viewer viewer-01 --secrets-dir "$SECRETS")
generate secure-aggregator "${SECURE_AGGREGATOR[@]}"
generate secure-edge --role edge --log-dir "$APP" --from start --chain http --aggregator 127.0.0.1 \
--chain-port $PORT_S_CHAIN --node deploy-secure --tls --ca-file "$PKI/ca.crt" --auth scram \
--password-file "$SECRETS/edge-01.pass"
info "generated with --yes --runtime manual: aggregator, edge, standalone, secure-aggregator, secure-edge"
section "Startup"
for name in aggregator standalone secure-aggregator edge secure-edge; do
if [[ ${GEN[$name]} != 0 ]]; then
tail -n 5 "$LOG/deploy-$name.out" | sed 's/^/ /'
abort "lw-deploy.sh did not generate $name (exit ${GEN[$name]})"
fi
spawn "$name" "$BIN" -c "$RUN/$name/logwisp.toml"
done
for p in $PORT_CHAIN $PORT_HTTP $PORT_TCP $PORT_SA_HTTP $PORT_S_CHAIN $PORT_S_HTTP; do
wait_port "$p" || abort "port $p is not listening"
done
daemons_up
write_env LW="$BIN" CA="$PKI/ca.crt" PW="$SECRETS/viewer-01.pass" APP="$APP"
guide "lw-deploy.sh test" <<EOF
Ports (generated by lw-deploy.sh --yes --runtime manual):
$PORT_CHAIN aggregator tcp_chain ingest, from edge (node deploy-edge)
$PORT_HTTP aggregator http sink: SSE /stream, JSON /status
$PORT_TCP aggregator tcp sink
$PORT_SA_HTTP standalone http sink, json entries of the same log
$PORT_S_CHAIN secure-aggregator http_chain ingest, TLS + scram user edge-01
$PORT_S_HTTP secure-aggregator http sink, TLS + scram viewer viewer-01
$PORT_D_CHAIN docker aggregator chain ingest ($CONTAINER, --auto only)
$PORT_D_HTTP docker aggregator http sink
Configurations: $(short "$RUN")/NAME/logwisp.toml; outputs: $(short "$OUT")/
Shell setup (LW, CA, PW: viewer-01's password file, APP: the tailed directory):
> . $(short "$RUN")/env
Append a line and watch it arrive:
> curl -N http://127.0.0.1:$PORT_HTTP/stream
> echo hello >>\$APP/app.log
The secured sink (bearer keeps the token off argv):
> token=\$(\$LW auth token -url https://127.0.0.1:$PORT_S_HTTP -user viewer-01 -password-file \$PW -ca-file \$CA)
> curl -N --cacert \$CA -H @<(bearer) https://127.0.0.1:$PORT_S_HTTP/stream
Logs: $(short "$LOG")/
EOF
manual_hold
total_processed() { # PORT: of a plain http sink
curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$1/status" 2>/dev/null |
grep -o '"total_processed":[0-9]*' | grep -o '[0-9]*$' || echo 0
}
section "Generated configurations"
for name in aggregator edge standalone secure-aggregator secure-edge; do
conf="$RUN/$name/logwisp.toml" ok=0
"$BIN" --check -c "$conf" >>"$LOG/lw-check.out" 2>&1 && ok=1
check "$name: --yes wrote $(short "$conf") (exit ${GEN[$name]}), lw --check accepts it" $ok
done
section "Plain chain and outputs"
wait_until 10 grep -q 'deploy-edge/app.log "first line' "$OUT/aggregator/logwisp.log"
n=$(grep -c 'deploy-edge/app.log "first line' "$OUT/aggregator/logwisp.log")
check "edge -> aggregator: the line there at start reached the file output ($n)" $((n == 1))
live="appended line $$"
tcp_read $PORT_TCP 4 >"$RUN/tcp.out" &
tcp_pid=$!
curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP/stream" >"$RUN/sse.out" 2>/dev/null &
sse_pid=$!
sleep 1
echo "$live" >>"$APP/app.log"
wait "$tcp_pid" "$sse_pid"
wait_until 10 grep -q "$live" "$OUT/aggregator/logwisp.log"
n=$(grep -c "deploy-edge/app.log \"$live" "$OUT/aggregator/logwisp.log")
check "edge -> aggregator: a line appended later reached the file output ($n)" $((n == 1))
n=$(grep -c "^data:.*deploy-edge/app.log \"$live" "$RUN/sse.out")
check "aggregator http sink: /stream delivered it live ($n events)" $((n == 1))
n=$(grep -c "deploy-edge/app.log \"$live" "$RUN/tcp.out")
check "aggregator tcp sink: delivered it live ($n lines)" $((n == 1))
n=$(total_processed $PORT_HTTP)
check "aggregator http sink: /status counts both lines (total_processed=$n)" $((n >= 2))
wait_until 10 grep -q "$live" "$OUT/standalone/logwisp.log"
n=$(grep -c -e '"fields":\["first line' -e "\"fields\":\[\"$live" "$OUT/standalone/logwisp.log")
check "standalone: both lines in its file output, as json ($n)" $((n == 2))
n=$(total_processed $PORT_SA_HTTP)
check "standalone http sink: /status counts them (total_processed=$n)" $((n >= 2))
section "TLS and scram"
wait_until 20 grep -q "edge-01/app.log \"$live" "$OUT/secure/logwisp.log"
n=$(grep -c 'edge-01/app.log' "$OUT/secure/logwisp.log")
check "secure-edge -> secure-aggregator: entries over http_chain, TLS and scram ($n lines)" $((n >= 2))
n=$(grep -c 'deploy-secure/' "$OUT/secure/logwisp.log")
check "secure-aggregator: the node label is the username, not the edge's own ($n deploy-secure lines)" $((n == 0))
modes="$(stat -c %a "$SECRETS/edge-01.pass" "$SECRETS/viewer-01.pass" 2>/dev/null | paste -sd' ')"
ok=$(is "$modes" "600 600")
grep -qx 'username = "edge-01"' "$RUN/secure-aggregator/users.toml" || ok=0
grep -qx 'username = "viewer-01"' "$RUN/secure-aggregator/viewers.toml" || ok=0
grep -q 'viewer-01' "$RUN/secure-aggregator/users.toml" && ok=0
check "secure-aggregator: passwords generated (modes $modes), edge-01 in users.toml, viewer-01 in viewers.toml" $ok
ok=1
cmp -s "$PKI/aggregator.crt" "$RUN/secure-aggregator/tls/node.crt" || ok=0
cmp -s "$PKI/aggregator.key" "$RUN/secure-aggregator/tls/node.key" || ok=0
cmp -s "$PKI/ca.crt" "$RUN/secure-edge/tls/ca.crt" || ok=0
cmp -s "$SECRETS/edge-01.pass" "$RUN/secure-edge/chain.pass" || ok=0
check "certificates, key and the edge's password copied into the configuration directories" $ok
token="$("$BIN" auth token -url "https://127.0.0.1:$PORT_S_HTTP" -user viewer-01 \
-password-file "$SECRETS/viewer-01.pass" -ca-file "$PKI/ca.crt" 2>>"$LOG/auth-cli.out")"
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt" \
-H @<(printf 'Authorization: Bearer %s\n' "$token") "https://127.0.0.1:$PORT_S_HTTP/status" 2>/dev/null)"
check "secure-aggregator http sink: viewer-01's token opens /status (HTTP $code)" $(is "$code" 200)
code="$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt" \
"https://127.0.0.1:$PORT_S_HTTP/status" 2>/dev/null)"
check "secure-aggregator http sink: /status refused without a token (HTTP $code)" $(is "$code" 401)
section "Re-running a deployment"
conf="$RUN/secure-aggregator/logwisp.toml"
before="$(cksum <"$conf")" pass_before="$(cksum <"$SECRETS/edge-01.pass")"
deploy rerun "${SECURE_AGGREGATOR[@]}" --runtime manual --config-dir "$RUN/secure-aggregator" --bin "$BIN"
rc=$?
ok=$((rc != 0))
grep -q 'exists (--force replaces it)' "$LOG/deploy-rerun.out" || ok=0
[[ $(cksum <"$conf") == "$before" ]] || ok=0
check "without --force, --yes refuses to replace a configuration (exit $rc)" $ok
deploy rerun-force "${SECURE_AGGREGATOR[@]}" --runtime manual --config-dir "$RUN/secure-aggregator" --bin "$BIN" --force
rc=$?
ok=$((rc == 0))
[[ -f $conf.bak && $(cksum <"$conf.bak") == "$before" ]] || ok=0
[[ $(cksum <"$SECRETS/edge-01.pass") == "$pass_before" ]] || ok=0
check "with --force: the old configuration kept as logwisp.toml.bak, the generated passwords reused (exit $rc)" $ok
section "--yes fails closed"
deploy fail-edge --runtime manual --role edge --config-dir "$RUN/fail-edge" --bin "$BIN" --log-dir "$APP"
rc=$?
ok=$((rc != 0))
grep -q 'missing value: --aggregator' "$LOG/deploy-fail-edge.out" || ok=0
[[ -e $RUN/fail-edge ]] && ok=0
check "an edge without --aggregator stops before writing anything (exit $rc)" $ok
deploy fail-aggregator --runtime manual --role aggregator --config-dir "$RUN/fail-aggregator" --bin "$BIN"
rc=$?
ok=$((rc != 0))
grep -q 'no output: give --file-dir, --http-port or --tcp-port' "$LOG/deploy-fail-aggregator.out" || ok=0
[[ -e $RUN/fail-aggregator ]] && ok=0
check "an aggregator without an output stops before writing anything (exit $rc)" $ok
deploy implied-auth --dry-run --runtime manual --role aggregator --config-dir "$RUN/dry/conf" --listen 127.0.0.1 \
--http-port $PORT_DRY --tls --cert-file "$PKI/aggregator.crt" --key-file "$PKI/aggregator.key" \
--ca-file "$PKI/ca.crt" --add-user edge-01 --sink-allow viewer-01 --secrets-dir "$RUN/dry/secrets"
rc=$?
ok=$((rc == 0))
said implied-auth "over tcp_chain (TLS, auth scram)" || ok=0
said implied-auth "/stream /status (txt, TLS, auth mtls)" || ok=0
check "without --auth and --sink-auth, --add-user implies scram and --sink-allow mtls (exit $rc)" $ok
deploy fail-auth --runtime manual --role aggregator --config-dir "$RUN/fail-auth" --bin "$BIN" \
--file-dir "$RUN/fail-auth/out" --tls --cert-file "$PKI/aggregator.crt" --key-file "$PKI/aggregator.key" \
--auth none --add-user edge-01 --secrets-dir "$RUN/fail-auth/secrets"
rc=$?
ok=$((rc != 0))
said fail-auth '--add-user needs --auth scram' || ok=0
[[ -e $RUN/fail-auth ]] && ok=0
check "an --add-user that --auth none contradicts stops before writing anything (exit $rc)" $ok
deploy fail-pattern --runtime manual --role standalone --config-dir "$RUN/fail-pattern" --bin "$BIN" --log-dir "$APP" \
--file-dir "$RUN/fail-pattern/out" --log-pattern $'*.log\nfrom = "start"'
rc=$?
ok=$((rc != 0))
said fail-pattern 'invalid --log-pattern' || ok=0
[[ -e $RUN/fail-pattern ]] && ok=0
check "a --log-pattern with a newline stops before writing anything (exit $rc)" $ok
(cd "$RUN" && deploy fail-secrets --runtime manual --role aggregator --config-dir "$RUN/fail-secrets" \
--bin "$BIN" --file-dir "$RUN/fail-secrets/out" --tls --cert-file "$PKI/aggregator.crt" \
--key-file "$PKI/aggregator.key" --auth scram --add-user edge-01 --secrets-dir '~/secrets')
rc=$?
ok=$((rc != 0))
said fail-secrets 'not an absolute path: ~/secrets' || ok=0
[[ -e $RUN/fail-secrets || -e "$RUN/~" ]] && ok=0
check "a --secrets-dir of ~/secrets stops before writing anything (exit $rc)" $ok
deploy fail-check --runtime manual --role aggregator --config-dir "$RUN/fail-check" --bin "$BIN" \
--file-dir "$RUN/fail-check/out" --tls --cert-file "$APP/app.log" --key-file "$APP/app.log"
rc=$?
ok=$((rc != 0))
said fail-check 'lw --check rejects' || ok=0
check "a configuration lw --check rejects (a log file as the certificate) stops the run (exit $rc)" $ok
deploy fail-tmp --dry-run --runtime native --role standalone --log-dir "$APP" --file-dir /tmp/lw-deploy-test \
--bin "$BIN"
rc=$?
ok=$((rc != 0))
said fail-tmp 'PrivateTmp=yes' || ok=0
check "native Linux: a --file-dir under /tmp, which the unit makes private, stops the run (exit $rc)" $ok
# A failing install stands in for the real one: a regression must not install onto this host
mkdir -p "$RUN/stub"
printf '#!/bin/sh\nexit 1\n' >"$RUN/stub/install"
chmod +x "$RUN/stub/install"
PATH="$RUN/stub:$PATH" deploy fail-os --runtime native --os freebsd --role standalone --log-dir "$APP" \
--file-dir "$RUN/fail-os" --bin "$BIN"
rc=$?
ok=$((rc != 0))
said fail-os 'needs a freebsd host' || ok=0
said fail-os '+ install' && ok=0
check "--os freebsd --runtime native on this Linux host stops before changing anything (exit $rc)" $ok
section "--dry-run changes nothing"
# What a dry run could touch: the run directory (minus what the daemons write)
# and what a native install writes
snapshot() {
find "$RUN" \( -path "$LOG" -o -path "$OUT" \) -prune -o -printf '%p %s %T@ %m\n' | sort | cksum
stat -c '%n %s %Y' /usr/local/bin/lw /etc/logwisp /etc/systemd/system/logwisp.service \
/etc/systemd/system/logwisp.service.d /etc/sysusers.d/logwisp.conf /etc/tmpfiles.d/logwisp.conf 2>&1
}
before="$(snapshot)"
deploy dry-manual --dry-run --runtime manual --role aggregator --config-dir "$RUN/dry/conf" --bin "$BIN" \
--listen 127.0.0.1 --http-port $PORT_DRY --file-dir "$RUN/dry/out" --tls --cert-file "$PKI/aggregator.crt" \
--key-file "$PKI/aggregator.key" --auth scram --add-user edge-01 --sink-auth scram --add-viewer viewer-01 \
--secrets-dir "$RUN/dry/secrets"
rc=$?
ok=$((rc == 0))
said dry-manual "+ write $RUN/dry/conf/logwisp.toml" || ok=0
said dry-manual "+ generate a password into $RUN/dry/secrets/edge-01.pass" || ok=0
[[ -e $RUN/dry || $(snapshot) != "$before" ]] && ok=0
check "manual: printed the configuration and the passwords it would generate, wrote nothing (exit $rc)" $ok
deploy dry-zone --dry-run --runtime manual --role aggregator --config-dir "$RUN/dry/conf" --listen fe80::1%lo \
--http-port $PORT_DRY --file-dir "$RUN/dry/out" --tls --cert-file "$PKI/aggregator.crt" \
--key-file "$PKI/aggregator.key" --sink-auth scram --add-viewer viewer-01 --secrets-dir "$RUN/dry/secrets"
rc=$?
ok=$((rc == 0))
said dry-zone "-url https://[fe80::1%25lo]:$PORT_DRY " || ok=0
said dry-zone 'host = "fe80::1%lo"' || ok=0
check "a link-local listener: its URLs escape the zone as %25, its host keeps it (exit $rc)" $ok
deploy dry-native --dry-run --runtime native --role edge --log-dir "$APP" --aggregator 127.0.0.1 \
--chain-port $PORT_CHAIN --node deploy-native --bin "$BIN"
rc=$?
ok=$((rc == 0))
said dry-native "+ install -m 0755 $BIN /usr/local/bin/lw" || ok=0
said dry-native "+ write /etc/logwisp/logwisp.toml" || ok=0
said dry-native "+ systemctl enable logwisp" || ok=0
[[ $(snapshot) != "$before" ]] && ok=0
check "native: printed the install, the configuration and systemctl, changed nothing (exit $rc)" $ok
deploy dry-docker --dry-run --runtime docker --role aggregator --name "$CONTAINER" --image "$IMAGE" --no-build \
--config-dir "$RUN/docker/conf" --file-dir "$OUT/docker" --listen 127.0.0.1 \
--chain-port $PORT_D_CHAIN --http-port $PORT_D_HTTP
rc=$?
# The command breaks before each option; joined, it is one line
cmd="$(sed -e ':a' -e '/\\$/N; s/\\\n//; ta' "$LOG/deploy-dry-docker.out" | tr -s ' ' | grep '^+ docker run -d ')"
ok=$((rc == 0))
for want in "--name $CONTAINER" "--restart unless-stopped" "--read-only" "--cap-drop ALL" \
"--security-opt no-new-privileges" "-u 65532:65532" "-v $RUN/docker/conf:/etc/logwisp:ro" \
"-v $OUT/docker:$OUT/docker" "-p 127.0.0.1:$PORT_D_CHAIN:$PORT_D_CHAIN" \
"-p 127.0.0.1:$PORT_D_HTTP:$PORT_D_HTTP" "$IMAGE -c /etc/logwisp/logwisp.toml"; do
[[ $cmd == *" $want"* ]] || { ok=0; info "the docker run command lacks: $want"; }
done
said dry-docker '| host = "0.0.0.0"' || ok=0
# The snapshot leaves out $OUT, where its --file-dir lies
[[ -e $OUT/docker || $(snapshot) != "$before" ]] && ok=0
check "docker: a hardened docker run publishing on --listen, the container binding 0.0.0.0, nothing written (exit $rc)" $ok
section "Docker runtime"
if ! command -v docker >/dev/null || ! docker info >/dev/null 2>&1; then
skip "docker: no docker daemon here"
elif ((EUID != 0)); then
skip "docker: lw-deploy.sh needs root to give the configuration to the image user"
elif ! docker image inspect "$IMAGE" >/dev/null 2>&1; then
skip "docker: no $IMAGE image here (make image)"
else
at_exit "docker rm -f $CONTAINER"
deploy docker --runtime docker --role aggregator --name "$CONTAINER" --image "$IMAGE" --no-build \
--config-dir "$RUN/docker/conf" --file-dir "$OUT/docker" --listen 127.0.0.1 \
--chain-port $PORT_D_CHAIN --http-port $PORT_D_HTTP
rc=$?
check "docker: the aggregator container is running (exit $rc)" $(is "$rc" 0)
generate docker-edge --role edge --log-dir "$APP" --aggregator 127.0.0.1 --chain-port $PORT_D_CHAIN \
--node deploy-docker
spawn docker-edge "$BIN" -c "$RUN/docker-edge/logwisp.toml"
for i in $(seq 1 15); do
echo "line $i for the container" >>"$APP/app.log"
sleep 1
grep -q 'deploy-docker/app.log' "$OUT/docker/logwisp.log" 2>/dev/null && break
done
n=$(grep -c 'deploy-docker/app.log' "$OUT/docker/logwisp.log" 2>/dev/null)
check "docker: a native edge's entries reached the container's file output (${n:-0} lines)" $((${n:-0} >= 1))
n=$(total_processed $PORT_D_HTTP)
check "docker: the published http sink answers /status (total_processed=$n)" $((n >= 1))
fi
summary
test/ipv6-test.sh
Executable
+244
View File
@@ -0,0 +1,244 @@
#!/usr/bin/env bash
# logwisp IPv6 test: a relay on ::1 takes both chain transports and serves
# viewers over TLS with an IPv6 SAN, and listeners keep to their address
# family. Skips (exit 77) on a host without an IPv6 loopback.
# Requires: bash 5+, openssl, curl, an IPv6 loopback. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
RUN=$E2E_DIR/run-ipv6
PKI=$RUN/pki
AUTH=$RUN/auth
OUT=$RUN/out
USERS=$AUTH/users.toml
PORT_TCP_CHAIN=15851
PORT_HTTP_CHAIN=15852
PORT_TCP_SINK=15853
PORT_HTTP_SINK=15854
PORT_FAMILY=15855
PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_FAMILY"
e2e_init "$@"
section "Setup"
# /proc/net/if_inet6 lists ::1 as 31 zeros and a 1; it is absent without IPv6
grep -qs '^0\{31\}1 ' /proc/net/if_inet6 || skip_all "this host has no IPv6 loopback (::1)"
need openssl curl
ports_free $PORTS
rm -rf "$RUN"
mkdir -p "$CONF" "$LOG" "$PKI" "$AUTH" "$OUT"
# The relay certificate names only ::1: every dialer verifies an IPv6 SAN
pki_ca
pki_leaf relay relay.internal serverAuth "IP:::1"
info "test PKI in $(short "$PKI")/: ca, relay (SAN IP ::1)"
add_users "$USERS" edge-01 viewer-01
info "users edge-01, viewer-01 in $(short "$USERS")"
tls_server() { printf '[%s.tls]\nenabled = true\ncert_file = "%s"\nkey_file = "%s"\n' "$1" "$PKI/relay.crt" "$PKI/relay.key"; }
scram_server() { printf '[%s.auth]\ntype = "scram"\ncredentials_file = "%s"\n' "$1" "$USERS"; }
relay_pipeline() { # name source_type source_port sink_type sink_port
cat <<EOF
[[pipelines]]
name = "$1"
[pipelines.flow.format]
type = "json"
sanitizer_policy = "json"
[[pipelines.plugin_sources]]
id = "in"
type = "$2"
[pipelines.plugin_sources.config]
host = "::1"
port = $3
EOF
tls_server pipelines.plugin_sources.config
scram_server pipelines.plugin_sources.config
cat <<EOF
[[pipelines.plugin_sinks]]
id = "file"
type = "file"
[pipelines.plugin_sinks.config]
directory = "$OUT"
name = "$2"
flush_interval_ms = 200
[[pipelines.plugin_sinks]]
id = "out"
type = "$4"
[pipelines.plugin_sinks.config]
host = "::1"
port = $5
EOF
tls_server pipelines.plugin_sinks.config
scram_server pipelines.plugin_sinks.config
}
{
printf 'status_reporter = false\n[logging]\noutput = "stdout"\nlevel = "info"\n'
relay_pipeline relay_tcp tcp_chain $PORT_TCP_CHAIN tcp $PORT_TCP_SINK
relay_pipeline relay_http http_chain $PORT_HTTP_CHAIN http $PORT_HTTP_SINK
} >"$CONF/relay.toml"
edge_conf() { # name sink_type port
cat >"$CONF/$1.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
level = "info"
[[pipelines]]
name = "$1"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "$2"
[pipelines.plugin_sinks.config]
host = "::1"
port = $3
node = "$1"
[pipelines.plugin_sinks.config.tls]
enabled = true
ca_file = "$PKI/ca.crt"
[pipelines.plugin_sinks.config.auth]
type = "scram"
username = "edge-01"
password_file = "$AUTH/edge-01.pass"
EOF
}
edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN
edge_conf edge_http http_chain $PORT_HTTP_CHAIN
# Plain http sinks whose /status names the listener that answered
family_conf() { # name host
cat >"$CONF/$1.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
level = "info"
[[pipelines]]
name = "$1"
[[pipelines.plugin_sources]]
id = "none"
type = "null"
[[pipelines.plugin_sinks]]
id = "$1"
type = "http"
[pipelines.plugin_sinks.config]
host = "$2"
port = $PORT_FAMILY
EOF
}
family_conf any6 "::"
family_conf any4 "0.0.0.0"
info "configurations in $(short "$CONF")/"
section "Startup"
start_daemon relay relay.toml
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
wait_port "$p" ::1 || abort "relay port $p is not listening on ::1" relay
done
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
start_daemon any6 any6.toml
ANY6_PID=${PIDS[-1]}
wait_port $PORT_FAMILY ::1 || abort "any6 is not listening on [::]:$PORT_FAMILY" any6
# The checks start any4 themselves, after testing :: alone
((AUTO)) || start_daemon any4 any4.toml
daemons_up
write_env LW="$BIN" CA="$PKI/ca.crt" PW="$AUTH/viewer-01.pass"
guide "logwisp IPv6 test" <<EOF
Ports (relay on ::1, TLS with SAN IP ::1, scram):
$PORT_TCP_CHAIN relay tcp_chain ingest, edge_tcp dials [::1]
$PORT_HTTP_CHAIN relay http_chain ingest, edge_http dials [::1]
$PORT_TCP_SINK tcp sink
$PORT_HTTP_SINK http sink
$PORT_FAMILY plain http sinks side by side: any6 on ::, any4 on 0.0.0.0
Shell setup (LW, CA, PW: viewer-01's password file):
> . $(short "$RUN")/env
Read the tcp sink as viewer-01:
> \$LW auth stream -addr [::1]:$PORT_TCP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA
Read the http sink (bearer keeps the token off argv; -g leaves the brackets to the URL):
> token=\$(\$LW auth token -url https://[::1]:$PORT_HTTP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA)
> curl -gN --cacert \$CA -H @<(bearer) https://[::1]:$PORT_HTTP_SINK/stream
Which listener answers on $PORT_FAMILY:
> curl -s http://127.0.0.1:$PORT_FAMILY/status; curl -gs http://[::1]:$PORT_FAMILY/status
Ingested entries: $(short "$OUT")/ Passwords: $(short "$AUTH")/
Logs: $(short "$LOG")/
EOF
manual_hold
relay_log="$LOG/relay.out"
curl_args=(-g -s --max-time 5 --noproxy '*')
answered_by() { # url -> the instance whose /status answered, empty if refused
curl "${curl_args[@]}" "$1/status" 2>/dev/null | grep -o '"instance_id":"[^"]*"' | cut -d'"' -f4
}
section "Scenario 1: chain links and viewers over [::1]"
# 1. Both chain transports deliver; the http_chain edge logged in at
# https://[::1]:PORT/auth first
wait_until 20 has_entry tcp_chain 'edge-01/'
n=$(ingested tcp_chain | grep -c 'edge-01/')
check "tcp_chain: entries from the edge dialing [::1] reached the relay ($n lines)" $((n >= 1))
wait_until 20 has_entry http_chain 'edge-01/'
n=$(ingested http_chain | grep -c 'edge-01/')
check "http_chain: entries from the edge dialing [::1] reached the relay ($n lines)" $((n >= 1))
# The logger quotes a value holding brackets
n=$(grep -cF "addr \"[::1]:$PORT_TCP_CHAIN\"" "$relay_log")
check "relay logged its listener as [::1]:$PORT_TCP_CHAIN ($n)" $((n >= 1))
# 2. HTTP sink: lw auth token at an IPv6 URL, the token carried by curl -g
token="$("$BIN" auth token -url "https://[::1]:$PORT_HTTP_SINK" -user viewer-01 \
-password-file "$AUTH/viewer-01.pass" -ca-file "$PKI/ca.crt" 2>>"$LOG/auth-cli.out")"
check "http sink: lw auth token -url https://[::1]:$PORT_HTTP_SINK issued a token" $(is_set "$token")
code="$(curl "${curl_args[@]}" -o /dev/null -w '%{http_code}' --cacert "$PKI/ca.crt" \
-H @<(printf 'Authorization: Bearer %s\n' "$token") "https://[::1]:$PORT_HTTP_SINK/status" 2>/dev/null || true)"
check "http sink: curl -g https://[::1]:$PORT_HTTP_SINK/status with the token (HTTP $code)" $(is "$code" 200)
sse="$(timeout 4 curl -gsN --noproxy '*' --cacert "$PKI/ca.crt" \
-H @<(printf 'Authorization: Bearer %s\n' "$token") \
"https://[::1]:$PORT_HTTP_SINK/stream" 2>/dev/null || true)"
n=$(grep -c '^data:.*edge-01/' <<<"$sse")
check "http sink: /stream delivered SSE events over [::1] ($n events)" $((n >= 1))
# 3. TCP sink: lw auth stream at an IPv6 address
timeout 4 "$BIN" auth stream -addr "[::1]:$PORT_TCP_SINK" -user viewer-01 \
-password-file "$AUTH/viewer-01.pass" -ca-file "$PKI/ca.crt" \
>"$RUN/stream.out" 2>"$LOG/stream.err"
n=$(grep -c 'edge-01/' "$RUN/stream.out")
check "tcp sink: lw auth stream -addr [::1]:$PORT_TCP_SINK streamed entries ($n lines)" $((n >= 1))
section "Scenario 2: each listener keeps to its address family"
# 4. :: is IPv6-only: it takes [::1] and leaves IPv4, and its port, alone
who="$(answered_by "http://[::1]:$PORT_FAMILY")"
check ":: listener accepts [::1] (answered by ${who:-nobody})" $(is "$who" any6)
who="$(answered_by "http://127.0.0.1:$PORT_FAMILY")"
check ":: listener refuses 127.0.0.1 (answered by ${who:-nobody})" $(is "$who" "")
start_daemon any4 any4.toml
wait_port $PORT_FAMILY 127.0.0.1
who="$(answered_by "http://127.0.0.1:$PORT_FAMILY")"
check "0.0.0.0 binds the same port beside :: and takes 127.0.0.1 (answered by ${who:-nobody})" $(is "$who" any4)
# 5. 0.0.0.0 is IPv4-only: with :: gone, [::1] finds nobody
kill -TERM "$ANY6_PID"
wait "$ANY6_PID" 2>/dev/null
who="$(answered_by "http://[::1]:$PORT_FAMILY")"
check "0.0.0.0 listener refuses [::1] (answered by ${who:-nobody})" $(is "$who" "")
summary
test/lib.sh
Executable
+288
View File
@@ -0,0 +1,288 @@
# Shared by the e2e scripts (test/*-test.sh): sourced after the script sets RUN,
# then e2e_init parses the common flags. Daemons log to $LOG/NAME.out only.
# Exit codes: 0 every check passed, 1 a check failed, 77 the script skipped.
E2E_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
BIN=${LOGWISP_BIN:-${E2E_DIR%/*}/bin/lw}
AUTO=0 KEEP=0 FOLLOW=0
PIDS=() DAEMONS=() AT_EXIT=()
E2E_PASSED=0 E2E_FAILED=0 E2E_SKIPPED=0
# What --follow shows besides WARN and ERROR lines; a script may extend it
FOLLOW_KEYS='LogWisp started|hot reload completed|Login accepted|rejected|refused'
PKI_DAYS=45 # past the 30-day expiry warning, which would fill the daemon logs
# https://no-color.org; FORCE_COLOR=1 keeps colour through a pipe
if [[ ${FORCE_COLOR:-0} != 0 ]] || [[ -t 1 && -z ${NO_COLOR:-} ]]; then
C_BOLD=$'\e[1m' C_DIM=$'\e[2m' C_RED=$'\e[31m' C_GREEN=$'\e[32m'
C_YELLOW=$'\e[33m' C_CYAN=$'\e[36m' C_OFF=$'\e[0m'
else
C_BOLD='' C_DIM='' C_RED='' C_GREEN='' C_YELLOW='' C_CYAN='' C_OFF=''
fi
usage() {
sed -n '2,/^[^#]/s/^# \{0,1\}//p' "$0"
cat <<EOF
Usage: $0 [--auto [--keep]] [--follow]
(none) start the daemons, print the guide, keep them up until Ctrl-C
--auto run the checks, then stop the daemons
--keep with --auto: leave them up when every check passes
--follow show the daemons' warnings, errors and key events as they log
Daemon logs: $(short "$RUN")/log/NAME.out. NO_COLOR=1 or FORCE_COLOR=1 sets
colour; LOGWISP_BIN names another lw binary.
EOF
}
e2e_init() { # "$@" of the script
local a
for a; do
case $a in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
--follow) FOLLOW=1 ;;
-h | --help) usage; exit 0 ;;
*) printf '%s: unknown argument %s (see --help)\n' "${0##*/}" "$a" >&2; exit 2 ;;
esac
done
CONF=$RUN/conf LOG=$RUN/log
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
trap 'exit 129' HUP
trap 'exit 141' PIPE
printf '%s%s%s\n' "$C_BOLD" "${0##*/}$( ((AUTO)) && echo ' --auto')" "$C_OFF"
}
# --- output ---
rule() { local s='' n=$1; while ((n-- > 0)); do s+='─'; done; printf '%s' "$s"; }
short() { if [[ $1 == "$PWD"/* ]]; then printf '%s' "${1#"$PWD"/}"; else printf '%s' "$1"; fi; }
section() { printf '\n%s── %s %s%s\n' "$C_BOLD$C_CYAN" "$1" "$(rule $((60 - ${#1})))" "$C_OFF"; }
info() { printf ' %s·%s %s\n' "$C_DIM" "$C_OFF" "$*"; }
pass() { E2E_PASSED=$((E2E_PASSED + 1)); printf ' %sPASS%s %s\n' "$C_BOLD$C_GREEN" "$C_OFF" "$1"; }
fail() { E2E_FAILED=$((E2E_FAILED + 1)); printf ' %sFAIL%s %s\n' "$C_BOLD$C_RED" "$C_OFF" "$1"; }
skip() { E2E_SKIPPED=$((E2E_SKIPPED + 1)); printf ' %sSKIP%s %s\n' "$C_BOLD$C_YELLOW" "$C_OFF" "$1"; }
check() { if (($2)); then pass "$1"; else fail "$1"; fi; } # LABEL 1|0
is() { [[ $1 == "$2" ]] && echo 1 || echo 0; } # ACTUAL EXPECTED
is_set() { [[ -n $1 ]] && echo 1 || echo 0; }
# guide TITLE, the body on stdin: "> " lines are commands, " NNNNN " lines
# ports, lines ending in ":" headings.
guide() {
local line
printf '\n%s┌── %s %s%s\n' "$C_BOLD" "$1" "$(rule $((59 - ${#1})))" "$C_OFF"
while IFS= read -r line; do
case $line in
'> '*) line=" $C_CYAN${line#> }$C_OFF" ;;
' '[0-9][0-9][0-9][0-9][0-9]' '*) line=" $C_YELLOW${line:2:5}$C_OFF${line:7}" ;;
*:) line="$C_BOLD$line$C_OFF" ;;
esac
printf '%s│%s %s\n' "$C_BOLD" "$C_OFF" "$line"
done
printf '%s└%s%s\n' "$C_BOLD" "$(rule 63)" "$C_OFF"
}
# write_env NAME=VALUE...: $RUN/env, which the guide's commands source
write_env() {
local kv
{
echo "# . $(short "$RUN")/env: what the guide of ${0##*/} refers to"
echo 'export no_proxy="127.0.0.1,127.0.0.2,::1${no_proxy:+,$no_proxy}"; export NO_PROXY=$no_proxy'
for kv; do printf 'export %s=%q\n' "${kv%%=*}" "${kv#*=}"; done
# keeps a bearer token off argv: curl -H @<(bearer)
echo "bearer() { printf 'Authorization: Bearer %s\\n' \"\$token\"; }"
} >"$RUN/env"
}
# --- results ---
# abort MESSAGE [DAEMON]: a failure the checks cannot run past; shows the
# daemon's last warnings and errors
abort() {
fail "$1"
ABORTED=${2:-}
[[ -z $ABORTED ]] || excerpt "$ABORTED" '^([^ ]+ (WARN|ERROR) |Error: )' 5
summary
}
summary() {
local name counts="$E2E_PASSED passed, $E2E_FAILED failed, $E2E_SKIPPED skipped"
stop_follow
if ((E2E_FAILED == 0 && KEEP)); then
info "--keep: left running: pids ${PIDS[*]}${AT_EXIT[*]:+; later: ${AT_EXIT[*]}}"
PIDS=() AT_EXIT=()
fi
stop_all
if ((E2E_FAILED)); then
for name in "${DAEMONS[@]}"; do
[[ $name == "${ABORTED:-}" ]] || excerpt "$name" '^([^ ]+ ERROR |Error: )'
done
fi
printf '%s%s%s\n' "$C_DIM" "$(rule 64)" "$C_OFF"
if ((E2E_FAILED)); then
printf '%sRESULT: FAILURES%s (%s); logs in %s/\n' "$C_BOLD$C_RED" "$C_OFF" "$counts" "$(short "$LOG")"
exit 1
fi
if ((E2E_SKIPPED)); then
printf '%sRESULT: PASS%s (%s)\n' "$C_BOLD$C_GREEN" "$C_OFF" "$counts"
else
printf '%sRESULT: ALL PASS%s (%s)\n' "$C_BOLD$C_GREEN" "$C_OFF" "$counts"
fi
exit 0
}
skip_all() { # REASON: nothing in the script applies to this host
skip "$1"
printf '%s%s%s\n%sRESULT: SKIPPED%s\n' "$C_DIM" "$(rule 64)" "$C_OFF" "$C_BOLD$C_YELLOW" "$C_OFF"
exit 77
}
# --- processes ---
spawn() { # NAME COMMAND...: in the background, output to $LOG/NAME.out
local name=$1
shift
"$@" >"$LOG/$name.out" 2>&1 &
PIDS+=($!) DAEMONS+=("$name")
}
start_daemon() { spawn "$1" "$BIN" -c "$CONF/$2"; } # NAME CONF_FILE
daemons_up() { info "running: ${DAEMONS[*]} (logs: $(short "$LOG")/NAME.out)"; }
at_exit() { AT_EXIT+=("$1"); } # COMMAND, run at teardown
stop_all() { # TERM every process, KILL what outlives 10 s
local pid deadline=$((SECONDS + 10))
((${#PIDS[@]})) || return 0
kill -TERM "${PIDS[@]}" 2>/dev/null
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && ((SECONDS < deadline)); do sleep 0.1; done
kill -KILL "$pid" 2>/dev/null
done
info "stopped ${#PIDS[@]} process(es)"
PIDS=()
}
cleanup() {
local rc=$? c
trap '' PIPE
trap - EXIT
stop_follow
stop_all
for c in "${AT_EXIT[@]}"; do eval "$c" >/dev/null 2>&1; done
exit "$rc"
}
# follow_logs: with --follow, WARN, ERROR and FOLLOW_KEYS lines of every daemon
# started so far, as they are logged
follow_logs() {
local name files=()
((FOLLOW)) || return 0
for name in "${DAEMONS[@]}"; do files+=("$LOG/$name.out"); done
tail -v -n +1 -F "${files[@]}" 2>/dev/null > >(follow_filter) &
FOLLOW_PID=$!
info "following warnings, errors and key events of: ${DAEMONS[*]}"
}
follow_filter() { # tail -v output -> "NAME HH:MM:SS LEVEL message"; read, not awk, for line buffering
local line name='' ts lvl rest c
while IFS= read -r line; do
case $line in '==> '*' <==')
name=${line#==> } name=${name% <==} name=${name##*/} name=${name%.out}
continue ;;
esac
read -r ts lvl rest <<<"$line"
# lw's own startup failure, before its logger runs: "Error: ..."
[[ $ts == Error: ]] && rest="$lvl $rest" lvl=ERROR ts=''
case $lvl in
ERROR) c=$C_RED ;;
WARN) c=$C_YELLOW ;;
*) [[ $line =~ $FOLLOW_KEYS ]] || continue; c=$C_CYAN ;;
esac
printf ' %s%-12s %s%s %s%-5s %s%s\n' "$C_DIM" "$name" "${ts:11:8}" "$C_OFF" "$c" "$lvl" "${rest#msg }" "$C_OFF"
done
}
stop_follow() { [[ -z ${FOLLOW_PID:-} ]] || kill "$FOLLOW_PID" 2>/dev/null; FOLLOW_PID=''; }
# manual_hold: starts --follow; without --auto, keeps the daemons up until Ctrl-C
manual_hold() {
follow_logs
((AUTO)) && return 0
info "manual mode: the daemons stay up until Ctrl-C (--auto runs the checks)"
while :; do sleep 1; done
}
# --- network and files ---
# listening PORT [HOST]: a listener on PORT in HOST's address family. It reads
# the kernel's table: a probe connection would land in the daemon logs, as a
# failed TLS handshake or hello the checks count.
listening() {
local table=/proc/net/tcp
[[ ${2:-} == *:* ]] && table=/proc/net/tcp6
awk -v port="$(printf ':%04X' "$1")" '$4 == "0A" && substr($2, length($2) - 4) == port { found = 1 }
END { exit !found }' "$table" 2>/dev/null
}
wait_port() { # PORT [HOST]: up to 10 s
local i
for ((i = 0; i < 100; i++)); do
listening "$@" && return 0
sleep 0.1
done
return 1
}
wait_until() { # SECONDS COMMAND...
local deadline=$((SECONDS + $1))
shift
until "$@"; do
((SECONDS >= deadline)) && return 1
sleep 0.2
done
}
ports_free() { # PORT...: in either family, as a dual-stack [::] listener holds IPv4 too
local p
for p; do
listening "$p" || listening "$p" :: && abort "port $p is already in use"
done
return 0
}
need() { # TOOL...: the binary and each tool, or abort
[[ -x $BIN ]] || abort "no lw binary at $(short "$BIN"): make build"
local t
for t; do command -v "$t" >/dev/null || abort "$t not found"; done
}
# What a plain TCP sink client receives in SECONDS
tcp_read() { timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true; }
ingested() { cat "$OUT/$1"* 2>/dev/null; } # FILE_PREFIX: the relay's file sinks
has_entry() { ingested "$1" | grep -q -- "$2"; } # FILE_PREFIX PATTERN
count() { grep -c -- "$2" "$LOG/$1.out" 2>/dev/null; } # DAEMON PATTERN
logged() { local n; n=$(count "$1" "$2"); ((${n:-0} >= ${3:-1})); } # DAEMON PATTERN [MIN]
excerpt() { # DAEMON ERE [LINES]: its last matching WARN or ERROR lines
{ echo "==> $1 <=="; grep -E "$2" "$LOG/$1.out" 2>/dev/null | tail -n "${3:-3}"; } | follow_filter
}
# --- credentials ---
pki_key() { openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1" 2>/dev/null; }
pki_ca() { # the test CA in $PKI
mkdir -p "$PKI"
pki_key "$PKI/ca.key"
openssl req -x509 -new -key "$PKI/ca.key" -days "$PKI_DAYS" -out "$PKI/ca.crt" \
-subj "/CN=LogWisp Test CA" 2>/dev/null || abort "PKI: cannot create the CA"
}
pki_leaf() { # NAME CN EKU [SAN]: $PKI/NAME.crt and NAME.key from the test CA
local name=$1 ext="extendedKeyUsage=$3"
[[ -n ${4:-} ]] && ext+=$'\n'"subjectAltName=$4"
printf '%s\n' "$ext" >"$PKI/$name.ext"
pki_key "$PKI/$name.key"
openssl req -new -key "$PKI/$name.key" -out "$PKI/$name.csr" -subj "/CN=$2" 2>/dev/null
openssl x509 -req -in "$PKI/$name.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" -CAcreateserial \
-out "$PKI/$name.crt" -days "$PKI_DAYS" -extfile "$PKI/$name.ext" 2>/dev/null
[[ -s $PKI/$name.crt ]] || abort "PKI: cannot issue $name"
}
add_users() { # CREDENTIALS USER...: through the CLI, passwords in $AUTH/USER.pass
local u
for u in "${@:2}"; do
"$BIN" auth add-user -credentials "$1" -user "$u" -password-file "$AUTH/$u.pass" \
>>"$LOG/auth-cli.out" 2>&1 || abort "lw auth add-user $u failed" auth-cli
done
}
test/mtls-chain-test.sh
+111 -303
View File
@@ -1,123 +1,43 @@
#!/usr/bin/env bash
# logwisp mTLS authentication test
#
# Scenario 1 — chained instances, client authenticates with mTLS:
# edge-01 cert --> tcp_chain sink --> :15811 tcp_chain src --> file sink
# edge-01 cert --> http_chain sink --> :15812 http_chain src --> file sink
# edge-99 cert --> tcp_chain sink --> :15811 rejected by the allow list
#
# Scenario 2 — a viewer client reads a streaming sink over mTLS:
# viewer-01 cert --> :15813 tcp sink (openssl s_client)
# viewer-01 cert --> :15814 http sink (curl, /stream and /status)
# rogue cert --> both, rejected by the allow list
#
# Also covers: node binding (a peer holding the edge-01 certificate cannot
# label its entries anything else), dialer-side server identity pinning, and
# a peer presenting no certificate at all.
#
# Usage:
# ./mtls-chain-test.sh manual mode: relay + edges up, guide printed
# ./mtls-chain-test.sh --auto automated checks and teardown
# ./mtls-chain-test.sh --keep (with --auto) skip teardown on success
#
# logwisp mTLS test: edges authenticate to a relay with client certificates,
# viewers to its tcp and http sinks; covers the allow lists, node binding (an
# edge-01 certificate labels its entries edge-01), dialer-side server pinning
# and a peer without a certificate.
# Requires: bash 5+, coreutils (timeout), openssl, curl. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run-mtls"
CONF="$RUN/conf"
LOG="$RUN/log"
PKI="$RUN/pki"
OUT="$RUN/out"
RUN=$E2E_DIR/run-mtls
PKI=$RUN/pki
OUT=$RUN/out
PORT_TCP_CHAIN=15811
PORT_HTTP_CHAIN=15812
PORT_TCP_SINK=15813
PORT_HTTP_SINK=15814
PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK"
e2e_init "$@"
AUTO=0; KEEP=0
for a in "$@"; do case "$a" in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
*) echo "unknown arg: $a" >&2; exit 1 ;;
esac; done
PIDS=()
cleanup() {
local rc=$?
trap - EXIT INT TERM
if (( ${#PIDS[@]} )); then
echo "--- teardown: stopping ${#PIDS[@]} daemon(s)"
kill -TERM "${PIDS[@]}" 2>/dev/null
local deadline=$(( SECONDS + 10 ))
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
kill -KILL "$pid" 2>/dev/null
done
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
wait_port() { # port timeout_s
local i; for (( i=0; i < $2 * 10; i++ )); do
port_open "$1" && return 0
sleep 0.1
done
return 1
}
start_daemon() { # name conf
"$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 &
PIDS+=($!)
echo "started $1 (pid $!)"
}
# --- Preflight ---
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
command -v openssl >/dev/null || { echo "openssl not found" >&2; exit 1; }
command -v curl >/dev/null || { echo "curl not found" >&2; exit 1; }
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
done
section "Setup"
need openssl curl
ports_free $PORTS
rm -rf "$RUN"
mkdir -p "$CONF" "$LOG" "$PKI" "$OUT"
# --- PKI ---
# One CA for every peer: the point of the test is that CA membership alone is
# no longer sufficient, so the identities must all be issued by the same CA.
gen_key() { openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1" 2>/dev/null; }
pki_ca
pki_leaf relay relay.internal serverAuth "IP:127.0.0.1,DNS:relay.internal"
pki_leaf edge-01 edge-01 clientAuth
pki_leaf edge-02 edge-02 clientAuth
pki_leaf edge-99 edge-99 clientAuth
pki_leaf viewer-01 viewer-01 clientAuth
pki_leaf rogue rogue-viewer clientAuth
info "test PKI in $(short "$PKI")/: ca, relay, edge-01, edge-02, edge-99, viewer-01, rogue"
gen_leaf() { # name CN eku [SAN]
local name=$1 cn=$2 eku=$3 san=${4:-}
gen_key "$PKI/$name.key"
openssl req -new -key "$PKI/$name.key" -out "$PKI/$name.csr" -subj "/CN=$cn" 2>/dev/null
local ext="extendedKeyUsage=$eku"
[[ -n $san ]] && ext+=$'\n'"subjectAltName=$san"
printf '%s\n' "$ext" > "$PKI/$name.ext"
openssl x509 -req -in "$PKI/$name.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" \
-CAcreateserial -out "$PKI/$name.crt" -days 2 -extfile "$PKI/$name.ext" 2>/dev/null
}
echo "--- generating test PKI in $PKI"
gen_key "$PKI/ca.key"
openssl req -x509 -new -key "$PKI/ca.key" -days 2 -out "$PKI/ca.crt" \
-subj "/CN=LogWisp Test CA" 2>/dev/null
gen_leaf relay relay.internal serverAuth "IP:127.0.0.1,DNS:relay.internal"
gen_leaf edge-01 edge-01 clientAuth
gen_leaf edge-99 edge-99 clientAuth
gen_leaf viewer-01 viewer-01 clientAuth
gen_leaf rogue rogue-viewer clientAuth
[[ -s "$PKI/rogue.crt" ]] || { echo "PKI generation failed" >&2; exit 1; }
# --- Config generation ---
# Relay: both ingest ports authorize edge-01 only and bind the node label to
# the certificate identity; both streaming sinks authorize viewer-01 only.
cat > "$CONF/relay.toml" <<EOF
cat >"$CONF/relay.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
@@ -144,7 +64,7 @@ client_ca_file = "$PKI/ca.crt"
[pipelines.plugin_sources.config.auth]
type = "mtls"
identity = "cn"
allow = ["edge-01"]
allow = ["edge-01", "edge-02"]
node_binding = "force"
[[pipelines.plugin_sinks]]
@@ -219,123 +139,15 @@ type = "mtls"
allow = ["viewer-01"]
EOF
# edge_tcp holds the edge-01 certificate but declares node "edge-tcp":
# node_binding = "force" must relabel its entries to "edge-01".
cat > "$CONF/edge_tcp.toml" <<EOF
status_reporter = false
[logging]
output = "file"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_tcp"
[[pipelines]]
name = "edge_tcp"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "tcp_chain"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_TCP_CHAIN
node = "edge-tcp"
[pipelines.plugin_sinks.config.tls]
enabled = true
ca_file = "$PKI/ca.crt"
cert_file = "$PKI/edge-01.crt"
key_file = "$PKI/edge-01.key"
[pipelines.plugin_sinks.config.auth]
type = "mtls"
allow = ["relay.internal"]
EOF
cat > "$CONF/edge_http.toml" <<EOF
status_reporter = false
[logging]
output = "file"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_http"
[[pipelines]]
name = "edge_http"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "http_chain"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_HTTP_CHAIN
node = "edge-http"
flush_interval_ms = 500
[pipelines.plugin_sinks.config.tls]
enabled = true
ca_file = "$PKI/ca.crt"
cert_file = "$PKI/edge-01.crt"
key_file = "$PKI/edge-01.key"
[pipelines.plugin_sinks.config.auth]
type = "mtls"
allow = ["relay.internal"]
EOF
# edge_rogue holds a CA-issued certificate the relay does not authorize, and
# claims to be edge-01 on top of it.
cat > "$CONF/edge_rogue.toml" <<EOF
status_reporter = false
[logging]
output = "file"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_rogue"
[[pipelines]]
name = "edge_rogue"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "tcp_chain"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_TCP_CHAIN
node = "edge-01"
[pipelines.plugin_sinks.config.tls]
enabled = true
ca_file = "$PKI/ca.crt"
cert_file = "$PKI/edge-99.crt"
key_file = "$PKI/edge-99.key"
EOF
# edge_pinfail pins a server identity the relay does not have: the dialer must
# refuse the handshake even though the certificate chains to the trusted CA.
cat > "$CONF/edge_pinfail.toml" <<EOF
edge_conf() { # name sink_type port node cert level [sink options]: the auth block follows
cat >"$CONF/$1.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
level = "debug"
level = "$6"
[[pipelines]]
name = "edge_pinfail"
name = "$1"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
@@ -345,68 +157,72 @@ format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "tcp_chain"
type = "$2"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_TCP_CHAIN
node = "edge-pinfail"
backoff_max_ms = 1000
port = $3
node = "$4"
${7:-}
[pipelines.plugin_sinks.config.tls]
enabled = true
ca_file = "$PKI/ca.crt"
cert_file = "$PKI/edge-01.crt"
key_file = "$PKI/edge-01.key"
[pipelines.plugin_sinks.config.auth]
type = "mtls"
allow = ["some-other-relay.internal"]
cert_file = "$PKI/$5.crt"
key_file = "$PKI/$5.key"
EOF
}
pin() { printf '[pipelines.plugin_sinks.config.auth]\ntype = "mtls"\nallow = ["%s"]\n' "$2" >>"$CONF/$1.toml"; }
# --- Guide ---
cat <<EOF
================================================================
logwisp mTLS auth test — port map
$PORT_TCP_CHAIN relay ingest (tcp_chain, mTLS, allow = edge-01)
$PORT_HTTP_CHAIN relay ingest (http_chain, mTLS, allow = edge-01)
$PORT_TCP_SINK TCP sink (mTLS, allow = viewer-01)
$PORT_HTTP_SINK HTTP sink (mTLS, allow = viewer-01)
# edge_tcp holds the edge-01 certificate but declares node "edge-tcp":
# node_binding = "force" must relabel its entries to "edge-01".
edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN edge-tcp edge-01 info
pin edge_tcp relay.internal
edge_conf edge_http http_chain $PORT_HTTP_CHAIN edge-http edge-01 info "flush_interval_ms = 500"
pin edge_http relay.internal
# edge_rogue holds a CA-issued certificate the relay does not authorize, and
# claims to be edge-01 on top of it.
edge_conf edge_rogue tcp_chain $PORT_TCP_CHAIN edge-01 edge-99 info
# edge_pinfail pins a server identity the relay does not have: the dialer must
# refuse the handshake even though the certificate chains to the trusted CA.
# The relay admits its edge-02, so only the pin can keep its entries out.
edge_conf edge_pinfail tcp_chain $PORT_TCP_CHAIN edge-pinfail edge-02 debug "backoff_max_ms = 1000"
pin edge_pinfail some-other-relay.internal
info "configurations in $(short "$CONF")/"
Read the TCP sink as an authorized viewer:
openssl s_client -quiet -connect 127.0.0.1:$PORT_TCP_SINK \\
-CAfile $PKI/ca.crt -cert $PKI/viewer-01.crt -key $PKI/viewer-01.key
Read the HTTP sink:
curl -N --noproxy '*' --cacert $PKI/ca.crt \\
--cert $PKI/viewer-01.crt --key $PKI/viewer-01.key \\
https://127.0.0.1:$PORT_HTTP_SINK/stream
Swap in rogue.crt/rogue.key for either and the policy refuses it.
Ingested entries land in $OUT/ (node label forced to the certificate CN).
Logs: $LOG/
================================================================
EOF
# --- Startup ---
section "Startup"
start_daemon relay relay.toml
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; }
for p in $PORTS; do
wait_port "$p" || abort "relay port $p is not listening" relay
done
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
start_daemon edge_rogue edge_rogue.toml
start_daemon edge_pinfail edge_pinfail.toml
daemons_up
write_env LW="$BIN" PKI="$PKI" CA="$PKI/ca.crt" CERT="$PKI/viewer-01.crt" KEY="$PKI/viewer-01.key"
if (( AUTO == 0 )); then
echo "--- daemons running; Ctrl-C to stop"
while :; do sleep 1; done
fi
guide "logwisp mTLS test" <<EOF
Ports:
$PORT_TCP_CHAIN relay tcp_chain ingest, mTLS, allow edge-01, edge-02
$PORT_HTTP_CHAIN relay http_chain ingest, mTLS, allow edge-01
$PORT_TCP_SINK tcp sink, mTLS, allow viewer-01
$PORT_HTTP_SINK http sink, mTLS, allow viewer-01
Edges: edge_tcp, edge_http (edge-01) deliver; edge_rogue (edge-99) is
refused; edge_pinfail (edge-02, admitted) pins a server name the relay does
not have.
Shell setup (CA, and viewer-01's CERT and KEY):
> . $(short "$RUN")/env
Read the tcp sink, then the http sink, as viewer-01:
> openssl s_client -quiet -connect 127.0.0.1:$PORT_TCP_SINK -CAfile \$CA -cert \$CERT -key \$KEY
> curl -N --cacert \$CA --cert \$CERT --key \$KEY https://127.0.0.1:$PORT_HTTP_SINK/stream
With \$PKI/rogue.crt and rogue.key instead, the policy refuses either.
Ingested entries (node label forced to the certificate CN): $(short "$OUT")/
Logs: $(short "$LOG")/
EOF
manual_hold
echo "--- settling 4s (connect + first http_chain flush)"
info "settling 4 s (connect, first http_chain flush)"
sleep 4
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
# Viewer helpers
tcp_view() { # cert_basename secs
timeout "$2" openssl s_client -quiet \
@@ -414,7 +230,7 @@ tcp_view() { # cert_basename secs
-cert "$PKI/$1.crt" -key "$PKI/$1.key" </dev/null 2>/dev/null || true
}
http_get() { # path cert_basename|"" -> "<http_code>|<body>"
http_get() { # path cert_basename|"" -> "<http_code>"
local path=$1 name=${2:-}
local args=(-s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*'
--cacert "$PKI/ca.crt")
@@ -423,91 +239,83 @@ http_get() { # path cert_basename|"" -> "<http_code>|<body>"
}
relay_log="$LOG/relay.out"
ingested() { cat "$OUT"/${1}* 2>/dev/null; }
echo "=== Scenario 1: chained instances over mTLS ==="
section "Scenario 1: chained instances over mTLS"
# 1. An authorized edge delivers entries into the relay's file sink
tcp_file="$(ingested tcp_chain)"
n=$(grep -c 'edge-01/' <<< "$tcp_file")
check "tcp_chain: authorized edge-01 entries reached the file sink ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-01/' <<<"$tcp_file")
check "tcp_chain: authorized edge-01 entries reached the file sink ($n lines)" $((n >= 1))
http_file="$(ingested http_chain)"
n=$(grep -c 'edge-01/' <<< "$http_file")
check "http_chain: authorized edge-01 entries reached the file sink ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-01/' <<<"$http_file")
check "http_chain: authorized edge-01 entries reached the file sink ($n lines)" $((n >= 1))
# 2. node_binding = "force" overrode the label the sender configured
n=$(grep -c 'edge-tcp/' <<< "$tcp_file")
check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $(( n == 0 ))
n=$(grep -c 'edge-http/' <<< "$http_file")
check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $(( n == 0 ))
n=$(grep -c 'edge-tcp/' <<<"$tcp_file")
check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $((n == 0))
n=$(grep -c 'edge-http/' <<<"$http_file")
check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $((n == 0))
# 3. An identity outside the allow list is refused, even claiming to be edge-01
n=$(grep -c 'Connection rejected by auth policy' "$relay_log")
check "allow list: unauthorized edge-99 connection rejected ($n rejections)" $(( n >= 1 ))
n=$(grep -c 'edge-99' <<< "$tcp_file")
check "allow list: no edge-99 entry was ingested" $(( n == 0 ))
check "allow list: unauthorized edge-99 connection rejected ($n rejections)" $((n >= 1))
n=$(grep -c 'edge-99' <<<"$tcp_file")
check "allow list: no edge-99 entry was ingested" $((n == 0))
# 4. A peer with no certificate cannot complete the handshake
# 4. A peer with no certificate cannot complete the handshake; edge_pinfail
# fails handshakes too, with "bad certificate"
timeout 5 openssl s_client -connect 127.0.0.1:$PORT_TCP_CHAIN \
-CAfile "$PKI/ca.crt" </dev/null >/dev/null 2>&1
sleep 0.5
n=$(grep -c 'TLS handshake failed' "$relay_log")
check "client_auth: a peer with no certificate was refused ($n handshake errors)" $(( n >= 1 ))
n=$(grep -c "TLS handshake failed.*didn't provide a certificate" "$relay_log")
check "client_auth: a peer with no certificate was refused ($n handshake errors)" $((n >= 1))
# 5. Dialer-side pinning: the relay's identity is not the one edge_pinfail pins
n=$(grep -c 'is not allowed' "$LOG/edge_pinfail.out")
check "server pinning: dialer refused a CA-valid server it does not pin ($n refusals)" $(( n >= 1 ))
n=$(grep -c 'edge-pinfail' <<< "$tcp_file")
check "server pinning: pin-failing edge delivered nothing" $(( n == 0 ))
check "server pinning: dialer refused a CA-valid server it does not pin ($n refusals)" $((n >= 1))
n=$(grep -c 'edge-02/' <<<"$tcp_file")
check "server pinning: pin-failing edge-02 delivered nothing ($n lines)" $((n == 0))
echo "=== Scenario 2: viewer clients on mTLS-gated sinks ==="
section "Scenario 2: viewer clients on mTLS-gated sinks"
# 6. TCP sink: authorized viewer streams, rogue gets nothing
out="$(tcp_view viewer-01 4)"
n=$(grep -c 'edge-01/' <<< "$out")
check "tcp sink: viewer-01 streamed entries ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-01/' <<<"$out")
check "tcp sink: viewer-01 streamed entries ($n lines)" $((n >= 1))
out="$(tcp_view rogue 4)"
n=$(grep -c '"message"' <<< "$out")
check "tcp sink: rogue viewer received no entries" $(( n == 0 ))
n=$(grep -c 'edge-01/' <<<"$out")
check "tcp sink: rogue viewer received no entries ($n lines)" $((n == 0))
# 7. HTTP sink: stream and status both gated
code="$(http_get /status viewer-01)"
check "http sink: /status served to viewer-01 (HTTP $code)" $([[ $code == 200 ]] && echo 1 || echo 0)
check "http sink: /status served to viewer-01 (HTTP $code)" $(is "$code" 200)
code="$(http_get /status rogue)"
check "http sink: /status refused to rogue viewer (HTTP $code)" $([[ $code == 403 ]] && echo 1 || echo 0)
check "http sink: /status refused to rogue viewer (HTTP $code)" $(is "$code" 403)
code="$(http_get /stream rogue)"
check "http sink: /stream refused to rogue viewer (HTTP $code)" $([[ $code == 403 ]] && echo 1 || echo 0)
check "http sink: /stream refused to rogue viewer (HTTP $code)" $(is "$code" 403)
# curl reports 000 when the handshake itself fails, which is what a client
# with no certificate must hit
code="$(http_get /status)"
check "http sink: client with no certificate failed the handshake (curl $code)" \
$([[ $code == 000 ]] && echo 1 || echo 0)
check "http sink: client with no certificate failed the handshake (curl $code)" $(is "$code" 000)
sse="$(timeout 4 curl -sN --noproxy '*' --cacert "$PKI/ca.crt" \
--cert "$PKI/viewer-01.crt" --key "$PKI/viewer-01.key" \
"https://127.0.0.1:$PORT_HTTP_SINK/stream" 2>/dev/null || true)"
n=$(grep -c '^data:.*edge-01/' <<< "$sse")
check "http sink: viewer-01 received SSE events ($n events)" $(( n >= 1 ))
n=$(grep -c '^data:.*edge-01/' <<<"$sse")
check "http sink: viewer-01 received SSE events ($n events)" $((n >= 1))
# 8. The status endpoint reports the policy and its rejection count
status="$(curl -s --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt" \
--cert "$PKI/viewer-01.crt" --key "$PKI/viewer-01.key" \
"https://127.0.0.1:$PORT_HTTP_SINK/status" 2>/dev/null || true)"
n=$(grep -c 'mtls' <<< "$status")
check "http sink: status endpoint reports the auth policy" $(( n >= 1 ))
rej=$(grep -o '"auth_rejected"[ :]*[0-9]*' <<< "$status" | grep -o '[0-9]*$' || echo 0)
check "http sink: status endpoint counts auth rejections (auth_rejected=$rej)" $(( rej >= 1 ))
n=$(grep -c 'mtls' <<<"$status")
check "http sink: status endpoint reports the auth policy" $((n >= 1))
rej=$(grep -o '"auth_rejected"[ :]*[0-9]*' <<<"$status" | grep -o '[0-9]*$' || echo 0)
check "http sink: status endpoint counts auth rejections (auth_rejected=$rej)" $((rej >= 1))
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
else
echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log"
fi
exit "$fail"
summary
test/passthrough-test.sh
+22 -36
View File
@@ -1,27 +1,21 @@
#!/usr/bin/env bash
# logwisp file source pass-through test
#
# file src (raw, from=start) --> raw format --> file sink byte-exact relay
# file src (defaults) --> raw format --> file sink no key dropped
#
# The fixture is a record whose envelope is wider than time/level/msg/fields,
# which is what the narrow JSON branch used to reduce to an empty message.
#
# Usage: ./passthrough-test.sh
# logwisp file pass-through test: a file source relays a record whose envelope
# is wider than time/level/msg/fields byte for byte (raw = true) and, parsed,
# with no key dropped. One-shot: both modes run the checks; no ports.
# Requires: bash 5+, coreutils (timeout). Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run/passthrough"
[[ -x $BIN ]] || { echo "no binary at $BIN; run make build" >&2; exit 1; }
RUN=$E2E_DIR/run/passthrough
e2e_init "$@"
section "Setup"
need
rm -rf "$RUN"
mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed"
mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed" "$CONF" "$LOG"
cat > "$RUN/in/wide.jsonl" <<'EOF'
cat >"$RUN/in/wide.jsonl" <<'EOF'
{"time":"2026-09-08T21:35:44.178372768-04:00","level":"INFO","sub":"app","run":0,"tick":0,"frame":0,"fields":{"msg":"init begin","mode":"play"}}
{"time":"2026-09-08T21:35:44.178581366-04:00","level":"PROC","run":0,"tick":0,"frame":0,"fields":{"seq":1,"seed":1788917744178374836}}
plain text line, not JSON at all
@@ -56,37 +50,29 @@ name = "relay"
flush_interval_ms = 100
EOF
}
conf "$RUN/out-raw" true >"$CONF/raw.toml"
conf "$RUN/out-parsed" false >"$CONF/parsed.toml"
info "fixture $(short "$RUN/in/wide.jsonl"), configurations in $(short "$CONF")/"
conf "$RUN/out-raw" true > "$RUN/raw.toml"
conf "$RUN/out-parsed" false > "$RUN/parsed.toml"
section "Relay"
for c in raw parsed; do
timeout 5 "$BIN" -c "$RUN/$c.toml" > "$RUN/$c.out" 2>&1
info "lw -c $(short "$CONF/$c.toml"), 5 s"
timeout 5 "$BIN" -c "$CONF/$c.toml" >"$LOG/$c.out" 2>&1
done
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
section "Checks"
# 1. raw = true relays the file byte for byte
if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" > /dev/null; then
if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" >/dev/null; then
check "raw = true: output identical to input" 1
else
check "raw = true: output identical to input" 0
diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6
diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6 | sed 's/^/ /'
fi
# 2. the default parse keeps every key, JSON branch refused on the wide envelope
n=$(grep -c '"sub":"app"' "$RUN/out-parsed/relay.log")
check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $(( n == 1 ))
check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $((n == 1))
n=$(grep -c '1788917744178374836' "$RUN/out-parsed/relay.log")
check "defaults: large integers are not re-encoded through float64 ($n)" $(( n == 1 ))
check "defaults: large integers are not re-encoded through float64 ($n)" $((n == 1))
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
else
echo "RESULT: FAILURES — inspect $RUN/"
fi
exit "$fail"
summary
test/scram-chain-test.sh
+106 -209
View File
@@ -1,122 +1,46 @@
#!/usr/bin/env bash
# logwisp SCRAM authentication test: chain edges and sink viewers log in with
# passwords managed by `logwisp auth`; the guide printed below maps the ports.
# Usage: ./scram-chain-test.sh [--auto [--keep]] manual mode keeps the daemons
# up; --auto runs the checks and tears down, --keep skips that on success.
# logwisp SCRAM test: chain edges and sink viewers log in with passwords
# managed by `lw auth`: wrong and missing credentials, a certificate bound to
# its user, a stream past the exchange deadline, revocation and throttling.
# Requires: bash 5+, coreutils (timeout), openssl, curl. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run-scram"
CONF="$RUN/conf"
LOG="$RUN/log"
PKI="$RUN/pki"
AUTH="$RUN/auth"
OUT="$RUN/out"
USERS="$AUTH/users.toml"
RUN=$E2E_DIR/run-scram
PKI=$RUN/pki
AUTH=$RUN/auth
OUT=$RUN/out
USERS=$AUTH/users.toml
PORT_TCP_CHAIN=15821
PORT_HTTP_CHAIN=15822
PORT_TCP_SINK=15823
PORT_HTTP_SINK=15824
PORT_BOUND=15825
PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_BOUND"
e2e_init "$@"
AUTO=0; KEEP=0
for a in "$@"; do case "$a" in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
*) echo "unknown arg: $a" >&2; exit 1 ;;
esac; done
PIDS=()
cleanup() {
local rc=$?
trap - EXIT INT TERM
if (( ${#PIDS[@]} )); then
echo "--- teardown: stopping ${#PIDS[@]} process(es)"
kill -TERM "${PIDS[@]}" 2>/dev/null
local deadline=$(( SECONDS + 10 ))
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
kill -KILL "$pid" 2>/dev/null
done
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
wait_port() { # port timeout_s
local i; for (( i=0; i < $2 * 10; i++ )); do
port_open "$1" && return 0
sleep 0.1
done
return 1
}
wait_until() { # timeout_s command...
local deadline=$(( SECONDS + $1 )); shift
until "$@"; do
(( SECONDS >= deadline )) && return 1
sleep 0.2
done
}
start_daemon() { # name conf
"$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 &
PIDS+=($!)
echo "started $1 (pid $!)"
}
# --- Preflight ---
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
command -v openssl >/dev/null || { echo "openssl not found" >&2; exit 1; }
command -v curl >/dev/null || { echo "curl not found" >&2; exit 1; }
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_BOUND; do
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
done
section "Setup"
need openssl curl
ports_free $PORTS
rm -rf "$RUN"
mkdir -p "$CONF" "$LOG" "$PKI" "$AUTH" "$OUT"
# --- PKI ---
gen_key() { openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$1" 2>/dev/null; }
pki_ca
pki_leaf relay relay.internal serverAuth "IP:127.0.0.1"
pki_leaf edge-01 edge-01 clientAuth
pki_leaf edge-02 edge-02 clientAuth
info "test PKI in $(short "$PKI")/: ca, relay, edge-01, edge-02"
gen_leaf() { # name CN eku [SAN]
local name=$1 cn=$2 eku=$3 san=${4:-}
gen_key "$PKI/$name.key"
openssl req -new -key "$PKI/$name.key" -out "$PKI/$name.csr" -subj "/CN=$cn" 2>/dev/null
local ext="extendedKeyUsage=$eku"
[[ -n $san ]] && ext+=$'\n'"subjectAltName=$san"
printf '%s\n' "$ext" > "$PKI/$name.ext"
openssl x509 -req -in "$PKI/$name.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" \
-CAcreateserial -out "$PKI/$name.crt" -days 2 -extfile "$PKI/$name.ext" 2>/dev/null
}
# Credentials through the CLI only, and a password no user has, for the
# wrong-password and unknown-user edges
add_users "$USERS" edge-01 edge-02 viewer-01
(umask 077; openssl rand -base64 24 >"$AUTH/wrong.pass")
info "users edge-01, edge-02, viewer-01 in $(short "$USERS")"
echo "--- generating test PKI in $PKI"
gen_key "$PKI/ca.key"
openssl req -x509 -new -key "$PKI/ca.key" -days 2 -out "$PKI/ca.crt" \
-subj "/CN=LogWisp Test CA" 2>/dev/null
gen_leaf relay relay.internal serverAuth "IP:127.0.0.1"
gen_leaf edge-01 edge-01 clientAuth
gen_leaf edge-02 edge-02 clientAuth
[[ -s "$PKI/edge-02.crt" ]] || { echo "PKI generation failed" >&2; exit 1; }
# --- Credentials, through the CLI only ---
echo "--- adding users in $USERS"
for u in edge-01 edge-02 viewer-01; do
"$BIN" auth add-user -credentials "$USERS" -user "$u" -password-file "$AUTH/$u.pass" \
2>>"$LOG/auth-cli.out" || { echo "add-user $u failed (see $LOG/auth-cli.out)" >&2; exit 1; }
done
# A password no user has, for the wrong-password and unknown-user edges
(umask 077; openssl rand -base64 24 > "$AUTH/wrong.pass")
# --- Config generation ---
# One credentials file serves every listener, so revoking viewer-01 leaves the
# file with users (the CLI refuses to remove the last one).
cat > "$CONF/relay.toml" <<EOF
cat >"$CONF/relay.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
@@ -271,7 +195,7 @@ ca_file = "$PKI/ca.crt"
EOF
[[ -n $cert ]] && printf 'cert_file = "%s"\nkey_file = "%s"\n' "$PKI/$cert.crt" "$PKI/$cert.key"
[[ -n $user ]] && printf '[pipelines.plugin_sinks.config.auth]\ntype = "scram"\nusername = "%s"\npassword_file = "%s"\n' "$user" "$pass"
} > "$CONF/$name.toml"
} >"$CONF/$name.toml"
}
# edge_tcp declares node "edge-tcp": node_binding = "force" must relabel it edge-01.
@@ -284,66 +208,45 @@ edge_conf edge_unknown tcp_chain $PORT_TCP_CHAIN edge-unknown edge-99 "$AUTH/w
edge_conf edge_noauth tcp_chain $PORT_TCP_CHAIN edge-noauth "" ""
edge_conf edge_bound tcp_chain $PORT_BOUND edge-bound edge-01 "$AUTH/edge-01.pass" edge-01
edge_conf edge_cross tcp_chain $PORT_BOUND edge-cross edge-02 "$AUTH/edge-02.pass" edge-01
info "configurations in $(short "$CONF")/"
# --- Guide ---
cat <<EOF
================================================================
logwisp SCRAM auth test — port map
$PORT_TCP_CHAIN relay ingest (tcp_chain, scram, node_binding = force)
$PORT_HTTP_CHAIN relay ingest (http_chain, scram)
$PORT_TCP_SINK TCP sink (scram)
$PORT_HTTP_SINK HTTP sink (scram)
$PORT_BOUND relay ingest (tcp_chain, client_auth + scram, certificate CN = user)
Edges: edge_tcp and edge_http log in as edge-01 and deliver; edge_wrongpw,
edge_unknown and edge_noauth are refused on $PORT_TCP_CHAIN; on $PORT_BOUND
edge_bound (edge-01 cert, user edge-01) delivers and edge_cross (edge-01
cert, user edge-02) is refused; $PKI/edge-02.crt pairs with user edge-02.
Read the TCP sink as viewer-01:
$BIN auth stream -addr 127.0.0.1:$PORT_TCP_SINK -user viewer-01 \\
-password-file $AUTH/viewer-01.pass -ca-file $PKI/ca.crt
Read the HTTP sink (the token stays off argv):
token=\$($BIN auth token -url https://127.0.0.1:$PORT_HTTP_SINK -user viewer-01 \\
-password-file $AUTH/viewer-01.pass -ca-file $PKI/ca.crt)
curl -N --noproxy '*' --cacert $PKI/ca.crt \\
-H @<(printf 'Authorization: Bearer %s\n' "\$token") https://127.0.0.1:$PORT_HTTP_SINK/stream
Revoke: $BIN auth remove-user -credentials $USERS -user viewer-01
then kill -HUP the relay.
Ingested entries land in $OUT/ (node label forced to the username).
Passwords and credentials: $AUTH/ Logs: $LOG/
================================================================
EOF
# --- Startup ---
section "Startup"
start_daemon relay relay.toml
RELAY_PID=${PIDS[-1]}
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK $PORT_BOUND; do
wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; }
for p in $PORTS; do
wait_port "$p" || abort "relay port $p is not listening" relay
done
for e in edge_tcp edge_http edge_wrongpw edge_unknown edge_noauth edge_bound edge_cross; do
start_daemon "$e" "$e.toml"
done
daemons_up
write_env LW="$BIN" CA="$PKI/ca.crt" PW="$AUTH/viewer-01.pass" USERS="$USERS" RELAY="$RELAY_PID"
if (( AUTO == 0 )); then
echo "--- daemons running; Ctrl-C to stop"
while :; do sleep 1; done
fi
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
guide "logwisp SCRAM test" <<EOF
Ports:
$PORT_TCP_CHAIN relay tcp_chain ingest, scram, node_binding = force
$PORT_HTTP_CHAIN relay http_chain ingest, scram
$PORT_TCP_SINK tcp sink, scram
$PORT_HTTP_SINK http sink, scram
$PORT_BOUND relay tcp_chain ingest, client_auth + scram, certificate CN = user
Edges: edge_tcp and edge_http log in as edge-01 and deliver; edge_wrongpw,
edge_unknown and edge_noauth are refused on $PORT_TCP_CHAIN. On $PORT_BOUND edge_bound
(edge-01 certificate and user) delivers, edge_cross (user edge-02) is refused.
Shell setup (LW, CA, PW: viewer-01's password file, USERS, RELAY: its pid):
> . $(short "$RUN")/env
Read the tcp sink as viewer-01:
> \$LW auth stream -addr 127.0.0.1:$PORT_TCP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA
Read the http sink (bearer keeps the token off argv):
> token=\$(\$LW auth token -url https://127.0.0.1:$PORT_HTTP_SINK -user viewer-01 -password-file \$PW -ca-file \$CA)
> curl -N --cacert \$CA -H @<(bearer) https://127.0.0.1:$PORT_HTTP_SINK/stream
Revoke viewer-01:
> \$LW auth remove-user -credentials \$USERS -user viewer-01 && kill -HUP \$RELAY
Ingested entries (node label forced to the username): $(short "$OUT")/
Passwords: $(short "$AUTH")/ Logs: $(short "$LOG")/
EOF
manual_hold
relay_log="$LOG/relay.out"
ingested() { cat "$OUT"/${1}* 2>/dev/null; }
has_entry() { ingested "$1" | grep -q -- "$2"; } # file_prefix pattern
logged() { # log_name pattern [min_count]
local n; n=$(grep -c -- "$2" "$LOG/$1.out" 2>/dev/null)
(( ${n:-0} >= ${3:-1} ))
}
token_for() { # user -> token on stdout; the exit status is the CLI's
"$BIN" auth token -url "https://127.0.0.1:$PORT_HTTP_SINK" -user "$1" \
@@ -360,72 +263,73 @@ http_get() { # path [token] -> http_code; the header goes through a pipe, never
fi
}
echo "=== Scenario 1: chained instances over SCRAM ==="
section "Scenario 1: chained instances over SCRAM"
# 1. An edge with the right password delivers on both transports
wait_until 20 has_entry tcp_chain 'edge-01/'
tcp_file="$(ingested tcp_chain)"
n=$(grep -c 'edge-01/' <<< "$tcp_file")
check "tcp_chain: edge-01 entries reached the file sink ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-01/' <<<"$tcp_file")
check "tcp_chain: edge-01 entries reached the file sink ($n lines)" $((n >= 1))
wait_until 20 has_entry http_chain 'edge-01/'
http_file="$(ingested http_chain)"
n=$(grep -c 'edge-01/' <<< "$http_file")
check "http_chain: edge-01 entries reached the file sink ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-01/' <<<"$http_file")
check "http_chain: edge-01 entries reached the file sink ($n lines)" $((n >= 1))
# 2. node_binding = "force" replaced the label the sender configured
n=$(grep -c 'edge-tcp/' <<< "$tcp_file")
check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $(( n == 0 ))
n=$(grep -c 'edge-http/' <<< "$http_file")
check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $(( n == 0 ))
n=$(grep -c 'edge-tcp/' <<<"$tcp_file")
check "node binding: sender's own label \"edge-tcp\" was not honored ($n lines)" $((n == 0))
n=$(grep -c 'edge-http/' <<<"$http_file")
check "node binding: sender's own label \"edge-http\" was not honored ($n lines)" $((n == 0))
# 3. Wrong password, unknown user and no credentials are all refused
wait_until 20 logged edge_wrongpw 'Chain connect refused'
n=$(grep -c 'Chain connect refused' "$LOG/edge_wrongpw.out")
check "wrong password: edge-02 with a bad password was refused ($n refusals)" $(( n >= 1 ))
n=$(count edge_wrongpw 'Chain connect refused')
check "wrong password: edge-02 with a bad password was refused ($n refusals)" $((n >= 1))
wait_until 20 logged edge_unknown 'Chain connect refused'
n=$(grep -c 'Chain connect refused' "$LOG/edge_unknown.out")
check "unknown user: edge-99 was refused ($n refusals)" $(( n >= 1 ))
n=$(count edge_unknown 'Chain connect refused')
check "unknown user: edge-99 was refused ($n refusals)" $((n >= 1))
wait_until 20 logged relay 'peer offered no credentials'
n=$(grep -c 'peer offered no credentials' "$relay_log")
check "no auth: relay refused an edge that offered no credentials ($n refusals)" $(( n >= 1 ))
n=$(count relay 'peer offered no credentials')
check "no auth: relay refused an edge that offered no credentials ($n refusals)" $((n >= 1))
wait_until 20 logged edge_noauth 'Chain link refused'
n=$(grep -c 'Chain link refused' "$LOG/edge_noauth.out")
check "no auth: the edge read the refusal instead of writing into a closed link ($n)" $(( n >= 1 ))
n=$(count edge_noauth 'Chain link refused')
check "no auth: the edge read the refusal instead of writing into a closed link ($n)" $((n >= 1))
failed_login='Connection rejected by auth policy.*instance_id in_tcp .*invalid credentials'
wait_until 20 logged relay "$failed_login" 2
n=$(grep -c "$failed_login" "$relay_log")
check "relay logged the failed logins on :$PORT_TCP_CHAIN ($n rejections)" $(( n >= 2 ))
n=$(count relay "$failed_login")
check "relay logged the failed logins on :$PORT_TCP_CHAIN ($n rejections)" $((n >= 2))
tcp_file="$(ingested tcp_chain)"
n=$(grep -vc 'edge-01/' <<< "$tcp_file")
check "refused edges: nothing but edge-01 entries was ingested ($n foreign lines)" $(( n == 0 ))
n=$(grep -vc 'edge-01/' <<<"$tcp_file")
check "refused edges: nothing but edge-01 entries was ingested ($n foreign lines)" $((n == 0))
# 4. Certificate bound to the user: edge-01's certificate admits only edge-01
wait_until 20 has_entry bound 'edge-01/'
n=$(ingested bound | grep -c 'edge-01/')
check "certificate binding: edge-01 certificate + user edge-01 delivered ($n lines)" $(( n >= 1 ))
check "certificate binding: edge-01 certificate + user edge-01 delivered ($n lines)" $((n >= 1))
wait_until 20 logged edge_cross 'Chain connect refused'
n=$(grep -c 'Chain connect refused' "$LOG/edge_cross.out")
check "certificate binding: edge-01 certificate + user edge-02 was refused ($n refusals)" $(( n >= 1 ))
n=$(count edge_cross 'Chain connect refused')
check "certificate binding: edge-01 certificate + user edge-02 was refused ($n refusals)" $((n >= 1))
wait_until 20 logged relay 'does not match user \\"edge-02\\"'
n=$(grep -c 'does not match user \\"edge-02\\"' "$relay_log")
check "certificate binding: relay logged the certificate/user mismatch ($n)" $(( n >= 1 ))
n=$(count relay 'does not match user \\"edge-02\\"')
check "certificate binding: relay logged the certificate/user mismatch ($n)" $((n >= 1))
n=$(ingested bound | grep -vc 'edge-01/')
check "certificate binding: the cross-bound edge delivered nothing ($n foreign lines)" $(( n == 0 ))
check "certificate binding: the cross-bound edge delivered nothing ($n foreign lines)" $((n == 0))
echo "=== Scenario 2: viewers on SCRAM-gated sinks ==="
section "Scenario 2: viewers on SCRAM-gated sinks"
# 5. TCP sink: a logged-in viewer keeps streaming past the 10 s exchange
# deadline; a TLS client that sends no hello gets nothing and is dropped at it
info "streaming as viewer-01 beside a silent TLS client, 13 s"
"$BIN" auth stream -addr "127.0.0.1:$PORT_TCP_SINK" -user viewer-01 \
-password-file "$AUTH/viewer-01.pass" -ca-file "$PKI/ca.crt" \
> "$RUN/stream.out" 2>"$LOG/stream.err" &
>"$RUN/stream.out" 2>"$LOG/stream.err" &
stream_pid=$!
PIDS+=($stream_pid)
timeout 12 openssl s_client -quiet -connect "127.0.0.1:$PORT_TCP_SINK" \
@@ -437,47 +341,47 @@ sleep 2
n2=$(grep -c 'edge-01/' "$RUN/stream.out")
kill -TERM "$stream_pid" 2>/dev/null
wait "$stream_pid" "$raw_pid" 2>/dev/null
check "tcp sink: viewer-01 streamed entries through logwisp auth stream ($n1 lines at 11 s)" $(( n1 >= 1 ))
check "tcp sink: stream still live after the exchange deadline ($n1 -> $n2 lines, 11 s -> 13 s)" $(( n2 > n1 ))
n=$(wc -c < "$RUN/raw.out")
check "tcp sink: a TLS client with no hello received nothing ($n bytes)" $(( n == 0 ))
check "tcp sink: viewer-01 streamed entries through lw auth stream ($n1 lines at 11 s)" $((n1 >= 1))
check "tcp sink: stream still live after the exchange deadline ($n1 -> $n2 lines, 11 s -> 13 s)" $((n2 > n1))
n=$(wc -c <"$RUN/raw.out")
check "tcp sink: a TLS client with no hello received nothing ($n bytes)" $((n == 0))
n=$(grep 'component tcp_sink' "$relay_log" | grep -c 'read hello')
check "tcp sink: relay dropped the silent client at the exchange deadline ($n)" $(( n >= 1 ))
check "tcp sink: relay dropped the silent client at the exchange deadline ($n)" $((n >= 1))
# 6. HTTP sink: a bearer token from logwisp auth token, carried by curl
# 6. HTTP sink: a bearer token from lw auth token, carried by curl
token="$(token_for viewer-01)"
check "http sink: logwisp auth token issued a token for viewer-01" $([[ -n $token ]] && echo 1 || echo 0)
check "http sink: lw auth token issued a token for viewer-01" $(is_set "$token")
code="$(http_get /status "$token")"
check "http sink: /status served with the token (HTTP $code)" $([[ $code == 200 ]] && echo 1 || echo 0)
check "http sink: /status served with the token (HTTP $code)" $(is "$code" 200)
sse="$(timeout 4 curl -sN --noproxy '*' --cacert "$PKI/ca.crt" \
-H @<(printf 'Authorization: Bearer %s\n' "$token") \
"https://127.0.0.1:$PORT_HTTP_SINK/stream" 2>/dev/null || true)"
n=$(grep -c '^data:.*edge-01/' <<< "$sse")
check "http sink: /stream delivered SSE events with the token ($n events)" $(( n >= 1 ))
n=$(grep -c '^data:.*edge-01/' <<<"$sse")
check "http sink: /stream delivered SSE events with the token ($n events)" $((n >= 1))
code="$(http_get /status)"
check "http sink: /status refused without a token (HTTP $code)" $([[ $code == 401 ]] && echo 1 || echo 0)
check "http sink: /status refused without a token (HTTP $code)" $(is "$code" 401)
code="$(http_get /stream not.a.token)"
check "http sink: /stream refused with a garbage token (HTTP $code)" $([[ $code == 401 ]] && echo 1 || echo 0)
check "http sink: /stream refused with a garbage token (HTTP $code)" $(is "$code" 401)
# 7. Revocation: remove-user + SIGHUP kills the token and the login
reloads=$(grep -c 'Configuration hot reload completed successfully' "$relay_log")
reloaded() { (( $(grep -c 'Configuration hot reload completed successfully' "$relay_log") > reloads )); }
reloads=$(count relay 'Configuration hot reload completed successfully')
reloaded() { (($(count relay 'Configuration hot reload completed successfully') > reloads)); }
"$BIN" auth remove-user -credentials "$USERS" -user viewer-01 2>>"$LOG/auth-cli.out"
rc=$?
check "revocation: logwisp auth remove-user removed viewer-01 (exit $rc)" $(( rc == 0 ))
check "revocation: lw auth remove-user removed viewer-01 (exit $rc)" $((rc == 0))
kill -HUP "$RELAY_PID"
wait_until 20 reloaded && wait_port "$PORT_HTTP_SINK" 10
wait_until 20 reloaded && wait_port "$PORT_HTTP_SINK"
rc=$?
check "revocation: relay reloaded on SIGHUP" $(( rc == 0 ))
check "revocation: relay reloaded on SIGHUP" $((rc == 0))
code="$(http_get /status "$token")"
check "revocation: the old token is refused after the reload (HTTP $code)" $([[ $code == 401 ]] && echo 1 || echo 0)
check "revocation: the old token is refused after the reload (HTTP $code)" $(is "$code" 401)
token_for viewer-01 >/dev/null
rc=$?
check "revocation: logwisp auth token for viewer-01 now fails (exit $rc)" $(( rc == 1 ))
check "revocation: lw auth token for viewer-01 now fails (exit $rc)" $((rc == 1))
# 8. Throttling, last: it leaves this address throttled on the http sink.
# At most 4 exchanges may be unfinished per address, so hellos that are
@@ -492,13 +396,6 @@ for i in $(seq 1 12); do
[[ $code == 429 ]] && break
done
check "throttling: /auth answered 429 after ${#codes[@]} rapid hellos (${codes[*]})" \
$([[ $code == 429 ]] && (( ${#codes[@]} <= 6 )) && echo 1 || echo 0)
$([[ $code == 429 ]] && ((${#codes[@]} <= 6)) && echo 1 || echo 0)
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
else
echo "RESULT: FAILURES — inspect $LOG/*.out"
fi
exit "$fail"
summary
test/scram-proxy-test.sh
+229 -161
View File
@@ -1,95 +1,50 @@
#!/usr/bin/env bash
# logwisp browser login test: an http sink in proxy mode (auth.trusted_proxies)
# behind a TLS-terminating reverse proxy that mounts it under /logs/. Headless
# Chromium logs in through the shipped pages; the CLI logs in unbound.
# Usage: ./scram-proxy-test.sh [--auto [--keep]] manual mode keeps the daemons
# up; --auto runs the checks and tears down, --keep skips that on success.
# Requires: bash 5+, go, openssl, curl; node with playwright for the browser
# checks (skipped, and reported, without it). Linux dev host only.
# logwisp browser login test: an http sink in proxy mode behind a TLS-terminating
# reverse proxy that mounts it under /logs/; headless Chromium signs in through
# the shipped pages, with cookies and without (token mode); the CLI unbound.
# Requires: bash 5+, go, openssl, curl; node with playwright and both builds
# `playwright install chromium` fetches, or the browser checks skip.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run-proxy"
CONF="$RUN/conf"
LOG="$RUN/log"
PKI="$RUN/pki"
AUTH="$RUN/auth"
PROXY_SRC="$RUN/proxy"
USERS="$AUTH/users.toml"
RUN=$E2E_DIR/run-proxy
PKI=$RUN/pki
AUTH=$RUN/auth
PROXY_SRC=$RUN/proxy
USERS=$AUTH/users.toml
PORT_PROXY=15831 # the site: TLS ends here, logwisp is under /logs/
PORT_SINK=15832 # logwisp http sink, plaintext, only for 127.0.0.2
PROXY_ADDR=127.0.0.2
SITE="https://127.0.0.1:$PORT_PROXY"
e2e_init "$@"
AUTO=0; KEEP=0
for a in "$@"; do case "$a" in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
*) echo "unknown arg: $a" >&2; exit 1 ;;
esac; done
PIDS=()
cleanup() {
local rc=$?
trap - EXIT INT TERM
if (( ${#PIDS[@]} )); then
echo "--- teardown: stopping ${#PIDS[@]} process(es)"
kill -TERM "${PIDS[@]}" 2>/dev/null
local deadline=$(( SECONDS + 10 ))
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
kill -KILL "$pid" 2>/dev/null
done
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
wait_port() { # port timeout_s
local i; for (( i=0; i < $2 * 10; i++ )); do
port_open "$1" && return 0
sleep 0.1
done
return 1
}
# --- Preflight ---
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
for tool in go openssl curl; do
command -v "$tool" >/dev/null || { echo "$tool not found" >&2; exit 1; }
done
for p in $PORT_PROXY $PORT_SINK; do
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
done
BROWSER=0
command -v node >/dev/null && node -e 'require.resolve("playwright")' 2>/dev/null && BROWSER=1
section "Setup"
need go openssl curl
ports_free $PORT_PROXY $PORT_SINK
# The launch each scenario makes: the module resolves without its browsers, and
# chromium.executablePath() names full Chromium, not the default headless shell
launches() { timeout 60 node -e 'require("playwright").chromium.launch(JSON.parse(process.argv[1])).then((b) => b.close())' "$1" >/dev/null 2>&1; }
BROWSER=0 FULL_CHROMIUM=0
if command -v node >/dev/null; then
launches '{}' && BROWSER=1
launches '{"channel": "chromium"}' && FULL_CHROMIUM=1
fi
rm -rf "$RUN"
mkdir -p "$CONF" "$LOG" "$PKI" "$AUTH" "$PROXY_SRC"
# --- PKI: the site's certificate; logwisp holds none ---
echo "--- generating the site certificate in $PKI"
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$PKI/ca.key" 2>/dev/null
openssl req -x509 -new -key "$PKI/ca.key" -days 2 -out "$PKI/ca.crt" -subj "/CN=LogWisp Test CA" 2>/dev/null
openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$PKI/site.key" 2>/dev/null
openssl req -new -key "$PKI/site.key" -out "$PKI/site.csr" -subj "/CN=site.internal" 2>/dev/null
printf 'extendedKeyUsage=serverAuth\nsubjectAltName=IP:127.0.0.1\n' > "$PKI/site.ext"
openssl x509 -req -in "$PKI/site.csr" -CA "$PKI/ca.crt" -CAkey "$PKI/ca.key" -CAcreateserial \
-out "$PKI/site.crt" -days 2 -extfile "$PKI/site.ext" 2>/dev/null
[[ -s "$PKI/site.crt" ]] || { echo "PKI generation failed" >&2; exit 1; }
# The site's certificate; logwisp holds none
pki_ca
pki_leaf site site.internal serverAuth "IP:127.0.0.1"
info "site certificate in $(short "$PKI")/"
# --- Credentials, through the CLI only (the default 64 MiB profile) ---
"$BIN" auth add-user -credentials "$USERS" -user viewer-01 -password-file "$AUTH/viewer-01.pass" \
2>>"$LOG/auth-cli.out" || { echo "add-user failed (see $LOG/auth-cli.out)" >&2; exit 1; }
# The default 64 MiB Argon2 profile, as a deployment would have
add_users "$USERS" viewer-01
info "user viewer-01 in $(short "$USERS")"
# --- The site's reverse proxy ---
# /logs/ forwards to logwisp from $PROXY_ADDR; /plain/ forwards the same way
# but says the site was reached over http, which logwisp must refuse.
cat > "$PROXY_SRC/main.go" <<'EOF'
cat >"$PROXY_SRC/main.go" <<'EOF'
package main
import (
@@ -129,10 +84,13 @@ func main() {
log.Fatal(http.ListenAndServeTLS(listen, cert, key, mux))
}
EOF
(cd "$PROXY_SRC" && go build -o proxy main.go) || { echo "proxy build failed" >&2; exit 1; }
if ! (cd "$PROXY_SRC" && go build -o proxy main.go) >"$LOG/proxy-build.out" 2>&1; then
tail -n 5 "$LOG/proxy-build.out" | sed 's/^/ /'
abort "the reverse proxy did not build"
fi
info "reverse proxy built in $(short "$PROXY_SRC")/"
# --- Config ---
cat > "$CONF/logwisp.toml" <<EOF
cat >"$CONF/logwisp.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
@@ -162,35 +120,47 @@ credentials_file = "$USERS"
trusted_proxies = ["$PROXY_ADDR"]
EOF
cat > "$RUN/browser.cjs" <<'EOF'
// Logs in through the shipped pages, as a person would; prints key=value lines
cat >"$RUN/browser.cjs" <<'EOF'
// Signs in through the shipped pages, as a person would; prints key=value lines.
// cookie: a fresh browser. dropped: the session cookie never arrives. blocked:
// a profile that refuses every cookie; it also reloads logwisp (pid) mid-stream.
const { chromium } = require("playwright");
const [base, user, password] = process.argv.slice(2);
(async () => {
const out = { csp_violations: 0 };
const browser = await chromium.launch();
try {
const context = await browser.newContext({ ignoreHTTPSErrors: true });
const page = await context.newPage();
page.on("console", (m) => { if (/Content Security Policy/i.test(m.text())) out.csp_violations++; });
const status = () => page.evaluate(() => document.getElementById("status").textContent);
const fs = require("fs");
const [mode, base, user, passFile, profile, pid] = process.argv.slice(2);
const password = fs.readFileSync(passFile, "utf8").replace(/\n+$/, "");
const out = { csp_violations: 0 };
const lines = (page) => page.evaluate(() => document.querySelectorAll("#log > div").length);
const linesAtLeast = (page, n) =>
page.waitForFunction((n) => document.querySelectorAll("#log > div").length >= n, n, { timeout: 15000 });
const statusText = (page) => page.evaluate(() => document.getElementById("status").textContent);
const statusIs = (page, re) =>
page.waitForFunction((re) => new RegExp(re).test(document.getElementById("status").textContent), re.source, { timeout: 30000 });
const signIn = async (page, pass) => {
await page.fill('#login input[name="username"]', user);
await page.fill('#login input[name="password"]', pass);
await page.click('#login button[type="submit"]');
};
// The viewer's own form: the login page has a visible #login too, but no #log
const inlineLogin = (page) => page.waitForFunction(() =>
document.getElementById("log") && !document.getElementById("login").hidden, null, { timeout: 30000 });
const scenarios = {
async cookie(context, page) {
await page.goto(base + "auth/view");
await page.waitForURL(/\/auth\/login\?next=view$/, { timeout: 10000 });
await page.waitForURL(/\/auth\/login\?next=view%23signed-in$/, { timeout: 10000 });
out.redirected_to_login = 1;
await page.fill('input[name="username"]', user);
await page.fill('input[name="password"]', "not-the-password");
await page.click('button[type="submit"]');
await page.waitForFunction(() => /refused/.test(document.getElementById("status").textContent), null, { timeout: 30000 });
await signIn(page, "not-the-password");
await statusIs(page, /refused/);
out.wrong_password_refused = 1;
await page.fill('input[name="password"]', password);
const started = Date.now();
await page.click('button[type="submit"]');
await page.waitForURL(/\/auth\/view$/, { timeout: 30000 }).catch(async (e) => { throw new Error(`${e.message}: ${await status()}`); });
await signIn(page, password);
await page.waitForURL(/\/auth\/view$/, { timeout: 30000 }).catch(async (e) => { throw new Error(`${e.message}: ${await statusText(page)}`); });
out.login_ms = Date.now() - started;
await page.waitForFunction(() => document.querySelectorAll("#log > div").length >= 3, null, { timeout: 15000 });
out.events = await page.evaluate(() => document.querySelectorAll("#log > div").length);
await linesAtLeast(page, 3);
out.events = await lines(page);
out.inline_login = await page.evaluate(() => !document.getElementById("login").hidden);
const c = (await context.cookies()).find((c) => c.name === "logwisp_session");
out.cookie = c ? `path=${c.path};httponly=${c.httpOnly};secure=${c.secure};samesite=${c.sameSite}` : "none";
@@ -200,54 +170,120 @@ const [base, user, password] = process.argv.slice(2);
await page.waitForURL(/\/auth\/login$/, { timeout: 10000 });
out.cookie_after_logout = (await context.cookies()).some((c) => c.name === "logwisp_session") ? 1 : 0;
out.status_after_logout = await page.evaluate(async (b) => (await fetch(b + "status")).status, base);
},
// As if the browser dropped the cookie silently: the viewer must not send
// the person back to the login page in a loop
async dropped(context, page) {
await context.route(/\/auth$/, async (route) => {
const response = await route.fetch();
await context.clearCookies(); // route.fetch shares the context's cookie jar
const headers = response.headers();
delete headers["set-cookie"];
await route.fulfill({ response, headers });
});
await page.goto(base + "auth/view");
await page.waitForURL(/\/auth\/login\?next=/, { timeout: 10000 });
await signIn(page, password);
await page.waitForURL(/\/auth\/view$/, { timeout: 30000 });
await inlineLogin(page);
out.inline_login = 1;
const streamed = page.waitForRequest(/\/stream$/, { timeout: 30000 });
await signIn(page, password);
const token = ((await streamed).headers().authorization ?? "").replace(/^Bearer /, "");
await page.waitForFunction(() => document.getElementById("state").textContent === "live", null, { timeout: 30000 });
out.token_in_page_only = await page.evaluate((t) => t !== "" && localStorage.length === 0 &&
sessionStorage.length === 0 && !location.href.includes(t) && !document.cookie.includes(t), token) ? 1 : 0;
},
async blocked(context, page) {
out.proofs = out.proofs_with_session = 0;
page.on("request", (r) => {
const body = r.method() === "POST" && /\/auth$/.test(r.url()) ? JSON.parse(r.postData() ?? "{}") : {};
if (body.proof) out.proofs++;
if (body.proof && "session" in body) out.proofs_with_session++;
});
await page.goto(base + "auth/login");
out.cookie_enabled = await page.evaluate(() => navigator.cookieEnabled);
out.document_cookie_kept = JSON.stringify(await page.evaluate(() => { document.cookie = "t=1"; return document.cookie; }));
await page.waitForSelector('#status a[href="view"]', { timeout: 10000 });
out.login_page_disabled = await page.evaluate(() => document.querySelector("#login button").disabled);
await page.click('#status a[href="view"]');
await inlineLogin(page);
out.inline_login = 1;
const streamed = page.waitForRequest(/\/stream$/, { timeout: 30000 });
await signIn(page, password);
const bearer = (await streamed).headers().authorization ?? "";
out.stream_bearer = /^Bearer \S+$/.test(bearer) ? 1 : 0;
await linesAtLeast(page, 3);
out.events = await lines(page);
const statusWith = () => page.evaluate(async ([b, h]) =>
(await fetch(b + "status", { headers: { Authorization: h } })).status, [base, bearer]);
out.token_status = await statusWith();
await page.click("#logout");
await statusIs(page, /^Signed out\.$/);
await inlineLogin(page);
out.token_status_after_logout = await statusWith();
// A reload ends the open stream and every token; the reconnect's 401 asks again
await signIn(page, password);
await page.waitForFunction(() => document.getElementById("state").textContent === "live", null, { timeout: 30000 });
process.kill(Number(pid), "SIGHUP");
await statusIs(page, /session ended/);
await inlineLogin(page);
out.expired_asks_again = 1;
},
};
(async () => {
let context;
try {
if (mode === "blocked") {
// Full Chromium: the headless shell ignores a profile's Preferences
context = await chromium.launchPersistentContext(profile, { ignoreHTTPSErrors: true, channel: "chromium" });
} else {
context = await (await chromium.launch()).newContext({ ignoreHTTPSErrors: true });
}
const page = await context.newPage();
page.on("console", (m) => { if (/Content Security Policy/i.test(m.text())) out.csp_violations++; });
await scenarios[mode](context, page);
} catch (e) {
out.error = JSON.stringify(e.message.split("\n")[0]);
} finally {
await browser.close();
await (context?.browser() ?? context)?.close();
}
for (const [k, v] of Object.entries(out)) console.log(`${k}=${v}`);
})();
EOF
info "configuration in $(short "$CONF")/, browser driver $(short "$RUN/browser.cjs")"
# --- Guide ---
cat <<EOF
================================================================
logwisp browser login test — port map
section "Startup"
# logwisp stays PIDS[0]: the blocked-cookie scenario reloads it with SIGHUP
start_daemon logwisp logwisp.toml
spawn proxy "$PROXY_SRC/proxy" "127.0.0.1:$PORT_PROXY" "http://127.0.0.1:$PORT_SINK" "$PROXY_ADDR" \
"$PKI/site.crt" "$PKI/site.key"
wait_port $PORT_SINK || abort "logwisp port $PORT_SINK is not listening" logwisp
wait_port $PORT_PROXY || abort "proxy port $PORT_PROXY is not listening" proxy
daemons_up
write_env LW="$BIN" CA="$PKI/ca.crt" PW="$AUTH/viewer-01.pass" SITE="$SITE"
guide "logwisp browser login test" <<EOF
Ports:
$PORT_PROXY the site: TLS-terminating reverse proxy, logwisp under /logs/
$PORT_SINK logwisp http sink, plaintext, trusted_proxies = ["$PROXY_ADDR"]
Open $SITE/logs/auth/view (CA: $PKI/ca.crt) and sign in as viewer-01
with the password in $AUTH/viewer-01.pass.
CLI, unbound through the proxy (the token stays off argv):
token=\$($BIN auth token -unbound -url $SITE/logs -user viewer-01 \\
-password-file $AUTH/viewer-01.pass -ca-file $PKI/ca.crt)
curl -N --noproxy '*' --cacert $PKI/ca.crt \\
-H @<(printf 'Authorization: Bearer %s\n' "\$token") $SITE/logs/stream
Logs: $LOG/
================================================================
In a browser trusting $(short "$PKI/ca.crt"), sign in as viewer-01 with the
password in $(short "$AUTH/viewer-01.pass"):
> $SITE/logs/auth/view
Shell setup (LW, CA, PW: viewer-01's password file, SITE):
> . $(short "$RUN")/env
The CLI, unbound through the proxy (bearer keeps the token off argv):
> token=\$(\$LW auth token -unbound -url \$SITE/logs -user viewer-01 -password-file \$PW -ca-file \$CA)
> curl -N --cacert \$CA -H @<(bearer) \$SITE/logs/stream
Logs: $(short "$LOG")/
EOF
# --- Startup ---
"$BIN" -c "$CONF/logwisp.toml" > "$LOG/logwisp.out" 2>&1 &
PIDS+=($!)
"$PROXY_SRC/proxy" "127.0.0.1:$PORT_PROXY" "http://127.0.0.1:$PORT_SINK" "$PROXY_ADDR" \
"$PKI/site.crt" "$PKI/site.key" > "$LOG/proxy.out" 2>&1 &
PIDS+=($!)
for p in $PORT_SINK $PORT_PROXY; do
wait_port "$p" 10 || { echo "FAIL: port $p not listening (see $LOG/)"; exit 1; }
done
if (( AUTO == 0 )); then
echo "--- daemons running; Ctrl-C to stop"
while :; do sleep 1; done
fi
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
is() { [[ $1 == "$2" ]] && echo 1 || echo 0; } # actual expected -> 1 or 0
is_set() { [[ -n $1 ]] && echo 1 || echo 0; }
manual_hold
code_of() { # url [token] -> http_code; the header goes through a pipe, never argv
local args=(-s -o /dev/null -w '%{http_code}' --max-time 5 --noproxy '*' --cacert "$PKI/ca.crt")
@@ -258,48 +294,87 @@ code_of() { # url [token] -> http_code; the header goes through a pipe, never ar
fi
}
echo "=== Scenario 1: a person in a browser ==="
if (( BROWSER )); then
browser_out="$(timeout 120 node "$RUN/browser.cjs" "$SITE/logs/" viewer-01 "$(cat "$AUTH/viewer-01.pass")" 2>"$LOG/browser.err")"
echo "$browser_out" > "$LOG/browser.out"
val() { sed -n "s/^$1=//p" <<< "$browser_out"; }
[[ -n $(val error) ]] && echo "browser error: $(val error)"
browser() { # mode: runs browser.cjs, whose key=value lines val reads
info "browser: $1 (output in $(short "$LOG")/browser-$1.out)"
timeout 120 node "$RUN/browser.cjs" "$1" "$SITE/logs/" viewer-01 "$AUTH/viewer-01.pass" \
"$RUN/profile" "${PIDS[0]}" >"$LOG/browser-$1.out" 2>"$LOG/browser-$1.err"
browser_out="$(cat "$LOG/browser-$1.out")"
[[ -z $(val error) ]] || info "${C_RED}browser error ($1): $(val error)$C_OFF"
}
val() { sed -n "s/^$1=//p" <<<"$browser_out"; }
# The count covers the whole scenario only when it ran to its end without error
csp_clean() { echo $(($(is "$(val error)" "") && $(is "$(val csp_violations)" 0))); }
section "Scenario 1: a person in a browser"
if ((BROWSER)); then
browser cookie
check "browser: an unauthenticated viewer is sent to the login page" $(is "$(val redirected_to_login)" 1)
check "browser: a wrong password is refused on the page" $(is "$(val wrong_password_refused)" 1)
check "browser: login (Argon2 in the page, $(val login_ms) ms) opened the viewer" $(is_set "$(val login_ms)")
n=$(val events)
check "browser: the viewer showed events through EventSource (${n:-0} lines)" $(( ${n:-0} >= 3 ))
check "browser: the viewer showed events through EventSource (${n:-0} lines)" $((${n:-0} >= 3))
check "browser: with the cookie working the viewer shows no sign-in form" $(is "$(val inline_login)" false)
check "browser: session cookie $(val cookie)" \
$(is "$(val cookie)" "path=/logs;httponly=true;secure=true;samesite=Strict")
check "browser: page scripts cannot read the session (document.cookie $(val document_cookie))" \
$(is "$(val document_cookie)" '""')
check "browser: sign out cleared the cookie" $(is "$(val cookie_after_logout)" 0)
check "browser: status refused after sign out (HTTP $(val status_after_logout))" $(is "$(val status_after_logout)" 401)
check "browser: no CSP violations ($(val csp_violations))" $(is "$(val csp_violations)" 0)
check "browser: no CSP violations ($(val csp_violations))" $(csp_clean)
n=$(grep 'Login accepted' "$LOG/logwisp.out" | grep -c 'remote_addr 127.0.0.1')
check "logwisp logged the forwarded client, not the proxy ($n logins from 127.0.0.1)" $(( n >= 1 ))
check "logwisp logged the forwarded client, not the proxy ($n logins from 127.0.0.1)" $((n >= 1))
browser dropped
check "browser: a session cookie the browser dropped brings the viewer's own sign-in form, not a loop" \
$(is "$(val inline_login)" 1)
check "browser: its token stays out of storage, the URL and cookies" $(is "$(val token_in_page_only)" 1)
else
echo "SKIP: browser checks (node with playwright not found)"
skip "browser checks: node, playwright or its Chromium headless shell missing"
fi
echo "=== Scenario 2: the CLI through the proxy ==="
section "Scenario 2: a browser that keeps no cookies"
if ((FULL_CHROMIUM)); then
mkdir -p "$RUN/profile/Default"
echo '{"profile": {"default_content_setting_values": {"cookies": 2}}}' >"$RUN/profile/Default/Preferences"
browser blocked
check "no cookies: the profile blocks cookies (document.cookie kept $(val document_cookie_kept); navigator.cookieEnabled $(val cookie_enabled))" \
$(is "$(val document_cookie_kept)" '""')
check "no cookies: the login page says so, links to the viewer and stays disabled" \
$(is "$(val login_page_disabled)" true)
check "no cookies: the viewer shows its own sign-in form" $(is "$(val inline_login)" 1)
n=$(val events)
check "no cookies: the viewer streamed events with a bearer token (${n:-0} lines)" \
$((${n:-0} >= 3 && $(is "$(val stream_bearer)" 1)))
n=$(val proofs)
check "no cookies: the viewer's proofs asked for no cookie (${n:-0} sent, $(val proofs_with_session) with a session)" \
$((${n:-0} >= 1 && $(is "$(val proofs_with_session)" 0)))
check "no cookies: the page's token opened status (HTTP $(val token_status))" $(is "$(val token_status)" 200)
check "no cookies: sign out revoked it (HTTP $(val token_status_after_logout))" \
$(is "$(val token_status_after_logout)" 401)
check "no cookies: a reload ended the session and the viewer asked to sign in again" \
$(is "$(val expired_asks_again)" 1)
check "no cookies: no CSP violations ($(val csp_violations))" $(csp_clean)
else
skip "browser checks: node, playwright or its full Chromium missing"
fi
section "Scenario 3: the CLI through the proxy"
token="$("$BIN" auth token -unbound -url "$SITE/logs" -user viewer-01 -password-file "$AUTH/viewer-01.pass" \
-ca-file "$PKI/ca.crt" 2>>"$LOG/auth-cli.out")"
check "cli: logwisp auth token -unbound logged in through the proxy" $(is_set "$token")
check "cli: lw auth token -unbound logged in through the proxy" $(is_set "$token")
code="$(code_of "$SITE/logs/status" "$token")"
check "cli: /logs/status served with the token (HTTP $code)" $(is "$code" 200)
sse="$(timeout 3 curl -sN --noproxy '*' --cacert "$PKI/ca.crt" \
-H @<(printf 'Authorization: Bearer %s\n' "$token") "$SITE/logs/stream" 2>/dev/null || true)"
n=$(grep -c '^data:' <<< "$sse")
check "cli: /logs/stream delivered SSE events with the token ($n events)" $(( n >= 1 ))
n=$(grep -c '^data:' <<<"$sse")
check "cli: /logs/stream delivered SSE events with the token ($n events)" $((n >= 1))
code="$(code_of "$SITE/logs/status")"
check "cli: /logs/status refused without a token (HTTP $code)" $(is "$code" 401)
"$BIN" auth token -url "$SITE/logs" -user viewer-01 -password-file "$AUTH/viewer-01.pass" \
-ca-file "$PKI/ca.crt" >/dev/null 2>>"$LOG/auth-cli.out"
rc=$?
check "cli: a path without -unbound is a usage error (exit $rc)" $(( rc == 2 ))
check "cli: a path without -unbound is a usage error (exit $rc)" $((rc == 2))
echo "=== Scenario 3: around the proxy ==="
section "Scenario 4: around the proxy"
code="$(code_of "http://127.0.0.1:$PORT_SINK/auth/login")"
check "direct: the login page refuses a peer that is not the proxy (HTTP $code)" $(is "$code" 403)
code="$(code_of "http://127.0.0.1:$PORT_SINK/status" "$token")"
@@ -307,11 +382,4 @@ check "direct: a valid token does not open the sink around the proxy (HTTP $code
code="$(code_of "$SITE/plain/status" "$token")"
check "plaintext site: a request the proxy received over http is refused (HTTP $code)" $(is "$code" 403)
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
else
echo "RESULT: FAILURES — inspect $LOG/*"
fi
exit "$fail"
summary