From a1e9d21d06efaf29785f759924963793523e3a4d18a1c5c2a18d8b1a960e65fb Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sat, 10 Oct 2026 18:28:01 -0400 Subject: [PATCH] v0.32.0 lwconf engine update: config by path, schema from code, wasm --- AGENTS.md | 7 +- Makefile | 33 +++-- cmd/lw/bootstrap.go | 2 +- cmd/lw/cli.go | 2 +- cmd/lw/main.go | 7 +- cmd/lwconf/lwconf.js | 91 +++++++++++++ cmd/lwconf/lwconf.test.mjs | 204 +++++++++++++++++++++++------- cmd/lwconf/main.go | 200 ++++++++++++++++++++++++++--- deploy/package/completion/_lw | 14 +- deploy/package/completion/lw.fish | 14 +- doc/cli.md | 31 ++++- doc/installation.md | 9 +- doc/lw.1 | 15 ++- doc/operations.md | 15 ++- doc/todo.md | 68 +++++++--- go.mod | 2 +- go.sum | 2 + internal/authz/authz_test.go | 2 +- internal/compose/compose.go | 178 +++++++++++++++++++++----- internal/compose/compose_test.go | 90 +++++++++++-- internal/config/config.go | 6 +- internal/config/loader.go | 14 +- internal/config/preset.go | 189 ++++++++++++++++++--------- internal/config/preset_test.go | 57 ++++++++- internal/config/schema.go | 51 ++++++-- internal/config/schema_test.go | 61 ++++++++- internal/config/spec.go | 26 +++- internal/config/validate.go | 186 +++++++++++++++------------ internal/tui/canvas.go | 43 ++----- internal/tui/dialog.go | 6 +- internal/tui/inspector.go | 11 +- internal/tui/tui.go | 34 ++--- internal/tui/tui_test.go | 29 ++++- 33 files changed, 1278 insertions(+), 421 deletions(-) create mode 100644 cmd/lwconf/lwconf.js diff --git a/AGENTS.md b/AGENTS.md index 7f4946d..0812e03 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,10 +80,11 @@ change touches. refuses unknown keys at any depth; they keep only I/O. Rules that read nothing belong in a `Check` method, so `ValidateConfig` sees them offline. - `internal/compose` is the engine `lw --tui` and the website's WebAssembly - share: it edits pipelines and writes the command line, environment and file - forms, each loading back to the same pipelines; it reads no file and links + share: it edits pipelines, writes the command line, environment and file + forms and reads each back to the same pipelines; it reads no file and links no TLS, HTTP or terminal code (its test runs `go list -deps`). `cmd/lwconf` - hands it to the page as WebAssembly (`make wasm`; `node --test` there). + hands it to a page: `make wasm` writes the drop-in `bin/lwconf/`, which + `lwconf.js` loads and calls (`node --test` there). - `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands table whose groups (`auth`, `tls`, `preset`, `config`) share one flag mechanism; `config.Load` takes the parsed `config.Args`. Presets are rows of diff --git a/Makefile b/Makefile index 3f8f8ab..5a0703c 100644 --- a/Makefile +++ b/Makefile @@ -15,12 +15,13 @@ GIT_COMMIT ?= $(GIT_COMMIT_DEFAULT) BUILD_TIME_DEFAULT != f='+%Y-%m-%d_%H:%M:%S'; e="$${SOURCE_DATE_EPOCH:-}"; \ if [ -n "$$e" ]; then date -u -d "@$$e" "$$f" 2>/dev/null || date -u -r "$$e" "$$f"; else date -u "$$f"; fi BUILD_TIME ?= $(BUILD_TIME_DEFAULT) -# The image stamps the commit time, so a rebuild of one commit stays cached +# The image and the wasm stamp the commit time, so a rebuild of one commit +# stays cached, and the website's engine keeps its checksums IMAGE_BUILD_TIME_DEFAULT != TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%d_%H:%M:%S 2>/dev/null || echo unknown IMAGE_BUILD_TIME ?= $(IMAGE_BUILD_TIME_DEFAULT) -VERSION_LDFLAGS = -X 'github.com/lixenwraith/logwisp/internal/version.Version=$(VERSION)' \ - -X 'github.com/lixenwraith/logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \ - -X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(BUILD_TIME)' +COMMIT_LDFLAGS = -X 'github.com/lixenwraith/logwisp/internal/version.Version=$(VERSION)' \ + -X 'github.com/lixenwraith/logwisp/internal/version.GitCommit=$(GIT_COMMIT)' +VERSION_LDFLAGS = $(COMMIT_LDFLAGS) -X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(BUILD_TIME)' # Not LDFLAGS/GOFLAGS: packagers export those for the C linker and go itself. GO_BUILDFLAGS ?= GO_LDFLAGS ?= @@ -78,8 +79,8 @@ help: @echo " build Build $(BIN_DIR)/$(BINARY) with version metadata" @echo " release Build it static, stripped and trimmed (CGO_ENABLED=0)" @echo " dev Build it with the race detector" - @echo " wasm Build $(BIN_DIR)/lwconf.wasm, the engine for the website, beside its" - @echo " loader wasm_exec.js, and print both checksums" + @echo " wasm Write $(WASM_DIR)/, the engine a website drops in: lwconf.wasm, its" + @echo " loader wasm_exec.js, lwconf.js, schema.json, and lwconf.json, their manifest" @echo " version Print the version metadata a build embeds" @echo " completion Regenerate $(PKG_DIR)/completion from lw's command tables" @echo " clean Remove $(BIN_DIR)/" @@ -123,11 +124,21 @@ dev: check-go $(GO) build -race $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC) # The page loads the toolchain's own loader, which uses no eval: its CSP needs -# only 'wasm-unsafe-eval' +# only 'wasm-unsafe-eval'. schema.json is lw --schema stamped as the engine is, +# so it equals the engine's schema(); lwconf.json records the build. +WASM_DIR = $(BIN_DIR)/lwconf +WASM_FILES = lwconf.wasm wasm_exec.js lwconf.js schema.json +WASM_LDFLAGS = -s -w $(COMMIT_LDFLAGS) -X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(IMAGE_BUILD_TIME)' $(GO_LDFLAGS) wasm: check-go - GOOS=js GOARCH=wasm $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "-s -w $(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/lwconf.wasm ./cmd/lwconf - install -m 0644 "$$($(GO) env GOROOT)/lib/wasm/wasm_exec.js" $(BIN_DIR)/wasm_exec.js - @cd $(BIN_DIR) && { sha256sum lwconf.wasm wasm_exec.js 2>/dev/null || sha256 -r lwconf.wasm wasm_exec.js; } + @rm -rf $(WASM_DIR) && mkdir -p $(WASM_DIR) + GOOS=js GOARCH=wasm $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "$(WASM_LDFLAGS)" -o $(WASM_DIR)/lwconf.wasm ./cmd/lwconf + install -m 0644 "$$($(GO) env GOROOT)/lib/wasm/wasm_exec.js" cmd/lwconf/lwconf.js $(WASM_DIR) + $(GO) run -trimpath $(GO_BUILDFLAGS) -ldflags "$(WASM_LDFLAGS)" $(SRC) --schema > $(WASM_DIR)/schema.json + @set -eu; go=$$($(GO) env GOVERSION); cd $(WASM_DIR); \ + sums=$$(sha256sum $(WASM_FILES) 2>/dev/null || sha256 -r $(WASM_FILES)); echo "$$sums"; \ + { printf '{\n "version": "%s",\n "commit": "%s",\n "go": "%s",\n "sha256": {' '$(VERSION)' '$(GIT_COMMIT)' "$$go"; \ + echo "$$sums" | awk '{ sub(/^\*/, "", $$2); printf "%s\n \"%s\": \"%s\"", (NR > 1 ? "," : ""), $$2, $$1 }'; \ + printf '\n }\n}\n'; } > lwconf.json test: check-go $(GO) test ./... @@ -157,7 +168,7 @@ verify: test done @echo "cross build js/wasm"; GOOS=js GOARCH=wasm $(GO) build -o /dev/null ./cmd/lwconf @if command -v node >/dev/null 2>&1; then \ - (cd $(WEB_DIR) && node --test) && cd cmd/lwconf && node --test; \ + (cd $(WEB_DIR) && node --test) && cd cmd/lwconf && GO='$(GO)' node --test; \ else \ echo "node not found: web client and lwconf tests skipped"; \ fi diff --git a/cmd/lw/bootstrap.go b/cmd/lw/bootstrap.go index 27abf92..9edeb0b 100644 --- a/cmd/lw/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -62,7 +62,7 @@ func handleReload(ctx context.Context, newCfg *config.Config, oldSvc *service.Se logger.Info("msg", "Starting configuration hot reload") if err := config.ValidateConfig(newCfg); err != nil { - logger.Error("msg", "Invalid reload configuration, keeping old service running", "error", err) + logger.Error("msg", "Invalid reload configuration, keeping old service running", "error", oneLine(err)) return nil, nil, nil, err } diff --git a/cmd/lw/cli.go b/cmd/lw/cli.go index 0e5c3b8..95b416b 100644 --- a/cmd/lw/cli.go +++ b/cmd/lw/cli.go @@ -291,7 +291,7 @@ func presetCommands() []subcommand { s.define = func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { values := map[string]*string{} for _, k := range p.Params { - values[k.Name] = fs.String(k.Name, k.Default, k.Help) + values[k.Name] = fs.String(k.Name, k.Default, k.Usage()) } return func(stdout, _ io.Writer) error { set := map[string]string{} diff --git a/cmd/lw/main.go b/cmd/lw/main.go index c8737eb..0f38a5a 100644 --- a/cmd/lw/main.go +++ b/cmd/lw/main.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "reflect" + "strings" "syscall" "github.com/lixenwraith/logwisp/internal/compose" @@ -147,7 +148,7 @@ func main() { next, err = manager.Snapshot() } if err != nil { - logger.Error("msg", "Failed to read reload configuration", "error", err, "action", "keeping current service") + logger.Error("msg", "Failed to read reload configuration", "error", oneLine(err), "action", "keeping current service") return } // Watch events are hints and may be duplicated or concern startup-only @@ -219,6 +220,10 @@ func readsStdin(cfg *config.Config) bool { return false } +// oneLine is an error for a log record, which is a line: the problems +// validation joins, "; " between them +func oneLine(err error) string { return strings.ReplaceAll(err.Error(), "\n", "; ") } + // loadFailed reports a configuration that did not load and returns lw's exit // code: a named file that is missing is a usage error func loadFailed(err error) int { diff --git a/cmd/lwconf/lwconf.js b/cmd/lwconf/lwconf.js new file mode 100644 index 0000000..32f8574 --- /dev/null +++ b/cmd/lwconf/lwconf.js @@ -0,0 +1,91 @@ +// LogWisp's configuration engine for a page: one dependency-free ES module +// beside lwconf.wasm, which make wasm writes with it. It knows the engine's +// calls and the shapes of its schema and pipelines, never a plugin, a key or +// a rule, so the page draws whatever the engine it loads describes. + +// load starts the engine: the toolchain's wasm_exec.js by a script element +// (outside a page the caller imports it first), the module fetched with its +// integrity, then its main, which sets globalThis.lwconf +export async function load({ execURL, execSRI, wasmURL, wasmSRI }) { + if (!globalThis.Go) await script(execURL, execSRI); + const go = new globalThis.Go(); + const fetched = async () => { + const res = await fetch(wasmURL, { integrity: wasmSRI }); + if (!res.ok) throw new Error(`${wasmURL}: ${res.status}`); + return res; + }; + const { instance } = await WebAssembly.instantiateStreaming(fetched(), go.importObject) + .catch(async () => WebAssembly.instantiate(await (await fetched()).arrayBuffer(), go.importObject)); // a server's wrong type + go.run(instance); + if (!globalThis.lwconf) throw new Error("lwconf.wasm did not start"); + return new Engine(globalThis.lwconf); +} + +function script(src, integrity) { + return new Promise((resolve, reject) => { + const s = Object.assign(document.createElement("script"), { src, onload: resolve }); + s.onerror = () => reject(new Error(`${src} did not load`)); + if (integrity) s.integrity = integrity; + document.head.append(s); + }); +} + +// Engine's calls take objects or JSON text and return the engine's answer, +// {value} or {error, path?, pipeline?, node?, key?}; dirs answers whether a +// preset path is a directory, {"/var/log/app.log": false} +export class Engine { + #api; + constructor(api) { this.#api = api; } + #call(name, ...args) { + return JSON.parse(this.#api[name](...args.map(a => (typeof a === "string" ? a : JSON.stringify(a))))); + } + schema() { return this.#call("schema"); } + preset(name, values, dirs = {}) { return this.#call("preset", name, values, dirs); } + parse(text, form, dirs = {}) { return this.#call("parse", text, form, dirs); } + validate(pipelines) { return this.#call("validate", pipelines); } + emit(pipelines, form) { return this.#call("emit", pipelines, form); } + edit(pipelines, op) { return this.#call("edit", pipelines, op); } +} + +// fields are the rows a form draws for a node, by its role (and a source's +// or sink's type): a value row a key, with what leaving it unset means, and a +// group a table, holding its own rows; paths are dotted as edit takes them +export function fields(schema, role, type) { + const plugin = schema[`${role}s`]?.find(p => p.type === type); + const rows = (keys, prefix) => keys.map(key => { + const path = prefix + key.key; + if (Array.isArray(key.keys)) return { path, key, rows: rows(key.keys, path + ".") }; + return { path, key, placeholder: key.zero ? `(${key.zero})` : (key.default ?? key.hint ?? "") }; + }); + return rows(plugin?.keys ?? schema.flow.find(s => s.key === role)?.keys ?? [], ""); +} + +// value is a node's value at a dotted path, undefined while unset; a node is +// {role, id, index} as the engine returns it, a list stage's by its index +export function value(pipelines, pipeline, node, path = "") { + const p = pipelines[pipeline]; + const list = { source: p?.plugin_sources, sink: p?.plugin_sinks }[node.role]; + const stage = p?.flow?.[node.role]; + let v = list ? list.find(s => s.id === node.id)?.config : Array.isArray(stage) ? stage[node.index] : stage; + for (const part of path ? path.split(".") : []) v = v?.[part]; + return v; +} + +// at names where problems lie: a pipeline's node, however value takes it, the +// pipeline, or nowhere +export function at(pipeline, node) { + if (pipeline === undefined) return ""; + return node ? `${pipeline}/${node.role}/${node.id ?? ""}/${node.index ?? 0}` : `${pipeline}`; +} + +// group sorts validate's problems by at(pipeline, node), then by key, "" for +// the node or pipeline itself +export function group(problems) { + const out = new Map(); + for (const p of problems) { + const keys = out.get(at(p.pipeline, p.node)) ?? new Map(); + keys.set(p.key ?? "", [...(keys.get(p.key ?? "") ?? []), p]); + out.set(at(p.pipeline, p.node), keys); + } + return out; +} diff --git a/cmd/lwconf/lwconf.test.mjs b/cmd/lwconf/lwconf.test.mjs index f50e6d5..a6c64da 100644 --- a/cmd/lwconf/lwconf.test.mjs +++ b/cmd/lwconf/lwconf.test.mjs @@ -2,63 +2,181 @@ import { after, test } from "node:test"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { pathToFileURL } from "node:url"; -// Built by make wasm and started as the page starts it: the toolchain's loader -// beside the module, main running until the page closes. The make run is its -// own: a parent GNU make's flags would stop a BSD one. -const dir = mkdtempSync(join(tmpdir(), "lwconf-")); -after(() => rmSync(dir, { recursive: true, force: true })); -const made = execFileSync("make", ["-s", "wasm", `BIN_DIR=${dir}`, "VERSION=v0.0.0-node"], { - cwd: join(import.meta.dirname, "../.."), encoding: "utf8", - env: { ...process.env, MAKEFLAGS: "", MFLAGS: "", MAKELEVEL: "" }, +// The drop-in directory make wasm writes over an earlier one's, served as a +// site serves it (?type= another content type) and loaded through lwconf.js; +// wasm_exec.js is imported first, as a page's script element runs it. make runs +// on its own, with make verify's toolchain: a GNU make's flags stop a BSD one. +const root = join(import.meta.dirname, "../.."); +const tmp = mkdtempSync(join(tmpdir(), "lwconf-")); +const dir = join(tmp, "lwconf"); +const go = process.env.GO ?? "go"; +const env = { ...process.env, MAKEFLAGS: "", MFLAGS: "", MAKELEVEL: "" }; +const wasm = (bin, ...vars) => execFileSync("make", ["-s", "wasm", `BIN_DIR=${bin}`, "VERSION=v0.0.0-node", `GO=${go}`, ...vars], + { cwd: root, env, stdio: "ignore" }); +mkdirSync(dir); +writeFileSync(join(dir, "stale.js"), ""); +wasm(tmp); +const server = createServer((req, res) => { + const { pathname, searchParams } = new URL(req.url, "http://localhost"); + const file = join(dir, basename(pathname)); + if (!existsSync(file)) return res.writeHead(404).end(); + res.setHeader("content-type", searchParams.get("type") ?? (file.endsWith(".wasm") ? "application/wasm" : "text/javascript")); + res.end(readFileSync(file)); }); +await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); +after(() => { server.close(); rmSync(tmp, { recursive: true, force: true }); }); +const url = file => `http://127.0.0.1:${server.address().port}/${file}`; +const sri = file => "sha256-" + createHash("sha256").update(readFileSync(join(dir, file))).digest("base64"); + +const { load, fields, value, at, group } = await import(pathToFileURL(join(dir, "lwconf.js"))); await import(pathToFileURL(join(dir, "wasm_exec.js"))); -const go = new globalThis.Go(); -const { instance } = await WebAssembly.instantiate(readFileSync(join(dir, "lwconf.wasm")), go.importObject); -go.run(instance); -const call = (name, ...args) => JSON.parse(globalThis.lwconf[name](...args)); +const engine = await load({ wasmURL: url("lwconf.wasm"), wasmSRI: sri("lwconf.wasm") }); +const schema = engine.schema().value; -test("make wasm stamps the version and prints both checksums", () => { - for (const file of ["lwconf.wasm", "wasm_exec.js"]) { - const sum = createHash("sha256").update(readFileSync(join(dir, file))).digest("hex"); - assert.match(made, new RegExp(`^${sum} +${file.replace(".", "\\.")}$`, "m")); +test("make wasm writes one directory, which its manifest describes", () => { + const manifest = JSON.parse(readFileSync(join(dir, "lwconf.json"), "utf8")); + const files = readdirSync(dir).filter(f => f !== "lwconf.json").sort(); + assert.deepEqual(Object.keys(manifest.sha256).sort(), files); + for (const file of files) { + assert.equal(manifest.sha256[file], createHash("sha256").update(readFileSync(join(dir, file))).digest("hex"), file); } - assert.equal(call("schema").value.version, "v0.0.0-node"); + const version = execFileSync(go, ["env", "GOVERSION"], { cwd: root, encoding: "utf8" }).trim(); + assert.deepEqual([manifest.version, schema.version, manifest.go], ["v0.0.0-node", "v0.0.0-node", version]); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "schema.json"), "utf8")), schema); }); -// Off the host a directory path ends in '/' -test("a preset's pipelines, written as a command line, parse and validate back to themselves", () => { - const { value: pipelines } = call("preset", "tail", JSON.stringify({ path: "/var/log/" })); - assert.equal(pipelines[0].plugin_sources[0].config.directory, "/var/log/"); - const { value: line } = call("emit", JSON.stringify(pipelines), "command"); - assert.match(line, /^lw \\\n {2}--pipeline tail/); - assert.deepEqual(call("parse", line).value, pipelines); - assert.deepEqual(call("validate", JSON.stringify(pipelines)).value, pipelines); - assert.match(call("emit", JSON.stringify(pipelines), "file").value, /^\[\[pipelines\]\]/m); +test("make wasm writes a commit's files the same at any time", () => { + wasm(join(tmp, "again"), "BUILD_TIME=another"); + assert.equal(readFileSync(join(tmp, "again/lwconf/lwconf.json"), "utf8"), readFileSync(join(dir, "lwconf.json"), "utf8")); }); -test("a refused call answers why, and the engine keeps answering", () => { +test("load refuses an engine its integrity does not match, or a server does not have", async () => { + await assert.rejects(load({ wasmURL: url("lwconf.wasm"), wasmSRI: sri("schema.json") })); + await assert.rejects(load({ wasmURL: url("missing.wasm") }), { message: `${url("missing.wasm")}: 404` }); +}); + +test("load takes an engine a server sends as plain bytes", async () => { + const api = globalThis.lwconf; + delete globalThis.lwconf; + try { + const bytes = await load({ wasmURL: url("lwconf.wasm?type=application/octet-stream"), wasmSRI: sri("lwconf.wasm") }); + assert.deepEqual(bytes.schema().value, schema); + } finally { + globalThis.lwconf = api; + } +}); + +// The environment holds one pipeline, which a preset makes +test("a preset's pipeline, a single file answered by dirs, reads back from every form", () => { + const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false }; + assert.deepEqual(engine.preset("tail", path), { error: "path: a directory or a file pattern?", path: "path" }); + const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs); + assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" }); + assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]); + assert.deepEqual(engine.parse("lw -p serve:path=/var/log/app.log,allow=192.0.2.0/24,allow=198.51.100.7", "command", dirs).value, pipelines); + for (const { name } of schema.forms) { + assert.deepEqual(engine.parse(engine.emit(pipelines, name).value, name).value, pipelines, name); + } + assert.deepEqual(engine.validate(pipelines).value, { pipelines, problems: [] }); +}); + +// A value of the wrong kind is a problem as any other is, and the parts after +// it are still read +test("validate places every problem at its pipeline, node and key", () => { + const { value: { problems } } = engine.validate([ + { name: "a", flow: { rate_limit: { policy: "bogus" }, filters: [{ patterns: ["("] }] }, plugin_sources: [{ type: "file" }], + plugin_sinks: [{ id: "out", type: "tcp", config: { port: 9000, tls: { enabled: true } } }, { type: "http", config: { port: "x" } }] }, + { name: "a", plugin_sinks: [{ type: "tcp" }] }, + ]); + const node = (role, id, index = 0) => ({ role, id, index }); + assert.deepEqual(problems.map(({ path, pipeline, node, key }) => ({ path, pipeline, node, key })), [ + { path: "pipelines[0].flow.rate_limit.policy", pipeline: 0, node: node("rate_limit", ""), key: "policy" }, + { path: "pipelines[0].flow.filters[0].patterns[0]", pipeline: 0, node: node("filters", "", 0), key: "patterns" }, + { path: "pipelines[0].plugin_sources[file].config.directory", pipeline: 0, node: node("source", "file"), key: "directory" }, + { path: "pipelines[0].plugin_sinks[out].config.tls", pipeline: 0, node: node("sink", "out"), key: "tls" }, + { path: "pipelines[0].plugin_sinks[http].config.port", pipeline: 0, node: node("sink", "http"), key: "port" }, + { path: "pipelines[1].name", pipeline: 1, node: undefined, key: "name" }, + { path: "pipelines[1].plugin_sources", pipeline: 1, node: undefined, key: "plugin_sources" }, + { path: "pipelines[1].plugin_sinks[tcp].config.port", pipeline: 1, node: node("sink", "tcp"), key: "port" }, + ]); + assert.match(problems[3].text, /^listeners need cert_file/); + assert.equal(problems[4].text, "not an integer"); + const placed = group(problems); + assert.deepEqual(placed.get(at(0, { role: "sink", id: "out" })).get("tls"), [problems[3]]); + assert.deepEqual(placed.get(at(0, { role: "rate_limit" })).get("policy"), [problems[0]]); + assert.deepEqual([...placed.get(at(1)).keys()], ["name", "plugin_sources"]); +}); + +test("edits keep the engine's rules: free ids, retype's carry-over, a stage turned on", () => { + let pipelines = [], node; + const edit = op => { + const r = engine.edit(pipelines, { pipeline: 0, ...op }); + assert.equal(r.error, undefined, JSON.stringify(op)); + ({ pipelines, node } = r.value); + return node; + }; + edit({ op: "add_pipeline", name: "p" }); + edit({ op: "add", role: "source", type: "null" }); + assert.deepEqual(edit({ op: "add", role: "source", type: "null" }), { role: "source", id: "null_2", index: 0 }); + const tcp = edit({ op: "add", role: "sink", type: "tcp" }); + edit({ op: "set", node: tcp, key: "port", values: ["9000"] }); + edit({ op: "set", node: tcp, key: "tls.enabled", values: ["true"] }); + assert.deepEqual(edit({ op: "retype", node: tcp, type: "http" }), { role: "sink", id: "http", index: 0 }); + assert.deepEqual(value(pipelines, 0, node), { port: 9000, tls: { enabled: true } }); + edit({ op: "unset", node, key: "tls.enabled" }); + edit({ op: "set", node: { role: "rate_limit" }, key: "entries_per_second", values: ["5"] }); + assert.deepEqual(value(pipelines, 0, { role: "rate_limit" }, "policy"), schema.flow[0].on.policy); + edit({ op: "add", role: "filters", type: "exclude" }); + edit({ op: "add", role: "filters", type: "include" }); + assert.deepEqual(edit({ op: "move_filter", from: 1, to: 0 }), { role: "filters", id: "", index: 0 }); + assert.deepEqual([value(pipelines, 0, node, "type"), value(pipelines, 0, { role: "filters", index: 1 }, "type")], ["include", "exclude"]); + edit({ op: "remove", node: { role: "source", id: "null" } }); + edit({ op: "rename_pipeline", name: "q" }); + assert.deepEqual([pipelines[0].name, pipelines[0].plugin_sources.map(s => s.id), value(pipelines, 0, { role: "sink", id: "http" })], + ["q", ["null_2"], { port: 9000 }]); + edit({ op: "remove_pipeline" }); + assert.deepEqual(pipelines, []); +}); + +test("fields are a node's keys, its tables' of the plugin's side as groups", () => { + assert.deepEqual(schema.sources.find(p => p.type === "null").keys, []); + const rows = fields(schema, "sink", "tcp_chain"); + const tls = rows.find(r => r.path === "tls"); + assert.ok(tls.rows.some(r => r.path === "tls.pin_sha256") && !tls.rows.some(r => r.path === "tls.self_signed")); + assert.deepEqual(rows.find(r => r.path === "port").key.max, 65535); + assert.equal(fields(schema, "rate_limit").find(r => r.path === "entries_per_second").placeholder, "(off)"); + assert.equal(fields(schema, "heartbeat").find(r => r.path === "interval_ms").placeholder, "1000"); +}); + +test("a refused call answers why and where, and the engine keeps answering", () => { const refused = [ - [["parse", "lw --nope x"], /not a pipeline flag/], - [["parse", "lw"], /no pipeline flags/], - [["parse"], /^usage: lwconf\.parse\(line\)/], - [["parse", 5], /^usage: /], - [["preset", "tail", JSON.stringify({ path: "/var/log" })], /path/], - [["validate", JSON.stringify([{ name: "p" }])], /no sources/], - [["validate", JSON.stringify(["a", "b"].map((name, i) => ({ - name, plugin_sources: [{ type: "tcp_chain", config: { port: [9000, "x"][i] } }], plugin_sinks: [{ type: "null" }], - })))], /^pipelines\[1\]\.plugin_sources\[tcp_chain\]\.config\.port: not an integer$/], - [["emit", JSON.stringify([{ name: "p" }]), "command"], /no sources/], - [["emit", "[]", "yaml"], /no form "yaml"/], + [["parse", "lw --nope x", "command", "{}"], { error: /not a pipeline flag/ }], + [["parse", "x=1", "environment", "{}"], { error: /^no LOGWISP_ pipeline variables$/ }], + [["parse", "", "file", "{}"], { error: /no \[\[pipelines\]\]/ }], + [["parse", "lw"], { error: /^usage: lwconf\.parse\(text, form, dirs\)/ }], + [["parse", 5, "command", "{}"], { error: /^usage: / }], + [["preset", "serve", JSON.stringify({ allow: 5 }), "{}"], { error: /^allow: takes a string/, path: "allow" }], + [["edit", JSON.stringify([{ name: "p", plugin_sinks: [{ type: "tcp" }] }]), + JSON.stringify({ op: "set", pipeline: 0, node: { role: "sink", id: "tcp" }, key: "port", values: ["x"] })], + { error: /port: not an integer$/, path: "pipelines[0].plugin_sinks[tcp].config.port", pipeline: 0, key: "port" }], + [["emit", JSON.stringify([{ name: "p" }]), "command"], { error: /no sources defined/, path: "pipelines[0].plugin_sources" }], + [["emit", "[]", "yaml"], { error: /no form "yaml"/ }], + [["edit", "[]", JSON.stringify({ op: "add_pipeline", nmae: "p" })], { error: /unknown field "nmae"/ }], + [["edit", "[]", JSON.stringify({ op: "rename" })], { error: /^no op "rename" \(valid: add, add_pipeline/ }], ]; for (const [args, want] of refused) { - const got = call(...args); + const got = JSON.parse(globalThis.lwconf[args[0]](...args.slice(1))); assert.equal(got.value, undefined, `${args}`); - assert.match(got.error, want, `${args}`); + for (const [field, w] of Object.entries(want)) { + if (w instanceof RegExp) assert.match(got[field], w, `${args}`); + else assert.equal(got[field], w, `${args}`); + } } - assert.ok(call("schema").value.sources.length > 0); + assert.ok(engine.schema().value.sources.length > 0); }); diff --git a/cmd/lwconf/main.go b/cmd/lwconf/main.go index ab07fbb..e752159 100644 --- a/cmd/lwconf/main.go +++ b/cmd/lwconf/main.go @@ -1,19 +1,24 @@ //go:build js && wasm -// Command lwconf is the configuration engine as WebAssembly, for the website. -// It sets globalThis.lwconf, whose calls take strings and return a JSON object: -// {"value": ...}, or {"error": "..."} for a call refused. Pipelines travel in -// compose's JSON form; off the host a preset's directory path ends in '/'. +// Command lwconf is the configuration engine as WebAssembly: it sets +// globalThis.lwconf for lwconf.js, calls taking strings and returning JSON, +// {"value"} or {"error"} with its path, one in a pipeline placed on its node +// and key, any other a preset's or flag's key. Pipelines are compose's JSON +// form; dirs answers config.IsDir by path, {} leaving '/' and globs to tell. package main import ( + "bytes" "encoding/json" + "errors" "fmt" + "maps" "slices" "strings" "syscall/js" "github.com/lixenwraith/logwisp/internal/compose" + "github.com/lixenwraith/logwisp/internal/config" ) // calls are lwconf's, by name, with the string arguments each takes @@ -22,34 +27,187 @@ var calls = map[string]struct { call func(args []string) (any, error) }{ "schema": {nil, func([]string) (any, error) { return compose.NewSchema(), nil }}, - "preset": {[]string{"name", "values"}, func(a []string) (any, error) { - var values map[string]string - if err := json.Unmarshal([]byte(a[1]), &values); err != nil { - return nil, fmt.Errorf("values: %w", err) + "preset": {[]string{"name", "values", "dirs"}, func(a []string) (any, error) { + values, err := presetValues(a[1]) + if err != nil { + return nil, err } - return pipelines(compose.FromPreset(a[0], values, nil)) + isDir, err := answers(a[2]) + if err != nil { + return nil, err + } + return pipelines(compose.FromPreset(a[0], values, isDir)) }}, - "parse": {[]string{"line"}, func(a []string) (any, error) { - return pipelines(compose.FromCommandLine(a[0], nil)) + "parse": {[]string{"text", "form", "dirs"}, func(a []string) (any, error) { + f, err := form(a[1]) + if err != nil { + return nil, err + } + isDir, err := answers(a[2]) + if err != nil { + return nil, err + } + return pipelines(f.Read(a[0], isDir)) }}, "validate": {[]string{"pipelines"}, func(a []string) (any, error) { c, err := compose.FromJSON([]byte(a[0])) - if err == nil { - err = c.Validate() + if c == nil { + return nil, err } - return pipelines(c, err) + value, err := pipelines(c, nil) // a value of the wrong kind is a problem Validate names + if err != nil { + return nil, err + } + problems := []problem{} + for _, p := range compose.Problems(c.Validate()) { + problems = append(problems, place(p)) + } + return map[string]any{"pipelines": value, "problems": problems}, nil }}, "emit": {[]string{"pipelines", "form"}, func(a []string) (any, error) { c, err := compose.FromJSON([]byte(a[0])) if err != nil { return nil, err } - forms := compose.Forms() - if i := slices.IndexFunc(forms, func(f compose.Form) bool { return f.Name == a[1] }); i >= 0 { - return forms[i].Write(c) + f, err := form(a[1]) + if err != nil { + return nil, err } - return nil, fmt.Errorf("no form %q", a[1]) + return f.Write(c) }}, + "edit": {[]string{"pipelines", "op"}, func(a []string) (any, error) { + c, err := compose.FromJSON([]byte(a[0])) + if err != nil { + return nil, err + } + var o op + in := json.NewDecoder(bytes.NewReader([]byte(a[1]))) + in.DisallowUnknownFields() + if err := in.Decode(&o); err != nil { + return nil, fmt.Errorf("op: %w", err) + } + edit, ok := edits[o.Op] + if !ok { + return nil, fmt.Errorf("no op %q (valid: %s)", o.Op, strings.Join(slices.Sorted(maps.Keys(edits)), ", ")) + } + n, err := edit(c, o) + if _, ok := errors.AsType[*config.KeyError](err); ok && o.Node.Role != "" { + err = config.At(o.Node.Path(o.Pipeline), err) // a key set or unset in the node + } + value, err := pipelines(c, err) + if err != nil { + return nil, err + } + return map[string]any{"pipelines": value, "node": n}, nil + }}, +} + +// op is one edit: the compose method Op names, and its arguments +type op struct { + Op string `json:"op"` + Pipeline int `json:"pipeline"` + Node compose.Node `json:"node"` + Role string `json:"role"` + Type string `json:"type"` + Key string `json:"key"` + Values []string `json:"values"` + Name string `json:"name"` + From int `json:"from"` + To int `json:"to"` +} + +// edits call compose's edit methods, each returning the node it leaves chosen +var edits = map[string]func(*compose.Composition, op) (compose.Node, error){ + "add_pipeline": func(c *compose.Composition, o op) (compose.Node, error) { + return compose.Node{}, c.AddPipeline(o.Name) + }, + "rename_pipeline": func(c *compose.Composition, o op) (compose.Node, error) { + return compose.Node{}, c.RenamePipeline(o.Pipeline, o.Name) + }, + "remove_pipeline": func(c *compose.Composition, o op) (compose.Node, error) { + return compose.Node{}, c.RemovePipeline(o.Pipeline) + }, + "add": func(c *compose.Composition, o op) (compose.Node, error) { return c.Add(o.Pipeline, o.Role, o.Type) }, + "retype": func(c *compose.Composition, o op) (compose.Node, error) { + return c.Retype(o.Pipeline, o.Node, o.Type) + }, + "remove": func(c *compose.Composition, o op) (compose.Node, error) { + return compose.Node{}, c.Remove(o.Pipeline, o.Node) + }, + "move_filter": func(c *compose.Composition, o op) (compose.Node, error) { + return compose.Node{Role: "filters", Index: o.To}, c.MoveFilter(o.Pipeline, o.From, o.To) + }, + "set": func(c *compose.Composition, o op) (compose.Node, error) { + return o.Node, c.Set(o.Pipeline, o.Node, o.Key, o.Values...) + }, + "unset": func(c *compose.Composition, o op) (compose.Node, error) { + return o.Node, c.Unset(o.Pipeline, o.Node, o.Key) + }, +} + +// problem is where a problem, or a call's refusal, lies as far as the engine +// knows: its path, and the pipeline, node and key Locate reads from it +type problem struct { + Error string `json:"error,omitempty"` + Path string `json:"path,omitempty"` + Pipeline *int `json:"pipeline,omitempty"` + Node *compose.Node `json:"node,omitempty"` + Key string `json:"key,omitempty"` + Text string `json:"text,omitempty"` +} + +func place(err error) problem { + p := problem{Text: err.Error()} + if ke, ok := errors.AsType[*config.KeyError](err); ok { + p.Path = ke.Key + } + if pi, n, key, text, ok := compose.Locate(err); ok { + p.Pipeline, p.Key, p.Text = &pi, key, text + if n.Role != "" { + p.Node = &n + } + } + return p +} + +// presetValues reads a preset's values: a string each, or a list key's +// strings, which the preset takes ',' between them +func presetValues(data string) (map[string]string, error) { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(data), &raw); err != nil { + return nil, fmt.Errorf("values: %w", err) + } + values := map[string]string{} + for k, v := range raw { + var one string + var many []string + switch { + case json.Unmarshal(v, &one) == nil: + values[k] = one + case json.Unmarshal(v, &many) == nil: + values[k] = strings.Join(many, ",") + default: + return nil, &config.KeyError{Key: k, Err: errors.New("takes a string, or a list of strings")} + } + } + return values, nil +} + +// answers reads dirs: whether each path a preset names is a directory +func answers(dirs string) (config.IsDir, error) { + var m map[string]bool + if err := json.Unmarshal([]byte(dirs), &m); err != nil { + return nil, fmt.Errorf("dirs: %w", err) + } + return config.Answers(m), nil +} + +func form(name string) (compose.Form, error) { + forms := compose.Forms() + if i := slices.IndexFunc(forms, func(f compose.Form) bool { return f.Name == name }); i >= 0 { + return forms[i], nil + } + return compose.Form{}, fmt.Errorf("no form %q", name) } // pipelines is a composition's value: its pipelines settled, ids given @@ -81,9 +239,11 @@ func answer(name string, args []js.Value) string { } value, err = c.call(strs) } - out := map[string]any{"value": value} + var out any = map[string]any{"value": value} if err != nil { - out = map[string]any{"error": err.Error()} + p := place(err) + p.Error, p.Text = err.Error(), "" + out = p } data, _ := json.Marshal(out) return string(data) diff --git a/deploy/package/completion/_lw b/deploy/package/completion/_lw index ac7555b..ad0bc51 100644 --- a/deploy/package/completion/_lw +++ b/deploy/package/completion/_lw @@ -94,10 +94,10 @@ preset:*) (( CURRENT -= 2 )) case ${words[1]} in pipe) _arguments -S \ - '--format=[raw|txt|json]:value:_default' \ + '--format=[raw|txt|json: entry format]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; tail) _arguments -S \ - '--format=[raw|txt|json]:value:_default' \ + '--format=[raw|txt|json: entry format]:value:_default' \ '--from=[start|end: where reading a file starts]:value:_default' \ '--path=[file, directory (its files) or glob to follow]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; @@ -105,7 +105,7 @@ preset:*) '--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \ '--cert=[certificate file, for tls=files]:value:_default' \ '--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \ - '--format=[raw|txt|json]:value:_default' \ + '--format=[raw|txt|json: entry format]:value:_default' \ '--from=[start|end: where reading a file starts]:value:_default' \ '--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \ '--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \ @@ -115,10 +115,10 @@ preset:*) '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ '--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \ - '--tls=[off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \ + '--tls=[off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \ '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \ - '--viewer=[true: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \ + '--viewer=[true|false: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; edge) _arguments -S \ '--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \ @@ -139,7 +139,7 @@ preset:*) '--cert=[certificate file, for tls=files]:value:_default' \ '--client_ca=[CA file that verifies edge certificates (mTLS)]:value:_default' \ '--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \ - '--format=[raw|txt|json]:value:_default' \ + '--format=[raw|txt|json: entry format]:value:_default' \ '--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \ '--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \ '--issuer_key=[CA key file, for tls=issuer]:value:_default' \ @@ -147,7 +147,7 @@ preset:*) '--listen=[HOST:PORT; [IPV6\]:PORT; [::\] for IPv6 only]:value:_default' \ '--out=[directory for the received entries; default: standard output]:value:_default' \ '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ - '--tls=[self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \ + '--tls=[self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \ '--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \ diff --git a/deploy/package/completion/lw.fish b/deploy/package/completion/lw.fish index 61d7ff7..b50baa7 100644 --- a/deploy/package/completion/lw.fish +++ b/deploy/package/completion/lw.fish @@ -96,16 +96,16 @@ complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a serve -d 'Serve files or standard input live: a browser viewer at /, SSE at /stream' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a edge -d 'Forward files or standard input to an aggregator over TLS, authenticated' complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a aggregator -d 'Receive from edges over TLS, authenticated, to files or standard output' -complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json' +complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset pipe' -l help -o h -d 'print the flags' -complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json' +complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset tail' -l from -r -F -d 'start|end: where reading a file starts' complete -c lw -n '__lw_in preset tail' -l path -r -F -d 'file, directory (its files) or glob to follow' complete -c lw -n '__lw_in preset tail' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset serve' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them' complete -c lw -n '__lw_in preset serve' -l cert -r -F -d 'certificate file, for tls=files' complete -c lw -n '__lw_in preset serve' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them' -complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json' +complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset serve' -l from -r -F -d 'start|end: where reading a file starts' complete -c lw -n '__lw_in preset serve' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them' complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer' @@ -115,10 +115,10 @@ complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT; [IPV6]:P complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them' -complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files' +complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files' complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM' -complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true: the login page and viewer, behind proxy; without user or users the viewer is always on' +complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true|false: the login page and viewer, behind proxy; without user or users the viewer is always on' complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots' complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca' @@ -137,7 +137,7 @@ complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CI complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files' complete -c lw -n '__lw_in preset aggregator' -l client_ca -r -F -d 'CA file that verifies edge certificates (mTLS)' complete -c lw -n '__lw_in preset aggregator' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them' -complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json' +complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json: entry format' complete -c lw -n '__lw_in preset aggregator' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer' complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key file, for tls=issuer' @@ -145,7 +145,7 @@ complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT; [::] for IPv6 only' complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output' complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' -complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files' +complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files' complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM' diff --git a/doc/cli.md b/doc/cli.md index 8f0715d..feecd6f 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -69,9 +69,18 @@ pipelines have [their own flags](#pipelines): resolved into it, as TOML that `-c` reads back, and exit - `-T`, `--tui`: compose the pipelines on screen, then run or print them ([below](#lw---tui)) -- `--schema`: print every setting, pipeline flag, flow stage, plugin option - and preset as one JSON document (kinds, defaults, rules, help), and exit; - tools build their option lists from it +- `--schema`: print every setting, pipeline flag, flow stage, plugin option, + preset and output form as one JSON document, and exit; tools build their + option lists from it: + - a key: its kind, default, rules and help; a table (`tls`, `auth`, `acl`) + holds its own keys, those of the plugin's side (listener or dialer), + HTTP's only on a plugin that speaks HTTP. Every listener lists + `auth.node_binding` and `auth.trusted_proxies`, which validation admits + only on chain sources and on the http sink + - a flow stage: its keys, and `on`, the values that turn it on + - a preset key: its default and help, and the choices (`enum`) and `hint` + of the option it sets + - `forms`: `command`, `environment` and `file`, each with its hint - `--color [WHEN]`, WHEN `auto`, `always` or `never`: level names in color on console sinks, default `auto` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`); bare, `always`. A console sink's own `color` wins @@ -421,14 +430,22 @@ lw -c dev.toml --logging.level=debug `lw --check` validates without running: it builds every pipeline and plugin as a start would (options, TLS files, credentials files, startup warnings), but binds no port, reads no stdin, and opens, reads or creates no log file; it -prints `configuration ok: N pipeline(s)` and exits 0, or the error and exits 1 -(2 when a named file is missing, see [Exit Codes](#exit-codes)): +prints `configuration ok: N pipeline(s)` and exits 0. Otherwise it prints the +first problem of each setting, flow stage, source, sink and pipeline, a line +each named by its key's path, or else the first file that fails to open, and +exits 1 (2 when a named file is missing, see [Exit Codes](#exit-codes)): ```bash lw --check -c /etc/logwisp/logwisp.toml lw --check --source file:directory=/var/log/app --sink http:port=8080 ``` +```text +Error: Failed to load config: invalid configuration: +pipelines[0].plugin_sources[file].config.directory: must be set +pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000 +``` + **Production** ```bash @@ -472,8 +489,8 @@ WebAssembly. Space flips it; Esc goes back. `a` adds a source, sink or filter; `c` changes a source's or sink's type in its place; `d` deletes, or returns a value to its default; Space turns a flow stage on or off; `J`/`K` move a - filter; `[` `]` switch pipelines; `p` presets; `!` shows the first - problem; `r` runs; `o` output; `?` keys; `q` quits. + filter; `[` `]` switch pipelines; `p` presets; `!` shows every problem, + moving to the first; `r` runs; `o` output; `?` keys; `q` quits. - Pipelines: Up from a column's top row reaches the bar, where `[` `]` or Left/Right switch, `a` adds one (empty or from a preset), Enter renames and `d` deletes, asking first when it has parts. diff --git a/doc/installation.md b/doc/installation.md index 2def19a..19ce192 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -26,9 +26,12 @@ The Makefile works with GNU make and BSD make alike. Targets: - `make build`: `bin/lw`, with version, commit and build time from git - `make release`: the same, static (`CGO_ENABLED=0`), `-trimpath`, stripped - `make dev`: built with the race detector - - `make wasm`: `bin/lwconf.wasm`, the configuration engine for the - website, version stamped, beside the toolchain's `wasm_exec.js`; prints - both checksums + - `make wasm`: `bin/lwconf/`, the configuration engine a website drops in: + `lwconf.wasm` (version stamped), the toolchain's `wasm_exec.js`, + `lwconf.js` that loads and calls it, `schema.json` (`lw --schema`) and + `lwconf.json`, their manifest: version, commit, Go version and each + file's sha256. Stamped with the commit's time, a commit's files are the + same on every build - `make version`: the metadata a build would embed - `make completion`: regenerate the bash, zsh and fish completion in `deploy/package/completion/` from `cmd/lw`'s `shorts` and `commands` diff --git a/doc/lw.1 b/doc/lw.1 index 5676b97..8915695 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -141,7 +141,10 @@ Build every pipeline and plugin as a start would, loading options, TLS files and credentials files and printing startup warnings, without binding a port or opening, reading or creating a log file; print .Dq configuration ok -and exit 0, or the error and exit 1, or 2 when a named configuration file is +and exit 0. +Otherwise print the first problem of each setting, flow stage, source, sink +and pipeline, a line each named by its key's path, or else the first file +that fails to open, and exit 1, or 2 when a named configuration file is missing .Pq see Sx EXIT STATUS . .It Fl \-dump @@ -167,9 +170,13 @@ A file is read, never written; the shell empties a redirect's target before .Nm reads it, so the redirect names a file it does not load. .It Fl \-schema -Print every setting, pipeline flag, flow stage, plugin option and preset as -one JSON document, with kinds, defaults, rules and help, and exit; tools build -their option lists from it. +Print every setting, pipeline flag, flow stage, plugin option, preset and +output form as one JSON document, with kinds, defaults, rules and help, and +exit; tools build their option lists from it. +A plugin's table holds the keys of its side; a flow stage carries +.Ql on , +the values that turn it on; a preset key carries the choices and hint of the +option it sets. .It Fl q , Fl \-quiet Turn off the log of .Nm diff --git a/doc/operations.md b/doc/operations.md index aceec07..35729f3 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -100,15 +100,16 @@ lw --check -c candidate.toml ``` Before a reload, check the edited file this way: a failed reload keeps the -running service, but `--check` says why without touching it. Failures name the -pipeline and the offending key: +running service, but `--check` says why without touching it. It prints the +first problem of each setting, flow stage, source, sink and pipeline, a line +each named by its key's path; once none is left, the files they name are +opened as a start opens them: +```text +Error: Failed to load config: invalid configuration: +pipelines[0].plugin_sources[in].config.directory: must be set +pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000 ``` -ERROR msg="Failed to create pipeline" pipeline=app error="failed to create sink out: port: must be 1-65535, got 0" -``` - -Remember that most validation lives in plugin constructors, so a config only -proves itself when the pipeline is actually built. ## Monitoring diff --git a/doc/todo.md b/doc/todo.md index 004e23a..d40aff0 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -89,17 +89,18 @@ What exists: on the option structs, one catalogue row per plugin type, `Decode` (every constructor decodes through it and keeps only I/O), `Coerce` for typed command-line values, and a `ValidateConfig` that decodes every plugin - offline and names the refused key's path. + offline and names the first problem of each part by its key's path. - `internal/config` already compiles for `GOOS=js`; it links no net/http, crypto/tls or terminal code. - The engine, `internal/compose`: starts empty, from a preset (whose path - `config.IsDir` resolves, nil off the host), a loaded configuration or a - pasted command line; adds, removes and moves sources, sinks and filters, - sets and unsets keys, validates; writes a command line, environment - variables or a file, each loading back to the same pipelines. Its test - fails when it links TLS, HTTP, process, signal or terminal code. -- `lw --schema`: settings, pipeline flags, flow stages in order, the tls, - auth and acl tables, sources, sinks and presets, as one JSON document. + `config.IsDir` resolves, from answers off the host), a loaded + configuration or pasted text of any form; adds, removes and moves sources, + sinks and filters, sets and unsets keys, validates; writes a command line, + environment variables or a file, each reading back to the same pipelines. + Its test fails when it links TLS, HTTP, process, signal or terminal code. +- `lw --schema`: settings, pipeline flags, flow stages in order with what + turns each on, sources and sinks with their tables' keys of their side, + presets and forms, as one JSON document. - `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions, layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal, ConfirmDialog, scrolling, mouse; themed Radio, Field, TextInput, @@ -118,21 +119,46 @@ Steps: radio buttons, narrow form fields and option lists, menus, frames, rules, wires and windows, moved from `lw --tui`. 5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)). -6. WebAssembly: done. `make wasm` builds `bin/lwconf.wasm`, version stamped, - beside the toolchain's `wasm_exec.js`, and prints both checksums: about - 6.9 MB, 1.9 MB gzipped. `wasm_exec.js` uses no eval, so the page needs only - `'wasm-unsafe-eval'` in its `script-src`. The page runs the module with - `new Go().run(instance)`, then calls `globalThis.lwconf`; each call takes - strings and returns a JSON object, `{"value": ...}` or `{"error": "..."}`: +6. WebAssembly: done. `make wasm` writes `bin/lwconf/`, which a site drops + in whole: + - `lwconf.wasm`, version stamped: 7.1 MB, 1.9 MB at gzip -9. + - `wasm_exec.js`, the toolchain's, which uses no eval: the page needs only + `'wasm-unsafe-eval'` in its `script-src`. + - `lwconf.js`, one dependency-free ES module that knows the calls and + shapes, never a key: `load({execURL, execSRI, wasmURL, wasmSRI})` + fetches both by their integrity and resolves to the engine; + `fields(schema, role, type)` is a node's form rows, a table's a group; + `value`, `at` and `group` read a node's value and sort problems by node. + - `schema.json`, equal to `schema()`, and `lwconf.json`, the manifest: + version, commit, Go version and each file's sha256. + + The engine's calls take strings (`lwconf.js` writes objects as JSON) and + answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`, + as far as the engine places the refusal: - `schema()`: what `lw --schema` prints. - - `preset(name, values)`: a preset's pipelines; `values` is a JSON object - of strings, and a directory path ends in `/`. - - `parse(line)`: a pasted command line's pipelines. - - `validate(pipelines)`: the pipelines settled (defaults, kinds, ids), or - the first problem by its key path. - - `emit(pipelines, form)`: `command`, `environment` or `file`. + - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key + to a string, a list key to strings. + - `parse(text, form, dirs)`: the pipelines of a form's pasted text. + - `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled + (defaults, kinds, ids) and the first problem of each part (a value of + the wrong kind too) as `{path, pipeline, node, key, text}`. + - `emit(pipelines, form)`: the text of `command`, `environment` or `file`. + - `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is + `{op, pipeline, ...}`: `add_pipeline` and `rename_pipeline` (`name`), + `remove_pipeline`, `add` (`role`, `type`), `retype` (`node`, `type`), + `remove` (`node`), `move_filter` (`from`, `to`), `set` (`node`, `key`, + `values`), `unset` (`node`, `key`). + - `dirs` answers whether a path a preset names is a directory, + `{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one, + one with a glob a pattern, and any other is refused at `path`. - Pipelines are a JSON list in the file's shape (`name`, `flow`, - `plugin_sources`, `plugin_sinks`); a misspelled key is refused. + `plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node + is `{role, id, index}`: a source's or sink's id, a filter's index. Still to verify: a scripted session on a pseudo-terminal for Run and each output. + +Still to derive: `AuthOptions.Check` alone limits `auth.node_binding` to chain +sources and `auth.trusted_proxies` to the http sink, so the schema lists both +on every listener; a side tag naming those sides would filter them as `http` +filters HTTP's keys. diff --git a/go.mod b/go.mod index 5f9505f..b7cd20d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/lixenwraith/color v0.1.3 github.com/lixenwraith/config v0.2.2 github.com/lixenwraith/log v0.1.11 - github.com/lixenwraith/terminal v0.2.1 + github.com/lixenwraith/terminal v0.2.2-0.20261010220430-c657af606d34 github.com/lixenwraith/toml v0.1.3 golang.org/x/term v0.47.0 ) diff --git a/go.sum b/go.sum index d2177f4..796192d 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKh github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg= github.com/lixenwraith/terminal v0.2.1 h1:KsMfmxrRicBFWMdXJIEJ3cpLWVw6IoNF8GwgsxbD3kQ= github.com/lixenwraith/terminal v0.2.1/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY= +github.com/lixenwraith/terminal v0.2.2-0.20261010220430-c657af606d34 h1:GasarGRKYbAXzEz1utbcf9n4+N7WXPKyTbUtQZj8v74= +github.com/lixenwraith/terminal v0.2.2-0.20261010220430-c657af606d34/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY= github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI= github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= diff --git a/internal/authz/authz_test.go b/internal/authz/authz_test.go index 36d1ca9..6f0fbd5 100644 --- a/internal/authz/authz_test.go +++ b/internal/authz/authz_test.go @@ -154,7 +154,7 @@ func TestNewValidation(t *testing.T) { {"trusted proxies without a type", &config.AuthOptions{TrustedProxies: proxies}, nil, config.Listener, HTTP, "trusted_proxies is set"}, {"trusted proxies under mtls", &config.AuthOptions{Type: MethodMTLS, TrustedProxies: proxies}, mtlsListenerTLS(), config.Listener, HTTP, "trusted_proxies applies only"}, {"trusted proxies on a chain source", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.ChainListener, HTTP, "only to the http sink"}, - {"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to the http sink"}, + {"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to HTTP listeners"}, {"trusted proxies with certificate binding", listener(config.AuthOptions{TrustedProxies: proxies, Identity: "cn"}), f.serverTLS, config.Listener, HTTP, "TLS-terminating proxy"}, {"malformed trusted proxy", listener(config.AuthOptions{TrustedProxies: []string{"localhost"}}), nil, config.Listener, HTTP, "neither an address"}, {"trusted proxies on a dialer", dialer(config.AuthOptions{TrustedProxies: proxies}), &tls.Config{}, config.Dialer, HTTP, "only to listeners"}, diff --git a/internal/compose/compose.go b/internal/compose/compose.go index 16eb1c3..b4a6ffe 100644 --- a/internal/compose/compose.go +++ b/internal/compose/compose.go @@ -32,9 +32,9 @@ type Composition struct { // by its key in the flow table: filters with an index, format, rate_limit or // heartbeat type Node struct { - Role string - ID string - Index int + Role string `json:"role"` + ID string `json:"id"` + Index int `json:"index"` } // FromConfig starts from a loaded configuration's pipelines, its console sinks @@ -100,8 +100,10 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) { } // settle gives every flow stage its defaults, every plugin's options their -// kinds and every source and sink an id; the rules are Validate's +// kinds and every source and sink an id; the rules are Validate's. Its error +// joins a plugin's each holding a value of the wrong kind, kept as given. func (c *Composition) settle() error { + var errs []error for i := range c.Pipelines { p := &c.Pipelines[i] if p.Flow == nil { @@ -122,17 +124,17 @@ func (c *Composition) settle() error { for j, s := range p.PluginSources { p.PluginSources[j].Config = orEmpty(s.Config) if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil { - return config.At(fmt.Sprintf("pipelines[%d].plugin_sources[%s].config", i, s.ID), err) + errs = append(errs, config.At(Node{Role: "source", ID: s.ID}.Path(i), err)) } } for j, s := range p.PluginSinks { p.PluginSinks[j].Config = orEmpty(s.Config) if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil { - return config.At(fmt.Sprintf("pipelines[%d].plugin_sinks[%s].config", i, s.ID), err) + errs = append(errs, config.At(Node{Role: "sink", ID: s.ID}.Path(i), err)) } } } - return nil + return errors.Join(errs...) } // name gives the parts a file left unnamed the ids their specs would get: the @@ -340,6 +342,17 @@ func index(p *config.PipelineConfig, n Node) int { return -1 } +// Path is the path of node n's options in pipeline pi, as Locate reads it +func (n Node) Path(pi int) string { + switch n.Role { + case "source", "sink": + return fmt.Sprintf("pipelines[%d].plugin_%ss[%s].config", pi, n.Role, n.ID) + case "filters": + return fmt.Sprintf("pipelines[%d].flow.filters[%d]", pi, n.Index) + } + return fmt.Sprintf("pipelines[%d].flow.%s", pi, n.Role) +} + func (n Node) missing() error { if n.Role == "filters" { return fmt.Errorf("no filter %d", n.Index) @@ -476,9 +489,69 @@ func (c *Composition) Options(pi int, n Node) (map[string]any, bool, error) { return nil, false, n.missing() } -// Validate applies every rule lw applies at load, naming the refused key's path +// Validate applies every rule lw applies at load: its error joins every +// problem, one a part, which Problems splits and Locate places func (c *Composition) Validate() error { return config.ValidatePipelines(c.Pipelines) } +// Problems are the problems an error joins, in order +func Problems(err error) []error { + if joined, ok := err.(interface{ Unwrap() []error }); ok { + var out []error + for _, e := range joined.Unwrap() { + out = append(out, Problems(e)...) + } + return out + } + if err == nil { + return nil + } + return []error{err} +} + +// Locate reads where a problem lies from its path: the pipeline, the node, +// and the key in the node's options, a list's without its index; text is +// what is wrong. ok is false for a problem no pipeline holds. +func Locate(err error) (pi int, n Node, key, text string, ok bool) { + ke, _ := errors.AsType[*config.KeyError](err) + rest, found := "", false + if ke != nil { + rest, found = strings.CutPrefix(ke.Key, "pipelines[") + } + end := strings.IndexByte(rest, ']') + if !found || end < 0 { + return 0, n, "", err.Error(), false + } + if pi, err = strconv.Atoi(rest[:end]); err != nil { + return 0, n, "", ke.Error(), false + } + rest = strings.TrimPrefix(rest[end+1:], ".") + for role, list := range map[string]string{"source": "plugin_sources[", "sink": "plugin_sinks["} { + if after, ok := strings.CutPrefix(rest, list); ok { + id, more := cutID(after) + n.Role, n.ID, rest = role, id, strings.TrimPrefix(strings.TrimPrefix(more, "config"), ".") + } + } + if stage, ok := strings.CutPrefix(rest, "flow."); ok { + name, more, _ := strings.Cut(stage, ".") + name, index, _ := strings.Cut(name, "[") + n.Role, rest = name, more + n.Index, _ = strconv.Atoi(strings.TrimSuffix(index, "]")) + } + key, _, _ = strings.Cut(rest, "[") + return pi, n, key, ke.Err.Error(), true +} + +// cutID cuts a plugin path at its id's ']', the first that ends the path or +// starts its type or config: ids are free text, and keys hold list indexes +func cutID(path string) (id, rest string) { + for i := range len(path) { + if after := path[i+1:]; path[i] == ']' && (after == "" || after == ".type" || after == ".config" || strings.HasPrefix(after, ".config.")) { + return path[:i], strings.TrimPrefix(after, ".") + } + } + return path, "" +} + // CommandLine writes the composition as an lw command line, a flag a line, // quoted for a POSIX shell func (c *Composition) CommandLine() (string, error) { @@ -549,28 +622,77 @@ func (c *Composition) File() ([]byte, error) { return toml.Marshal(map[string]any{"pipelines": c.Pipelines}) } -// Form is a way to write a composition, named for what it is: each loads back -// to the same pipelines +// Form is a way to write a composition, named for what it is, and to read it +// back to the same pipelines type Form struct { - Name, Hint string - Write func(*Composition) (string, error) + Name string `json:"name"` + Hint string `json:"hint"` + Write func(*Composition) (string, error) `json:"-"` + Read func(string, config.IsDir) (*Composition, error) `json:"-"` } var forms = []Form{ - {"command", "lw's flags", (*Composition).CommandLine}, - {"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment}, + {"command", "lw's flags", (*Composition).CommandLine, FromCommandLine}, + {"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment, FromEnvironment}, {"file", "a configuration file's pipelines", func(c *Composition) (string, error) { data, err := c.File() return string(data), err - }}, + }, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }}, } -// Forms are what a composition writes: lw --tui prints one, the website shows them +// Forms are what a composition writes and reads: lw --tui prints one, the +// website shows them and reads them pasted func Forms() []Form { return slices.Clone(forms) } +// FromEnvironment starts from NAME='value' lines as an environment file holds +// them, quoted as a POSIX shell reads them: '#' starts a comment line, a word +// without '=', such as export, assigns nothing, and a later assignment wins +func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) { + vars := map[string]string{} + for line := range strings.Lines(text) { + if line = strings.TrimSpace(line); line == "" || line[0] == '#' { + continue + } + words, err := split(line) + if err != nil { + return nil, err + } + for _, w := range words { + if name, value, ok := strings.Cut(w, "="); ok { + vars[name] = value + } + } + } + var environ []string + for name, value := range vars { + environ = append(environ, name+"="+value) + } + pipelines, err := config.EnvPipelines(environ, isDir) + if err != nil { + return nil, err + } + c := &Composition{Pipelines: pipelines} + return c, c.settle() +} + +// FromFile starts from a configuration file's pipelines, refusing a misspelled +// key as lw does; its settings are the host's and its console sinks keep the +// color they name +func FromFile(data []byte) (*Composition, error) { + cfg, err := config.ParseFile(data) + if err != nil { + return nil, err + } + if len(cfg.Pipelines) == 0 { + return nil, errors.New("no [[pipelines]]") + } + return FromConfig(&config.Config{Pipelines: cfg.Pipelines}) +} + // FromJSON starts from pipelines as the website keeps them: a JSON list of // tables in a file's shape, a misspelled key refused as a file's is, a null -// a value left unset +// a value left unset. A value of the wrong kind returns the composition too, +// which Validate names among the rest. func FromJSON(data []byte) (*Composition, error) { var tables any if err := json.Unmarshal(data, &tables); err != nil { @@ -710,14 +832,14 @@ func copyPipeline(p config.PipelineConfig) config.PipelineConfig { // Schema is what lw --schema prints, and the website builds its catalogue // from. A setting's flag is --KEY, a pipeline flag's --FLAG. type Schema struct { - Version string `json:"version"` - Settings []Setting `json:"settings"` - Flags []config.SpecFlag `json:"pipeline_flags"` - Flow []config.Stage `json:"flow"` - Tables map[string][]config.Key `json:"tables"` - Sources []Plugin `json:"sources"` - Sinks []Plugin `json:"sinks"` - Presets []config.Preset `json:"presets"` + Version string `json:"version"` + Settings []Setting `json:"settings"` + Flags []config.SpecFlag `json:"pipeline_flags"` + Flow []config.Stage `json:"flow"` + Sources []Plugin `json:"sources"` + Sinks []Plugin `json:"sinks"` + Presets []config.Preset `json:"presets"` + Forms []Form `json:"forms"` } // Setting is a settings key with its environment variable @@ -737,12 +859,8 @@ func NewSchema() Schema { Version: version.Short(), Flags: config.SpecFlags(), Flow: config.FlowStages(), - Tables: map[string][]config.Key{ - "tls": config.KeysOf[config.TLSOptions](), - "auth": config.KeysOf[config.AuthOptions](), - "acl": config.KeysOf[config.ACLOptions](), - }, Presets: config.Presets(), + Forms: Forms(), } for _, k := range config.SettingKeys() { s.Settings = append(s.Settings, Setting{k, "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(k.Name, ".", "_"))}) diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go index a86bcf2..a07787f 100644 --- a/internal/compose/compose_test.go +++ b/internal/compose/compose_test.go @@ -1,6 +1,7 @@ package compose import ( + "errors" "fmt" "os" "os/exec" @@ -35,11 +36,12 @@ func TestEngineLinksNoHostCode(t *testing.T) { } } -// A pasted command line, its line ends LF or CRLF, and the website's JSON each -// start the composition that wrote them: escaped and quoted values, nested -// keys, lists, moved filters, stages whose flags' defaults differ from a -// file's, a value starting with '-', JSON's numbers for integers -func TestWrittenFormsStartTheSameComposition(t *testing.T) { +// Every form a composition writes, its line ends LF or CRLF, and the +// website's JSON read back to it: escaped and quoted values, nested keys, +// lists, moved filters, stages whose flags' defaults differ from a file's, a +// value starting with '-', JSON's numbers for integers. The environment holds +// one pipeline, so each form is written from one or both. +func TestWrittenFormsReadBack(t *testing.T) { c := &Composition{} must := func(err error) { t.Helper() @@ -68,18 +70,29 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) { must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100")) must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000")) must(c.Set(0, Node{Role: "heartbeat"}, "enabled", "true")) + one := &Composition{Pipelines: slices.Clone(c.Pipelines)} must(c.AddPipeline("-b")) for _, role := range []string{"source", "sink"} { _, err = c.Add(1, role, "null") must(err) } must(c.Set(1, Node{Role: "heartbeat"}, "interval_ms", "2000")) - line, err := c.CommandLine() - must(err) - for _, pasted := range []string{line, strings.ReplaceAll(line, "\n", "\r\n")} { - got, err := FromCommandLine(pasted, nil) - if err != nil || !reflect.DeepEqual(got, c) { - t.Fatalf("%q\n got %+v %v\nwant %+v", pasted, got, err, c) + for _, f := range Forms() { + written := 0 + for _, c := range []*Composition{c, one} { + text, err := f.Write(c) + if err != nil { + continue + } + written++ + for _, pasted := range []string{text, strings.ReplaceAll(text, "\n", "\r\n")} { + if got, err := f.Read(pasted, nil); err != nil || !reflect.DeepEqual(got, c) { + t.Fatalf("%s %q\n got %+v %v\nwant %+v", f.Name, pasted, got, err, c) + } + } + } + if written == 0 { + t.Errorf("%s wrote neither composition", f.Name) } } data, err := c.JSON() @@ -89,6 +102,52 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) { } } +// Pasted environment text reads as a shell runs it: a comment line assigns +// nothing, export is no assignment, and a later assignment wins +func TestPastedEnvironmentReadsAsAShellRunsIt(t *testing.T) { + c, err := FromEnvironment("export LOGWISP_SOURCE='null'\n# LOGWISP_SOURCE='tcp_chain'\n\nLOGWISP_SINK='tcp:port=1'\nLOGWISP_SINK='null'\n", nil) + if err != nil { + t.Fatal(err) + } + p := c.Pipelines[0] + if len(c.Pipelines) != 1 || len(p.PluginSources) != 1 || p.PluginSources[0].Type != "null" || len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "null" { + t.Fatalf("pipelines %+v", c.Pipelines) + } +} + +// Locate places a problem by its path: a plugin's key, a key of its table, a +// rule between its table's keys, a list entry, the plugin itself under an id +// holding ']', a filter's list entry, a stage and the pipeline; and nothing +// outside one +func TestLocatePlacesAProblem(t *testing.T) { + sink := func(id string) Node { return Node{Role: "sink", ID: id} } + for path, want := range map[string]struct { + pi int + n Node + key string + }{ + "pipelines[1].plugin_sinks[out].config.port": {1, sink("out"), "port"}, + "pipelines[0].plugin_sinks[out].config.tls.min_version": {0, sink("out"), "tls.min_version"}, + "pipelines[0].plugin_sinks[out].config.tls": {0, sink("out"), "tls"}, + "pipelines[0].plugin_sinks[out].config.acl.allow[1]": {0, sink("out"), "acl.allow"}, + "pipelines[0].plugin_sinks[a]b].config": {0, sink("a]b"), ""}, + "pipelines[0].plugin_sources[in].type": {0, Node{Role: "source", ID: "in"}, "type"}, + "pipelines[0].flow.filters[2].patterns[1]": {0, Node{Role: "filters", Index: 2}, "patterns"}, + "pipelines[0].flow.rate_limit.policy": {0, Node{Role: "rate_limit"}, "policy"}, + "pipelines[3].name": {3, Node{}, "name"}, + } { + pi, n, key, text, ok := Locate(fmt.Errorf("checked: %w", &config.KeyError{Key: path, Err: errors.New("wrong")})) + if !ok || pi != want.pi || n != want.n || key != want.key || text != "wrong" { + t.Errorf("%s: %d %+v %q %q %v", path, pi, n, key, text, ok) + } + } + for _, err := range []error{errors.New("no pipelines configured"), &config.KeyError{Key: "color", Err: errors.New("wrong")}} { + if _, _, _, text, ok := Locate(err); ok || text != err.Error() { + t.Errorf("%v: placed, text %q", err, text) + } + } +} + // A null in the website's JSON is a value left unset, which every form then // leaves out, in a table or a list func TestANullInJSONIsUnset(t *testing.T) { @@ -105,13 +164,18 @@ func TestANullInJSONIsUnset(t *testing.T) { } } -// The website's JSON refuses a misspelled key as a file does, at any depth -func TestAMisspelledJSONKeyIsRefused(t *testing.T) { +// The website's JSON and a pasted file refuse a misspelled key as lw does, +// at any depth +func TestAMisspelledKeyIsRefused(t *testing.T) { for _, data := range []string{`[{"name":"p","plugin_source":[]}]`, `[{"name":"p","flow":{"rate_limt":{}}}]`} { if _, err := FromJSON([]byte(data)); err == nil || !strings.Contains(err.Error(), "unknown key") { t.Errorf("%s: %v", data, err) } } + file := "[[pipelines]]\nname = \"p\"\nnmae = \"q\"\n[[pipelines.plugin_sources]]\ntype = \"null\"\n" + if _, err := FromFile([]byte(file)); err == nil || !strings.Contains(err.Error(), `unknown key "pipelines[0].nmae"`) { + t.Errorf("file: %v", err) + } } // The command line and environment carry no control character, which a diff --git a/internal/config/config.go b/internal/config/config.go index 37675af..f55db89 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -319,8 +319,8 @@ type AuthOptions struct { CredentialsFile string `toml:"credentials_file" lw:"listener,hint=file" help:"scram: verifiers, from lw auth add-user"` // exp has whole seconds, a sub-second expires_in reads as unknown, and 10 s // leaves room for renewal ahead of expiry - TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"` - TrustedProxies []string `toml:"trusted_proxies" lw:"listener,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"` + TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,http,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"` + TrustedProxies []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"` Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"` PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"` } @@ -335,7 +335,7 @@ type ACLOptions struct { ProxyProtocol string `toml:"proxy_protocol" default:"off" lw:"enum=off|optional|required" help:"read a PROXY header from proxy_from"` ProxyFrom []string `toml:"proxy_from" lw:"hint=cidr" help:"proxies that may send a PROXY header"` MaxConnectionsPerClient int64 `toml:"max_connections_per_client" lw:"min=0,zero=unlimited" help:"connections a client may hold"` - RequestsPerSecondPerClient float64 `toml:"requests_per_second_per_client" lw:"min=0,zero=unlimited" help:"HTTP: requests a client may make each second"` + RequestsPerSecondPerClient float64 `toml:"requests_per_second_per_client" lw:"http,min=0,zero=unlimited" help:"HTTP: requests a client may make each second"` } // --- TLS Options --- diff --git a/internal/config/loader.go b/internal/config/loader.go index 22f60ba..163690e 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -51,7 +51,7 @@ func Load(args Args) (*Manager, error) { return nil, err } if len(specs) == 0 { - specs = envPipelineSpecs() + specs = envPipelineSpecs(os.Environ()) } if _, err := buildPipelines(specs, HostIsDir); err != nil { return nil, err @@ -63,6 +63,7 @@ func Load(args Args) (*Manager, error) { initial.Logging.Level = "warn" initial.StatusReporter = false } + var invalid error // the validator's problems, which lw prints a line each cfg, err := lconfig.NewBuilder(). WithTarget(initial). WithEnvPrefix("LOGWISP_"). @@ -70,10 +71,10 @@ func Load(args Args) (*Manager, error) { WithFile(configPath). WithTypedValidator(func(cfg *Config) error { effective := *cfg // the builder keeps cfg; only the copy is replaced - if err := m.usePipelines(&effective); err != nil { - return err + if invalid = m.usePipelines(&effective); invalid == nil { + invalid = ValidateConfig(&effective) } - return ValidateConfig(&effective) + return invalid }). WithSecurityOptions(lconfig.SecurityOptions{ PreventPathTraversal: true, @@ -81,7 +82,10 @@ func Load(args Args) (*Manager, error) { }). Build() if err != nil { - if !errors.Is(err, lconfig.ErrConfigNotFound) { + switch { + case invalid != nil: + return nil, fmt.Errorf("invalid configuration:\n%w", invalid) + case !errors.Is(err, lconfig.ErrConfigNotFound): return nil, fmt.Errorf("failed to load or validate config: %w", err) } if isExplicit { diff --git a/internal/config/preset.go b/internal/config/preset.go index 39fb52e..71a3c59 100644 --- a/internal/config/preset.go +++ b/internal/config/preset.go @@ -8,6 +8,7 @@ import ( "net" "os" "path/filepath" + "reflect" "slices" "strconv" "strings" @@ -24,22 +25,48 @@ type Preset struct { } // PresetParam is one key. A list takes several values, ',' between them or -// the key repeated; any other key takes one. +// the key repeated; any other key takes one. Enum and Hint are those of the +// option the key sets. type PresetParam struct { - Name string `json:"name"` - Default string `json:"default,omitempty"` - Help string `json:"help"` - Required bool `json:"required,omitempty"` - List bool `json:"list,omitempty"` + Name string `json:"name"` + Default string `json:"default,omitempty"` + Help string `json:"help"` + Required bool `json:"required,omitempty"` + List bool `json:"list,omitempty"` + Enum []string `json:"enum,omitempty"` + Hint string `json:"hint,omitempty"` +} + +// Usage is the key's help after its choices, as lw preset and lw --tui show it +func (p PresetParam) Usage() string { + if len(p.Enum) == 0 { + return p.Help + } + return strings.Join(p.Enum, "|") + ": " + p.Help +} + +// mirror gives a key the enum and hint of option key of table T, which it +// sets; a switch takes true or false +func mirror[T any](p PresetParam, key string) PresetParam { + opts := options(reflect.TypeFor[T]()) + i := slices.IndexFunc(opts, func(o option) bool { return o.Name == key }) + if i < 0 { + panic(fmt.Sprintf("preset key %s: %s has no key %q", p.Name, reflect.TypeFor[T]().Name(), key)) + } + p.Enum, p.Hint = opts[i].Enum, opts[i].Hint + if opts[i].Kind == "bool" { + p.Enum = []string{"true", "false"} + } + return p } // IsDir tells whether a preset's path is a directory, whose files it follows, -// or a file pattern. Off the target host it is the user's answer, or nil: a +// or a file pattern. Off the target host it is the user's Answers, or nil: a // path ending in '/' is then a directory, one with a glob a pattern, and any // other fails with ErrPathKind. type IsDir func(path string) (bool, error) -var ErrPathKind = errors.New("path: a directory or a file pattern?") +var ErrPathKind error = &KeyError{"path", errors.New("a directory or a file pattern?")} // HostIsDir asks this host: a path that does not exist is a pattern func HostIsDir(path string) (bool, error) { @@ -47,29 +74,67 @@ func HostIsDir(path string) (bool, error) { return err == nil && fi.IsDir(), nil } +// Answers is IsDir off the host: whether each path answered is a directory, +// and nil's rule for the others +func Answers(dirs map[string]bool) IsDir { + return func(path string) (bool, error) { + if dir, ok := dirs[path]; ok { + return dir, nil + } + if dir := strings.HasSuffix(path, "/"); dir || strings.ContainsAny(path, `*?[\`) { + return dir, nil + } + return false, ErrPathKind + } +} + // Keys several presets share var ( pathParam = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"} - fromParam = PresetParam{Name: "from", Default: "end", Help: "start|end: where reading a file starts"} - formatParam = func(def string) PresetParam { return PresetParam{Name: "format", Default: def, Help: "raw|txt|json"} } - tlsParams = []PresetParam{ - {Name: "cert", Help: "certificate file, for tls=files"}, - {Name: "key", Help: "key file, for tls=files"}, - {Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"}, - {Name: "issuer_key", Help: "CA key file, for tls=issuer"}, + fromParam = mirror[FileSourceOptions](PresetParam{Name: "from", Default: "end", Help: "where reading a file starts"}, "from") + formatParam = func(def string) PresetParam { + return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "entry format"}, "type") + } + tlsParams = []PresetParam{ + mirror[TLSOptions](PresetParam{Name: "cert", Help: "certificate file, for tls=files"}, "cert_file"), + mirror[TLSOptions](PresetParam{Name: "key", Help: "key file, for tls=files"}, "key_file"), + mirror[TLSOptions](PresetParam{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"}, "issuer_cert_file"), + mirror[TLSOptions](PresetParam{Name: "issuer_key", Help: "CA key file, for tls=issuer"}, "issuer_key_file"), {Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"}, } aclParams = []PresetParam{ - {Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, - {Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, + mirror[ACLOptions](PresetParam{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, "allow"), + mirror[ACLOptions](PresetParam{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, "deny"), } - transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"} - userParams = []PresetParam{ + transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp_chain or http_chain", Enum: transports()} + usersParam = func(who string) PresetParam { + return mirror[AuthOptions](PresetParam{Name: "users", Help: "credentials file (lw auth add-user): " + who + " log in with SCRAM"}, "credentials_file") + } + userParams = []PresetParam{ {Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"}, - {Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, + mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file"), } ) +// tlsParam is a listener's tls key: a tlsModes name, off only where the +// listener may serve plain text +func tlsParam(def string, off bool) PresetParam { + p := PresetParam{Name: "tls", Default: def, Help: "a certificate made at startup (self-signed, or from issuer_*), or files"} + if off { + p.Help = "plain text, " + p.Help + } + for _, m := range tlsModes { + if m.name != "off" || off { + p.Enum = append(p.Enum, m.name) + } + } + return p +} + +// The listeners' tls keys: the aggregator always serves TLS, as edges send +// credentials and logs +var serveTLS, aggregatorTLS = tlsParam("off", true), tlsParam("self", false) + var presets = []Preset{ {"pipe", "Standard input to standard output, line for line: lw without a file", []PresetParam{formatParam("raw")}, @@ -90,11 +155,11 @@ var presets = []Preset{ {"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream", slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"), {Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"}, - {Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"}, - {Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"}, + serveTLS, + usersParam("readers"), }, userParams, []PresetParam{ - {Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, - {Name: "viewer", Default: "false", Help: "true: the login page and viewer, behind proxy; without user or users the viewer is always on"}, + mirror[AuthOptions](PresetParam{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, "trusted_proxies"), + mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "the login page and viewer, behind proxy; without user or users the viewer is always on"}, "viewer_page"), }, aclParams, tlsParams), func(p *PipelineConfig, v map[string]string, isDir IsDir) error { sources, err := pathOrStdin(v, isDir) @@ -102,7 +167,7 @@ var presets = []Preset{ return err } p.PluginSources = sources - sink, err := listener(v, "off") + sink, err := listener(v, serveTLS) if err != nil { return err } @@ -139,12 +204,12 @@ var presets = []Preset{ slices.Concat([]PresetParam{pathParam, fromParam, {Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"}, transportParam, - {Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, + mirror[TLSOptions](PresetParam{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, "ca_file"), {Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"}, {Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"}, }, userParams, []PresetParam{ - {Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, - {Name: "key", Help: "client key file"}, + mirror[TLSOptions](PresetParam{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, "cert_file"), + mirror[TLSOptions](PresetParam{Name: "key", Help: "client key file"}, "key_file"), {Name: "node", Help: "origin label; default: this host's name"}, }), func(p *PipelineConfig, v map[string]string, isDir IsDir) error { @@ -188,11 +253,11 @@ var presets = []Preset{ slices.Concat([]PresetParam{ {Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"}, transportParam, - {Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"}, - {Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"}, + aggregatorTLS, + usersParam("edges"), }, userParams, []PresetParam{ - {Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, - {Name: "out", Help: "directory for the received entries; default: standard output"}, + mirror[TLSOptions](PresetParam{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, "client_ca_file"), + mirror[FileSinkOptions](PresetParam{Name: "out", Help: "directory for the received entries; default: standard output"}, "directory"), formatParam("json"), }, aclParams, tlsParams), func(p *PipelineConfig, v map[string]string, isDir IsDir) error { @@ -200,10 +265,7 @@ var presets = []Preset{ if err != nil { return err } - if v["tls"] == "off" { - return fmt.Errorf("tls=off: edges send credentials and logs, the aggregator always serves TLS") - } - src, err := listener(v, "self") + src, err := listener(v, aggregatorTLS) if err != nil { return err } @@ -288,13 +350,10 @@ func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error) if path == "" { return pipeDefault("").PluginSources, nil } - dir, err := strings.HasSuffix(path, "/"), error(nil) - switch { - case isDir != nil: - dir, err = isDir(path) - case !dir && !strings.ContainsAny(path, `*?[\`): - err = ErrPathKind + if isDir == nil { + isDir = Answers(nil) } + dir, err := isDir(path) if err != nil { return nil, err } @@ -308,20 +367,23 @@ func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error) // tlsModes is what each preset tls mode sets, and the preset keys it needs // by the tls option each fills; made marks a certificate made at startup, // which takes hosts. -var tlsModes = map[string]struct { +var tlsModes = []tlsMode{ + {name: "off"}, + {name: "self", set: map[string]any{"self_signed": true}, made: true}, + {name: "issuer", needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true}, + {name: "files", needs: map[string]string{"cert": "cert_file", "key": "key_file"}}, +} + +type tlsMode struct { + name string set map[string]any needs map[string]string made bool -}{ - "off": {}, - "self": {set: map[string]any{"self_signed": true}, made: true}, - "issuer": {needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true}, - "files": {needs: map[string]string{"cert": "cert_file", "key": "key_file"}}, } -// listener starts a listening plugin's options: host, port, acl and tls, -// whose mode decides which certificate keys apply. -func listener(v map[string]string, def string) (map[string]any, error) { +// listener starts a listening plugin's options: host, port, acl and tls, in +// a mode its key offers, which decides the certificate keys that apply +func listener(v map[string]string, key PresetParam) (map[string]any, error) { host, port, err := splitAddr("listen", v["listen"]) if err != nil { return nil, err @@ -335,11 +397,12 @@ func listener(v map[string]string, def string) (map[string]any, error) { if len(acl) > 0 { opts["acl"] = acl } - mode := cmp.Or(v["tls"], def) - m, ok := tlsModes[mode] - if !ok { - return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(slices.Sorted(maps.Keys(tlsModes)), ", ")) + mode := cmp.Or(v["tls"], key.Default) + i := slices.IndexFunc(tlsModes, func(m tlsMode) bool { return m.name == mode }) + if i < 0 || !slices.Contains(key.Enum, mode) { + return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(key.Enum, ", ")) } + m := tlsModes[i] for _, other := range tlsModes { for k := range other.needs { if _, needed := m.needs[k]; needed && v[k] == "" { @@ -392,13 +455,25 @@ func splitAddr(key, addr string) (string, int64, error) { return host, int64(port), nil } +// chainType is a transport's chain plugin type, both a source and a sink func chainType(transport string) (string, error) { - if transport != "tcp" && transport != "http" { - return "", fmt.Errorf("transport %q (valid: tcp, http)", transport) + if !slices.Contains(transports(), transport) { + return "", fmt.Errorf("transport %q (valid: %s)", transport, strings.Join(transports(), ", ")) } return transport + "_chain", nil } +// transports name the chain plugins: the catalogue's TRANSPORT_chain types +func transports() []string { + var out []string + for _, p := range plugins { + if t, ok := strings.CutSuffix(p.Type, "_chain"); ok && p.Role == "sink" { + out = append(out, t) + } + } + return out +} + func list(s string) []any { var out []any for e := range strings.SplitSeq(s, ",") { diff --git a/internal/config/preset_test.go b/internal/config/preset_test.go index 72ed093..6c4b5b7 100644 --- a/internal/config/preset_test.go +++ b/internal/config/preset_test.go @@ -3,6 +3,7 @@ package config import ( "errors" "reflect" + "slices" "strings" "testing" ) @@ -56,7 +57,7 @@ func TestPresetKeysAreChecked(t *testing.T) { "edge:to=agg:9000": "it never sends unauthenticated", "edge:to=agg:9000,password_file=p": "password_file needs user", "edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`, - "aggregator:users=u,tls=off": "the aggregator always serves TLS", + "aggregator:users=u,tls=off": `tls "off" (valid: `, "aggregator:user=u,users=f": "user or users, not both", "aggregator": "it never receives unauthenticated", } { @@ -67,6 +68,49 @@ func TestPresetKeysAreChecked(t *testing.T) { } } +// A preset key that sets an option takes its choices and hint, a switch true +// or false; a listener's tls offers off only where it may serve plain text +func TestPresetKeysTakeTheirOptionsChoices(t *testing.T) { + param := func(preset, key string) PresetParam { + ps := Presets() + p := ps[slices.IndexFunc(ps, func(p Preset) bool { return p.Name == preset })] + return p.Params[slices.IndexFunc(p.Params, func(k PresetParam) bool { return k.Name == key })] + } + for _, c := range []struct { + preset, key string + table reflect.Type + option string + }{ + {"tail", "from", reflect.TypeFor[FileSourceOptions](), "from"}, + {"pipe", "format", reflect.TypeFor[FormatConfig](), "type"}, + {"serve", "proxy", reflect.TypeFor[AuthOptions](), "trusted_proxies"}, + {"aggregator", "users", reflect.TypeFor[AuthOptions](), "credentials_file"}, + } { + opts := options(c.table) + o := opts[slices.IndexFunc(opts, func(o option) bool { return o.Name == c.option })] + if p := param(c.preset, c.key); !slices.Equal(p.Enum, o.Enum) || p.Hint != o.Hint || len(o.Enum) == 0 && o.Hint == "" { + t.Errorf("%s %s: enum %v hint %q, want %s's %v %q", c.preset, c.key, p.Enum, p.Hint, c.option, o.Enum, o.Hint) + } + } + var modes []string + for _, m := range tlsModes { + modes = append(modes, m.name) + } + for _, c := range []struct { + preset, key string + enum []string + }{ + {"serve", "viewer", []string{"true", "false"}}, + {"edge", "transport", transports()}, + {"serve", "tls", modes}, + {"aggregator", "tls", slices.DeleteFunc(slices.Clone(modes), func(m string) bool { return m == "off" })}, + } { + if p := param(c.preset, c.key); !slices.Equal(p.Enum, c.enum) { + t.Errorf("%s %s: enum %v, want %v", c.preset, c.key, p.Enum, c.enum) + } + } +} + // A preset user without password_file is asked for on the terminal func TestPresetUserWithoutPasswordFileAsks(t *testing.T) { p, err := ExpandPreset("edge", map[string]string{"to": "agg:9000", "user": "u"}, nil) @@ -78,16 +122,19 @@ func TestPresetUserWithoutPasswordFileAsks(t *testing.T) { } } -// A preset path is what isDir answers, a directory named with brackets too; -// without isDir, off the target host, it is a directory when it ends in '/', -// a pattern when it holds a glob, and otherwise ErrPathKind asks +// A preset path is what isDir answers, a directory named with brackets too, +// and off the target host the user's answer, a single file too; without one +// it is a directory when it ends in '/', a pattern when it holds a glob, and +// otherwise ErrPathKind asks func TestPresetPathKind(t *testing.T) { yes := func(string) (bool, error) { return true, nil } + file := Answers(map[string]bool{"/srv/app.log": false}) for _, c := range []struct { path string isDir IsDir want string - }{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""}} { + }{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""}, + {"/srv/app.log", file, "/srv app.log"}, {"/srv/app", file, ""}, {"/srv/app/", file, "/srv/app/ *"}} { p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir) if c.want == "" { if !errors.Is(err, ErrPathKind) { diff --git a/internal/config/schema.go b/internal/config/schema.go index 6e5f0c1..8c0b1ff 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "maps" "math" "reflect" "slices" @@ -57,8 +58,25 @@ var plugins = []Plugin{ // Plugins lists the catalogue func Plugins() []Plugin { return slices.Clone(plugins) } -// Keys describes the plugin's options -func (p Plugin) Keys() []Key { return keys(p.options) } +// Keys describes the plugin's options, its tables holding the keys of its +// side, and HTTP's keys only when it speaks HTTP +func (p Plugin) Keys() []Key { return p.applying(keys(p.options)) } + +func (p Plugin) applying(all []Key) []Key { + side, out := "listener", []Key{} + if p.Side == Dialer { + side = "dialer" + } + for _, k := range all { + if (k.Side == "" || k.Side == side) && (!k.HTTP || p.HTTP) { + if k.Keys != nil { + k.Keys = p.applying(k.Keys) + } + out = append(out, k) + } + } + return out +} // LookupPlugin finds the catalogue row of a source or sink type func LookupPlugin(role, typ string) (Plugin, bool) { @@ -82,7 +100,8 @@ func (e *KeyError) Unwrap() error { return e.Err } // option is one key of an options table, from its field's tags: toml names // it, default: holds the default as typed on a command line, help: says what // it does, and lw: lists its rules: required, enum=a|b, min=N, max=N, -// zero=MEANING (zero is a value, not the default), hint=KIND, listener or dialer. +// zero=MEANING (zero is a value, not the default), hint=KIND, listener or +// dialer, and http (only plugins that speak HTTP take it). type option struct { Key index int @@ -107,6 +126,8 @@ func options(t reflect.Type) []option { o.Required = true case "listener", "dialer": o.Side = name + case "http": + o.HTTP = true case "enum": o.Enum = strings.Split(value, "|") case "min", "max": @@ -133,7 +154,7 @@ func options(t reflect.Type) []option { } // Key describes one option, for lw --schema and the composer: its tags, and -// its kind, a table's being "table" (tls, auth or acl) +// its kind, a table's being "table" (tls, auth or acl) with the table's keys type Key struct { Name string `json:"key"` Kind string `json:"kind"` @@ -146,17 +167,21 @@ type Key struct { Zero string `json:"zero,omitempty"` Hint string `json:"hint,omitempty"` Side string `json:"side,omitempty"` + HTTP bool `json:"http,omitempty"` + Keys []Key `json:"keys,omitempty"` } var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer", reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"} // Stage is a flow stage: its key in the flow table, whether it is a list (the -// filters), and its options +// filters), its options, and the values that turn it on, which a file's +// defaults leave off type Stage struct { - Key string `json:"key"` - List bool `json:"list,omitempty"` - Keys []Key `json:"keys"` + Key string `json:"key"` + List bool `json:"list,omitempty"` + Keys []Key `json:"keys"` + On map[string]any `json:"on,omitempty"` } // FlowStages lists the flow table's stages, in the order entries pass them @@ -165,20 +190,20 @@ func FlowStages() []Stage { t := reflect.TypeFor[FlowConfig]() for _, o := range options(t) { ft := t.Field(o.index).Type - out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem())}) + out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])}) } return out } -// KeysOf describes the options of a table: tls, auth or acl -func KeysOf[T any]() []Key { return keys(reflect.TypeFor[T]()) } - func keys(t reflect.Type) []Key { - var out []Key + out := []Key{} for _, o := range options(t) { if o.Kind == "" { panic(fmt.Sprintf("%s.%s: no kind", t.Name(), o.Name)) } + if ft := t.Field(o.index).Type; ft.Kind() == reflect.Pointer { + o.Keys = keys(ft.Elem()) + } out = append(out, o.Key) } return out diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index e27478d..bda93ad 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -3,6 +3,7 @@ package config import ( "math" "reflect" + "slices" "strings" "testing" ) @@ -79,7 +80,8 @@ func TestCoerceGivesTheDeclaredKinds(t *testing.T) { } // ValidateConfig decodes every plugin and flow stage without opening a file, -// and names the path of what it refuses +// and names the path of what it refuses: a rule between a table's keys by +// the table's func TestValidateConfigNamesThePath(t *testing.T) { valid := func() *Config { return &Config{Color: "auto", Pipelines: []PipelineConfig{{ @@ -97,9 +99,13 @@ func TestValidateConfigNamesThePath(t *testing.T) { "pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000": func(c *Config) { c.Pipelines[0].PluginSinks[0].Config["port"] = int64(70000) }, - "pipelines[0].plugin_sinks[out].config: tls: client_auth requires client_ca_file": func(c *Config) { + "pipelines[0].plugin_sinks[out].config.tls: client_auth requires client_ca_file": func(c *Config) { c.Pipelines[0].PluginSinks[0].Config["tls"].(map[string]any)["client_auth"] = true }, + "pipelines[0].plugin_sinks[out].config.acl: requests_per_second_per_client applies only to HTTP listeners": func(c *Config) { + c.Pipelines[0].PluginSinks[0].Type = "tcp" + c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"requests_per_second_per_client": 2.5} + }, "pipelines[0].plugin_sinks[out]: duplicate id": func(c *Config) { c.Pipelines[0].PluginSinks = append(c.Pipelines[0].PluginSinks, PluginSinkConfig{ID: "out", Type: "null"}) }, @@ -109,7 +115,7 @@ func TestValidateConfigNamesThePath(t *testing.T) { "pipelines[0].plugin_sinks[out].config.port: not an integer": func(c *Config) { c.Pipelines[0].PluginSinks[0].Config["port"] = "http" }, - `pipelines[0].plugin_sinks[up].config: auth: type "mtls" cannot be used with tls.insecure_skip_verify`: func(c *Config) { + `pipelines[0].plugin_sinks[up].config.auth: type "mtls" cannot be used with tls.insecure_skip_verify`: func(c *Config) { c.Pipelines[0].PluginSinks[0] = PluginSinkConfig{ID: "up", Type: "tcp_chain", Config: map[string]any{"host": "h", "port": int64(1), "tls": map[string]any{"enabled": true, "insecure_skip_verify": true}, "auth": map[string]any{"type": "mtls"}}} }, @@ -133,3 +139,52 @@ func TestValidateConfigNamesThePath(t *testing.T) { } } } + +// A configuration that does not load names every problem, a line each, each +// part's first: the settings, each flow stage, each plugin, each pipeline +func TestLoadReportsEveryProblemALine(t *testing.T) { + isolateConfig(t) + _, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file", + "sink", "tcp:port=70000", "rate-limit", "policy=bogus", "filter", "include:patterns=(", + "pipeline", "a", "sink", "null")}) + want := strings.Join([]string{"invalid configuration:", + `color: must be one of auto, always, never, got "bogus"`, + `pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "bogus"`, + "pipelines[0].flow.filters[0].patterns[0]: \"(\": error parsing regexp: missing closing ): `(`", + "pipelines[0].plugin_sources[file].config.directory: must be set", + "pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000", + `pipelines[1].name: duplicate name "a"`, + }, "\n") + if err == nil || err.Error() != want { + t.Fatalf("got\n%v\nwant\n%s", err, want) + } +} + +// A plugin lists the table keys that apply to it: those of its side, a chain +// source's being a listener's, and the HTTP ones only when it speaks HTTP +func TestTableKeysFollowThePlugin(t *testing.T) { + has := func(role, typ, path string) bool { + p, _ := LookupPlugin(role, typ) + keys := p.Keys() + for part := range strings.SplitSeq(path, ".") { + i := slices.IndexFunc(keys, func(k Key) bool { return k.Name == part }) + if i < 0 { + return false + } + keys = keys[i].Keys + } + return true + } + for path, want := range map[string]bool{ + "sink tcp_chain tls.pin_sha256": true, "sink tcp_chain tls.self_signed": false, + "source tcp_chain tls.self_signed": true, "source tcp_chain tls.pin_sha256": false, + "source tcp_chain auth.node_binding": true, "sink tcp_chain auth.credentials_file": false, + "sink tcp acl.requests_per_second_per_client": false, "sink tcp auth.token_lifetime_ms": false, + "sink http acl.requests_per_second_per_client": true, "sink http auth.trusted_proxies": true, + } { + f := strings.Fields(path) + if has(f[0], f[1], f[2]) != want { + t.Errorf("%s: listed %v, want %v", path, !want, want) + } + } +} diff --git a/internal/config/spec.go b/internal/config/spec.go index 7e0cb95..abba5f4 100644 --- a/internal/config/spec.go +++ b/internal/config/spec.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "maps" - "os" "reflect" "slices" "strconv" @@ -183,12 +182,15 @@ func cliSpecs(in []Spec) ([]pipelineSpec, error) { return specs, nil } -// envPipelineSpecs reads the one-pipeline environment form. An empty variable -// counts as unset, so container templates may leave optional ones blank. -func envPipelineSpecs() []pipelineSpec { +// envPipelineSpecs reads the one-pipeline environment form from environ, +// NAME=value each. An empty variable counts as unset, so container templates +// may leave optional ones blank. +func envPipelineSpecs(environ []string) []pipelineSpec { var specs []pipelineSpec - if name := os.Getenv(specVariable("pipeline")); name != "" { - specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name}) + for _, kv := range environ { + if name, value, _ := strings.Cut(kv, "="); name == specVariable("pipeline") && value != "" { + specs = append(specs, pipelineSpec{name, nil, value}) + } } for i := range specKinds { kind := &specKinds[i] @@ -198,7 +200,7 @@ func envPipelineSpecs() []pipelineSpec { name, spec string } var found []numbered - for _, kv := range os.Environ() { + for _, kv := range environ { name, spec, _ := strings.Cut(kv, "=") suffix, ok := strings.CutPrefix(name, base+"_") n, err := strconv.ParseUint(suffix, 10, 64) @@ -216,6 +218,16 @@ func envPipelineSpecs() []pipelineSpec { return specs } +// EnvPipelines builds the pipeline of LOGWISP_ variables, NAME=value each, as +// Load does without pipeline flags +func EnvPipelines(environ []string, isDir IsDir) ([]PipelineConfig, error) { + specs := envPipelineSpecs(environ) + if len(specs) == 0 { + return nil, errors.New("no LOGWISP_ pipeline variables") + } + return buildPipelines(specs, isDir) +} + // SpecPipelines builds the pipelines of command-line specs, as Load does func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) { s, err := cliSpecs(specs) diff --git a/internal/config/validate.go b/internal/config/validate.go index fdca93f..6875e66 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -24,26 +24,20 @@ import ( // ValidateConfig checks a configuration without reading a file it names: // the settings, then each pipeline's flow stages, and its sources and sinks -// decoded in full, naming the path of a refused key. lw --check opens them. +// decoded in full. Its error joins every problem, one a part, each a +// *KeyError naming the path of the refused key. lw --check opens the files. func ValidateConfig(cfg *Config) error { if cfg == nil { return fmt.Errorf("config is nil") } - if err := checkTags(reflect.ValueOf(cfg).Elem(), ""); err != nil { - return err - } + errs := []error{checkTags(reflect.ValueOf(cfg).Elem(), "")} if l := cfg.Logging; l != nil { - if err := checkTags(reflect.ValueOf(l).Elem(), "logging."); err != nil { - return err - } + errs = append(errs, checkTags(reflect.ValueOf(l).Elem(), "logging.")) if l.Console != nil { - if err := checkTags(reflect.ValueOf(l.Console).Elem(), "logging.console."); err != nil { - return err - } + errs = append(errs, checkTags(reflect.ValueOf(l.Console).Elem(), "logging.console.")) } } - - return ValidatePipelines(cfg.Pipelines) + return errors.Join(append(errs, ValidatePipelines(cfg.Pipelines))...) } // ValidatePipelines is ValidateConfig's check of the pipelines @@ -51,97 +45,101 @@ func ValidatePipelines(pipelines []PipelineConfig) error { if len(pipelines) == 0 { return fmt.Errorf("no pipelines configured") } - // Reject duplicate pipeline names (service map is keyed by name) - names := make(map[string]struct{}, len(pipelines)) - for i, p := range pipelines { - if _, dup := names[p.Name]; dup { - return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name) - } - names[p.Name] = struct{}{} - } + var errs []error + at := func(path string, err error) { errs = append(errs, At(path, err)) } + names := map[string]bool{} // the service is keyed by name single := map[string]string{} // role and type of a single-instance plugin: its pipeline for i, p := range pipelines { - if err := lconfig.NonEmpty(p.Name); err != nil { - return fmt.Errorf("pipeline[%d].name: %w", i, err) + path := fmt.Sprintf("pipelines[%d]", i) + switch { + case strings.TrimSpace(p.Name) == "": + at(path+".name", errors.New("must be set")) + case names[p.Name]: + at(path+".name", fmt.Errorf("duplicate name %q", p.Name)) } + names[p.Name] = true if len(p.PluginSources) == 0 { - return fmt.Errorf("pipeline[%d]: no sources defined", i) + at(path+".plugin_sources", errors.New("no sources defined")) } if len(p.PluginSinks) == 0 { - return fmt.Errorf("pipeline[%d]: no sinks defined", i) + at(path+".plugin_sinks", errors.New("no sinks defined")) } - if err := checkFlow(p.Flow, fmt.Sprintf("pipelines[%d].flow", i)); err != nil { - return err + if err := checkFlow(p.Flow, path+".flow"); err != nil { + errs = append(errs, err) } ids := map[string]bool{} - check := func(role, list, id, typ string, m map[string]any) error { - path := fmt.Sprintf("pipelines[%d].%s[%s]", i, list, id) + check := func(role, list, id, typ string, m map[string]any) { + path := fmt.Sprintf("%s.%s[%s]", path, list, id) plugin, ok := LookupPlugin(role, typ) switch { case ids[list+id]: - return fmt.Errorf("%s: duplicate id", path) + at(path, errors.New("duplicate id")) case !ok: - return fmt.Errorf("%s.type: unknown %s type %q", path, role, typ) + at(path+".type", fmt.Errorf("unknown %s type %q", role, typ)) case plugin.Single && single[role+typ] != "": - return fmt.Errorf("pipeline %q: one %s %s may run, and pipeline %q has it", p.Name, typ, role, single[role+typ]) + at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it", typ, role, single[role+typ])) + default: + ids[list+id] = true + if plugin.Single { + single[role+typ] = p.Name + } + if _, err := decode(role, typ, m); err != nil { + at(path+".config", err) + } } - ids[list+id] = true - if plugin.Single { - single[role+typ] = p.Name - } - if _, err := decode(role, typ, m); err != nil { - return At(path+".config", err) - } - return nil } for _, s := range p.PluginSources { - if err := check("source", "plugin_sources", s.ID, s.Type, s.Config); err != nil { - return err - } + check("source", "plugin_sources", s.ID, s.Type, s.Config) } for _, s := range p.PluginSinks { - if err := check("sink", "plugin_sinks", s.ID, s.Type, s.Config); err != nil { - return err - } + check("sink", "plugin_sinks", s.ID, s.Type, s.Config) } } - - return nil + return errors.Join(errs...) } // checkFlow settles copies of a pipeline's flow stages, which keep their -// defaults for the dump +// defaults for the dump: a problem a stage func checkFlow(f *FlowConfig, path string) error { if f == nil { return nil } - stages := map[string]any{} - if f.Heartbeat != nil { - stages["heartbeat"] = new(*f.Heartbeat) - } - if f.RateLimit != nil { - stages["rate_limit"] = new(*f.RateLimit) - } - if f.Format != nil { - stages["format"] = new(*f.Format) - } - for i, filter := range f.Filters { - stages[fmt.Sprintf("filters[%d]", i)] = &filter - } - for _, key := range slices.Sorted(maps.Keys(stages)) { - if err := Settle(stages[key]); err != nil { - return At(path+"."+key, err) + var errs []error + settle := func(key string, stage any) { + if err := Settle(stage); err != nil { + errs = append(errs, At(path+"."+key, err)) } } - return nil + if f.RateLimit != nil { + settle("rate_limit", new(*f.RateLimit)) + } + for i, filter := range f.Filters { + settle(fmt.Sprintf("filters[%d]", i), &filter) + } + if f.Format != nil { + settle("format", new(*f.Format)) + } + if f.Heartbeat != nil { + settle("heartbeat", new(*f.Heartbeat)) + } + return errors.Join(errs...) } -// At prefixes err with the path of the table it was found in +// At addresses err to the path of the table it was found in func At(path string, err error) error { if ke, ok := errors.AsType[*KeyError](err); ok { - return fmt.Errorf("%s.%s: %w", path, ke.Key, ke.Err) + return &KeyError{path + "." + ke.Key, ke.Err} } - return fmt.Errorf("%s: %w", path, err) + return &KeyError{path, err} +} + +// table addresses a rule between a table's keys, which its Check names by +// a "tls: " prefix, to the table +func table(key string, err error) error { + if _, ok := errors.AsType[*KeyError](err); err == nil || ok { + return err + } + return &KeyError{key, errors.New(strings.TrimPrefix(err.Error(), key+": "))} } // networked is the options of a plugin that listens or dials @@ -175,12 +173,12 @@ func (p Plugin) checkNetwork(n networked) error { return &KeyError{"host", err} } if err := t.Check(p.Side != Dialer); err != nil { - return err + return table("tls", err) } if err := acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0); err != nil { - return err + return table("acl", err) } - return a.Check(p.Side, p.HTTP, t.State()) + return table("auth", a.Check(p.Side, p.HTTP, t.State())) } // Check holds the rules between tls keys on a listener or a dialer; a table @@ -311,9 +309,9 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error { return errors.New("auth: username and password_file go together") case o.CredentialsFile == "" && o.Username == "": return fmt.Errorf("auth: type %q requires credentials_file, or username and password_file", "scram") - case o.TokenLifetimeMS > 0 && !http: - return errors.New("auth: token_lifetime_ms applies only to HTTP listeners") - case len(o.TrustedProxies) > 0 && (side != Listener || !http): + case !http && httpKey(o) != "": + return fmt.Errorf("auth: %s applies only to HTTP listeners", httpKey(o)) + case len(o.TrustedProxies) > 0 && side != Listener: return errors.New("auth: trusted_proxies applies only to the http sink") case len(o.TrustedProxies) > 0 && o.Identity != "": return errors.New("auth: identity binds a client certificate, which a TLS-terminating proxy does not pass on; drop it or trusted_proxies") @@ -367,14 +365,13 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error { return err } header := o.ProxyProtocol == "optional" || o.ProxyProtocol == "required" - rps := o.RequestsPerSecondPerClient switch { case header && len(o.ProxyFrom) == 0: return fmt.Errorf("acl: proxy_protocol %s needs proxy_from, the proxies that may send the header", o.ProxyProtocol) case !header && len(o.ProxyFrom) > 0: return errors.New("acl: proxy_from needs proxy_protocol optional or required") - case rps > 0 && !http: - return errors.New("acl: requests_per_second_per_client applies only to HTTP listeners") + case !http && httpKey(o) != "": + return fmt.Errorf("acl: %s applies only to HTTP listeners", httpKey(o)) case o.MaxConnectionsPerClient > 0 && proxied: return errors.New("acl: max_connections_per_client counts connections, behind trusted_proxies the proxy's; cap clients at the proxy") } @@ -399,6 +396,17 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error { return nil } +// httpKey names the first key of table v tagged http that is set +func httpKey(v any) string { + rv := reflect.ValueOf(v).Elem() + for _, o := range options(rv.Type()) { + if o.HTTP && !rv.Field(o.index).IsZero() { + return o.Name + } + } + return "" +} + // ParsePrefix reads an address or CIDR of network's family ("tcp": either) func ParsePrefix(entry, network string) (netip.Prefix, error) { entry = strings.TrimSpace(entry) @@ -575,13 +583,33 @@ func checkFileKeys(path string) error { if err != nil { return err } - root, err := toml.NewParser(data).Parse() + root, err := fileTable(data) if err != nil { return err } + return checkKeys(root, reflect.TypeOf(Config{}), "") +} + +// ParseFile reads a configuration file's text, refusing the keys lw refuses +// there, and opens nothing it names +func ParseFile(data []byte) (*Config, error) { + root, err := fileTable(data) + if err != nil { + return nil, err + } + cfg := &Config{} + return cfg, Scan(root, cfg) +} + +// fileTable is a file's root table without the keys lw ignores there +func fileTable(data []byte) (map[string]any, error) { + root, err := toml.NewParser(data).Parse() + if err != nil { + return nil, err + } delete(root, "config_file") // runtime metadata, documented as ignored in the file for _, key := range []string{"version", "check", "dump"} { delete(root, key) // command-line switches, which earlier lw --dump output wrote } - return checkKeys(root, reflect.TypeOf(Config{}), "") + return root, nil } diff --git a/internal/tui/canvas.go b/internal/tui/canvas.go index c51f088..35258f3 100644 --- a/internal/tui/canvas.go +++ b/internal/tui/canvas.go @@ -2,7 +2,6 @@ package tui import ( "fmt" - "regexp" "slices" "strconv" "strings" @@ -164,7 +163,7 @@ func join(g ui.Wires, l layout, col int) { // drawNode draws a source or sink: a tinted bar, its type and whether it // listens or dials, then a summary of its options. The chosen node's bar is -// solid; a node the first problem names has its bar in the error style. +// solid; a node a problem names has its bar in the error style. func (a *app) drawNode(r ui.Region, s spot, n compose.Node, tint ui.Style, chosen bool) { th := a.th bg, bar := th.Text, a.font.bar @@ -295,41 +294,15 @@ func show(v any) string { return fmt.Sprint(v) } -// problemPath reads the part a validation error names: its pipeline, node and -// key, as config's key paths write them -var problemPath = regexp.MustCompile(`^pipelines?\[(\d+)\](?:\.(plugin_sources|plugin_sinks)\[([^\]]*)\](?:\.config\.?([\w.]*))?|\.flow\.(\w+)(?:\[(\d+)\])?\.?([\w.]*))?`) - -// locate reads a problem's part, and its text past the path -func locate(err error) (pi int, n compose.Node, key, text string, ok bool) { - at := problemPath.FindStringSubmatchIndex(err.Error()) - if at == nil { - return 0, n, "", err.Error(), false - } - m := problemPath.FindStringSubmatch(err.Error()) - text = strings.TrimPrefix(err.Error()[at[1]:], ": ") - pi, _ = strconv.Atoi(m[1]) - switch { - case m[2] != "": - n = compose.Node{Role: map[string]string{"plugin_sources": "source", "plugin_sinks": "sink"}[m[2]], ID: m[3]} - key = m[4] - case m[5] != "": - n = compose.Node{Role: m[5]} - if m[6] != "" { - n.Index, _ = strconv.Atoi(m[6]) - } - key = m[7] - } - return pi, n, key, text, true -} - -// faulty reports whether the first problem names this node, or a filter -// when n is the filters stage +// faulty reports whether a problem names this node, or a filter when n is +// the filters stage func (a *app) faulty(n compose.Node) bool { - if a.problem == nil { - return false + for _, p := range compose.Problems(a.problem) { + if pi, m, _, _, ok := compose.Locate(p); ok && pi == a.pi && m.Role == n.Role && m.ID == n.ID { + return true + } } - pi, m, _, _, ok := locate(a.problem) - return ok && pi == a.pi && m.Role == n.Role && m.ID == n.ID + return false } // where names a problem's part as the screen shows it: its pipeline when diff --git a/internal/tui/dialog.go b/internal/tui/dialog.go index 6b93400..b8746b8 100644 --- a/internal/tui/dialog.go +++ b/internal/tui/dialog.go @@ -265,7 +265,7 @@ func (f *form) draw(a *app, r ui.Region) { } summary, noteRows := wrap(f.summary, w, 2), len(wrap(f.err, w, 3)) for _, p := range f.params { - noteRows = max(noteRows, len(wrap(p.Help, w, 3))) + noteRows = max(noteRows, len(wrap(p.Usage(), w, 3))) } top, bottom := len(summary)+min(1, len(summary)), noteRows+min(1, noteRows) in := a.frame(r, w, top+len(f.fields)*tall+bottom, f.title) @@ -281,7 +281,7 @@ func (f *form) draw(a *app, r ui.Region) { v.TextInput(f.fields[i], p.Default, i == f.focus.Index, th) } }) - note, style := f.params[f.focus.Index].Help, th.Muted + note, style := f.params[f.focus.Index].Usage(), th.Muted if f.err != "" { note, style = f.err, th.Error } @@ -456,7 +456,7 @@ func help(f font) dialog { {"p", "start over from a preset"}, {"r", "run the pipelines"}, {"o", "run, or print a command line, environment or file"}, - {"!", "the first problem"}, + {"!", "every problem, moving to the first"}, {"paste", "an lw command line replaces the pipelines"}, {"q", "quit"}, } diff --git a/internal/tui/inspector.go b/internal/tui/inspector.go index db8caf9..ae6af79 100644 --- a/internal/tui/inspector.go +++ b/internal/tui/inspector.go @@ -69,8 +69,7 @@ func (a *app) keyLines(out []line, n compose.Node, keys []config.Key, prefix str case "table": out = append(out, line{node: n, key: k, depth: depth, group: name, item: -1}) if a.insp.open[a.foldKey(n, k.Name)] { - p, _ := a.plugin(n) - out = a.keyLines(out, n, sided(a.tables[name], p.Side), k.Name+".", depth+1) + out = a.keyLines(out, n, k.Keys, k.Name+".", depth+1) } case "list": out = append(out, line{node: n, key: k, depth: depth, group: name, item: -1}) @@ -180,10 +179,10 @@ func (a *app) nodeTitle(n compose.Node) string { return n.Role + " · flow" } -// help is the focused row's problem, or its key's help and default +// help is the focused row's first problem, or its key's help and default func (a *app) help(l line) (string, ui.Style) { - if a.problem != nil { - if pi, n, key, text, ok := locate(a.problem); ok && pi == a.pi && n == l.node && key == l.key.Name { + for _, p := range compose.Problems(a.problem) { + if pi, n, key, text, ok := compose.Locate(p); ok && pi == a.pi && n == l.node && key == l.key.Name { return text, a.th.Error } } @@ -264,7 +263,7 @@ func (a *app) groupSummary(l line) string { return show(v) case map[string]any: var out []string - for _, k := range a.tables[l.group] { + for _, k := range l.key.Keys { if s, ok := v[k.Name]; ok { out = append(out, k.Name+" "+show(s)) } diff --git a/internal/tui/tui.go b/internal/tui/tui.go index bf4fe52..17f4504 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -85,7 +85,6 @@ type app struct { look look th ui.Theme font font - tables map[string][]config.Key pi int // the pipeline shown col int // the chosen column row [3]int // the chosen node of each column @@ -96,14 +95,14 @@ type app struct { dialog dialog // takes every key while open status string // the last edit's refusal note string // what the last action did - problem error // the composition's first problem + problem error // the composition's problems, joined changed bool done bool result Result } func newApp(c *compose.Composition, l look, f font) *app { - a := &app{comp: c, look: l, th: l.Theme, font: f, tables: compose.NewSchema().Tables} + a := &app{comp: c, look: l, th: l.Theme, font: f} a.th.Glyphs = f.Glyphs a.insp.open = map[string]bool{} if len(c.Pipelines) == 0 { @@ -167,10 +166,10 @@ func (a *app) plugin(n compose.Node) (config.Plugin, bool) { return config.LookupPlugin(n.Role, typ) } -// keys are the options a node takes: a plugin's of its side, a stage's +// keys are the options a node takes: a plugin's, a stage's func (a *app) keys(n compose.Node) []config.Key { if p, ok := a.plugin(n); ok { - return sided(p.Keys(), p.Side) + return p.Keys() } for _, s := range config.FlowStages() { if s.Key == n.Role { @@ -180,18 +179,6 @@ func (a *app) keys(n compose.Node) []config.Key { return nil } -// sided drops the keys of the other network side -func sided(keys []config.Key, side config.Side) []config.Key { - name := map[config.Side]string{config.Listener: "listener", config.ChainListener: "listener", config.Dialer: "dialer"}[side] - var out []config.Key - for _, k := range keys { - if k.Side == "" || k.Side == name { - out = append(out, k) - } - } - return out -} - // edit applies one engine edit: a refusal is the status, a change revalidates func (a *app) edit(err error) bool { if err != nil { @@ -418,13 +405,13 @@ func (a *app) quit() { a.dialog = &confirm{question: "Quit without running or printing the pipelines?", yes: func(a *app) { a.done = true }} } -// showProblem chooses the part the first problem names and shows it whole +// showProblem chooses the part the first problem names and shows them all func (a *app) showProblem() { if a.problem == nil { a.note = "the pipelines are valid" return } - if pi, n, key, _, ok := locate(a.problem); ok && pi < len(a.comp.Pipelines) { + if pi, n, key, _, ok := compose.Locate(compose.Problems(a.problem)[0]); ok && pi < len(a.comp.Pipelines) { a.pi = pi a.choose(n) a.inspectAt(func(l line) bool { return l.node == n && l.key.Name == key }) @@ -539,7 +526,8 @@ func (a *app) paint(cells []terminal.Cell, w, h int) { } // bottom is the status rows, wrapped to three at most: an edit's refusal, a -// note, the first problem by its part, or the keys while all is valid +// note, the first problem by its part and how many follow, or the keys while +// all is valid func (a *app) bottom(w int) ([]string, ui.Style) { th, f := a.th, a.font text, style := "", th.Error @@ -549,10 +537,14 @@ func (a *app) bottom(w int) ([]string, ui.Style) { case a.note != "": text, style = a.note, th.Muted case a.problem != nil: - pi, n, key, msg, _ := locate(a.problem) + problems := compose.Problems(a.problem) + pi, n, key, msg, _ := compose.Locate(problems[0]) if where := a.where(pi, n, key); where != "" { msg = where + ": " + msg } + if len(problems) > 1 { + msg += fmt.Sprintf(" (%d more: ! shows them)", len(problems)-1) + } text = f.invalid + " " + msg default: return []string{f.valid + " valid"}, th.Accent diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index 87e07b7..04ac1bd 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -399,18 +399,28 @@ func TestPasteReplacesThePipelines(t *testing.T) { } } -// The first problem marks the part it names, and ! moves there, to the key +// Every problem marks the part it names and its key's row, the status counts +// those after the first, and ! moves to the first, to the key func TestProblemNamesItsPart(t *testing.T) { - a := mono(pipeline(t, []string{"null"}, []string{"null", "tcp"})) + a := mono(pipeline(t, []string{"null"}, []string{"null", "tcp", "http"})) sink := compose.Node{Role: "sink", ID: "tcp"} - if !a.faulty(sink) || a.faulty(compose.Node{Role: "source", ID: "null"}) || a.faulty(compose.Node{Role: "sink", ID: "null"}) { + if !a.faulty(sink) || !a.faulty(compose.Node{Role: "sink", ID: "http"}) || + a.faulty(compose.Node{Role: "source", ID: "null"}) || a.faulty(compose.Node{Role: "sink", ID: "null"}) { t.Fatalf("problem %v", a.problem) } + if status, _ := a.bottom(200); !strings.Contains(strings.Join(status, " "), "sink tcp · port: must be set (1 more") { + t.Fatalf("status %q", status) + } press(a, "!") n, _ := a.node() if l := a.lines()[a.insp.cursor]; n != sink || !a.inspect || l.key.Name != "port" { t.Fatalf("on %+v, row %+v", n, l) } + a.choose(compose.Node{Role: "sink", ID: "http"}) + lines := a.lines() + if text, style := a.help(lines[slices.IndexFunc(lines, func(l line) bool { return l.key.Name == "port" })]); text != "must be set" || style != a.th.Error { + t.Fatalf("http's port row: %q", text) + } } // ! and c on the pipeline bar open the inspector, which then takes the keys: @@ -515,6 +525,19 @@ func presetForm(a *app, name string) *form { return f } +// A preset form's note shows the focused key's choices before its help +func TestPresetFormShowsTheChoices(t *testing.T) { + a := mono(pipeline(t, nil, nil)) + presetForm(a, "tail") + press(a, terminal.KeyBacktab) + ps := config.Presets() + tail := ps[slices.IndexFunc(ps, func(p config.Preset) bool { return p.Name == "tail" })].Params + format := tail[slices.IndexFunc(tail, func(p config.PresetParam) bool { return p.Name == "format" })] + if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.Contains(screen, format.Usage()) || len(format.Enum) == 0 { + t.Fatalf("format's choices not shown:\n%s", screen) + } +} + // A preset form is usable at 25 columns: labels stack above their values, // which take the row, so typed text shows whole func TestPresetFormFitsANarrowScreen(t *testing.T) {