v0.32.0 lwconf engine update: config by path, schema from code, wasm

AGENTS.md

@@ -80,10 +80,11 @@ change touches.
   refuses unknown keys at any depth; they keep only I/O. Rules that read
   nothing belong in a `Check` method, so `ValidateConfig` sees them offline.
 - `internal/compose` is the engine `lw --tui` and the website's WebAssembly
-  share: it edits pipelines and writes the command line, environment and file
+  share: it edits pipelines, writes the command line, environment and file
-  forms, each loading back to the same pipelines; it reads no file and links
+  forms and reads each back to the same pipelines; it reads no file and links
   no TLS, HTTP or terminal code (its test runs `go list -deps`). `cmd/lwconf`
-  hands it to the page as WebAssembly (`make wasm`; `node --test` there).
+  hands it to a page: `make wasm` writes the drop-in `bin/lwconf/`, which
+  `lwconf.js` loads and calls (`node --test` there).
 - `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands
   table whose groups (`auth`, `tls`, `preset`, `config`) share one flag
   mechanism; `config.Load` takes the parsed `config.Args`. Presets are rows of

Makefile

@@ -15,12 +15,13 @@ GIT_COMMIT ?= $(GIT_COMMIT_DEFAULT)
 BUILD_TIME_DEFAULT != f='+%Y-%m-%d_%H:%M:%S'; e="$${SOURCE_DATE_EPOCH:-}"; \
 	if [ -n "$$e" ]; then date -u -d "@$$e" "$$f" 2>/dev/null || date -u -r "$$e" "$$f"; else date -u "$$f"; fi
 BUILD_TIME ?= $(BUILD_TIME_DEFAULT)
-# The image stamps the commit time, so a rebuild of one commit stays cached
+# The image and the wasm stamp the commit time, so a rebuild of one commit
+# stays cached, and the website's engine keeps its checksums
 IMAGE_BUILD_TIME_DEFAULT != TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%d_%H:%M:%S 2>/dev/null || echo unknown
 IMAGE_BUILD_TIME ?= $(IMAGE_BUILD_TIME_DEFAULT)
-VERSION_LDFLAGS = -X 'github.com/lixenwraith/logwisp/internal/version.Version=$(VERSION)' \
+COMMIT_LDFLAGS = -X 'github.com/lixenwraith/logwisp/internal/version.Version=$(VERSION)' \
-	-X 'github.com/lixenwraith/logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \
+	-X 'github.com/lixenwraith/logwisp/internal/version.GitCommit=$(GIT_COMMIT)'
-	-X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(BUILD_TIME)'
+VERSION_LDFLAGS = $(COMMIT_LDFLAGS) -X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(BUILD_TIME)'
 # Not LDFLAGS/GOFLAGS: packagers export those for the C linker and go itself.
 GO_BUILDFLAGS ?=
 GO_LDFLAGS ?=
@@ -78,8 +79,8 @@ help:
 	@echo "  build        Build $(BIN_DIR)/$(BINARY) with version metadata"
 	@echo "  release      Build it static, stripped and trimmed (CGO_ENABLED=0)"
 	@echo "  dev          Build it with the race detector"
-	@echo "  wasm         Build $(BIN_DIR)/lwconf.wasm, the engine for the website, beside its"
+	@echo "  wasm         Write $(WASM_DIR)/, the engine a website drops in: lwconf.wasm, its"
-	@echo "               loader wasm_exec.js, and print both checksums"
+	@echo "               loader wasm_exec.js, lwconf.js, schema.json, and lwconf.json, their manifest"
 	@echo "  version      Print the version metadata a build embeds"
 	@echo "  completion   Regenerate $(PKG_DIR)/completion from lw's command tables"
 	@echo "  clean        Remove $(BIN_DIR)/"
@@ -123,11 +124,21 @@ dev: check-go
 	$(GO) build -race $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC)
 
 # The page loads the toolchain's own loader, which uses no eval: its CSP needs
-# only 'wasm-unsafe-eval'
+# only 'wasm-unsafe-eval'. schema.json is lw --schema stamped as the engine is,
+# so it equals the engine's schema(); lwconf.json records the build.
+WASM_DIR = $(BIN_DIR)/lwconf
+WASM_FILES = lwconf.wasm wasm_exec.js lwconf.js schema.json
+WASM_LDFLAGS = -s -w $(COMMIT_LDFLAGS) -X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(IMAGE_BUILD_TIME)' $(GO_LDFLAGS)
 wasm: check-go
-	GOOS=js GOARCH=wasm $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "-s -w $(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/lwconf.wasm ./cmd/lwconf
+	@rm -rf $(WASM_DIR) && mkdir -p $(WASM_DIR)
-	install -m 0644 "$$($(GO) env GOROOT)/lib/wasm/wasm_exec.js" $(BIN_DIR)/wasm_exec.js
+	GOOS=js GOARCH=wasm $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "$(WASM_LDFLAGS)" -o $(WASM_DIR)/lwconf.wasm ./cmd/lwconf
-	@cd $(BIN_DIR) && { sha256sum lwconf.wasm wasm_exec.js 2>/dev/null || sha256 -r lwconf.wasm wasm_exec.js; }
+	install -m 0644 "$$($(GO) env GOROOT)/lib/wasm/wasm_exec.js" cmd/lwconf/lwconf.js $(WASM_DIR)
+	$(GO) run -trimpath $(GO_BUILDFLAGS) -ldflags "$(WASM_LDFLAGS)" $(SRC) --schema > $(WASM_DIR)/schema.json
+	@set -eu; go=$$($(GO) env GOVERSION); cd $(WASM_DIR); \
+	sums=$$(sha256sum $(WASM_FILES) 2>/dev/null || sha256 -r $(WASM_FILES)); echo "$$sums"; \
+	{ printf '{\n  "version": "%s",\n  "commit": "%s",\n  "go": "%s",\n  "sha256": {' '$(VERSION)' '$(GIT_COMMIT)' "$$go"; \
+		echo "$$sums" | awk '{ sub(/^\*/, "", $$2); printf "%s\n    \"%s\": \"%s\"", (NR > 1 ? "," : ""), $$2, $$1 }'; \
+		printf '\n  }\n}\n'; } > lwconf.json
 
 test: check-go
 	$(GO) test ./...
@@ -157,7 +168,7 @@ verify: test
 	done
 	@echo "cross build js/wasm"; GOOS=js GOARCH=wasm $(GO) build -o /dev/null ./cmd/lwconf
 	@if command -v node >/dev/null 2>&1; then \
-		(cd $(WEB_DIR) && node --test) && cd cmd/lwconf && node --test; \
+		(cd $(WEB_DIR) && node --test) && cd cmd/lwconf && GO='$(GO)' node --test; \
 	else \
 		echo "node not found: web client and lwconf tests skipped"; \
 	fi

cmd/lw/bootstrap.go

@@ -62,7 +62,7 @@ func handleReload(ctx context.Context, newCfg *config.Config, oldSvc *service.Se
 	logger.Info("msg", "Starting configuration hot reload")
 
 	if err := config.ValidateConfig(newCfg); err != nil {
-		logger.Error("msg", "Invalid reload configuration, keeping old service running", "error", err)
+		logger.Error("msg", "Invalid reload configuration, keeping old service running", "error", oneLine(err))
 		return nil, nil, nil, err
 	}
 

cmd/lw/cli.go

@@ -291,7 +291,7 @@ func presetCommands() []subcommand {
 		s.define = func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 			values := map[string]*string{}
 			for _, k := range p.Params {
-				values[k.Name] = fs.String(k.Name, k.Default, k.Help)
+				values[k.Name] = fs.String(k.Name, k.Default, k.Usage())
 			}
 			return func(stdout, _ io.Writer) error {
 				set := map[string]string{}

cmd/lw/main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/signal"
 	"reflect"
+	"strings"
 	"syscall"
 
 	"github.com/lixenwraith/logwisp/internal/compose"
@@ -147,7 +148,7 @@ func main() {
 			next, err = manager.Snapshot()
 		}
 		if err != nil {
-			logger.Error("msg", "Failed to read reload configuration", "error", err, "action", "keeping current service")
+			logger.Error("msg", "Failed to read reload configuration", "error", oneLine(err), "action", "keeping current service")
 			return
 		}
 		// Watch events are hints and may be duplicated or concern startup-only
@@ -219,6 +220,10 @@ func readsStdin(cfg *config.Config) bool {
 	return false
 }
 
+// oneLine is an error for a log record, which is a line: the problems
+// validation joins, "; " between them
+func oneLine(err error) string { return strings.ReplaceAll(err.Error(), "\n", "; ") }
+
 // loadFailed reports a configuration that did not load and returns lw's exit
 // code: a named file that is missing is a usage error
 func loadFailed(err error) int {

cmd/lwconf/lwconf.js

@@ -0,0 +1,91 @@
+// LogWisp's configuration engine for a page: one dependency-free ES module
+// beside lwconf.wasm, which make wasm writes with it. It knows the engine's
+// calls and the shapes of its schema and pipelines, never a plugin, a key or
+// a rule, so the page draws whatever the engine it loads describes.
+
+// load starts the engine: the toolchain's wasm_exec.js by a script element
+// (outside a page the caller imports it first), the module fetched with its
+// integrity, then its main, which sets globalThis.lwconf
+export async function load({ execURL, execSRI, wasmURL, wasmSRI }) {
+  if (!globalThis.Go) await script(execURL, execSRI);
+  const go = new globalThis.Go();
+  const fetched = async () => {
+    const res = await fetch(wasmURL, { integrity: wasmSRI });
+    if (!res.ok) throw new Error(`${wasmURL}: ${res.status}`);
+    return res;
+  };
+  const { instance } = await WebAssembly.instantiateStreaming(fetched(), go.importObject)
+    .catch(async () => WebAssembly.instantiate(await (await fetched()).arrayBuffer(), go.importObject)); // a server's wrong type
+  go.run(instance);
+  if (!globalThis.lwconf) throw new Error("lwconf.wasm did not start");
+  return new Engine(globalThis.lwconf);
+}
+
+function script(src, integrity) {
+  return new Promise((resolve, reject) => {
+    const s = Object.assign(document.createElement("script"), { src, onload: resolve });
+    s.onerror = () => reject(new Error(`${src} did not load`));
+    if (integrity) s.integrity = integrity;
+    document.head.append(s);
+  });
+}
+
+// Engine's calls take objects or JSON text and return the engine's answer,
+// {value} or {error, path?, pipeline?, node?, key?}; dirs answers whether a
+// preset path is a directory, {"/var/log/app.log": false}
+export class Engine {
+  #api;
+  constructor(api) { this.#api = api; }
+  #call(name, ...args) {
+    return JSON.parse(this.#api[name](...args.map(a => (typeof a === "string" ? a : JSON.stringify(a)))));
+  }
+  schema() { return this.#call("schema"); }
+  preset(name, values, dirs = {}) { return this.#call("preset", name, values, dirs); }
+  parse(text, form, dirs = {}) { return this.#call("parse", text, form, dirs); }
+  validate(pipelines) { return this.#call("validate", pipelines); }
+  emit(pipelines, form) { return this.#call("emit", pipelines, form); }
+  edit(pipelines, op) { return this.#call("edit", pipelines, op); }
+}
+
+// fields are the rows a form draws for a node, by its role (and a source's
+// or sink's type): a value row a key, with what leaving it unset means, and a
+// group a table, holding its own rows; paths are dotted as edit takes them
+export function fields(schema, role, type) {
+  const plugin = schema[`${role}s`]?.find(p => p.type === type);
+  const rows = (keys, prefix) => keys.map(key => {
+    const path = prefix + key.key;
+    if (Array.isArray(key.keys)) return { path, key, rows: rows(key.keys, path + ".") };
+    return { path, key, placeholder: key.zero ? `(${key.zero})` : (key.default ?? key.hint ?? "") };
+  });
+  return rows(plugin?.keys ?? schema.flow.find(s => s.key === role)?.keys ?? [], "");
+}
+
+// value is a node's value at a dotted path, undefined while unset; a node is
+// {role, id, index} as the engine returns it, a list stage's by its index
+export function value(pipelines, pipeline, node, path = "") {
+  const p = pipelines[pipeline];
+  const list = { source: p?.plugin_sources, sink: p?.plugin_sinks }[node.role];
+  const stage = p?.flow?.[node.role];
+  let v = list ? list.find(s => s.id === node.id)?.config : Array.isArray(stage) ? stage[node.index] : stage;
+  for (const part of path ? path.split(".") : []) v = v?.[part];
+  return v;
+}
+
+// at names where problems lie: a pipeline's node, however value takes it, the
+// pipeline, or nowhere
+export function at(pipeline, node) {
+  if (pipeline === undefined) return "";
+  return node ? `${pipeline}/${node.role}/${node.id ?? ""}/${node.index ?? 0}` : `${pipeline}`;
+}
+
+// group sorts validate's problems by at(pipeline, node), then by key, "" for
+// the node or pipeline itself
+export function group(problems) {
+  const out = new Map();
+  for (const p of problems) {
+    const keys = out.get(at(p.pipeline, p.node)) ?? new Map();
+    keys.set(p.key ?? "", [...(keys.get(p.key ?? "") ?? []), p]);
+    out.set(at(p.pipeline, p.node), keys);
+  }
+  return out;
+}

cmd/lwconf/lwconf.test.mjs

@@ -2,63 +2,181 @@ import { after, test } from "node:test";
 import assert from "node:assert/strict";
 import { execFileSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { mkdtempSync, readFileSync, rmSync } from "node:fs";
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { basename, join } from "node:path";
 import { pathToFileURL } from "node:url";
 
-// Built by make wasm and started as the page starts it: the toolchain's loader
+// The drop-in directory make wasm writes over an earlier one's, served as a
-// beside the module, main running until the page closes. The make run is its
+// site serves it (?type= another content type) and loaded through lwconf.js;
-// own: a parent GNU make's flags would stop a BSD one.
+// wasm_exec.js is imported first, as a page's script element runs it. make runs
-const dir = mkdtempSync(join(tmpdir(), "lwconf-"));
+// on its own, with make verify's toolchain: a GNU make's flags stop a BSD one.
-after(() => rmSync(dir, { recursive: true, force: true }));
+const root = join(import.meta.dirname, "../..");
-const made = execFileSync("make", ["-s", "wasm", `BIN_DIR=${dir}`, "VERSION=v0.0.0-node"], {
+const tmp = mkdtempSync(join(tmpdir(), "lwconf-"));
-  cwd: join(import.meta.dirname, "../.."), encoding: "utf8",
+const dir = join(tmp, "lwconf");
-  env: { ...process.env, MAKEFLAGS: "", MFLAGS: "", MAKELEVEL: "" },
+const go = process.env.GO ?? "go";
+const env = { ...process.env, MAKEFLAGS: "", MFLAGS: "", MAKELEVEL: "" };
+const wasm = (bin, ...vars) => execFileSync("make", ["-s", "wasm", `BIN_DIR=${bin}`, "VERSION=v0.0.0-node", `GO=${go}`, ...vars],
+  { cwd: root, env, stdio: "ignore" });
+mkdirSync(dir);
+writeFileSync(join(dir, "stale.js"), "");
+wasm(tmp);
+const server = createServer((req, res) => {
+  const { pathname, searchParams } = new URL(req.url, "http://localhost");
+  const file = join(dir, basename(pathname));
+  if (!existsSync(file)) return res.writeHead(404).end();
+  res.setHeader("content-type", searchParams.get("type") ?? (file.endsWith(".wasm") ? "application/wasm" : "text/javascript"));
+  res.end(readFileSync(file));
 });
+await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
+after(() => { server.close(); rmSync(tmp, { recursive: true, force: true }); });
+const url = file => `http://127.0.0.1:${server.address().port}/${file}`;
+const sri = file => "sha256-" + createHash("sha256").update(readFileSync(join(dir, file))).digest("base64");
+
+const { load, fields, value, at, group } = await import(pathToFileURL(join(dir, "lwconf.js")));
 await import(pathToFileURL(join(dir, "wasm_exec.js")));
-const go = new globalThis.Go();
+const engine = await load({ wasmURL: url("lwconf.wasm"), wasmSRI: sri("lwconf.wasm") });
-const { instance } = await WebAssembly.instantiate(readFileSync(join(dir, "lwconf.wasm")), go.importObject);
+const schema = engine.schema().value;
-go.run(instance);
-const call = (name, ...args) => JSON.parse(globalThis.lwconf[name](...args));
 
-test("make wasm stamps the version and prints both checksums", () => {
+test("make wasm writes one directory, which its manifest describes", () => {
-  for (const file of ["lwconf.wasm", "wasm_exec.js"]) {
+  const manifest = JSON.parse(readFileSync(join(dir, "lwconf.json"), "utf8"));
-    const sum = createHash("sha256").update(readFileSync(join(dir, file))).digest("hex");
+  const files = readdirSync(dir).filter(f => f !== "lwconf.json").sort();
-    assert.match(made, new RegExp(`^${sum} +${file.replace(".", "\\.")}$`, "m"));
+  assert.deepEqual(Object.keys(manifest.sha256).sort(), files);
+  for (const file of files) {
+    assert.equal(manifest.sha256[file], createHash("sha256").update(readFileSync(join(dir, file))).digest("hex"), file);
   }
-  assert.equal(call("schema").value.version, "v0.0.0-node");
+  const version = execFileSync(go, ["env", "GOVERSION"], { cwd: root, encoding: "utf8" }).trim();
+  assert.deepEqual([manifest.version, schema.version, manifest.go], ["v0.0.0-node", "v0.0.0-node", version]);
+  assert.deepEqual(JSON.parse(readFileSync(join(dir, "schema.json"), "utf8")), schema);
 });
 
-// Off the host a directory path ends in '/'
+test("make wasm writes a commit's files the same at any time", () => {
-test("a preset's pipelines, written as a command line, parse and validate back to themselves", () => {
+  wasm(join(tmp, "again"), "BUILD_TIME=another");
-  const { value: pipelines } = call("preset", "tail", JSON.stringify({ path: "/var/log/" }));
+  assert.equal(readFileSync(join(tmp, "again/lwconf/lwconf.json"), "utf8"), readFileSync(join(dir, "lwconf.json"), "utf8"));
-  assert.equal(pipelines[0].plugin_sources[0].config.directory, "/var/log/");
-  const { value: line } = call("emit", JSON.stringify(pipelines), "command");
-  assert.match(line, /^lw \\\n {2}--pipeline tail/);
-  assert.deepEqual(call("parse", line).value, pipelines);
-  assert.deepEqual(call("validate", JSON.stringify(pipelines)).value, pipelines);
-  assert.match(call("emit", JSON.stringify(pipelines), "file").value, /^\[\[pipelines\]\]/m);
 });
 
-test("a refused call answers why, and the engine keeps answering", () => {
+test("load refuses an engine its integrity does not match, or a server does not have", async () => {
+  await assert.rejects(load({ wasmURL: url("lwconf.wasm"), wasmSRI: sri("schema.json") }));
+  await assert.rejects(load({ wasmURL: url("missing.wasm") }), { message: `${url("missing.wasm")}: 404` });
+});
+
+test("load takes an engine a server sends as plain bytes", async () => {
+  const api = globalThis.lwconf;
+  delete globalThis.lwconf;
+  try {
+    const bytes = await load({ wasmURL: url("lwconf.wasm?type=application/octet-stream"), wasmSRI: sri("lwconf.wasm") });
+    assert.deepEqual(bytes.schema().value, schema);
+  } finally {
+    globalThis.lwconf = api;
+  }
+});
+
+// The environment holds one pipeline, which a preset makes
+test("a preset's pipeline, a single file answered by dirs, reads back from every form", () => {
+  const path = { path: "/var/log/app.log" }, dirs = { "/var/log/app.log": false };
+  assert.deepEqual(engine.preset("tail", path), { error: "path: a directory or a file pattern?", path: "path" });
+  const { value: pipelines } = engine.preset("serve", { ...path, allow: ["192.0.2.0/24", "198.51.100.7"] }, dirs);
+  assert.deepEqual(value(pipelines, 0, { role: "source", id: "file" }), { directory: "/var/log", pattern: "app.log", from: "end" });
+  assert.deepEqual(value(pipelines, 0, { role: "sink", id: "http" }, "acl.allow"), ["192.0.2.0/24", "198.51.100.7"]);
+  assert.deepEqual(engine.parse("lw -p serve:path=/var/log/app.log,allow=192.0.2.0/24,allow=198.51.100.7", "command", dirs).value, pipelines);
+  for (const { name } of schema.forms) {
+    assert.deepEqual(engine.parse(engine.emit(pipelines, name).value, name).value, pipelines, name);
+  }
+  assert.deepEqual(engine.validate(pipelines).value, { pipelines, problems: [] });
+});
+
+// A value of the wrong kind is a problem as any other is, and the parts after
+// it are still read
+test("validate places every problem at its pipeline, node and key", () => {
+  const { value: { problems } } = engine.validate([
+    { name: "a", flow: { rate_limit: { policy: "bogus" }, filters: [{ patterns: ["("] }] }, plugin_sources: [{ type: "file" }],
+      plugin_sinks: [{ id: "out", type: "tcp", config: { port: 9000, tls: { enabled: true } } }, { type: "http", config: { port: "x" } }] },
+    { name: "a", plugin_sinks: [{ type: "tcp" }] },
+  ]);
+  const node = (role, id, index = 0) => ({ role, id, index });
+  assert.deepEqual(problems.map(({ path, pipeline, node, key }) => ({ path, pipeline, node, key })), [
+    { path: "pipelines[0].flow.rate_limit.policy", pipeline: 0, node: node("rate_limit", ""), key: "policy" },
+    { path: "pipelines[0].flow.filters[0].patterns[0]", pipeline: 0, node: node("filters", "", 0), key: "patterns" },
+    { path: "pipelines[0].plugin_sources[file].config.directory", pipeline: 0, node: node("source", "file"), key: "directory" },
+    { path: "pipelines[0].plugin_sinks[out].config.tls", pipeline: 0, node: node("sink", "out"), key: "tls" },
+    { path: "pipelines[0].plugin_sinks[http].config.port", pipeline: 0, node: node("sink", "http"), key: "port" },
+    { path: "pipelines[1].name", pipeline: 1, node: undefined, key: "name" },
+    { path: "pipelines[1].plugin_sources", pipeline: 1, node: undefined, key: "plugin_sources" },
+    { path: "pipelines[1].plugin_sinks[tcp].config.port", pipeline: 1, node: node("sink", "tcp"), key: "port" },
+  ]);
+  assert.match(problems[3].text, /^listeners need cert_file/);
+  assert.equal(problems[4].text, "not an integer");
+  const placed = group(problems);
+  assert.deepEqual(placed.get(at(0, { role: "sink", id: "out" })).get("tls"), [problems[3]]);
+  assert.deepEqual(placed.get(at(0, { role: "rate_limit" })).get("policy"), [problems[0]]);
+  assert.deepEqual([...placed.get(at(1)).keys()], ["name", "plugin_sources"]);
+});
+
+test("edits keep the engine's rules: free ids, retype's carry-over, a stage turned on", () => {
+  let pipelines = [], node;
+  const edit = op => {
+    const r = engine.edit(pipelines, { pipeline: 0, ...op });
+    assert.equal(r.error, undefined, JSON.stringify(op));
+    ({ pipelines, node } = r.value);
+    return node;
+  };
+  edit({ op: "add_pipeline", name: "p" });
+  edit({ op: "add", role: "source", type: "null" });
+  assert.deepEqual(edit({ op: "add", role: "source", type: "null" }), { role: "source", id: "null_2", index: 0 });
+  const tcp = edit({ op: "add", role: "sink", type: "tcp" });
+  edit({ op: "set", node: tcp, key: "port", values: ["9000"] });
+  edit({ op: "set", node: tcp, key: "tls.enabled", values: ["true"] });
+  assert.deepEqual(edit({ op: "retype", node: tcp, type: "http" }), { role: "sink", id: "http", index: 0 });
+  assert.deepEqual(value(pipelines, 0, node), { port: 9000, tls: { enabled: true } });
+  edit({ op: "unset", node, key: "tls.enabled" });
+  edit({ op: "set", node: { role: "rate_limit" }, key: "entries_per_second", values: ["5"] });
+  assert.deepEqual(value(pipelines, 0, { role: "rate_limit" }, "policy"), schema.flow[0].on.policy);
+  edit({ op: "add", role: "filters", type: "exclude" });
+  edit({ op: "add", role: "filters", type: "include" });
+  assert.deepEqual(edit({ op: "move_filter", from: 1, to: 0 }), { role: "filters", id: "", index: 0 });
+  assert.deepEqual([value(pipelines, 0, node, "type"), value(pipelines, 0, { role: "filters", index: 1 }, "type")], ["include", "exclude"]);
+  edit({ op: "remove", node: { role: "source", id: "null" } });
+  edit({ op: "rename_pipeline", name: "q" });
+  assert.deepEqual([pipelines[0].name, pipelines[0].plugin_sources.map(s => s.id), value(pipelines, 0, { role: "sink", id: "http" })],
+    ["q", ["null_2"], { port: 9000 }]);
+  edit({ op: "remove_pipeline" });
+  assert.deepEqual(pipelines, []);
+});
+
+test("fields are a node's keys, its tables' of the plugin's side as groups", () => {
+  assert.deepEqual(schema.sources.find(p => p.type === "null").keys, []);
+  const rows = fields(schema, "sink", "tcp_chain");
+  const tls = rows.find(r => r.path === "tls");
+  assert.ok(tls.rows.some(r => r.path === "tls.pin_sha256") && !tls.rows.some(r => r.path === "tls.self_signed"));
+  assert.deepEqual(rows.find(r => r.path === "port").key.max, 65535);
+  assert.equal(fields(schema, "rate_limit").find(r => r.path === "entries_per_second").placeholder, "(off)");
+  assert.equal(fields(schema, "heartbeat").find(r => r.path === "interval_ms").placeholder, "1000");
+});
+
+test("a refused call answers why and where, and the engine keeps answering", () => {
   const refused = [
-    [["parse", "lw --nope x"], /not a pipeline flag/],
+    [["parse", "lw --nope x", "command", "{}"], { error: /not a pipeline flag/ }],
-    [["parse", "lw"], /no pipeline flags/],
+    [["parse", "x=1", "environment", "{}"], { error: /^no LOGWISP_ pipeline variables$/ }],
-    [["parse"], /^usage: lwconf\.parse\(line\)/],
+    [["parse", "", "file", "{}"], { error: /no \[\[pipelines\]\]/ }],
-    [["parse", 5], /^usage: /],
+    [["parse", "lw"], { error: /^usage: lwconf\.parse\(text, form, dirs\)/ }],
-    [["preset", "tail", JSON.stringify({ path: "/var/log" })], /path/],
+    [["parse", 5, "command", "{}"], { error: /^usage: / }],
-    [["validate", JSON.stringify([{ name: "p" }])], /no sources/],
+    [["preset", "serve", JSON.stringify({ allow: 5 }), "{}"], { error: /^allow: takes a string/, path: "allow" }],
-    [["validate", JSON.stringify(["a", "b"].map((name, i) => ({
+    [["edit", JSON.stringify([{ name: "p", plugin_sinks: [{ type: "tcp" }] }]),
-      name, plugin_sources: [{ type: "tcp_chain", config: { port: [9000, "x"][i] } }], plugin_sinks: [{ type: "null" }],
+      JSON.stringify({ op: "set", pipeline: 0, node: { role: "sink", id: "tcp" }, key: "port", values: ["x"] })],
-    })))], /^pipelines\[1\]\.plugin_sources\[tcp_chain\]\.config\.port: not an integer$/],
+      { error: /port: not an integer$/, path: "pipelines[0].plugin_sinks[tcp].config.port", pipeline: 0, key: "port" }],
-    [["emit", JSON.stringify([{ name: "p" }]), "command"], /no sources/],
+    [["emit", JSON.stringify([{ name: "p" }]), "command"], { error: /no sources defined/, path: "pipelines[0].plugin_sources" }],
-    [["emit", "[]", "yaml"], /no form "yaml"/],
+    [["emit", "[]", "yaml"], { error: /no form "yaml"/ }],
+    [["edit", "[]", JSON.stringify({ op: "add_pipeline", nmae: "p" })], { error: /unknown field "nmae"/ }],
+    [["edit", "[]", JSON.stringify({ op: "rename" })], { error: /^no op "rename" \(valid: add, add_pipeline/ }],
   ];
   for (const [args, want] of refused) {
-    const got = call(...args);
+    const got = JSON.parse(globalThis.lwconf[args[0]](...args.slice(1)));
     assert.equal(got.value, undefined, `${args}`);
-    assert.match(got.error, want, `${args}`);
+    for (const [field, w] of Object.entries(want)) {
+      if (w instanceof RegExp) assert.match(got[field], w, `${args}`);
+      else assert.equal(got[field], w, `${args}`);
     }
-  assert.ok(call("schema").value.sources.length > 0);
+  }
+  assert.ok(engine.schema().value.sources.length > 0);
 });

cmd/lwconf/main.go

@@ -1,19 +1,24 @@
 //go:build js && wasm
 
-// Command lwconf is the configuration engine as WebAssembly, for the website.
+// Command lwconf is the configuration engine as WebAssembly: it sets
-// It sets globalThis.lwconf, whose calls take strings and return a JSON object:
+// globalThis.lwconf for lwconf.js, calls taking strings and returning JSON,
-// {"value": ...}, or {"error": "..."} for a call refused. Pipelines travel in
+// {"value"} or {"error"} with its path, one in a pipeline placed on its node
-// compose's JSON form; off the host a preset's directory path ends in '/'.
+// and key, any other a preset's or flag's key. Pipelines are compose's JSON
+// form; dirs answers config.IsDir by path, {} leaving '/' and globs to tell.
 package main
 
 import (
+	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"maps"
 	"slices"
 	"strings"
 	"syscall/js"
 
 	"github.com/lixenwraith/logwisp/internal/compose"
+	"github.com/lixenwraith/logwisp/internal/config"
 )
 
 // calls are lwconf's, by name, with the string arguments each takes
@@ -22,34 +27,187 @@ var calls = map[string]struct {
 	call func(args []string) (any, error)
 }{
 	"schema": {nil, func([]string) (any, error) { return compose.NewSchema(), nil }},
-	"preset": {[]string{"name", "values"}, func(a []string) (any, error) {
+	"preset": {[]string{"name", "values", "dirs"}, func(a []string) (any, error) {
-		var values map[string]string
+		values, err := presetValues(a[1])
-		if err := json.Unmarshal([]byte(a[1]), &values); err != nil {
+		if err != nil {
-			return nil, fmt.Errorf("values: %w", err)
+			return nil, err
 		}
-		return pipelines(compose.FromPreset(a[0], values, nil))
+		isDir, err := answers(a[2])
+		if err != nil {
+			return nil, err
+		}
+		return pipelines(compose.FromPreset(a[0], values, isDir))
 	}},
-	"parse": {[]string{"line"}, func(a []string) (any, error) {
+	"parse": {[]string{"text", "form", "dirs"}, func(a []string) (any, error) {
-		return pipelines(compose.FromCommandLine(a[0], nil))
+		f, err := form(a[1])
+		if err != nil {
+			return nil, err
+		}
+		isDir, err := answers(a[2])
+		if err != nil {
+			return nil, err
+		}
+		return pipelines(f.Read(a[0], isDir))
 	}},
 	"validate": {[]string{"pipelines"}, func(a []string) (any, error) {
 		c, err := compose.FromJSON([]byte(a[0]))
-		if err == nil {
+		if c == nil {
-			err = c.Validate()
+			return nil, err
 		}
-		return pipelines(c, err)
+		value, err := pipelines(c, nil) // a value of the wrong kind is a problem Validate names
+		if err != nil {
+			return nil, err
+		}
+		problems := []problem{}
+		for _, p := range compose.Problems(c.Validate()) {
+			problems = append(problems, place(p))
+		}
+		return map[string]any{"pipelines": value, "problems": problems}, nil
 	}},
 	"emit": {[]string{"pipelines", "form"}, func(a []string) (any, error) {
 		c, err := compose.FromJSON([]byte(a[0]))
 		if err != nil {
 			return nil, err
 		}
-		forms := compose.Forms()
+		f, err := form(a[1])
-		if i := slices.IndexFunc(forms, func(f compose.Form) bool { return f.Name == a[1] }); i >= 0 {
+		if err != nil {
-			return forms[i].Write(c)
+			return nil, err
 		}
-		return nil, fmt.Errorf("no form %q", a[1])
+		return f.Write(c)
 	}},
+	"edit": {[]string{"pipelines", "op"}, func(a []string) (any, error) {
+		c, err := compose.FromJSON([]byte(a[0]))
+		if err != nil {
+			return nil, err
+		}
+		var o op
+		in := json.NewDecoder(bytes.NewReader([]byte(a[1])))
+		in.DisallowUnknownFields()
+		if err := in.Decode(&o); err != nil {
+			return nil, fmt.Errorf("op: %w", err)
+		}
+		edit, ok := edits[o.Op]
+		if !ok {
+			return nil, fmt.Errorf("no op %q (valid: %s)", o.Op, strings.Join(slices.Sorted(maps.Keys(edits)), ", "))
+		}
+		n, err := edit(c, o)
+		if _, ok := errors.AsType[*config.KeyError](err); ok && o.Node.Role != "" {
+			err = config.At(o.Node.Path(o.Pipeline), err) // a key set or unset in the node
+		}
+		value, err := pipelines(c, err)
+		if err != nil {
+			return nil, err
+		}
+		return map[string]any{"pipelines": value, "node": n}, nil
+	}},
+}
+
+// op is one edit: the compose method Op names, and its arguments
+type op struct {
+	Op       string       `json:"op"`
+	Pipeline int          `json:"pipeline"`
+	Node     compose.Node `json:"node"`
+	Role     string       `json:"role"`
+	Type     string       `json:"type"`
+	Key      string       `json:"key"`
+	Values   []string     `json:"values"`
+	Name     string       `json:"name"`
+	From     int          `json:"from"`
+	To       int          `json:"to"`
+}
+
+// edits call compose's edit methods, each returning the node it leaves chosen
+var edits = map[string]func(*compose.Composition, op) (compose.Node, error){
+	"add_pipeline": func(c *compose.Composition, o op) (compose.Node, error) {
+		return compose.Node{}, c.AddPipeline(o.Name)
+	},
+	"rename_pipeline": func(c *compose.Composition, o op) (compose.Node, error) {
+		return compose.Node{}, c.RenamePipeline(o.Pipeline, o.Name)
+	},
+	"remove_pipeline": func(c *compose.Composition, o op) (compose.Node, error) {
+		return compose.Node{}, c.RemovePipeline(o.Pipeline)
+	},
+	"add": func(c *compose.Composition, o op) (compose.Node, error) { return c.Add(o.Pipeline, o.Role, o.Type) },
+	"retype": func(c *compose.Composition, o op) (compose.Node, error) {
+		return c.Retype(o.Pipeline, o.Node, o.Type)
+	},
+	"remove": func(c *compose.Composition, o op) (compose.Node, error) {
+		return compose.Node{}, c.Remove(o.Pipeline, o.Node)
+	},
+	"move_filter": func(c *compose.Composition, o op) (compose.Node, error) {
+		return compose.Node{Role: "filters", Index: o.To}, c.MoveFilter(o.Pipeline, o.From, o.To)
+	},
+	"set": func(c *compose.Composition, o op) (compose.Node, error) {
+		return o.Node, c.Set(o.Pipeline, o.Node, o.Key, o.Values...)
+	},
+	"unset": func(c *compose.Composition, o op) (compose.Node, error) {
+		return o.Node, c.Unset(o.Pipeline, o.Node, o.Key)
+	},
+}
+
+// problem is where a problem, or a call's refusal, lies as far as the engine
+// knows: its path, and the pipeline, node and key Locate reads from it
+type problem struct {
+	Error    string        `json:"error,omitempty"`
+	Path     string        `json:"path,omitempty"`
+	Pipeline *int          `json:"pipeline,omitempty"`
+	Node     *compose.Node `json:"node,omitempty"`
+	Key      string        `json:"key,omitempty"`
+	Text     string        `json:"text,omitempty"`
+}
+
+func place(err error) problem {
+	p := problem{Text: err.Error()}
+	if ke, ok := errors.AsType[*config.KeyError](err); ok {
+		p.Path = ke.Key
+	}
+	if pi, n, key, text, ok := compose.Locate(err); ok {
+		p.Pipeline, p.Key, p.Text = &pi, key, text
+		if n.Role != "" {
+			p.Node = &n
+		}
+	}
+	return p
+}
+
+// presetValues reads a preset's values: a string each, or a list key's
+// strings, which the preset takes ',' between them
+func presetValues(data string) (map[string]string, error) {
+	var raw map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(data), &raw); err != nil {
+		return nil, fmt.Errorf("values: %w", err)
+	}
+	values := map[string]string{}
+	for k, v := range raw {
+		var one string
+		var many []string
+		switch {
+		case json.Unmarshal(v, &one) == nil:
+			values[k] = one
+		case json.Unmarshal(v, &many) == nil:
+			values[k] = strings.Join(many, ",")
+		default:
+			return nil, &config.KeyError{Key: k, Err: errors.New("takes a string, or a list of strings")}
+		}
+	}
+	return values, nil
+}
+
+// answers reads dirs: whether each path a preset names is a directory
+func answers(dirs string) (config.IsDir, error) {
+	var m map[string]bool
+	if err := json.Unmarshal([]byte(dirs), &m); err != nil {
+		return nil, fmt.Errorf("dirs: %w", err)
+	}
+	return config.Answers(m), nil
+}
+
+func form(name string) (compose.Form, error) {
+	forms := compose.Forms()
+	if i := slices.IndexFunc(forms, func(f compose.Form) bool { return f.Name == name }); i >= 0 {
+		return forms[i], nil
+	}
+	return compose.Form{}, fmt.Errorf("no form %q", name)
 }
 
 // pipelines is a composition's value: its pipelines settled, ids given
@@ -81,9 +239,11 @@ func answer(name string, args []js.Value) string {
 		}
 		value, err = c.call(strs)
 	}
-	out := map[string]any{"value": value}
+	var out any = map[string]any{"value": value}
 	if err != nil {
-		out = map[string]any{"error": err.Error()}
+		p := place(err)
+		p.Error, p.Text = err.Error(), ""
+		out = p
 	}
 	data, _ := json.Marshal(out)
 	return string(data)

deploy/package/completion/_lw

@@ -94,10 +94,10 @@ preset:*)
 	(( CURRENT -= 2 ))
 	case ${words[1]} in
 	pipe) _arguments -S \
-		'--format=[raw|txt|json]:value:_default' \
+		'--format=[raw|txt|json: entry format]:value:_default' \
 		'(-h --help)'{-h,--help}'[print the flags]' ;;
 	tail) _arguments -S \
-		'--format=[raw|txt|json]:value:_default' \
+		'--format=[raw|txt|json: entry format]:value:_default' \
 		'--from=[start|end: where reading a file starts]:value:_default' \
 		'--path=[file, directory (its files) or glob to follow]:value:_default' \
 		'(-h --help)'{-h,--help}'[print the flags]' ;;
@@ -105,7 +105,7 @@ preset:*)
 		'--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \
 		'--cert=[certificate file, for tls=files]:value:_default' \
 		'--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \
-		'--format=[raw|txt|json]:value:_default' \
+		'--format=[raw|txt|json: entry format]:value:_default' \
 		'--from=[start|end: where reading a file starts]:value:_default' \
 		'--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \
 		'--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \
@@ -115,10 +115,10 @@ preset:*)
 		'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
 		'--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \
 		'--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \
-		'--tls=[off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \
+		'--tls=[off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \
 		'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \
 		'--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \
-		'--viewer=[true: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \
+		'--viewer=[true|false: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \
 		'(-h --help)'{-h,--help}'[print the flags]' ;;
 	edge) _arguments -S \
 		'--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \
@@ -139,7 +139,7 @@ preset:*)
 		'--cert=[certificate file, for tls=files]:value:_default' \
 		'--client_ca=[CA file that verifies edge certificates (mTLS)]:value:_default' \
 		'--deny=[addresses or CIDRs refused, allow or not; '\'','\'' between them]:value:_default' \
-		'--format=[raw|txt|json]:value:_default' \
+		'--format=[raw|txt|json: entry format]:value:_default' \
 		'--hosts=[names and addresses, beyond this host'\''s, of a self or issuer certificate; '\'','\'' between them]:value:_default' \
 		'--issuer_cert=[CA certificate file (lw tls ca), for tls=issuer]:value:_default' \
 		'--issuer_key=[CA key file, for tls=issuer]:value:_default' \
@@ -147,7 +147,7 @@ preset:*)
 		'--listen=[HOST:PORT; [IPV6\]:PORT; [::\] for IPv6 only]:value:_default' \
 		'--out=[directory for the received entries; default: standard output]:value:_default' \
 		'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
-		'--tls=[self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \
+		'--tls=[self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files]:value:_default' \
 		'--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \
 		'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \
 		'--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \

deploy/package/completion/lw.fish

@@ -96,16 +96,16 @@ complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a
 complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a serve -d 'Serve files or standard input live: a browser viewer at /, SSE at /stream'
 complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a edge -d 'Forward files or standard input to an aggregator over TLS, authenticated'
 complete -c lw -n '__lw_in preset; and test (count (commandline -opc)) -eq 2' -a aggregator -d 'Receive from edges over TLS, authenticated, to files or standard output'
-complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json'
+complete -c lw -n '__lw_in preset pipe' -l format -r -F -d 'raw|txt|json: entry format'
 complete -c lw -n '__lw_in preset pipe' -l help -o h -d 'print the flags'
-complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json'
+complete -c lw -n '__lw_in preset tail' -l format -r -F -d 'raw|txt|json: entry format'
 complete -c lw -n '__lw_in preset tail' -l from -r -F -d 'start|end: where reading a file starts'
 complete -c lw -n '__lw_in preset tail' -l path -r -F -d 'file, directory (its files) or glob to follow'
 complete -c lw -n '__lw_in preset tail' -l help -o h -d 'print the flags'
 complete -c lw -n '__lw_in preset serve' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them'
 complete -c lw -n '__lw_in preset serve' -l cert -r -F -d 'certificate file, for tls=files'
 complete -c lw -n '__lw_in preset serve' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them'
-complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json'
+complete -c lw -n '__lw_in preset serve' -l format -r -F -d 'raw|txt|json: entry format'
 complete -c lw -n '__lw_in preset serve' -l from -r -F -d 'start|end: where reading a file starts'
 complete -c lw -n '__lw_in preset serve' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them'
 complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer'
@@ -115,10 +115,10 @@ complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT; [IPV6]:P
 complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
 complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input'
 complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them'
-complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files'
+complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: plain text, a certificate made at startup (self-signed, or from issuer_*), or files'
 complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users'
 complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM'
-complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true: the login page and viewer, behind proxy; without user or users the viewer is always on'
+complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true|false: the login page and viewer, behind proxy; without user or users the viewer is always on'
 complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags'
 complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots'
 complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca'
@@ -137,7 +137,7 @@ complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CI
 complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files'
 complete -c lw -n '__lw_in preset aggregator' -l client_ca -r -F -d 'CA file that verifies edge certificates (mTLS)'
 complete -c lw -n '__lw_in preset aggregator' -l deny -r -F -d 'addresses or CIDRs refused, allow or not; \',\' between them'
-complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json'
+complete -c lw -n '__lw_in preset aggregator' -l format -r -F -d 'raw|txt|json: entry format'
 complete -c lw -n '__lw_in preset aggregator' -l hosts -r -F -d 'names and addresses, beyond this host\'s, of a self or issuer certificate; \',\' between them'
 complete -c lw -n '__lw_in preset aggregator' -l issuer_cert -r -F -d 'CA certificate file (lw tls ca), for tls=issuer'
 complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key file, for tls=issuer'
@@ -145,7 +145,7 @@ complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls
 complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT; [::] for IPv6 only'
 complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output'
 complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
-complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files'
+complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed, or from issuer_*), or files'
 complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain'
 complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users'
 complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM'

doc/cli.md

@@ -69,9 +69,18 @@ pipelines have [their own flags](#pipelines):
   resolved into it, as TOML that `-c` reads back, and exit
 - `-T`, `--tui`: compose the pipelines on screen, then run or print them
   ([below](#lw---tui))
-- `--schema`: print every setting, pipeline flag, flow stage, plugin option
+- `--schema`: print every setting, pipeline flag, flow stage, plugin option,
-  and preset as one JSON document (kinds, defaults, rules, help), and exit;
+  preset and output form as one JSON document, and exit; tools build their
-  tools build their option lists from it
+  option lists from it:
+  - a key: its kind, default, rules and help; a table (`tls`, `auth`, `acl`)
+    holds its own keys, those of the plugin's side (listener or dialer),
+    HTTP's only on a plugin that speaks HTTP. Every listener lists
+    `auth.node_binding` and `auth.trusted_proxies`, which validation admits
+    only on chain sources and on the http sink
+  - a flow stage: its keys, and `on`, the values that turn it on
+  - a preset key: its default and help, and the choices (`enum`) and `hint`
+    of the option it sets
+  - `forms`: `command`, `environment` and `file`, each with its hint
 - `--color [WHEN]`, WHEN `auto`, `always` or `never`: level names in color on
   console sinks, default `auto` (a terminal, `NO_COLOR` unset, `TERM` not
   `dumb`); bare, `always`. A console sink's own `color` wins
@@ -421,14 +430,22 @@ lw -c dev.toml --logging.level=debug
 `lw --check` validates without running: it builds every pipeline and plugin as
 a start would (options, TLS files, credentials files, startup warnings), but
 binds no port, reads no stdin, and opens, reads or creates no log file; it
-prints `configuration ok: N pipeline(s)` and exits 0, or the error and exits 1
+prints `configuration ok: N pipeline(s)` and exits 0. Otherwise it prints the
-(2 when a named file is missing, see [Exit Codes](#exit-codes)):
+first problem of each setting, flow stage, source, sink and pipeline, a line
+each named by its key's path, or else the first file that fails to open, and
+exits 1 (2 when a named file is missing, see [Exit Codes](#exit-codes)):
 
 ```bash
 lw --check -c /etc/logwisp/logwisp.toml
 lw --check --source file:directory=/var/log/app --sink http:port=8080
 ```
 
+```text
+Error: Failed to load config: invalid configuration:
+pipelines[0].plugin_sources[file].config.directory: must be set
+pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000
+```
+
 **Production**
 
 ```bash
@@ -472,8 +489,8 @@ WebAssembly.
   Space flips it; Esc goes back. `a` adds a source, sink or filter; `c`
   changes a source's or sink's type in its place; `d` deletes, or returns a
   value to its default; Space turns a flow stage on or off; `J`/`K` move a
-  filter; `[` `]` switch pipelines; `p` presets; `!` shows the first
+  filter; `[` `]` switch pipelines; `p` presets; `!` shows every problem,
-  problem; `r` runs; `o` output; `?` keys; `q` quits.
+  moving to the first; `r` runs; `o` output; `?` keys; `q` quits.
 - Pipelines: Up from a column's top row reaches the bar, where `[` `]` or
   Left/Right switch, `a` adds one (empty or from a preset), Enter renames and
   `d` deletes, asking first when it has parts.

doc/installation.md

@@ -26,9 +26,12 @@ The Makefile works with GNU make and BSD make alike. Targets:
   - `make build`: `bin/lw`, with version, commit and build time from git
   - `make release`: the same, static (`CGO_ENABLED=0`), `-trimpath`, stripped
   - `make dev`: built with the race detector
-  - `make wasm`: `bin/lwconf.wasm`, the configuration engine for the
+  - `make wasm`: `bin/lwconf/`, the configuration engine a website drops in:
-    website, version stamped, beside the toolchain's `wasm_exec.js`; prints
+    `lwconf.wasm` (version stamped), the toolchain's `wasm_exec.js`,
-    both checksums
+    `lwconf.js` that loads and calls it, `schema.json` (`lw --schema`) and
+    `lwconf.json`, their manifest: version, commit, Go version and each
+    file's sha256. Stamped with the commit's time, a commit's files are the
+    same on every build
   - `make version`: the metadata a build would embed
   - `make completion`: regenerate the bash, zsh and fish completion in
     `deploy/package/completion/` from `cmd/lw`'s `shorts` and `commands`

doc/lw.1

@@ -141,7 +141,10 @@ Build every pipeline and plugin as a start would, loading options, TLS files
 and credentials files and printing startup warnings, without binding a port
 or opening, reading or creating a log file; print
 .Dq configuration ok
-and exit 0, or the error and exit 1, or 2 when a named configuration file is
+and exit 0.
+Otherwise print the first problem of each setting, flow stage, source, sink
+and pipeline, a line each named by its key's path, or else the first file
+that fails to open, and exit 1, or 2 when a named configuration file is
 missing
 .Pq see Sx EXIT STATUS .
 .It Fl \-dump
@@ -167,9 +170,13 @@ A file is read, never written; the shell empties a redirect's target before
 .Nm
 reads it, so the redirect names a file it does not load.
 .It Fl \-schema
-Print every setting, pipeline flag, flow stage, plugin option and preset as
+Print every setting, pipeline flag, flow stage, plugin option, preset and
-one JSON document, with kinds, defaults, rules and help, and exit; tools build
+output form as one JSON document, with kinds, defaults, rules and help, and
-their option lists from it.
+exit; tools build their option lists from it.
+A plugin's table holds the keys of its side; a flow stage carries
+.Ql on ,
+the values that turn it on; a preset key carries the choices and hint of the
+option it sets.
 .It Fl q , Fl \-quiet
 Turn off the log of
 .Nm

doc/operations.md

@@ -100,15 +100,16 @@ lw --check -c candidate.toml
 ```
 
 Before a reload, check the edited file this way: a failed reload keeps the
-running service, but `--check` says why without touching it. Failures name the
+running service, but `--check` says why without touching it. It prints the
-pipeline and the offending key:
+first problem of each setting, flow stage, source, sink and pipeline, a line
+each named by its key's path; once none is left, the files they name are
+opened as a start opens them:
 
+```text
+Error: Failed to load config: invalid configuration:
+pipelines[0].plugin_sources[in].config.directory: must be set
+pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000
 ```
-ERROR msg="Failed to create pipeline" pipeline=app error="failed to create sink out: port: must be 1-65535, got 0"
-```
-
-Remember that most validation lives in plugin constructors, so a config only
-proves itself when the pipeline is actually built.
 
 ## Monitoring
 

doc/todo.md

@@ -89,17 +89,18 @@ What exists:
   on the option structs, one catalogue row per plugin type, `Decode` (every
   constructor decodes through it and keeps only I/O), `Coerce` for typed
   command-line values, and a `ValidateConfig` that decodes every plugin
-  offline and names the refused key's path.
+  offline and names the first problem of each part by its key's path.
 - `internal/config` already compiles for `GOOS=js`; it links no net/http,
   crypto/tls or terminal code.
 - The engine, `internal/compose`: starts empty, from a preset (whose path
-  `config.IsDir` resolves, nil off the host), a loaded configuration or a
+  `config.IsDir` resolves, from answers off the host), a loaded
-  pasted command line; adds, removes and moves sources, sinks and filters,
+  configuration or pasted text of any form; adds, removes and moves sources,
-  sets and unsets keys, validates; writes a command line, environment
+  sinks and filters, sets and unsets keys, validates; writes a command line,
-  variables or a file, each loading back to the same pipelines. Its test
+  environment variables or a file, each reading back to the same pipelines.
-  fails when it links TLS, HTTP, process, signal or terminal code.
+  Its test fails when it links TLS, HTTP, process, signal or terminal code.
-- `lw --schema`: settings, pipeline flags, flow stages in order, the tls,
+- `lw --schema`: settings, pipeline flags, flow stages in order with what
-  auth and acl tables, sources, sinks and presets, as one JSON document.
+  turns each on, sources and sinks with their tables' keys of their side,
+  presets and forms, as one JSON document.
 - `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions,
   layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal,
   ConfirmDialog, scrolling, mouse; themed Radio, Field, TextInput,
@@ -118,21 +119,46 @@ Steps:
    radio buttons, narrow form fields and option lists, menus, frames, rules,
    wires and windows, moved from `lw --tui`.
 5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)).
-6. WebAssembly: done. `make wasm` builds `bin/lwconf.wasm`, version stamped,
+6. WebAssembly: done. `make wasm` writes `bin/lwconf/`, which a site drops
-   beside the toolchain's `wasm_exec.js`, and prints both checksums: about
+   in whole:
-   6.9 MB, 1.9 MB gzipped. `wasm_exec.js` uses no eval, so the page needs only
+   - `lwconf.wasm`, version stamped: 7.1 MB, 1.9 MB at gzip -9.
-   `'wasm-unsafe-eval'` in its `script-src`. The page runs the module with
+   - `wasm_exec.js`, the toolchain's, which uses no eval: the page needs only
-   `new Go().run(instance)`, then calls `globalThis.lwconf`; each call takes
+     `'wasm-unsafe-eval'` in its `script-src`.
-   strings and returns a JSON object, `{"value": ...}` or `{"error": "..."}`:
+   - `lwconf.js`, one dependency-free ES module that knows the calls and
+     shapes, never a key: `load({execURL, execSRI, wasmURL, wasmSRI})`
+     fetches both by their integrity and resolves to the engine;
+     `fields(schema, role, type)` is a node's form rows, a table's a group;
+     `value`, `at` and `group` read a node's value and sort problems by node.
+   - `schema.json`, equal to `schema()`, and `lwconf.json`, the manifest:
+     version, commit, Go version and each file's sha256.
+
+   The engine's calls take strings (`lwconf.js` writes objects as JSON) and
+   answer `{"value": ...}` or `{"error", "path", "pipeline", "node", "key"}`,
+   as far as the engine places the refusal:
    - `schema()`: what `lw --schema` prints.
-   - `preset(name, values)`: a preset's pipelines; `values` is a JSON object
+   - `preset(name, values, dirs)`: a preset's pipelines; `values` maps a key
-     of strings, and a directory path ends in `/`.
+     to a string, a list key to strings.
-   - `parse(line)`: a pasted command line's pipelines.
+   - `parse(text, form, dirs)`: the pipelines of a form's pasted text.
-   - `validate(pipelines)`: the pipelines settled (defaults, kinds, ids), or
+   - `validate(pipelines)`: `{pipelines, problems}`, the pipelines settled
-     the first problem by its key path.
+     (defaults, kinds, ids) and the first problem of each part (a value of
-   - `emit(pipelines, form)`: `command`, `environment` or `file`.
+     the wrong kind too) as `{path, pipeline, node, key, text}`.
+   - `emit(pipelines, form)`: the text of `command`, `environment` or `file`.
+   - `edit(pipelines, op)`: `{pipelines, node}`, the node left chosen. `op` is
+     `{op, pipeline, ...}`: `add_pipeline` and `rename_pipeline` (`name`),
+     `remove_pipeline`, `add` (`role`, `type`), `retype` (`node`, `type`),
+     `remove` (`node`), `move_filter` (`from`, `to`), `set` (`node`, `key`,
+     `values`), `unset` (`node`, `key`).
+   - `dirs` answers whether a path a preset names is a directory,
+     `{"/var/log/app.log": false}`; unanswered, a path ending in `/` is one,
+     one with a glob a pattern, and any other is refused at `path`.
    - Pipelines are a JSON list in the file's shape (`name`, `flow`,
-     `plugin_sources`, `plugin_sinks`); a misspelled key is refused.
+     `plugin_sources`, `plugin_sinks`); a misspelled key is refused. A node
+     is `{role, id, index}`: a source's or sink's id, a filter's index.
 
 Still to verify: a scripted session on a pseudo-terminal for Run and each
 output.
+
+Still to derive: `AuthOptions.Check` alone limits `auth.node_binding` to chain
+sources and `auth.trusted_proxies` to the http sink, so the schema lists both
+on every listener; a side tag naming those sides would filter them as `http`
+filters HTTP's keys.

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/lixenwraith/color v0.1.3
 	github.com/lixenwraith/config v0.2.2
 	github.com/lixenwraith/log v0.1.11
-	github.com/lixenwraith/terminal v0.2.1
+	github.com/lixenwraith/terminal v0.2.2-0.20261010220430-c657af606d34
 	github.com/lixenwraith/toml v0.1.3
 	golang.org/x/term v0.47.0
 )

go.sum

@@ -10,6 +10,8 @@ github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKh
 github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg=
 github.com/lixenwraith/terminal v0.2.1 h1:KsMfmxrRicBFWMdXJIEJ3cpLWVw6IoNF8GwgsxbD3kQ=
 github.com/lixenwraith/terminal v0.2.1/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY=
+github.com/lixenwraith/terminal v0.2.2-0.20261010220430-c657af606d34 h1:GasarGRKYbAXzEz1utbcf9n4+N7WXPKyTbUtQZj8v74=
+github.com/lixenwraith/terminal v0.2.2-0.20261010220430-c657af606d34/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY=
 github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI=
 github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0=
 golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=

internal/authz/authz_test.go

@@ -154,7 +154,7 @@ func TestNewValidation(t *testing.T) {
 		{"trusted proxies without a type", &config.AuthOptions{TrustedProxies: proxies}, nil, config.Listener, HTTP, "trusted_proxies is set"},
 		{"trusted proxies under mtls", &config.AuthOptions{Type: MethodMTLS, TrustedProxies: proxies}, mtlsListenerTLS(), config.Listener, HTTP, "trusted_proxies applies only"},
 		{"trusted proxies on a chain source", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.ChainListener, HTTP, "only to the http sink"},
-		{"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to the http sink"},
+		{"trusted proxies on tcp", listener(config.AuthOptions{TrustedProxies: proxies}), nil, config.Listener, TCP, "only to HTTP listeners"},
 		{"trusted proxies with certificate binding", listener(config.AuthOptions{TrustedProxies: proxies, Identity: "cn"}), f.serverTLS, config.Listener, HTTP, "TLS-terminating proxy"},
 		{"malformed trusted proxy", listener(config.AuthOptions{TrustedProxies: []string{"localhost"}}), nil, config.Listener, HTTP, "neither an address"},
 		{"trusted proxies on a dialer", dialer(config.AuthOptions{TrustedProxies: proxies}), &tls.Config{}, config.Dialer, HTTP, "only to listeners"},

internal/compose/compose.go

@@ -32,9 +32,9 @@ type Composition struct {
 // by its key in the flow table: filters with an index, format, rate_limit or
 // heartbeat
 type Node struct {
-	Role  string
+	Role  string `json:"role"`
-	ID    string
+	ID    string `json:"id"`
-	Index int
+	Index int    `json:"index"`
 }
 
 // FromConfig starts from a loaded configuration's pipelines, its console sinks
@@ -100,8 +100,10 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
 }
 
 // settle gives every flow stage its defaults, every plugin's options their
-// kinds and every source and sink an id; the rules are Validate's
+// kinds and every source and sink an id; the rules are Validate's. Its error
+// joins a plugin's each holding a value of the wrong kind, kept as given.
 func (c *Composition) settle() error {
+	var errs []error
 	for i := range c.Pipelines {
 		p := &c.Pipelines[i]
 		if p.Flow == nil {
@@ -122,17 +124,17 @@ func (c *Composition) settle() error {
 		for j, s := range p.PluginSources {
 			p.PluginSources[j].Config = orEmpty(s.Config)
 			if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
-				return config.At(fmt.Sprintf("pipelines[%d].plugin_sources[%s].config", i, s.ID), err)
+				errs = append(errs, config.At(Node{Role: "source", ID: s.ID}.Path(i), err))
 			}
 		}
 		for j, s := range p.PluginSinks {
 			p.PluginSinks[j].Config = orEmpty(s.Config)
 			if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
-				return config.At(fmt.Sprintf("pipelines[%d].plugin_sinks[%s].config", i, s.ID), err)
+				errs = append(errs, config.At(Node{Role: "sink", ID: s.ID}.Path(i), err))
 			}
 		}
 	}
-	return nil
+	return errors.Join(errs...)
 }
 
 // name gives the parts a file left unnamed the ids their specs would get: the
@@ -340,6 +342,17 @@ func index(p *config.PipelineConfig, n Node) int {
 	return -1
 }
 
+// Path is the path of node n's options in pipeline pi, as Locate reads it
+func (n Node) Path(pi int) string {
+	switch n.Role {
+	case "source", "sink":
+		return fmt.Sprintf("pipelines[%d].plugin_%ss[%s].config", pi, n.Role, n.ID)
+	case "filters":
+		return fmt.Sprintf("pipelines[%d].flow.filters[%d]", pi, n.Index)
+	}
+	return fmt.Sprintf("pipelines[%d].flow.%s", pi, n.Role)
+}
+
 func (n Node) missing() error {
 	if n.Role == "filters" {
 		return fmt.Errorf("no filter %d", n.Index)
@@ -476,9 +489,69 @@ func (c *Composition) Options(pi int, n Node) (map[string]any, bool, error) {
 	return nil, false, n.missing()
 }
 
-// Validate applies every rule lw applies at load, naming the refused key's path
+// Validate applies every rule lw applies at load: its error joins every
+// problem, one a part, which Problems splits and Locate places
 func (c *Composition) Validate() error { return config.ValidatePipelines(c.Pipelines) }
 
+// Problems are the problems an error joins, in order
+func Problems(err error) []error {
+	if joined, ok := err.(interface{ Unwrap() []error }); ok {
+		var out []error
+		for _, e := range joined.Unwrap() {
+			out = append(out, Problems(e)...)
+		}
+		return out
+	}
+	if err == nil {
+		return nil
+	}
+	return []error{err}
+}
+
+// Locate reads where a problem lies from its path: the pipeline, the node,
+// and the key in the node's options, a list's without its index; text is
+// what is wrong. ok is false for a problem no pipeline holds.
+func Locate(err error) (pi int, n Node, key, text string, ok bool) {
+	ke, _ := errors.AsType[*config.KeyError](err)
+	rest, found := "", false
+	if ke != nil {
+		rest, found = strings.CutPrefix(ke.Key, "pipelines[")
+	}
+	end := strings.IndexByte(rest, ']')
+	if !found || end < 0 {
+		return 0, n, "", err.Error(), false
+	}
+	if pi, err = strconv.Atoi(rest[:end]); err != nil {
+		return 0, n, "", ke.Error(), false
+	}
+	rest = strings.TrimPrefix(rest[end+1:], ".")
+	for role, list := range map[string]string{"source": "plugin_sources[", "sink": "plugin_sinks["} {
+		if after, ok := strings.CutPrefix(rest, list); ok {
+			id, more := cutID(after)
+			n.Role, n.ID, rest = role, id, strings.TrimPrefix(strings.TrimPrefix(more, "config"), ".")
+		}
+	}
+	if stage, ok := strings.CutPrefix(rest, "flow."); ok {
+		name, more, _ := strings.Cut(stage, ".")
+		name, index, _ := strings.Cut(name, "[")
+		n.Role, rest = name, more
+		n.Index, _ = strconv.Atoi(strings.TrimSuffix(index, "]"))
+	}
+	key, _, _ = strings.Cut(rest, "[")
+	return pi, n, key, ke.Err.Error(), true
+}
+
+// cutID cuts a plugin path at its id's ']', the first that ends the path or
+// starts its type or config: ids are free text, and keys hold list indexes
+func cutID(path string) (id, rest string) {
+	for i := range len(path) {
+		if after := path[i+1:]; path[i] == ']' && (after == "" || after == ".type" || after == ".config" || strings.HasPrefix(after, ".config.")) {
+			return path[:i], strings.TrimPrefix(after, ".")
+		}
+	}
+	return path, ""
+}
+
 // CommandLine writes the composition as an lw command line, a flag a line,
 // quoted for a POSIX shell
 func (c *Composition) CommandLine() (string, error) {
@@ -549,28 +622,77 @@ func (c *Composition) File() ([]byte, error) {
 	return toml.Marshal(map[string]any{"pipelines": c.Pipelines})
 }
 
-// Form is a way to write a composition, named for what it is: each loads back
+// Form is a way to write a composition, named for what it is, and to read it
-// to the same pipelines
+// back to the same pipelines
 type Form struct {
-	Name, Hint string
+	Name  string                                           `json:"name"`
-	Write      func(*Composition) (string, error)
+	Hint  string                                           `json:"hint"`
+	Write func(*Composition) (string, error)               `json:"-"`
+	Read  func(string, config.IsDir) (*Composition, error) `json:"-"`
 }
 
 var forms = []Form{
-	{"command", "lw's flags", (*Composition).CommandLine},
+	{"command", "lw's flags", (*Composition).CommandLine, FromCommandLine},
-	{"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment},
+	{"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment, FromEnvironment},
 	{"file", "a configuration file's pipelines", func(c *Composition) (string, error) {
 		data, err := c.File()
 		return string(data), err
-	}},
+	}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }},
 }
 
-// Forms are what a composition writes: lw --tui prints one, the website shows them
+// Forms are what a composition writes and reads: lw --tui prints one, the
+// website shows them and reads them pasted
 func Forms() []Form { return slices.Clone(forms) }
 
+// FromEnvironment starts from NAME='value' lines as an environment file holds
+// them, quoted as a POSIX shell reads them: '#' starts a comment line, a word
+// without '=', such as export, assigns nothing, and a later assignment wins
+func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) {
+	vars := map[string]string{}
+	for line := range strings.Lines(text) {
+		if line = strings.TrimSpace(line); line == "" || line[0] == '#' {
+			continue
+		}
+		words, err := split(line)
+		if err != nil {
+			return nil, err
+		}
+		for _, w := range words {
+			if name, value, ok := strings.Cut(w, "="); ok {
+				vars[name] = value
+			}
+		}
+	}
+	var environ []string
+	for name, value := range vars {
+		environ = append(environ, name+"="+value)
+	}
+	pipelines, err := config.EnvPipelines(environ, isDir)
+	if err != nil {
+		return nil, err
+	}
+	c := &Composition{Pipelines: pipelines}
+	return c, c.settle()
+}
+
+// FromFile starts from a configuration file's pipelines, refusing a misspelled
+// key as lw does; its settings are the host's and its console sinks keep the
+// color they name
+func FromFile(data []byte) (*Composition, error) {
+	cfg, err := config.ParseFile(data)
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.Pipelines) == 0 {
+		return nil, errors.New("no [[pipelines]]")
+	}
+	return FromConfig(&config.Config{Pipelines: cfg.Pipelines})
+}
+
 // FromJSON starts from pipelines as the website keeps them: a JSON list of
 // tables in a file's shape, a misspelled key refused as a file's is, a null
-// a value left unset
+// a value left unset. A value of the wrong kind returns the composition too,
+// which Validate names among the rest.
 func FromJSON(data []byte) (*Composition, error) {
 	var tables any
 	if err := json.Unmarshal(data, &tables); err != nil {
@@ -714,10 +836,10 @@ type Schema struct {
 	Settings []Setting         `json:"settings"`
 	Flags    []config.SpecFlag `json:"pipeline_flags"`
 	Flow     []config.Stage    `json:"flow"`
-	Tables   map[string][]config.Key `json:"tables"`
 	Sources  []Plugin          `json:"sources"`
 	Sinks    []Plugin          `json:"sinks"`
 	Presets  []config.Preset   `json:"presets"`
+	Forms    []Form            `json:"forms"`
 }
 
 // Setting is a settings key with its environment variable
@@ -737,12 +859,8 @@ func NewSchema() Schema {
 		Version: version.Short(),
 		Flags:   config.SpecFlags(),
 		Flow:    config.FlowStages(),
-		Tables: map[string][]config.Key{
-			"tls":  config.KeysOf[config.TLSOptions](),
-			"auth": config.KeysOf[config.AuthOptions](),
-			"acl":  config.KeysOf[config.ACLOptions](),
-		},
 		Presets: config.Presets(),
+		Forms:   Forms(),
 	}
 	for _, k := range config.SettingKeys() {
 		s.Settings = append(s.Settings, Setting{k, "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(k.Name, ".", "_"))})

internal/compose/compose_test.go

@@ -1,6 +1,7 @@
 package compose
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -35,11 +36,12 @@ func TestEngineLinksNoHostCode(t *testing.T) {
 	}
 }
 
-// A pasted command line, its line ends LF or CRLF, and the website's JSON each
+// Every form a composition writes, its line ends LF or CRLF, and the
-// start the composition that wrote them: escaped and quoted values, nested
+// website's JSON read back to it: escaped and quoted values, nested keys,
-// keys, lists, moved filters, stages whose flags' defaults differ from a
+// lists, moved filters, stages whose flags' defaults differ from a file's, a
-// file's, a value starting with '-', JSON's numbers for integers
+// value starting with '-', JSON's numbers for integers. The environment holds
-func TestWrittenFormsStartTheSameComposition(t *testing.T) {
+// one pipeline, so each form is written from one or both.
+func TestWrittenFormsReadBack(t *testing.T) {
 	c := &Composition{}
 	must := func(err error) {
 		t.Helper()
@@ -68,18 +70,29 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) {
 	must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100"))
 	must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000"))
 	must(c.Set(0, Node{Role: "heartbeat"}, "enabled", "true"))
+	one := &Composition{Pipelines: slices.Clone(c.Pipelines)}
 	must(c.AddPipeline("-b"))
 	for _, role := range []string{"source", "sink"} {
 		_, err = c.Add(1, role, "null")
 		must(err)
 	}
 	must(c.Set(1, Node{Role: "heartbeat"}, "interval_ms", "2000"))
-	line, err := c.CommandLine()
+	for _, f := range Forms() {
-	must(err)
+		written := 0
-	for _, pasted := range []string{line, strings.ReplaceAll(line, "\n", "\r\n")} {
+		for _, c := range []*Composition{c, one} {
-		got, err := FromCommandLine(pasted, nil)
+			text, err := f.Write(c)
-		if err != nil || !reflect.DeepEqual(got, c) {
+			if err != nil {
-			t.Fatalf("%q\n got %+v %v\nwant %+v", pasted, got, err, c)
+				continue
+			}
+			written++
+			for _, pasted := range []string{text, strings.ReplaceAll(text, "\n", "\r\n")} {
+				if got, err := f.Read(pasted, nil); err != nil || !reflect.DeepEqual(got, c) {
+					t.Fatalf("%s %q\n got %+v %v\nwant %+v", f.Name, pasted, got, err, c)
+				}
+			}
+		}
+		if written == 0 {
+			t.Errorf("%s wrote neither composition", f.Name)
 		}
 	}
 	data, err := c.JSON()
@@ -89,6 +102,52 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) {
 	}
 }
 
+// Pasted environment text reads as a shell runs it: a comment line assigns
+// nothing, export is no assignment, and a later assignment wins
+func TestPastedEnvironmentReadsAsAShellRunsIt(t *testing.T) {
+	c, err := FromEnvironment("export LOGWISP_SOURCE='null'\n# LOGWISP_SOURCE='tcp_chain'\n\nLOGWISP_SINK='tcp:port=1'\nLOGWISP_SINK='null'\n", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := c.Pipelines[0]
+	if len(c.Pipelines) != 1 || len(p.PluginSources) != 1 || p.PluginSources[0].Type != "null" || len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "null" {
+		t.Fatalf("pipelines %+v", c.Pipelines)
+	}
+}
+
+// Locate places a problem by its path: a plugin's key, a key of its table, a
+// rule between its table's keys, a list entry, the plugin itself under an id
+// holding ']', a filter's list entry, a stage and the pipeline; and nothing
+// outside one
+func TestLocatePlacesAProblem(t *testing.T) {
+	sink := func(id string) Node { return Node{Role: "sink", ID: id} }
+	for path, want := range map[string]struct {
+		pi  int
+		n   Node
+		key string
+	}{
+		"pipelines[1].plugin_sinks[out].config.port":            {1, sink("out"), "port"},
+		"pipelines[0].plugin_sinks[out].config.tls.min_version": {0, sink("out"), "tls.min_version"},
+		"pipelines[0].plugin_sinks[out].config.tls":             {0, sink("out"), "tls"},
+		"pipelines[0].plugin_sinks[out].config.acl.allow[1]":    {0, sink("out"), "acl.allow"},
+		"pipelines[0].plugin_sinks[a]b].config":                 {0, sink("a]b"), ""},
+		"pipelines[0].plugin_sources[in].type":                  {0, Node{Role: "source", ID: "in"}, "type"},
+		"pipelines[0].flow.filters[2].patterns[1]":              {0, Node{Role: "filters", Index: 2}, "patterns"},
+		"pipelines[0].flow.rate_limit.policy":                   {0, Node{Role: "rate_limit"}, "policy"},
+		"pipelines[3].name":                                     {3, Node{}, "name"},
+	} {
+		pi, n, key, text, ok := Locate(fmt.Errorf("checked: %w", &config.KeyError{Key: path, Err: errors.New("wrong")}))
+		if !ok || pi != want.pi || n != want.n || key != want.key || text != "wrong" {
+			t.Errorf("%s: %d %+v %q %q %v", path, pi, n, key, text, ok)
+		}
+	}
+	for _, err := range []error{errors.New("no pipelines configured"), &config.KeyError{Key: "color", Err: errors.New("wrong")}} {
+		if _, _, _, text, ok := Locate(err); ok || text != err.Error() {
+			t.Errorf("%v: placed, text %q", err, text)
+		}
+	}
+}
+
 // A null in the website's JSON is a value left unset, which every form then
 // leaves out, in a table or a list
 func TestANullInJSONIsUnset(t *testing.T) {
@@ -105,13 +164,18 @@ func TestANullInJSONIsUnset(t *testing.T) {
 	}
 }
 
-// The website's JSON refuses a misspelled key as a file does, at any depth
+// The website's JSON and a pasted file refuse a misspelled key as lw does,
-func TestAMisspelledJSONKeyIsRefused(t *testing.T) {
+// at any depth
+func TestAMisspelledKeyIsRefused(t *testing.T) {
 	for _, data := range []string{`[{"name":"p","plugin_source":[]}]`, `[{"name":"p","flow":{"rate_limt":{}}}]`} {
 		if _, err := FromJSON([]byte(data)); err == nil || !strings.Contains(err.Error(), "unknown key") {
 			t.Errorf("%s: %v", data, err)
 		}
 	}
+	file := "[[pipelines]]\nname = \"p\"\nnmae = \"q\"\n[[pipelines.plugin_sources]]\ntype = \"null\"\n"
+	if _, err := FromFile([]byte(file)); err == nil || !strings.Contains(err.Error(), `unknown key "pipelines[0].nmae"`) {
+		t.Errorf("file: %v", err)
+	}
 }
 
 // The command line and environment carry no control character, which a

internal/config/config.go

@@ -319,8 +319,8 @@ type AuthOptions struct {
 	CredentialsFile string   `toml:"credentials_file" lw:"listener,hint=file" help:"scram: verifiers, from lw auth add-user"`
 	// exp has whole seconds, a sub-second expires_in reads as unknown, and 10 s
 	// leaves room for renewal ahead of expiry
-	TokenLifetimeMS int64    `toml:"token_lifetime_ms" lw:"listener,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
+	TokenLifetimeMS int64    `toml:"token_lifetime_ms" lw:"listener,http,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
-	TrustedProxies  []string `toml:"trusted_proxies" lw:"listener,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"`
+	TrustedProxies  []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"`
 	Username        string   `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"`
 	PasswordFile    string   `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"`
 }
@@ -335,7 +335,7 @@ type ACLOptions struct {
 	ProxyProtocol              string   `toml:"proxy_protocol" default:"off" lw:"enum=off|optional|required" help:"read a PROXY header from proxy_from"`
 	ProxyFrom                  []string `toml:"proxy_from" lw:"hint=cidr" help:"proxies that may send a PROXY header"`
 	MaxConnectionsPerClient    int64    `toml:"max_connections_per_client" lw:"min=0,zero=unlimited" help:"connections a client may hold"`
-	RequestsPerSecondPerClient float64  `toml:"requests_per_second_per_client" lw:"min=0,zero=unlimited" help:"HTTP: requests a client may make each second"`
+	RequestsPerSecondPerClient float64  `toml:"requests_per_second_per_client" lw:"http,min=0,zero=unlimited" help:"HTTP: requests a client may make each second"`
 }
 
 // --- TLS Options ---

internal/config/loader.go

@@ -51,7 +51,7 @@ func Load(args Args) (*Manager, error) {
 		return nil, err
 	}
 	if len(specs) == 0 {
-		specs = envPipelineSpecs()
+		specs = envPipelineSpecs(os.Environ())
 	}
 	if _, err := buildPipelines(specs, HostIsDir); err != nil {
 		return nil, err
@@ -63,6 +63,7 @@ func Load(args Args) (*Manager, error) {
 		initial.Logging.Level = "warn"
 		initial.StatusReporter = false
 	}
+	var invalid error // the validator's problems, which lw prints a line each
 	cfg, err := lconfig.NewBuilder().
 		WithTarget(initial).
 		WithEnvPrefix("LOGWISP_").
@@ -70,10 +71,10 @@ func Load(args Args) (*Manager, error) {
 		WithFile(configPath).
 		WithTypedValidator(func(cfg *Config) error {
 			effective := *cfg // the builder keeps cfg; only the copy is replaced
-			if err := m.usePipelines(&effective); err != nil {
+			if invalid = m.usePipelines(&effective); invalid == nil {
-				return err
+				invalid = ValidateConfig(&effective)
 			}
-			return ValidateConfig(&effective)
+			return invalid
 		}).
 		WithSecurityOptions(lconfig.SecurityOptions{
 			PreventPathTraversal: true,
@@ -81,7 +82,10 @@ func Load(args Args) (*Manager, error) {
 		}).
 		Build()
 	if err != nil {
-		if !errors.Is(err, lconfig.ErrConfigNotFound) {
+		switch {
+		case invalid != nil:
+			return nil, fmt.Errorf("invalid configuration:\n%w", invalid)
+		case !errors.Is(err, lconfig.ErrConfigNotFound):
 			return nil, fmt.Errorf("failed to load or validate config: %w", err)
 		}
 		if isExplicit {

internal/config/preset.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"reflect"
 	"slices"
 	"strconv"
 	"strings"
@@ -24,22 +25,48 @@ type Preset struct {
 }
 
 // PresetParam is one key. A list takes several values, ',' between them or
-// the key repeated; any other key takes one.
+// the key repeated; any other key takes one. Enum and Hint are those of the
+// option the key sets.
 type PresetParam struct {
 	Name     string   `json:"name"`
 	Default  string   `json:"default,omitempty"`
 	Help     string   `json:"help"`
 	Required bool     `json:"required,omitempty"`
 	List     bool     `json:"list,omitempty"`
+	Enum     []string `json:"enum,omitempty"`
+	Hint     string   `json:"hint,omitempty"`
+}
+
+// Usage is the key's help after its choices, as lw preset and lw --tui show it
+func (p PresetParam) Usage() string {
+	if len(p.Enum) == 0 {
+		return p.Help
+	}
+	return strings.Join(p.Enum, "|") + ": " + p.Help
+}
+
+// mirror gives a key the enum and hint of option key of table T, which it
+// sets; a switch takes true or false
+func mirror[T any](p PresetParam, key string) PresetParam {
+	opts := options(reflect.TypeFor[T]())
+	i := slices.IndexFunc(opts, func(o option) bool { return o.Name == key })
+	if i < 0 {
+		panic(fmt.Sprintf("preset key %s: %s has no key %q", p.Name, reflect.TypeFor[T]().Name(), key))
+	}
+	p.Enum, p.Hint = opts[i].Enum, opts[i].Hint
+	if opts[i].Kind == "bool" {
+		p.Enum = []string{"true", "false"}
+	}
+	return p
 }
 
 // IsDir tells whether a preset's path is a directory, whose files it follows,
-// or a file pattern. Off the target host it is the user's answer, or nil: a
+// or a file pattern. Off the target host it is the user's Answers, or nil: a
 // path ending in '/' is then a directory, one with a glob a pattern, and any
 // other fails with ErrPathKind.
 type IsDir func(path string) (bool, error)
 
-var ErrPathKind = errors.New("path: a directory or a file pattern?")
+var ErrPathKind error = &KeyError{"path", errors.New("a directory or a file pattern?")}
 
 // HostIsDir asks this host: a path that does not exist is a pattern
 func HostIsDir(path string) (bool, error) {
@@ -47,29 +74,67 @@ func HostIsDir(path string) (bool, error) {
 	return err == nil && fi.IsDir(), nil
 }
 
+// Answers is IsDir off the host: whether each path answered is a directory,
+// and nil's rule for the others
+func Answers(dirs map[string]bool) IsDir {
+	return func(path string) (bool, error) {
+		if dir, ok := dirs[path]; ok {
+			return dir, nil
+		}
+		if dir := strings.HasSuffix(path, "/"); dir || strings.ContainsAny(path, `*?[\`) {
+			return dir, nil
+		}
+		return false, ErrPathKind
+	}
+}
+
 // Keys several presets share
 var (
 	pathParam   = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"}
-	fromParam   = PresetParam{Name: "from", Default: "end", Help: "start|end: where reading a file starts"}
+	fromParam   = mirror[FileSourceOptions](PresetParam{Name: "from", Default: "end", Help: "where reading a file starts"}, "from")
-	formatParam = func(def string) PresetParam { return PresetParam{Name: "format", Default: def, Help: "raw|txt|json"} }
+	formatParam = func(def string) PresetParam {
+		return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "entry format"}, "type")
+	}
 	tlsParams = []PresetParam{
-		{Name: "cert", Help: "certificate file, for tls=files"},
+		mirror[TLSOptions](PresetParam{Name: "cert", Help: "certificate file, for tls=files"}, "cert_file"),
-		{Name: "key", Help: "key file, for tls=files"},
+		mirror[TLSOptions](PresetParam{Name: "key", Help: "key file, for tls=files"}, "key_file"),
-		{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"},
+		mirror[TLSOptions](PresetParam{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"}, "issuer_cert_file"),
-		{Name: "issuer_key", Help: "CA key file, for tls=issuer"},
+		mirror[TLSOptions](PresetParam{Name: "issuer_key", Help: "CA key file, for tls=issuer"}, "issuer_key_file"),
 		{Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"},
 	}
 	aclParams = []PresetParam{
-		{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"},
+		mirror[ACLOptions](PresetParam{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, "allow"),
-		{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"},
+		mirror[ACLOptions](PresetParam{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, "deny"),
+	}
+	transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp_chain or http_chain", Enum: transports()}
+	usersParam     = func(who string) PresetParam {
+		return mirror[AuthOptions](PresetParam{Name: "users", Help: "credentials file (lw auth add-user): " + who + " log in with SCRAM"}, "credentials_file")
 	}
-	transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
 	userParams = []PresetParam{
 		{Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"},
-		{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"},
+		mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file"),
 	}
 )
 
+// tlsParam is a listener's tls key: a tlsModes name, off only where the
+// listener may serve plain text
+func tlsParam(def string, off bool) PresetParam {
+	p := PresetParam{Name: "tls", Default: def, Help: "a certificate made at startup (self-signed, or from issuer_*), or files"}
+	if off {
+		p.Help = "plain text, " + p.Help
+	}
+	for _, m := range tlsModes {
+		if m.name != "off" || off {
+			p.Enum = append(p.Enum, m.name)
+		}
+	}
+	return p
+}
+
+// The listeners' tls keys: the aggregator always serves TLS, as edges send
+// credentials and logs
+var serveTLS, aggregatorTLS = tlsParam("off", true), tlsParam("self", false)
+
 var presets = []Preset{
 	{"pipe", "Standard input to standard output, line for line: lw without a file",
 		[]PresetParam{formatParam("raw")},
@@ -90,11 +155,11 @@ var presets = []Preset{
 	{"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream",
 		slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"),
 			{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
-			{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
+			serveTLS,
-			{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
+			usersParam("readers"),
 		}, userParams, []PresetParam{
-			{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
+			mirror[AuthOptions](PresetParam{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, "trusted_proxies"),
-			{Name: "viewer", Default: "false", Help: "true: the login page and viewer, behind proxy; without user or users the viewer is always on"},
+			mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "the login page and viewer, behind proxy; without user or users the viewer is always on"}, "viewer_page"),
 		}, aclParams, tlsParams),
 		func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
 			sources, err := pathOrStdin(v, isDir)
@@ -102,7 +167,7 @@ var presets = []Preset{
 				return err
 			}
 			p.PluginSources = sources
-			sink, err := listener(v, "off")
+			sink, err := listener(v, serveTLS)
 			if err != nil {
 				return err
 			}
@@ -139,12 +204,12 @@ var presets = []Preset{
 		slices.Concat([]PresetParam{pathParam, fromParam,
 			{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"},
 			transportParam,
-			{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"},
+			mirror[TLSOptions](PresetParam{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, "ca_file"),
 			{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
 			{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
 		}, userParams, []PresetParam{
-			{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"},
+			mirror[TLSOptions](PresetParam{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, "cert_file"),
-			{Name: "key", Help: "client key file"},
+			mirror[TLSOptions](PresetParam{Name: "key", Help: "client key file"}, "key_file"),
 			{Name: "node", Help: "origin label; default: this host's name"},
 		}),
 		func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
@@ -188,11 +253,11 @@ var presets = []Preset{
 		slices.Concat([]PresetParam{
 			{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"},
 			transportParam,
-			{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
+			aggregatorTLS,
-			{Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"},
+			usersParam("edges"),
 		}, userParams, []PresetParam{
-			{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
+			mirror[TLSOptions](PresetParam{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, "client_ca_file"),
-			{Name: "out", Help: "directory for the received entries; default: standard output"},
+			mirror[FileSinkOptions](PresetParam{Name: "out", Help: "directory for the received entries; default: standard output"}, "directory"),
 			formatParam("json"),
 		}, aclParams, tlsParams),
 		func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
@@ -200,10 +265,7 @@ var presets = []Preset{
 			if err != nil {
 				return err
 			}
-			if v["tls"] == "off" {
+			src, err := listener(v, aggregatorTLS)
-				return fmt.Errorf("tls=off: edges send credentials and logs, the aggregator always serves TLS")
-			}
-			src, err := listener(v, "self")
 			if err != nil {
 				return err
 			}
@@ -288,13 +350,10 @@ func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error)
 	if path == "" {
 		return pipeDefault("").PluginSources, nil
 	}
-	dir, err := strings.HasSuffix(path, "/"), error(nil)
+	if isDir == nil {
-	switch {
+		isDir = Answers(nil)
-	case isDir != nil:
-		dir, err = isDir(path)
-	case !dir && !strings.ContainsAny(path, `*?[\`):
-		err = ErrPathKind
 	}
+	dir, err := isDir(path)
 	if err != nil {
 		return nil, err
 	}
@@ -308,20 +367,23 @@ func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error)
 // tlsModes is what each preset tls mode sets, and the preset keys it needs
 // by the tls option each fills; made marks a certificate made at startup,
 // which takes hosts.
-var tlsModes = map[string]struct {
+var tlsModes = []tlsMode{
+	{name: "off"},
+	{name: "self", set: map[string]any{"self_signed": true}, made: true},
+	{name: "issuer", needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true},
+	{name: "files", needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
+}
+
+type tlsMode struct {
+	name  string
 	set   map[string]any
 	needs map[string]string
 	made  bool
-}{
-	"off":    {},
-	"self":   {set: map[string]any{"self_signed": true}, made: true},
-	"issuer": {needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true},
-	"files":  {needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
 }
 
-// listener starts a listening plugin's options: host, port, acl and tls,
+// listener starts a listening plugin's options: host, port, acl and tls, in
-// whose mode decides which certificate keys apply.
+// a mode its key offers, which decides the certificate keys that apply
-func listener(v map[string]string, def string) (map[string]any, error) {
+func listener(v map[string]string, key PresetParam) (map[string]any, error) {
 	host, port, err := splitAddr("listen", v["listen"])
 	if err != nil {
 		return nil, err
@@ -335,11 +397,12 @@ func listener(v map[string]string, def string) (map[string]any, error) {
 	if len(acl) > 0 {
 		opts["acl"] = acl
 	}
-	mode := cmp.Or(v["tls"], def)
+	mode := cmp.Or(v["tls"], key.Default)
-	m, ok := tlsModes[mode]
+	i := slices.IndexFunc(tlsModes, func(m tlsMode) bool { return m.name == mode })
-	if !ok {
+	if i < 0 || !slices.Contains(key.Enum, mode) {
-		return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(slices.Sorted(maps.Keys(tlsModes)), ", "))
+		return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(key.Enum, ", "))
 	}
+	m := tlsModes[i]
 	for _, other := range tlsModes {
 		for k := range other.needs {
 			if _, needed := m.needs[k]; needed && v[k] == "" {
@@ -392,13 +455,25 @@ func splitAddr(key, addr string) (string, int64, error) {
 	return host, int64(port), nil
 }
 
+// chainType is a transport's chain plugin type, both a source and a sink
 func chainType(transport string) (string, error) {
-	if transport != "tcp" && transport != "http" {
+	if !slices.Contains(transports(), transport) {
-		return "", fmt.Errorf("transport %q (valid: tcp, http)", transport)
+		return "", fmt.Errorf("transport %q (valid: %s)", transport, strings.Join(transports(), ", "))
 	}
 	return transport + "_chain", nil
 }
 
+// transports name the chain plugins: the catalogue's TRANSPORT_chain types
+func transports() []string {
+	var out []string
+	for _, p := range plugins {
+		if t, ok := strings.CutSuffix(p.Type, "_chain"); ok && p.Role == "sink" {
+			out = append(out, t)
+		}
+	}
+	return out
+}
+
 func list(s string) []any {
 	var out []any
 	for e := range strings.SplitSeq(s, ",") {

internal/config/preset_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"errors"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 )
@@ -56,7 +57,7 @@ func TestPresetKeysAreChecked(t *testing.T) {
 		"edge:to=agg:9000":                 "it never sends unauthenticated",
 		"edge:to=agg:9000,password_file=p": "password_file needs user",
 		"edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
-		"aggregator:users=u,tls=off":                            "the aggregator always serves TLS",
+		"aggregator:users=u,tls=off":                            `tls "off" (valid: `,
 		"aggregator:user=u,users=f":                             "user or users, not both",
 		"aggregator":                                            "it never receives unauthenticated",
 	} {
@@ -67,6 +68,49 @@ func TestPresetKeysAreChecked(t *testing.T) {
 	}
 }
 
+// A preset key that sets an option takes its choices and hint, a switch true
+// or false; a listener's tls offers off only where it may serve plain text
+func TestPresetKeysTakeTheirOptionsChoices(t *testing.T) {
+	param := func(preset, key string) PresetParam {
+		ps := Presets()
+		p := ps[slices.IndexFunc(ps, func(p Preset) bool { return p.Name == preset })]
+		return p.Params[slices.IndexFunc(p.Params, func(k PresetParam) bool { return k.Name == key })]
+	}
+	for _, c := range []struct {
+		preset, key string
+		table       reflect.Type
+		option      string
+	}{
+		{"tail", "from", reflect.TypeFor[FileSourceOptions](), "from"},
+		{"pipe", "format", reflect.TypeFor[FormatConfig](), "type"},
+		{"serve", "proxy", reflect.TypeFor[AuthOptions](), "trusted_proxies"},
+		{"aggregator", "users", reflect.TypeFor[AuthOptions](), "credentials_file"},
+	} {
+		opts := options(c.table)
+		o := opts[slices.IndexFunc(opts, func(o option) bool { return o.Name == c.option })]
+		if p := param(c.preset, c.key); !slices.Equal(p.Enum, o.Enum) || p.Hint != o.Hint || len(o.Enum) == 0 && o.Hint == "" {
+			t.Errorf("%s %s: enum %v hint %q, want %s's %v %q", c.preset, c.key, p.Enum, p.Hint, c.option, o.Enum, o.Hint)
+		}
+	}
+	var modes []string
+	for _, m := range tlsModes {
+		modes = append(modes, m.name)
+	}
+	for _, c := range []struct {
+		preset, key string
+		enum        []string
+	}{
+		{"serve", "viewer", []string{"true", "false"}},
+		{"edge", "transport", transports()},
+		{"serve", "tls", modes},
+		{"aggregator", "tls", slices.DeleteFunc(slices.Clone(modes), func(m string) bool { return m == "off" })},
+	} {
+		if p := param(c.preset, c.key); !slices.Equal(p.Enum, c.enum) {
+			t.Errorf("%s %s: enum %v, want %v", c.preset, c.key, p.Enum, c.enum)
+		}
+	}
+}
+
 // A preset user without password_file is asked for on the terminal
 func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
 	p, err := ExpandPreset("edge", map[string]string{"to": "agg:9000", "user": "u"}, nil)
@@ -78,16 +122,19 @@ func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
 	}
 }
 
-// A preset path is what isDir answers, a directory named with brackets too;
+// A preset path is what isDir answers, a directory named with brackets too,
-// without isDir, off the target host, it is a directory when it ends in '/',
+// and off the target host the user's answer, a single file too; without one
-// a pattern when it holds a glob, and otherwise ErrPathKind asks
+// it is a directory when it ends in '/', a pattern when it holds a glob, and
+// otherwise ErrPathKind asks
 func TestPresetPathKind(t *testing.T) {
 	yes := func(string) (bool, error) { return true, nil }
+	file := Answers(map[string]bool{"/srv/app.log": false})
 	for _, c := range []struct {
 		path  string
 		isDir IsDir
 		want  string
-	}{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""}} {
+	}{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""},
+		{"/srv/app.log", file, "/srv app.log"}, {"/srv/app", file, ""}, {"/srv/app/", file, "/srv/app/ *"}} {
 		p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir)
 		if c.want == "" {
 			if !errors.Is(err, ErrPathKind) {

internal/config/schema.go

@@ -3,6 +3,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"maps"
 	"math"
 	"reflect"
 	"slices"
@@ -57,8 +58,25 @@ var plugins = []Plugin{
 // Plugins lists the catalogue
 func Plugins() []Plugin { return slices.Clone(plugins) }
 
-// Keys describes the plugin's options
+// Keys describes the plugin's options, its tables holding the keys of its
-func (p Plugin) Keys() []Key { return keys(p.options) }
+// side, and HTTP's keys only when it speaks HTTP
+func (p Plugin) Keys() []Key { return p.applying(keys(p.options)) }
+
+func (p Plugin) applying(all []Key) []Key {
+	side, out := "listener", []Key{}
+	if p.Side == Dialer {
+		side = "dialer"
+	}
+	for _, k := range all {
+		if (k.Side == "" || k.Side == side) && (!k.HTTP || p.HTTP) {
+			if k.Keys != nil {
+				k.Keys = p.applying(k.Keys)
+			}
+			out = append(out, k)
+		}
+	}
+	return out
+}
 
 // LookupPlugin finds the catalogue row of a source or sink type
 func LookupPlugin(role, typ string) (Plugin, bool) {
@@ -82,7 +100,8 @@ func (e *KeyError) Unwrap() error { return e.Err }
 // option is one key of an options table, from its field's tags: toml names
 // it, default: holds the default as typed on a command line, help: says what
 // it does, and lw: lists its rules: required, enum=a|b, min=N, max=N,
-// zero=MEANING (zero is a value, not the default), hint=KIND, listener or dialer.
+// zero=MEANING (zero is a value, not the default), hint=KIND, listener or
+// dialer, and http (only plugins that speak HTTP take it).
 type option struct {
 	Key
 	index int
@@ -107,6 +126,8 @@ func options(t reflect.Type) []option {
 				o.Required = true
 			case "listener", "dialer":
 				o.Side = name
+			case "http":
+				o.HTTP = true
 			case "enum":
 				o.Enum = strings.Split(value, "|")
 			case "min", "max":
@@ -133,7 +154,7 @@ func options(t reflect.Type) []option {
 }
 
 // Key describes one option, for lw --schema and the composer: its tags, and
-// its kind, a table's being "table" (tls, auth or acl)
+// its kind, a table's being "table" (tls, auth or acl) with the table's keys
 type Key struct {
 	Name     string   `json:"key"`
 	Kind     string   `json:"kind"`
@@ -146,17 +167,21 @@ type Key struct {
 	Zero     string   `json:"zero,omitempty"`
 	Hint     string   `json:"hint,omitempty"`
 	Side     string   `json:"side,omitempty"`
+	HTTP     bool     `json:"http,omitempty"`
+	Keys     []Key    `json:"keys,omitempty"`
 }
 
 var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer",
 	reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"}
 
 // Stage is a flow stage: its key in the flow table, whether it is a list (the
-// filters), and its options
+// filters), its options, and the values that turn it on, which a file's
+// defaults leave off
 type Stage struct {
 	Key  string         `json:"key"`
 	List bool           `json:"list,omitempty"`
 	Keys []Key          `json:"keys"`
+	On   map[string]any `json:"on,omitempty"`
 }
 
 // FlowStages lists the flow table's stages, in the order entries pass them
@@ -165,20 +190,20 @@ func FlowStages() []Stage {
 	t := reflect.TypeFor[FlowConfig]()
 	for _, o := range options(t) {
 		ft := t.Field(o.index).Type
-		out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem())})
+		out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])})
 	}
 	return out
 }
 
-// KeysOf describes the options of a table: tls, auth or acl
-func KeysOf[T any]() []Key { return keys(reflect.TypeFor[T]()) }
-
 func keys(t reflect.Type) []Key {
-	var out []Key
+	out := []Key{}
 	for _, o := range options(t) {
 		if o.Kind == "" {
 			panic(fmt.Sprintf("%s.%s: no kind", t.Name(), o.Name))
 		}
+		if ft := t.Field(o.index).Type; ft.Kind() == reflect.Pointer {
+			o.Keys = keys(ft.Elem())
+		}
 		out = append(out, o.Key)
 	}
 	return out

internal/config/schema_test.go

@@ -3,6 +3,7 @@ package config
 import (
 	"math"
 	"reflect"
+	"slices"
 	"strings"
 	"testing"
 )
@@ -79,7 +80,8 @@ func TestCoerceGivesTheDeclaredKinds(t *testing.T) {
 }
 
 // ValidateConfig decodes every plugin and flow stage without opening a file,
-// and names the path of what it refuses
+// and names the path of what it refuses: a rule between a table's keys by
+// the table's
 func TestValidateConfigNamesThePath(t *testing.T) {
 	valid := func() *Config {
 		return &Config{Color: "auto", Pipelines: []PipelineConfig{{
@@ -97,9 +99,13 @@ func TestValidateConfigNamesThePath(t *testing.T) {
 		"pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000": func(c *Config) {
 			c.Pipelines[0].PluginSinks[0].Config["port"] = int64(70000)
 		},
-		"pipelines[0].plugin_sinks[out].config: tls: client_auth requires client_ca_file": func(c *Config) {
+		"pipelines[0].plugin_sinks[out].config.tls: client_auth requires client_ca_file": func(c *Config) {
 			c.Pipelines[0].PluginSinks[0].Config["tls"].(map[string]any)["client_auth"] = true
 		},
+		"pipelines[0].plugin_sinks[out].config.acl: requests_per_second_per_client applies only to HTTP listeners": func(c *Config) {
+			c.Pipelines[0].PluginSinks[0].Type = "tcp"
+			c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"requests_per_second_per_client": 2.5}
+		},
 		"pipelines[0].plugin_sinks[out]: duplicate id": func(c *Config) {
 			c.Pipelines[0].PluginSinks = append(c.Pipelines[0].PluginSinks, PluginSinkConfig{ID: "out", Type: "null"})
 		},
@@ -109,7 +115,7 @@ func TestValidateConfigNamesThePath(t *testing.T) {
 		"pipelines[0].plugin_sinks[out].config.port: not an integer": func(c *Config) {
 			c.Pipelines[0].PluginSinks[0].Config["port"] = "http"
 		},
-		`pipelines[0].plugin_sinks[up].config: auth: type "mtls" cannot be used with tls.insecure_skip_verify`: func(c *Config) {
+		`pipelines[0].plugin_sinks[up].config.auth: type "mtls" cannot be used with tls.insecure_skip_verify`: func(c *Config) {
 			c.Pipelines[0].PluginSinks[0] = PluginSinkConfig{ID: "up", Type: "tcp_chain", Config: map[string]any{"host": "h", "port": int64(1),
 				"tls": map[string]any{"enabled": true, "insecure_skip_verify": true}, "auth": map[string]any{"type": "mtls"}}}
 		},
@@ -133,3 +139,52 @@ func TestValidateConfigNamesThePath(t *testing.T) {
 		}
 	}
 }
+
+// A configuration that does not load names every problem, a line each, each
+// part's first: the settings, each flow stage, each plugin, each pipeline
+func TestLoadReportsEveryProblemALine(t *testing.T) {
+	isolateConfig(t)
+	_, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file",
+		"sink", "tcp:port=70000", "rate-limit", "policy=bogus", "filter", "include:patterns=(",
+		"pipeline", "a", "sink", "null")})
+	want := strings.Join([]string{"invalid configuration:",
+		`color: must be one of auto, always, never, got "bogus"`,
+		`pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "bogus"`,
+		"pipelines[0].flow.filters[0].patterns[0]: \"(\": error parsing regexp: missing closing ): `(`",
+		"pipelines[0].plugin_sources[file].config.directory: must be set",
+		"pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000",
+		`pipelines[1].name: duplicate name "a"`,
+	}, "\n")
+	if err == nil || err.Error() != want {
+		t.Fatalf("got\n%v\nwant\n%s", err, want)
+	}
+}
+
+// A plugin lists the table keys that apply to it: those of its side, a chain
+// source's being a listener's, and the HTTP ones only when it speaks HTTP
+func TestTableKeysFollowThePlugin(t *testing.T) {
+	has := func(role, typ, path string) bool {
+		p, _ := LookupPlugin(role, typ)
+		keys := p.Keys()
+		for part := range strings.SplitSeq(path, ".") {
+			i := slices.IndexFunc(keys, func(k Key) bool { return k.Name == part })
+			if i < 0 {
+				return false
+			}
+			keys = keys[i].Keys
+		}
+		return true
+	}
+	for path, want := range map[string]bool{
+		"sink tcp_chain tls.pin_sha256": true, "sink tcp_chain tls.self_signed": false,
+		"source tcp_chain tls.self_signed": true, "source tcp_chain tls.pin_sha256": false,
+		"source tcp_chain auth.node_binding": true, "sink tcp_chain auth.credentials_file": false,
+		"sink tcp acl.requests_per_second_per_client": false, "sink tcp auth.token_lifetime_ms": false,
+		"sink http acl.requests_per_second_per_client": true, "sink http auth.trusted_proxies": true,
+	} {
+		f := strings.Fields(path)
+		if has(f[0], f[1], f[2]) != want {
+			t.Errorf("%s: listed %v, want %v", path, !want, want)
+		}
+	}
+}

internal/config/spec.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"maps"
-	"os"
 	"reflect"
 	"slices"
 	"strconv"
@@ -183,12 +182,15 @@ func cliSpecs(in []Spec) ([]pipelineSpec, error) {
 	return specs, nil
 }
 
-// envPipelineSpecs reads the one-pipeline environment form. An empty variable
+// envPipelineSpecs reads the one-pipeline environment form from environ,
-// counts as unset, so container templates may leave optional ones blank.
+// NAME=value each. An empty variable counts as unset, so container templates
-func envPipelineSpecs() []pipelineSpec {
+// may leave optional ones blank.
+func envPipelineSpecs(environ []string) []pipelineSpec {
 	var specs []pipelineSpec
-	if name := os.Getenv(specVariable("pipeline")); name != "" {
+	for _, kv := range environ {
-		specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name})
+		if name, value, _ := strings.Cut(kv, "="); name == specVariable("pipeline") && value != "" {
+			specs = append(specs, pipelineSpec{name, nil, value})
+		}
 	}
 	for i := range specKinds {
 		kind := &specKinds[i]
@@ -198,7 +200,7 @@ func envPipelineSpecs() []pipelineSpec {
 			name, spec string
 		}
 		var found []numbered
-		for _, kv := range os.Environ() {
+		for _, kv := range environ {
 			name, spec, _ := strings.Cut(kv, "=")
 			suffix, ok := strings.CutPrefix(name, base+"_")
 			n, err := strconv.ParseUint(suffix, 10, 64)
@@ -216,6 +218,16 @@ func envPipelineSpecs() []pipelineSpec {
 	return specs
 }
 
+// EnvPipelines builds the pipeline of LOGWISP_ variables, NAME=value each, as
+// Load does without pipeline flags
+func EnvPipelines(environ []string, isDir IsDir) ([]PipelineConfig, error) {
+	specs := envPipelineSpecs(environ)
+	if len(specs) == 0 {
+		return nil, errors.New("no LOGWISP_ pipeline variables")
+	}
+	return buildPipelines(specs, isDir)
+}
+
 // SpecPipelines builds the pipelines of command-line specs, as Load does
 func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) {
 	s, err := cliSpecs(specs)

internal/config/validate.go

@@ -24,26 +24,20 @@ import (
 
 // ValidateConfig checks a configuration without reading a file it names:
 // the settings, then each pipeline's flow stages, and its sources and sinks
-// decoded in full, naming the path of a refused key. lw --check opens them.
+// decoded in full. Its error joins every problem, one a part, each a
+// *KeyError naming the path of the refused key. lw --check opens the files.
 func ValidateConfig(cfg *Config) error {
 	if cfg == nil {
 		return fmt.Errorf("config is nil")
 	}
-	if err := checkTags(reflect.ValueOf(cfg).Elem(), ""); err != nil {
+	errs := []error{checkTags(reflect.ValueOf(cfg).Elem(), "")}
-		return err
-	}
 	if l := cfg.Logging; l != nil {
-		if err := checkTags(reflect.ValueOf(l).Elem(), "logging."); err != nil {
+		errs = append(errs, checkTags(reflect.ValueOf(l).Elem(), "logging."))
-			return err
-		}
 		if l.Console != nil {
-			if err := checkTags(reflect.ValueOf(l.Console).Elem(), "logging.console."); err != nil {
+			errs = append(errs, checkTags(reflect.ValueOf(l.Console).Elem(), "logging.console."))
-				return err
 		}
 	}
-	}
+	return errors.Join(append(errs, ValidatePipelines(cfg.Pipelines))...)
-
-	return ValidatePipelines(cfg.Pipelines)
 }
 
 // ValidatePipelines is ValidateConfig's check of the pipelines
@@ -51,97 +45,101 @@ func ValidatePipelines(pipelines []PipelineConfig) error {
 	if len(pipelines) == 0 {
 		return fmt.Errorf("no pipelines configured")
 	}
-	// Reject duplicate pipeline names (service map is keyed by name)
+	var errs []error
-	names := make(map[string]struct{}, len(pipelines))
+	at := func(path string, err error) { errs = append(errs, At(path, err)) }
-	for i, p := range pipelines {
+	names := map[string]bool{}    // the service is keyed by name
-		if _, dup := names[p.Name]; dup {
-			return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
-		}
-		names[p.Name] = struct{}{}
-	}
 	single := map[string]string{} // role and type of a single-instance plugin: its pipeline
 	for i, p := range pipelines {
-		if err := lconfig.NonEmpty(p.Name); err != nil {
+		path := fmt.Sprintf("pipelines[%d]", i)
-			return fmt.Errorf("pipeline[%d].name: %w", i, err)
+		switch {
+		case strings.TrimSpace(p.Name) == "":
+			at(path+".name", errors.New("must be set"))
+		case names[p.Name]:
+			at(path+".name", fmt.Errorf("duplicate name %q", p.Name))
 		}
+		names[p.Name] = true
 		if len(p.PluginSources) == 0 {
-			return fmt.Errorf("pipeline[%d]: no sources defined", i)
+			at(path+".plugin_sources", errors.New("no sources defined"))
 		}
 		if len(p.PluginSinks) == 0 {
-			return fmt.Errorf("pipeline[%d]: no sinks defined", i)
+			at(path+".plugin_sinks", errors.New("no sinks defined"))
 		}
-		if err := checkFlow(p.Flow, fmt.Sprintf("pipelines[%d].flow", i)); err != nil {
+		if err := checkFlow(p.Flow, path+".flow"); err != nil {
-			return err
+			errs = append(errs, err)
 		}
 		ids := map[string]bool{}
-		check := func(role, list, id, typ string, m map[string]any) error {
+		check := func(role, list, id, typ string, m map[string]any) {
-			path := fmt.Sprintf("pipelines[%d].%s[%s]", i, list, id)
+			path := fmt.Sprintf("%s.%s[%s]", path, list, id)
 			plugin, ok := LookupPlugin(role, typ)
 			switch {
 			case ids[list+id]:
-				return fmt.Errorf("%s: duplicate id", path)
+				at(path, errors.New("duplicate id"))
 			case !ok:
-				return fmt.Errorf("%s.type: unknown %s type %q", path, role, typ)
+				at(path+".type", fmt.Errorf("unknown %s type %q", role, typ))
 			case plugin.Single && single[role+typ] != "":
-				return fmt.Errorf("pipeline %q: one %s %s may run, and pipeline %q has it", p.Name, typ, role, single[role+typ])
+				at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it", typ, role, single[role+typ]))
-			}
+			default:
 				ids[list+id] = true
 				if plugin.Single {
 					single[role+typ] = p.Name
 				}
 				if _, err := decode(role, typ, m); err != nil {
-				return At(path+".config", err)
+					at(path+".config", err)
+				}
 			}
-			return nil
 		}
 		for _, s := range p.PluginSources {
-			if err := check("source", "plugin_sources", s.ID, s.Type, s.Config); err != nil {
+			check("source", "plugin_sources", s.ID, s.Type, s.Config)
-				return err
-			}
 		}
 		for _, s := range p.PluginSinks {
-			if err := check("sink", "plugin_sinks", s.ID, s.Type, s.Config); err != nil {
+			check("sink", "plugin_sinks", s.ID, s.Type, s.Config)
-				return err
 		}
 	}
-	}
+	return errors.Join(errs...)
-
-	return nil
 }
 
 // checkFlow settles copies of a pipeline's flow stages, which keep their
-// defaults for the dump
+// defaults for the dump: a problem a stage
 func checkFlow(f *FlowConfig, path string) error {
 	if f == nil {
 		return nil
 	}
-	stages := map[string]any{}
+	var errs []error
-	if f.Heartbeat != nil {
+	settle := func(key string, stage any) {
-		stages["heartbeat"] = new(*f.Heartbeat)
+		if err := Settle(stage); err != nil {
+			errs = append(errs, At(path+"."+key, err))
+		}
 	}
 	if f.RateLimit != nil {
-		stages["rate_limit"] = new(*f.RateLimit)
+		settle("rate_limit", new(*f.RateLimit))
-	}
-	if f.Format != nil {
-		stages["format"] = new(*f.Format)
 	}
 	for i, filter := range f.Filters {
-		stages[fmt.Sprintf("filters[%d]", i)] = &filter
+		settle(fmt.Sprintf("filters[%d]", i), &filter)
 	}
-	for _, key := range slices.Sorted(maps.Keys(stages)) {
+	if f.Format != nil {
-		if err := Settle(stages[key]); err != nil {
+		settle("format", new(*f.Format))
-			return At(path+"."+key, err)
 	}
+	if f.Heartbeat != nil {
+		settle("heartbeat", new(*f.Heartbeat))
 	}
-	return nil
+	return errors.Join(errs...)
 }
 
-// At prefixes err with the path of the table it was found in
+// At addresses err to the path of the table it was found in
 func At(path string, err error) error {
 	if ke, ok := errors.AsType[*KeyError](err); ok {
-		return fmt.Errorf("%s.%s: %w", path, ke.Key, ke.Err)
+		return &KeyError{path + "." + ke.Key, ke.Err}
 	}
-	return fmt.Errorf("%s: %w", path, err)
+	return &KeyError{path, err}
+}
+
+// table addresses a rule between a table's keys, which its Check names by
+// a "tls: " prefix, to the table
+func table(key string, err error) error {
+	if _, ok := errors.AsType[*KeyError](err); err == nil || ok {
+		return err
+	}
+	return &KeyError{key, errors.New(strings.TrimPrefix(err.Error(), key+": "))}
 }
 
 // networked is the options of a plugin that listens or dials
@@ -175,12 +173,12 @@ func (p Plugin) checkNetwork(n networked) error {
 		return &KeyError{"host", err}
 	}
 	if err := t.Check(p.Side != Dialer); err != nil {
-		return err
+		return table("tls", err)
 	}
 	if err := acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0); err != nil {
-		return err
+		return table("acl", err)
 	}
-	return a.Check(p.Side, p.HTTP, t.State())
+	return table("auth", a.Check(p.Side, p.HTTP, t.State()))
 }
 
 // Check holds the rules between tls keys on a listener or a dialer; a table
@@ -311,9 +309,9 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
 		return errors.New("auth: username and password_file go together")
 	case o.CredentialsFile == "" && o.Username == "":
 		return fmt.Errorf("auth: type %q requires credentials_file, or username and password_file", "scram")
-	case o.TokenLifetimeMS > 0 && !http:
+	case !http && httpKey(o) != "":
-		return errors.New("auth: token_lifetime_ms applies only to HTTP listeners")
+		return fmt.Errorf("auth: %s applies only to HTTP listeners", httpKey(o))
-	case len(o.TrustedProxies) > 0 && (side != Listener || !http):
+	case len(o.TrustedProxies) > 0 && side != Listener:
 		return errors.New("auth: trusted_proxies applies only to the http sink")
 	case len(o.TrustedProxies) > 0 && o.Identity != "":
 		return errors.New("auth: identity binds a client certificate, which a TLS-terminating proxy does not pass on; drop it or trusted_proxies")
@@ -367,14 +365,13 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error {
 		return err
 	}
 	header := o.ProxyProtocol == "optional" || o.ProxyProtocol == "required"
-	rps := o.RequestsPerSecondPerClient
 	switch {
 	case header && len(o.ProxyFrom) == 0:
 		return fmt.Errorf("acl: proxy_protocol %s needs proxy_from, the proxies that may send the header", o.ProxyProtocol)
 	case !header && len(o.ProxyFrom) > 0:
 		return errors.New("acl: proxy_from needs proxy_protocol optional or required")
-	case rps > 0 && !http:
+	case !http && httpKey(o) != "":
-		return errors.New("acl: requests_per_second_per_client applies only to HTTP listeners")
+		return fmt.Errorf("acl: %s applies only to HTTP listeners", httpKey(o))
 	case o.MaxConnectionsPerClient > 0 && proxied:
 		return errors.New("acl: max_connections_per_client counts connections, behind trusted_proxies the proxy's; cap clients at the proxy")
 	}
@@ -399,6 +396,17 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error {
 	return nil
 }
 
+// httpKey names the first key of table v tagged http that is set
+func httpKey(v any) string {
+	rv := reflect.ValueOf(v).Elem()
+	for _, o := range options(rv.Type()) {
+		if o.HTTP && !rv.Field(o.index).IsZero() {
+			return o.Name
+		}
+	}
+	return ""
+}
+
 // ParsePrefix reads an address or CIDR of network's family ("tcp": either)
 func ParsePrefix(entry, network string) (netip.Prefix, error) {
 	entry = strings.TrimSpace(entry)
@@ -575,13 +583,33 @@ func checkFileKeys(path string) error {
 	if err != nil {
 		return err
 	}
-	root, err := toml.NewParser(data).Parse()
+	root, err := fileTable(data)
 	if err != nil {
 		return err
 	}
+	return checkKeys(root, reflect.TypeOf(Config{}), "")
+}
+
+// ParseFile reads a configuration file's text, refusing the keys lw refuses
+// there, and opens nothing it names
+func ParseFile(data []byte) (*Config, error) {
+	root, err := fileTable(data)
+	if err != nil {
+		return nil, err
+	}
+	cfg := &Config{}
+	return cfg, Scan(root, cfg)
+}
+
+// fileTable is a file's root table without the keys lw ignores there
+func fileTable(data []byte) (map[string]any, error) {
+	root, err := toml.NewParser(data).Parse()
+	if err != nil {
+		return nil, err
+	}
 	delete(root, "config_file") // runtime metadata, documented as ignored in the file
 	for _, key := range []string{"version", "check", "dump"} {
 		delete(root, key) // command-line switches, which earlier lw --dump output wrote
 	}
-	return checkKeys(root, reflect.TypeOf(Config{}), "")
+	return root, nil
 }

internal/tui/canvas.go

@@ -2,7 +2,6 @@ package tui
 
 import (
 	"fmt"
-	"regexp"
 	"slices"
 	"strconv"
 	"strings"
@@ -164,7 +163,7 @@ func join(g ui.Wires, l layout, col int) {
 
 // drawNode draws a source or sink: a tinted bar, its type and whether it
 // listens or dials, then a summary of its options. The chosen node's bar is
-// solid; a node the first problem names has its bar in the error style.
+// solid; a node a problem names has its bar in the error style.
 func (a *app) drawNode(r ui.Region, s spot, n compose.Node, tint ui.Style, chosen bool) {
 	th := a.th
 	bg, bar := th.Text, a.font.bar
@@ -295,41 +294,15 @@ func show(v any) string {
 	return fmt.Sprint(v)
 }
 
-// problemPath reads the part a validation error names: its pipeline, node and
+// faulty reports whether a problem names this node, or a filter when n is
-// key, as config's key paths write them
+// the filters stage
-var problemPath = regexp.MustCompile(`^pipelines?\[(\d+)\](?:\.(plugin_sources|plugin_sinks)\[([^\]]*)\](?:\.config\.?([\w.]*))?|\.flow\.(\w+)(?:\[(\d+)\])?\.?([\w.]*))?`)
-
-// locate reads a problem's part, and its text past the path
-func locate(err error) (pi int, n compose.Node, key, text string, ok bool) {
-	at := problemPath.FindStringSubmatchIndex(err.Error())
-	if at == nil {
-		return 0, n, "", err.Error(), false
-	}
-	m := problemPath.FindStringSubmatch(err.Error())
-	text = strings.TrimPrefix(err.Error()[at[1]:], ": ")
-	pi, _ = strconv.Atoi(m[1])
-	switch {
-	case m[2] != "":
-		n = compose.Node{Role: map[string]string{"plugin_sources": "source", "plugin_sinks": "sink"}[m[2]], ID: m[3]}
-		key = m[4]
-	case m[5] != "":
-		n = compose.Node{Role: m[5]}
-		if m[6] != "" {
-			n.Index, _ = strconv.Atoi(m[6])
-		}
-		key = m[7]
-	}
-	return pi, n, key, text, true
-}
-
-// faulty reports whether the first problem names this node, or a filter
-// when n is the filters stage
 func (a *app) faulty(n compose.Node) bool {
-	if a.problem == nil {
+	for _, p := range compose.Problems(a.problem) {
-		return false
+		if pi, m, _, _, ok := compose.Locate(p); ok && pi == a.pi && m.Role == n.Role && m.ID == n.ID {
+			return true
 		}
-	pi, m, _, _, ok := locate(a.problem)
+	}
-	return ok && pi == a.pi && m.Role == n.Role && m.ID == n.ID
+	return false
 }
 
 // where names a problem's part as the screen shows it: its pipeline when

internal/tui/dialog.go

@@ -265,7 +265,7 @@ func (f *form) draw(a *app, r ui.Region) {
 	}
 	summary, noteRows := wrap(f.summary, w, 2), len(wrap(f.err, w, 3))
 	for _, p := range f.params {
-		noteRows = max(noteRows, len(wrap(p.Help, w, 3)))
+		noteRows = max(noteRows, len(wrap(p.Usage(), w, 3)))
 	}
 	top, bottom := len(summary)+min(1, len(summary)), noteRows+min(1, noteRows)
 	in := a.frame(r, w, top+len(f.fields)*tall+bottom, f.title)
@@ -281,7 +281,7 @@ func (f *form) draw(a *app, r ui.Region) {
 			v.TextInput(f.fields[i], p.Default, i == f.focus.Index, th)
 		}
 	})
-	note, style := f.params[f.focus.Index].Help, th.Muted
+	note, style := f.params[f.focus.Index].Usage(), th.Muted
 	if f.err != "" {
 		note, style = f.err, th.Error
 	}
@@ -456,7 +456,7 @@ func help(f font) dialog {
 		{"p", "start over from a preset"},
 		{"r", "run the pipelines"},
 		{"o", "run, or print a command line, environment or file"},
-		{"!", "the first problem"},
+		{"!", "every problem, moving to the first"},
 		{"paste", "an lw command line replaces the pipelines"},
 		{"q", "quit"},
 	}

internal/tui/inspector.go

@@ -69,8 +69,7 @@ func (a *app) keyLines(out []line, n compose.Node, keys []config.Key, prefix str
 		case "table":
 			out = append(out, line{node: n, key: k, depth: depth, group: name, item: -1})
 			if a.insp.open[a.foldKey(n, k.Name)] {
-				p, _ := a.plugin(n)
+				out = a.keyLines(out, n, k.Keys, k.Name+".", depth+1)
-				out = a.keyLines(out, n, sided(a.tables[name], p.Side), k.Name+".", depth+1)
 			}
 		case "list":
 			out = append(out, line{node: n, key: k, depth: depth, group: name, item: -1})
@@ -180,10 +179,10 @@ func (a *app) nodeTitle(n compose.Node) string {
 	return n.Role + " · flow"
 }
 
-// help is the focused row's problem, or its key's help and default
+// help is the focused row's first problem, or its key's help and default
 func (a *app) help(l line) (string, ui.Style) {
-	if a.problem != nil {
+	for _, p := range compose.Problems(a.problem) {
-		if pi, n, key, text, ok := locate(a.problem); ok && pi == a.pi && n == l.node && key == l.key.Name {
+		if pi, n, key, text, ok := compose.Locate(p); ok && pi == a.pi && n == l.node && key == l.key.Name {
 			return text, a.th.Error
 		}
 	}
@@ -264,7 +263,7 @@ func (a *app) groupSummary(l line) string {
 		return show(v)
 	case map[string]any:
 		var out []string
-		for _, k := range a.tables[l.group] {
+		for _, k := range l.key.Keys {
 			if s, ok := v[k.Name]; ok {
 				out = append(out, k.Name+" "+show(s))
 			}

internal/tui/tui.go

@@ -85,7 +85,6 @@ type app struct {
 	look    look
 	th      ui.Theme
 	font    font
-	tables  map[string][]config.Key
 	pi      int               // the pipeline shown
 	col     int               // the chosen column
 	row     [3]int            // the chosen node of each column
@@ -96,14 +95,14 @@ type app struct {
 	dialog  dialog // takes every key while open
 	status  string // the last edit's refusal
 	note    string // what the last action did
-	problem error  // the composition's first problem
+	problem error  // the composition's problems, joined
 	changed bool
 	done    bool
 	result  Result
 }
 
 func newApp(c *compose.Composition, l look, f font) *app {
-	a := &app{comp: c, look: l, th: l.Theme, font: f, tables: compose.NewSchema().Tables}
+	a := &app{comp: c, look: l, th: l.Theme, font: f}
 	a.th.Glyphs = f.Glyphs
 	a.insp.open = map[string]bool{}
 	if len(c.Pipelines) == 0 {
@@ -167,10 +166,10 @@ func (a *app) plugin(n compose.Node) (config.Plugin, bool) {
 	return config.LookupPlugin(n.Role, typ)
 }
 
-// keys are the options a node takes: a plugin's of its side, a stage's
+// keys are the options a node takes: a plugin's, a stage's
 func (a *app) keys(n compose.Node) []config.Key {
 	if p, ok := a.plugin(n); ok {
-		return sided(p.Keys(), p.Side)
+		return p.Keys()
 	}
 	for _, s := range config.FlowStages() {
 		if s.Key == n.Role {
@@ -180,18 +179,6 @@ func (a *app) keys(n compose.Node) []config.Key {
 	return nil
 }
 
-// sided drops the keys of the other network side
-func sided(keys []config.Key, side config.Side) []config.Key {
-	name := map[config.Side]string{config.Listener: "listener", config.ChainListener: "listener", config.Dialer: "dialer"}[side]
-	var out []config.Key
-	for _, k := range keys {
-		if k.Side == "" || k.Side == name {
-			out = append(out, k)
-		}
-	}
-	return out
-}
-
 // edit applies one engine edit: a refusal is the status, a change revalidates
 func (a *app) edit(err error) bool {
 	if err != nil {
@@ -418,13 +405,13 @@ func (a *app) quit() {
 	a.dialog = &confirm{question: "Quit without running or printing the pipelines?", yes: func(a *app) { a.done = true }}
 }
 
-// showProblem chooses the part the first problem names and shows it whole
+// showProblem chooses the part the first problem names and shows them all
 func (a *app) showProblem() {
 	if a.problem == nil {
 		a.note = "the pipelines are valid"
 		return
 	}
-	if pi, n, key, _, ok := locate(a.problem); ok && pi < len(a.comp.Pipelines) {
+	if pi, n, key, _, ok := compose.Locate(compose.Problems(a.problem)[0]); ok && pi < len(a.comp.Pipelines) {
 		a.pi = pi
 		a.choose(n)
 		a.inspectAt(func(l line) bool { return l.node == n && l.key.Name == key })
@@ -539,7 +526,8 @@ func (a *app) paint(cells []terminal.Cell, w, h int) {
 }
 
 // bottom is the status rows, wrapped to three at most: an edit's refusal, a
-// note, the first problem by its part, or the keys while all is valid
+// note, the first problem by its part and how many follow, or the keys while
+// all is valid
 func (a *app) bottom(w int) ([]string, ui.Style) {
 	th, f := a.th, a.font
 	text, style := "", th.Error
@@ -549,10 +537,14 @@ func (a *app) bottom(w int) ([]string, ui.Style) {
 	case a.note != "":
 		text, style = a.note, th.Muted
 	case a.problem != nil:
-		pi, n, key, msg, _ := locate(a.problem)
+		problems := compose.Problems(a.problem)
+		pi, n, key, msg, _ := compose.Locate(problems[0])
 		if where := a.where(pi, n, key); where != "" {
 			msg = where + ": " + msg
 		}
+		if len(problems) > 1 {
+			msg += fmt.Sprintf(" (%d more: ! shows them)", len(problems)-1)
+		}
 		text = f.invalid + " " + msg
 	default:
 		return []string{f.valid + " valid"}, th.Accent

internal/tui/tui_test.go

@@ -399,18 +399,28 @@ func TestPasteReplacesThePipelines(t *testing.T) {
 	}
 }
 
-// The first problem marks the part it names, and ! moves there, to the key
+// Every problem marks the part it names and its key's row, the status counts
+// those after the first, and ! moves to the first, to the key
 func TestProblemNamesItsPart(t *testing.T) {
-	a := mono(pipeline(t, []string{"null"}, []string{"null", "tcp"}))
+	a := mono(pipeline(t, []string{"null"}, []string{"null", "tcp", "http"}))
 	sink := compose.Node{Role: "sink", ID: "tcp"}
-	if !a.faulty(sink) || a.faulty(compose.Node{Role: "source", ID: "null"}) || a.faulty(compose.Node{Role: "sink", ID: "null"}) {
+	if !a.faulty(sink) || !a.faulty(compose.Node{Role: "sink", ID: "http"}) ||
+		a.faulty(compose.Node{Role: "source", ID: "null"}) || a.faulty(compose.Node{Role: "sink", ID: "null"}) {
 		t.Fatalf("problem %v", a.problem)
 	}
+	if status, _ := a.bottom(200); !strings.Contains(strings.Join(status, " "), "sink tcp · port: must be set (1 more") {
+		t.Fatalf("status %q", status)
+	}
 	press(a, "!")
 	n, _ := a.node()
 	if l := a.lines()[a.insp.cursor]; n != sink || !a.inspect || l.key.Name != "port" {
 		t.Fatalf("on %+v, row %+v", n, l)
 	}
+	a.choose(compose.Node{Role: "sink", ID: "http"})
+	lines := a.lines()
+	if text, style := a.help(lines[slices.IndexFunc(lines, func(l line) bool { return l.key.Name == "port" })]); text != "must be set" || style != a.th.Error {
+		t.Fatalf("http's port row: %q", text)
+	}
 }
 
 // ! and c on the pipeline bar open the inspector, which then takes the keys:
@@ -515,6 +525,19 @@ func presetForm(a *app, name string) *form {
 	return f
 }
 
+// A preset form's note shows the focused key's choices before its help
+func TestPresetFormShowsTheChoices(t *testing.T) {
+	a := mono(pipeline(t, nil, nil))
+	presetForm(a, "tail")
+	press(a, terminal.KeyBacktab)
+	ps := config.Presets()
+	tail := ps[slices.IndexFunc(ps, func(p config.Preset) bool { return p.Name == "tail" })].Params
+	format := tail[slices.IndexFunc(tail, func(p config.PresetParam) bool { return p.Name == "format" })]
+	if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.Contains(screen, format.Usage()) || len(format.Enum) == 0 {
+		t.Fatalf("format's choices not shown:\n%s", screen)
+	}
+}
+
 // A preset form is usable at 25 columns: labels stack above their values,
 // which take the row, so typed text shows whole
 func TestPresetFormFitsANarrowScreen(t *testing.T) {