v0.37.0 lwconf live and fixes

cmd/lwconf/lwconf.js

@@ -4,23 +4,45 @@
 // a rule, so the page draws whatever the engine it loads describes.
 
 // load starts the engine: the toolchain's wasm_exec.js by a script element
-// (outside a page the caller imports it first), the module fetched with its
+// (outside a page the caller imports it first), the module fetched and held to
-// integrity, then its main, which sets globalThis.lwconf
+// its integrity, then its main, which sets globalThis.lwconf; progress, if
-export async function load({ execURL, execSRI, wasmURL, wasmSRI }) {
+// given, hears the module's bytes received so far as they arrive
+export async function load({ execURL, execSRI, wasmURL, wasmSRI, progress }) {
   if (!globalThis.Go) await script(execURL, execSRI);
   const go = new globalThis.Go();
-  const fetched = async () => {
+  const { instance } = await WebAssembly.instantiate(await fetchHeld(wasmURL, wasmSRI, progress), go.importObject);
-    const res = await fetch(wasmURL, { integrity: wasmSRI });
-    if (!res.ok) throw new Error(`${wasmURL}: ${res.status}`);
-    return res;
-  };
-  const { instance } = await WebAssembly.instantiateStreaming(fetched(), go.importObject)
-    .catch(async () => WebAssembly.instantiate(await (await fetched()).arrayBuffer(), go.importObject)); // a server's wrong type
   go.run(instance);
   if (!globalThis.lwconf) throw new Error("lwconf.wasm did not start");
   return new Engine(globalThis.lwconf);
 }
 
+// fetchHeld counts a file as it arrives, then holds it to its integrity:
+// fetch's own check would keep the body back until its last byte. Without
+// SubtleCrypto (a page not on https) fetch checks it, and progress hears the end.
+async function fetchHeld(url, integrity, progress) {
+  const subtle = integrity && globalThis.crypto?.subtle;
+  const res = await fetch(url, subtle ? {} : { integrity });
+  if (!res.ok) throw new Error(`${url}: ${res.status}`);
+  let received = 0;
+  const counted = res.body.pipeThrough(new TransformStream({
+    transform(chunk, out) { received += chunk.byteLength; progress?.(received); out.enqueue(chunk); },
+  }));
+  const bytes = await new Response(counted).arrayBuffer();
+  if (subtle && !(await held(subtle, bytes, integrity))) throw new Error(`${url}: not the file its integrity names`);
+  return bytes;
+}
+
+// held reads integrity as a browser does: of its hashes in the strongest
+// algorithm it names, one must be the file's; naming none refuses the file
+async function held(subtle, bytes, integrity) {
+  const algs = { sha256: "SHA-256", sha384: "SHA-384", sha512: "SHA-512" }; // weakest first
+  const hashes = integrity.trim().split(/\s+/).map(h => h.split("?")[0].split("-")).filter(([alg]) => alg in algs);
+  const alg = Object.keys(algs).findLast(a => hashes.some(([b]) => b === a));
+  if (!alg) return false;
+  const digest = btoa(String.fromCharCode(...new Uint8Array(await subtle.digest(algs[alg], bytes))));
+  return hashes.some(([a, value]) => a === alg && value === digest);
+}
+
 function script(src, integrity) {
   return new Promise((resolve, reject) => {
     const s = Object.assign(document.createElement("script"), { src, onload: resolve });

cmd/lwconf/lwconf.test.mjs

@@ -6,10 +6,12 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync,
 import { createServer } from "node:http";
 import { tmpdir } from "node:os";
 import { basename, join } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
 import { pathToFileURL } from "node:url";
 
 // The drop-in directory make wasm writes over an earlier one's, served as a
-// site serves it (?type= another content type) and loaded through lwconf.js;
+// site serves it (?type= another content type, ?held the second half of a file
+// once rest settles) and loaded through lwconf.js;
 // wasm_exec.js is imported first, as a page's script element runs it. make runs
 // on its own, with make verify's toolchain: a GNU make's flags stop a BSD one.
 const root = join(import.meta.dirname, "../..");
@@ -22,17 +24,22 @@ const wasm = (bin, ...vars) => execFileSync("make", ["-s", "wasm", `BIN_DIR=${bi
 mkdirSync(dir);
 writeFileSync(join(dir, "stale.js"), "");
 wasm(tmp);
-const server = createServer((req, res) => {
+let rest = Promise.resolve();
+const server = createServer(async (req, res) => {
   const { pathname, searchParams } = new URL(req.url, "http://localhost");
   const file = join(dir, basename(pathname));
   if (!existsSync(file)) return res.writeHead(404).end();
   res.setHeader("content-type", searchParams.get("type") ?? (file.endsWith(".wasm") ? "application/wasm" : "text/javascript"));
-  res.end(readFileSync(file));
+  const body = readFileSync(file);
+  if (!searchParams.has("held")) return res.end(body);
+  res.write(body.subarray(0, body.length >> 1));
+  await rest;
+  res.end(body.subarray(body.length >> 1));
 });
 await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
 after(() => { server.close(); rmSync(tmp, { recursive: true, force: true }); });
 const url = file => `http://127.0.0.1:${server.address().port}/${file}`;
-const sri = file => "sha256-" + createHash("sha256").update(readFileSync(join(dir, file))).digest("base64");
+const sri = (file, alg = "sha256") => `${alg}-` + createHash(alg).update(readFileSync(join(dir, file))).digest("base64");
 
 const { load, fields, value, at, group } = await import(pathToFileURL(join(dir, "lwconf.js")));
 await import(pathToFileURL(join(dir, "wasm_exec.js")));
@@ -56,20 +63,45 @@ test("make wasm writes a commit's files the same at any time", () => {
   assert.equal(readFileSync(join(tmp, "again/lwconf/lwconf.json"), "utf8"), readFileSync(join(dir, "lwconf.json"), "utf8"));
 });
 
-test("load refuses an engine its integrity does not match, or a server does not have", async () => {
+// load again, as a page loads its engine
-  await assert.rejects(load({ wasmURL: url("lwconf.wasm"), wasmSRI: sri("schema.json") }));
+async function fresh(opts) {
-  await assert.rejects(load({ wasmURL: url("missing.wasm") }), { message: `${url("missing.wasm")}: 404` });
-});
-
-test("load takes an engine a server sends as plain bytes", async () => {
   const api = globalThis.lwconf;
   delete globalThis.lwconf;
   try {
-    const bytes = await load({ wasmURL: url("lwconf.wasm?type=application/octet-stream"), wasmSRI: sri("lwconf.wasm") });
+    return await load(opts);
-    assert.deepEqual(bytes.schema().value, schema);
   } finally {
     globalThis.lwconf = api;
   }
+}
+
+// By the strongest algorithm integrity names; where a page has no SubtleCrypto
+// (not on https: crypto.getRandomValues, which Go's runtime needs, stays) fetch checks
+test("load refuses an engine its integrity does not match, or a server does not have", async () => {
+  const wasmURL = url("lwconf.wasm");
+  await assert.rejects(fresh({ wasmURL, wasmSRI: sri("schema.json") }), { message: `${wasmURL}: not the file its integrity names` });
+  await assert.rejects(fresh({ wasmURL, wasmSRI: `${sri("lwconf.wasm")} ${sri("schema.json", "sha512")}` }));
+  assert.deepEqual((await fresh({ wasmURL, wasmSRI: `${sri("schema.json")} ${sri("lwconf.wasm", "sha512")}` })).schema().value, schema);
+  Object.defineProperty(globalThis.crypto, "subtle", { value: undefined, configurable: true });
+  try {
+    await assert.rejects(fresh({ wasmURL, wasmSRI: sri("schema.json") }));
+    assert.deepEqual((await fresh({ wasmURL, wasmSRI: sri("lwconf.wasm") })).schema().value, schema);
+  } finally {
+    delete globalThis.crypto.subtle;
+  }
+  await assert.rejects(load({ wasmURL: url("missing.wasm") }), { message: `${url("missing.wasm")}: 404` });
+});
+
+// The second half is sent once the first is counted, or two seconds on
+test("load takes an engine a server sends as plain bytes, counting them as they arrive", async () => {
+  const heard = [];
+  let first, sent = false, early = false;
+  rest = Promise.race([new Promise(resolve => { first = resolve; }), delay(2000, undefined, { ref: false })]).then(() => { sent = true; });
+  const again = await fresh({ wasmURL: url("lwconf.wasm?type=application/octet-stream&held"), wasmSRI: sri("lwconf.wasm"),
+    progress: n => { heard.push(n); early ||= !sent; first(); } });
+  assert.deepEqual(again.schema().value, schema);
+  assert.ok(early, "counted before the last byte was sent");
+  assert.ok(heard.every((n, i) => i === 0 || n > heard[i - 1]));
+  assert.equal(heard.at(-1), readFileSync(join(dir, "lwconf.wasm")).length);
 });
 
 // A path neither ending in '/' nor holding a glob is a question, in a preset

config/logwisp.toml

@@ -79,7 +79,7 @@
 # [[pipelines.flow.filters]]
 # type = "include"                      # include|exclude
 # logic = "or"                          # or|and: any pattern, or all
-# patterns = []                         # RE2, e.g. [".*ERROR.*", ".*WARN.*"]; none passes every entry
+# patterns = [".*ERROR.*", ".*WARN.*"]  # Required (example); RE2, at least one
 
 # [pipelines.flow.format]
 # type = "raw"                          # raw|txt|json

doc/architecture.md

@@ -265,8 +265,9 @@ so the page needs only `'wasm-unsafe-eval'` in its `script-src`.
 `lwconf.js` is one dependency-free ES module that knows the calls and shapes,
 never a key:
 
-- `load({execURL, execSRI, wasmURL, wasmSRI})` fetches both by their
+- `load({execURL, execSRI, wasmURL, wasmSRI, progress})` fetches both by
-  integrity and resolves to the engine.
+  their integrity and resolves to the engine; `progress` hears the module's
+  bytes as they arrive, which fetch's own integrity check would hold back.
 - `fields(schema, role, type)` is a node's form rows, a table's a group.
 - `value`, `at` and `group` read a node's value and sort problems by node.
 

doc/configuration.md

@@ -217,8 +217,9 @@ pipelines[0].plugin_sinks[http].config: unknown key "tls.enabeld"
   key's default, or the meaning its help gives zero (off, none, unlimited). A
   number below its key's minimum is refused, a negative one included where
   zero means the default (`buffer_size = -1`).
-- Flow stages are checked whether enabled or not: kinds, enums and ranges,
+- Flow stages are checked whether enabled or not: kinds, enums and ranges;
-  and filter patterns must compile ([Filters](filters.md)).
+  a filter needs at least one pattern, and each must compile
+  ([Filters](filters.md)).
 - `auth` enums (`identity`, `node_binding`) are checked whatever `type` is; a
   block naming peers or credentials with `type` `none` or unset is refused
   ([Security](security.md#validation-1)).

doc/filters.md

@@ -16,10 +16,10 @@ patterns = ["ERROR", "WARN"]
   `exclude` (matches are dropped).
 - `logic`, string, default `or`: `or` (any pattern matches) or `and` (every
   pattern matches).
-- `patterns`, []string, default `[]`: Go RE2 regular expressions.
+- `patterns`, []string, required: Go RE2 regular expressions.
 
-A filter with no patterns passes everything. An invalid pattern is refused
+A filter needs at least one pattern. A filter without one, or with an invalid
-when the configuration loads, named by its path
+pattern, is refused when the configuration loads, named by its path
 (`pipelines[0].flow.filters[0].patterns[1]`).
 
 ## What Gets Matched

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/lixenwraith/color v0.1.3
 	github.com/lixenwraith/config v0.2.2
 	github.com/lixenwraith/log v0.1.12
-	github.com/lixenwraith/terminal v0.2.2
+	github.com/lixenwraith/terminal v0.3.0
 	github.com/lixenwraith/toml v0.1.3
 	golang.org/x/term v0.47.0
 )

go.sum

@@ -8,8 +8,8 @@ github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZ
 github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg=
 github.com/lixenwraith/log v0.1.12 h1:0poqOWKPTvxiyv83czKMkBrc4ZShaywq3G/dSb34Jpw=
 github.com/lixenwraith/log v0.1.12/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg=
-github.com/lixenwraith/terminal v0.2.2 h1:ZyAGE+QV10sFyNoEThEkba+ozfEZyns1A6PO84bsbV4=
+github.com/lixenwraith/terminal v0.3.0 h1:PEKsobnZHXKJXf2sUu8VxRfU3/1eG+mKugAo+XbaOVw=
-github.com/lixenwraith/terminal v0.2.2/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY=
+github.com/lixenwraith/terminal v0.3.0/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY=
 github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI=
 github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0=
 golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=

internal/compose/compose_test.go

@@ -64,8 +64,9 @@ func TestWrittenFormsReadBack(t *testing.T) {
 	exclude, err := c.Add(0, "filters", "exclude")
 	must(err)
 	must(c.Set(0, exclude, "patterns", `password=\S{8,64}`, `"quoted"`))
-	_, err = c.Add(0, "filters", "include")
+	include, err := c.Add(0, "filters", "include")
 	must(err)
+	must(c.Set(0, include, "patterns", "ERROR"))
 	must(c.MoveFilter(0, 1, 0))
 	must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100"))
 	must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000"))

internal/config/config.go

@@ -138,7 +138,7 @@ const (
 type FilterConfig struct {
 	Type     FilterType  `toml:"type" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"`
 	Logic    FilterLogic `toml:"logic" default:"or" lw:"enum=or|and" help:"match any pattern, or all"`
-	Patterns []string    `toml:"patterns,omitempty" lw:"hint=regex" help:"RE2 patterns"`
+	Patterns []string    `toml:"patterns,omitempty" lw:"required,hint=regex" help:"RE2 patterns"`
 }
 
 // --- Sources ---

internal/config/schema.go

@@ -370,15 +370,15 @@ func setDefault(f reflect.Value, o option) {
 }
 
 // checkTags applies the tag rules to a table's own keys, naming each by
-// prefix and key. A zero number is the default or its zero meaning, and
+// prefix and key. A required list needs an entry. A zero number is the
-// meets the range unless required; an empty enum is one only by its zero
+// default or its zero meaning, and meets the range unless required; an empty
-// or default.
+// enum is one only by its zero or default.
 func checkTags(v reflect.Value, prefix string) error {
 	for _, o := range options(v.Type()) {
 		f := v.Field(o.index)
 		var err error
 		switch {
-		case o.Required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == ""):
+		case o.Required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == "" || f.Kind() == reflect.Slice && f.Len() == 0):
 			err = errors.New("must be set")
 		case f.Kind() == reflect.String && len(o.Enum) > 0 && (f.String() != "" || o.Zero == "" && o.Default == "") && !slices.Contains(o.Enum, f.String()):
 			err = fmt.Errorf("must be one of %s, got %q", strings.Join(o.Enum, ", "), f.String())

internal/config/schema_test.go

@@ -26,7 +26,7 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) {
 			t.Errorf("%s %s from %v: %v", p.Type, p.Role, m, err)
 		}
 	}
-	for _, stage := range []any{&HeartbeatConfig{}, &RateLimitConfig{}, &FormatConfig{}, &FilterConfig{}} {
+	for _, stage := range []any{&HeartbeatConfig{}, &RateLimitConfig{}, &FormatConfig{}, &FilterConfig{Patterns: []string{"1"}}} {
 		if err := Settle(stage); err != nil {
 			t.Errorf("%T: %v", stage, err)
 		}

internal/filter/filter.go

@@ -58,14 +58,6 @@ func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) {
 func (f *Filter) Apply(entry core.LogEntry) bool {
 	f.totalProcessed.Add(1)
 
-	// No patterns means pass everything
-	if len(f.patterns) == 0 {
-		f.logger.Debug("msg", "No patterns configured, passing entry",
-			"component", "filter",
-			"type", f.config.Type)
-		return true
-	}
-
 	// Check against all fields that might contain the log content
 	text := entry.Message
 	if entry.Level != "" {