From 24c3eb1d6ef2830bea0a5d67bc832a78d8fd9939455f5416f3f84c6b97c29d75 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Wed, 14 Oct 2026 23:26:57 -0400 Subject: [PATCH] v0.37.0 lwconf live and fixes --- cmd/lwconf/lwconf.js | 42 ++++++++++++++++++------ cmd/lwconf/lwconf.test.mjs | 56 +++++++++++++++++++++++++------- config/logwisp.toml | 2 +- doc/architecture.md | 5 +-- doc/configuration.md | 5 +-- doc/filters.md | 6 ++-- go.mod | 2 +- go.sum | 4 +-- internal/compose/compose_test.go | 3 +- internal/config/config.go | 2 +- internal/config/schema.go | 8 ++--- internal/config/schema_test.go | 2 +- internal/filter/filter.go | 8 ----- 13 files changed, 97 insertions(+), 48 deletions(-) diff --git a/cmd/lwconf/lwconf.js b/cmd/lwconf/lwconf.js index cfdda5d..31da0b6 100644 --- a/cmd/lwconf/lwconf.js +++ b/cmd/lwconf/lwconf.js @@ -4,23 +4,45 @@ // a rule, so the page draws whatever the engine it loads describes. // load starts the engine: the toolchain's wasm_exec.js by a script element -// (outside a page the caller imports it first), the module fetched with its -// integrity, then its main, which sets globalThis.lwconf -export async function load({ execURL, execSRI, wasmURL, wasmSRI }) { +// (outside a page the caller imports it first), the module fetched and held to +// its integrity, then its main, which sets globalThis.lwconf; progress, if +// given, hears the module's bytes received so far as they arrive +export async function load({ execURL, execSRI, wasmURL, wasmSRI, progress }) { if (!globalThis.Go) await script(execURL, execSRI); const go = new globalThis.Go(); - const fetched = async () => { - const res = await fetch(wasmURL, { integrity: wasmSRI }); - if (!res.ok) throw new Error(`${wasmURL}: ${res.status}`); - return res; - }; - const { instance } = await WebAssembly.instantiateStreaming(fetched(), go.importObject) - .catch(async () => WebAssembly.instantiate(await (await fetched()).arrayBuffer(), go.importObject)); // a server's wrong type + const { instance } = await WebAssembly.instantiate(await fetchHeld(wasmURL, wasmSRI, progress), go.importObject); go.run(instance); if (!globalThis.lwconf) throw new Error("lwconf.wasm did not start"); return new Engine(globalThis.lwconf); } +// fetchHeld counts a file as it arrives, then holds it to its integrity: +// fetch's own check would keep the body back until its last byte. Without +// SubtleCrypto (a page not on https) fetch checks it, and progress hears the end. +async function fetchHeld(url, integrity, progress) { + const subtle = integrity && globalThis.crypto?.subtle; + const res = await fetch(url, subtle ? {} : { integrity }); + if (!res.ok) throw new Error(`${url}: ${res.status}`); + let received = 0; + const counted = res.body.pipeThrough(new TransformStream({ + transform(chunk, out) { received += chunk.byteLength; progress?.(received); out.enqueue(chunk); }, + })); + const bytes = await new Response(counted).arrayBuffer(); + if (subtle && !(await held(subtle, bytes, integrity))) throw new Error(`${url}: not the file its integrity names`); + return bytes; +} + +// held reads integrity as a browser does: of its hashes in the strongest +// algorithm it names, one must be the file's; naming none refuses the file +async function held(subtle, bytes, integrity) { + const algs = { sha256: "SHA-256", sha384: "SHA-384", sha512: "SHA-512" }; // weakest first + const hashes = integrity.trim().split(/\s+/).map(h => h.split("?")[0].split("-")).filter(([alg]) => alg in algs); + const alg = Object.keys(algs).findLast(a => hashes.some(([b]) => b === a)); + if (!alg) return false; + const digest = btoa(String.fromCharCode(...new Uint8Array(await subtle.digest(algs[alg], bytes)))); + return hashes.some(([a, value]) => a === alg && value === digest); +} + function script(src, integrity) { return new Promise((resolve, reject) => { const s = Object.assign(document.createElement("script"), { src, onload: resolve }); diff --git a/cmd/lwconf/lwconf.test.mjs b/cmd/lwconf/lwconf.test.mjs index 21ae60a..460b0c2 100644 --- a/cmd/lwconf/lwconf.test.mjs +++ b/cmd/lwconf/lwconf.test.mjs @@ -6,10 +6,12 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { basename, join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; import { pathToFileURL } from "node:url"; // The drop-in directory make wasm writes over an earlier one's, served as a -// site serves it (?type= another content type) and loaded through lwconf.js; +// site serves it (?type= another content type, ?held the second half of a file +// once rest settles) and loaded through lwconf.js; // wasm_exec.js is imported first, as a page's script element runs it. make runs // on its own, with make verify's toolchain: a GNU make's flags stop a BSD one. const root = join(import.meta.dirname, "../.."); @@ -22,17 +24,22 @@ const wasm = (bin, ...vars) => execFileSync("make", ["-s", "wasm", `BIN_DIR=${bi mkdirSync(dir); writeFileSync(join(dir, "stale.js"), ""); wasm(tmp); -const server = createServer((req, res) => { +let rest = Promise.resolve(); +const server = createServer(async (req, res) => { const { pathname, searchParams } = new URL(req.url, "http://localhost"); const file = join(dir, basename(pathname)); if (!existsSync(file)) return res.writeHead(404).end(); res.setHeader("content-type", searchParams.get("type") ?? (file.endsWith(".wasm") ? "application/wasm" : "text/javascript")); - res.end(readFileSync(file)); + const body = readFileSync(file); + if (!searchParams.has("held")) return res.end(body); + res.write(body.subarray(0, body.length >> 1)); + await rest; + res.end(body.subarray(body.length >> 1)); }); await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); after(() => { server.close(); rmSync(tmp, { recursive: true, force: true }); }); const url = file => `http://127.0.0.1:${server.address().port}/${file}`; -const sri = file => "sha256-" + createHash("sha256").update(readFileSync(join(dir, file))).digest("base64"); +const sri = (file, alg = "sha256") => `${alg}-` + createHash(alg).update(readFileSync(join(dir, file))).digest("base64"); const { load, fields, value, at, group } = await import(pathToFileURL(join(dir, "lwconf.js"))); await import(pathToFileURL(join(dir, "wasm_exec.js"))); @@ -56,20 +63,45 @@ test("make wasm writes a commit's files the same at any time", () => { assert.equal(readFileSync(join(tmp, "again/lwconf/lwconf.json"), "utf8"), readFileSync(join(dir, "lwconf.json"), "utf8")); }); -test("load refuses an engine its integrity does not match, or a server does not have", async () => { - await assert.rejects(load({ wasmURL: url("lwconf.wasm"), wasmSRI: sri("schema.json") })); - await assert.rejects(load({ wasmURL: url("missing.wasm") }), { message: `${url("missing.wasm")}: 404` }); -}); - -test("load takes an engine a server sends as plain bytes", async () => { +// load again, as a page loads its engine +async function fresh(opts) { const api = globalThis.lwconf; delete globalThis.lwconf; try { - const bytes = await load({ wasmURL: url("lwconf.wasm?type=application/octet-stream"), wasmSRI: sri("lwconf.wasm") }); - assert.deepEqual(bytes.schema().value, schema); + return await load(opts); } finally { globalThis.lwconf = api; } +} + +// By the strongest algorithm integrity names; where a page has no SubtleCrypto +// (not on https: crypto.getRandomValues, which Go's runtime needs, stays) fetch checks +test("load refuses an engine its integrity does not match, or a server does not have", async () => { + const wasmURL = url("lwconf.wasm"); + await assert.rejects(fresh({ wasmURL, wasmSRI: sri("schema.json") }), { message: `${wasmURL}: not the file its integrity names` }); + await assert.rejects(fresh({ wasmURL, wasmSRI: `${sri("lwconf.wasm")} ${sri("schema.json", "sha512")}` })); + assert.deepEqual((await fresh({ wasmURL, wasmSRI: `${sri("schema.json")} ${sri("lwconf.wasm", "sha512")}` })).schema().value, schema); + Object.defineProperty(globalThis.crypto, "subtle", { value: undefined, configurable: true }); + try { + await assert.rejects(fresh({ wasmURL, wasmSRI: sri("schema.json") })); + assert.deepEqual((await fresh({ wasmURL, wasmSRI: sri("lwconf.wasm") })).schema().value, schema); + } finally { + delete globalThis.crypto.subtle; + } + await assert.rejects(load({ wasmURL: url("missing.wasm") }), { message: `${url("missing.wasm")}: 404` }); +}); + +// The second half is sent once the first is counted, or two seconds on +test("load takes an engine a server sends as plain bytes, counting them as they arrive", async () => { + const heard = []; + let first, sent = false, early = false; + rest = Promise.race([new Promise(resolve => { first = resolve; }), delay(2000, undefined, { ref: false })]).then(() => { sent = true; }); + const again = await fresh({ wasmURL: url("lwconf.wasm?type=application/octet-stream&held"), wasmSRI: sri("lwconf.wasm"), + progress: n => { heard.push(n); early ||= !sent; first(); } }); + assert.deepEqual(again.schema().value, schema); + assert.ok(early, "counted before the last byte was sent"); + assert.ok(heard.every((n, i) => i === 0 || n > heard[i - 1])); + assert.equal(heard.at(-1), readFileSync(join(dir, "lwconf.wasm")).length); }); // A path neither ending in '/' nor holding a glob is a question, in a preset diff --git a/config/logwisp.toml b/config/logwisp.toml index b6101d9..370e9ad 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -79,7 +79,7 @@ # [[pipelines.flow.filters]] # type = "include" # include|exclude # logic = "or" # or|and: any pattern, or all -# patterns = [] # RE2, e.g. [".*ERROR.*", ".*WARN.*"]; none passes every entry +# patterns = [".*ERROR.*", ".*WARN.*"] # Required (example); RE2, at least one # [pipelines.flow.format] # type = "raw" # raw|txt|json diff --git a/doc/architecture.md b/doc/architecture.md index 6dc9cc0..7b45644 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -265,8 +265,9 @@ so the page needs only `'wasm-unsafe-eval'` in its `script-src`. `lwconf.js` is one dependency-free ES module that knows the calls and shapes, never a key: -- `load({execURL, execSRI, wasmURL, wasmSRI})` fetches both by their - integrity and resolves to the engine. +- `load({execURL, execSRI, wasmURL, wasmSRI, progress})` fetches both by + their integrity and resolves to the engine; `progress` hears the module's + bytes as they arrive, which fetch's own integrity check would hold back. - `fields(schema, role, type)` is a node's form rows, a table's a group. - `value`, `at` and `group` read a node's value and sort problems by node. diff --git a/doc/configuration.md b/doc/configuration.md index 3c2564b..ce35d4d 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -217,8 +217,9 @@ pipelines[0].plugin_sinks[http].config: unknown key "tls.enabeld" key's default, or the meaning its help gives zero (off, none, unlimited). A number below its key's minimum is refused, a negative one included where zero means the default (`buffer_size = -1`). -- Flow stages are checked whether enabled or not: kinds, enums and ranges, - and filter patterns must compile ([Filters](filters.md)). +- Flow stages are checked whether enabled or not: kinds, enums and ranges; + a filter needs at least one pattern, and each must compile + ([Filters](filters.md)). - `auth` enums (`identity`, `node_binding`) are checked whatever `type` is; a block naming peers or credentials with `type` `none` or unset is refused ([Security](security.md#validation-1)). diff --git a/doc/filters.md b/doc/filters.md index 7210bcc..fd8d141 100644 --- a/doc/filters.md +++ b/doc/filters.md @@ -16,10 +16,10 @@ patterns = ["ERROR", "WARN"] `exclude` (matches are dropped). - `logic`, string, default `or`: `or` (any pattern matches) or `and` (every pattern matches). -- `patterns`, []string, default `[]`: Go RE2 regular expressions. +- `patterns`, []string, required: Go RE2 regular expressions. -A filter with no patterns passes everything. An invalid pattern is refused -when the configuration loads, named by its path +A filter needs at least one pattern. A filter without one, or with an invalid +pattern, is refused when the configuration loads, named by its path (`pipelines[0].flow.filters[0].patterns[1]`). ## What Gets Matched diff --git a/go.mod b/go.mod index 3f02b7b..2e47d24 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/lixenwraith/color v0.1.3 github.com/lixenwraith/config v0.2.2 github.com/lixenwraith/log v0.1.12 - github.com/lixenwraith/terminal v0.2.2 + github.com/lixenwraith/terminal v0.3.0 github.com/lixenwraith/toml v0.1.3 golang.org/x/term v0.47.0 ) diff --git a/go.sum b/go.sum index 08fd1a2..e028659 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZ github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg= github.com/lixenwraith/log v0.1.12 h1:0poqOWKPTvxiyv83czKMkBrc4ZShaywq3G/dSb34Jpw= github.com/lixenwraith/log v0.1.12/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg= -github.com/lixenwraith/terminal v0.2.2 h1:ZyAGE+QV10sFyNoEThEkba+ozfEZyns1A6PO84bsbV4= -github.com/lixenwraith/terminal v0.2.2/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY= +github.com/lixenwraith/terminal v0.3.0 h1:PEKsobnZHXKJXf2sUu8VxRfU3/1eG+mKugAo+XbaOVw= +github.com/lixenwraith/terminal v0.3.0/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY= github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI= github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go index f1ce454..f5e5d9e 100644 --- a/internal/compose/compose_test.go +++ b/internal/compose/compose_test.go @@ -64,8 +64,9 @@ func TestWrittenFormsReadBack(t *testing.T) { exclude, err := c.Add(0, "filters", "exclude") must(err) must(c.Set(0, exclude, "patterns", `password=\S{8,64}`, `"quoted"`)) - _, err = c.Add(0, "filters", "include") + include, err := c.Add(0, "filters", "include") must(err) + must(c.Set(0, include, "patterns", "ERROR")) must(c.MoveFilter(0, 1, 0)) must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100")) must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000")) diff --git a/internal/config/config.go b/internal/config/config.go index b7f6db6..76b8676 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -138,7 +138,7 @@ const ( type FilterConfig struct { Type FilterType `toml:"type" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"` Logic FilterLogic `toml:"logic" default:"or" lw:"enum=or|and" help:"match any pattern, or all"` - Patterns []string `toml:"patterns,omitempty" lw:"hint=regex" help:"RE2 patterns"` + Patterns []string `toml:"patterns,omitempty" lw:"required,hint=regex" help:"RE2 patterns"` } // --- Sources --- diff --git a/internal/config/schema.go b/internal/config/schema.go index f111987..63e0c64 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -370,15 +370,15 @@ func setDefault(f reflect.Value, o option) { } // checkTags applies the tag rules to a table's own keys, naming each by -// prefix and key. A zero number is the default or its zero meaning, and -// meets the range unless required; an empty enum is one only by its zero -// or default. +// prefix and key. A required list needs an entry. A zero number is the +// default or its zero meaning, and meets the range unless required; an empty +// enum is one only by its zero or default. func checkTags(v reflect.Value, prefix string) error { for _, o := range options(v.Type()) { f := v.Field(o.index) var err error switch { - case o.Required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == ""): + case o.Required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == "" || f.Kind() == reflect.Slice && f.Len() == 0): err = errors.New("must be set") case f.Kind() == reflect.String && len(o.Enum) > 0 && (f.String() != "" || o.Zero == "" && o.Default == "") && !slices.Contains(o.Enum, f.String()): err = fmt.Errorf("must be one of %s, got %q", strings.Join(o.Enum, ", "), f.String()) diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index 615f7a0..442e12e 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -26,7 +26,7 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) { t.Errorf("%s %s from %v: %v", p.Type, p.Role, m, err) } } - for _, stage := range []any{&HeartbeatConfig{}, &RateLimitConfig{}, &FormatConfig{}, &FilterConfig{}} { + for _, stage := range []any{&HeartbeatConfig{}, &RateLimitConfig{}, &FormatConfig{}, &FilterConfig{Patterns: []string{"1"}}} { if err := Settle(stage); err != nil { t.Errorf("%T: %v", stage, err) } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index cf43362..8e00195 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -58,14 +58,6 @@ func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) { func (f *Filter) Apply(entry core.LogEntry) bool { f.totalProcessed.Add(1) - // No patterns means pass everything - if len(f.patterns) == 0 { - f.logger.Debug("msg", "No patterns configured, passing entry", - "component", "filter", - "type", f.config.Type) - return true - } - // Check against all fields that might contain the log content text := entry.Message if entry.Level != "" {