v0.30.1 configuration engine as wasm
Package / arch (push) Canceled after 0s
Package / user (push) Canceled after 0s
Package / deb (push) Canceled after 0s
Package / freebsd (push) Canceled after 0s
Package / image (push) Canceled after 0s
Test / test (push) Canceled after 0s
Test / e2e (push) Canceled after 0s
Package / arch (push) Canceled after 0s
Package / user (push) Canceled after 0s
Package / deb (push) Canceled after 0s
Package / freebsd (push) Canceled after 0s
Package / image (push) Canceled after 0s
Test / test (push) Canceled after 0s
Test / e2e (push) Canceled after 0s
This commit is contained in:
1 parent
883af0fcaa
commit
bfcc15613a
13 files changed
+320
-52
No files matched your search
@@ -20,6 +20,9 @@ jobs:
|
|||||||
- run: GOOS=freebsd GOARCH=amd64 go build -o /tmp/lw-freebsd ./cmd/lw
|
- run: GOOS=freebsd GOARCH=amd64 go build -o /tmp/lw-freebsd ./cmd/lw
|
||||||
- run: node --test
|
- run: node --test
|
||||||
working-directory: internal/sink/http/web
|
working-directory: internal/sink/http/web
|
||||||
|
# make wasm builds the engine for the page, then node runs its calls
|
||||||
|
- run: node --test
|
||||||
|
working-directory: cmd/lwconf
|
||||||
# The end-to-end scripts on a host with an IPv6 loopback and playwright's
|
# The end-to-end scripts on a host with an IPv6 loopback and playwright's
|
||||||
# Chromium; the docker run needs root, so it skips
|
# Chromium; the docker run needs root, so it skips
|
||||||
e2e:
|
e2e:
|
||||||
|
|||||||
@@ -82,7 +82,8 @@ change touches.
|
|||||||
- `internal/compose` is the engine `lw --tui` and the website's WebAssembly
|
- `internal/compose` is the engine `lw --tui` and the website's WebAssembly
|
||||||
share: it edits pipelines and writes the command line, environment and file
|
share: it edits pipelines and writes the command line, environment and file
|
||||||
forms, each loading back to the same pipelines; it reads no file and links
|
forms, each loading back to the same pipelines; it reads no file and links
|
||||||
no TLS, HTTP or terminal code (its test runs `go list -deps`).
|
no TLS, HTTP or terminal code (its test runs `go list -deps`). `cmd/lwconf`
|
||||||
|
hands it to the page as WebAssembly (`make wasm`; `node --test` there).
|
||||||
- `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands
|
- `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands
|
||||||
table whose groups (`auth`, `tls`, `preset`, `config`) share one flag
|
table whose groups (`auth`, `tls`, `preset`, `config`) share one flag
|
||||||
mechanism; `config.Load` takes the parsed `config.Args`. Presets are rows of
|
mechanism; `config.Load` takes the parsed `config.Args`. Presets are rows of
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ DEB_MAINTAINER ?= Maintainer <maintainer@example.org>
|
|||||||
|
|
||||||
.DEFAULT_GOAL := help
|
.DEFAULT_GOAL := help
|
||||||
|
|
||||||
.PHONY: help check-go build release dev test verify e2e image image-check install uninstall deb arch completion clean version
|
.PHONY: help check-go build release dev wasm test verify e2e image image-check install uninstall deb arch completion clean version
|
||||||
|
|
||||||
help:
|
help:
|
||||||
@echo "Usage: make <target> [VARIABLE=value ...]"
|
@echo "Usage: make <target> [VARIABLE=value ...]"
|
||||||
@@ -78,12 +78,14 @@ help:
|
|||||||
@echo " build Build $(BIN_DIR)/$(BINARY) with version metadata"
|
@echo " build Build $(BIN_DIR)/$(BINARY) with version metadata"
|
||||||
@echo " release Build it static, stripped and trimmed (CGO_ENABLED=0)"
|
@echo " release Build it static, stripped and trimmed (CGO_ENABLED=0)"
|
||||||
@echo " dev Build it with the race detector"
|
@echo " dev Build it with the race detector"
|
||||||
|
@echo " wasm Build $(BIN_DIR)/lwconf.wasm, the engine for the website, beside its"
|
||||||
|
@echo " loader wasm_exec.js, and print both checksums"
|
||||||
@echo " version Print the version metadata a build embeds"
|
@echo " version Print the version metadata a build embeds"
|
||||||
@echo " completion Regenerate $(PKG_DIR)/completion from lw's command tables"
|
@echo " completion Regenerate $(PKG_DIR)/completion from lw's command tables"
|
||||||
@echo " clean Remove $(BIN_DIR)/"
|
@echo " clean Remove $(BIN_DIR)/"
|
||||||
@echo "Check:"
|
@echo "Check:"
|
||||||
@echo " test Run the Go tests"
|
@echo " test Run the Go tests"
|
||||||
@echo " verify vet, gofmt, tests, linux and freebsd cross builds, web client tests"
|
@echo " verify vet, gofmt, tests, linux, freebsd and wasm cross builds, node tests"
|
||||||
@echo " e2e Build, then run each of $(E2E) with --auto and report"
|
@echo " e2e Build, then run each of $(E2E) with --auto and report"
|
||||||
@echo "Container:"
|
@echo "Container:"
|
||||||
@echo " image Build $(IMAGE):$(IMAGE_TAG) with $(CONTAINER_ENGINE) (scratch, static, UID 65532)"
|
@echo " image Build $(IMAGE):$(IMAGE_TAG) with $(CONTAINER_ENGINE) (scratch, static, UID 65532)"
|
||||||
@@ -120,6 +122,13 @@ release: check-go
|
|||||||
dev: check-go
|
dev: check-go
|
||||||
$(GO) build -race $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC)
|
$(GO) build -race $(GO_BUILDFLAGS) -ldflags "$(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/$(BINARY) $(SRC)
|
||||||
|
|
||||||
|
# The page loads the toolchain's own loader, which uses no eval: its CSP needs
|
||||||
|
# only 'wasm-unsafe-eval'
|
||||||
|
wasm: check-go
|
||||||
|
GOOS=js GOARCH=wasm $(GO) build -trimpath $(GO_BUILDFLAGS) -ldflags "-s -w $(VERSION_LDFLAGS) $(GO_LDFLAGS)" -o $(BIN_DIR)/lwconf.wasm ./cmd/lwconf
|
||||||
|
install -m 0644 "$$($(GO) env GOROOT)/lib/wasm/wasm_exec.js" $(BIN_DIR)/wasm_exec.js
|
||||||
|
@cd $(BIN_DIR) && { sha256sum lwconf.wasm wasm_exec.js 2>/dev/null || sha256 -r lwconf.wasm wasm_exec.js; }
|
||||||
|
|
||||||
test: check-go
|
test: check-go
|
||||||
$(GO) test ./...
|
$(GO) test ./...
|
||||||
|
|
||||||
@@ -146,10 +155,11 @@ verify: test
|
|||||||
echo "cross build $$target"; \
|
echo "cross build $$target"; \
|
||||||
CGO_ENABLED=0 GOOS=$${target%/*} GOARCH=$${target#*/} $(GO) build -o /dev/null $(SRC) || exit 1; \
|
CGO_ENABLED=0 GOOS=$${target%/*} GOARCH=$${target#*/} $(GO) build -o /dev/null $(SRC) || exit 1; \
|
||||||
done
|
done
|
||||||
|
@echo "cross build js/wasm"; GOOS=js GOARCH=wasm $(GO) build -o /dev/null ./cmd/lwconf
|
||||||
@if command -v node >/dev/null 2>&1; then \
|
@if command -v node >/dev/null 2>&1; then \
|
||||||
cd $(WEB_DIR) && node --test; \
|
(cd $(WEB_DIR) && node --test) && cd cmd/lwconf && node --test; \
|
||||||
else \
|
else \
|
||||||
echo "node not found: web client tests skipped"; \
|
echo "node not found: web client and lwconf tests skipped"; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# A script exits 0 (passed), 77 (skipped: the host lacks what it tests) or
|
# A script exits 0 (passed), 77 (skipped: the host lacks what it tests) or
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import { after, test } from "node:test";
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { execFileSync } from "node:child_process";
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||||
|
import { tmpdir } from "node:os";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { pathToFileURL } from "node:url";
|
||||||
|
|
||||||
|
// Built by make wasm and started as the page starts it: the toolchain's loader
|
||||||
|
// beside the module, main running until the page closes. The make run is its
|
||||||
|
// own: a parent GNU make's flags would stop a BSD one.
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "lwconf-"));
|
||||||
|
after(() => rmSync(dir, { recursive: true, force: true }));
|
||||||
|
const made = execFileSync("make", ["-s", "wasm", `BIN_DIR=${dir}`, "VERSION=v0.0.0-node"], {
|
||||||
|
cwd: join(import.meta.dirname, "../.."), encoding: "utf8",
|
||||||
|
env: { ...process.env, MAKEFLAGS: "", MFLAGS: "", MAKELEVEL: "" },
|
||||||
|
});
|
||||||
|
await import(pathToFileURL(join(dir, "wasm_exec.js")));
|
||||||
|
const go = new globalThis.Go();
|
||||||
|
const { instance } = await WebAssembly.instantiate(readFileSync(join(dir, "lwconf.wasm")), go.importObject);
|
||||||
|
go.run(instance);
|
||||||
|
const call = (name, ...args) => JSON.parse(globalThis.lwconf[name](...args));
|
||||||
|
|
||||||
|
test("make wasm stamps the version and prints both checksums", () => {
|
||||||
|
for (const file of ["lwconf.wasm", "wasm_exec.js"]) {
|
||||||
|
const sum = createHash("sha256").update(readFileSync(join(dir, file))).digest("hex");
|
||||||
|
assert.match(made, new RegExp(`^${sum} +${file.replace(".", "\\.")}$`, "m"));
|
||||||
|
}
|
||||||
|
assert.equal(call("schema").value.version, "v0.0.0-node");
|
||||||
|
});
|
||||||
|
|
||||||
|
// Off the host a directory path ends in '/'
|
||||||
|
test("a preset's pipelines, written as a command line, parse and validate back to themselves", () => {
|
||||||
|
const { value: pipelines } = call("preset", "tail", JSON.stringify({ path: "/var/log/" }));
|
||||||
|
assert.equal(pipelines[0].plugin_sources[0].config.directory, "/var/log/");
|
||||||
|
const { value: line } = call("emit", JSON.stringify(pipelines), "command");
|
||||||
|
assert.match(line, /^lw \\\n {2}--pipeline tail/);
|
||||||
|
assert.deepEqual(call("parse", line).value, pipelines);
|
||||||
|
assert.deepEqual(call("validate", JSON.stringify(pipelines)).value, pipelines);
|
||||||
|
assert.match(call("emit", JSON.stringify(pipelines), "file").value, /^\[\[pipelines\]\]/m);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("a refused call answers why, and the engine keeps answering", () => {
|
||||||
|
const refused = [
|
||||||
|
[["parse", "lw --nope x"], /not a pipeline flag/],
|
||||||
|
[["parse", "lw"], /no pipeline flags/],
|
||||||
|
[["parse"], /^usage: lwconf\.parse\(line\)/],
|
||||||
|
[["parse", 5], /^usage: /],
|
||||||
|
[["preset", "tail", JSON.stringify({ path: "/var/log" })], /path/],
|
||||||
|
[["validate", JSON.stringify([{ name: "p" }])], /no sources/],
|
||||||
|
[["validate", JSON.stringify(["a", "b"].map((name, i) => ({
|
||||||
|
name, plugin_sources: [{ type: "tcp_chain", config: { port: [9000, "x"][i] } }], plugin_sinks: [{ type: "null" }],
|
||||||
|
})))], /^pipelines\[1\]\.plugin_sources\[tcp_chain\]\.config\.port: not an integer$/],
|
||||||
|
[["emit", JSON.stringify([{ name: "p" }]), "command"], /no sources/],
|
||||||
|
[["emit", "[]", "yaml"], /no form "yaml"/],
|
||||||
|
];
|
||||||
|
for (const [args, want] of refused) {
|
||||||
|
const got = call(...args);
|
||||||
|
assert.equal(got.value, undefined, `${args}`);
|
||||||
|
assert.match(got.error, want, `${args}`);
|
||||||
|
}
|
||||||
|
assert.ok(call("schema").value.sources.length > 0);
|
||||||
|
});
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
//go:build js && wasm
|
||||||
|
|
||||||
|
// Command lwconf is the configuration engine as WebAssembly, for the website.
|
||||||
|
// It sets globalThis.lwconf, whose calls take strings and return a JSON object:
|
||||||
|
// {"value": ...}, or {"error": "..."} for a call refused. Pipelines travel in
|
||||||
|
// compose's JSON form; off the host a preset's directory path ends in '/'.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"syscall/js"
|
||||||
|
|
||||||
|
"github.com/lixenwraith/logwisp/internal/compose"
|
||||||
|
)
|
||||||
|
|
||||||
|
// calls are lwconf's, by name, with the string arguments each takes
|
||||||
|
var calls = map[string]struct {
|
||||||
|
args []string
|
||||||
|
call func(args []string) (any, error)
|
||||||
|
}{
|
||||||
|
"schema": {nil, func([]string) (any, error) { return compose.NewSchema(), nil }},
|
||||||
|
"preset": {[]string{"name", "values"}, func(a []string) (any, error) {
|
||||||
|
var values map[string]string
|
||||||
|
if err := json.Unmarshal([]byte(a[1]), &values); err != nil {
|
||||||
|
return nil, fmt.Errorf("values: %w", err)
|
||||||
|
}
|
||||||
|
return pipelines(compose.FromPreset(a[0], values, nil))
|
||||||
|
}},
|
||||||
|
"parse": {[]string{"line"}, func(a []string) (any, error) {
|
||||||
|
return pipelines(compose.FromCommandLine(a[0], nil))
|
||||||
|
}},
|
||||||
|
"validate": {[]string{"pipelines"}, func(a []string) (any, error) {
|
||||||
|
c, err := compose.FromJSON([]byte(a[0]))
|
||||||
|
if err == nil {
|
||||||
|
err = c.Validate()
|
||||||
|
}
|
||||||
|
return pipelines(c, err)
|
||||||
|
}},
|
||||||
|
"emit": {[]string{"pipelines", "form"}, func(a []string) (any, error) {
|
||||||
|
c, err := compose.FromJSON([]byte(a[0]))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
forms := compose.Forms()
|
||||||
|
if i := slices.IndexFunc(forms, func(f compose.Form) bool { return f.Name == a[1] }); i >= 0 {
|
||||||
|
return forms[i].Write(c)
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("no form %q", a[1])
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
// pipelines is a composition's value: its pipelines settled, ids given
|
||||||
|
func pipelines(c *compose.Composition, err error) (any, error) {
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
data, err := c.JSON()
|
||||||
|
return json.RawMessage(data), err
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
api := map[string]any{}
|
||||||
|
for name := range calls {
|
||||||
|
api[name] = js.FuncOf(func(_ js.Value, args []js.Value) any { return answer(name, args) })
|
||||||
|
}
|
||||||
|
js.Global().Set("lwconf", js.ValueOf(api))
|
||||||
|
select {} // the page calls in until it closes
|
||||||
|
}
|
||||||
|
|
||||||
|
// answer is a call's JSON for the page: its value, or why it was refused
|
||||||
|
func answer(name string, args []js.Value) string {
|
||||||
|
c := calls[name]
|
||||||
|
value, err := any(nil), fmt.Errorf("usage: lwconf.%s(%s), each a string", name, strings.Join(c.args, ", "))
|
||||||
|
if len(args) == len(c.args) && !slices.ContainsFunc(args, func(v js.Value) bool { return v.Type() != js.TypeString }) {
|
||||||
|
strs := make([]string, len(args))
|
||||||
|
for i, arg := range args {
|
||||||
|
strs[i] = arg.String()
|
||||||
|
}
|
||||||
|
value, err = c.call(strs)
|
||||||
|
}
|
||||||
|
out := map[string]any{"value": value}
|
||||||
|
if err != nil {
|
||||||
|
out = map[string]any{"error": err.Error()}
|
||||||
|
}
|
||||||
|
data, _ := json.Marshal(out)
|
||||||
|
return string(data)
|
||||||
|
}
|
||||||
+6
-2
@@ -26,6 +26,9 @@ The Makefile works with GNU make and BSD make alike. Targets:
|
|||||||
- `make build`: `bin/lw`, with version, commit and build time from git
|
- `make build`: `bin/lw`, with version, commit and build time from git
|
||||||
- `make release`: the same, static (`CGO_ENABLED=0`), `-trimpath`, stripped
|
- `make release`: the same, static (`CGO_ENABLED=0`), `-trimpath`, stripped
|
||||||
- `make dev`: built with the race detector
|
- `make dev`: built with the race detector
|
||||||
|
- `make wasm`: `bin/lwconf.wasm`, the configuration engine for the
|
||||||
|
website, version stamped, beside the toolchain's `wasm_exec.js`; prints
|
||||||
|
both checksums
|
||||||
- `make version`: the metadata a build would embed
|
- `make version`: the metadata a build would embed
|
||||||
- `make completion`: regenerate the bash, zsh and fish completion in
|
- `make completion`: regenerate the bash, zsh and fish completion in
|
||||||
`deploy/package/completion/` from `cmd/lw`'s `shorts` and `commands`
|
`deploy/package/completion/` from `cmd/lw`'s `shorts` and `commands`
|
||||||
@@ -34,8 +37,9 @@ The Makefile works with GNU make and BSD make alike. Targets:
|
|||||||
- Check
|
- Check
|
||||||
- `make test`: the Go tests
|
- `make test`: the Go tests
|
||||||
- `make verify`: tests, `go vet`, `gofmt -l` on the Go files changed since
|
- `make verify`: tests, `go vet`, `gofmt -l` on the Go files changed since
|
||||||
`main`, linux and freebsd cross builds for amd64 and arm64, and the web
|
`main`, linux and freebsd cross builds for amd64 and arm64 and the js/wasm
|
||||||
client's `node --test` when node is installed
|
one, and, when node is installed, `node --test` for the web client and
|
||||||
|
for `cmd/lwconf` (which runs `make wasm`)
|
||||||
- `make e2e`: builds, then runs every `test/*-test.sh --auto` in turn and
|
- `make e2e`: builds, then runs every `test/*-test.sh --auto` in turn and
|
||||||
reports passes, failures and skips; `E2E='test/scram-*-test.sh'` runs a
|
reports passes, failures and skips; `E2E='test/scram-*-test.sh'` runs a
|
||||||
subset; see [Test Scripts](#test-scripts)
|
subset; see [Test Scripts](#test-scripts)
|
||||||
|
|||||||
+17
-10
@@ -114,14 +114,21 @@ Steps:
|
|||||||
optional, in `color`: export the 16 xterm reference shades (RGBTo16 maps
|
optional, in `color`: export the 16 xterm reference shades (RGBTo16 maps
|
||||||
the site's hues badly, so the 16-color tier is chosen by hand).
|
the site's hues badly, so the 16-color tier is chosen by hand).
|
||||||
5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)).
|
5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)).
|
||||||
6. WebAssembly: `cmd/lwconf` (`js && wasm`) exposes schema, preset, parse,
|
6. WebAssembly: done. `make wasm` builds `bin/lwconf.wasm`, version stamped,
|
||||||
validate and emit to the page as JSON strings. `make wasm` builds
|
beside the toolchain's `wasm_exec.js`, and prints both checksums: about
|
||||||
`bin/lwconf.wasm` with the version stamped, copies the toolchain's
|
6.9 MB, 1.9 MB gzipped. `wasm_exec.js` uses no eval, so the page needs only
|
||||||
`wasm_exec.js`, and prints both checksums. A prototype measured about 6 MB,
|
`'wasm-unsafe-eval'` in its `script-src`. The page runs the module with
|
||||||
1.7 MB gzipped, ready in about 60 ms in Chromium; `wasm_exec.js` uses no
|
`new Go().run(instance)`, then calls `globalThis.lwconf`; each call takes
|
||||||
eval, so the page needs only `'wasm-unsafe-eval'` in its `script-src`.
|
strings and returns a JSON object, `{"value": ...}` or `{"error": "..."}`:
|
||||||
|
- `schema()`: what `lw --schema` prints.
|
||||||
|
- `preset(name, values)`: a preset's pipelines; `values` is a JSON object
|
||||||
|
of strings, and a directory path ends in `/`.
|
||||||
|
- `parse(line)`: a pasted command line's pipelines.
|
||||||
|
- `validate(pipelines)`: the pipelines settled (defaults, kinds, ids), or
|
||||||
|
the first problem by its key path.
|
||||||
|
- `emit(pipelines, form)`: `command`, `environment` or `file`.
|
||||||
|
- Pipelines are a JSON list in the file's shape (`name`, `flow`,
|
||||||
|
`plugin_sources`, `plugin_sinks`); a misspelled key is refused.
|
||||||
|
|
||||||
Verify:
|
Still to verify: a scripted session on a pseudo-terminal for Run and each
|
||||||
- CI builds `cmd/lwconf` for `GOOS=js` and runs a node test of its calls.
|
output.
|
||||||
- The diagram rendered into a cell buffer at 80x24 and 64x24, per tier; a
|
|
||||||
scripted session on a pseudo-terminal for Run and each output.
|
|
||||||
@@ -5,6 +5,7 @@
|
|||||||
package compose
|
package compose
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
@@ -59,7 +60,8 @@ func FromPreset(name string, values map[string]string, isDir config.IsDir) (*Com
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FromCommandLine starts from a pasted lw command line, quoted as a POSIX
|
// FromCommandLine starts from a pasted lw command line, quoted as a POSIX
|
||||||
// shell reads it; it takes pipeline flags only, as a composition holds no settings
|
// shell reads it; it takes pipeline flags only, at least one, as a composition
|
||||||
|
// holds no settings
|
||||||
func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
|
func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
|
||||||
words, err := split(strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(line)) // pasted line ends
|
words, err := split(strings.NewReplacer("\r\n", "\n", "\r", "\n").Replace(line)) // pasted line ends
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -86,6 +88,9 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
|
|||||||
}
|
}
|
||||||
specs = append(specs, config.Spec{Flag: flag, Value: value})
|
specs = append(specs, config.Spec{Flag: flag, Value: value})
|
||||||
}
|
}
|
||||||
|
if len(specs) == 0 {
|
||||||
|
return nil, errors.New("no pipeline flags")
|
||||||
|
}
|
||||||
pipelines, err := config.SpecPipelines(specs, isDir)
|
pipelines, err := config.SpecPipelines(specs, isDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -117,13 +122,13 @@ func (c *Composition) settle() error {
|
|||||||
for j, s := range p.PluginSources {
|
for j, s := range p.PluginSources {
|
||||||
p.PluginSources[j].Config = orEmpty(s.Config)
|
p.PluginSources[j].Config = orEmpty(s.Config)
|
||||||
if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
|
if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
|
||||||
return fmt.Errorf("source %s: %w", s.ID, err)
|
return config.At(fmt.Sprintf("pipelines[%d].plugin_sources[%s].config", i, s.ID), err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for j, s := range p.PluginSinks {
|
for j, s := range p.PluginSinks {
|
||||||
p.PluginSinks[j].Config = orEmpty(s.Config)
|
p.PluginSinks[j].Config = orEmpty(s.Config)
|
||||||
if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
|
if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
|
||||||
return fmt.Errorf("sink %s: %w", s.ID, err)
|
return config.At(fmt.Sprintf("pipelines[%d].plugin_sinks[%s].config", i, s.ID), err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -477,6 +482,75 @@ func (c *Composition) File() ([]byte, error) {
|
|||||||
return toml.Marshal(map[string]any{"pipelines": c.Pipelines})
|
return toml.Marshal(map[string]any{"pipelines": c.Pipelines})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Form is a way to write a composition, named for what it is: each loads back
|
||||||
|
// to the same pipelines
|
||||||
|
type Form struct {
|
||||||
|
Name, Hint string
|
||||||
|
Write func(*Composition) (string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
var forms = []Form{
|
||||||
|
{"command", "lw's flags", (*Composition).CommandLine},
|
||||||
|
{"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment},
|
||||||
|
{"file", "a configuration file's pipelines", func(c *Composition) (string, error) {
|
||||||
|
data, err := c.File()
|
||||||
|
return string(data), err
|
||||||
|
}},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Forms are what a composition writes: lw --tui prints one, the website shows them
|
||||||
|
func Forms() []Form { return slices.Clone(forms) }
|
||||||
|
|
||||||
|
// FromJSON starts from pipelines as the website keeps them: a JSON list of
|
||||||
|
// tables in a file's shape, a misspelled key refused as a file's is, a null
|
||||||
|
// a value left unset
|
||||||
|
func FromJSON(data []byte) (*Composition, error) {
|
||||||
|
var tables any
|
||||||
|
if err := json.Unmarshal(data, &tables); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cfg := &config.Config{}
|
||||||
|
if err := config.Scan(map[string]any{"pipelines": dropNulls(tables)}, cfg); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return FromConfig(cfg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropNulls removes JSON's nulls from tables and lists, which the shell forms
|
||||||
|
// would write as text
|
||||||
|
func dropNulls(v any) any {
|
||||||
|
switch v := v.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
for k, e := range v {
|
||||||
|
if e == nil {
|
||||||
|
delete(v, k)
|
||||||
|
} else {
|
||||||
|
v[k] = dropNulls(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case []any:
|
||||||
|
v = slices.DeleteFunc(v, func(e any) bool { return e == nil })
|
||||||
|
for i, e := range v {
|
||||||
|
v[i] = dropNulls(e)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// JSON writes the pipelines as FromJSON reads them, keyed as File writes them
|
||||||
|
func (c *Composition) JSON() ([]byte, error) {
|
||||||
|
data, err := toml.Marshal(map[string]any{"pipelines": c.Pipelines})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
root, err := toml.NewParser(data).Parse()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return json.Marshal(root["pipelines"])
|
||||||
|
}
|
||||||
|
|
||||||
// quote leaves a word of these characters bare, and single-quotes any other,
|
// quote leaves a word of these characters bare, and single-quotes any other,
|
||||||
// or one starting with '=', which zsh expands
|
// or one starting with '=', which zsh expands
|
||||||
func quote(word string) string {
|
func quote(word string) string {
|
||||||
|
|||||||
@@ -34,10 +34,11 @@ func TestEngineLinksNoHostCode(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A pasted command line, its line ends LF or CRLF, starts the composition that
|
// A pasted command line, its line ends LF or CRLF, and the website's JSON each
|
||||||
// wrote it: escaped and quoted values, nested keys, lists, moved filters,
|
// start the composition that wrote them: escaped and quoted values, nested
|
||||||
// stages whose flags' defaults differ from a file's, a value starting with '-'
|
// keys, lists, moved filters, stages whose flags' defaults differ from a
|
||||||
func TestPastedCommandLineStartsTheSameComposition(t *testing.T) {
|
// file's, a value starting with '-', JSON's numbers for integers
|
||||||
|
func TestWrittenFormsStartTheSameComposition(t *testing.T) {
|
||||||
c := &Composition{}
|
c := &Composition{}
|
||||||
must := func(err error) {
|
must := func(err error) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -80,6 +81,36 @@ func TestPastedCommandLineStartsTheSameComposition(t *testing.T) {
|
|||||||
t.Fatalf("%q\n got %+v %v\nwant %+v", pasted, got, err, c)
|
t.Fatalf("%q\n got %+v %v\nwant %+v", pasted, got, err, c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
data, err := c.JSON()
|
||||||
|
must(err)
|
||||||
|
if got, err := FromJSON(data); err != nil || !reflect.DeepEqual(got, c) {
|
||||||
|
t.Fatalf("%s\n got %+v %v\nwant %+v", data, got, err, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A null in the website's JSON is a value left unset, which every form then
|
||||||
|
// leaves out, in a table or a list
|
||||||
|
func TestANullInJSONIsUnset(t *testing.T) {
|
||||||
|
with, err := FromJSON([]byte(`[{"name":"p","flow":{"rate_limit":null,"filters":[{"patterns":["a",null]}]},
|
||||||
|
"plugin_sources":[{"type":"file","config":{"directory":"/x/","pattern":null}}],
|
||||||
|
"plugin_sinks":[{"type":"console","config":{"color":null}}]}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
without, err := FromJSON([]byte(`[{"name":"p","flow":{"filters":[{"patterns":["a"]}]},
|
||||||
|
"plugin_sources":[{"type":"file","config":{"directory":"/x/"}}],"plugin_sinks":[{"type":"console"}]}]`))
|
||||||
|
if err != nil || !reflect.DeepEqual(with, without) {
|
||||||
|
t.Fatalf("%v\n got %+v\nwant %+v", err, with, without)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The website's JSON refuses a misspelled key as a file does, at any depth
|
||||||
|
func TestAMisspelledJSONKeyIsRefused(t *testing.T) {
|
||||||
|
for _, data := range []string{`[{"name":"p","plugin_source":[]}]`, `[{"name":"p","flow":{"rate_limt":{}}}]`} {
|
||||||
|
if _, err := FromJSON([]byte(data)); err == nil || !strings.Contains(err.Error(), "unknown key") {
|
||||||
|
t.Errorf("%s: %v", data, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The command line and environment carry no control character, which a
|
// The command line and environment carry no control character, which a
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ const (
|
|||||||
type FilterConfig struct {
|
type FilterConfig struct {
|
||||||
Type FilterType `toml:"type" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"`
|
Type FilterType `toml:"type" default:"include" lw:"enum=include|exclude" help:"pass only matching entries, or drop them"`
|
||||||
Logic FilterLogic `toml:"logic" default:"or" lw:"enum=or|and" help:"match any pattern, or all"`
|
Logic FilterLogic `toml:"logic" default:"or" lw:"enum=or|and" help:"match any pattern, or all"`
|
||||||
Patterns []string `toml:"patterns" lw:"hint=regex" help:"RE2 patterns"`
|
Patterns []string `toml:"patterns,omitempty" lw:"hint=regex" help:"RE2 patterns"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- Sources ---
|
// --- Sources ---
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ func ValidatePipelines(pipelines []PipelineConfig) error {
|
|||||||
single[role+typ] = p.Name
|
single[role+typ] = p.Name
|
||||||
}
|
}
|
||||||
if _, err := decode(role, typ, m); err != nil {
|
if _, err := decode(role, typ, m); err != nil {
|
||||||
return at(path+".config", err)
|
return At(path+".config", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -130,14 +130,14 @@ func checkFlow(f *FlowConfig, path string) error {
|
|||||||
}
|
}
|
||||||
for _, key := range slices.Sorted(maps.Keys(stages)) {
|
for _, key := range slices.Sorted(maps.Keys(stages)) {
|
||||||
if err := Settle(stages[key]); err != nil {
|
if err := Settle(stages[key]); err != nil {
|
||||||
return at(path+"."+key, err)
|
return At(path+"."+key, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// at prefixes err with the path of the table it was found in
|
// At prefixes err with the path of the table it was found in
|
||||||
func at(path string, err error) error {
|
func At(path string, err error) error {
|
||||||
if ke, ok := errors.AsType[*KeyError](err); ok {
|
if ke, ok := errors.AsType[*KeyError](err); ok {
|
||||||
return fmt.Errorf("%s.%s: %w", path, ke.Key, ke.Err)
|
return fmt.Errorf("%s.%s: %w", path, ke.Key, ke.Err)
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-20
@@ -240,29 +240,16 @@ func (f *presetForm) apply(a *app) {
|
|||||||
a.use(c.Pipelines[0])
|
a.use(c.Pipelines[0])
|
||||||
}
|
}
|
||||||
|
|
||||||
// outputs are what o offers: run, or print one form
|
// outputMenu is what o offers: run, or print one of the forms
|
||||||
var outputs = []struct {
|
|
||||||
name, hint string
|
|
||||||
form func(*compose.Composition) (string, error) // nil runs
|
|
||||||
}{
|
|
||||||
{"run", "start the pipelines now, with lw's other settings", nil},
|
|
||||||
{"command line", "print lw's flags", (*compose.Composition).CommandLine},
|
|
||||||
{"environment", "print LOGWISP_ variables: one pipeline", (*compose.Composition).Environment},
|
|
||||||
{"file", "print a configuration file's pipelines", func(c *compose.Composition) (string, error) {
|
|
||||||
data, err := c.File()
|
|
||||||
return string(data), err
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *app) outputMenu() dialog {
|
func (a *app) outputMenu() dialog {
|
||||||
var options []ui.Option
|
forms := compose.Forms()
|
||||||
for _, o := range outputs {
|
options := []ui.Option{{Name: "run", Hint: "start the pipelines now, with lw's other settings"}}
|
||||||
options = append(options, ui.Option{Name: o.name, Hint: o.hint})
|
for _, f := range forms {
|
||||||
|
options = append(options, ui.Option{Name: f.Name, Hint: "print " + f.Hint})
|
||||||
}
|
}
|
||||||
return &chooser{title: "Output, once the screen closes", list: ui.NewOptionListState(options),
|
return &chooser{title: "Output, once the screen closes", list: ui.NewOptionListState(options),
|
||||||
pick: func(a *app, i int) {
|
pick: func(a *app, i int) {
|
||||||
o := outputs[i]
|
if i == 0 {
|
||||||
if o.form == nil {
|
|
||||||
if a.problem != nil {
|
if a.problem != nil {
|
||||||
a.status = "not valid yet: c shows the problem"
|
a.status = "not valid yet: c shows the problem"
|
||||||
return
|
return
|
||||||
@@ -270,7 +257,7 @@ func (a *app) outputMenu() dialog {
|
|||||||
a.result, a.done = Result{Exit: Start, Pipelines: a.comp.Pipelines}, true
|
a.result, a.done = Result{Exit: Start, Pipelines: a.comp.Pipelines}, true
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
text, err := o.form(a.comp)
|
text, err := forms[i-1].Write(a.comp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.status = err.Error()
|
a.status = err.Error()
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -311,9 +311,6 @@ func slicesHasKey(keys []config.Key, name string) bool {
|
|||||||
// paste starts over from a pasted command line
|
// paste starts over from a pasted command line
|
||||||
func (a *app) paste(text string) {
|
func (a *app) paste(text string) {
|
||||||
c, err := compose.FromCommandLine(text, config.HostIsDir)
|
c, err := compose.FromCommandLine(text, config.HostIsDir)
|
||||||
if err == nil && len(c.Pipelines) == 0 {
|
|
||||||
err = errors.New("no pipeline flags")
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
a.status = "paste: " + err.Error()
|
a.status = "paste: " + err.Error()
|
||||||
return
|
return
|
||||||
|
|||||||
Reference in new issue
Block a user