v0.29.0 the configuration engine, and lw --schema

cmd/lw/bootstrap.go

@@ -135,14 +135,7 @@ func checkConfig(cfg *config.Config) int {
 // dumpConfig prints the effective configuration, flags and environment
 // resolved into it, as a file lw -c reads back.
 func dumpConfig(cfg *config.Config, w io.Writer) int {
-	// Snapshot gave each console sink the top-level color: keep it inherited
-	for _, p := range cfg.Pipelines {
-		for _, s := range p.PluginSinks {
-			if s.Type == "console" && s.Config["color"] == cfg.Color {
-				delete(s.Config, "color")
-			}
-		}
-	}
+	config.Uninherit(cfg.Pipelines, cfg.Color)
 	data, err := toml.Marshal(cfg)
 	if err == nil {
 		_, err = w.Write(data)

cmd/lw/cli.go

@@ -58,7 +58,7 @@ var shorts = map[string]struct {
 
 // switches are lw's own options that take no value and are no setting, so no
 // file or environment variable turns one on
-var switches = []string{"check", "dump", "version"}
+var switches = []string{"check", "dump", "version", "schema"}
 
 // usageError is a command-line mistake: exit status 2 rather than 1
 type usageError string
@@ -241,7 +241,7 @@ func parseCommandLine(argv []string) (inv invocation, err error) {
 		case arg == "--color":
 			// Bare, it means always: config takes no bare flag for a string key
 			inv.load.Overrides = append(inv.load.Overrides, "--color="+cmp.Or(next(&i), "always"))
-		case strings.HasPrefix(name, "--") && slices.Contains(specFlags, name[2:]):
+		case strings.HasPrefix(name, "--") && slices.ContainsFunc(specFlags, isFlag(name[2:])):
 			if !inline {
 				value = next(&i)
 			}
@@ -271,6 +271,10 @@ func parseCommandLine(argv []string) (inv invocation, err error) {
 	return inv, err
 }
 
+func isFlag(name string) func(config.SpecFlag) bool {
+	return func(f config.SpecFlag) bool { return f.Flag == name }
+}
+
 // presetCommands makes each preset a subcommand whose flags are its keys
 func presetCommands() []subcommand {
 	var subs []subcommand
@@ -294,7 +298,7 @@ func presetCommands() []subcommand {
 				for name, v := range values {
 					set[name] = *v
 				}
-				pipeline, err := config.ExpandPreset(p.Name, set)
+				pipeline, err := config.ExpandPreset(p.Name, set, config.HostIsDir)
 				if err != nil {
 					return usageError(err.Error())
 				}

cmd/lw/cli_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"flag"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"reflect"
 	"regexp"
@@ -12,7 +13,9 @@ import (
 	"testing"
 
 	shipped "github.com/lixenwraith/logwisp/config"
+	"github.com/lixenwraith/logwisp/internal/compose"
 	"github.com/lixenwraith/logwisp/internal/config"
+	"github.com/lixenwraith/logwisp/internal/plugin"
 	"github.com/lixenwraith/logwisp/internal/testutil"
 	"github.com/lixenwraith/logwisp/internal/tlsx"
 )
@@ -187,6 +190,123 @@ func TestDumpReadsBack(t *testing.T) {
 	}
 }
 
+// The command line, file and environment a composition writes, read by a
+// POSIX shell and loaded as lw loads them, give its pipelines back: every
+// preset, a second sink, a file's stage without its defaults, an integer for
+// a number, an edge and aggregator pair, and values to escape and quote. The
+// environment holds one pipeline.
+func TestEveryFormLoadsTheSamePipelines(t *testing.T) {
+	dir := t.TempDir()
+	empty := filepath.Join(dir, "empty.toml")
+	if err := os.WriteFile(empty, nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	users, pass := filepath.Join(dir, "users.toml"), filepath.Join(dir, "it's.pass")
+	values := map[string]map[string]string{
+		"pipe":  {"format": "txt"},
+		"tail":  {"path": dir},
+		"serve": {"path": filepath.Join(dir, "*.log"), "listen": "127.0.0.1:15841", "tls": "self", "users": users, "allow": "127.0.0.0/8", "deny": "127.0.0.2"},
+		"edge": {"path": dir + "/", "to": "127.0.0.1:15842", "transport": "http", "pin": "sha256//" + strings.Repeat("A", 43) + "=",
+			"user": `edge,01=a\b`, "password_file": pass},
+		"aggregator": {"listen": "127.0.0.1:15843", "users": users, "out": filepath.Join(dir, "out dir"), "allow": "10.0.0.0/8,127.0.0.1"},
+	}
+	compositions := map[string]*compose.Composition{}
+	for _, p := range config.Presets() {
+		c, err := compose.FromPreset(p.Name, values[p.Name], config.HostIsDir)
+		if err != nil {
+			t.Fatalf("%s: %v", p.Name, err)
+		}
+		compositions[p.Name] = c
+	}
+	if _, err := compositions["tail"].Add(0, "sink", "null"); err != nil {
+		t.Fatal(err)
+	}
+	written := filepath.Join(dir, "written.toml")
+	if err := os.WriteFile(written, []byte(`[[pipelines]]
+name = "written"
+flow.rate_limit.rate = 5
+plugin_sources = [{id = "in", type = "null"}]
+plugin_sinks = [{id = "out", type = "http", config = {port = 15844, acl = {requests_per_second_per_client = 2}}}]
+`), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	c, err := compose.FromConfig(loadTestConfig(t, written))
+	if err != nil {
+		t.Fatal(err)
+	}
+	compositions["written"] = c
+	pair := &compose.Composition{Pipelines: append(compositions["edge"].Pipelines, compositions["aggregator"].Pipelines...)}
+	filter, err := pair.Add(0, "filters", "exclude")
+	if err == nil {
+		err = pair.Set(0, filter, "patterns", `password=\S{8,64}`, `it's "quoted"`)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	compositions["pair"] = pair
+	shell := func(script string) []string {
+		out, err := exec.Command("sh", "-c", script).Output()
+		if err != nil {
+			t.Fatalf("sh: %v\n%s", err, script)
+		}
+		return strings.Split(strings.TrimSuffix(string(out), "\x00"), "\x00")
+	}
+	for name, c := range compositions {
+		line, err := c.CommandLine()
+		if err != nil {
+			t.Fatalf("%s: %v", name, err)
+		}
+		words := shell("set --" + strings.TrimPrefix(line, "lw") + `printf '%s\0' "$@"`)
+		loaded := []*config.Config{loadTestConfig(t, empty, words...)}
+		file := filepath.Join(dir, name+".toml")
+		data, err := c.File()
+		if err == nil {
+			err = os.WriteFile(file, data, 0o600)
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		loaded = append(loaded, loadTestConfig(t, file))
+		env, err := c.Environment()
+		if len(c.Pipelines) > 1 {
+			if err == nil {
+				t.Errorf("%s: the environment took %d pipelines", name, len(c.Pipelines))
+			}
+		} else {
+			var names []string
+			for _, line := range strings.Split(strings.TrimSpace(env), "\n") {
+				names = append(names, `"$`+line[:strings.IndexByte(line, '=')]+`"`)
+			}
+			testutil.ClearEnvPrefix(t, "LOGWISP_")
+			for i, value := range shell(env + "printf '%s\\0' " + strings.Join(names, " ")) {
+				t.Setenv(strings.Trim(names[i], `"$`), value)
+			}
+			loaded = append(loaded, loadWithEnvironment(t, empty))
+		}
+		for _, cfg := range loaded {
+			back, err := compose.FromConfig(cfg)
+			if err != nil || !reflect.DeepEqual(back, c) {
+				t.Errorf("%s: loads back as\n%+v %v\nwant %+v\nfrom %s\n%s\n%s", name, back, err, c, line, data, env)
+			}
+		}
+	}
+}
+
+// lw --schema lists the sources and sinks lw builds
+func TestSchemaListsWhatLwBuilds(t *testing.T) {
+	s := compose.NewSchema()
+	for _, p := range s.Sources {
+		if _, ok := plugin.GetSource(p.Type); !ok {
+			t.Errorf("source %s has no factory", p.Type)
+		}
+	}
+	for _, p := range s.Sinks {
+		if _, ok := plugin.GetSink(p.Type); !ok {
+			t.Errorf("sink %s has no factory", p.Type)
+		}
+	}
+}
+
 // Every preset expands to plugins that build, as lw --check builds them
 func TestEveryPresetBuilds(t *testing.T) {
 	testLogger(t)

cmd/lw/completion.go

@@ -77,8 +77,8 @@ func (s shell) script() []byte {
 // options maps each long option lw itself takes to its short letter, if any
 func options() map[string]string {
 	options := map[string]string{}
-	for _, long := range config.SpecFlags() {
-		options[long] = ""
+	for _, f := range config.SpecFlags() {
+		options[f.Flag] = ""
 	}
 	for long := range config.Settings() {
 		options[long] = ""
@@ -99,7 +99,7 @@ func options() map[string]string {
 func completionModel() (top []compFlag, cmds []compCommand, presets string) {
 	settings, specFlags, shortOf := config.Settings(), config.SpecFlags(), options()
 	for _, long := range slices.Sorted(maps.Keys(shortOf)) {
-		f := compFlag{long: long, short: shortOf[long], value: "text", repeat: slices.Contains(specFlags, long)}
+		f := compFlag{long: long, short: shortOf[long], value: "text", repeat: slices.ContainsFunc(specFlags, isFlag(long))}
 		if v, ok := valueOf[long]; ok {
 			f.value = v
 		} else if settings[long] || slices.Contains(switches, long) {

cmd/lw/help.go

@@ -19,7 +19,7 @@ Usage:
   lw -t|--check [options]       Build every pipeline and plugin, report, exit
   lw --dump [options]           Print the effective configuration as TOML, exit
   lw help | -h | --help
-  lw -V | --version
+  lw -V | --version | --schema
 
 Commands (lw COMMAND -h lists its subcommands):
 %s
@@ -37,6 +37,8 @@ after '='; short options do not combine. lw takes options only.
                                 (on a terminal; default), always (a bare
                                 --color) or never
   -V, --version                 Print the version and exit
+      --schema                  Print every setting, flag, plugin option and
+                                preset as JSON, exit
   -h, --help                    Print this help and exit
       --                        End the options
 

cmd/lw/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -9,6 +10,7 @@ import (
 	"reflect"
 	"syscall"
 
+	"github.com/lixenwraith/logwisp/internal/compose"
 	"github.com/lixenwraith/logwisp/internal/config"
 	"github.com/lixenwraith/logwisp/internal/core"
 	"github.com/lixenwraith/logwisp/internal/version"
@@ -33,6 +35,14 @@ func main() {
 	case inv.on["version"]:
 		fmt.Println(version.String())
 		os.Exit(0)
+	case inv.on["schema"]:
+		out := json.NewEncoder(os.Stdout)
+		out.SetIndent("", "  ")
+		if err := out.Encode(compose.NewSchema()); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+		os.Exit(0)
 	}
 
 	// --- 1. Initial setup ---

cmd/lw/test_helpers_test.go

@@ -20,6 +20,11 @@ func testLogger(t *testing.T) {
 func loadTestConfig(t *testing.T, path string, args ...string) *config.Config {
 	t.Helper()
 	testutil.ClearEnvPrefix(t, "LOGWISP_")
+	return loadWithEnvironment(t, path, args...)
+}
+
+func loadWithEnvironment(t *testing.T, path string, args ...string) *config.Config {
+	t.Helper()
 	inv, err := parseCommandLine(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...))
 	if err != nil {
 		t.Fatal(err)