v0.32.0 lwconf engine update: config by path, schema from code, wasm

internal/compose/compose.go

@@ -32,9 +32,9 @@ type Composition struct {
 // by its key in the flow table: filters with an index, format, rate_limit or
 // heartbeat
 type Node struct {
-	Role  string
-	ID    string
-	Index int
+	Role  string `json:"role"`
+	ID    string `json:"id"`
+	Index int    `json:"index"`
 }
 
 // FromConfig starts from a loaded configuration's pipelines, its console sinks
@@ -100,8 +100,10 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
 }
 
 // settle gives every flow stage its defaults, every plugin's options their
-// kinds and every source and sink an id; the rules are Validate's
+// kinds and every source and sink an id; the rules are Validate's. Its error
+// joins a plugin's each holding a value of the wrong kind, kept as given.
 func (c *Composition) settle() error {
+	var errs []error
 	for i := range c.Pipelines {
 		p := &c.Pipelines[i]
 		if p.Flow == nil {
@@ -122,17 +124,17 @@ func (c *Composition) settle() error {
 		for j, s := range p.PluginSources {
 			p.PluginSources[j].Config = orEmpty(s.Config)
 			if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
-				return config.At(fmt.Sprintf("pipelines[%d].plugin_sources[%s].config", i, s.ID), err)
+				errs = append(errs, config.At(Node{Role: "source", ID: s.ID}.Path(i), err))
 			}
 		}
 		for j, s := range p.PluginSinks {
 			p.PluginSinks[j].Config = orEmpty(s.Config)
 			if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
-				return config.At(fmt.Sprintf("pipelines[%d].plugin_sinks[%s].config", i, s.ID), err)
+				errs = append(errs, config.At(Node{Role: "sink", ID: s.ID}.Path(i), err))
 			}
 		}
 	}
-	return nil
+	return errors.Join(errs...)
 }
 
 // name gives the parts a file left unnamed the ids their specs would get: the
@@ -340,6 +342,17 @@ func index(p *config.PipelineConfig, n Node) int {
 	return -1
 }
 
+// Path is the path of node n's options in pipeline pi, as Locate reads it
+func (n Node) Path(pi int) string {
+	switch n.Role {
+	case "source", "sink":
+		return fmt.Sprintf("pipelines[%d].plugin_%ss[%s].config", pi, n.Role, n.ID)
+	case "filters":
+		return fmt.Sprintf("pipelines[%d].flow.filters[%d]", pi, n.Index)
+	}
+	return fmt.Sprintf("pipelines[%d].flow.%s", pi, n.Role)
+}
+
 func (n Node) missing() error {
 	if n.Role == "filters" {
 		return fmt.Errorf("no filter %d", n.Index)
@@ -476,9 +489,69 @@ func (c *Composition) Options(pi int, n Node) (map[string]any, bool, error) {
 	return nil, false, n.missing()
 }
 
-// Validate applies every rule lw applies at load, naming the refused key's path
+// Validate applies every rule lw applies at load: its error joins every
+// problem, one a part, which Problems splits and Locate places
 func (c *Composition) Validate() error { return config.ValidatePipelines(c.Pipelines) }
 
+// Problems are the problems an error joins, in order
+func Problems(err error) []error {
+	if joined, ok := err.(interface{ Unwrap() []error }); ok {
+		var out []error
+		for _, e := range joined.Unwrap() {
+			out = append(out, Problems(e)...)
+		}
+		return out
+	}
+	if err == nil {
+		return nil
+	}
+	return []error{err}
+}
+
+// Locate reads where a problem lies from its path: the pipeline, the node,
+// and the key in the node's options, a list's without its index; text is
+// what is wrong. ok is false for a problem no pipeline holds.
+func Locate(err error) (pi int, n Node, key, text string, ok bool) {
+	ke, _ := errors.AsType[*config.KeyError](err)
+	rest, found := "", false
+	if ke != nil {
+		rest, found = strings.CutPrefix(ke.Key, "pipelines[")
+	}
+	end := strings.IndexByte(rest, ']')
+	if !found || end < 0 {
+		return 0, n, "", err.Error(), false
+	}
+	if pi, err = strconv.Atoi(rest[:end]); err != nil {
+		return 0, n, "", ke.Error(), false
+	}
+	rest = strings.TrimPrefix(rest[end+1:], ".")
+	for role, list := range map[string]string{"source": "plugin_sources[", "sink": "plugin_sinks["} {
+		if after, ok := strings.CutPrefix(rest, list); ok {
+			id, more := cutID(after)
+			n.Role, n.ID, rest = role, id, strings.TrimPrefix(strings.TrimPrefix(more, "config"), ".")
+		}
+	}
+	if stage, ok := strings.CutPrefix(rest, "flow."); ok {
+		name, more, _ := strings.Cut(stage, ".")
+		name, index, _ := strings.Cut(name, "[")
+		n.Role, rest = name, more
+		n.Index, _ = strconv.Atoi(strings.TrimSuffix(index, "]"))
+	}
+	key, _, _ = strings.Cut(rest, "[")
+	return pi, n, key, ke.Err.Error(), true
+}
+
+// cutID cuts a plugin path at its id's ']', the first that ends the path or
+// starts its type or config: ids are free text, and keys hold list indexes
+func cutID(path string) (id, rest string) {
+	for i := range len(path) {
+		if after := path[i+1:]; path[i] == ']' && (after == "" || after == ".type" || after == ".config" || strings.HasPrefix(after, ".config.")) {
+			return path[:i], strings.TrimPrefix(after, ".")
+		}
+	}
+	return path, ""
+}
+
 // CommandLine writes the composition as an lw command line, a flag a line,
 // quoted for a POSIX shell
 func (c *Composition) CommandLine() (string, error) {
@@ -549,28 +622,77 @@ func (c *Composition) File() ([]byte, error) {
 	return toml.Marshal(map[string]any{"pipelines": c.Pipelines})
 }
 
-// Form is a way to write a composition, named for what it is: each loads back
-// to the same pipelines
+// Form is a way to write a composition, named for what it is, and to read it
+// back to the same pipelines
 type Form struct {
-	Name, Hint string
-	Write      func(*Composition) (string, error)
+	Name  string                                           `json:"name"`
+	Hint  string                                           `json:"hint"`
+	Write func(*Composition) (string, error)               `json:"-"`
+	Read  func(string, config.IsDir) (*Composition, error) `json:"-"`
 }
 
 var forms = []Form{
-	{"command", "lw's flags", (*Composition).CommandLine},
-	{"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment},
+	{"command", "lw's flags", (*Composition).CommandLine, FromCommandLine},
+	{"environment", "LOGWISP_ variables: one pipeline", (*Composition).Environment, FromEnvironment},
 	{"file", "a configuration file's pipelines", func(c *Composition) (string, error) {
 		data, err := c.File()
 		return string(data), err
-	}},
+	}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }},
 }
 
-// Forms are what a composition writes: lw --tui prints one, the website shows them
+// Forms are what a composition writes and reads: lw --tui prints one, the
+// website shows them and reads them pasted
 func Forms() []Form { return slices.Clone(forms) }
 
+// FromEnvironment starts from NAME='value' lines as an environment file holds
+// them, quoted as a POSIX shell reads them: '#' starts a comment line, a word
+// without '=', such as export, assigns nothing, and a later assignment wins
+func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) {
+	vars := map[string]string{}
+	for line := range strings.Lines(text) {
+		if line = strings.TrimSpace(line); line == "" || line[0] == '#' {
+			continue
+		}
+		words, err := split(line)
+		if err != nil {
+			return nil, err
+		}
+		for _, w := range words {
+			if name, value, ok := strings.Cut(w, "="); ok {
+				vars[name] = value
+			}
+		}
+	}
+	var environ []string
+	for name, value := range vars {
+		environ = append(environ, name+"="+value)
+	}
+	pipelines, err := config.EnvPipelines(environ, isDir)
+	if err != nil {
+		return nil, err
+	}
+	c := &Composition{Pipelines: pipelines}
+	return c, c.settle()
+}
+
+// FromFile starts from a configuration file's pipelines, refusing a misspelled
+// key as lw does; its settings are the host's and its console sinks keep the
+// color they name
+func FromFile(data []byte) (*Composition, error) {
+	cfg, err := config.ParseFile(data)
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.Pipelines) == 0 {
+		return nil, errors.New("no [[pipelines]]")
+	}
+	return FromConfig(&config.Config{Pipelines: cfg.Pipelines})
+}
+
 // FromJSON starts from pipelines as the website keeps them: a JSON list of
 // tables in a file's shape, a misspelled key refused as a file's is, a null
-// a value left unset
+// a value left unset. A value of the wrong kind returns the composition too,
+// which Validate names among the rest.
 func FromJSON(data []byte) (*Composition, error) {
 	var tables any
 	if err := json.Unmarshal(data, &tables); err != nil {
@@ -710,14 +832,14 @@ func copyPipeline(p config.PipelineConfig) config.PipelineConfig {
 // Schema is what lw --schema prints, and the website builds its catalogue
 // from. A setting's flag is --KEY, a pipeline flag's --FLAG.
 type Schema struct {
-	Version  string                  `json:"version"`
-	Settings []Setting               `json:"settings"`
-	Flags    []config.SpecFlag       `json:"pipeline_flags"`
-	Flow     []config.Stage          `json:"flow"`
-	Tables   map[string][]config.Key `json:"tables"`
-	Sources  []Plugin                `json:"sources"`
-	Sinks    []Plugin                `json:"sinks"`
-	Presets  []config.Preset         `json:"presets"`
+	Version  string            `json:"version"`
+	Settings []Setting         `json:"settings"`
+	Flags    []config.SpecFlag `json:"pipeline_flags"`
+	Flow     []config.Stage    `json:"flow"`
+	Sources  []Plugin          `json:"sources"`
+	Sinks    []Plugin          `json:"sinks"`
+	Presets  []config.Preset   `json:"presets"`
+	Forms    []Form            `json:"forms"`
 }
 
 // Setting is a settings key with its environment variable
@@ -737,12 +859,8 @@ func NewSchema() Schema {
 		Version: version.Short(),
 		Flags:   config.SpecFlags(),
 		Flow:    config.FlowStages(),
-		Tables: map[string][]config.Key{
-			"tls":  config.KeysOf[config.TLSOptions](),
-			"auth": config.KeysOf[config.AuthOptions](),
-			"acl":  config.KeysOf[config.ACLOptions](),
-		},
 		Presets: config.Presets(),
+		Forms:   Forms(),
 	}
 	for _, k := range config.SettingKeys() {
 		s.Settings = append(s.Settings, Setting{k, "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(k.Name, ".", "_"))})

internal/compose/compose_test.go

@@ -1,6 +1,7 @@
 package compose
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
@@ -35,11 +36,12 @@ func TestEngineLinksNoHostCode(t *testing.T) {
 	}
 }
 
-// A pasted command line, its line ends LF or CRLF, and the website's JSON each
-// start the composition that wrote them: escaped and quoted values, nested
-// keys, lists, moved filters, stages whose flags' defaults differ from a
-// file's, a value starting with '-', JSON's numbers for integers
-func TestWrittenFormsStartTheSameComposition(t *testing.T) {
+// Every form a composition writes, its line ends LF or CRLF, and the
+// website's JSON read back to it: escaped and quoted values, nested keys,
+// lists, moved filters, stages whose flags' defaults differ from a file's, a
+// value starting with '-', JSON's numbers for integers. The environment holds
+// one pipeline, so each form is written from one or both.
+func TestWrittenFormsReadBack(t *testing.T) {
 	c := &Composition{}
 	must := func(err error) {
 		t.Helper()
@@ -68,18 +70,29 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) {
 	must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100"))
 	must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000"))
 	must(c.Set(0, Node{Role: "heartbeat"}, "enabled", "true"))
+	one := &Composition{Pipelines: slices.Clone(c.Pipelines)}
 	must(c.AddPipeline("-b"))
 	for _, role := range []string{"source", "sink"} {
 		_, err = c.Add(1, role, "null")
 		must(err)
 	}
 	must(c.Set(1, Node{Role: "heartbeat"}, "interval_ms", "2000"))
-	line, err := c.CommandLine()
-	must(err)
-	for _, pasted := range []string{line, strings.ReplaceAll(line, "\n", "\r\n")} {
-		got, err := FromCommandLine(pasted, nil)
-		if err != nil || !reflect.DeepEqual(got, c) {
-			t.Fatalf("%q\n got %+v %v\nwant %+v", pasted, got, err, c)
+	for _, f := range Forms() {
+		written := 0
+		for _, c := range []*Composition{c, one} {
+			text, err := f.Write(c)
+			if err != nil {
+				continue
+			}
+			written++
+			for _, pasted := range []string{text, strings.ReplaceAll(text, "\n", "\r\n")} {
+				if got, err := f.Read(pasted, nil); err != nil || !reflect.DeepEqual(got, c) {
+					t.Fatalf("%s %q\n got %+v %v\nwant %+v", f.Name, pasted, got, err, c)
+				}
+			}
+		}
+		if written == 0 {
+			t.Errorf("%s wrote neither composition", f.Name)
 		}
 	}
 	data, err := c.JSON()
@@ -89,6 +102,52 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) {
 	}
 }
 
+// Pasted environment text reads as a shell runs it: a comment line assigns
+// nothing, export is no assignment, and a later assignment wins
+func TestPastedEnvironmentReadsAsAShellRunsIt(t *testing.T) {
+	c, err := FromEnvironment("export LOGWISP_SOURCE='null'\n# LOGWISP_SOURCE='tcp_chain'\n\nLOGWISP_SINK='tcp:port=1'\nLOGWISP_SINK='null'\n", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	p := c.Pipelines[0]
+	if len(c.Pipelines) != 1 || len(p.PluginSources) != 1 || p.PluginSources[0].Type != "null" || len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "null" {
+		t.Fatalf("pipelines %+v", c.Pipelines)
+	}
+}
+
+// Locate places a problem by its path: a plugin's key, a key of its table, a
+// rule between its table's keys, a list entry, the plugin itself under an id
+// holding ']', a filter's list entry, a stage and the pipeline; and nothing
+// outside one
+func TestLocatePlacesAProblem(t *testing.T) {
+	sink := func(id string) Node { return Node{Role: "sink", ID: id} }
+	for path, want := range map[string]struct {
+		pi  int
+		n   Node
+		key string
+	}{
+		"pipelines[1].plugin_sinks[out].config.port":            {1, sink("out"), "port"},
+		"pipelines[0].plugin_sinks[out].config.tls.min_version": {0, sink("out"), "tls.min_version"},
+		"pipelines[0].plugin_sinks[out].config.tls":             {0, sink("out"), "tls"},
+		"pipelines[0].plugin_sinks[out].config.acl.allow[1]":    {0, sink("out"), "acl.allow"},
+		"pipelines[0].plugin_sinks[a]b].config":                 {0, sink("a]b"), ""},
+		"pipelines[0].plugin_sources[in].type":                  {0, Node{Role: "source", ID: "in"}, "type"},
+		"pipelines[0].flow.filters[2].patterns[1]":              {0, Node{Role: "filters", Index: 2}, "patterns"},
+		"pipelines[0].flow.rate_limit.policy":                   {0, Node{Role: "rate_limit"}, "policy"},
+		"pipelines[3].name":                                     {3, Node{}, "name"},
+	} {
+		pi, n, key, text, ok := Locate(fmt.Errorf("checked: %w", &config.KeyError{Key: path, Err: errors.New("wrong")}))
+		if !ok || pi != want.pi || n != want.n || key != want.key || text != "wrong" {
+			t.Errorf("%s: %d %+v %q %q %v", path, pi, n, key, text, ok)
+		}
+	}
+	for _, err := range []error{errors.New("no pipelines configured"), &config.KeyError{Key: "color", Err: errors.New("wrong")}} {
+		if _, _, _, text, ok := Locate(err); ok || text != err.Error() {
+			t.Errorf("%v: placed, text %q", err, text)
+		}
+	}
+}
+
 // A null in the website's JSON is a value left unset, which every form then
 // leaves out, in a table or a list
 func TestANullInJSONIsUnset(t *testing.T) {
@@ -105,13 +164,18 @@ func TestANullInJSONIsUnset(t *testing.T) {
 	}
 }
 
-// The website's JSON refuses a misspelled key as a file does, at any depth
-func TestAMisspelledJSONKeyIsRefused(t *testing.T) {
+// The website's JSON and a pasted file refuse a misspelled key as lw does,
+// at any depth
+func TestAMisspelledKeyIsRefused(t *testing.T) {
 	for _, data := range []string{`[{"name":"p","plugin_source":[]}]`, `[{"name":"p","flow":{"rate_limt":{}}}]`} {
 		if _, err := FromJSON([]byte(data)); err == nil || !strings.Contains(err.Error(), "unknown key") {
 			t.Errorf("%s: %v", data, err)
 		}
 	}
+	file := "[[pipelines]]\nname = \"p\"\nnmae = \"q\"\n[[pipelines.plugin_sources]]\ntype = \"null\"\n"
+	if _, err := FromFile([]byte(file)); err == nil || !strings.Contains(err.Error(), `unknown key "pipelines[0].nmae"`) {
+		t.Errorf("file: %v", err)
+	}
 }
 
 // The command line and environment carry no control character, which a