v0.38.0 lwconf updates for deployment

lwconf: sparse stages, every problem of a part, pastes read by form

internal/compose/compose.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"reflect"
+	"regexp"
 	"slices"
 	"strconv"
 	"strings"
@@ -22,11 +23,13 @@ import (
 	"github.com/lixenwraith/toml"
 )
 
-// Composition is the pipelines being composed. Its flow stages hold their
-// defaults and its plugin options the kinds they declare, so each form it
-// writes loads back to the same pipelines.
+// Composition is the pipelines being composed. Its flow stages and plugin
+// options hold the keys set, the options in the kinds they declare, so each
+// form it writes loads back to the same pipelines. LeftOut is what a pasted
+// form held besides pipelines: a file's settings, other LOGWISP_ variables.
 type Composition struct {
 	Pipelines []config.PipelineConfig
+	LeftOut   []string
 }
 
 // Node is a part of a pipeline: a source or sink by its id, or a flow stage
@@ -112,9 +115,10 @@ func FromCommandLine(line string, isDir config.IsDir) (*Composition, error) {
 	return c, c.settle()
 }
 
-// settle gives every flow stage its defaults, every plugin's options their
-// kinds and every source and sink an id; the rules are Validate's. Its error
-// joins a plugin's each holding a value of the wrong kind, kept as given.
+// settle gives every stage the values its spec writes out (FillOn), every
+// plugin's options their kinds, unsetting a zero that takes its key's
+// default, and every source and sink an id; the rules are Validate's. Its
+// error joins a plugin's each holding a value of the wrong kind, kept as given.
 func (c *Composition) settle() error {
 	var errs []error
 	for i := range c.Pipelines {
@@ -126,23 +130,20 @@ func (c *Composition) settle() error {
 			func(j int) string { return p.PluginSources[j].Type })
 		name(len(p.PluginSinks), func(j int) *string { return &p.PluginSinks[j].ID },
 			func(j int) string { return p.PluginSinks[j].Type })
-		for _, stage := range stages(p.Flow) {
+		for key, stage := range stages(p.Flow) {
 			if !stage.IsNil() {
-				config.Fill(stage.Interface())
+				config.FillOn(key, stage.Interface())
 			}
 		}
-		for j := range p.Flow.Filters {
-			config.Fill(&p.Flow.Filters[j])
-		}
 		for j, s := range p.PluginSources {
 			p.PluginSources[j].Config = orEmpty(s.Config)
-			if err := config.Coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
+			if err := coerce("source", s.Type, p.PluginSources[j].Config); err != nil {
 				errs = append(errs, config.At(Node{Role: "source", ID: s.ID}.Path(i), err))
 			}
 		}
 		for j, s := range p.PluginSinks {
 			p.PluginSinks[j].Config = orEmpty(s.Config)
-			if err := config.Coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
+			if err := coerce("sink", s.Type, p.PluginSinks[j].Config); err != nil {
 				errs = append(errs, config.At(Node{Role: "sink", ID: s.ID}.Path(i), err))
 			}
 		}
@@ -150,6 +151,32 @@ func (c *Composition) settle() error {
 	return errors.Join(errs...)
 }
 
+// coerce gives a plugin's options their kinds, then unsets a number at zero
+// where zero takes the key's default: lw reads it so, and the options then
+// show the default rather than a 0 that means it
+func coerce(role, typ string, m map[string]any) error {
+	err := config.Coerce(role, typ, m)
+	if p, ok := config.LookupPlugin(role, typ); ok {
+		unsetDefaults(p.Keys(), m)
+	}
+	return err
+}
+
+func unsetDefaults(keys []config.Key, m map[string]any) {
+	for _, k := range keys {
+		switch v := m[k.Name]; v {
+		case int64(0), 0.0:
+			if k.Default != "" && k.Zero == "" {
+				delete(m, k.Name)
+			}
+		default:
+			if sub, ok := v.(map[string]any); ok {
+				unsetDefaults(k.Keys, sub)
+			}
+		}
+	}
+}
+
 // name gives the parts a file left unnamed the ids their specs would get: the
 // shell forms write no empty id
 func name(n int, id func(int) *string, typ func(int) string) {
@@ -236,7 +263,8 @@ func (c *Composition) RemovePipeline(i int) error {
 }
 
 // Add appends a source or sink of a catalogue type under config.FreeID, or a
-// filter of type include or exclude, and returns its node
+// filter of type include or exclude, or of the default for "", and returns
+// its node
 func (c *Composition) Add(pi int, role, typ string) (Node, error) {
 	p, err := c.pipeline(pi)
 	if err != nil {
@@ -253,9 +281,7 @@ func (c *Composition) Add(pi int, role, typ string) (Node, error) {
 			ids = append(ids, s.ID)
 		}
 	case "filters":
-		f := config.FilterConfig{Type: config.FilterType(typ)}
-		config.Fill(&f)
-		p.Flow.Filters = append(p.Flow.Filters, f)
+		p.Flow.Filters = append(p.Flow.Filters, config.FilterConfig{Type: config.FilterType(typ)})
 		return Node{Role: role, Index: len(p.Flow.Filters) - 1}, nil
 	default:
 		return Node{}, fmt.Errorf("cannot add to %s", role)
@@ -392,7 +418,8 @@ func (c *Composition) MoveFilter(pi, from, to int) error {
 
 // Set gives a key at a dotted path its values: one, or several for a list. A
 // flow stage that is off turns on. A value of the wrong kind leaves the key as
-// it was; any other rule is Validate's.
+// it was, and a zero that takes the key's default unsets it; any other rule
+// is Validate's.
 func (c *Composition) Set(pi int, n Node, key string, values ...string) error {
 	return c.edit(pi, n, func(m map[string]any) error {
 		unset(m, strings.Split(key, "."))
@@ -435,7 +462,7 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er
 		if err := change(next); err != nil {
 			return err
 		}
-		if err := config.Coerce(n.Role, typ, next); err != nil {
+		if err := coerce(n.Role, typ, next); err != nil {
 			return err
 		}
 		*m = next
@@ -462,7 +489,8 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er
 }
 
 // editStage edits a stage's keys as a spec sets them; the table they scan into
-// replaces the stage, which turns on as a spec turns it on
+// replaces the stage, which turns on as a spec turns it on and holds the keys
+// set, a key at its zero unset
 func editStage(stage reflect.Value, key string, change func(map[string]any) error) error {
 	m := config.StageOn(key, nil)
 	if !stage.IsNil() {
@@ -475,7 +503,7 @@ func editStage(stage reflect.Value, key string, change func(map[string]any) erro
 	if err := config.Scan(m, next.Interface()); err != nil {
 		return err
 	}
-	config.Fill(next.Interface())
+	config.FillOn(key, next.Interface())
 	stage.Set(next)
 	return nil
 }
@@ -639,21 +667,40 @@ func (c *Composition) File() ([]byte, error) {
 }
 
 // Form is a way to write a composition, named for what it is, and to read it
-// back to the same pipelines
+// back to the same pipelines; looks matches text in its shape
 type Form struct {
 	Name  string                                           `json:"name"`
 	Hint  string                                           `json:"hint"`
 	Write func(*Composition) (string, error)               `json:"-"`
 	Read  func(string, config.IsDir) (*Composition, error) `json:"-"`
+	looks *regexp.Regexp
 }
 
 var forms = []Form{
-	{"command", "an lw command line", (*Composition).CommandLine, FromCommandLine},
-	{"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment},
+	{"command", "an lw command line", (*Composition).CommandLine, FromCommandLine,
+		regexp.MustCompile(`^\s*(\S*/)?lw(\s|$)|(^|\s)(--[a-z]|-p\s)`)},
+	{"environment", "LOGWISP_ variables for one pipeline", (*Composition).Environment, FromEnvironment,
+		regexp.MustCompile(`(?m)^\s*(export\s+)?LOGWISP_[A-Z0-9_]*=`)},
 	{"file", "the pipelines of a configuration file", func(c *Composition) (string, error) {
 		data, err := c.File()
 		return string(data), err
-	}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) }},
+	}, func(text string, _ config.IsDir) (*Composition, error) { return FromFile([]byte(text)) },
+		regexp.MustCompile(`(?m)^\s*(\[\[?pipelines\b|pipelines\s*=)`)},
+}
+
+// Parse reads text pasted as the form. Refused, text in another form's shape
+// and not in this one's is refused as that form, which reads it.
+func (f Form) Parse(text string, isDir config.IsDir) (*Composition, error) {
+	c, err := f.Read(text, isDir)
+	if err == nil || f.looks.MatchString(text) {
+		return c, err
+	}
+	for _, g := range forms {
+		if g.looks.MatchString(text) {
+			return nil, fmt.Errorf("this is %s, not %s: read it as the %s form", g.Hint, f.Hint, g.Name)
+		}
+	}
+	return nil, err
 }
 
 // Forms are what a composition writes and reads: lw --tui prints one, the
@@ -662,7 +709,8 @@ func Forms() []Form { return slices.Clone(forms) }
 
 // FromEnvironment starts from NAME='value' lines as an environment file holds
 // them, quoted as a POSIX shell reads them: '#' starts a comment line, a word
-// without '=', such as export, assigns nothing, and a later assignment wins
+// without '=', such as export, assigns nothing, and a later assignment wins.
+// LeftOut names the LOGWISP_ variables no pipeline flag has.
 func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) {
 	vars := map[string]string{}
 	for line := range strings.Lines(text) {
@@ -683,26 +731,28 @@ func FromEnvironment(text string, isDir config.IsDir) (*Composition, error) {
 	for name, value := range vars {
 		environ = append(environ, name+"="+value)
 	}
-	pipelines, err := config.EnvPipelines(environ, isDir)
+	pipelines, unread, err := config.EnvPipelines(environ, isDir)
 	if err != nil {
 		return nil, err
 	}
-	c := &Composition{Pipelines: pipelines}
+	c := &Composition{Pipelines: pipelines, LeftOut: unread}
 	return c, c.settle()
 }
 
 // FromFile starts from a configuration file's pipelines, refusing a misspelled
-// key as lw does; its settings are the host's and its console sinks keep the
-// color they name
+// key as lw does; its settings are the host's, which LeftOut names, and its
+// console sinks keep the color they name
 func FromFile(data []byte) (*Composition, error) {
-	cfg, err := config.ParseFile(data)
+	cfg, settings, err := config.ParseFile(data)
 	if err != nil {
 		return nil, err
 	}
 	if len(cfg.Pipelines) == 0 {
 		return nil, errors.New("no [[pipelines]]")
 	}
-	return FromConfig(&config.Config{Pipelines: cfg.Pipelines})
+	c, err := FromConfig(&config.Config{Pipelines: cfg.Pipelines})
+	c.LeftOut = settings
+	return c, err
 }
 
 // FromJSON starts from pipelines as the website keeps them: a JSON list of

internal/compose/compose_test.go

@@ -206,8 +206,8 @@ func TestShellFormsRefuseControlCharacters(t *testing.T) {
 }
 
 // Set refuses a value of the wrong kind, a plugin's and a stage's alike, and
-// leaves the key as it was; Unset returns a key to its default, dropping a
-// table it leaves empty
+// leaves the key as it was; Unset leaves a key unset, dropping a table it
+// leaves empty
 func TestSetKeepsAKeyItCannotTake(t *testing.T) {
 	c := &Composition{}
 	if err := c.AddPipeline("p"); err != nil {
@@ -228,7 +228,7 @@ func TestSetKeepsAKeyItCannotTake(t *testing.T) {
 		}
 	}
 	if m, beat := c.Pipelines[0].PluginSinks[0].Config, c.Pipelines[0].Flow.Heartbeat; !reflect.DeepEqual(m, map[string]any{"port": int64(8080)}) ||
-		beat.IntervalMS != 1000 {
+		beat.IntervalMS != 0 {
 		t.Fatalf("options %v, heartbeat %+v", m, beat)
 	}
 }
@@ -325,18 +325,67 @@ func TestPipelineNamesAreUnique(t *testing.T) {
 }
 
 // A stage an edit turns on does what its spec would: a rate limit drops what
-// passes its rate, a heartbeat beats
+// passes its rate, a heartbeat beats. A stage and a filter hold only the keys
+// set, so the forms carry no default the edit did not set.
 func TestAStageTurnsOnAsItsSpecDoes(t *testing.T) {
 	c := &Composition{}
 	if err := c.AddPipeline("p"); err != nil {
 		t.Fatal(err)
 	}
-	for _, err := range []error{c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500")} {
+	_, err := c.Add(0, "filters", "")
+	for _, err := range []error{err, c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500"),
+		c.Set(0, Node{Role: "filters"}, "patterns", "x")} {
 		if err != nil {
 			t.Fatal(err)
 		}
 	}
-	if f := c.Pipelines[0].Flow; f.RateLimit.Policy != "drop" || !f.Heartbeat.Enabled {
-		t.Fatalf("%+v %+v", f.RateLimit, f.Heartbeat)
+	f := c.Pipelines[0].Flow
+	if *f.RateLimit != (config.RateLimitConfig{EntriesPerSecond: 5, Policy: "drop"}) || *f.Heartbeat != (config.HeartbeatConfig{Enabled: true, IntervalMS: 500}) ||
+		!reflect.DeepEqual(f.Filters, []config.FilterConfig{{Patterns: []string{"x"}}}) {
+		t.Fatalf("%+v %+v %+v", f.RateLimit, f.Heartbeat, f.Filters)
+	}
+}
+
+// A zero where zero takes the key's default is unset, as lw reads it; where
+// zero means something of its own (retention_hours 0 keeps files), it stays
+func TestAZeroTakingTheDefaultIsUnset(t *testing.T) {
+	c, err := FromJSON([]byte(`[{"name":"p","plugin_sources":[{"type":"file","config":{"directory":"/x","check_interval_ms":0}}],
+		"plugin_sinks":[{"type":"file","config":{"directory":"/y","name":"n","retention_hours":0}}]}]`))
+	if err == nil {
+		err = c.Set(0, Node{Role: "sink", ID: "file"}, "buffer_size", "0")
+	}
+	if src, sink := c.Pipelines[0].PluginSources[0].Config, c.Pipelines[0].PluginSinks[0].Config; err != nil ||
+		!reflect.DeepEqual(src, map[string]any{"directory": "/x"}) || !reflect.DeepEqual(sink, map[string]any{"directory": "/y", "name": "n", "retention_hours": 0.0}) {
+		t.Fatalf("%v %v %v", src, sink, err)
+	}
+}
+
+// Text pasted as one form that has another's shape is refused as that form;
+// text in no form's shape, or in this one's, gets this form's own refusal.
+// The settings a file holds, and an environment's LOGWISP_ variables no
+// pipeline flag has, are left out by name.
+func TestAPasteIsReadAsItsForm(t *testing.T) {
+	form := map[string]Form{}
+	for _, f := range Forms() {
+		form[f.Name] = f
+	}
+	for text, want := range map[[2]string]string{
+		{"file", "lw --source null --sink null"}:         "this is an lw command line, not the pipelines of a configuration file: read it as the command form",
+		{"command", "LOGWISP_SOURCE='null'\n"}:           "this is LOGWISP_ variables for one pipeline, not an lw command line: read it as the environment form",
+		{"environment", "[[pipelines]]\nname = \"p\"\n"}: "this is the pipelines of a configuration file, not LOGWISP_ variables for one pipeline: read it as the file form",
+		{"command", "lw -c lw.toml"}:                     "-c: not a pipeline flag",
+		{"file", "hello world"}:                          "expected '=' after key",
+	} {
+		if _, err := form[text[0]].Parse(text[1], nil); err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("%s %q: %v", text[0], text[1], err)
+		}
+	}
+	file, err := form["file"].Parse("quiet = true\n[logging]\nlevel = \"info\"\n[[pipelines]]\nname = \"p\"\n", nil)
+	if err != nil || !slices.Equal(file.LeftOut, []string{"logging.level", "quiet"}) {
+		t.Fatalf("file: %+v %v", file, err)
+	}
+	env, err := form["environment"].Parse("LOGWISP_SORCE_1='null'\nLOGWISP_QUIET=true\nLOGWISP_SINK=''\nLOGWISP_SOURCE_1=null\nHOME=/root\n", nil)
+	if err != nil || !slices.Equal(env.LeftOut, []string{"LOGWISP_QUIET", "LOGWISP_SORCE_1"}) {
+		t.Fatalf("environment: %+v %v", env, err)
 	}
 }