v0.29.0 the configuration engine, and lw --schema

internal/config/spec.go

@@ -4,7 +4,9 @@ import (
 	"cmp"
 	"errors"
 	"fmt"
+	"maps"
 	"os"
+	"reflect"
 	"slices"
 	"strconv"
 	"strings"
@@ -17,11 +19,12 @@ type specKind struct {
 	typed bool // SPEC starts with TYPE
 	many  bool // repeatable within a pipeline; the environment adds _1.._N
 	first bool // starts its pipeline, and names it by TYPE unless --pipeline did
-	apply func(p *PipelineConfig, typ string, opts map[string]any) error
+	apply func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error
+	write func(p *PipelineConfig) []string // its specs that build p's part; nil for preset
 }
 
 var specKinds = []specKind{
-	{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error {
 		values := map[string][]string{}
 		for k, v := range opts {
 			switch v := v.(type) {
@@ -35,9 +38,9 @@ var specKinds = []specKind{
 				return fmt.Errorf("preset keys do not nest: %q", k)
 			}
 		}
-		return applyPreset(p, typ, values)
-	}},
-	{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
+		return applyPreset(p, typ, values, isDir)
+	}, nil},
+	{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
 		ids := make([]string, len(p.PluginSources))
 		for i, s := range p.PluginSources {
 			ids[i] = s.ID
@@ -45,8 +48,13 @@ var specKinds = []specKind{
 		id, err := instanceID(typ, opts, ids)
 		p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
 		return cmp.Or(err, Coerce("source", typ, opts))
+	}, func(p *PipelineConfig) (specs []string) {
+		for _, s := range p.PluginSources {
+			specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
+		}
+		return specs
 	}},
-	{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
 		ids := make([]string, len(p.PluginSinks))
 		for i, s := range p.PluginSinks {
 			ids[i] = s.ID
@@ -54,27 +62,61 @@ var specKinds = []specKind{
 		id, err := instanceID(typ, opts, ids)
 		p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
 		return cmp.Or(err, Coerce("sink", typ, opts))
+	}, func(p *PipelineConfig) (specs []string) {
+		for _, s := range p.PluginSinks {
+			specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
+		}
+		return specs
 	}},
-	{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
 		opts["type"] = typ
 		f := FilterConfig{}
 		err := Scan(opts, &f)
 		p.Flow.Filters = append(p.Flow.Filters, f)
 		return err
+	}, func(p *PipelineConfig) (specs []string) {
+		for _, f := range p.Flow.Filters {
+			Fill(&f)
+			specs = append(specs, typedSpec(&f))
+		}
+		return specs
 	}},
-	{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
 		opts["type"] = typ
 		p.Flow.Format = &FormatConfig{}
 		return Scan(opts, p.Flow.Format)
+	}, func(p *PipelineConfig) []string {
+		if p.Flow.Format == nil {
+			return nil
+		}
+		format := *p.Flow.Format
+		Fill(&format)
+		return []string{typedSpec(&format)}
 	}},
 	// Naming a stage turns it on: the file defaults (pass, disabled) would not.
-	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
+	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
 		p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
 		return Scan(opts, p.Flow.RateLimit)
+	}, func(p *PipelineConfig) []string {
+		if p.Flow.RateLimit == nil {
+			return nil
+		}
+		rate := *p.Flow.RateLimit
+		Fill(&rate) // the policy, whose default here is drop
+		return []string{formatSpec("", Values(&rate))}
 	}},
-	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
+	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
 		p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
 		return Scan(opts, p.Flow.Heartbeat)
+	}, func(p *PipelineConfig) []string {
+		if p.Flow.Heartbeat == nil {
+			return nil
+		}
+		beat := *p.Flow.Heartbeat
+		Fill(&beat)
+		m := Values(&beat)
+		m["enabled"] = beat.Enabled
+		return []string{formatSpec("", m)}
 	}},
 }
 
@@ -88,15 +130,28 @@ type pipelineSpec struct {
 // Spec is one pipeline flag: its name without dashes, and its value.
 type Spec struct{ Flag, Value string }
 
-// SpecFlags names the pipeline flags, for the command-line parser.
-func SpecFlags() []string {
-	flags := []string{"pipeline"}
+// SpecFlag is a pipeline flag, for the parser and lw --schema: its name
+// without dashes, its environment variable, which takes _1.._N when Many, and
+// whether SPEC starts with TYPE
+type SpecFlag struct {
+	Flag     string `json:"flag"`
+	Variable string `json:"variable"`
+	Typed    bool   `json:"typed"`
+	Many     bool   `json:"many"`
+}
+
+func SpecFlags() []SpecFlag {
+	flags := []SpecFlag{{"pipeline", specVariable("pipeline"), false, false}}
 	for _, k := range specKinds {
-		flags = append(flags, k.flag)
+		flags = append(flags, SpecFlag{k.flag, specVariable(k.flag), k.typed, k.many})
 	}
 	return flags
 }
 
+func specVariable(flag string) string {
+	return "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(flag, "-", "_"))
+}
+
 // cliSpecs looks up the kind of each command-line spec.
 func cliSpecs(in []Spec) ([]pipelineSpec, error) {
 	var specs []pipelineSpec
@@ -121,12 +176,12 @@ func cliSpecs(in []Spec) ([]pipelineSpec, error) {
 // counts as unset, so container templates may leave optional ones blank.
 func envPipelineSpecs() []pipelineSpec {
 	var specs []pipelineSpec
-	if name := os.Getenv("LOGWISP_PIPELINE"); name != "" {
-		specs = append(specs, pipelineSpec{"LOGWISP_PIPELINE", nil, name})
+	if name := os.Getenv(specVariable("pipeline")); name != "" {
+		specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name})
 	}
 	for i := range specKinds {
 		kind := &specKinds[i]
-		base := "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(kind.flag, "-", "_"))
+		base := specVariable(kind.flag)
 		type numbered struct {
 			n          uint64
 			name, spec string
@@ -150,9 +205,18 @@ func envPipelineSpecs() []pipelineSpec {
 	return specs
 }
 
+// SpecPipelines builds the pipelines of command-line specs, as Load does
+func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) {
+	s, err := cliSpecs(specs)
+	if err != nil {
+		return nil, err
+	}
+	return buildPipelines(s, isDir)
+}
+
 // buildPipelines turns specs, in order, into pipelines; specs before the first
 // --pipeline go to one named "cli". Every call returns fresh maps.
-func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
+func buildPipelines(specs []pipelineSpec, isDir IsDir) ([]PipelineConfig, error) {
 	var pipelines []PipelineConfig
 	var seen map[*specKind]bool
 	for _, s := range specs {
@@ -181,7 +245,7 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
 			return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value)
 		}
 		seen[s.kind] = true
-		if err := s.kind.apply(p, typ, opts); err != nil {
+		if err := s.kind.apply(p, typ, opts, isDir); err != nil {
 			return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
 		}
 	}
@@ -191,6 +255,104 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
 	return pipelines, nil
 }
 
+// PipelineSpecs writes pipelines as the flags that build them: --pipeline,
+// ids, flow stage defaults and heartbeat's enabled explicit, as some flags'
+// defaults differ from a file's
+func PipelineSpecs(pipelines []PipelineConfig) []Spec {
+	var specs []Spec
+	for _, p := range pipelines {
+		specs = append(specs, Spec{"pipeline", p.Name})
+		p.Flow = cmp.Or(p.Flow, &FlowConfig{})
+		for _, k := range specKinds {
+			if k.write != nil {
+				for _, value := range k.write(&p) {
+					specs = append(specs, Spec{k.flag, value})
+				}
+			}
+		}
+	}
+	return specs
+}
+
+// typedSpec writes a stage whose type key is its spec's TYPE
+func typedSpec(stage any) string {
+	m := Values(stage)
+	typ, _ := m["type"].(string)
+	delete(m, "type")
+	return formatSpec(typ, m)
+}
+
+// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a
+// repeated key
+func formatSpec(typ string, tables ...map[string]any) string {
+	var parts []string
+	if typ != "" {
+		parts = append(parts, escape(typ))
+	}
+	var add func(prefix string, m map[string]any)
+	add = func(prefix string, m map[string]any) {
+		for _, k := range slices.Sorted(maps.Keys(m)) {
+			switch v := m[k].(type) {
+			case map[string]any:
+				add(prefix+k+".", v)
+			case []any:
+				for _, e := range v {
+					parts = append(parts, escape(prefix+k)+"="+escape(text(e)))
+				}
+			default:
+				parts = append(parts, escape(prefix+k)+"="+escape(text(v)))
+			}
+		}
+	}
+	for _, m := range tables {
+		add("", m)
+	}
+	return strings.Join(parts, ",")
+}
+
+func text(v any) string {
+	if f, ok := v.(float64); ok {
+		return strconv.FormatFloat(f, 'g', -1, 64)
+	}
+	return fmt.Sprint(v)
+}
+
+// escape is unescape's inverse: ',' and '=' escaped, and '\' where an escape
+// would otherwise start, so a regex's \d stays as typed
+func escape(s string) string {
+	var b strings.Builder
+	followed := s + "," // as a value is, unless it ends the spec
+	for i := range len(s) {
+		if s[i] == ',' || s[i] == '=' || escapes(followed, i) {
+			b.WriteByte('\\')
+		}
+		b.WriteByte(s[i])
+	}
+	return b.String()
+}
+
+// Values gives a table's keys that are not zero, as a spec sets them, for the
+// composer to edit and Scan back
+func Values(v any) map[string]any {
+	rv, m := reflect.ValueOf(v).Elem(), map[string]any{}
+	for _, o := range options(rv.Type()) {
+		switch f := rv.Field(o.index); {
+		case f.IsZero():
+		case f.Kind() == reflect.String:
+			m[o.Name] = f.String()
+		case f.Kind() == reflect.Slice:
+			var list []any
+			for i := range f.Len() {
+				list = append(list, f.Index(i).Interface())
+			}
+			m[o.Name] = list
+		default:
+			m[o.Name] = f.Interface()
+		}
+	}
+	return m
+}
+
 // useStdio makes a pipeline without a source read stdin, and one without a
 // sink write stdout, as a filter does.
 func useStdio(p *PipelineConfig) {
@@ -225,7 +387,7 @@ func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err er
 		case typed && key == "type":
 			err = errors.New(`TYPE already sets "type"`)
 		default:
-			err = setOption(opts, key, unescape(value))
+			err = SetOption(opts, key, unescape(value))
 		}
 		if err != nil {
 			return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
@@ -262,8 +424,8 @@ func escapes(s string, i int) bool {
 	return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0
 }
 
-// setOption stores value at a dotted key; a second value turns it into a list.
-func setOption(opts map[string]any, key, value string) error {
+// SetOption stores value at a dotted key; a second value turns it into a list.
+func SetOption(opts map[string]any, key, value string) error {
 	segments := strings.Split(key, ".")
 	if slices.Contains(segments, "") {
 		return fmt.Errorf("empty key in %q", key)
@@ -293,8 +455,7 @@ func setOption(opts map[string]any, key, value string) error {
 	return nil
 }
 
-// instanceID takes the id option; an unnamed instance gets the first of TYPE,
-// TYPE_2, TYPE_3... that no earlier instance of its role holds.
+// instanceID takes the id option, or FreeID for an unnamed instance
 func instanceID(typ string, opts map[string]any, taken []string) (string, error) {
 	id, _ := opts["id"].(string)
 	if _, set := opts["id"]; set && id == "" {
@@ -304,9 +465,15 @@ func instanceID(typ string, opts map[string]any, taken []string) (string, error)
 	if id != "" {
 		return id, nil
 	}
-	id = typ
+	return FreeID(typ, taken), nil
+}
+
+// FreeID is the first of TYPE, TYPE_2, TYPE_3... that no earlier instance of
+// its role holds
+func FreeID(typ string, taken []string) string {
+	id := typ
 	for n := 2; slices.Contains(taken, id); n++ {
 		id = typ + "_" + strconv.Itoa(n)
 	}
-	return id, nil
+	return id
 }