logwisp/internal/config/spec.go

package config

import (
	"cmp"
	"errors"
	"fmt"
	"maps"
	"os"
	"reflect"
	"slices"
	"strconv"
	"strings"
)

// specKind is one pipeline spec flag. Its environment form is LOGWISP_ plus
// the flag uppercased, '-' as '_'.
type specKind struct {
	flag  string
	typed bool // SPEC starts with TYPE
	many  bool // repeatable within a pipeline; the environment adds _1.._N
	first bool // starts its pipeline, and names it by TYPE unless --pipeline did
	apply func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error
	write func(p *PipelineConfig) []string // its specs that build p's part; nil for preset
}

var specKinds = []specKind{
	{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error {
		values := map[string][]string{}
		for k, v := range opts {
			switch v := v.(type) {
			case string:
				values[k] = []string{v}
			case []any: // a repeated key
				for _, e := range v {
					values[k] = append(values[k], e.(string))
				}
			default:
				return fmt.Errorf("preset keys do not nest: %q", k)
			}
		}
		return applyPreset(p, typ, values, isDir)
	}, nil},
	{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
		ids := make([]string, len(p.PluginSources))
		for i, s := range p.PluginSources {
			ids[i] = s.ID
		}
		id, err := instanceID(typ, opts, ids)
		p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
		return cmp.Or(err, Coerce("source", typ, opts))
	}, func(p *PipelineConfig) (specs []string) {
		for _, s := range p.PluginSources {
			specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
		}
		return specs
	}},
	{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
		ids := make([]string, len(p.PluginSinks))
		for i, s := range p.PluginSinks {
			ids[i] = s.ID
		}
		id, err := instanceID(typ, opts, ids)
		p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
		return cmp.Or(err, Coerce("sink", typ, opts))
	}, func(p *PipelineConfig) (specs []string) {
		for _, s := range p.PluginSinks {
			specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
		}
		return specs
	}},
	{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
		opts["type"] = typ
		f := FilterConfig{}
		err := Scan(opts, &f)
		p.Flow.Filters = append(p.Flow.Filters, f)
		return err
	}, func(p *PipelineConfig) (specs []string) {
		for _, f := range p.Flow.Filters {
			Fill(&f)
			specs = append(specs, typedSpec(&f))
		}
		return specs
	}},
	{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
		opts["type"] = typ
		p.Flow.Format = &FormatConfig{}
		return Scan(opts, p.Flow.Format)
	}, func(p *PipelineConfig) []string {
		if p.Flow.Format == nil {
			return nil
		}
		format := *p.Flow.Format
		Fill(&format)
		return []string{typedSpec(&format)}
	}},
	// Naming a stage turns it on: the file defaults (pass, disabled) would not.
	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
		p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
		return Scan(opts, p.Flow.RateLimit)
	}, func(p *PipelineConfig) []string {
		if p.Flow.RateLimit == nil {
			return nil
		}
		rate := *p.Flow.RateLimit
		Fill(&rate) // the policy, whose default here is drop
		return []string{formatSpec("", Values(&rate))}
	}},
	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
		p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
		return Scan(opts, p.Flow.Heartbeat)
	}, func(p *PipelineConfig) []string {
		if p.Flow.Heartbeat == nil {
			return nil
		}
		beat := *p.Flow.Heartbeat
		Fill(&beat)
		m := Values(&beat)
		m["enabled"] = beat.Enabled
		return []string{formatSpec("", m)}
	}},
}

// pipelineSpec is one pipeline flag or variable; a nil kind starts a pipeline.
type pipelineSpec struct {
	name  string // "--sink" or "LOGWISP_SINK_2", for errors
	kind  *specKind
	value string
}

// Spec is one pipeline flag: its name without dashes, and its value.
type Spec struct{ Flag, Value string }

// SpecFlag is a pipeline flag, for the parser and lw --schema: its name
// without dashes, its environment variable, which takes _1.._N when Many, and
// whether SPEC starts with TYPE
type SpecFlag struct {
	Flag     string `json:"flag"`
	Variable string `json:"variable"`
	Typed    bool   `json:"typed"`
	Many     bool   `json:"many"`
}

func SpecFlags() []SpecFlag {
	flags := []SpecFlag{{"pipeline", specVariable("pipeline"), false, false}}
	for _, k := range specKinds {
		flags = append(flags, SpecFlag{k.flag, specVariable(k.flag), k.typed, k.many})
	}
	return flags
}

func specVariable(flag string) string {
	return "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(flag, "-", "_"))
}

// cliSpecs looks up the kind of each command-line spec.
func cliSpecs(in []Spec) ([]pipelineSpec, error) {
	var specs []pipelineSpec
	for _, s := range in {
		var kind *specKind
		if s.Flag != "pipeline" {
			k := slices.IndexFunc(specKinds, func(k specKind) bool { return k.flag == s.Flag })
			if k < 0 {
				return nil, fmt.Errorf("--%s is no pipeline flag", s.Flag)
			}
			kind = &specKinds[k]
		}
		if s.Value == "" {
			return nil, fmt.Errorf("--%s requires a value", s.Flag)
		}
		specs = append(specs, pipelineSpec{"--" + s.Flag, kind, s.Value})
	}
	return specs, nil
}

// envPipelineSpecs reads the one-pipeline environment form. An empty variable
// counts as unset, so container templates may leave optional ones blank.
func envPipelineSpecs() []pipelineSpec {
	var specs []pipelineSpec
	if name := os.Getenv(specVariable("pipeline")); name != "" {
		specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name})
	}
	for i := range specKinds {
		kind := &specKinds[i]
		base := specVariable(kind.flag)
		type numbered struct {
			n          uint64
			name, spec string
		}
		var found []numbered
		for _, kv := range os.Environ() {
			name, spec, _ := strings.Cut(kv, "=")
			suffix, ok := strings.CutPrefix(name, base+"_")
			n, err := strconv.ParseUint(suffix, 10, 64)
			if name == base || ok && kind.many && err == nil {
				found = append(found, numbered{n, name, spec})
			}
		}
		slices.SortFunc(found, func(a, b numbered) int { return cmp.Or(cmp.Compare(a.n, b.n), cmp.Compare(a.name, b.name)) })
		for _, f := range found {
			if f.spec != "" {
				specs = append(specs, pipelineSpec{f.name, kind, f.spec})
			}
		}
	}
	return specs
}

// SpecPipelines builds the pipelines of command-line specs, as Load does
func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) {
	s, err := cliSpecs(specs)
	if err != nil {
		return nil, err
	}
	return buildPipelines(s, isDir)
}

// buildPipelines turns specs, in order, into pipelines; specs before the first
// --pipeline go to one named "cli". Every call returns fresh maps.
func buildPipelines(specs []pipelineSpec, isDir IsDir) ([]PipelineConfig, error) {
	var pipelines []PipelineConfig
	var seen map[*specKind]bool
	for _, s := range specs {
		if s.kind == nil {
			pipelines = append(pipelines, PipelineConfig{Name: s.value, Flow: &FlowConfig{}})
			seen = map[*specKind]bool{}
			continue
		}
		typ, opts, err := parseSpec(s.value, s.kind.typed)
		if err != nil {
			return nil, fmt.Errorf("%s %w", s.name, err)
		}
		if pipelines == nil {
			name := "cli"
			if s.kind.first {
				name = typ
			}
			pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
			seen = map[*specKind]bool{}
		}
		p := &pipelines[len(pipelines)-1]
		switch {
		case seen[s.kind] && !s.kind.many:
			return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
		case s.kind.first && len(seen) > 0:
			return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value)
		}
		seen[s.kind] = true
		if err := s.kind.apply(p, typ, opts, isDir); err != nil {
			return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
		}
	}
	for i := range pipelines {
		useStdio(&pipelines[i])
	}
	return pipelines, nil
}

// PipelineSpecs writes pipelines as the flags that build them: --pipeline,
// ids, flow stage defaults and heartbeat's enabled explicit, as some flags'
// defaults differ from a file's
func PipelineSpecs(pipelines []PipelineConfig) []Spec {
	var specs []Spec
	for _, p := range pipelines {
		specs = append(specs, Spec{"pipeline", p.Name})
		p.Flow = cmp.Or(p.Flow, &FlowConfig{})
		for _, k := range specKinds {
			if k.write != nil {
				for _, value := range k.write(&p) {
					specs = append(specs, Spec{k.flag, value})
				}
			}
		}
	}
	return specs
}

// typedSpec writes a stage whose type key is its spec's TYPE
func typedSpec(stage any) string {
	m := Values(stage)
	typ, _ := m["type"].(string)
	delete(m, "type")
	return formatSpec(typ, m)
}

// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a
// repeated key
func formatSpec(typ string, tables ...map[string]any) string {
	var parts []string
	if typ != "" {
		parts = append(parts, escape(typ))
	}
	var add func(prefix string, m map[string]any)
	add = func(prefix string, m map[string]any) {
		for _, k := range slices.Sorted(maps.Keys(m)) {
			switch v := m[k].(type) {
			case map[string]any:
				add(prefix+k+".", v)
			case []any:
				for _, e := range v {
					parts = append(parts, escape(prefix+k)+"="+escape(text(e)))
				}
			default:
				parts = append(parts, escape(prefix+k)+"="+escape(text(v)))
			}
		}
	}
	for _, m := range tables {
		add("", m)
	}
	return strings.Join(parts, ",")
}

func text(v any) string {
	if f, ok := v.(float64); ok {
		return strconv.FormatFloat(f, 'g', -1, 64)
	}
	return fmt.Sprint(v)
}

// escape is unescape's inverse: ',' and '=' escaped, and '\' where an escape
// would otherwise start, so a regex's \d stays as typed
func escape(s string) string {
	var b strings.Builder
	followed := s + "," // as a value is, unless it ends the spec
	for i := range len(s) {
		if s[i] == ',' || s[i] == '=' || escapes(followed, i) {
			b.WriteByte('\\')
		}
		b.WriteByte(s[i])
	}
	return b.String()
}

// Values gives a table's keys that are not zero, as a spec sets them, for the
// composer to edit and Scan back
func Values(v any) map[string]any {
	rv, m := reflect.ValueOf(v).Elem(), map[string]any{}
	for _, o := range options(rv.Type()) {
		switch f := rv.Field(o.index); {
		case f.IsZero():
		case f.Kind() == reflect.String:
			m[o.Name] = f.String()
		case f.Kind() == reflect.Slice:
			var list []any
			for i := range f.Len() {
				list = append(list, f.Index(i).Interface())
			}
			m[o.Name] = list
		default:
			m[o.Name] = f.Interface()
		}
	}
	return m
}

// useStdio makes a pipeline without a source read stdin, and one without a
// sink write stdout, as a filter does.
func useStdio(p *PipelineConfig) {
	pipe := pipeDefault(p.Name)
	if len(p.PluginSources) == 0 {
		p.PluginSources = pipe.PluginSources
	}
	if len(p.PluginSinks) == 0 {
		p.PluginSinks = pipe.PluginSinks
	}
}

// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
// a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {
	rest, more := spec, true
	if typed {
		typ, rest, more = cutUnescaped(spec, ',')
		if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq {
			return "", nil, fmt.Errorf("%s: missing TYPE", spec)
		}
		typ = unescape(typ)
	}
	opts = map[string]any{}
	for more {
		var part string
		part, rest, more = cutUnescaped(rest, ',')
		key, value, eq := cutUnescaped(part, '=')
		switch key = unescape(key); {
		case !eq:
			err = errors.New(`missing "="`)
		case typed && key == "type":
			err = errors.New(`TYPE already sets "type"`)
		default:
			err = SetOption(opts, key, unescape(value))
		}
		if err != nil {
			return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
		}
	}
	return typ, opts, nil
}

// cutUnescaped cuts s at its first unescaped sep, leaving escapes in place.
func cutUnescaped(s string, sep byte) (before, after string, found bool) {
	for i := 0; i < len(s); i++ {
		if escapes(s, i) {
			i++
		} else if s[i] == sep {
			return s[:i], s[i+1:], true
		}
	}
	return s, "", false
}

func unescape(s string) string {
	var b strings.Builder
	for i := 0; i < len(s); i++ {
		if escapes(s, i) {
			i++
		}
		b.WriteByte(s[i])
	}
	return b.String()
}

// escapes keeps any other backslash literal, so regex escapes such as \d pass.
func escapes(s string, i int) bool {
	return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0
}

// SetOption stores value at a dotted key; a second value turns it into a list.
func SetOption(opts map[string]any, key, value string) error {
	segments := strings.Split(key, ".")
	if slices.Contains(segments, "") {
		return fmt.Errorf("empty key in %q", key)
	}
	table, last := opts, len(segments)-1
	for i, segment := range segments[:last] {
		switch existing := table[segment].(type) {
		case nil:
			next := map[string]any{}
			table[segment], table = next, next
		case map[string]any:
			table = existing
		default:
			return fmt.Errorf("%q is both a value and a table", strings.Join(segments[:i+1], "."))
		}
	}
	switch existing := table[segments[last]].(type) {
	case nil:
		table[segments[last]] = value
	case string:
		table[segments[last]] = []any{existing, value}
	case []any:
		table[segments[last]] = append(existing, value)
	default:
		return fmt.Errorf("%q is both a table and a value", key)
	}
	return nil
}

// instanceID takes the id option, or FreeID for an unnamed instance
func instanceID(typ string, opts map[string]any, taken []string) (string, error) {
	id, _ := opts["id"].(string)
	if _, set := opts["id"]; set && id == "" {
		return "", errors.New(`"id" takes one non-empty value`)
	}
	delete(opts, "id")
	if id != "" {
		return id, nil
	}
	return FreeID(typ, taken), nil
}

// FreeID is the first of TYPE, TYPE_2, TYPE_3... that no earlier instance of
// its role holds
func FreeID(typ string, taken []string) string {
	id := typ
	for n := 2; slices.Contains(taken, id); n++ {
		id = typ + "_" + strconv.Itoa(n)
	}
	return id
}