lixen/logwisp
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.29.0 the configuration engine, and lw --schema

Browse Source
This commit is contained in:
lixen committed 2026-10-09 18:42:11 -04:00
1 parent e8e5f05e47
commit 2d72772943
28 files changed
+1386 -240
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

internal/config/config.go
+18 -28
View File
@@ -4,15 +4,13 @@ package config
// Config is the top-level configuration structure for the LogWisp application
type Config struct {
Quiet bool `toml:"quiet"`
Quiet bool `toml:"quiet" help:"silence lw's own log and notices; pipeline output still flows"`
// Console sinks without their own color: "auto" is a terminal with
// NO_COLOR unset and TERM not dumb
Color string `toml:"color" lw:"enum=auto|always|never"`
// "auto" is a terminal with NO_COLOR unset and TERM not dumb
Color string `toml:"color" lw:"enum=auto|always|never" help:"level names in color on console sinks without their own"`
// Runtime behavior flags
StatusReporter bool `toml:"status_reporter"`
ConfigAutoReload bool `toml:"auto_reload"`
StatusReporter bool `toml:"status_reporter" help:"log pipeline statistics every 30 s at debug; without a file, off"`
ConfigAutoReload bool `toml:"auto_reload" help:"reload when the file changes"`
// Selected file path is runtime metadata, never a configurable override.
ConfigFile string `toml:"-"`
@@ -26,10 +24,10 @@ type Config struct {
// LogConfig represents the logging configuration for the LogWisp application itself
type LogConfig struct {
Output string `toml:"output" lw:"enum=file|stdout|stderr|split|all|none"`
Level string `toml:"level" lw:"enum=debug|info|warn|error"`
Format string `toml:"format" lw:"enum=raw|txt|json,zero=the log default"`
Sanitization string `toml:"sanitization" lw:"enum=raw|json|txt|shell,zero=the log default"` // console output
Output string `toml:"output" lw:"enum=file|stdout|stderr|split|all|none" help:"where lw's own log goes; split: warn and error to stderr"`
Level string `toml:"level" lw:"enum=debug|info|warn|error" help:"lowest level logged; without a file, warn"`
Format string `toml:"format" lw:"enum=raw|txt|json,zero=the log default" help:"format of lw's own log"`
Sanitization string `toml:"sanitization" lw:"enum=raw|json|txt|shell,zero=the log default" help:"how the console log escapes control characters"`
// File output settings (when Output includes "file" or "all")
File *LogFileConfig `toml:"file"`
@@ -40,25 +38,16 @@ type LogConfig struct {
// LogFileConfig defines settings for file-based application logging
type LogFileConfig struct {
// Directory for log files
Directory string `toml:"directory"`
// Base name for log files
Name string `toml:"name"`
// Maximum size per log file in MB
MaxSizeMB int64 `toml:"max_size_mb"`
// Maximum total size of all logs in MB
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
// Log retention in hours (0 = disabled)
RetentionHours float64 `toml:"retention_hours"`
Directory string `toml:"directory" lw:"hint=dir" help:"where file and all write; the limits below delete old .log files in it"`
Name string `toml:"name" help:"base name of the files"`
MaxSizeMB int64 `toml:"max_size_mb" help:"size at which a file rotates"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb" help:"size of all .log files there, beyond which the oldest go"`
RetentionHours float64 `toml:"retention_hours" help:"hours a .log file is kept; 0 keeps them"`
}
// LogConsoleConfig defines settings for console-based application logging
type LogConsoleConfig struct {
Target string `toml:"target" lw:"enum=stdout|stderr|split"`
Target string `toml:"target" lw:"enum=stdout|stderr|split" help:"accepted without effect: logging.output chooses the console"`
}
// --- Pipeline ---
@@ -74,12 +63,13 @@ type PipelineConfig struct {
// --- Flow ---
// FlowConfig consolidates all processing stages between sources and sinks
// FlowConfig holds the stages between sources and sinks, in the order entries
// pass them; the heartbeat runs beside them
type FlowConfig struct {
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
RateLimit *RateLimitConfig `toml:"rate_limit"`
Filters []FilterConfig `toml:"filters,omitempty"`
Format *FormatConfig `toml:"format"`
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
}
// --- Flow stages ---
internal/config/loader.go
+16 -10
View File
@@ -35,15 +35,9 @@ type Args struct {
// Settings maps each --path setting Load takes to whether it is a switch, which
// takes no separate value; the parser must not leave config to guess.
func Settings() map[string]bool {
c := lconfig.New()
if err := c.RegisterStructWithTags("", defaults(), lconfig.FormatTOML); err != nil {
panic(err) // Config is static: every run fails or none does
}
settings := map[string]bool{}
for path, value := range c.GetRegisteredPathsWithDefaults() {
if _, tables := value.([]PipelineConfig); !tables { // the pipeline flags set those
_, settings[path] = value.(bool)
}
for _, k := range SettingKeys() {
settings[k.Name] = k.Kind == "bool"
}
return settings
}
@@ -59,7 +53,7 @@ func Load(args Args) (*Manager, error) {
if len(specs) == 0 {
specs = envPipelineSpecs()
}
if _, err := buildPipelines(specs); err != nil {
if _, err := buildPipelines(specs, HostIsDir); err != nil {
return nil, err
}
m := &Manager{path: configPath, explicit: isExplicit, specs: specs}
@@ -132,12 +126,24 @@ func (m *Manager) Snapshot() (*Config, error) {
return cfg, nil
}
// Uninherit removes the color Snapshot gave each console sink, so a dump or a
// composition leaves it inherited
func Uninherit(pipelines []PipelineConfig, color string) {
for _, p := range pipelines {
for _, s := range p.PluginSinks {
if s.Type == "console" && s.Config["color"] == color {
delete(s.Config, "color")
}
}
}
}
// usePipelines replaces the file's or default pipelines with the spec ones.
func (m *Manager) usePipelines(cfg *Config) error {
if len(m.specs) == 0 {
return nil
}
pipelines, err := buildPipelines(m.specs)
pipelines, err := buildPipelines(m.specs, HostIsDir)
cfg.Pipelines = pipelines
return err
}
internal/config/preset.go
+66 -31
View File
@@ -2,6 +2,7 @@ package config
import (
"cmp"
"errors"
"fmt"
"maps"
"net"
@@ -16,16 +17,34 @@ import (
// a pipeline with it. Later flags add sources, sinks and filters, or replace
// its format, rate limit and heartbeat.
type Preset struct {
Name, Summary string
Params []PresetParam
build func(p *PipelineConfig, v map[string]string) error
Name string `json:"name"`
Summary string `json:"summary"`
Params []PresetParam `json:"keys"`
build func(p *PipelineConfig, v map[string]string, isDir IsDir) error
}
// PresetParam is one key. A list takes several values, ',' between them or
// the key repeated; any other key takes one.
type PresetParam struct {
Name, Default, Help string
Required, List bool
Name string `json:"name"`
Default string `json:"default,omitempty"`
Help string `json:"help"`
Required bool `json:"required,omitempty"`
List bool `json:"list,omitempty"`
}
// IsDir tells whether a preset's path is a directory, whose files it follows,
// or a file pattern. Off the target host it is the user's answer, or nil: a
// path ending in '/' is then a directory, one with a glob a pattern, and any
// other fails with ErrPathKind.
type IsDir func(path string) (bool, error)
var ErrPathKind = errors.New("path: a directory or a file pattern?")
// HostIsDir asks this host: a path that does not exist is a pattern
func HostIsDir(path string) (bool, error) {
fi, err := os.Stat(path)
return err == nil && fi.IsDir(), nil
}
// Keys several presets share
@@ -50,7 +69,7 @@ var (
var presets = []Preset{
{"pipe", "Standard input to standard output, line for line: lw without a file",
[]PresetParam{formatParam("raw")},
func(p *PipelineConfig, v map[string]string) error {
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
stdio := pipeDefault("")
p.PluginSources, p.PluginSinks = stdio.PluginSources, stdio.PluginSinks
p.Flow.Format = &FormatConfig{Type: v["format"]}
@@ -58,10 +77,11 @@ var presets = []Preset{
}},
{"tail", "Follow files to standard output, like tail -F",
[]PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")},
func(p *PipelineConfig, v map[string]string) error {
p.PluginSources, p.PluginSinks = pathOrStdin(v), pipeDefault("").PluginSinks
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir)
p.PluginSources, p.PluginSinks = sources, pipeDefault("").PluginSinks
p.Flow.Format = &FormatConfig{Type: v["format"]}
return nil
return err
}},
{"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream",
slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"),
@@ -71,8 +91,12 @@ var presets = []Preset{
{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
{Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"},
}, aclParams, tlsParams),
func(p *PipelineConfig, v map[string]string) error {
p.PluginSources = pathOrStdin(v)
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir)
if err != nil {
return err
}
p.PluginSources = sources
sink, err := listener(v, "off")
if err != nil {
return err
@@ -115,8 +139,12 @@ var presets = []Preset{
{Name: "key", Help: "client key file"},
{Name: "node", Help: "origin label; default: this host's name"},
},
func(p *PipelineConfig, v map[string]string) error {
p.PluginSources = pathOrStdin(v)
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir)
if err != nil {
return err
}
p.PluginSources = sources
typ, err := chainType(v["transport"])
if err != nil {
return err
@@ -159,7 +187,7 @@ var presets = []Preset{
{Name: "out", Help: "directory for the received entries; default: standard output"},
formatParam("json"),
}, aclParams, tlsParams),
func(p *PipelineConfig, v map[string]string) error {
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
typ, err := chainType(v["transport"])
if err != nil {
return err
@@ -197,18 +225,18 @@ var presets = []Preset{
func Presets() []Preset { return slices.Clone(presets) }
// ExpandPreset builds the pipeline a preset makes of values, as --preset does
func ExpandPreset(name string, values map[string]string) (PipelineConfig, error) {
func ExpandPreset(name string, values map[string]string, isDir IsDir) (PipelineConfig, error) {
p := PipelineConfig{Flow: &FlowConfig{}}
each := map[string][]string{}
for k, v := range values {
each[k] = []string{v}
}
err := applyPreset(&p, name, each)
err := applyPreset(&p, name, each, isDir)
return p, err
}
// applyPreset fills defaults and checks keys before the preset's build
func applyPreset(p *PipelineConfig, name string, values map[string][]string) error {
func applyPreset(p *PipelineConfig, name string, values map[string][]string, isDir IsDir) error {
i := slices.IndexFunc(presets, func(r Preset) bool { return r.Name == name })
if i < 0 {
var names []string
@@ -239,24 +267,31 @@ func applyPreset(p *PipelineConfig, name string, values map[string][]string) err
}
}
p.Name = cmp.Or(p.Name, name)
return r.build(p, v)
return r.build(p, v, isDir)
}
func pathOrStdin(v map[string]string) []PluginSourceConfig {
if v["path"] == "" {
return pipeDefault("").PluginSources
// pathOrStdin follows path: a directory's files, or the files a glob or a
// file name matches in its directory; without one, standard input
func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error) {
path := v["path"]
if path == "" {
return pipeDefault("").PluginSources, nil
}
return []PluginSourceConfig{fileSource(v["path"], v["from"])}
}
// fileSource follows path: a directory's files, or the files a glob or a
// file name matches in its directory.
func fileSource(path, from string) PluginSourceConfig {
dir, pattern := path, "*"
if fi, err := os.Stat(path); err != nil || !fi.IsDir() {
dir, pattern = filepath.Dir(path), filepath.Base(path)
dir, err := strings.HasSuffix(path, "/"), error(nil)
switch {
case isDir != nil:
dir, err = isDir(path)
case !dir && !strings.ContainsAny(path, `*?[\`):
err = ErrPathKind
}
return PluginSourceConfig{ID: "file", Type: "file", Config: map[string]any{"directory": dir, "pattern": pattern, "from": from}}
if err != nil {
return nil, err
}
directory, pattern := path, "*"
if !dir {
directory, pattern = filepath.Dir(path), filepath.Base(path)
}
return []PluginSourceConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": directory, "pattern": pattern, "from": v["from"]}}}, nil
}
// tlsModes is what each preset tls mode sets, and the preset keys it needs
internal/config/preset_test.go
+27
View File
@@ -1,6 +1,7 @@
package config
import (
"errors"
"reflect"
"strings"
"testing"
@@ -62,3 +63,29 @@ func TestPresetKeysAreChecked(t *testing.T) {
}
}
}
// A preset path is what isDir answers, a directory named with brackets too;
// without isDir, off the target host, it is a directory when it ends in '/',
// a pattern when it holds a glob, and otherwise ErrPathKind asks
func TestPresetPathKind(t *testing.T) {
yes := func(string) (bool, error) { return true, nil }
for _, c := range []struct {
path string
isDir IsDir
want string
}{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""}} {
p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir)
if c.want == "" {
if !errors.Is(err, ErrPathKind) {
t.Errorf("%s: %v, want ErrPathKind", c.path, err)
}
continue
}
if err != nil {
t.Fatal(err)
}
if m := p.PluginSources[0].Config; m["directory"].(string)+" "+m["pattern"].(string) != c.want {
t.Errorf("%s: %v, want %s", c.path, m, c.want)
}
}
}
internal/config/schema.go
+152 -58
View File
@@ -16,6 +16,10 @@ import (
// keys it takes
type Side int
var sideNames = []string{"local", "listener", "chain_listener", "dialer"}
func (s Side) MarshalText() ([]byte, error) { return []byte(sideNames[s]), nil }
const (
Local Side = iota // no network
Listener // the tcp and http sinks
@@ -25,12 +29,13 @@ const (
// Plugin is a catalogue row: a source or sink type and its options table
type Plugin struct {
Role, Type string // "source" or "sink", and the type its config names
Side Side
HTTP bool // speaks HTTP: takes bearer tokens and request limits
Single bool // one instance per process
Summary string
options reflect.Type
Role string `json:"-"` // "source" or "sink"
Type string `json:"type"`
Side Side `json:"side"`
HTTP bool `json:"http"` // speaks HTTP: takes bearer tokens and request limits
Single bool `json:"single"` // one instance per process
Summary string `json:"summary"`
options reflect.Type
}
var plugins = []Plugin{
@@ -49,6 +54,12 @@ var plugins = []Plugin{
{"sink", "http_chain", Dialer, true, false, "Post entry batches to an http_chain source", reflect.TypeFor[HTTPChainSinkOptions]()},
}
// Plugins lists the catalogue
func Plugins() []Plugin { return slices.Clone(plugins) }
// Keys describes the plugin's options
func (p Plugin) Keys() []Key { return keys(p.options) }
// LookupPlugin finds the catalogue row of a source or sink type
func LookupPlugin(role, typ string) (Plugin, bool) {
i := slices.IndexFunc(plugins, func(p Plugin) bool { return p.Role == role && p.Type == typ })
@@ -73,13 +84,8 @@ func (e *KeyError) Unwrap() error { return e.Err }
// it does, and lw: lists its rules: required, enum=a|b, min=N, max=N,
// zero=MEANING (zero is a value, not the default), hint=KIND, listener or dialer.
type option struct {
key, def, help string
required bool
enum []string
min, max *float64
zero string
hint, side string
index int
Key
index int
}
// options reads the keys of an options table; a malformed tag panics, as the
@@ -92,31 +98,31 @@ func options(t reflect.Type) []option {
if key == "" || key == "-" {
continue
}
o := option{key: key, def: f.Tag.Get("default"), help: f.Tag.Get("help"), index: i}
o := option{Key{Name: key, Kind: kinds[f.Type.Kind()], Default: f.Tag.Get("default"), Help: f.Tag.Get("help")}, i}
for rule := range strings.SplitSeq(f.Tag.Get("lw"), ",") {
name, value, _ := strings.Cut(rule, "=")
switch name {
case "":
case "required":
o.required = true
o.Required = true
case "listener", "dialer":
o.side = name
o.Side = name
case "enum":
o.enum = strings.Split(value, "|")
o.Enum = strings.Split(value, "|")
case "min", "max":
n, err := strconv.ParseFloat(value, 64)
if err != nil {
panic(fmt.Sprintf("%s.%s: lw %s=%q", t.Name(), key, name, value))
}
if name == "min" {
o.min = &n
o.Min = &n
} else {
o.max = &n
o.Max = &n
}
case "zero":
o.zero = value
o.Zero = value
case "hint":
o.hint = value
o.Hint = value
default:
panic(fmt.Sprintf("%s.%s: unknown lw rule %q", t.Name(), key, rule))
}
@@ -126,6 +132,81 @@ func options(t reflect.Type) []option {
return opts
}
// Key describes one option, for lw --schema and the composer: its tags, and
// its kind, a table's being "table" (tls, auth or acl)
type Key struct {
Name string `json:"key"`
Kind string `json:"kind"`
Default string `json:"default,omitempty"`
Help string `json:"help,omitempty"`
Required bool `json:"required,omitempty"`
Enum []string `json:"enum,omitempty"`
Min *float64 `json:"min,omitempty"`
Max *float64 `json:"max,omitempty"`
Zero string `json:"zero,omitempty"`
Hint string `json:"hint,omitempty"`
Side string `json:"side,omitempty"`
}
var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer",
reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"}
// Stage is a flow stage: its key in the flow table, whether it is a list (the
// filters), and its options
type Stage struct {
Key string `json:"key"`
List bool `json:"list,omitempty"`
Keys []Key `json:"keys"`
}
// FlowStages lists the flow table's stages, in the order entries pass them
func FlowStages() []Stage {
var out []Stage
t := reflect.TypeFor[FlowConfig]()
for _, o := range options(t) {
ft := t.Field(o.index).Type
out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem())})
}
return out
}
// KeysOf describes the options of a table: tls, auth or acl
func KeysOf[T any]() []Key { return keys(reflect.TypeFor[T]()) }
func keys(t reflect.Type) []Key {
var out []Key
for _, o := range options(t) {
if o.Kind == "" {
panic(fmt.Sprintf("%s.%s: no kind", t.Name(), o.Name))
}
out = append(out, o.Key)
}
return out
}
// SettingKeys describes the settings by their dotted paths, with the defaults
// lw has when it reads a file
func SettingKeys() []Key { return settingKeys(reflect.ValueOf(defaults()).Elem(), "") }
func settingKeys(v reflect.Value, prefix string) []Key {
var out []Key
for _, o := range options(v.Type()) {
switch f := v.Field(o.index); f.Kind() {
case reflect.Pointer:
out = append(out, settingKeys(f.Elem(), prefix+o.Name+".")...)
case reflect.Slice: // the pipelines, which their flags set
default:
k := o.Key
k.Name = prefix + k.Name
if !f.IsZero() {
k.Default = fmt.Sprint(f.Interface())
}
out = append(out, k)
}
}
return out
}
// Decode reads a plugin's config map into its options: unknown keys, kinds,
// the defaults, the tag rules, the tls, auth and acl rules of its side, then the
// options' own Check. It reads no file.
@@ -145,7 +226,7 @@ func decode(role, typ string, m map[string]any) (any, error) {
if err := checkKeys(m, p.options, ""); err != nil {
return nil, err
}
m = clone(m)
m = Clone(m)
if err := coerce(p.options, m, ""); err != nil {
return nil, err // a file's wrong kind, named by its key
}
@@ -168,9 +249,8 @@ func decode(role, typ string, m map[string]any) (any, error) {
// Settle gives a decoded table its defaults, applies the tag rules, then its
// own Check: Decode's last steps, for the flow stages a file decodes typed
func Settle(v any) error {
rv := reflect.ValueOf(v).Elem()
settle(rv)
if err := checkTags(rv, ""); err != nil {
Fill(v)
if err := checkTags(reflect.ValueOf(v).Elem(), ""); err != nil {
return err
}
if c, ok := v.(interface{ Check() error }); ok {
@@ -179,15 +259,18 @@ func Settle(v any) error {
return nil
}
// Fill gives a table its defaults and checks nothing: Settle's first step
func Fill(v any) { settle(reflect.ValueOf(v).Elem()) }
// preset sets every default before decoding, so a switch that defaults on
// stays on unless set off; it does so in the tables m sets
func preset(v reflect.Value, m map[string]any) {
for _, o := range options(v.Type()) {
f := v.Field(o.index)
if sub, ok := m[o.key].(map[string]any); ok && f.Kind() == reflect.Pointer && f.Type().Elem().Kind() == reflect.Struct {
if sub, ok := m[o.Name].(map[string]any); ok && f.Kind() == reflect.Pointer && f.Type().Elem().Kind() == reflect.Struct {
f.Set(reflect.New(f.Type().Elem()))
preset(f.Elem(), sub)
} else if o.def != "" {
} else if o.Default != "" {
setDefault(f, o)
}
}
@@ -203,7 +286,7 @@ func settle(v reflect.Value) {
if !f.IsNil() {
settle(f.Elem())
}
case o.def != "" && o.zero == "" && f.Kind() != reflect.Bool && f.IsZero():
case o.Default != "" && o.Zero == "" && f.Kind() != reflect.Bool && f.IsZero():
setDefault(f, o)
}
}
@@ -213,24 +296,24 @@ func setDefault(f reflect.Value, o option) {
var err error
switch f.Kind() {
case reflect.String:
f.SetString(o.def)
f.SetString(o.Default)
case reflect.Int64:
var n int64
n, err = strconv.ParseInt(o.def, 10, 64)
n, err = strconv.ParseInt(o.Default, 10, 64)
f.SetInt(n)
case reflect.Float64:
var n float64
n, err = strconv.ParseFloat(o.def, 64)
n, err = strconv.ParseFloat(o.Default, 64)
f.SetFloat(n)
case reflect.Bool:
var b bool
b, err = strconv.ParseBool(o.def)
b, err = strconv.ParseBool(o.Default)
f.SetBool(b)
default:
err = errors.New("no default for this kind")
}
if err != nil {
panic(fmt.Sprintf("%s: default %q: %v", o.key, o.def, err))
panic(fmt.Sprintf("%s: default %q: %v", o.Name, o.Default, err))
}
}
@@ -243,12 +326,12 @@ func checkTags(v reflect.Value, prefix string) error {
f := v.Field(o.index)
var err error
switch {
case o.required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == ""):
case o.Required && (f.IsZero() || f.Kind() == reflect.String && strings.TrimSpace(f.String()) == ""):
err = errors.New("must be set")
case f.Kind() == reflect.String && len(o.enum) > 0 && (f.String() != "" || o.zero == "" && o.def == "") && !slices.Contains(o.enum, f.String()):
err = fmt.Errorf("must be one of %s, got %q", strings.Join(o.enum, ", "), f.String())
case f.Kind() == reflect.String && len(o.Enum) > 0 && (f.String() != "" || o.Zero == "" && o.Default == "") && !slices.Contains(o.Enum, f.String()):
err = fmt.Errorf("must be one of %s, got %q", strings.Join(o.Enum, ", "), f.String())
case f.CanFloat() || f.CanInt():
if f.IsZero() && !o.required {
if f.IsZero() && !o.Required {
break
}
n := float64(0)
@@ -260,16 +343,16 @@ func checkTags(v reflect.Value, prefix string) error {
switch {
case math.IsNaN(n) || math.IsInf(n, 0):
err = errors.New("must be a finite number")
case o.min != nil && o.max != nil && !(n >= *o.min && n <= *o.max):
err = fmt.Errorf("must be from %s to %s, got %s", num(*o.min), num(*o.max), num(n))
case o.min != nil && !(n >= *o.min):
err = fmt.Errorf("must be at least %s, got %s", num(*o.min), num(n))
case o.max != nil && !(n <= *o.max):
err = fmt.Errorf("must be at most %s, got %s", num(*o.max), num(n))
case o.Min != nil && o.Max != nil && !(n >= *o.Min && n <= *o.Max):
err = fmt.Errorf("must be from %s to %s, got %s", num(*o.Min), num(*o.Max), num(n))
case o.Min != nil && !(n >= *o.Min):
err = fmt.Errorf("must be at least %s, got %s", num(*o.Min), num(n))
case o.Max != nil && !(n <= *o.Max):
err = fmt.Errorf("must be at most %s, got %s", num(*o.Max), num(n))
}
}
if err != nil {
return &KeyError{prefix + o.key, err}
return &KeyError{prefix + o.Name, err}
}
}
return nil
@@ -281,8 +364,8 @@ func num(n float64) string { return strconv.FormatFloat(n, 'f', -1, 64) }
func sideKeys(t reflect.Type, side string) []string {
var keys []string
for _, o := range options(t) {
if o.side == side {
keys = append(keys, o.key)
if o.Side == side {
keys = append(keys, o.Name)
}
}
return keys
@@ -301,7 +384,7 @@ func Coerce(role, typ string, m map[string]any) error {
func coerce(t reflect.Type, m map[string]any, prefix string) error {
for _, o := range options(t) {
value, ok := m[o.key]
value, ok := m[o.Name]
if !ok {
continue
}
@@ -310,25 +393,28 @@ func coerce(t reflect.Type, m map[string]any, prefix string) error {
ft = ft.Elem()
}
if sub, ok := value.(map[string]any); ok && ft.Kind() == reflect.Struct {
if err := coerce(ft, sub, prefix+o.key+"."); err != nil {
if err := coerce(ft, sub, prefix+o.Name+"."); err != nil {
return err
}
continue
}
v, err := coerceValue(ft.Kind(), value)
if err != nil {
return &KeyError{prefix + o.key, err}
return &KeyError{prefix + o.Name, err}
}
m[o.key] = v
m[o.Name] = v
}
return nil
}
// coerceValue converts text, and a JSON float for an integer; its errors
// never hold the value, which may be a misplaced secret
// coerceValue converts text, a JSON float for an integer and a TOML integer
// for a number; its errors never hold the value, which may be a misplaced secret
func coerceValue(kind reflect.Kind, value any) (any, error) {
s, text := value.(string)
n, integer := value.(int64)
switch {
case kind == reflect.Float64 && integer:
return float64(n), nil
case kind == reflect.Slice && text:
return []any{s}, nil
case kind == reflect.Int64 && text:
@@ -357,14 +443,22 @@ func coerceValue(kind reflect.Kind, value any) (any, error) {
return value, nil
}
// clone copies m and the tables in it, so decoding leaves the caller's map be
func clone(m map[string]any) map[string]any {
// Clone copies an options map with its tables and lists, so decoding or an
// edit leaves the caller's map be
func Clone(m map[string]any) map[string]any {
c := make(map[string]any, len(m))
for k, v := range m {
if sub, ok := v.(map[string]any); ok {
v = clone(sub)
}
c[k] = v
c[k] = cloneValue(v)
}
return c
}
func cloneValue(v any) any {
switch v := v.(type) {
case map[string]any:
return Clone(v)
case []any:
return slices.Clone(v)
}
return v
}
internal/config/schema_test.go
+2 -2
View File
@@ -13,8 +13,8 @@ func TestEveryRowDecodesFromItsRequiredKeys(t *testing.T) {
for _, p := range plugins {
m := map[string]any{}
for _, o := range options(p.options) {
if o.required {
m[o.key] = "1"
if o.Required {
m[o.Name] = "1"
}
}
if err := Coerce(p.Role, p.Type, m); err != nil {
internal/config/spec.go
+193 -26
View File
@@ -4,7 +4,9 @@ import (
"cmp"
"errors"
"fmt"
"maps"
"os"
"reflect"
"slices"
"strconv"
"strings"
@@ -17,11 +19,12 @@ type specKind struct {
typed bool // SPEC starts with TYPE
many bool // repeatable within a pipeline; the environment adds _1.._N
first bool // starts its pipeline, and names it by TYPE unless --pipeline did
apply func(p *PipelineConfig, typ string, opts map[string]any) error
apply func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error
write func(p *PipelineConfig) []string // its specs that build p's part; nil for preset
}
var specKinds = []specKind{
{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any, isDir IsDir) error {
values := map[string][]string{}
for k, v := range opts {
switch v := v.(type) {
@@ -35,9 +38,9 @@ var specKinds = []specKind{
return fmt.Errorf("preset keys do not nest: %q", k)
}
}
return applyPreset(p, typ, values)
}},
{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
return applyPreset(p, typ, values, isDir)
}, nil},
{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
ids := make([]string, len(p.PluginSources))
for i, s := range p.PluginSources {
ids[i] = s.ID
@@ -45,8 +48,13 @@ var specKinds = []specKind{
id, err := instanceID(typ, opts, ids)
p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
return cmp.Or(err, Coerce("source", typ, opts))
}, func(p *PipelineConfig) (specs []string) {
for _, s := range p.PluginSources {
specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
}
return specs
}},
{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
ids := make([]string, len(p.PluginSinks))
for i, s := range p.PluginSinks {
ids[i] = s.ID
@@ -54,27 +62,61 @@ var specKinds = []specKind{
id, err := instanceID(typ, opts, ids)
p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
return cmp.Or(err, Coerce("sink", typ, opts))
}, func(p *PipelineConfig) (specs []string) {
for _, s := range p.PluginSinks {
specs = append(specs, formatSpec(s.Type, map[string]any{"id": s.ID}, s.Config))
}
return specs
}},
{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
opts["type"] = typ
f := FilterConfig{}
err := Scan(opts, &f)
p.Flow.Filters = append(p.Flow.Filters, f)
return err
}, func(p *PipelineConfig) (specs []string) {
for _, f := range p.Flow.Filters {
Fill(&f)
specs = append(specs, typedSpec(&f))
}
return specs
}},
{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any, _ IsDir) error {
opts["type"] = typ
p.Flow.Format = &FormatConfig{}
return Scan(opts, p.Flow.Format)
}, func(p *PipelineConfig) []string {
if p.Flow.Format == nil {
return nil
}
format := *p.Flow.Format
Fill(&format)
return []string{typedSpec(&format)}
}},
// Naming a stage turns it on: the file defaults (pass, disabled) would not.
{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
return Scan(opts, p.Flow.RateLimit)
}, func(p *PipelineConfig) []string {
if p.Flow.RateLimit == nil {
return nil
}
rate := *p.Flow.RateLimit
Fill(&rate) // the policy, whose default here is drop
return []string{formatSpec("", Values(&rate))}
}},
{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
return Scan(opts, p.Flow.Heartbeat)
}, func(p *PipelineConfig) []string {
if p.Flow.Heartbeat == nil {
return nil
}
beat := *p.Flow.Heartbeat
Fill(&beat)
m := Values(&beat)
m["enabled"] = beat.Enabled
return []string{formatSpec("", m)}
}},
}
@@ -88,15 +130,28 @@ type pipelineSpec struct {
// Spec is one pipeline flag: its name without dashes, and its value.
type Spec struct{ Flag, Value string }
// SpecFlags names the pipeline flags, for the command-line parser.
func SpecFlags() []string {
flags := []string{"pipeline"}
// SpecFlag is a pipeline flag, for the parser and lw --schema: its name
// without dashes, its environment variable, which takes _1.._N when Many, and
// whether SPEC starts with TYPE
type SpecFlag struct {
Flag string `json:"flag"`
Variable string `json:"variable"`
Typed bool `json:"typed"`
Many bool `json:"many"`
}
func SpecFlags() []SpecFlag {
flags := []SpecFlag{{"pipeline", specVariable("pipeline"), false, false}}
for _, k := range specKinds {
flags = append(flags, k.flag)
flags = append(flags, SpecFlag{k.flag, specVariable(k.flag), k.typed, k.many})
}
return flags
}
func specVariable(flag string) string {
return "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(flag, "-", "_"))
}
// cliSpecs looks up the kind of each command-line spec.
func cliSpecs(in []Spec) ([]pipelineSpec, error) {
var specs []pipelineSpec
@@ -121,12 +176,12 @@ func cliSpecs(in []Spec) ([]pipelineSpec, error) {
// counts as unset, so container templates may leave optional ones blank.
func envPipelineSpecs() []pipelineSpec {
var specs []pipelineSpec
if name := os.Getenv("LOGWISP_PIPELINE"); name != "" {
specs = append(specs, pipelineSpec{"LOGWISP_PIPELINE", nil, name})
if name := os.Getenv(specVariable("pipeline")); name != "" {
specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name})
}
for i := range specKinds {
kind := &specKinds[i]
base := "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(kind.flag, "-", "_"))
base := specVariable(kind.flag)
type numbered struct {
n uint64
name, spec string
@@ -150,9 +205,18 @@ func envPipelineSpecs() []pipelineSpec {
return specs
}
// SpecPipelines builds the pipelines of command-line specs, as Load does
func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) {
s, err := cliSpecs(specs)
if err != nil {
return nil, err
}
return buildPipelines(s, isDir)
}
// buildPipelines turns specs, in order, into pipelines; specs before the first
// --pipeline go to one named "cli". Every call returns fresh maps.
func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
func buildPipelines(specs []pipelineSpec, isDir IsDir) ([]PipelineConfig, error) {
var pipelines []PipelineConfig
var seen map[*specKind]bool
for _, s := range specs {
@@ -181,7 +245,7 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value)
}
seen[s.kind] = true
if err := s.kind.apply(p, typ, opts); err != nil {
if err := s.kind.apply(p, typ, opts, isDir); err != nil {
return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
}
}
@@ -191,6 +255,104 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
return pipelines, nil
}
// PipelineSpecs writes pipelines as the flags that build them: --pipeline,
// ids, flow stage defaults and heartbeat's enabled explicit, as some flags'
// defaults differ from a file's
func PipelineSpecs(pipelines []PipelineConfig) []Spec {
var specs []Spec
for _, p := range pipelines {
specs = append(specs, Spec{"pipeline", p.Name})
p.Flow = cmp.Or(p.Flow, &FlowConfig{})
for _, k := range specKinds {
if k.write != nil {
for _, value := range k.write(&p) {
specs = append(specs, Spec{k.flag, value})
}
}
}
}
return specs
}
// typedSpec writes a stage whose type key is its spec's TYPE
func typedSpec(stage any) string {
m := Values(stage)
typ, _ := m["type"].(string)
delete(m, "type")
return formatSpec(typ, m)
}
// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a
// repeated key
func formatSpec(typ string, tables ...map[string]any) string {
var parts []string
if typ != "" {
parts = append(parts, escape(typ))
}
var add func(prefix string, m map[string]any)
add = func(prefix string, m map[string]any) {
for _, k := range slices.Sorted(maps.Keys(m)) {
switch v := m[k].(type) {
case map[string]any:
add(prefix+k+".", v)
case []any:
for _, e := range v {
parts = append(parts, escape(prefix+k)+"="+escape(text(e)))
}
default:
parts = append(parts, escape(prefix+k)+"="+escape(text(v)))
}
}
}
for _, m := range tables {
add("", m)
}
return strings.Join(parts, ",")
}
func text(v any) string {
if f, ok := v.(float64); ok {
return strconv.FormatFloat(f, 'g', -1, 64)
}
return fmt.Sprint(v)
}
// escape is unescape's inverse: ',' and '=' escaped, and '\' where an escape
// would otherwise start, so a regex's \d stays as typed
func escape(s string) string {
var b strings.Builder
followed := s + "," // as a value is, unless it ends the spec
for i := range len(s) {
if s[i] == ',' || s[i] == '=' || escapes(followed, i) {
b.WriteByte('\\')
}
b.WriteByte(s[i])
}
return b.String()
}
// Values gives a table's keys that are not zero, as a spec sets them, for the
// composer to edit and Scan back
func Values(v any) map[string]any {
rv, m := reflect.ValueOf(v).Elem(), map[string]any{}
for _, o := range options(rv.Type()) {
switch f := rv.Field(o.index); {
case f.IsZero():
case f.Kind() == reflect.String:
m[o.Name] = f.String()
case f.Kind() == reflect.Slice:
var list []any
for i := range f.Len() {
list = append(list, f.Index(i).Interface())
}
m[o.Name] = list
default:
m[o.Name] = f.Interface()
}
}
return m
}
// useStdio makes a pipeline without a source read stdin, and one without a
// sink write stdout, as a filter does.
func useStdio(p *PipelineConfig) {
@@ -225,7 +387,7 @@ func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err er
case typed && key == "type":
err = errors.New(`TYPE already sets "type"`)
default:
err = setOption(opts, key, unescape(value))
err = SetOption(opts, key, unescape(value))
}
if err != nil {
return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
@@ -262,8 +424,8 @@ func escapes(s string, i int) bool {
return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0
}
// setOption stores value at a dotted key; a second value turns it into a list.
func setOption(opts map[string]any, key, value string) error {
// SetOption stores value at a dotted key; a second value turns it into a list.
func SetOption(opts map[string]any, key, value string) error {
segments := strings.Split(key, ".")
if slices.Contains(segments, "") {
return fmt.Errorf("empty key in %q", key)
@@ -293,8 +455,7 @@ func setOption(opts map[string]any, key, value string) error {
return nil
}
// instanceID takes the id option; an unnamed instance gets the first of TYPE,
// TYPE_2, TYPE_3... that no earlier instance of its role holds.
// instanceID takes the id option, or FreeID for an unnamed instance
func instanceID(typ string, opts map[string]any, taken []string) (string, error) {
id, _ := opts["id"].(string)
if _, set := opts["id"]; set && id == "" {
@@ -304,9 +465,15 @@ func instanceID(typ string, opts map[string]any, taken []string) (string, error)
if id != "" {
return id, nil
}
id = typ
return FreeID(typ, taken), nil
}
// FreeID is the first of TYPE, TYPE_2, TYPE_3... that no earlier instance of
// its role holds
func FreeID(typ string, taken []string) string {
id := typ
for n := 2; slices.Contains(taken, id); n++ {
id = typ + "_" + strconv.Itoa(n)
}
return id, nil
return id
}
internal/config/spec_test.go
+14
View File
@@ -213,3 +213,17 @@ func TestReloadKeepsSpecPipelines(t *testing.T) {
t.Fatalf("reload after the file was removed: %v", err)
}
}
// Written specs build the pipelines a file would: each flow stage with its
// defaults, so the rate limit keeps its file policy and the heartbeat its off
func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) {
p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{Rate: 5}, Heartbeat: &HeartbeatConfig{}},
PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}},
PluginSinks: []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}}
got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil)
p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{Rate: 5, Policy: "pass"},
Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}}
if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) {
t.Fatalf("got %+v %v\nwant %+v", got, err, p)
}
}
internal/config/validate.go
+18 -16
View File
@@ -29,20 +29,6 @@ func ValidateConfig(cfg *Config) error {
if cfg == nil {
return fmt.Errorf("config is nil")
}
if len(cfg.Pipelines) == 0 {
return fmt.Errorf("no pipelines configured")
}
// Reject duplicate pipeline names (service map is keyed by name)
names := make(map[string]struct{}, len(cfg.Pipelines))
for i, p := range cfg.Pipelines {
if _, dup := names[p.Name]; dup {
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
}
names[p.Name] = struct{}{}
}
if err := checkTags(reflect.ValueOf(cfg).Elem(), ""); err != nil {
return err
}
@@ -57,8 +43,24 @@ func ValidateConfig(cfg *Config) error {
}
}
return ValidatePipelines(cfg.Pipelines)
}
// ValidatePipelines is ValidateConfig's check of the pipelines
func ValidatePipelines(pipelines []PipelineConfig) error {
if len(pipelines) == 0 {
return fmt.Errorf("no pipelines configured")
}
// Reject duplicate pipeline names (service map is keyed by name)
names := make(map[string]struct{}, len(pipelines))
for i, p := range pipelines {
if _, dup := names[p.Name]; dup {
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
}
names[p.Name] = struct{}{}
}
single := map[string]string{} // role and type of a single-instance plugin: its pipeline
for i, p := range cfg.Pipelines {
for i, p := range pipelines {
if err := lconfig.NonEmpty(p.Name); err != nil {
return fmt.Errorf("pipeline[%d].name: %w", i, err)
}
@@ -196,7 +198,7 @@ func (o *TLSOptions) Check(listener bool) error {
other = "listener"
}
for _, opt := range options(v.Type()) {
if f := v.Field(opt.index); opt.side == other && !f.IsZero() && (f.Kind() != reflect.Slice || f.Len() > 0) {
if f := v.Field(opt.index); opt.Side == other && !f.IsZero() && (f.Kind() != reflect.Slice || f.Len() > 0) {
return fmt.Errorf("tls: %s apply to %ss", and(sideKeys(v.Type(), other)), other)
}
}