v0.32.0 lwconf engine update: config by path, schema from code, wasm

This commit is contained in:
lixen committed 2026-10-10 18:28:01 -04:00
1 parent c2e3d669da
commit a1e9d21d06
33 files changed
+1278 -421

No files matched your search

+3 -3
View File
@@ -319,8 +319,8 @@ type AuthOptions struct {
CredentialsFile string `toml:"credentials_file" lw:"listener,hint=file" help:"scram: verifiers, from lw auth add-user"`
// exp has whole seconds, a sub-second expires_in reads as unknown, and 10 s
// leaves room for renewal ahead of expiry
TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
TrustedProxies []string `toml:"trusted_proxies" lw:"listener,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"`
TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,http,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
TrustedProxies []string `toml:"trusted_proxies" lw:"listener,http,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"`
Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"`
PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"`
}
@@ -335,7 +335,7 @@ type ACLOptions struct {
ProxyProtocol string `toml:"proxy_protocol" default:"off" lw:"enum=off|optional|required" help:"read a PROXY header from proxy_from"`
ProxyFrom []string `toml:"proxy_from" lw:"hint=cidr" help:"proxies that may send a PROXY header"`
MaxConnectionsPerClient int64 `toml:"max_connections_per_client" lw:"min=0,zero=unlimited" help:"connections a client may hold"`
RequestsPerSecondPerClient float64 `toml:"requests_per_second_per_client" lw:"min=0,zero=unlimited" help:"HTTP: requests a client may make each second"`
RequestsPerSecondPerClient float64 `toml:"requests_per_second_per_client" lw:"http,min=0,zero=unlimited" help:"HTTP: requests a client may make each second"`
}
// --- TLS Options ---
+9 -5
View File
@@ -51,7 +51,7 @@ func Load(args Args) (*Manager, error) {
return nil, err
}
if len(specs) == 0 {
specs = envPipelineSpecs()
specs = envPipelineSpecs(os.Environ())
}
if _, err := buildPipelines(specs, HostIsDir); err != nil {
return nil, err
@@ -63,6 +63,7 @@ func Load(args Args) (*Manager, error) {
initial.Logging.Level = "warn"
initial.StatusReporter = false
}
var invalid error // the validator's problems, which lw prints a line each
cfg, err := lconfig.NewBuilder().
WithTarget(initial).
WithEnvPrefix("LOGWISP_").
@@ -70,10 +71,10 @@ func Load(args Args) (*Manager, error) {
WithFile(configPath).
WithTypedValidator(func(cfg *Config) error {
effective := *cfg // the builder keeps cfg; only the copy is replaced
if err := m.usePipelines(&effective); err != nil {
return err
if invalid = m.usePipelines(&effective); invalid == nil {
invalid = ValidateConfig(&effective)
}
return ValidateConfig(&effective)
return invalid
}).
WithSecurityOptions(lconfig.SecurityOptions{
PreventPathTraversal: true,
@@ -81,7 +82,10 @@ func Load(args Args) (*Manager, error) {
}).
Build()
if err != nil {
if !errors.Is(err, lconfig.ErrConfigNotFound) {
switch {
case invalid != nil:
return nil, fmt.Errorf("invalid configuration:\n%w", invalid)
case !errors.Is(err, lconfig.ErrConfigNotFound):
return nil, fmt.Errorf("failed to load or validate config: %w", err)
}
if isExplicit {
+132 -57
View File
@@ -8,6 +8,7 @@ import (
"net"
"os"
"path/filepath"
"reflect"
"slices"
"strconv"
"strings"
@@ -24,22 +25,48 @@ type Preset struct {
}
// PresetParam is one key. A list takes several values, ',' between them or
// the key repeated; any other key takes one.
// the key repeated; any other key takes one. Enum and Hint are those of the
// option the key sets.
type PresetParam struct {
Name string `json:"name"`
Default string `json:"default,omitempty"`
Help string `json:"help"`
Required bool `json:"required,omitempty"`
List bool `json:"list,omitempty"`
Name string `json:"name"`
Default string `json:"default,omitempty"`
Help string `json:"help"`
Required bool `json:"required,omitempty"`
List bool `json:"list,omitempty"`
Enum []string `json:"enum,omitempty"`
Hint string `json:"hint,omitempty"`
}
// Usage is the key's help after its choices, as lw preset and lw --tui show it
func (p PresetParam) Usage() string {
if len(p.Enum) == 0 {
return p.Help
}
return strings.Join(p.Enum, "|") + ": " + p.Help
}
// mirror gives a key the enum and hint of option key of table T, which it
// sets; a switch takes true or false
func mirror[T any](p PresetParam, key string) PresetParam {
opts := options(reflect.TypeFor[T]())
i := slices.IndexFunc(opts, func(o option) bool { return o.Name == key })
if i < 0 {
panic(fmt.Sprintf("preset key %s: %s has no key %q", p.Name, reflect.TypeFor[T]().Name(), key))
}
p.Enum, p.Hint = opts[i].Enum, opts[i].Hint
if opts[i].Kind == "bool" {
p.Enum = []string{"true", "false"}
}
return p
}
// IsDir tells whether a preset's path is a directory, whose files it follows,
// or a file pattern. Off the target host it is the user's answer, or nil: a
// or a file pattern. Off the target host it is the user's Answers, or nil: a
// path ending in '/' is then a directory, one with a glob a pattern, and any
// other fails with ErrPathKind.
type IsDir func(path string) (bool, error)
var ErrPathKind = errors.New("path: a directory or a file pattern?")
var ErrPathKind error = &KeyError{"path", errors.New("a directory or a file pattern?")}
// HostIsDir asks this host: a path that does not exist is a pattern
func HostIsDir(path string) (bool, error) {
@@ -47,29 +74,67 @@ func HostIsDir(path string) (bool, error) {
return err == nil && fi.IsDir(), nil
}
// Answers is IsDir off the host: whether each path answered is a directory,
// and nil's rule for the others
func Answers(dirs map[string]bool) IsDir {
return func(path string) (bool, error) {
if dir, ok := dirs[path]; ok {
return dir, nil
}
if dir := strings.HasSuffix(path, "/"); dir || strings.ContainsAny(path, `*?[\`) {
return dir, nil
}
return false, ErrPathKind
}
}
// Keys several presets share
var (
pathParam = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"}
fromParam = PresetParam{Name: "from", Default: "end", Help: "start|end: where reading a file starts"}
formatParam = func(def string) PresetParam { return PresetParam{Name: "format", Default: def, Help: "raw|txt|json"} }
tlsParams = []PresetParam{
{Name: "cert", Help: "certificate file, for tls=files"},
{Name: "key", Help: "key file, for tls=files"},
{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"},
{Name: "issuer_key", Help: "CA key file, for tls=issuer"},
fromParam = mirror[FileSourceOptions](PresetParam{Name: "from", Default: "end", Help: "where reading a file starts"}, "from")
formatParam = func(def string) PresetParam {
return mirror[FormatConfig](PresetParam{Name: "format", Default: def, Help: "entry format"}, "type")
}
tlsParams = []PresetParam{
mirror[TLSOptions](PresetParam{Name: "cert", Help: "certificate file, for tls=files"}, "cert_file"),
mirror[TLSOptions](PresetParam{Name: "key", Help: "key file, for tls=files"}, "key_file"),
mirror[TLSOptions](PresetParam{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"}, "issuer_cert_file"),
mirror[TLSOptions](PresetParam{Name: "issuer_key", Help: "CA key file, for tls=issuer"}, "issuer_key_file"),
{Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"},
}
aclParams = []PresetParam{
{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"},
{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"},
mirror[ACLOptions](PresetParam{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family (either behind proxy=); default: all; ',' between them"}, "allow"),
mirror[ACLOptions](PresetParam{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, "deny"),
}
transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
userParams = []PresetParam{
transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp_chain or http_chain", Enum: transports()}
usersParam = func(who string) PresetParam {
return mirror[AuthOptions](PresetParam{Name: "users", Help: "credentials file (lw auth add-user): " + who + " log in with SCRAM"}, "credentials_file")
}
userParams = []PresetParam{
{Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"},
{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"},
mirror[AuthOptions](PresetParam{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, "password_file"),
}
)
// tlsParam is a listener's tls key: a tlsModes name, off only where the
// listener may serve plain text
func tlsParam(def string, off bool) PresetParam {
p := PresetParam{Name: "tls", Default: def, Help: "a certificate made at startup (self-signed, or from issuer_*), or files"}
if off {
p.Help = "plain text, " + p.Help
}
for _, m := range tlsModes {
if m.name != "off" || off {
p.Enum = append(p.Enum, m.name)
}
}
return p
}
// The listeners' tls keys: the aggregator always serves TLS, as edges send
// credentials and logs
var serveTLS, aggregatorTLS = tlsParam("off", true), tlsParam("self", false)
var presets = []Preset{
{"pipe", "Standard input to standard output, line for line: lw without a file",
[]PresetParam{formatParam("raw")},
@@ -90,11 +155,11 @@ var presets = []Preset{
{"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream",
slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"),
{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
serveTLS,
usersParam("readers"),
}, userParams, []PresetParam{
{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
{Name: "viewer", Default: "false", Help: "true: the login page and viewer, behind proxy; without user or users the viewer is always on"},
mirror[AuthOptions](PresetParam{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, "trusted_proxies"),
mirror[HTTPSinkOptions](PresetParam{Name: "viewer", Default: "false", Help: "the login page and viewer, behind proxy; without user or users the viewer is always on"}, "viewer_page"),
}, aclParams, tlsParams),
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir)
@@ -102,7 +167,7 @@ var presets = []Preset{
return err
}
p.PluginSources = sources
sink, err := listener(v, "off")
sink, err := listener(v, serveTLS)
if err != nil {
return err
}
@@ -139,12 +204,12 @@ var presets = []Preset{
slices.Concat([]PresetParam{pathParam, fromParam,
{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"},
transportParam,
{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"},
mirror[TLSOptions](PresetParam{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, "ca_file"),
{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
}, userParams, []PresetParam{
{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"},
{Name: "key", Help: "client key file"},
mirror[TLSOptions](PresetParam{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, "cert_file"),
mirror[TLSOptions](PresetParam{Name: "key", Help: "client key file"}, "key_file"),
{Name: "node", Help: "origin label; default: this host's name"},
}),
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
@@ -188,11 +253,11 @@ var presets = []Preset{
slices.Concat([]PresetParam{
{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"},
transportParam,
{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
{Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"},
aggregatorTLS,
usersParam("edges"),
}, userParams, []PresetParam{
{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
{Name: "out", Help: "directory for the received entries; default: standard output"},
mirror[TLSOptions](PresetParam{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, "client_ca_file"),
mirror[FileSinkOptions](PresetParam{Name: "out", Help: "directory for the received entries; default: standard output"}, "directory"),
formatParam("json"),
}, aclParams, tlsParams),
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
@@ -200,10 +265,7 @@ var presets = []Preset{
if err != nil {
return err
}
if v["tls"] == "off" {
return fmt.Errorf("tls=off: edges send credentials and logs, the aggregator always serves TLS")
}
src, err := listener(v, "self")
src, err := listener(v, aggregatorTLS)
if err != nil {
return err
}
@@ -288,13 +350,10 @@ func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error)
if path == "" {
return pipeDefault("").PluginSources, nil
}
dir, err := strings.HasSuffix(path, "/"), error(nil)
switch {
case isDir != nil:
dir, err = isDir(path)
case !dir && !strings.ContainsAny(path, `*?[\`):
err = ErrPathKind
if isDir == nil {
isDir = Answers(nil)
}
dir, err := isDir(path)
if err != nil {
return nil, err
}
@@ -308,20 +367,23 @@ func pathOrStdin(v map[string]string, isDir IsDir) ([]PluginSourceConfig, error)
// tlsModes is what each preset tls mode sets, and the preset keys it needs
// by the tls option each fills; made marks a certificate made at startup,
// which takes hosts.
var tlsModes = map[string]struct {
var tlsModes = []tlsMode{
{name: "off"},
{name: "self", set: map[string]any{"self_signed": true}, made: true},
{name: "issuer", needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true},
{name: "files", needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
}
type tlsMode struct {
name string
set map[string]any
needs map[string]string
made bool
}{
"off": {},
"self": {set: map[string]any{"self_signed": true}, made: true},
"issuer": {needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true},
"files": {needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
}
// listener starts a listening plugin's options: host, port, acl and tls,
// whose mode decides which certificate keys apply.
func listener(v map[string]string, def string) (map[string]any, error) {
// listener starts a listening plugin's options: host, port, acl and tls, in
// a mode its key offers, which decides the certificate keys that apply
func listener(v map[string]string, key PresetParam) (map[string]any, error) {
host, port, err := splitAddr("listen", v["listen"])
if err != nil {
return nil, err
@@ -335,11 +397,12 @@ func listener(v map[string]string, def string) (map[string]any, error) {
if len(acl) > 0 {
opts["acl"] = acl
}
mode := cmp.Or(v["tls"], def)
m, ok := tlsModes[mode]
if !ok {
return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(slices.Sorted(maps.Keys(tlsModes)), ", "))
mode := cmp.Or(v["tls"], key.Default)
i := slices.IndexFunc(tlsModes, func(m tlsMode) bool { return m.name == mode })
if i < 0 || !slices.Contains(key.Enum, mode) {
return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(key.Enum, ", "))
}
m := tlsModes[i]
for _, other := range tlsModes {
for k := range other.needs {
if _, needed := m.needs[k]; needed && v[k] == "" {
@@ -392,13 +455,25 @@ func splitAddr(key, addr string) (string, int64, error) {
return host, int64(port), nil
}
// chainType is a transport's chain plugin type, both a source and a sink
func chainType(transport string) (string, error) {
if transport != "tcp" && transport != "http" {
return "", fmt.Errorf("transport %q (valid: tcp, http)", transport)
if !slices.Contains(transports(), transport) {
return "", fmt.Errorf("transport %q (valid: %s)", transport, strings.Join(transports(), ", "))
}
return transport + "_chain", nil
}
// transports name the chain plugins: the catalogue's TRANSPORT_chain types
func transports() []string {
var out []string
for _, p := range plugins {
if t, ok := strings.CutSuffix(p.Type, "_chain"); ok && p.Role == "sink" {
out = append(out, t)
}
}
return out
}
func list(s string) []any {
var out []any
for e := range strings.SplitSeq(s, ",") {
+52 -5
View File
@@ -3,6 +3,7 @@ package config
import (
"errors"
"reflect"
"slices"
"strings"
"testing"
)
@@ -56,7 +57,7 @@ func TestPresetKeysAreChecked(t *testing.T) {
"edge:to=agg:9000": "it never sends unauthenticated",
"edge:to=agg:9000,password_file=p": "password_file needs user",
"edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
"aggregator:users=u,tls=off": "the aggregator always serves TLS",
"aggregator:users=u,tls=off": `tls "off" (valid: `,
"aggregator:user=u,users=f": "user or users, not both",
"aggregator": "it never receives unauthenticated",
} {
@@ -67,6 +68,49 @@ func TestPresetKeysAreChecked(t *testing.T) {
}
}
// A preset key that sets an option takes its choices and hint, a switch true
// or false; a listener's tls offers off only where it may serve plain text
func TestPresetKeysTakeTheirOptionsChoices(t *testing.T) {
param := func(preset, key string) PresetParam {
ps := Presets()
p := ps[slices.IndexFunc(ps, func(p Preset) bool { return p.Name == preset })]
return p.Params[slices.IndexFunc(p.Params, func(k PresetParam) bool { return k.Name == key })]
}
for _, c := range []struct {
preset, key string
table reflect.Type
option string
}{
{"tail", "from", reflect.TypeFor[FileSourceOptions](), "from"},
{"pipe", "format", reflect.TypeFor[FormatConfig](), "type"},
{"serve", "proxy", reflect.TypeFor[AuthOptions](), "trusted_proxies"},
{"aggregator", "users", reflect.TypeFor[AuthOptions](), "credentials_file"},
} {
opts := options(c.table)
o := opts[slices.IndexFunc(opts, func(o option) bool { return o.Name == c.option })]
if p := param(c.preset, c.key); !slices.Equal(p.Enum, o.Enum) || p.Hint != o.Hint || len(o.Enum) == 0 && o.Hint == "" {
t.Errorf("%s %s: enum %v hint %q, want %s's %v %q", c.preset, c.key, p.Enum, p.Hint, c.option, o.Enum, o.Hint)
}
}
var modes []string
for _, m := range tlsModes {
modes = append(modes, m.name)
}
for _, c := range []struct {
preset, key string
enum []string
}{
{"serve", "viewer", []string{"true", "false"}},
{"edge", "transport", transports()},
{"serve", "tls", modes},
{"aggregator", "tls", slices.DeleteFunc(slices.Clone(modes), func(m string) bool { return m == "off" })},
} {
if p := param(c.preset, c.key); !slices.Equal(p.Enum, c.enum) {
t.Errorf("%s %s: enum %v, want %v", c.preset, c.key, p.Enum, c.enum)
}
}
}
// A preset user without password_file is asked for on the terminal
func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
p, err := ExpandPreset("edge", map[string]string{"to": "agg:9000", "user": "u"}, nil)
@@ -78,16 +122,19 @@ func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
}
}
// A preset path is what isDir answers, a directory named with brackets too;
// without isDir, off the target host, it is a directory when it ends in '/',
// a pattern when it holds a glob, and otherwise ErrPathKind asks
// A preset path is what isDir answers, a directory named with brackets too,
// and off the target host the user's answer, a single file too; without one
// it is a directory when it ends in '/', a pattern when it holds a glob, and
// otherwise ErrPathKind asks
func TestPresetPathKind(t *testing.T) {
yes := func(string) (bool, error) { return true, nil }
file := Answers(map[string]bool{"/srv/app.log": false})
for _, c := range []struct {
path string
isDir IsDir
want string
}{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""}} {
}{{"/srv/app/", nil, "/srv/app/ *"}, {"/srv/app/*.log", nil, "/srv/app *.log"}, {"/srv/app[1]", yes, "/srv/app[1] *"}, {"/srv/app", nil, ""},
{"/srv/app.log", file, "/srv app.log"}, {"/srv/app", file, ""}, {"/srv/app/", file, "/srv/app/ *"}} {
p, err := ExpandPreset("tail", map[string]string{"path": c.path}, c.isDir)
if c.want == "" {
if !errors.Is(err, ErrPathKind) {
+38 -13
View File
@@ -3,6 +3,7 @@ package config
import (
"errors"
"fmt"
"maps"
"math"
"reflect"
"slices"
@@ -57,8 +58,25 @@ var plugins = []Plugin{
// Plugins lists the catalogue
func Plugins() []Plugin { return slices.Clone(plugins) }
// Keys describes the plugin's options
func (p Plugin) Keys() []Key { return keys(p.options) }
// Keys describes the plugin's options, its tables holding the keys of its
// side, and HTTP's keys only when it speaks HTTP
func (p Plugin) Keys() []Key { return p.applying(keys(p.options)) }
func (p Plugin) applying(all []Key) []Key {
side, out := "listener", []Key{}
if p.Side == Dialer {
side = "dialer"
}
for _, k := range all {
if (k.Side == "" || k.Side == side) && (!k.HTTP || p.HTTP) {
if k.Keys != nil {
k.Keys = p.applying(k.Keys)
}
out = append(out, k)
}
}
return out
}
// LookupPlugin finds the catalogue row of a source or sink type
func LookupPlugin(role, typ string) (Plugin, bool) {
@@ -82,7 +100,8 @@ func (e *KeyError) Unwrap() error { return e.Err }
// option is one key of an options table, from its field's tags: toml names
// it, default: holds the default as typed on a command line, help: says what
// it does, and lw: lists its rules: required, enum=a|b, min=N, max=N,
// zero=MEANING (zero is a value, not the default), hint=KIND, listener or dialer.
// zero=MEANING (zero is a value, not the default), hint=KIND, listener or
// dialer, and http (only plugins that speak HTTP take it).
type option struct {
Key
index int
@@ -107,6 +126,8 @@ func options(t reflect.Type) []option {
o.Required = true
case "listener", "dialer":
o.Side = name
case "http":
o.HTTP = true
case "enum":
o.Enum = strings.Split(value, "|")
case "min", "max":
@@ -133,7 +154,7 @@ func options(t reflect.Type) []option {
}
// Key describes one option, for lw --schema and the composer: its tags, and
// its kind, a table's being "table" (tls, auth or acl)
// its kind, a table's being "table" (tls, auth or acl) with the table's keys
type Key struct {
Name string `json:"key"`
Kind string `json:"kind"`
@@ -146,17 +167,21 @@ type Key struct {
Zero string `json:"zero,omitempty"`
Hint string `json:"hint,omitempty"`
Side string `json:"side,omitempty"`
HTTP bool `json:"http,omitempty"`
Keys []Key `json:"keys,omitempty"`
}
var kinds = map[reflect.Kind]string{reflect.String: "string", reflect.Int64: "integer",
reflect.Float64: "number", reflect.Bool: "bool", reflect.Slice: "list", reflect.Pointer: "table"}
// Stage is a flow stage: its key in the flow table, whether it is a list (the
// filters), and its options
// filters), its options, and the values that turn it on, which a file's
// defaults leave off
type Stage struct {
Key string `json:"key"`
List bool `json:"list,omitempty"`
Keys []Key `json:"keys"`
Key string `json:"key"`
List bool `json:"list,omitempty"`
Keys []Key `json:"keys"`
On map[string]any `json:"on,omitempty"`
}
// FlowStages lists the flow table's stages, in the order entries pass them
@@ -165,20 +190,20 @@ func FlowStages() []Stage {
t := reflect.TypeFor[FlowConfig]()
for _, o := range options(t) {
ft := t.Field(o.index).Type
out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem())})
out = append(out, Stage{o.Name, ft.Kind() == reflect.Slice, keys(ft.Elem()), maps.Clone(stageOn[o.Name])})
}
return out
}
// KeysOf describes the options of a table: tls, auth or acl
func KeysOf[T any]() []Key { return keys(reflect.TypeFor[T]()) }
func keys(t reflect.Type) []Key {
var out []Key
out := []Key{}
for _, o := range options(t) {
if o.Kind == "" {
panic(fmt.Sprintf("%s.%s: no kind", t.Name(), o.Name))
}
if ft := t.Field(o.index).Type; ft.Kind() == reflect.Pointer {
o.Keys = keys(ft.Elem())
}
out = append(out, o.Key)
}
return out
+58 -3
View File
@@ -3,6 +3,7 @@ package config
import (
"math"
"reflect"
"slices"
"strings"
"testing"
)
@@ -79,7 +80,8 @@ func TestCoerceGivesTheDeclaredKinds(t *testing.T) {
}
// ValidateConfig decodes every plugin and flow stage without opening a file,
// and names the path of what it refuses
// and names the path of what it refuses: a rule between a table's keys by
// the table's
func TestValidateConfigNamesThePath(t *testing.T) {
valid := func() *Config {
return &Config{Color: "auto", Pipelines: []PipelineConfig{{
@@ -97,9 +99,13 @@ func TestValidateConfigNamesThePath(t *testing.T) {
"pipelines[0].plugin_sinks[out].config.port: must be from 1 to 65535, got 70000": func(c *Config) {
c.Pipelines[0].PluginSinks[0].Config["port"] = int64(70000)
},
"pipelines[0].plugin_sinks[out].config: tls: client_auth requires client_ca_file": func(c *Config) {
"pipelines[0].plugin_sinks[out].config.tls: client_auth requires client_ca_file": func(c *Config) {
c.Pipelines[0].PluginSinks[0].Config["tls"].(map[string]any)["client_auth"] = true
},
"pipelines[0].plugin_sinks[out].config.acl: requests_per_second_per_client applies only to HTTP listeners": func(c *Config) {
c.Pipelines[0].PluginSinks[0].Type = "tcp"
c.Pipelines[0].PluginSinks[0].Config["acl"] = map[string]any{"requests_per_second_per_client": 2.5}
},
"pipelines[0].plugin_sinks[out]: duplicate id": func(c *Config) {
c.Pipelines[0].PluginSinks = append(c.Pipelines[0].PluginSinks, PluginSinkConfig{ID: "out", Type: "null"})
},
@@ -109,7 +115,7 @@ func TestValidateConfigNamesThePath(t *testing.T) {
"pipelines[0].plugin_sinks[out].config.port: not an integer": func(c *Config) {
c.Pipelines[0].PluginSinks[0].Config["port"] = "http"
},
`pipelines[0].plugin_sinks[up].config: auth: type "mtls" cannot be used with tls.insecure_skip_verify`: func(c *Config) {
`pipelines[0].plugin_sinks[up].config.auth: type "mtls" cannot be used with tls.insecure_skip_verify`: func(c *Config) {
c.Pipelines[0].PluginSinks[0] = PluginSinkConfig{ID: "up", Type: "tcp_chain", Config: map[string]any{"host": "h", "port": int64(1),
"tls": map[string]any{"enabled": true, "insecure_skip_verify": true}, "auth": map[string]any{"type": "mtls"}}}
},
@@ -133,3 +139,52 @@ func TestValidateConfigNamesThePath(t *testing.T) {
}
}
}
// A configuration that does not load names every problem, a line each, each
// part's first: the settings, each flow stage, each plugin, each pipeline
func TestLoadReportsEveryProblemALine(t *testing.T) {
isolateConfig(t)
_, err := Load(Args{Overrides: []string{"--color=bogus"}, Specs: specs("pipeline", "a", "source", "file",
"sink", "tcp:port=70000", "rate-limit", "policy=bogus", "filter", "include:patterns=(",
"pipeline", "a", "sink", "null")})
want := strings.Join([]string{"invalid configuration:",
`color: must be one of auto, always, never, got "bogus"`,
`pipelines[0].flow.rate_limit.policy: must be one of pass, drop, got "bogus"`,
"pipelines[0].flow.filters[0].patterns[0]: \"(\": error parsing regexp: missing closing ): `(`",
"pipelines[0].plugin_sources[file].config.directory: must be set",
"pipelines[0].plugin_sinks[tcp].config.port: must be from 1 to 65535, got 70000",
`pipelines[1].name: duplicate name "a"`,
}, "\n")
if err == nil || err.Error() != want {
t.Fatalf("got\n%v\nwant\n%s", err, want)
}
}
// A plugin lists the table keys that apply to it: those of its side, a chain
// source's being a listener's, and the HTTP ones only when it speaks HTTP
func TestTableKeysFollowThePlugin(t *testing.T) {
has := func(role, typ, path string) bool {
p, _ := LookupPlugin(role, typ)
keys := p.Keys()
for part := range strings.SplitSeq(path, ".") {
i := slices.IndexFunc(keys, func(k Key) bool { return k.Name == part })
if i < 0 {
return false
}
keys = keys[i].Keys
}
return true
}
for path, want := range map[string]bool{
"sink tcp_chain tls.pin_sha256": true, "sink tcp_chain tls.self_signed": false,
"source tcp_chain tls.self_signed": true, "source tcp_chain tls.pin_sha256": false,
"source tcp_chain auth.node_binding": true, "sink tcp_chain auth.credentials_file": false,
"sink tcp acl.requests_per_second_per_client": false, "sink tcp auth.token_lifetime_ms": false,
"sink http acl.requests_per_second_per_client": true, "sink http auth.trusted_proxies": true,
} {
f := strings.Fields(path)
if has(f[0], f[1], f[2]) != want {
t.Errorf("%s: listed %v, want %v", path, !want, want)
}
}
}
+19 -7
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"maps"
"os"
"reflect"
"slices"
"strconv"
@@ -183,12 +182,15 @@ func cliSpecs(in []Spec) ([]pipelineSpec, error) {
return specs, nil
}
// envPipelineSpecs reads the one-pipeline environment form. An empty variable
// counts as unset, so container templates may leave optional ones blank.
func envPipelineSpecs() []pipelineSpec {
// envPipelineSpecs reads the one-pipeline environment form from environ,
// NAME=value each. An empty variable counts as unset, so container templates
// may leave optional ones blank.
func envPipelineSpecs(environ []string) []pipelineSpec {
var specs []pipelineSpec
if name := os.Getenv(specVariable("pipeline")); name != "" {
specs = append(specs, pipelineSpec{specVariable("pipeline"), nil, name})
for _, kv := range environ {
if name, value, _ := strings.Cut(kv, "="); name == specVariable("pipeline") && value != "" {
specs = append(specs, pipelineSpec{name, nil, value})
}
}
for i := range specKinds {
kind := &specKinds[i]
@@ -198,7 +200,7 @@ func envPipelineSpecs() []pipelineSpec {
name, spec string
}
var found []numbered
for _, kv := range os.Environ() {
for _, kv := range environ {
name, spec, _ := strings.Cut(kv, "=")
suffix, ok := strings.CutPrefix(name, base+"_")
n, err := strconv.ParseUint(suffix, 10, 64)
@@ -216,6 +218,16 @@ func envPipelineSpecs() []pipelineSpec {
return specs
}
// EnvPipelines builds the pipeline of LOGWISP_ variables, NAME=value each, as
// Load does without pipeline flags
func EnvPipelines(environ []string, isDir IsDir) ([]PipelineConfig, error) {
specs := envPipelineSpecs(environ)
if len(specs) == 0 {
return nil, errors.New("no LOGWISP_ pipeline variables")
}
return buildPipelines(specs, isDir)
}
// SpecPipelines builds the pipelines of command-line specs, as Load does
func SpecPipelines(specs []Spec, isDir IsDir) ([]PipelineConfig, error) {
s, err := cliSpecs(specs)
+107 -79
View File
@@ -24,26 +24,20 @@ import (
// ValidateConfig checks a configuration without reading a file it names:
// the settings, then each pipeline's flow stages, and its sources and sinks
// decoded in full, naming the path of a refused key. lw --check opens them.
// decoded in full. Its error joins every problem, one a part, each a
// *KeyError naming the path of the refused key. lw --check opens the files.
func ValidateConfig(cfg *Config) error {
if cfg == nil {
return fmt.Errorf("config is nil")
}
if err := checkTags(reflect.ValueOf(cfg).Elem(), ""); err != nil {
return err
}
errs := []error{checkTags(reflect.ValueOf(cfg).Elem(), "")}
if l := cfg.Logging; l != nil {
if err := checkTags(reflect.ValueOf(l).Elem(), "logging."); err != nil {
return err
}
errs = append(errs, checkTags(reflect.ValueOf(l).Elem(), "logging."))
if l.Console != nil {
if err := checkTags(reflect.ValueOf(l.Console).Elem(), "logging.console."); err != nil {
return err
}
errs = append(errs, checkTags(reflect.ValueOf(l.Console).Elem(), "logging.console."))
}
}
return ValidatePipelines(cfg.Pipelines)
return errors.Join(append(errs, ValidatePipelines(cfg.Pipelines))...)
}
// ValidatePipelines is ValidateConfig's check of the pipelines
@@ -51,97 +45,101 @@ func ValidatePipelines(pipelines []PipelineConfig) error {
if len(pipelines) == 0 {
return fmt.Errorf("no pipelines configured")
}
// Reject duplicate pipeline names (service map is keyed by name)
names := make(map[string]struct{}, len(pipelines))
for i, p := range pipelines {
if _, dup := names[p.Name]; dup {
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
}
names[p.Name] = struct{}{}
}
var errs []error
at := func(path string, err error) { errs = append(errs, At(path, err)) }
names := map[string]bool{} // the service is keyed by name
single := map[string]string{} // role and type of a single-instance plugin: its pipeline
for i, p := range pipelines {
if err := lconfig.NonEmpty(p.Name); err != nil {
return fmt.Errorf("pipeline[%d].name: %w", i, err)
path := fmt.Sprintf("pipelines[%d]", i)
switch {
case strings.TrimSpace(p.Name) == "":
at(path+".name", errors.New("must be set"))
case names[p.Name]:
at(path+".name", fmt.Errorf("duplicate name %q", p.Name))
}
names[p.Name] = true
if len(p.PluginSources) == 0 {
return fmt.Errorf("pipeline[%d]: no sources defined", i)
at(path+".plugin_sources", errors.New("no sources defined"))
}
if len(p.PluginSinks) == 0 {
return fmt.Errorf("pipeline[%d]: no sinks defined", i)
at(path+".plugin_sinks", errors.New("no sinks defined"))
}
if err := checkFlow(p.Flow, fmt.Sprintf("pipelines[%d].flow", i)); err != nil {
return err
if err := checkFlow(p.Flow, path+".flow"); err != nil {
errs = append(errs, err)
}
ids := map[string]bool{}
check := func(role, list, id, typ string, m map[string]any) error {
path := fmt.Sprintf("pipelines[%d].%s[%s]", i, list, id)
check := func(role, list, id, typ string, m map[string]any) {
path := fmt.Sprintf("%s.%s[%s]", path, list, id)
plugin, ok := LookupPlugin(role, typ)
switch {
case ids[list+id]:
return fmt.Errorf("%s: duplicate id", path)
at(path, errors.New("duplicate id"))
case !ok:
return fmt.Errorf("%s.type: unknown %s type %q", path, role, typ)
at(path+".type", fmt.Errorf("unknown %s type %q", role, typ))
case plugin.Single && single[role+typ] != "":
return fmt.Errorf("pipeline %q: one %s %s may run, and pipeline %q has it", p.Name, typ, role, single[role+typ])
at(path, fmt.Errorf("one %s %s may run, and pipeline %q has it", typ, role, single[role+typ]))
default:
ids[list+id] = true
if plugin.Single {
single[role+typ] = p.Name
}
if _, err := decode(role, typ, m); err != nil {
at(path+".config", err)
}
}
ids[list+id] = true
if plugin.Single {
single[role+typ] = p.Name
}
if _, err := decode(role, typ, m); err != nil {
return At(path+".config", err)
}
return nil
}
for _, s := range p.PluginSources {
if err := check("source", "plugin_sources", s.ID, s.Type, s.Config); err != nil {
return err
}
check("source", "plugin_sources", s.ID, s.Type, s.Config)
}
for _, s := range p.PluginSinks {
if err := check("sink", "plugin_sinks", s.ID, s.Type, s.Config); err != nil {
return err
}
check("sink", "plugin_sinks", s.ID, s.Type, s.Config)
}
}
return nil
return errors.Join(errs...)
}
// checkFlow settles copies of a pipeline's flow stages, which keep their
// defaults for the dump
// defaults for the dump: a problem a stage
func checkFlow(f *FlowConfig, path string) error {
if f == nil {
return nil
}
stages := map[string]any{}
if f.Heartbeat != nil {
stages["heartbeat"] = new(*f.Heartbeat)
}
if f.RateLimit != nil {
stages["rate_limit"] = new(*f.RateLimit)
}
if f.Format != nil {
stages["format"] = new(*f.Format)
}
for i, filter := range f.Filters {
stages[fmt.Sprintf("filters[%d]", i)] = &filter
}
for _, key := range slices.Sorted(maps.Keys(stages)) {
if err := Settle(stages[key]); err != nil {
return At(path+"."+key, err)
var errs []error
settle := func(key string, stage any) {
if err := Settle(stage); err != nil {
errs = append(errs, At(path+"."+key, err))
}
}
return nil
if f.RateLimit != nil {
settle("rate_limit", new(*f.RateLimit))
}
for i, filter := range f.Filters {
settle(fmt.Sprintf("filters[%d]", i), &filter)
}
if f.Format != nil {
settle("format", new(*f.Format))
}
if f.Heartbeat != nil {
settle("heartbeat", new(*f.Heartbeat))
}
return errors.Join(errs...)
}
// At prefixes err with the path of the table it was found in
// At addresses err to the path of the table it was found in
func At(path string, err error) error {
if ke, ok := errors.AsType[*KeyError](err); ok {
return fmt.Errorf("%s.%s: %w", path, ke.Key, ke.Err)
return &KeyError{path + "." + ke.Key, ke.Err}
}
return fmt.Errorf("%s: %w", path, err)
return &KeyError{path, err}
}
// table addresses a rule between a table's keys, which its Check names by
// a "tls: " prefix, to the table
func table(key string, err error) error {
if _, ok := errors.AsType[*KeyError](err); err == nil || ok {
return err
}
return &KeyError{key, errors.New(strings.TrimPrefix(err.Error(), key+": "))}
}
// networked is the options of a plugin that listens or dials
@@ -175,12 +173,12 @@ func (p Plugin) checkNetwork(n networked) error {
return &KeyError{"host", err}
}
if err := t.Check(p.Side != Dialer); err != nil {
return err
return table("tls", err)
}
if err := acl.Check(host, p.HTTP, a != nil && len(a.TrustedProxies) > 0); err != nil {
return err
return table("acl", err)
}
return a.Check(p.Side, p.HTTP, t.State())
return table("auth", a.Check(p.Side, p.HTTP, t.State()))
}
// Check holds the rules between tls keys on a listener or a dialer; a table
@@ -311,9 +309,9 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
return errors.New("auth: username and password_file go together")
case o.CredentialsFile == "" && o.Username == "":
return fmt.Errorf("auth: type %q requires credentials_file, or username and password_file", "scram")
case o.TokenLifetimeMS > 0 && !http:
return errors.New("auth: token_lifetime_ms applies only to HTTP listeners")
case len(o.TrustedProxies) > 0 && (side != Listener || !http):
case !http && httpKey(o) != "":
return fmt.Errorf("auth: %s applies only to HTTP listeners", httpKey(o))
case len(o.TrustedProxies) > 0 && side != Listener:
return errors.New("auth: trusted_proxies applies only to the http sink")
case len(o.TrustedProxies) > 0 && o.Identity != "":
return errors.New("auth: identity binds a client certificate, which a TLS-terminating proxy does not pass on; drop it or trusted_proxies")
@@ -367,14 +365,13 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error {
return err
}
header := o.ProxyProtocol == "optional" || o.ProxyProtocol == "required"
rps := o.RequestsPerSecondPerClient
switch {
case header && len(o.ProxyFrom) == 0:
return fmt.Errorf("acl: proxy_protocol %s needs proxy_from, the proxies that may send the header", o.ProxyProtocol)
case !header && len(o.ProxyFrom) > 0:
return errors.New("acl: proxy_from needs proxy_protocol optional or required")
case rps > 0 && !http:
return errors.New("acl: requests_per_second_per_client applies only to HTTP listeners")
case !http && httpKey(o) != "":
return fmt.Errorf("acl: %s applies only to HTTP listeners", httpKey(o))
case o.MaxConnectionsPerClient > 0 && proxied:
return errors.New("acl: max_connections_per_client counts connections, behind trusted_proxies the proxy's; cap clients at the proxy")
}
@@ -399,6 +396,17 @@ func (o *ACLOptions) Check(host string, http, proxied bool) error {
return nil
}
// httpKey names the first key of table v tagged http that is set
func httpKey(v any) string {
rv := reflect.ValueOf(v).Elem()
for _, o := range options(rv.Type()) {
if o.HTTP && !rv.Field(o.index).IsZero() {
return o.Name
}
}
return ""
}
// ParsePrefix reads an address or CIDR of network's family ("tcp": either)
func ParsePrefix(entry, network string) (netip.Prefix, error) {
entry = strings.TrimSpace(entry)
@@ -575,13 +583,33 @@ func checkFileKeys(path string) error {
if err != nil {
return err
}
root, err := toml.NewParser(data).Parse()
root, err := fileTable(data)
if err != nil {
return err
}
return checkKeys(root, reflect.TypeOf(Config{}), "")
}
// ParseFile reads a configuration file's text, refusing the keys lw refuses
// there, and opens nothing it names
func ParseFile(data []byte) (*Config, error) {
root, err := fileTable(data)
if err != nil {
return nil, err
}
cfg := &Config{}
return cfg, Scan(root, cfg)
}
// fileTable is a file's root table without the keys lw ignores there
func fileTable(data []byte) (map[string]any, error) {
root, err := toml.NewParser(data).Parse()
if err != nil {
return nil, err
}
delete(root, "config_file") // runtime metadata, documented as ignored in the file
for _, key := range []string{"version", "check", "dump"} {
delete(root, key) // command-line switches, which earlier lw --dump output wrote
}
return checkKeys(root, reflect.TypeOf(Config{}), "")
return root, nil
}