v0.22.0 several enhancements and fixes for security, network, doc, test, packaging
This commit is contained in:
1 parent
9ab8c356d2
commit
71d1b82f83
102 files changed
+7102
-2178
No files matched your search
@@ -6,6 +6,7 @@ package config
|
||||
type Config struct {
|
||||
// Top-level flags for application control
|
||||
ShowVersion bool `toml:"version"`
|
||||
Check bool `toml:"check"` // build every plugin, report, exit: lw --check
|
||||
Quiet bool `toml:"quiet"`
|
||||
|
||||
// Runtime behavior flags
|
||||
@@ -347,7 +348,7 @@ type AuthOptions struct {
|
||||
// Overrides trust_node.
|
||||
NodeBinding string `toml:"node_binding"`
|
||||
|
||||
// scram listeners: verifier file written by `logwisp auth add-user`, and
|
||||
// scram listeners: verifier file written by `lw auth add-user`, and
|
||||
// the bearer token lifetime on HTTP listeners (default 15 minutes)
|
||||
CredentialsFile string `toml:"credentials_file"`
|
||||
TokenLifetimeMS int64 `toml:"token_lifetime_ms"`
|
||||
|
||||
@@ -18,8 +18,11 @@ var ErrConfigNotFound = lconfig.ErrConfigNotFound
|
||||
// Manager owns the configuration sources and watcher for one application instance.
|
||||
// Its snapshots are detached from each other and from the running service.
|
||||
type Manager struct {
|
||||
config *lconfig.Config
|
||||
path string
|
||||
config *lconfig.Config
|
||||
path string
|
||||
explicit bool
|
||||
read bool // the file was loaded once
|
||||
specs []pipelineSpec // command-line or environment pipelines, kept across reloads
|
||||
}
|
||||
|
||||
// Load reads the startup sources and validates the initial configuration.
|
||||
@@ -29,13 +32,30 @@ func Load(args []string) (*Manager, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
specs, configArgs, err := takePipelineSpecs(configArgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(specs) == 0 {
|
||||
specs = envPipelineSpecs()
|
||||
}
|
||||
if _, err := buildPipelines(specs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := &Manager{path: configPath, explicit: isExplicit, specs: specs}
|
||||
initial := defaults()
|
||||
cfg, err := lconfig.NewBuilder().
|
||||
WithTarget(initial).
|
||||
WithEnvPrefix("LOGWISP_").
|
||||
WithArgs(configArgs).
|
||||
WithFile(configPath).
|
||||
WithTypedValidator(ValidateConfig).
|
||||
WithTypedValidator(func(cfg *Config) error {
|
||||
effective := *cfg // the builder keeps cfg; only the copy is replaced
|
||||
if err := m.usePipelines(&effective); err != nil {
|
||||
return err
|
||||
}
|
||||
return ValidateConfig(&effective)
|
||||
}).
|
||||
WithSecurityOptions(lconfig.SecurityOptions{
|
||||
PreventPathTraversal: true,
|
||||
MaxFileSize: 10 * 1024 * 1024,
|
||||
@@ -50,10 +70,12 @@ func Load(args []string) (*Manager, error) {
|
||||
}
|
||||
// A missing discovered default still permits valid CLI/env/default values.
|
||||
}
|
||||
m.read = err == nil
|
||||
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !initial.Quiet {
|
||||
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
|
||||
}
|
||||
return &Manager{config: cfg, path: configPath}, nil
|
||||
m.config = cfg
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// Snapshot validates a detached candidate before it is used to build a service.
|
||||
@@ -65,6 +87,9 @@ func (m *Manager) Snapshot() (*Config, error) {
|
||||
}
|
||||
cfg := value.(*Config)
|
||||
cfg.ConfigFile = m.path
|
||||
if err := m.usePipelines(cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := checkFileKeys(m.path); err != nil {
|
||||
return nil, fmt.Errorf("config file %q: %w", m.path, err)
|
||||
}
|
||||
@@ -74,12 +99,27 @@ func (m *Manager) Snapshot() (*Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// usePipelines replaces the file's or default pipelines with the spec ones.
|
||||
func (m *Manager) usePipelines(cfg *Config) error {
|
||||
if len(m.specs) == 0 {
|
||||
return nil
|
||||
}
|
||||
pipelines, err := buildPipelines(m.specs)
|
||||
cfg.Pipelines = pipelines
|
||||
return err
|
||||
}
|
||||
|
||||
// Reload rereads the selected file, retaining the startup CLI/environment sources.
|
||||
// Signals must call this even when automatic watching is disabled.
|
||||
func (m *Manager) Reload() (*Config, error) {
|
||||
if err := m.config.LoadFile(m.path); err != nil {
|
||||
// As at startup, a discovered default that does not exist is no error, so a
|
||||
// file-less instance still rebuilds and rotates its certificates. Once read,
|
||||
// its removal is one: the loader would keep the removed file's values.
|
||||
err := m.config.LoadFile(m.path)
|
||||
if err != nil && (m.explicit || m.read || !errors.Is(err, ErrConfigNotFound)) {
|
||||
return nil, fmt.Errorf("reload %q: %w", m.path, err)
|
||||
}
|
||||
m.read = m.read || err == nil
|
||||
return m.Snapshot()
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -124,6 +125,13 @@ func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
|
||||
if err != nil || !recovered.StatusReporter {
|
||||
t.Fatalf("override removal/recovery: %+v %v", recovered, err)
|
||||
}
|
||||
// Only a missing discovered default is tolerated; a named file must exist.
|
||||
if err := os.Remove("reload.toml"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.Reload(); !errors.Is(err, ErrConfigNotFound) {
|
||||
t.Fatalf("reload without the named file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// specKind is one pipeline spec flag. Its environment form is LOGWISP_ plus
|
||||
// the flag uppercased, '-' as '_'.
|
||||
type specKind struct {
|
||||
flag string
|
||||
typed bool // SPEC starts with TYPE
|
||||
many bool // repeatable within a pipeline; the environment adds _1.._N
|
||||
apply func(p *PipelineConfig, typ string, opts map[string]any) error
|
||||
}
|
||||
|
||||
var specKinds = []specKind{
|
||||
{"source", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
|
||||
ids := make([]string, len(p.PluginSources))
|
||||
for i, s := range p.PluginSources {
|
||||
ids[i] = s.ID
|
||||
}
|
||||
id, err := instanceID(typ, opts, ids)
|
||||
p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
|
||||
return err
|
||||
}},
|
||||
{"sink", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
|
||||
ids := make([]string, len(p.PluginSinks))
|
||||
for i, s := range p.PluginSinks {
|
||||
ids[i] = s.ID
|
||||
}
|
||||
id, err := instanceID(typ, opts, ids)
|
||||
p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
|
||||
return err
|
||||
}},
|
||||
{"filter", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
|
||||
opts["type"] = typ
|
||||
f := FilterConfig{}
|
||||
err := Scan(opts, &f)
|
||||
p.Flow.Filters = append(p.Flow.Filters, f)
|
||||
return err
|
||||
}},
|
||||
{"format", true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
|
||||
opts["type"] = typ
|
||||
p.Flow.Format = &FormatConfig{}
|
||||
return Scan(opts, p.Flow.Format)
|
||||
}},
|
||||
// Naming a stage turns it on: the file defaults (pass, disabled) would not.
|
||||
{"rate-limit", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
|
||||
p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
|
||||
return Scan(opts, p.Flow.RateLimit)
|
||||
}},
|
||||
{"heartbeat", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
|
||||
p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
|
||||
return Scan(opts, p.Flow.Heartbeat)
|
||||
}},
|
||||
}
|
||||
|
||||
// pipelineSpec is one pipeline flag or variable; a nil kind starts a pipeline.
|
||||
type pipelineSpec struct {
|
||||
name string // "--sink" or "LOGWISP_SINK_2", for errors
|
||||
kind *specKind
|
||||
value string
|
||||
}
|
||||
|
||||
// takePipelineSpecs consumes the pipeline flags, which lixenwraith/config
|
||||
// would report as unrecognized.
|
||||
func takePipelineSpecs(args []string) (specs []pipelineSpec, rest []string, err error) {
|
||||
for i := 0; i < len(args); i++ {
|
||||
if args[i] == "--" {
|
||||
return specs, append(rest, args[i:]...), nil
|
||||
}
|
||||
name, value, inline := strings.Cut(args[i], "=")
|
||||
var kind *specKind
|
||||
if name != "--pipeline" {
|
||||
k := slices.IndexFunc(specKinds, func(k specKind) bool { return "--"+k.flag == name })
|
||||
if k < 0 {
|
||||
rest = append(rest, args[i])
|
||||
continue
|
||||
}
|
||||
kind = &specKinds[k]
|
||||
}
|
||||
if !inline && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") {
|
||||
i++
|
||||
value = args[i]
|
||||
}
|
||||
if value == "" {
|
||||
return nil, nil, fmt.Errorf("%s requires a value", name)
|
||||
}
|
||||
specs = append(specs, pipelineSpec{name, kind, value})
|
||||
}
|
||||
return specs, rest, nil
|
||||
}
|
||||
|
||||
// envPipelineSpecs reads the one-pipeline environment form. An empty variable
|
||||
// counts as unset, so container templates may leave optional ones blank.
|
||||
func envPipelineSpecs() []pipelineSpec {
|
||||
var specs []pipelineSpec
|
||||
if name := os.Getenv("LOGWISP_PIPELINE"); name != "" {
|
||||
specs = append(specs, pipelineSpec{"LOGWISP_PIPELINE", nil, name})
|
||||
}
|
||||
for i := range specKinds {
|
||||
kind := &specKinds[i]
|
||||
base := "LOGWISP_" + strings.ToUpper(strings.ReplaceAll(kind.flag, "-", "_"))
|
||||
type numbered struct {
|
||||
n uint64
|
||||
name, spec string
|
||||
}
|
||||
var found []numbered
|
||||
for _, kv := range os.Environ() {
|
||||
name, spec, _ := strings.Cut(kv, "=")
|
||||
suffix, ok := strings.CutPrefix(name, base+"_")
|
||||
n, err := strconv.ParseUint(suffix, 10, 64)
|
||||
if name == base || ok && kind.many && err == nil {
|
||||
found = append(found, numbered{n, name, spec})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(found, func(a, b numbered) int { return cmp.Or(cmp.Compare(a.n, b.n), cmp.Compare(a.name, b.name)) })
|
||||
for _, f := range found {
|
||||
if f.spec != "" {
|
||||
specs = append(specs, pipelineSpec{f.name, kind, f.spec})
|
||||
}
|
||||
}
|
||||
}
|
||||
return specs
|
||||
}
|
||||
|
||||
// buildPipelines turns specs, in order, into pipelines; specs before the first
|
||||
// --pipeline go to one named "cli". Every call returns fresh maps.
|
||||
func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
|
||||
var pipelines []PipelineConfig
|
||||
var seen map[*specKind]bool
|
||||
for _, s := range specs {
|
||||
if s.kind == nil || pipelines == nil {
|
||||
name := "cli"
|
||||
if s.kind == nil {
|
||||
name = s.value
|
||||
}
|
||||
pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
|
||||
seen = map[*specKind]bool{}
|
||||
if s.kind == nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
p := &pipelines[len(pipelines)-1]
|
||||
if seen[s.kind] && !s.kind.many {
|
||||
return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
|
||||
}
|
||||
seen[s.kind] = true
|
||||
typ, opts, err := parseSpec(s.value, s.kind.typed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s %w", s.name, err)
|
||||
}
|
||||
if err := s.kind.apply(p, typ, opts); err != nil {
|
||||
return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
|
||||
}
|
||||
}
|
||||
return pipelines, nil
|
||||
}
|
||||
|
||||
// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
|
||||
// a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
|
||||
func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {
|
||||
rest, more := spec, true
|
||||
if typed {
|
||||
typ, rest, more = cutUnescaped(spec, ',')
|
||||
if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq {
|
||||
return "", nil, fmt.Errorf("%s: missing TYPE", spec)
|
||||
}
|
||||
typ = unescape(typ)
|
||||
}
|
||||
opts = map[string]any{}
|
||||
for more {
|
||||
var part string
|
||||
part, rest, more = cutUnescaped(rest, ',')
|
||||
key, value, eq := cutUnescaped(part, '=')
|
||||
switch key = unescape(key); {
|
||||
case !eq:
|
||||
err = errors.New(`missing "="`)
|
||||
case typed && key == "type":
|
||||
err = errors.New(`TYPE already sets "type"`)
|
||||
default:
|
||||
err = setOption(opts, key, unescape(value))
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
|
||||
}
|
||||
}
|
||||
return typ, opts, nil
|
||||
}
|
||||
|
||||
// cutUnescaped cuts s at its first unescaped sep, leaving escapes in place.
|
||||
func cutUnescaped(s string, sep byte) (before, after string, found bool) {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if escapes(s, i) {
|
||||
i++
|
||||
} else if s[i] == sep {
|
||||
return s[:i], s[i+1:], true
|
||||
}
|
||||
}
|
||||
return s, "", false
|
||||
}
|
||||
|
||||
func unescape(s string) string {
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
if escapes(s, i) {
|
||||
i++
|
||||
}
|
||||
b.WriteByte(s[i])
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// escapes keeps any other backslash literal, so regex escapes such as \d pass.
|
||||
func escapes(s string, i int) bool {
|
||||
return s[i] == '\\' && i+1 < len(s) && strings.IndexByte(`,=\`, s[i+1]) >= 0
|
||||
}
|
||||
|
||||
// setOption stores value at a dotted key; a second value turns it into a list.
|
||||
func setOption(opts map[string]any, key, value string) error {
|
||||
segments := strings.Split(key, ".")
|
||||
if slices.Contains(segments, "") {
|
||||
return fmt.Errorf("empty key in %q", key)
|
||||
}
|
||||
table, last := opts, len(segments)-1
|
||||
for i, segment := range segments[:last] {
|
||||
switch existing := table[segment].(type) {
|
||||
case nil:
|
||||
next := map[string]any{}
|
||||
table[segment], table = next, next
|
||||
case map[string]any:
|
||||
table = existing
|
||||
default:
|
||||
return fmt.Errorf("%q is both a value and a table", strings.Join(segments[:i+1], "."))
|
||||
}
|
||||
}
|
||||
switch existing := table[segments[last]].(type) {
|
||||
case nil:
|
||||
table[segments[last]] = value
|
||||
case string:
|
||||
table[segments[last]] = []any{existing, value}
|
||||
case []any:
|
||||
table[segments[last]] = append(existing, value)
|
||||
default:
|
||||
return fmt.Errorf("%q is both a table and a value", key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// instanceID takes the id option; an unnamed instance gets the first of TYPE,
|
||||
// TYPE_2, TYPE_3... that no earlier instance of its role holds.
|
||||
func instanceID(typ string, opts map[string]any, taken []string) (string, error) {
|
||||
id, _ := opts["id"].(string)
|
||||
if _, set := opts["id"]; set && id == "" {
|
||||
return "", errors.New(`"id" takes one non-empty value`)
|
||||
}
|
||||
delete(opts, "id")
|
||||
if id != "" {
|
||||
return id, nil
|
||||
}
|
||||
id = typ
|
||||
for n := 2; slices.Contains(taken, id); n++ {
|
||||
id = typ + "_" + strconv.Itoa(n)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"logwisp/internal/testutil"
|
||||
)
|
||||
|
||||
func loadPipelines(t *testing.T, args ...string) []PipelineConfig {
|
||||
t.Helper()
|
||||
m, err := Load(args)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
cfg, err := m.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cfg.Pipelines
|
||||
}
|
||||
|
||||
func TestPipelineSpecGrammar(t *testing.T) {
|
||||
isolateConfig(t)
|
||||
got := loadPipelines(t,
|
||||
"--source", `file,directory=/var/log/a\,b,pattern=*.log`,
|
||||
`--source=file,id=app,directory=x\=y`,
|
||||
"--source", `file,directory=c:\\logs\\`,
|
||||
"--sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c",
|
||||
"--filter", "include,patterns=ERROR,patterns=WARN",
|
||||
"--filter", `exclude,patterns=\d{1\,3}`,
|
||||
"--format", `txt,timestamp_format=Jan 2\, 2006`,
|
||||
"--pipeline", "b", "--source", "null", "--sink", "null", "--sink", "null", "--heartbeat", "interval_ms=1000",
|
||||
"--", "--source", "random")
|
||||
want := []PipelineConfig{{
|
||||
Name: "cli",
|
||||
Flow: &FlowConfig{
|
||||
Filters: []FilterConfig{
|
||||
{Type: "include", Patterns: []string{"ERROR", "WARN"}},
|
||||
{Type: "exclude", Patterns: []string{`\d{1,3}`}},
|
||||
},
|
||||
Format: &FormatConfig{Type: "txt", TimestampFormat: "Jan 2, 2006"},
|
||||
},
|
||||
PluginSources: []PluginSourceConfig{
|
||||
{ID: "file", Type: "file", Config: map[string]any{"directory": "/var/log/a,b", "pattern": "*.log"}},
|
||||
{ID: "app", Type: "file", Config: map[string]any{"directory": "x=y"}},
|
||||
{ID: "file_2", Type: "file", Config: map[string]any{"directory": `c:\logs\`}},
|
||||
},
|
||||
PluginSinks: []PluginSinkConfig{{ID: "http", Type: "http", Config: map[string]any{
|
||||
"port": "8080",
|
||||
"auth": map[string]any{"type": "scram", "credentials_file": "users.toml"},
|
||||
"tls": map[string]any{"cert_file": "c"},
|
||||
}}},
|
||||
}, {
|
||||
Name: "b",
|
||||
Flow: &FlowConfig{Heartbeat: &HeartbeatConfig{Enabled: true, IntervalMS: 1000}},
|
||||
PluginSources: []PluginSourceConfig{{ID: "null", Type: "null", Config: map[string]any{}}},
|
||||
PluginSinks: []PluginSinkConfig{
|
||||
{ID: "null", Type: "null", Config: map[string]any{}},
|
||||
{ID: "null_2", Type: "null", Config: map[string]any{}},
|
||||
},
|
||||
}}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
|
||||
}
|
||||
for want, args := range map[string][]string{
|
||||
`--sink http,port: missing "="`: {"--sink", "http,port"},
|
||||
`--source directory=/x: missing TYPE`: {"--source", "directory=/x"},
|
||||
`--source file,type=x: TYPE already sets "type"`: {"--source", "file,type=x"},
|
||||
`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"--sink", "http,tls=1,tls.cert_file=c"},
|
||||
`--sink requires a value`: {"--sink="},
|
||||
`--format txt: pipeline "cli" already has one`: {"--format", "json", "--format", "txt"},
|
||||
`--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"--rate-limit", "rate=1,polcy=drop"},
|
||||
} {
|
||||
if _, err := Load(args); err == nil || err.Error() != want {
|
||||
t.Errorf("%q: err = %v, want %s", args, err, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A spec value is one list entry, commas included: a regex quantifier or a DN
|
||||
// must not split into several, wider entries.
|
||||
func TestSpecValueIsOneListEntry(t *testing.T) {
|
||||
isolateConfig(t)
|
||||
sink := loadPipelines(t, "--source", "null", "--sink",
|
||||
`tcp,auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0]
|
||||
var opts TCPSinkOptions
|
||||
if err := Scan(sink.Config, &opts); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(opts.Auth.AllowPatterns, []string{`^edge-\d{1,3}$`}) ||
|
||||
!reflect.DeepEqual(opts.Auth.Allow, []string{"CN=a,O=b", "CN=c"}) {
|
||||
t.Fatalf("allow_patterns %q, allow %q", opts.Auth.AllowPatterns, opts.Auth.Allow)
|
||||
}
|
||||
}
|
||||
|
||||
// One pipeline from LOGWISP_*: numbered variables follow the bare one in
|
||||
// numeric order, only repeatable kinds are numbered, and empty means unset.
|
||||
func TestEnvironmentPipeline(t *testing.T) {
|
||||
isolateConfig(t)
|
||||
for name, value := range map[string]string{
|
||||
"LOGWISP_PIPELINE": "edge",
|
||||
"LOGWISP_SOURCE": "random",
|
||||
"LOGWISP_SOURCE_10": "null,id=ten",
|
||||
"LOGWISP_SOURCE_2": "null,id=two",
|
||||
"LOGWISP_SINK_1": "console",
|
||||
"LOGWISP_FILTER": "",
|
||||
"LOGWISP_FORMAT_1": "raw",
|
||||
"LOGWISP_RATE_LIMIT": "rate=10",
|
||||
"LOGWISP_LOGGING_LEVEL": "debug",
|
||||
} {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
m, err := Load(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
cfg, err := m.Snapshot()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := cfg.Pipelines
|
||||
if len(p) != 1 || p[0].Name != "edge" || len(p[0].PluginSources) != 3 || p[0].PluginSources[1].ID != "two" ||
|
||||
p[0].PluginSources[2].ID != "ten" || len(p[0].PluginSinks) != 1 || p[0].Flow.Filters != nil || p[0].Flow.Format != nil ||
|
||||
*p[0].Flow.RateLimit != (RateLimitConfig{Rate: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" {
|
||||
t.Fatalf("environment pipeline: %+v %+v", p, cfg.Logging)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) {
|
||||
isolateConfig(t)
|
||||
t.Setenv("LOGWISP_SOURCE", "random")
|
||||
t.Setenv("LOGWISP_SINK", "console")
|
||||
got := loadPipelines(t, "--source", "null", "--sink", "null")
|
||||
if len(got) != 1 || len(got[0].PluginSources) != 1 || got[0].PluginSources[0].Type != "null" ||
|
||||
len(got[0].PluginSinks) != 1 || got[0].PluginSinks[0].Type != "null" {
|
||||
t.Fatalf("environment specs mixed into command-line pipelines: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// The file's pipelines are dropped unvalidated; its other keys still apply.
|
||||
func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
|
||||
isolateConfig(t)
|
||||
testutil.WriteFile(t, "file.toml", "[logging]\nlevel = \"warn\"\n[[pipelines]]\nname = \"file\"\n")
|
||||
m, err := Load([]string{"-c", "file.toml", "--source", "null", "--sink", "null", "--status_reporter=false"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
if unknown := m.config.UnknownCLIKeys(); len(unknown) != 0 {
|
||||
t.Fatalf("spec flags reached the schema parser: %v", unknown)
|
||||
}
|
||||
cfg, err := m.Snapshot()
|
||||
if err != nil || len(cfg.Pipelines) != 1 || cfg.Pipelines[0].Name != "cli" || cfg.Logging.Level != "warn" || cfg.StatusReporter {
|
||||
t.Fatalf("snapshot: %+v %v", cfg, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Without a file a reload still rebuilds; once the discovered default
|
||||
// appears it is read, and its pipelines still yield to the specs. Its later
|
||||
// removal fails the reload instead of keeping the removed file's values.
|
||||
func TestReloadKeepsSpecPipelines(t *testing.T) {
|
||||
isolateConfig(t)
|
||||
m, err := Load([]string{"--source", "random,special=true", "--sink", "null"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer m.Close()
|
||||
first, err := m.Reload()
|
||||
if err != nil || !first.StatusReporter {
|
||||
t.Fatalf("reload without a file: %+v %v", first, err)
|
||||
}
|
||||
first.Pipelines[0].PluginSources[0].Config["special"] = "mutated"
|
||||
testutil.WriteFile(t, "logwisp.toml", "status_reporter = false\n[[pipelines]]\nname = \"file\"\n")
|
||||
next, err := m.Reload()
|
||||
if err != nil || next.StatusReporter || len(next.Pipelines) != 1 || next.Pipelines[0].Name != "cli" ||
|
||||
next.Pipelines[0].PluginSources[0].Config["special"] != "true" {
|
||||
t.Fatalf("reload: %+v %v", next, err)
|
||||
}
|
||||
if err := os.Remove("logwisp.toml"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.Reload(); !errors.Is(err, ErrConfigNotFound) {
|
||||
t.Fatalf("reload after the file was removed: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -94,15 +94,17 @@ func validateLogConfig(cfg *LogConfig) error {
|
||||
// not declare: a misspelled tls or auth key must fail, not silently disable
|
||||
// the protection it was meant to configure.
|
||||
func Scan(configMap map[string]any, target any) error {
|
||||
if err := unknownKeys(configMap, reflect.TypeOf(target), ""); err != nil {
|
||||
if err := checkKeys(configMap, reflect.TypeOf(target), ""); err != nil {
|
||||
return err
|
||||
}
|
||||
return lconfig.ScanMap(configMap, target)
|
||||
}
|
||||
|
||||
// unknownKeys walks nested tables against the toml tags of t. Map-typed fields
|
||||
// hold free-form keys and are not descended into.
|
||||
func unknownKeys(m map[string]any, t reflect.Type, prefix string) error {
|
||||
// checkKeys walks nested tables against the toml tags of t, refusing unknown
|
||||
// keys and making a lone string for a list field a one-entry list: the weakly
|
||||
// typed decoder would split it at commas, which a pattern or a DN may hold.
|
||||
// Map-typed fields hold free-form keys and are not descended into.
|
||||
func checkKeys(m map[string]any, t reflect.Type, prefix string) error {
|
||||
for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice {
|
||||
t = t.Elem()
|
||||
}
|
||||
@@ -122,8 +124,12 @@ func unknownKeys(m map[string]any, t reflect.Type, prefix string) error {
|
||||
}
|
||||
var tables []map[string]any
|
||||
switch v := m[key].(type) {
|
||||
case string:
|
||||
if ft.Kind() == reflect.Slice && ft.Elem().Kind() == reflect.String {
|
||||
m[key] = []string{v}
|
||||
}
|
||||
case map[string]any:
|
||||
if err := unknownKeys(v, ft, prefix+key+"."); err != nil {
|
||||
if err := checkKeys(v, ft, prefix+key+"."); err != nil {
|
||||
return err
|
||||
}
|
||||
case []map[string]any:
|
||||
@@ -136,7 +142,7 @@ func unknownKeys(m map[string]any, t reflect.Type, prefix string) error {
|
||||
}
|
||||
}
|
||||
for i, table := range tables {
|
||||
if err := unknownKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil {
|
||||
if err := checkKeys(table, ft, fmt.Sprintf("%s%s[%d].", prefix, key, i)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -160,5 +166,5 @@ func checkFileKeys(path string) error {
|
||||
return err
|
||||
}
|
||||
delete(root, "config_file") // runtime metadata, documented as ignored in the file
|
||||
return unknownKeys(root, reflect.TypeOf(Config{}), "")
|
||||
return checkKeys(root, reflect.TypeOf(Config{}), "")
|
||||
}
|
||||
Reference in new issue
Block a user