From bf410e12257777650f3fd17c4667cad4f02d21c52ac203bf4ae218e5cef44625 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sun, 4 Oct 2026 03:44:12 -0400 Subject: [PATCH] v0.23.0 color, presets, certificates made at startup, cli.go, sink flush --- AGENTS.md | 7 +- README.md | 15 +- cmd/lw/auth.go | 82 +----- cmd/lw/auth_test.go | 5 +- cmd/lw/bootstrap.go | 25 ++ cmd/lw/cli.go | 213 ++++++++++++++++ cmd/lw/cli_test.go | 139 +++++++++++ cmd/lw/help.go | 46 ++-- cmd/lw/main.go | 21 +- cmd/lw/test_helpers_test.go | 19 +- cmd/lw/tls.go | 147 +++++++++++ config/logwisp.toml | 18 +- doc/README.md | 3 +- doc/chaining.md | 4 +- doc/cli.md | 124 ++++++++- doc/configuration.md | 8 +- doc/lw.1 | 215 +++++++++++++++- doc/networking.md | 3 + doc/operations.md | 6 +- doc/security.md | 85 ++++++- doc/sinks.md | 25 +- doc/todo.md | 18 +- go.mod | 2 + go.sum | 4 + internal/authz/authz.go | 5 +- internal/authz/authz_test.go | 9 +- internal/config/config.go | 30 ++- internal/config/loader.go | 74 +++--- internal/config/loader_test.go | 79 +++--- internal/config/preset.go | 332 +++++++++++++++++++++++++ internal/config/preset_test.go | 63 +++++ internal/config/spec.go | 128 ++++++---- internal/config/spec_test.go | 64 ++--- internal/config/validate.go | 4 + internal/core/const.go | 3 + internal/sink/console/console.go | 98 +++++++- internal/sink/console/console_test.go | 26 +- internal/sink/http/http.go | 82 ++++-- internal/sink/http/http_test.go | 71 ++++++ internal/sink/sink.go | 24 ++ internal/sink/tcp/tcp.go | 113 +++++++-- internal/sink/tcp/tcp_test.go | 82 ++++++ internal/sink/tcpchain/tcpchain.go | 5 +- internal/source/httpchain/httpchain.go | 2 +- internal/source/tcpchain/tcpchain.go | 2 +- internal/testutil/fixtures.go | 35 +++ internal/tlsx/pki.go | 231 +++++++++++++++++ internal/tlsx/tlsx.go | 66 ++++- internal/tlsx/tlsx_test.go | 105 ++++++++ test/preset-test.sh | 108 ++++++++ 50 files changed, 2649 insertions(+), 426 deletions(-) create mode 100644 cmd/lw/cli.go create mode 100644 cmd/lw/cli_test.go create mode 100644 cmd/lw/tls.go create mode 100644 internal/config/preset.go create mode 100644 internal/config/preset_test.go create mode 100644 internal/tlsx/pki.go create mode 100755 test/preset-test.sh diff --git a/AGENTS.md b/AGENTS.md index fe0b9ca..ee21246 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,7 +76,12 @@ change touches. - Plugin options decode through `config.Scan`, never `lconfig.ScanMap`: unknown keys must fail at any depth. Set defaults on the options struct before the call; `TestEveryPluginRejectsUnknownKeys` covers every registered plugin. -- Network security has two seams: `tlsx` (TLS configs, startup warnings) and +- `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands + table whose groups (`auth`, `tls`, `preset`) share one flag mechanism; + `config.Load` takes the parsed `config.Args`. Presets are rows of one table + in `internal/config/preset.go`. +- Network security has two seams: `tlsx` (TLS configs, certificates made at + startup or by `lw tls`, pins, startup warnings) and `authz`. Listeners admit only through `Admit`/`AuthorizeRequest`, dialers through `Greet`/`Prepare`; `Authorize` refuses under scram, so a plugin gated on it alone fails closed. Nil policies are valid: keep call sites branch-free. diff --git a/README.md b/README.md index 900d8c9..7c1da55 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ streams, or downstream LogWisp nodes. - **A Unix filter out of the box**: with no configuration, `lw` copies stdin to stdout line for line and exits at the end of input; flags add filters, formats, sources and sinks +- **Presets**: `tail`, `serve`, `edge` and `aggregator` build the common + pipelines from a few keys (`lw --preset tail,path=/var/log/app`); + `lw --dump` turns any command line into a configuration file - **Independent pipelines**, each `sources → flow → sinks`, running concurrently in one process - **Fan-in and fan-out**: many sources and many sinks per pipeline @@ -39,7 +42,8 @@ streams, or downstream LogWisp nodes. ### Outputs -`console` (control characters escaped on a terminal), `file` (rotating with +`console` (level names in color and control characters escaped on a +terminal), `file` (rotating with retention), `http` (Server-Sent Events plus a JSON status endpoint), `tcp` (broadcast server), `null`, and the chain forwarders `tcp_chain` and `http_chain`. @@ -64,6 +68,9 @@ exponential backoff and jitter. - TLS 1.2/1.3 on every network source and sink, listener and dialer alike - Mutual TLS: listeners can require and verify client certificates; dialers can present a client identity +- Certificates without a PKI: `lw tls` makes a CA and certificates, and a + listener can make its own at startup, self-signed (dialers pin its key with + `pin_sha256`) or signed by a CA - Authorization by certificate identity: an `auth` block admits named peers (exact or RE2) rather than everything the CA issued, gates the `http` sink's stream and status endpoints, and lets a dialer pin the server it talks to @@ -109,8 +116,7 @@ authentication designs for the rationale and what is deliberately left out. - [Operations](doc/operations.md): running, monitoring, tuning, troubleshooting - [To Do](doc/todo.md): planned work in priority order: network access - control and the PROXY protocol, a hardening review of the config libraries, - packaging + control and the PROXY protocol, packaging A fully annotated configuration covering every option ships as [`config/logwisp.toml`](config/logwisp.toml). @@ -135,6 +141,9 @@ journalctl -f | ./bin/lw --sink http,host=127.0.0.1,port=8080 # tail a directory instead of stdin ./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \ --format json,sanitizer_policy=json > all.json + +# the same with a preset; on a terminal the level names are in color +./bin/lw --preset tail,path=/var/log/myapp ``` As a service, a configuration file holds the pipelines: diff --git a/cmd/lw/auth.go b/cmd/lw/auth.go index 6e86e89..3b1c366 100644 --- a/cmd/lw/auth.go +++ b/cmd/lw/auth.go @@ -28,15 +28,7 @@ import ( const reloadHint = "send SIGHUP to logwisp to apply (auto_reload does not watch the credentials file)" -// authCommand is one `lw auth` subcommand: define registers its flags -// and returns the command, which runs once they are parsed and checked. -type authCommand struct { - name, synopsis, summary string - required []string - define func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error -} - -var authCommands = []authCommand{ +var authCommands = []subcommand{ {"add-user", "-credentials FILE -user NAME [-password-file FILE] [-generate]", "Add a user to a credentials file, or replace its password", []string{"credentials", "user"}, defineAddUser}, @@ -51,77 +43,6 @@ var authCommands = []authCommand{ []string{"addr", "user", "password-file"}, defineStream}, } -// usageError is a command-line mistake: exit status 2 rather than 1 -type usageError string - -func (e usageError) Error() string { return string(e) } - -// runAuth runs `lw auth` and returns the exit status: 0 success or -// help, 1 failure, 2 usage error. -func runAuth(args []string, stdout, stderr io.Writer) int { - if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) { - printAuthUsage(stderr) - if len(args) == 0 { - return 2 - } - return 0 - } - i := slices.IndexFunc(authCommands, func(c authCommand) bool { return c.name == args[0] }) - if i < 0 { - fmt.Fprintf(stderr, "lw auth: unknown command %q\n\n", args[0]) - printAuthUsage(stderr) - return 2 - } - c := authCommands[i] - fs := flag.NewFlagSet("lw auth "+c.name, flag.ContinueOnError) - fs.SetOutput(stderr) - fs.Usage = func() { - fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), c.synopsis, c.summary) - fs.PrintDefaults() - } - run := c.define(fs) - if err := fs.Parse(args[1:]); err != nil { - if errors.Is(err, flag.ErrHelp) { - return 0 - } - return 2 // already reported by flag, with the usage - } - err := checkArgs(fs, c.required) - if err == nil { - err = run(stdout, stderr) - } - if err == nil { - return 0 - } - fmt.Fprintf(stderr, "%s: %v\n", fs.Name(), err) - if _, ok := errors.AsType[usageError](err); ok { - fs.Usage() - return 2 - } - return 1 -} - -func printAuthUsage(w io.Writer) { - fmt.Fprint(w, "Usage: lw auth [flags]\n\n") - for _, c := range authCommands { - fmt.Fprintf(w, " %-12s %s\n", c.name, c.summary) - } - fmt.Fprint(w, "\nRun lw auth -h for its flags. Credential changes apply on SIGHUP.\n"+ - "Exit status: 0 success, 1 failure, 2 usage error.\n") -} - -func checkArgs(fs *flag.FlagSet, required []string) error { - if fs.NArg() > 0 { - return usageError(fmt.Sprintf("unexpected argument %q", fs.Arg(0))) - } - for _, name := range required { - if fs.Lookup(name).Value.String() == "" { - return usageError("-" + name + " is required") - } - } - return nil -} - func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { path := fs.String("credentials", "", "credentials `file`, created when missing") user := fs.String("user", "", "user `name`") @@ -323,6 +244,7 @@ func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { func tlsFlags(fs *flag.FlagSet) *config.TLSOptions { o := &config.TLSOptions{Enabled: true} fs.StringVar(&o.CAFile, "ca-file", "", "CA `file` that verifies the server (default: system roots)") + fs.StringVar(&o.PinSHA256, "pin-sha256", "", "sha256//BASE64 `pin` of the server key, in place of -ca-file (a self_signed listener logs it)") fs.StringVar(&o.ServerName, "server-name", "", "`name` the server certificate must carry (default: the host)") fs.StringVar(&o.CertFile, "cert-file", "", "client certificate `file`, for listeners with tls.client_auth") fs.StringVar(&o.KeyFile, "key-file", "", "client key `file`") diff --git a/cmd/lw/auth_test.go b/cmd/lw/auth_test.go index 5151635..2a19234 100644 --- a/cmd/lw/auth_test.go +++ b/cmd/lw/auth_test.go @@ -14,10 +14,7 @@ import ( ) func runAuthTest(t *testing.T, args ...string) (code int, stdout, stderr string) { - t.Helper() - var out, errOut bytes.Buffer - code = runAuth(args, &out, &errOut) - return code, out.String(), errOut.String() + return runCommand(t, "auth", args...) } // writeCredentials writes users sharing a cheap Argon2 profile. add-user diff --git a/cmd/lw/bootstrap.go b/cmd/lw/bootstrap.go index 0ead3e9..cbf5e98 100644 --- a/cmd/lw/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "os" _ "github.com/lixenwraith/logwisp/internal/source/console" @@ -27,6 +28,7 @@ import ( "github.com/lixenwraith/log" "github.com/lixenwraith/log/sanitizer" + "github.com/lixenwraith/toml" ) // bootstrapInitial handles initial service startup with status reporter @@ -130,6 +132,29 @@ func checkConfig(cfg *config.Config) int { return 0 } +// dumpConfig prints the effective configuration, flags and environment +// resolved into it, as a file lw -c reads back. +func dumpConfig(cfg *config.Config, w io.Writer) int { + cfg.Dump = false + // Snapshot gave each console sink the top-level color: keep it inherited + for _, p := range cfg.Pipelines { + for _, s := range p.PluginSinks { + if s.Type == "console" && s.Config["color"] == cfg.Color { + delete(s.Config, "color") + } + } + } + data, err := toml.Marshal(cfg) + if err == nil { + _, err = w.Write(data) + } + if err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return 1 + } + return 0 +} + // initializeLogger sets up the global logger based on the application's configuration func initializeLogger(cfg *config.Config) error { logger = log.NewLogger() diff --git a/cmd/lw/cli.go b/cmd/lw/cli.go new file mode 100644 index 0000000..6cd5fcc --- /dev/null +++ b/cmd/lw/cli.go @@ -0,0 +1,213 @@ +package main + +import ( + "cmp" + "errors" + "flag" + "fmt" + "io" + "slices" + "strings" + + "github.com/lixenwraith/logwisp/internal/config" + + "github.com/lixenwraith/toml" +) + +// command is a group of subcommands, recognized only as lw's first argument +type command struct { + name, summary, footer string + subcommands []subcommand +} + +// subcommand is one `lw COMMAND NAME`: define registers its flags and +// returns the work, which runs once they are parsed and checked. +type subcommand struct { + name, synopsis, summary string + required []string + define func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error +} + +var commands = []command{ + {"auth", "SCRAM users, bearer tokens, tcp sink viewing", + "Credential changes apply on SIGHUP.", authCommands}, + {"tls", "A CA and certificates for TLS and mTLS", + "Files are never overwritten; keys are written mode 0600.", tlsCommands}, + {"preset", "Show the pipeline a preset makes, as TOML", + "lw --preset NAME,key=value runs one; flags here are its keys.", presetCommands()}, +} + +// usageError is a command-line mistake: exit status 2 rather than 1 +type usageError string + +func (e usageError) Error() string { return string(e) } + +// run runs a subcommand and returns the exit status: 0 success or help, +// 1 failure, 2 usage error. +func (c *command) run(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) { + c.usage(stderr) + if len(args) == 0 { + return 2 + } + return 0 + } + i := slices.IndexFunc(c.subcommands, func(s subcommand) bool { return s.name == args[0] }) + if i < 0 { + fmt.Fprintf(stderr, "lw %s: unknown command %q\n\n", c.name, args[0]) + c.usage(stderr) + return 2 + } + s := c.subcommands[i] + fs := flag.NewFlagSet("lw "+c.name+" "+s.name, flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { + fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), s.synopsis, s.summary) + fs.PrintDefaults() + } + work := s.define(fs) + if err := fs.Parse(args[1:]); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 // already reported by flag, with the usage + } + err := checkArgs(fs, s.required) + if err == nil { + err = work(stdout, stderr) + } + if err == nil { + return 0 + } + fmt.Fprintf(stderr, "%s: %v\n", fs.Name(), err) + if _, ok := errors.AsType[usageError](err); ok { + fs.Usage() + return 2 + } + return 1 +} + +func (c *command) usage(w io.Writer) { + fmt.Fprintf(w, "Usage: lw %s [flags]\n\n", c.name) + for _, s := range c.subcommands { + fmt.Fprintf(w, " %-12s %s\n", s.name, s.summary) + } + fmt.Fprintf(w, "\nRun lw %s -h for its flags. %s\n"+ + "Exit status: 0 success, 1 failure, 2 usage error.\n", c.name, c.footer) +} + +func checkArgs(fs *flag.FlagSet, required []string) error { + if fs.NArg() > 0 { + return usageError(fmt.Sprintf("unexpected argument %q", fs.Arg(0))) + } + for _, name := range required { + if fs.Lookup(name).Value.String() == "" { + return usageError("-" + name + " is required") + } + } + return nil +} + +// invocation is a parsed command line: a command with its arguments, a help +// request, or what config.Load reads. +type invocation struct { + command *command + args []string + help bool + load config.Args +} + +// parseCommandLine reads lw's own flags; anything else is a --path=value +// setting for lixenwraith/config, which reports what it does not know. A +// pipeline flag takes the next argument unless it starts with '-'. A help +// request wins over a malformed flag. +func parseCommandLine(argv []string) (inv invocation, err error) { + defer func() { + if inv.help { + err = nil + } + }() + if len(argv) > 0 { + if i := slices.IndexFunc(commands, func(c command) bool { return c.name == argv[0] }); i >= 0 { + return invocation{command: &commands[i], args: argv[1:]}, nil + } + inv.help = argv[0] == "help" + } + specFlags := config.SpecFlags() + next := func(i *int) string { + if *i+1 < len(argv) && !strings.HasPrefix(argv[*i+1], "-") { + *i++ + return argv[*i] + } + return "" + } + for i := 0; i < len(argv); i++ { + arg := argv[i] + name, value, inline := strings.Cut(arg, "=") + switch { + case arg == "--": + inv.load.Overrides = append(inv.load.Overrides, argv[i:]...) + return inv, err + case arg == "-h" || arg == "--help": + inv.help = true + case name == "-c" || name == "--config": + if !inline { + value = next(&i) + } + if value == "" { + err = cmp.Or(err, fmt.Errorf("%s requires a configuration file path", name)) + } + inv.load.File = value + case arg == "--color": + // Bare, it means always: config takes no bare flag for a string key + inv.load.Overrides = append(inv.load.Overrides, "--color="+cmp.Or(next(&i), "always")) + case strings.HasPrefix(name, "--") && slices.Contains(specFlags, name[2:]): + if !inline { + value = next(&i) + } + inv.load.Specs = append(inv.load.Specs, config.Spec{Flag: name[2:], Value: value}) + default: + inv.load.Overrides = append(inv.load.Overrides, arg) + } + } + return inv, err +} + +// presetCommands makes each preset a subcommand whose flags are its keys +func presetCommands() []subcommand { + var subs []subcommand + for _, p := range config.Presets() { + s := subcommand{name: p.Name, summary: p.Summary} + var synopsis []string + for _, k := range p.Params { + if k.Required { + s.required = append(s.required, k.Name) + synopsis = append(synopsis, "-"+k.Name+" VALUE") + } + } + s.synopsis = strings.Join(append(synopsis, "[-KEY VALUE...]"), " ") + s.define = func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { + values := map[string]*string{} + for _, k := range p.Params { + values[k.Name] = fs.String(k.Name, k.Default, k.Help) + } + return func(stdout, _ io.Writer) error { + set := map[string]string{} + for name, v := range values { + set[name] = *v + } + pipeline, err := config.ExpandPreset(p.Name, set) + if err != nil { + return usageError(err.Error()) + } + data, err := toml.Marshal(map[string]any{"pipelines": []config.PipelineConfig{pipeline}}) + if err == nil { + _, err = stdout.Write(data) + } + return err + } + } + subs = append(subs, s) + } + return subs +} diff --git a/cmd/lw/cli_test.go b/cmd/lw/cli_test.go new file mode 100644 index 0000000..2ef9c93 --- /dev/null +++ b/cmd/lw/cli_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/tlsx" +) + +// lw's own flags: the last -c wins, a pipeline flag or --color takes the next +// argument unless it starts with '-', -- ends lw's flags, and a command is +// only the first argument. Everything else is a setting for config. +func TestCommandLineGrammar(t *testing.T) { + for _, c := range []struct { + argv []string + want invocation + }{ + {[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}}, + {[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}}, + {[]string{"--source=null", "--sink", "http,port=1", "--filter", "--sink"}, + invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}, {Flag: "filter"}, {Flag: "sink"}}}}}, + {[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}}, + {[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}}, + {[]string{"--quiet", "--", "-c", "x", "--source", "null"}, invocation{load: config.Args{Overrides: []string{"--quiet", "--", "-c", "x", "--source", "null"}}}}, + {[]string{"--quiet", "-h"}, invocation{help: true, load: config.Args{Overrides: []string{"--quiet"}}}}, + {[]string{"-c", "-h"}, invocation{help: true}}, + {[]string{"help"}, invocation{help: true, load: config.Args{Overrides: []string{"help"}}}}, + {[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}}, + {[]string{"--quiet", "tls"}, invocation{load: config.Args{Overrides: []string{"--quiet", "tls"}}}}, + } { + got, err := parseCommandLine(c.argv) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("%q:\n got %+v %v\nwant %+v", c.argv, got, err, c.want) + } + } + for _, argv := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}} { + if _, err := parseCommandLine(argv); err == nil { + t.Errorf("%q: missing path accepted", argv) + } + } +} + +// --dump prints a file that lw -c reads back to the same configuration, its +// console sinks still taking the top-level color +func TestDumpReadsBack(t *testing.T) { + testLogger(t) + dir := t.TempDir() + empty := filepath.Join(dir, "empty.toml") + if err := os.WriteFile(empty, nil, 0o600); err != nil { + t.Fatal(err) + } + args := []string{"--color", "--preset", "serve,listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example", + "--filter", `exclude,patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"} + cfg := loadTestConfig(t, empty, args...) + var dump bytes.Buffer + if code := dumpConfig(loadTestConfig(t, empty, args...), &dump); code != 0 { + t.Fatalf("dump exit %d", code) + } + path := filepath.Join(dir, "dump.toml") + if err := os.WriteFile(path, dump.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + again := loadTestConfig(t, path) + if !reflect.DeepEqual(again.Pipelines, cfg.Pipelines) || again.Color != "always" || again.Logging.Level != "debug" || + strings.Count(dump.String(), "color = ") != 1 { + t.Fatalf("dump does not read back:\n%s\n got %+v\nwant %+v", dump.String(), again.Pipelines, cfg.Pipelines) + } +} + +// Every preset expands to plugins that build, as lw --check builds them +func TestEveryPresetBuilds(t *testing.T) { + testLogger(t) + dir := t.TempDir() + users, pass, empty := filepath.Join(dir, "users.toml"), filepath.Join(dir, "edge.pass"), filepath.Join(dir, "empty.toml") + writeCredentials(t, users, "edge-01") + for path, data := range map[string]string{pass: "password-edge-01\n", empty: ""} { + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + } + pin := "sha256//" + strings.Repeat("A", 43) + "=" + samples := map[string]string{ + "pipe": "pipe,format=txt", + "tail": "tail,path=" + dir, + "serve": "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users, + "edge": "edge,to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass, + "aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out"), + } + for _, p := range config.Presets() { + sample, ok := samples[p.Name] + if !ok { + t.Errorf("preset %s has no sample here", p.Name) + continue + } + if code := checkConfig(loadTestConfig(t, empty, "--preset", sample)); code != 0 { + t.Errorf("%s: lw --check exit %d", sample, code) + } + } +} + +// lw tls ca and lw tls cert write a chain a dialer verifies by the CA file, +// keys private, and never replace a file +func TestTLSCommandsIssueAVerifiableChain(t *testing.T) { + dir := t.TempDir() + steps := [][]string{ + {"ca", "-dir", dir, "-name", "test CA"}, + {"cert", "-ca-dir", dir, "-name", "agg", "-server", "-host", "127.0.0.1"}, + } + for _, args := range steps { + if code, _, stderr := runCommand(t, "tls", args...); code != 0 { + t.Fatalf("%v: exit %d: %s", args, code, stderr) + } + } + for _, key := range []string{"ca.key", "agg.key"} { + if fi, err := os.Stat(filepath.Join(dir, key)); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("%s: %v %v", key, fi.Mode(), err) + } + } + server, err := tlsx.Server(&config.TLSOptions{Enabled: true, CertFile: filepath.Join(dir, "agg.crt"), KeyFile: filepath.Join(dir, "agg.key")}, "") + if err != nil { + t.Fatal(err) + } + client, err := tlsx.Client(&config.TLSOptions{Enabled: true, CAFile: filepath.Join(dir, "ca.crt")}, "127.0.0.1") + if err != nil { + t.Fatal(err) + } + if err := testutil.Handshake(t, server, client); err != nil { + t.Fatalf("issued chain: %v", err) + } + if code, _, stderr := runCommand(t, "tls", steps[0]...); code != 1 || !strings.Contains(stderr, "exists") { + t.Fatalf("second ca: exit %d: %s", code, stderr) + } +} diff --git a/cmd/lw/help.go b/cmd/lw/help.go index a9a4b61..4ccb274 100644 --- a/cmd/lw/help.go +++ b/cmd/lw/help.go @@ -2,7 +2,7 @@ package main import ( "fmt" - "os" + "strings" "github.com/lixenwraith/logwisp/internal/version" ) @@ -15,19 +15,21 @@ Usage: line for line, like cat; exits at end of input lw -c [options] Run the file's pipelines lw --check [options] Build every pipeline and plugin, report, exit + lw --dump [options] Print the effective configuration as TOML, exit lw help | -h | --help lw --version -Subcommands: - lw auth SCRAM credentials files, bearer tokens and tcp - sink viewing; lw auth -h lists commands - +Commands (lw -h lists its subcommands): +%s Scalar configuration keys are settable as flags using their TOML paths: --= e.g. --logging.level=debug Common options: -c, --config Configuration file (default: ./logwisp.toml) --quiet Suppress console output + --color [] Level names in color on console sinks: + auto (a terminal, NO_COLOR unset; default), + always (bare --color) or never --status_reporter= Periodic status logging (default: true with a configuration file) --auto_reload= Config hot reload on file change (default: false) @@ -39,6 +41,9 @@ at warn without one): --logging.file.directory= Pipelines (replace the file's pipelines; see doc/cli.md): + --preset [,k=v...] Start a pipeline with a preset: pipe, tail, + serve, edge, aggregator; lw preset -h + lists its keys, lw preset prints it --pipeline Start a pipeline; earlier specs go to "cli" --source TYPE[,key=value...], repeatable --sink e.g. http,host=0.0.0.0,port=8080, repeatable @@ -56,14 +61,19 @@ Examples: journalctl -f | lw --sink http,host=127.0.0.1,port=8080 Serve stdin as a live stream lw --source file,directory=/var/log/app,pattern='*.log' --format txt + lw --preset tail,path=/var/log/app Follow a directory's files + lw --preset serve,path=/var/log/app,tls=self,users=users.toml + HTTPS stream, SCRAM logins + lw --preset edge,path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\ +password_file=edge-01.pass Forward to an aggregator Environment: LOGWISP_ Config path, '.' -> '_', uppercase e.g. LOGWISP_LOGGING_LEVEL=debug LOGWISP_CONFIG_FILE Configuration file path LOGWISP_CONFIG_DIR Configuration directory - LOGWISP_PIPELINE, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N], LOGWISP_FILTER[_N], - LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT + LOGWISP_PIPELINE, LOGWISP_PRESET, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N], + LOGWISP_FILTER[_N], LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT One pipeline, specs as the flags (_N adds more); ignored when a pipeline flag is given @@ -77,22 +87,10 @@ Exit codes: 2 configuration file not found ` -// handleHelp prints usage and exits if a help request is present in args -func handleHelp(args []string) { - if len(args) > 0 && args[0] == "help" { - printHelp() - } - for _, arg := range args { - if arg == "--" { - break // end of flags - } - if arg == "-h" || arg == "--help" { - printHelp() - } - } -} - func printHelp() { - fmt.Printf(helpText, version.Short()) - os.Exit(0) + var list strings.Builder + for _, c := range commands { + fmt.Fprintf(&list, " lw %-26s %s\n", c.name+" ", c.summary) + } + fmt.Printf(helpText, version.Short(), list.String()) } diff --git a/cmd/lw/main.go b/cmd/lw/main.go index 0accc10..1284958 100644 --- a/cmd/lw/main.go +++ b/cmd/lw/main.go @@ -20,19 +20,23 @@ import ( var logger *log.Logger func main() { - // Before handleHelp, so `lw auth -h` prints the auth usage - if len(os.Args) > 1 && os.Args[1] == "auth" { - os.Exit(runAuth(os.Args[2:], os.Stdout, os.Stderr)) + inv, err := parseCommandLine(os.Args[1:]) + switch { + case err != nil: + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + case inv.command != nil: + os.Exit(inv.command.run(inv.args, os.Stdout, os.Stderr)) + case inv.help: + printHelp() + os.Exit(0) } // --- 1. Initial setup --- // Emulates nohup signal.Ignore(syscall.SIGHUP) - // Before config parsing: the loader has no help flag - handleHelp(os.Args[1:]) - - manager, err := config.Load(os.Args[1:]) + manager, err := config.Load(inv.load) if err != nil { if errors.Is(err, config.ErrConfigNotFound) { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -57,6 +61,9 @@ func main() { if cfg.Check { os.Exit(checkConfig(cfg)) } + if cfg.Dump { + os.Exit(dumpConfig(cfg, os.Stdout)) + } if err := initializeLogger(cfg); err != nil { FatalError(1, "Failed to initialize logger: %v\n", err) diff --git a/cmd/lw/test_helpers_test.go b/cmd/lw/test_helpers_test.go index d4d7507..15885ef 100644 --- a/cmd/lw/test_helpers_test.go +++ b/cmd/lw/test_helpers_test.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "testing" "github.com/lixenwraith/logwisp/internal/config" @@ -19,7 +20,11 @@ func testLogger(t *testing.T) { func loadTestConfig(t *testing.T, path string, args ...string) *config.Config { t.Helper() testutil.ClearEnvPrefix(t, "LOGWISP_") - m, err := config.Load(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...)) + inv, err := parseCommandLine(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...)) + if err != nil { + t.Fatal(err) + } + m, err := config.Load(inv.load) if err != nil { t.Fatal(err) } @@ -30,3 +35,15 @@ func loadTestConfig(t *testing.T, path string, args ...string) *config.Config { } return cfg } + +// runCommand runs `lw NAME ARGS...` and returns its exit status and output +func runCommand(t *testing.T, name string, args ...string) (code int, stdout, stderr string) { + t.Helper() + inv, err := parseCommandLine(append([]string{name}, args...)) + if err != nil || inv.command == nil { + t.Fatalf("lw %s: not a command: %v", name, err) + } + var out, errOut bytes.Buffer + code = inv.command.run(inv.args, &out, &errOut) + return code, out.String(), errOut.String() +} diff --git a/cmd/lw/tls.go b/cmd/lw/tls.go new file mode 100644 index 0000000..c18bc2a --- /dev/null +++ b/cmd/lw/tls.go @@ -0,0 +1,147 @@ +package main + +import ( + "crypto" + "crypto/x509" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/lixenwraith/logwisp/internal/tlsx" +) + +// maxDays keeps a validity well inside time.Duration, whose overflow would +// date a certificate in the past +const maxDays = 36500 + +var tlsCommands = []subcommand{ + {"ca", "-dir DIR [-name NAME] [-days N]", + "Create a certificate authority, DIR/ca.crt and DIR/ca.key, that signs certificates only", + []string{"dir"}, defineCA}, + {"cert", "-ca-dir DIR -name NAME [-host NAME,...] [-server] [-client] [-days N] [-out DIR]", + "Issue a certificate and key, OUT/NAME.crt and OUT/NAME.key, from DIR/ca.crt and DIR/ca.key", + []string{"ca-dir", "name"}, defineCert}, +} + +func defineCA(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { + dir := fs.String("dir", "", "`directory` to write ca.crt and ca.key in, created when missing") + name := fs.String("name", "logwisp CA", "subject common `name`") + days := fs.Int("days", int(tlsx.CAValidity/(24*time.Hour)), "validity in `days`") + return func(_, stderr io.Writer) error { + if *days < 1 || *days > maxDays { + return usageError(fmt.Sprintf("-days must be 1 to %d", maxDays)) + } + cert, key, err := tlsx.NewCA(*name, time.Duration(*days)*24*time.Hour) + if err != nil { + return err + } + if err := writePair(*dir, "ca", cert, key); err != nil { + return err + } + fmt.Fprintf(stderr, "wrote %s and %s, valid until %s\n"+ + "ca.crt verifies: dialers' tls.ca_file, listeners' tls.client_ca_file; with ca.key, it signs: lw tls cert, tls.issuer_cert_file\n", + filepath.Join(*dir, "ca.crt"), filepath.Join(*dir, "ca.key"), cert.NotAfter.UTC().Format(time.DateOnly)) + return nil + } +} + +func defineCert(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { + caDir := fs.String("ca-dir", "", "`directory` holding ca.crt and ca.key (lw tls ca)") + name := fs.String("name", "", "subject common `name`, the mtls identity, and the file names") + hosts := fs.String("host", "", "DNS `names` and IP addresses, ',' between them (default with -server: -name)") + server := fs.Bool("server", false, "for listeners (TLS server authentication)") + client := fs.Bool("client", false, "for dialers presenting a certificate (TLS client authentication)") + days := fs.Int("days", int(tlsx.LeafValidity/(24*time.Hour)), "validity in `days`, at most the CA's") + out := fs.String("out", "", "`directory` to write in (default: -ca-dir)") + return func(_, stderr io.Writer) error { + switch { + case !*server && !*client: + return usageError("-server, -client or both is required") + case *days < 1 || *days > maxDays: + return usageError(fmt.Sprintf("-days must be 1 to %d", maxDays)) + case strings.ContainsAny(*name, `/\`) || *name == "ca" || strings.HasPrefix(*name, "."): + return usageError(fmt.Sprintf("-name %q cannot name the files", *name)) + } + issuer, signer, err := tlsx.LoadIssuer(filepath.Join(*caDir, "ca.crt"), filepath.Join(*caDir, "ca.key")) + if err != nil { + return err + } + leaf := tlsx.Leaf{Name: *name, Server: *server, Client: *client, Validity: time.Duration(*days) * 24 * time.Hour} + for h := range strings.SplitSeq(*hosts, ",") { + if h = strings.TrimSpace(h); h != "" { + leaf.Hosts = append(leaf.Hosts, h) + } + } + if len(leaf.Hosts) == 0 && *server { + leaf.Hosts = []string{*name} + } + key, err := tlsx.NewKey() + if err != nil { + return err + } + cert, err := leaf.Issue(&key.PublicKey, issuer, signer) + if err != nil { + return err + } + dir := *out + if dir == "" { + dir = *caDir + } + if err := writePair(dir, *name, cert, key); err != nil { + return err + } + fmt.Fprintf(stderr, "wrote %s and %s for %v, valid until %s, pin_sha256 %s\n", + filepath.Join(dir, *name+".crt"), filepath.Join(dir, *name+".key"), + append(leaf.Hosts, "CN="+*name), cert.NotAfter.UTC().Format(time.DateOnly), tlsx.PinSHA256(cert)) + return nil + } +} + +// writePair writes NAME.crt (0644) and NAME.key (0600) in dir, refusing to +// replace either: a reissued key would silently break every pin and peer. +func writePair(dir, name string, cert *x509.Certificate, key crypto.PrivateKey) error { + keyPEM, err := tlsx.EncodeKey(key) + if err != nil { + return err + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + certPath, keyPath := filepath.Join(dir, name+".crt"), filepath.Join(dir, name+".key") + for _, p := range []string{certPath, keyPath} { + if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s exists; remove it to replace it", p) + } + } + if err := writeNew(keyPath, keyPEM, 0o600); err != nil { + return err + } + if err := writeNew(certPath, tlsx.EncodeCert(cert), 0o644); err != nil { + os.Remove(keyPath) + return err + } + return nil +} + +func writeNew(path string, data []byte, mode os.FileMode) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode) + if err != nil { + return err + } + _, err = f.Write(data) + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + os.Remove(path) + } + return err +} diff --git a/config/logwisp.toml b/config/logwisp.toml index 2c0db26..0c45847 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -26,6 +26,7 @@ quiet = false # No log of lw's own; pipeline output still flows status_reporter = true # Periodic status logging (30s, DEBUG level); false without a file auto_reload = false # Watch file; read at startup (signals always reread it) +color = "auto" # auto|always|never: level names on console sinks; auto on a terminal, NO_COLOR unset ############################################################################### ### Logging (LogWisp's internal operational logging) @@ -93,10 +94,12 @@ name = "default" ### TLS (shared shape; applies to every network source and sink) ### ### Listeners (tcp/http sinks, tcp_chain/http_chain sources): -### cert_file + key_file are REQUIRED; client_auth + client_ca_file enable mTLS. +### cert_file + key_file, self_signed, or issuer_cert_file + issuer_key_file +### (exactly one); client_auth + client_ca_file enable mTLS. ### Dialers (tcp_chain/http_chain sinks): -### ca_file + server_name verify the server; cert_file + key_file present a -### client identity for mTLS. +### ca_file + server_name, or pin_sha256, verify the server; cert_file + +### key_file present a client identity for mTLS. +### `lw tls ca` / `lw tls cert` make a CA and certificates. ### ### enabled = false Master switch ### cert_file = "" Local certificate @@ -105,7 +108,15 @@ name = "default" ### client_ca_file = "" Listener: CA bundle verifying client certs ### ca_file = "" Dialer: CA bundle verifying the server (empty = system) ### server_name = "" Dialer: SNI / name to verify (empty = configured host) +### self_signed = false Listener: make a self-signed certificate at startup; +### it logs the pin_sha256 dialers set (changes on restart) +### issuer_cert_file = "" Listener: make one at startup signed by this CA ... +### issuer_key_file = "" ... and its key (lw tls ca writes both) +### hosts = [] Listener: names/addresses of a made certificate, beyond +### host, os.Hostname(), localhost, 127.0.0.1, ::1 ### insecure_skip_verify = false Dialer: disable verification (never in prod) +### pin_sha256 = "" Dialer: sha256//BASE64 of the server key, in place of +### ca_file; ';' between several ### min_version = "1.3" "1.2" or "1.3". No max_version, no cipher_suites. ### ### TLS alone is a CA membership check: ANY certificate the CA signed is @@ -257,6 +268,7 @@ type = "console" target = "stdout" # stdout|stderr ("split" NOT supported) # buffer_size = 1000 # escape = "auto" # auto|always|never: control characters as , auto on a terminal +# color = "auto" # auto|always|never: level names in color (default: top-level color) ## File sink (rotating) # [[pipelines.plugin_sinks]] diff --git a/doc/README.md b/doc/README.md index 0d08da0..34d96a4 100644 --- a/doc/README.md +++ b/doc/README.md @@ -31,8 +31,7 @@ streams, or downstream LogWisp nodes. manual, [`lw.1`](lw.1) - [Operations](operations.md): running, monitoring, tuning, troubleshooting - [To Do](todo.md): planned work in priority order: network access control - and the PROXY protocol, a hardening review of the config libraries, - packaging + and the PROXY protocol, packaging A fully annotated configuration covering every option lives at [`config/logwisp.toml`](../config/logwisp.toml). diff --git a/doc/chaining.md b/doc/chaining.md index d33c1f9..6db10d0 100644 --- a/doc/chaining.md +++ b/doc/chaining.md @@ -26,7 +26,9 @@ nodes can filter and reformat as if the entries were local. Both chain sources can feed a single pipeline (fan-in) whose sinks then fan the merged stream out. `test/chain-test.sh` builds the two-independent-pipelines -variant; `test/chain-aggregate-test.sh` builds the fan-in variant. +variant; `test/chain-aggregate-test.sh` builds the fan-in variant. The `edge` +and `aggregator` [presets](cli.md#presets) build an authenticated TLS link of +either transport from a few keys. ## Node Identity diff --git a/doc/cli.md b/doc/cli.md index 77ee278..62ed9b0 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -4,7 +4,11 @@ lw [options] [< input] lw -c [options] lw --check [options] +lw --dump [options] +lw --preset NAME[,key=value...] [options] lw auth [flags] +lw tls [flags] +lw preset [flags] lw help | -h | --help lw --version ``` @@ -14,9 +18,10 @@ stdout, line for line, until the end of input ([Built-in Defaults](#built-in-defaults)). With a file it runs the file's pipelines as a service. -`lw auth` manages SCRAM credentials and logs in to `scram` listeners; see -[below](#lw-auth). There is no certificate-generation command: use -`openssl` or your PKI tooling — see [Security](security.md#enabling-mtls). +A [preset](#presets) builds a common pipeline from a few keys. `lw auth` +manages SCRAM credentials and logs in to `scram` listeners +([below](#lw-auth)); `lw tls` makes a certificate authority and certificates +([below](#lw-tls)). A command is recognized only as the first argument. ## Options @@ -36,6 +41,11 @@ pipelines have [their own flags](#pipelines): `./logwisp.toml` ([Configuration](configuration.md#file-location)) - `--check`: build every pipeline and plugin, bind nothing, report and exit; see [Usage Patterns](#usage-patterns) +- `--dump`: print the effective configuration, flags, environment and presets + resolved into it, as TOML that `-c` reads back, and exit +- `--color [auto|always|never]`: level names in color on console sinks, + default `auto` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`); bare, + `always`. A console sink's own `color` wins ([console sink](sinks.md#console)) - `--quiet`: silence lw's own log and notices, default `false`; pipeline output still flows - `--status_reporter=`: periodic status logging, default `true` with a @@ -70,7 +80,10 @@ Pipeline flags define whole pipelines. They replace the configuration file's pipelines and the built-in default; every other key keeps its precedence. - `--pipeline NAME` starts a pipeline; specs before the first one go to a - pipeline named `cli` + pipeline named `cli`, or after the preset that starts them +- `--preset SPEC` starts a pipeline with a [preset](#presets); later flags add + sources, sinks and filters to it, or replace its format, rate limit and + heartbeat - `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable - `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline - both `--flag SPEC` and `--flag=SPEC` work; `--` ends options @@ -129,6 +142,61 @@ lw --pipeline app --source file,directory=/var/log/app \ Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns). +## Presets + +`--preset NAME,key=value,...` takes the [SPEC](#pipelines) syntax. A list +key (`hosts`, `proxy`) takes several values, repeated (`hosts=a,hosts=b`) or +`,`-separated; any other key takes one. `lw preset NAME [-key value ...]` +prints the pipeline a preset expands to, ready for a configuration file; +`lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An +unknown key fails and lists the valid ones. A `path` is a file, a directory +(its files) or a glob; a preset that takes one reads stdin without it, and +`from` (`end`, or `start`) is where reading a file starts. + +- `pipe`: stdin to stdout, the built-in default + - `format` (`raw`) +- `tail`: follow files to stdout, like `tail -F` + - `path` (required), `from`, `format` (`raw`) +- `serve`: a live HTTP stream (SSE), an [http sink](sinks.md#http) + - `path`, `from`, `format` (`json`), `listen` (`127.0.0.1:8080`) + - `tls`: `off` (default), `self` (a self-signed certificate made at + startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or + `files` (`cert` and `key`); `hosts` adds names to a made certificate + - `users`: a credentials file; readers then log in with SCRAM + - `proxy`, `viewer=true`: the built-in login and viewer pages, which work + only behind the TLS-terminating proxies `proxy` names (a browser cannot + bind its login to the TLS channel); there is no standalone viewer +- `edge`: forward to an aggregator over TLS (`tcp_chain`, or `http_chain` + with `transport=http`) + - `to` (required), `path`, `from`, `transport` (`tcp`), `node` + - verify the aggregator by `ca` (a CA file; default: system roots) or `pin` + (its `tls.pin_sha256`), and `server_name` + - authenticate by `user` and `password_file` (SCRAM), or a client `cert` and + `key` (mTLS): without either the preset refuses to run +- `aggregator`: receive from edges over TLS, to stdout or files + - `listen` (`0.0.0.0:9000`), `transport` (`tcp`), `format` (`json`) + - `tls`: `self` (default), `issuer` or `files`, as for `serve`; never `off` + - authenticate by `users` (SCRAM) or `client_ca` (mTLS): one is required + - `out`: a directory for `aggregate*.log` files; default stdout + +```bash +# follow a directory, level names in color on a terminal +lw --preset tail,path=/var/log/app + +# serve it over HTTPS, self-signed, to SCRAM users; keep the result as a file +lw --preset serve,path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml + +# an aggregator with a certificate made at startup, and an edge pinning it +lw --preset aggregator,users=/etc/logwisp/users.toml,out=/var/log/edges +lw --preset edge,path=/var/log/app,to=agg.example.org:9000,pin=sha256//BASE64,\ +user=edge-01,password_file=/etc/logwisp/edge-01.pass +``` + +A self-signed aggregator logs its pin at startup (`pin_sha256`, a warning, so +it shows without a configuration file too). The pin holds across reloads and +changes when lw restarts; a fleet signs with an issuer instead, and edges set +`ca`. See [Security](security.md#certificates-made-at-startup). + ## Environment Variables Configuration paths map to environment variables by replacing `.` with `_`, @@ -156,6 +224,7 @@ One pipeline can come from the environment, with the [SPEC](#pipelines) syntax. Any pipeline flag on the command line makes LogWisp ignore all of them. - `LOGWISP_PIPELINE`: the pipeline's name, default `cli` +- `LOGWISP_PRESET`: the [preset](#presets) that starts it - `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each - `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered one in numeric order; likewise `LOGWISP_SINK_N` and `LOGWISP_FILTER_N` @@ -211,7 +280,7 @@ service untouched; see [Configuration](configuration.md#hot-reload). ## Exit Codes - `0`: the end of input, a clean shutdown (`SIGINT`, `SIGTERM`), `--version` / - `--help`, or a valid `--check` + `--help`, `--dump`, or a valid `--check` - `1`: general error: a configuration load or validation failure (`--check` included), a logger init failure, a service bootstrap failure - `2`: an explicitly requested configuration file is not found @@ -239,8 +308,8 @@ lw < app.log > copy.log - A line over 1 MiB continues in the next entry; no byte is lost. - Nothing is dropped: a slow reader slows the reading of stdin. - On a terminal, control characters are written as `` (`ESC` becomes - `<1b>`); pipes and files get the bytes unchanged - ([console sink](sinks.md#console)). + `<1b>`) and level names are in color; pipes and files get the bytes + unchanged ([console sink](sinks.md#console)). - When a console source reads a terminal, lw says so once on stderr, unless `--quiet`: `lw: reading standard input; Ctrl-D ends it (lw --help for usage)`. @@ -255,10 +324,9 @@ values from the file, the environment or flags win. finishes when all its sources have ended, and once every pipeline has finished lw shuts down gracefully and exits 0. Only a console source ends on its own: file and network sources run until a signal, and so does a pipeline -holding one. Console and file sinks write what is queued before the -exit. The `http` and `tcp` sinks do not yet flush their clients' queues, so a -connected client can miss the last entries of a finite input -([To Do](todo.md)). +holding one. Every sink writes what is queued before the exit; the `http` +and `tcp` sinks give each connected client `write_timeout_ms`, at most 2 s, +to take it. ## Usage Patterns @@ -363,8 +431,9 @@ Neither touches a running LogWisp: send `SIGHUP`, since `auto_reload` does not watch the credentials file. **`token`** and **`stream`** build the same TLS and SCRAM client as a chain -sink. TLS flags: `-ca-file` (default: system roots), `-server-name` (default: -the host), and `-cert-file` / `-key-file` for a listener with `tls.client_auth`. +sink. TLS flags: `-ca-file` (default: system roots) or `-pin-sha256` (the +`tls.pin_sha256` a self-signed listener logs), `-server-name` (default: the +host), and `-cert-file` / `-key-file` for a listener with `tls.client_auth`. There is no flag to skip verification: an unverified server could relay the login. Redirects are not followed. `-unbound` logs in to an `http` sink behind a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's @@ -403,3 +472,32 @@ curl --cacert ca.crt -H @<(printf 'Authorization: Bearer %s\n' \ lw auth stream -addr HOST:PORT -user viewer -password-file viewer.pass \ -ca-file ca.crt ``` + +## `lw tls` + +Makes ECDSA P-256 certificates for `tls` blocks. It never replaces a file +(remove one to replace it) and writes keys `0600`, certificates `0644`. + +``` +lw tls ca -dir DIR [-name NAME] [-days 3650] +lw tls cert -ca-dir DIR -name NAME [-host NAME,...] [-server] [-client] + [-days 397] [-out DIR] +``` + +- `ca`: `DIR/ca.crt` and `DIR/ca.key`, a CA that signs leaves only (path + length 0). `ca.crt` is what dialers set as `tls.ca_file` and listeners as + `tls.client_ca_file`; with `ca.key` it is a listener's + `tls.issuer_cert_file` and `tls.issuer_key_file`. +- `cert`: `NAME.crt` and `NAME.key`, in `-out` (default: `-ca-dir`). + - `-server` for a listener: `-host` lists the DNS names and IP addresses it + carries, default `NAME`; `-client` for a dialer presenting a certificate; + both may be given. + - `NAME` is the subject CN, the identity `auth.type = "mtls"` matches. + - Valid 397 days (the longest browsers accept), never past the CA. It prints + the certificate's `pin_sha256`. + +```bash +lw tls ca -dir /etc/logwisp/pki +lw tls cert -ca-dir /etc/logwisp/pki -name agg.example.org -server +lw tls cert -ca-dir /etc/logwisp/pki -name edge-01 -client +``` diff --git a/doc/configuration.md b/doc/configuration.md index 74e1198..8051424 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -46,9 +46,12 @@ Top-level keys, each as type and default: emit a status report every 30 s at DEBUG level. - `auto_reload` (bool, `false`): watch the config file and reload pipelines on change. +- `color` (string, `"auto"`): level names in color on console sinks, which may + set their own: `"auto"` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`), + `"always"` or `"never"`; see the [console sink](sinks.md#console). -`--version` prints version information and exits; it is not a persistent -setting. +`--version`, `--check` and `--dump` print and exit; they are not persistent +settings. Note that `status_reporter` writes at DEBUG level, so it produces nothing unless `logging.level = "debug"`. @@ -311,6 +314,7 @@ What a reload applies: - pipelines, sources, sinks - filters, formatters, rate limits, heartbeats - `status_reporter` + - `color` - Not reloaded: - `logging.*`: applied once at startup - `quiet` diff --git a/doc/lw.1 b/doc/lw.1 index 06d6250..918293a 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -1,4 +1,4 @@ -.Dd October 3, 2026 +.Dd October 4, 2026 .Dt LW 1 .Os .Sh NAME @@ -6,10 +6,12 @@ .Nd LogWisp log transport .Sh SYNOPSIS .Nm -.Op Fl \-check +.Op Fl \-check | Fl \-dump .Op Fl c Ar file .Op Fl \-quiet +.Op Fl \-color Op Ar when .Op Fl \- Ns Ar key Ns = Ns Ar value ... +.Op Fl \-preset Ar spec .Op Fl \-pipeline Ar name .Op Fl \-source Ar spec ... .Op Fl \-sink Ar spec ... @@ -18,7 +20,7 @@ .Op Fl \-rate\-limit Ar spec .Op Fl \-heartbeat Ar spec .Nm -.Cm auth +.Cm auth | tls | preset .Ar command .Op Ar flags .Nm @@ -63,11 +65,13 @@ source runs until shutdown. When every pipeline has finished, .Nm shuts down gracefully and exits 0. -Console and file sinks first write what they hold; the +Every sink first writes what it holds; the .Cm http and .Cm tcp -sinks do not yet, so a client can miss the last lines of a finite input. +sinks give each client +.Cm write_timeout_ms , +at most 2 seconds, to take it. Under a supervisor, standard input is usually .Pa /dev/null : a console source there ends at once. @@ -88,6 +92,13 @@ option sets this: .Cm always or .Cm never . +Its +.Cm color +option, by default the top-level +.Cm color , +paints the level name in each line: debug cyan, info green, warn yellow, +error bold red, trace grey, in the 16 colors every terminal and text console +renders, with the terminal theme choosing the shades. .Pp .Nm stays in the foreground and logs to standard error; run a configuration as a @@ -109,6 +120,22 @@ Set any scalar configuration key by its TOML path, such as A bare .Fl \- Ns Ar key sets a boolean to true. +.It Fl \-color Op Ar when +Color the level names console sinks write: +.Cm auto +.Pq the default: on a terminal, with Ev NO_COLOR unset and Ev TERM not Cm dumb , +.Cm always +.Pq what a bare Fl \-color No means +or +.Cm never . +A console sink's own +.Cm color +option takes precedence. +The codes go into whatever the format makes, JSON included, so output another +program parses keeps +.Cm auto +or +.Cm never . .It Fl \-quiet Turn off the log of .Nm @@ -151,6 +178,11 @@ or opening, reading or creating a log file; print and exit 0, or the error and exit 1, or 2 when a named configuration file is missing .Pq see Sx EXIT STATUS . +.It Fl \-dump +Print the effective configuration, with the options, environment and +presets resolved into it, as a file +.Fl c +reads back, and exit. .It Fl \-version Print the version, commit and build time, and exit. .It Fl h , Fl \-help , Cm help @@ -176,7 +208,12 @@ Start a new pipeline. Specifications before the first .Fl \-pipeline belong to a pipeline named -.Dq cli . +.Dq cli , +or after the preset that starts them. +.It Fl \-preset Ar spec +Start the pipeline with a preset, described in +.Sx Presets ; +later options add sources, sinks and filters to it, or replace its format. .It Fl \-source Ar spec , Fl \-sink Ar spec , Fl \-filter Ar spec Add a source, sink or filter; each is repeatable. .It Fl \-format Ar spec , Fl \-rate\-limit Ar spec , Fl \-heartbeat Ar spec @@ -218,6 +255,125 @@ Any pipeline flag on the command line makes .Nm ignore the pipeline variables in .Sx ENVIRONMENT . +.Ss Presets +A preset builds a common pipeline from a few keys: +.Fl \-preset Ar name Ns Op , Ns Ar key Ns = Ns Ar value ... , +in the syntax of the other specifications; +.Nm +.Cm preset Ar name Op Fl Ns Ar key Ar value ... +prints the pipeline it expands to, and +.Fl h +there lists its keys and defaults. +A +.Ar path +is a file, a directory, whose files are followed, or a glob; without one, a +preset that takes it reads standard input. +.Bl -tag -width Ds +.It Cm pipe Op Cm format +Standard input to standard output, the pipeline +.Nm +runs without a configuration. +.It Cm tail path Op Cm from , format +Follow files to standard output, like +.Ql tail \-F . +.It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer Oc +Serve a live HTTP stream on +.Cm listen +.Pq default Li 127.0.0.1:8080 . +.Cm tls +is +.Cm off , +.Cm self +.Pq a self-signed certificate made at startup , +.Cm issuer +.Pq one signed by Cm issuer_cert No and Cm issuer_key +or +.Cm files +.Pq Cm cert No and Cm key ; +.Cm users +names a credentials file, so readers log in with SCRAM. +The built-in login and viewer pages, +.Cm viewer=true , +work only behind a TLS-terminating proxy named by +.Cm proxy , +since a browser cannot bind its login to the TLS channel. +.It Cm edge to Oo Cm path , transport , ca | pin , user , password_file , cert , key , node Oc +Forward to an aggregator at +.Cm to +over TLS, by +.Cm tcp_chain +or, with +.Cm transport=http , +.Cm http_chain . +It refuses to run without credentials: +.Cm user +and +.Cm password_file , +or a client +.Cm cert +and +.Cm key . +.It Cm aggregator Oo Cm listen , transport , tls , users , client_ca , out , format Oc +Receive from edges on +.Cm listen +.Pq default Li 0.0.0.0:9000 +over TLS, never off +.Pq default Cm tls=self , +authenticated by the +.Cm users +credentials file or the client certificates +.Cm client_ca +vouches for, and write to standard output or to files in +.Cm out . +.El +.Ss Certificate commands +.Nm +.Cm tls +makes ECDSA P-256 certificates; it never replaces a file, and writes keys +with mode 0600. +.Bl -tag -width Ds +.It Cm ca Fl dir Ar dir Oo Fl name Ar name Oc Op Fl days Ar n +Create a certificate authority, +.Pa dir/ca.crt +and +.Pa dir/ca.key , +valid 3650 days. +.It Cm cert Fl ca-dir Ar dir Fl name Ar name Oo Fl host Ar names Oc Oo Fl server Oc Oo Fl client Oc Oo Fl days Ar n Oc Op Fl out Ar dir +Issue +.Pa name.crt +and +.Pa name.key +from that authority, valid 397 days and never past it. +.Fl host +lists the DNS names and addresses a server certificate carries +.Pq default: Ar name ; +.Ar name +is also the subject, the identity +.Cm auth.type=mtls +checks. +.El +.Pp +A listener needs no files: +.Cm tls.self_signed=true , +or +.Cm tls.issuer_cert_file +with +.Cm tls.issuer_key_file , +make its certificate at startup, for +.Cm tls.hosts , +its +.Cm host , +this machine's name, localhost and the loopback addresses. +Its key lasts until +.Nm +exits, across reloads. +A self-signed listener logs the +.Cm tls.pin_sha256 +that dialers set in place of +.Cm tls.ca_file ; +it changes when +.Nm +restarts, so a fleet uses an issuer instead. .Ss Authentication commands .Nm .Cm auth @@ -258,7 +414,9 @@ The .Ar tls-flags are .Fl ca-file Ar file -.Pq default: the system roots , +.Pq default: the system roots +or +.Fl pin-sha256 Ar pin , .Fl server-name Ar name , and .Fl cert-file Ar file @@ -290,6 +448,8 @@ The directory holding .Pa logwisp.toml . .It Ev LOGWISP_PIPELINE The name of the one pipeline the variables below define. +.It Ev LOGWISP_PRESET +The preset that starts it. All of these are ignored when a pipeline option is on the command line. .It Ev LOGWISP_SOURCE , LOGWISP_SINK , LOGWISP_FILTER Source, sink and filter specifications, in the syntax of the matching @@ -299,6 +459,13 @@ counting from 1, adds more, as in .Ev LOGWISP_SINK_1 . .It Ev LOGWISP_FORMAT , LOGWISP_RATE_LIMIT , LOGWISP_HEARTBEAT The pipeline's format, rate limit and heartbeat specifications. +.It Ev NO_COLOR , TERM +Set to anything, +.Ev NO_COLOR +turns +.Cm auto +color off, as does +.Ev TERM=dumb . .El .Pp Secrets never go in the environment: TLS keys, credentials files and @@ -357,8 +524,11 @@ as it ends the shell reports status 141. .Pp .Nm -.Cm auth -exits 0 on success or help, 1 on failure, and 2 on a usage error. +.Cm auth , +.Cm tls +and +.Cm preset +exit 0 on success or help, 1 on failure, and 2 on a usage error. .Sh SIGNALS .Bl -tag -width "SIGINT, SIGTERM" .It Dv SIGINT , SIGTERM @@ -405,6 +575,33 @@ $ lw \-\-source 'file,directory=/var/log/app,pattern=*.log,raw=true' \e > all.log .Ed .Pp +Follow every file in a directory, with the level names in color on a +terminal: +.Bd -literal -offset indent +$ lw \-\-preset tail,path=/var/log/app +.Ed +.Pp +Print the configuration a set of options makes, to keep as a file: +.Bd -literal -offset indent +$ lw \-\-preset serve,path=/var/log/app,tls=self \-\-dump > serve.toml +.Ed +.Pp +Ship a directory from an edge to an aggregator: on the aggregator, create +the authority, its certificate and a user for the edge; copy +.Pa ca.crt +and the password file to the edge; then run both: +.Bd -literal -offset indent +# lw tls ca \-dir /etc/logwisp/pki +# lw tls cert \-ca-dir /etc/logwisp/pki \-name agg.example.org \-server +# lw auth add-user \-credentials /etc/logwisp/users.toml \e + \-user edge-01 \-password-file /root/edge-01.pass +agg# lw \-\-preset aggregator,tls=files,users=/etc/logwisp/users.toml,\e +cert=/etc/logwisp/pki/agg.example.org.crt,\e +key=/etc/logwisp/pki/agg.example.org.key,out=/var/log/edges +edge# lw \-\-preset edge,path=/var/log/app,to=agg.example.org:9000,\e +ca=/etc/logwisp/pki/ca.crt,user=edge-01,password_file=/root/edge-01.pass +.Ed +.Pp Check a configuration without binding a port or touching a log file: .Bd -literal -offset indent $ lw \-\-check \-c /etc/logwisp/logwisp.toml diff --git a/doc/networking.md b/doc/networking.md index ef5e898..2978e28 100644 --- a/doc/networking.md +++ b/doc/networking.md @@ -213,6 +213,9 @@ headers, and entry encoding. `client_ca_file` does not contain the issuer of the client certificate. - `certificate is not valid for any names` / SAN mismatch — the dialed `host` is not covered by the server certificate's SANs; set `server_name`. +- `server key sha256//... matches no tls.pin_sha256` — the listener is not the + one pinned, or a self-signed listener restarted and made a new key: take the + pin from its startup warning, or move it to an issuer and `ca_file`. - `protocol version not supported` — one side is pinned to `min_version = "1.3"` and the other cannot negotiate it. - Listeners log handshake failures at WARN with the remote address (the `tcp` diff --git a/doc/operations.md b/doc/operations.md index a366ff5..8748462 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -17,8 +17,10 @@ lw exits on its own only at the end of input: once every pipeline has finished, a pipeline finishing when all its sources have ended. Only a console source ends, when stdin does; file and network sources run until a signal, and lw shuts down gracefully, exit 0, either way. -Console and file sinks write what is queued before the exit; the `http` and -`tcp` sinks do not yet flush their clients' queues ([To Do](todo.md)). See +Every sink writes what is queued before the exit; the `http` and `tcp` sinks +give each client `write_timeout_ms`, at most 2 s, to take it. A reload stops +the old sinks the same way, so a client that stopped reading delays it by +that much. See [CLI](cli.md#built-in-defaults). There is no built-in daemon mode. Run LogWisp in the foreground under a diff --git a/doc/security.md b/doc/security.md index 93a0f62..117a246 100644 --- a/doc/security.md +++ b/doc/security.md @@ -8,7 +8,11 @@ configure it, and — equally important — what it does not yet do. **Implemented:** - TLS 1.2 / 1.3 on all network sources and sinks. -- Server certificate verification by dialers. +- Server certificate verification by dialers, by a CA or by the pin of the + server's key (`pin_sha256`). +- Certificates without files: listeners make theirs at startup, self-signed + or from an issuer CA, and `lw tls` makes a CA and certificates: see + [Certificates made at startup](#certificates-made-at-startup). - Mutual TLS (client certificate required and verified), at the transport layer. - Peer identity recorded per session. @@ -26,16 +30,20 @@ configure it, and — equally important — what it does not yet do. [Startup Warnings](#startup-warnings). - Unknown configuration keys rejected: a typo in `tls`, `auth` or a table path fails startup. -- Configuration input hardened (lixenwraith/config v0.2.2, reviewed with - adversarial tests and fuzzing like `auth`): +- Configuration input hardened (lixenwraith/config v0.2.2 and + lixenwraith/toml v0.1.3, reviewed with adversarial tests and fuzzing like + `auth`): + - a document creates at most 65536 tables, so a small file cannot exhaust + memory, and unescaped control characters are refused in strings and + comments; - a string in the file is one list entry, so a filter pattern holding a comma is not split into patterns that match nothing; - the file is opened without blocking, so a FIFO swapped in for it cannot hang the watcher or a `SIGHUP` reload; - a permission change on the file is reported once and does not stop auto-reload; - - conversion errors name the key, never the value, so a secret placed in - the wrong key does not reach the log. + - parse and conversion errors name the key, never the value, so a secret + placed in the wrong key does not reach the log. **Not implemented:** @@ -69,6 +77,11 @@ client_ca_file = "" ca_file = "" server_name = "" insecure_skip_verify = false +pin_sha256 = "" +self_signed = false +issuer_cert_file = "" +issuer_key_file = "" +hosts = [] min_version = "1.3" ``` @@ -76,11 +89,16 @@ Options by role, each with its default: - Both roles - `enabled` (`false`): master switch; when false the whole block is ignored. - - `cert_file`: local certificate; **required** for listeners, an optional - client identity for dialers. + - `cert_file`: local certificate; a listener needs it, `self_signed` or the + issuer files, a dialer may present it as its client identity. - `key_file`: private key for `cert_file`; set the two together. - `min_version` (`"1.3"`): `"1.2"` or `"1.3"`. - Listeners + - `self_signed` (`false`): make a self-signed certificate at startup. + - `issuer_cert_file`, `issuer_key_file`: make one at startup, signed by this + CA (`lw tls ca`). + - `hosts` (`[]`): names and addresses a made certificate carries, beyond the + listener's `host`, `os.Hostname()`, `localhost`, `127.0.0.1` and `::1`. - `client_auth` (`false`): require and verify a client certificate (mTLS). - `client_ca_file`: CA bundle that verifies client certificates; **required** when `client_auth` is true. @@ -89,6 +107,9 @@ Options by role, each with its default: - `server_name` (the configured `host`): SNI and certificate name to verify against. - `insecure_skip_verify` (`false`): disable server verification. + - `pin_sha256`: `sha256//BASE64` of the server's public key, curl's + `--pinnedpubkey` form, `;` between several; it replaces the CA and name + checks. Listeners are the `tcp` and `http` sinks and the `tcp_chain` and `http_chain` sources; dialers are the `tcp_chain` and `http_chain` sinks. @@ -102,7 +123,12 @@ sources; dialers are the `tcp_chain` and `http_chain` sinks. Misconfiguration fails at plugin construction, before the pipeline starts: -- a listener with `enabled = true` and no `cert_file`/`key_file` +- a listener with `enabled = true` and none of `cert_file`/`key_file`, + `self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one; an + issuer that is no CA (`lw tls ca` makes one) +- `pin_sha256` with `ca_file` or `insecure_skip_verify`, or not of the + `sha256//BASE64` form; a listener key on a dialer, or `pin_sha256` on a + listener - `client_auth = true` with no `client_ca_file` - a dialer with only one of `cert_file` / `key_file` - a certificate or key that will not load, or a CA file containing no @@ -298,20 +324,53 @@ Errors read like `auth: type "mtls" requires tls.client_auth`. Some settings work but are usually mistakes. Each plugin reports them at WARN when it is constructed, so startup and every reload repeat them: -- a certificate in `cert_file`, `ca_file` or `client_ca_file` that has expired, - is not yet valid, or expires within 30 days +- a certificate in `cert_file`, `ca_file`, `client_ca_file` or + `issuer_cert_file` that has expired, is not yet valid, or expires within 30 + days +- a self-signed listener, with the `pin_sha256` its dialers need - `insecure_skip_verify` on a dialer - an `allow_patterns` entry not anchored at both ends of every alternative (`^a|b$` admits `a…` and `…b`) -- a `key_file`, `credentials_file` or `password_file` every local user can read +- a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every + local user can read (once per path per process) +## Certificates made at startup + +A listener needs no certificate files. With `self_signed = true`, or with +`issuer_cert_file` and `issuer_key_file`, it makes an ECDSA P-256 key once per +process and, at startup and every reload, a certificate for it valid 397 days +(never past the issuer), for `hosts`, its `host` (unless a wildcard), +`os.Hostname()`, `localhost`, `127.0.0.1` and `::1`. + +- **Self-signed.** No CA can vouch for it, so dialers pin its key: + `pin_sha256`, which the listener logs at WARN on every start. A reload + keeps the key and so the pin; a restart changes both, and every dialer must + be given the new pin. Suits one aggregator and a few edges, a test, or a + stream read by `curl --pinnedpubkey`. +- **Issuer.** `lw tls ca` makes a CA once; the listener signs its own + certificate with it, and dialers verify it with `ca_file = ca.crt`, by name, + across restarts. The CA key then lives on the listener's host: give each + listener group its own CA, or issue certificates elsewhere with + `lw tls cert` and use `cert_file`. +- **Pins.** A pin is the SHA-256 of the server's public key, not of its + certificate, so it survives reissue. `pin_sha256` accepts several, `;` + between them, for a rotation. It replaces chain, name and validity checks: + whoever holds the key is the server. `scram` dialers still bind every login + to that certificate. + +```bash +lw tls ca -dir /etc/logwisp/pki # ca.crt, ca.key (0600) +lw tls cert -ca-dir /etc/logwisp/pki -name agg.example.org -server +lw tls cert -ca-dir /etc/logwisp/pki -name edge-01 -client +``` + ## Enabling mTLS ### 1. Generate a CA and certificates -LogWisp does not generate certificates; use `openssl`, `cfssl`, `step-cli`, or -your existing PKI. +`lw tls` makes them ([above](#certificates-made-at-startup)); with `openssl`, +`cfssl`, `step-cli` or an existing PKI: ```bash # CA diff --git a/doc/sinks.md b/doc/sinks.md index 513d5f3..e3ae603 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -27,6 +27,7 @@ type = "console" target = "stdout" buffer_size = 1000 escape = "auto" +color = "auto" ``` Options, each as type and default: @@ -36,6 +37,9 @@ Options, each as type and default: - `escape` (string, `auto`): write control characters as `` of their bytes, `ESC` as `<1b>`: `auto` when the output is a terminal, `always`, or `never`. +- `color` (string, the top-level `color`, itself `auto`): paint level names: + `auto` on a terminal with `NO_COLOR` unset and `TERM` not `dumb`, `always`, + or `never`. > `split` is **not** a valid target for this sink and is rejected at startup. > Level-based splitting exists only for LogWisp's own application log @@ -50,6 +54,14 @@ Payloads are written as the formatter made them, one record per line. characters, such as the joiners emoji and Indic scripts need, are kept. `never` keeps an application's own ANSI colours; under `auto`, pipes and files get the bytes unchanged. +- Color: the first whole word in a line, in any case, that names the entry's + level is painted: `DEBUG`/`DBG` cyan (blue is hard to read on black), + `INFO`/`INF` green, `WARNING`/`WARN` yellow, `ERROR`/`ERR`/`FATAL` bold red, + `TRACE` grey. The colors are ANSI 16, so the terminal's theme picks the + shades and a Linux or BSD text console renders them; the codes are the + sink's own, written after escaping. A line whose source found no level + stays plain. The codes go into whatever the format makes, JSON included: + for output another program parses keep `auto` or `never`, not `always`. - Backpressure: the sink never drops. When its output is slow (a slow reader, a paused terminal, a stalled log collector) the pipeline waits for it, with its other sinks. A service whose stdout may stall writes to a `file` sink @@ -205,10 +217,11 @@ Options, each as type and default: - Clients whose session has been idle-expired by the session manager are evicted by the broker. With the idle comment above, that reaches only a peer that has stopped accepting bytes on a sink configured `write_timeout_ms = 0`. -- On shutdown, connected clients receive - `event: disconnect / data: {"reason":"server_shutdown"}`. Entries still - queued for them are not flushed first, so a client can miss the last ones - of a finite input ([To Do](todo.md)). +- On shutdown (a reload too), each stream writes its queue and what remains + of the sink's input, then + `event: disconnect / data: {"reason":"server_shutdown"}`, on its own and + within `write_timeout_ms`, at most 2 s; a client still behind is cut then, + and one that stopped reading costs the others nothing. - HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1. **Status endpoint** returns service and version identity, host, port, TLS flag, @@ -287,8 +300,8 @@ Options, each as type and default: whole timeout, so the client is disconnected immediately rather than retried. - A client whose send queue is full has that event dropped (`dropped_writes`) and stays connected. -- On shutdown, entries still queued for clients are not flushed first, as - with the `http` sink. +- On shutdown, as with the `http` sink, each client receives what is queued, + within `write_timeout_ms`, at most 2 s, before it is disconnected. - With TLS enabled the handshake runs under a 10 s bound *after* the `max_connections` check, so concurrent handshakes are bounded too. - With an `auth` block, authorization runs after that handshake and *before* diff --git a/doc/todo.md b/doc/todo.md index 59dfb9f..a54c76e 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -122,23 +122,7 @@ connection level, below TLS, so a refused peer costs no handshake. - Denied and allowed clients; per-client SCRAM budgets behind one proxy address. -## 2. Flush network sinks at shutdown - -At the end of a finite input lw shuts down, and the `http` and `tcp` sinks -disconnect their clients without writing the entries still queued for them, so -a client can miss the last ones. On `Stop`, each sink's broker should first move -its input queue into the client queues, then let every client writer drain its -queue within a bound (`write_timeout_ms`) before the disconnect frame. Verify -with a finite stdin into each sink and a connected client counting lines. - -## 3. Bump `lixenwraith/toml` - -Its hardening commits (a table budget, TOML 1.1 escapes, control characters -refused outside strings, bounded errors that never print a value) wait to merge -in that repository. Then bump it here and drop this item; security.md's library -note already covers it. - -## 4. Packaging: AUR, FreeBSD ports, Debian +## 2. Packaging: AUR, FreeBSD ports, Debian The foundation exists: - The `lw` binary name, free in Arch (official repositories and AUR), Ubuntu diff --git a/go.mod b/go.mod index 5ba3877..c214fe8 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,10 @@ go 1.27.1 require ( github.com/lixenwraith/auth v0.0.0-20261002191639-3ed0e995807c + github.com/lixenwraith/color v0.1.3 github.com/lixenwraith/config v0.2.2 github.com/lixenwraith/log v0.1.11 + github.com/lixenwraith/terminal v0.1.9 github.com/lixenwraith/toml v0.1.3 golang.org/x/term v0.46.0 ) diff --git a/go.sum b/go.sum index c3d8f73..ddcf2ef 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,14 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/lixenwraith/auth v0.0.0-20261002191639-3ed0e995807c h1:zwpVa85g267vo13UCzCao4lfVgUPaU79CmxPW9OdF8w= github.com/lixenwraith/auth v0.0.0-20261002191639-3ed0e995807c/go.mod h1:411JUqnbQgy9jgoFV+fH91TfVUEfzYA4dyPv7f4c264= +github.com/lixenwraith/color v0.1.3 h1:vSuvtU4zh1mIZS/7UGHmMdsTr+a7eCjUYGbou7kzFn4= +github.com/lixenwraith/color v0.1.3/go.mod h1:L/I3Vxp6pqslAIzWyZGgKCm2lyStHCA4vKaUoMUm7bU= github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZgOc= github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg= github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKhU= github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg= +github.com/lixenwraith/terminal v0.1.9 h1:WPxLdqdzZoqtPgTb86oEwE66lF1+iPsJ/gpfo6h/rik= +github.com/lixenwraith/terminal v0.1.9/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44= github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI= github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= diff --git a/internal/authz/authz.go b/internal/authz/authz.go index 578b769..68560a2 100644 --- a/internal/authz/authz.go +++ b/internal/authz/authz.go @@ -129,8 +129,9 @@ func New(o *config.AuthOptions, tlsCfg *tls.Config, role Role, transport Transpo if tlsCfg == nil && (o.Type != MethodSCRAM || len(o.TrustedProxies) == 0) { return nil, fmt.Errorf("auth: type %q requires tls.enabled", o.Type) } - if role == RoleDialer && tlsCfg != nil && tlsCfg.InsecureSkipVerify { - // An unverified server makes identities claims and exposes credentials + if role == RoleDialer && tlsCfg != nil && tlsCfg.InsecureSkipVerify && tlsCfg.VerifyPeerCertificate == nil { + // An unverified server makes identities claims and exposes credentials; + // tls.pin_sha256 verifies it in VerifyPeerCertificate instead return nil, fmt.Errorf("auth: type %q cannot be used with tls.insecure_skip_verify", o.Type) } diff --git a/internal/authz/authz_test.go b/internal/authz/authz_test.go index d44f7d2..7d8bbd6 100644 --- a/internal/authz/authz_test.go +++ b/internal/authz/authz_test.go @@ -167,10 +167,13 @@ func TestNewValidation(t *testing.T) { } // The rows' bases are valid: a dialer needs TLS but not client_auth, as it - // pins the server's identity + // pins the server's identity; tls.pin_sha256 verifies in place of the chain + pinned := &tls.Config{InsecureSkipVerify: true, VerifyPeerCertificate: func([][]byte, [][]*x509.Certificate) error { return nil }} for _, o := range []*config.AuthOptions{{Type: MethodMTLS}, dialer(config.AuthOptions{})} { - if _, err := New(o, &tls.Config{}, RoleDialer, TCP); err != nil { - t.Fatalf("dialer policy %s rejected: %v", o.Type, err) + for _, c := range []*tls.Config{{}, pinned} { + if _, err := New(o, c, RoleDialer, TCP); err != nil { + t.Fatalf("dialer policy %s rejected: %v", o.Type, err) + } } } if _, err := New(listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, RoleListener, HTTP); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index 387693d..902c57f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,8 +7,13 @@ type Config struct { // Top-level flags for application control ShowVersion bool `toml:"version"` Check bool `toml:"check"` // build every plugin, report, exit: lw --check + Dump bool `toml:"dump"` // print the effective configuration, exit: lw --dump Quiet bool `toml:"quiet"` + // Console sinks without their own color: "auto" (a terminal, NO_COLOR + // unset, TERM not dumb), "always", "never" + Color string `toml:"color"` + // Runtime behavior flags StatusReporter bool `toml:"status_reporter"` ConfigAutoReload bool `toml:"auto_reload"` @@ -85,7 +90,7 @@ type PipelineConfig struct { type FlowConfig struct { Heartbeat *HeartbeatConfig `toml:"heartbeat"` RateLimit *RateLimitConfig `toml:"rate_limit"` - Filters []FilterConfig `toml:"filters"` + Filters []FilterConfig `toml:"filters,omitempty"` Format *FormatConfig `toml:"format"` } @@ -105,9 +110,9 @@ type HeartbeatConfig struct { // FormatConfig is a polymorphic struct representing log entry formatting options type FormatConfig struct { Type string `toml:"type"` // "json", "txt", "raw" - Flags int64 `toml:"flags"` - TimestampFormat string `toml:"timestamp_format"` - SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell" + Flags int64 `toml:"flags,omitempty"` + TimestampFormat string `toml:"timestamp_format,omitempty"` + SanitizerPolicy string `toml:"sanitizer_policy,omitempty"` // "raw", "json", "txt", "shell" } // --- Rate Limit Options --- @@ -245,6 +250,7 @@ type ConsoleSinkOptions struct { Target string `toml:"target"` // "stdout", "stderr" BufferSize int64 `toml:"buffer_size"` Escape string `toml:"escape"` // control characters as : "auto" (on a terminal), "always", "never" + Color string `toml:"color"` // level names in color: "auto", "always", "never"; default: the top-level color } // FileSinkOptions defines settings for a file-based sink @@ -366,9 +372,10 @@ type AuthOptions struct { // --- TLS Options --- // TLSOptions is one shape for both roles. Listeners present cert_file and -// key_file and verify clients with client_auth and client_ca_file; dialers -// verify the server with ca_file and server_name and may present -// cert_file and key_file. +// key_file, or a certificate issued at startup (self_signed, or from the +// issuer files), and verify clients with client_auth and client_ca_file; +// dialers verify the server with ca_file and server_name, or pin_sha256, and +// may present cert_file and key_file. type TLSOptions struct { Enabled bool `toml:"enabled"` @@ -376,6 +383,14 @@ type TLSOptions struct { CertFile string `toml:"cert_file"` KeyFile string `toml:"key_file"` + // Listeners without cert_file: a certificate on a key made at startup, + // self-signed or signed by the issuer CA, for hosts plus the host option, + // os.Hostname(), localhost and the loopback addresses + SelfSigned bool `toml:"self_signed"` + IssuerCertFile string `toml:"issuer_cert_file"` + IssuerKeyFile string `toml:"issuer_key_file"` + Hosts []string `toml:"hosts"` + // Listener-side peer verification (mTLS) ClientAuth bool `toml:"client_auth"` ClientCAFile string `toml:"client_ca_file"` @@ -384,6 +399,7 @@ type TLSOptions struct { CAFile string `toml:"ca_file"` // empty = system trust store ServerName string `toml:"server_name"` // default: config host InsecureSkipVerify bool `toml:"insecure_skip_verify"` + PinSHA256 string `toml:"pin_sha256"` // sha256//BASE64 of the server key, ';' between several // Minimum protocol version: "1.2" | "1.3" (default "1.3") MinVersion string `toml:"min_version"` diff --git a/internal/config/loader.go b/internal/config/loader.go index d1c83f7..e1cd145 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -6,7 +6,6 @@ import ( "io/fs" "os" "path/filepath" - "strings" "github.com/lixenwraith/logwisp/internal/core" @@ -26,14 +25,18 @@ type Manager struct { specs []pipelineSpec // command-line or environment pipelines, kept across reloads } +// Args is a parsed command line; cmd/lw owns its grammar. +type Args struct { + File string // -c: the configuration file, "" for the environment or discovery + Specs []Spec // pipeline flags, in order + Overrides []string // --path=value settings, for lixenwraith/config +} + // Load reads the startup sources and validates the initial configuration. // Watching starts only when Watch is called after successful service startup. -func Load(args []string) (*Manager, error) { - configPath, isExplicit, configArgs, err := resolveConfigPath(args) - if err != nil { - return nil, err - } - specs, configArgs, err := takePipelineSpecs(configArgs) +func Load(args Args) (*Manager, error) { + configPath, isExplicit := resolveConfigPath(args.File) + specs, err := cliSpecs(args.Specs) if err != nil { return nil, err } @@ -53,7 +56,7 @@ func Load(args []string) (*Manager, error) { cfg, err := lconfig.NewBuilder(). WithTarget(initial). WithEnvPrefix("LOGWISP_"). - WithArgs(configArgs). + WithArgs(args.Overrides). WithFile(configPath). WithTypedValidator(func(cfg *Config) error { effective := *cfg // the builder keeps cfg; only the copy is replaced @@ -96,6 +99,17 @@ func (m *Manager) Snapshot() (*Config, error) { if err := m.usePipelines(cfg); err != nil { return nil, err } + // The top-level color is each console sink's default; the maps are fresh + for _, p := range cfg.Pipelines { + for i, s := range p.PluginSinks { + if _, set := s.Config["color"]; s.Type == "console" && !set { + if s.Config == nil { + p.PluginSinks[i].Config = map[string]any{} + } + p.PluginSinks[i].Config["color"] = cfg.Color + } + } + } if err := checkFileKeys(m.path); err != nil { return nil, fmt.Errorf("config file %q: %w", m.path, err) } @@ -145,6 +159,7 @@ func (m *Manager) Close() { m.config.StopAutoUpdate() } func defaults() *Config { return &Config{ StatusReporter: true, + Color: "auto", Logging: &LogConfig{ Output: "stderr", Level: "info", @@ -174,51 +189,28 @@ func pipeDefault(name string) PipelineConfig { } } -// resolveConfigPath consumes file-selection flags before schema CLI parsing. -// The last selection wins, and -- terminates option handling. -func resolveConfigPath(args []string) (path string, isExplicit bool, remaining []string, err error) { - remaining = make([]string, 0, len(args)) - for i := 0; i < len(args); i++ { - arg := args[i] - if arg == "--" { - remaining = append(remaining, args[i:]...) - break - } - switch { - case arg == "-c" || arg == "--config": - if i+1 == len(args) || args[i+1] == "" || strings.HasPrefix(args[i+1], "-") { - return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg) - } - i++ - path, isExplicit = args[i], true - case strings.HasPrefix(arg, "--config=") || strings.HasPrefix(arg, "-c="): - _, path, _ = strings.Cut(arg, "=") - if path == "" { - return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg) - } - isExplicit = true - default: - remaining = append(remaining, arg) - } - } - if isExplicit { - return path, true, remaining, nil +// resolveConfigPath picks the file: -c, then LOGWISP_CONFIG_FILE and +// LOGWISP_CONFIG_DIR, then the first of the user's and the working directory's +// logwisp.toml. Only the discovered default may be missing. +func resolveConfigPath(file string) (path string, isExplicit bool) { + if file != "" { + return file, true } if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" { path = configFile if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" { path = filepath.Join(configDir, configFile) } - return path, true, remaining, nil + return path, true } if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" { - return filepath.Join(configDir, "logwisp.toml"), true, remaining, nil + return filepath.Join(configDir, "logwisp.toml"), true } if homeDir, err := os.UserHomeDir(); err == nil { path = filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml") if _, err := os.Stat(path); err == nil { - return path, false, remaining, nil + return path, false } } - return "logwisp.toml", false, remaining, nil + return "logwisp.toml", false } diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go index 4c60653..fc0c29e 100644 --- a/internal/config/loader_test.go +++ b/internal/config/loader_test.go @@ -12,29 +12,20 @@ import ( "github.com/lixenwraith/logwisp/internal/testutil" ) -func TestResolveConfigArguments(t *testing.T) { +// -c wins over the environment, which wins over discovery; only the +// discovered default is not explicit, so only it may be missing. +func TestResolveConfigPath(t *testing.T) { isolateConfig(t) - for _, args := range [][]string{{"-c", "chosen.toml"}, {"--config", "chosen.toml"}, {"--config=chosen.toml"}, {"-c=chosen.toml"}, {"-c", "old.toml", "--config=chosen.toml"}} { - path, explicit, remaining, err := resolveConfigPath(append(args, "--quiet")) - if err != nil || path != "chosen.toml" || !explicit || !reflect.DeepEqual(remaining, []string{"--quiet"}) { - t.Fatalf("%v: %q %v %v %v", args, path, explicit, remaining, err) - } - } - for _, args := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}} { - if _, _, _, err := resolveConfigPath(args); err == nil { - t.Fatalf("missing path accepted: %v", args) - } - } - args := []string{"--", "-c", "ignored.toml"} - path, explicit, remaining, err := resolveConfigPath(args) - if err != nil || explicit || path != "logwisp.toml" || !reflect.DeepEqual(remaining, args) { - t.Fatalf("terminator ignored: %q %v %v %v", path, explicit, remaining, err) + if path, explicit := resolveConfigPath(""); path != "logwisp.toml" || explicit { + t.Fatalf("discovered default: %q %v", path, explicit) } t.Setenv("LOGWISP_CONFIG_DIR", "configs") t.Setenv("LOGWISP_CONFIG_FILE", "env.toml") - path, explicit, _, err = resolveConfigPath(nil) - if err != nil || !explicit || path != filepath.Join("configs", "env.toml") { - t.Fatalf("environment path: %q %v %v", path, explicit, err) + if path, explicit := resolveConfigPath(""); path != filepath.Join("configs", "env.toml") || !explicit { + t.Fatalf("environment path: %q %v", path, explicit) + } + if path, explicit := resolveConfigPath("chosen.toml"); path != "chosen.toml" || !explicit { + t.Fatalf("-c path: %q %v", path, explicit) } } @@ -43,7 +34,7 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) { testutil.WriteFile(t, "selected.toml", "config_file = \"ignored.toml\"\n[logging]\nlevel = \"error\"\n") t.Setenv("LOGGING_LEVEL", "invalid-bare-variable") t.Setenv("LOGWISP_LOGGING_LEVEL", "debug") - m, err := Load([]string{"--config", "selected.toml", "--logging.level=warn", "--quiet"}) + m, err := Load(Args{File: "selected.toml", Overrides: []string{"--logging.level=warn", "--quiet"}}) if err != nil { t.Fatal(err) } @@ -61,7 +52,7 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) { if _, aliased := second.Pipelines[0].PluginSources[0].Config["buffer_size"]; err != nil || second.Logging.Level != "warn" || aliased { t.Fatalf("snapshot aliases another result: %+v %v", second, err) } - env, err := Load([]string{"-c=selected.toml"}) + env, err := Load(Args{File: "selected.toml"}) if err != nil { t.Fatal(err) } @@ -74,19 +65,19 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) { func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) { isolateConfig(t) - m, err := Load(nil) + m, err := Load(Args{}) if err != nil { t.Fatal(err) } m.Close() - if _, err := Load([]string{"-c", "missing.toml"}); !errors.Is(err, ErrConfigNotFound) { + if _, err := Load(Args{File: "missing.toml"}); !errors.Is(err, ErrConfigNotFound) { t.Fatalf("missing explicit path: %v", err) } - if _, err := Load([]string{"--logging.file.max_size_mb=1.5"}); err == nil || errors.Is(err, ErrConfigNotFound) { + if _, err := Load(Args{Overrides: []string{"--logging.file.max_size_mb=1.5"}}); err == nil || errors.Is(err, ErrConfigNotFound) { t.Fatalf("invalid CLI hidden by missing default: %v", err) } t.Setenv("LOGWISP_STATUS_REPORTER", "invalid") - if _, err := Load(nil); err == nil || errors.Is(err, ErrConfigNotFound) { + if _, err := Load(Args{}); err == nil || errors.Is(err, ErrConfigNotFound) { t.Fatalf("invalid environment hidden by missing default: %v", err) } } @@ -94,7 +85,7 @@ func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) { func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) { isolateConfig(t) testutil.WriteFile(t, "reload.toml", "status_reporter = true\n") - m, err := Load([]string{"-c", "reload.toml"}) + m, err := Load(Args{File: "reload.toml"}) if err != nil { t.Fatal(err) } @@ -137,7 +128,7 @@ func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) { func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) { isolateConfig(t) testutil.WriteFile(t, "watched.toml", "auto_reload = true\n") - m, err := Load([]string{"-c", "watched.toml"}) + m, err := Load(Args{File: "watched.toml"}) if err != nil { t.Fatal(err) } @@ -181,7 +172,7 @@ server_name = "relay.example" type = "none" allow = ["one", "two"] `) - m, err := Load([]string{"-c", "plugins.toml"}) + m, err := Load(Args{File: "plugins.toml"}) if err != nil { t.Fatal(err) } @@ -221,7 +212,7 @@ type = "tcp" [pipelines.plugin_sinks.config] port = 9000 `+table+"\n") - m, err := Load([]string{"-c", "plugins.toml"}) + m, err := Load(Args{File: "plugins.toml"}) if err != nil { t.Fatal(err) } @@ -257,7 +248,7 @@ type = "tcp" [pipelines.plugin_sinks.config] port = 9000 `+body+"\n") - m, err := Load([]string{"-c", "typo.toml"}) + m, err := Load(Args{File: "typo.toml"}) if err != nil { t.Fatal(err) } @@ -280,7 +271,7 @@ func TestNoConfigFileMeansQuietLogging(t *testing.T) { if c.file != "" { testutil.WriteFile(t, "logwisp.toml", c.file) } - m, err := Load(nil) + m, err := Load(Args{}) if err != nil { t.Fatal(err) } @@ -292,7 +283,7 @@ func TestNoConfigFileMeansQuietLogging(t *testing.T) { } t.Setenv("LOGWISP_LOGGING_LEVEL", "debug") os.Remove("logwisp.toml") - m, err := Load(nil) + m, err := Load(Args{}) if err != nil { t.Fatal(err) } @@ -309,7 +300,7 @@ func TestFileStringIsOneListEntry(t *testing.T) { testutil.WriteFile(t, "logwisp.toml", "[[pipelines]]\nname = \"p\"\n"+ "[[pipelines.plugin_sources]]\nid = \"in\"\ntype = \"null\"\n[[pipelines.plugin_sinks]]\nid = \"out\"\ntype = \"null\"\n"+ "[[pipelines.flow.filters]]\ntype = \"exclude\"\npatterns = \"password=\\\\S{8,64}\"\n") - m, err := Load(nil) + m, err := Load(Args{}) if err != nil { t.Fatal(err) } @@ -319,3 +310,25 @@ func TestFileStringIsOneListEntry(t *testing.T) { t.Fatalf("patterns: %v %v", cfg.Pipelines[0].Flow.Filters, err) } } + +// The top-level color is the default of every console sink, which may set +// its own; other sinks do not take the key. +func TestTopLevelColorIsEachConsoleSinksDefault(t *testing.T) { + isolateConfig(t) + m, err := Load(Args{Specs: specs("sink", "console", "sink", "console,color=never", "sink", "null"), Overrides: []string{"--color=always"}}) + if err != nil { + t.Fatal(err) + } + defer m.Close() + cfg, err := m.Snapshot() + if err != nil { + t.Fatal(err) + } + var colors []any + for _, s := range cfg.Pipelines[0].PluginSinks { + colors = append(colors, s.Config["color"]) + } + if !reflect.DeepEqual(colors, []any{"always", "never", nil}) { + t.Fatalf("console sink colors: %v", colors) + } +} diff --git a/internal/config/preset.go b/internal/config/preset.go new file mode 100644 index 0000000..c0fbf9e --- /dev/null +++ b/internal/config/preset.go @@ -0,0 +1,332 @@ +package config + +import ( + "cmp" + "fmt" + "maps" + "net" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +// Preset is a pipeline built from a few keys: --preset NAME,key=value starts +// a pipeline with it. Later flags add sources, sinks and filters, or replace +// its format, rate limit and heartbeat. +type Preset struct { + Name, Summary string + Params []PresetParam + build func(p *PipelineConfig, v map[string]string) error +} + +// PresetParam is one key. A list takes several values, ',' between them or +// the key repeated; any other key takes one. +type PresetParam struct { + Name, Default, Help string + Required, List bool +} + +// Keys several presets share +var ( + pathParam = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"} + fromParam = PresetParam{Name: "from", Default: "end", Help: "start|end: where reading a file starts"} + formatParam = func(def string) PresetParam { return PresetParam{Name: "format", Default: def, Help: "raw|txt|json"} } + tlsParams = []PresetParam{ + {Name: "cert", Help: "certificate file, for tls=files"}, + {Name: "key", Help: "key file, for tls=files"}, + {Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"}, + {Name: "issuer_key", Help: "CA key file, for tls=issuer"}, + {Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"}, + } + transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"} +) + +var presets = []Preset{ + {"pipe", "Standard input to standard output, line for line: lw without a file", + []PresetParam{formatParam("raw")}, + func(p *PipelineConfig, v map[string]string) error { + stdio := pipeDefault("") + p.PluginSources, p.PluginSinks = stdio.PluginSources, stdio.PluginSinks + p.Flow.Format = &FormatConfig{Type: v["format"]} + return nil + }}, + {"tail", "Follow files to standard output, like tail -F", + []PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")}, + func(p *PipelineConfig, v map[string]string) error { + p.PluginSources, p.PluginSinks = pathOrStdin(v), pipeDefault("").PluginSinks + p.Flow.Format = &FormatConfig{Type: v["format"]} + return nil + }}, + {"serve", "Serve files or standard input as a live HTTP stream (SSE)", + append([]PresetParam{pathParam, fromParam, formatParam("json"), + {Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"}, + {Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"}, + {Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"}, + {Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, + {Name: "viewer", Default: "false", Help: "true: the built-in login and viewer pages, behind proxy"}, + }, tlsParams...), + func(p *PipelineConfig, v map[string]string) error { + p.PluginSources = pathOrStdin(v) + sink, err := listener(v, "off") + if err != nil { + return err + } + if v["users"] != "" { + sink["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]} + } + if v["proxy"] != "" { + auth, ok := sink["auth"].(map[string]any) + if !ok { + return fmt.Errorf("proxy needs users: browsers log in with SCRAM") + } + auth["trusted_proxies"] = list(v["proxy"]) + } + switch v["viewer"] { + case "true": + sink["login_page"], sink["viewer_page"] = true, true + case "false": + default: + return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"]) + } + p.Flow.Format = &FormatConfig{Type: v["format"]} + p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "http", Type: "http", Config: sink}) + return nil + }}, + {"edge", "Forward files or standard input to an aggregator over TLS, authenticated", + []PresetParam{pathParam, fromParam, + {Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"}, + transportParam, + {Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, + {Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"}, + {Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"}, + {Name: "user", Help: "SCRAM user (lw auth add-user on the aggregator)"}, + {Name: "password_file", Help: "file holding the SCRAM password"}, + {Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, + {Name: "key", Help: "client key file"}, + {Name: "node", Help: "origin label; default: this host's name"}, + }, + func(p *PipelineConfig, v map[string]string) error { + p.PluginSources = pathOrStdin(v) + typ, err := chainType(v["transport"]) + if err != nil { + return err + } + host, port, err := splitAddr("to", v["to"]) + if err != nil { + return err + } + tls := map[string]any{"enabled": true} + for key, option := range map[string]string{"ca": "ca_file", "pin": "pin_sha256", "server_name": "server_name", "cert": "cert_file", "key": "key_file"} { + if v[key] != "" { + tls[option] = v[key] + } + } + var auth map[string]any + switch { + case v["user"] != "" && v["password_file"] != "": + auth = map[string]any{"type": "scram", "username": v["user"], "password_file": v["password_file"]} + case v["user"] != "" || v["password_file"] != "": + return fmt.Errorf("user and password_file go together") + case v["cert"] != "" && v["key"] != "": + auth = map[string]any{"type": "mtls"} + default: + return fmt.Errorf("edge needs user and password_file, or cert and key: it never sends unauthenticated") + } + sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth} + if v["node"] != "" { + sink["node"] = v["node"] + } + p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "edge", Type: typ, Config: sink}) + return nil + }}, + {"aggregator", "Receive from edges over TLS, authenticated, to files or standard output", + append([]PresetParam{ + {Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"}, + transportParam, + {Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"}, + {Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"}, + {Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, + {Name: "out", Help: "directory for the received entries; default: standard output"}, + formatParam("json"), + }, tlsParams...), + func(p *PipelineConfig, v map[string]string) error { + typ, err := chainType(v["transport"]) + if err != nil { + return err + } + if v["tls"] == "off" { + return fmt.Errorf("tls=off: edges send credentials and logs, the aggregator always serves TLS") + } + src, err := listener(v, "self") + if err != nil { + return err + } + if v["client_ca"] != "" { + tls := src["tls"].(map[string]any) + tls["client_auth"], tls["client_ca_file"] = true, v["client_ca"] + } + switch { + case v["users"] != "": + src["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]} + case v["client_ca"] != "": + src["auth"] = map[string]any{"type": "mtls"} + default: + return fmt.Errorf("aggregator needs users or client_ca: it never receives unauthenticated") + } + p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src}) + p.PluginSinks = pipeDefault("").PluginSinks + if v["out"] != "" { + p.PluginSinks = []PluginSinkConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": v["out"], "name": "aggregate"}}} + } + p.Flow.Format = &FormatConfig{Type: v["format"]} + return nil + }}, +} + +// Presets lists the presets, for lw preset +func Presets() []Preset { return slices.Clone(presets) } + +// ExpandPreset builds the pipeline a preset makes of values, as --preset does +func ExpandPreset(name string, values map[string]string) (PipelineConfig, error) { + p := PipelineConfig{Flow: &FlowConfig{}} + each := map[string][]string{} + for k, v := range values { + each[k] = []string{v} + } + err := applyPreset(&p, name, each) + return p, err +} + +// applyPreset fills defaults and checks keys before the preset's build +func applyPreset(p *PipelineConfig, name string, values map[string][]string) error { + i := slices.IndexFunc(presets, func(r Preset) bool { return r.Name == name }) + if i < 0 { + var names []string + for _, r := range presets { + names = append(names, r.Name) + } + return fmt.Errorf("no preset %q (valid: %s)", name, strings.Join(names, ", ")) + } + r := presets[i] + v, keys := map[string]string{}, []string{} + for _, k := range r.Params { + v[k.Name] = k.Default + keys = append(keys, k.Name) + } + for k, each := range values { + j := slices.IndexFunc(r.Params, func(p PresetParam) bool { return p.Name == k }) + switch { + case j < 0: + return fmt.Errorf("preset %s has no key %q (valid: %s)", name, k, strings.Join(keys, ", ")) + case len(each) > 1 && !r.Params[j].List: + return fmt.Errorf("preset %s key %q takes one value", name, k) + } + v[k] = cmp.Or(strings.Join(each, ","), v[k]) + } + for _, k := range r.Params { + if k.Required && v[k.Name] == "" { + return fmt.Errorf("preset %s needs %s", name, k.Name) + } + } + p.Name = cmp.Or(p.Name, name) + return r.build(p, v) +} + +func pathOrStdin(v map[string]string) []PluginSourceConfig { + if v["path"] == "" { + return pipeDefault("").PluginSources + } + return []PluginSourceConfig{fileSource(v["path"], v["from"])} +} + +// fileSource follows path: a directory's files, or the files a glob or a +// file name matches in its directory. +func fileSource(path, from string) PluginSourceConfig { + dir, pattern := path, "*" + if fi, err := os.Stat(path); err != nil || !fi.IsDir() { + dir, pattern = filepath.Dir(path), filepath.Base(path) + } + return PluginSourceConfig{ID: "file", Type: "file", Config: map[string]any{"directory": dir, "pattern": pattern, "from": from}} +} + +// tlsModes is what each preset tls mode sets, and the preset keys it needs +// by the tls option each fills; made marks a certificate made at startup, +// which takes hosts. +var tlsModes = map[string]struct { + set map[string]any + needs map[string]string + made bool +}{ + "off": {}, + "self": {set: map[string]any{"self_signed": true}, made: true}, + "issuer": {needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true}, + "files": {needs: map[string]string{"cert": "cert_file", "key": "key_file"}}, +} + +// listener starts a listening plugin's options: host, port and tls, whose +// mode decides which certificate keys apply. +func listener(v map[string]string, def string) (map[string]any, error) { + host, port, err := splitAddr("listen", v["listen"]) + if err != nil { + return nil, err + } + opts := map[string]any{"host": host, "port": port} + mode := cmp.Or(v["tls"], def) + m, ok := tlsModes[mode] + if !ok { + return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(slices.Sorted(maps.Keys(tlsModes)), ", ")) + } + for _, other := range tlsModes { + for k := range other.needs { + if _, needed := m.needs[k]; needed && v[k] == "" { + return nil, fmt.Errorf("tls=%s needs %s", mode, k) + } else if !needed && v[k] != "" { + return nil, fmt.Errorf("%s does not apply to tls=%s", k, mode) + } + } + } + if v["hosts"] != "" && !m.made { + return nil, fmt.Errorf("hosts does not apply to tls=%s", mode) + } + if mode == "off" { + return opts, nil + } + tls := map[string]any{"enabled": true} + maps.Copy(tls, m.set) + for k, option := range m.needs { + tls[option] = v[k] + } + if v["hosts"] != "" { + tls["hosts"] = list(v["hosts"]) + } + opts["tls"] = tls + return opts, nil +} + +func splitAddr(key, addr string) (string, int64, error) { + host, p, err := net.SplitHostPort(addr) + port, perr := strconv.ParseUint(p, 10, 16) + if err != nil || perr != nil || port == 0 { + return "", 0, fmt.Errorf("%s %q: want HOST:PORT, an IPv6 address in brackets", key, addr) + } + return host, int64(port), nil +} + +func chainType(transport string) (string, error) { + if transport != "tcp" && transport != "http" { + return "", fmt.Errorf("transport %q (valid: tcp, http)", transport) + } + return transport + "_chain", nil +} + +func list(s string) []any { + var out []any + for e := range strings.SplitSeq(s, ",") { + if e = strings.TrimSpace(e); e != "" { + out = append(out, e) + } + } + return out +} diff --git a/internal/config/preset_test.go b/internal/config/preset_test.go new file mode 100644 index 0000000..b4758bb --- /dev/null +++ b/internal/config/preset_test.go @@ -0,0 +1,63 @@ +package config + +import ( + "reflect" + "strings" + "testing" +) + +// A preset starts its pipeline and names it unless --pipeline did; later +// sinks add to its own, a later format replaces its own. +func TestPresetStartsAndNamesItsPipeline(t *testing.T) { + isolateConfig(t) + got := loadPipelines(t, "preset", "tail,path=/var/log/app.log", "sink", "null", "format", "json", + "pipeline", "mine", "preset", "pipe") + want := []PipelineConfig{{ + Name: "tail", + Flow: &FlowConfig{Format: &FormatConfig{Type: "json"}}, + PluginSources: []PluginSourceConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": "/var/log", "pattern": "app.log", "from": "end"}}}, + PluginSinks: []PluginSinkConfig{ + {ID: "stdout", Type: "console", Config: map[string]any{"color": "auto"}}, + {ID: "null", Type: "null", Config: map[string]any{}}, + }, + }, { + Name: "mine", + Flow: &FlowConfig{Format: &FormatConfig{Type: "raw"}}, + PluginSources: []PluginSourceConfig{{ID: "stdin", Type: "console", Config: map[string]any{}}}, + PluginSinks: []PluginSinkConfig{{ID: "stdout", Type: "console", Config: map[string]any{"color": "auto"}}}, + }} + if !reflect.DeepEqual(got, want) { + t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want) + } + if _, err := Load(Args{Specs: specs("sink", "null", "preset", "pipe")}); err == nil || !strings.Contains(err.Error(), "must start its pipeline") { + t.Fatalf("preset after a sink: %v", err) + } +} + +// Keys are checked against the preset's own, required ones enforced, and +// the edge and aggregator refuse to run without authentication. +func TestPresetKeysAreChecked(t *testing.T) { + isolateConfig(t) + for spec, want := range map[string]string{ + "nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`, + "tail,path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`, + "tail": "preset tail needs path", + "tail,path=x,format.type=json": "preset keys do not nest", + "tail,path=x,path=y": `key "path" takes one value`, + "serve,listen=8080": `listen "8080": want HOST:PORT`, + "serve,tls=self,cert=c": "cert does not apply to tls=self", + "serve,tls=files,cert=c": "tls=files needs key", + "serve,viewer=yes": `viewer "yes"`, + "serve,proxy=127.0.0.1": "proxy needs users", + "edge,to=agg:9000": "it never sends unauthenticated", + "edge,to=agg:9000,user=u": "user and password_file go together", + "edge,to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`, + "aggregator,users=u,tls=off": "the aggregator always serves TLS", + "aggregator": "it never receives unauthenticated", + } { + _, err := Load(Args{Specs: specs("preset", spec)}) + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: err = %v, want %s", spec, err, want) + } + } +} diff --git a/internal/config/spec.go b/internal/config/spec.go index e38fb26..c4e19de 100644 --- a/internal/config/spec.go +++ b/internal/config/spec.go @@ -16,11 +16,28 @@ type specKind struct { flag string typed bool // SPEC starts with TYPE many bool // repeatable within a pipeline; the environment adds _1.._N + first bool // starts its pipeline, and names it by TYPE unless --pipeline did apply func(p *PipelineConfig, typ string, opts map[string]any) error } var specKinds = []specKind{ - {"source", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + values := map[string][]string{} + for k, v := range opts { + switch v := v.(type) { + case string: + values[k] = []string{v} + case []any: // a repeated key + for _, e := range v { + values[k] = append(values[k], e.(string)) + } + default: + return fmt.Errorf("preset keys do not nest: %q", k) + } + } + return applyPreset(p, typ, values) + }}, + {"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { ids := make([]string, len(p.PluginSources)) for i, s := range p.PluginSources { ids[i] = s.ID @@ -29,7 +46,7 @@ var specKinds = []specKind{ p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts}) return err }}, - {"sink", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { ids := make([]string, len(p.PluginSinks)) for i, s := range p.PluginSinks { ids[i] = s.ID @@ -38,24 +55,24 @@ var specKinds = []specKind{ p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts}) return err }}, - {"filter", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { opts["type"] = typ f := FilterConfig{} err := Scan(opts, &f) p.Flow.Filters = append(p.Flow.Filters, f) return err }}, - {"format", true, false, func(p *PipelineConfig, typ string, opts map[string]any) error { + {"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any) error { opts["type"] = typ p.Flow.Format = &FormatConfig{} return Scan(opts, p.Flow.Format) }}, // Naming a stage turns it on: the file defaults (pass, disabled) would not. - {"rate-limit", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { + {"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"} return Scan(opts, p.Flow.RateLimit) }}, - {"heartbeat", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { + {"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error { p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true} return Scan(opts, p.Flow.Heartbeat) }}, @@ -68,33 +85,36 @@ type pipelineSpec struct { value string } -// takePipelineSpecs consumes the pipeline flags, which lixenwraith/config -// would report as unrecognized. -func takePipelineSpecs(args []string) (specs []pipelineSpec, rest []string, err error) { - for i := 0; i < len(args); i++ { - if args[i] == "--" { - return specs, append(rest, args[i:]...), nil - } - name, value, inline := strings.Cut(args[i], "=") +// Spec is one pipeline flag: its name without dashes, and its value. +type Spec struct{ Flag, Value string } + +// SpecFlags names the pipeline flags, for the command-line parser. +func SpecFlags() []string { + flags := []string{"pipeline"} + for _, k := range specKinds { + flags = append(flags, k.flag) + } + return flags +} + +// cliSpecs looks up the kind of each command-line spec. +func cliSpecs(in []Spec) ([]pipelineSpec, error) { + var specs []pipelineSpec + for _, s := range in { var kind *specKind - if name != "--pipeline" { - k := slices.IndexFunc(specKinds, func(k specKind) bool { return "--"+k.flag == name }) + if s.Flag != "pipeline" { + k := slices.IndexFunc(specKinds, func(k specKind) bool { return k.flag == s.Flag }) if k < 0 { - rest = append(rest, args[i]) - continue + return nil, fmt.Errorf("--%s is no pipeline flag", s.Flag) } kind = &specKinds[k] } - if !inline && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") { - i++ - value = args[i] + if s.Value == "" { + return nil, fmt.Errorf("--%s requires a value", s.Flag) } - if value == "" { - return nil, nil, fmt.Errorf("%s requires a value", name) - } - specs = append(specs, pipelineSpec{name, kind, value}) + specs = append(specs, pipelineSpec{"--" + s.Flag, kind, s.Value}) } - return specs, rest, nil + return specs, nil } // envPipelineSpecs reads the one-pipeline environment form. An empty variable @@ -131,48 +151,58 @@ func envPipelineSpecs() []pipelineSpec { } // buildPipelines turns specs, in order, into pipelines; specs before the first -// --pipeline go to one named "cli". A pipeline without a source reads stdin, -// one without a sink writes stdout. Every call returns fresh maps. +// --pipeline go to one named "cli". Every call returns fresh maps. func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { var pipelines []PipelineConfig var seen map[*specKind]bool for _, s := range specs { - if s.kind == nil || pipelines == nil { - name := "cli" - if s.kind == nil { - name = s.value - } - pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}}) + if s.kind == nil { + pipelines = append(pipelines, PipelineConfig{Name: s.value, Flow: &FlowConfig{}}) seen = map[*specKind]bool{} - if s.kind == nil { - continue - } + continue } - p := &pipelines[len(pipelines)-1] - if seen[s.kind] && !s.kind.many { - return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name) - } - seen[s.kind] = true typ, opts, err := parseSpec(s.value, s.kind.typed) if err != nil { return nil, fmt.Errorf("%s %w", s.name, err) } + if pipelines == nil { + name := "cli" + if s.kind.first { + name = typ + } + pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}}) + seen = map[*specKind]bool{} + } + p := &pipelines[len(pipelines)-1] + switch { + case seen[s.kind] && !s.kind.many: + return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name) + case s.kind.first && len(seen) > 0: + return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value) + } + seen[s.kind] = true if err := s.kind.apply(p, typ, opts); err != nil { return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err) } } for i := range pipelines { - pipe := pipeDefault(pipelines[i].Name) - if len(pipelines[i].PluginSources) == 0 { - pipelines[i].PluginSources = pipe.PluginSources - } - if len(pipelines[i].PluginSinks) == 0 { - pipelines[i].PluginSinks = pipe.PluginSinks - } + useStdio(&pipelines[i]) } return pipelines, nil } +// useStdio makes a pipeline without a source read stdin, and one without a +// sink write stdout, as a filter does. +func useStdio(p *PipelineConfig) { + pipe := pipeDefault(p.Name) + if len(p.PluginSources) == 0 { + p.PluginSources = pipe.PluginSources + } + if len(p.PluginSinks) == 0 { + p.PluginSinks = pipe.PluginSinks + } +} + // parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes // a list, and a backslash escapes ',', '=' and '\'. Errors name the part. func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) { diff --git a/internal/config/spec_test.go b/internal/config/spec_test.go index a0699f0..2a45450 100644 --- a/internal/config/spec_test.go +++ b/internal/config/spec_test.go @@ -10,9 +10,18 @@ import ( "github.com/lixenwraith/logwisp/internal/testutil" ) -func loadPipelines(t *testing.T, args ...string) []PipelineConfig { +// specs pairs flags with values: specs("sink", "null") is --sink null +func specs(pairs ...string) []Spec { + var out []Spec + for i := 0; i < len(pairs); i += 2 { + out = append(out, Spec{pairs[i], pairs[i+1]}) + } + return out +} + +func loadPipelines(t *testing.T, pairs ...string) []PipelineConfig { t.Helper() - m, err := Load(args) + m, err := Load(Args{Specs: specs(pairs...)}) if err != nil { t.Fatal(err) } @@ -27,15 +36,14 @@ func loadPipelines(t *testing.T, args ...string) []PipelineConfig { func TestPipelineSpecGrammar(t *testing.T) { isolateConfig(t) got := loadPipelines(t, - "--source", `file,directory=/var/log/a\,b,pattern=*.log`, - `--source=file,id=app,directory=x\=y`, - "--source", `file,directory=c:\\logs\\`, - "--sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c", - "--filter", "include,patterns=ERROR,patterns=WARN", - "--filter", `exclude,patterns=\d{1\,3}`, - "--format", `txt,timestamp_format=Jan 2\, 2006`, - "--pipeline", "b", "--source", "null", "--sink", "null", "--sink", "null", "--heartbeat", "interval_ms=1000", - "--", "--source", "random") + "source", `file,directory=/var/log/a\,b,pattern=*.log`, + "source", `file,id=app,directory=x\=y`, + "source", `file,directory=c:\\logs\\`, + "sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c", + "filter", "include,patterns=ERROR,patterns=WARN", + "filter", `exclude,patterns=\d{1\,3}`, + "format", `txt,timestamp_format=Jan 2\, 2006`, + "pipeline", "b", "source", "null", "sink", "null", "sink", "null", "heartbeat", "interval_ms=1000") want := []PipelineConfig{{ Name: "cli", Flow: &FlowConfig{ @@ -67,17 +75,17 @@ func TestPipelineSpecGrammar(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want) } - for want, args := range map[string][]string{ - `--sink http,port: missing "="`: {"--sink", "http,port"}, - `--source directory=/x: missing TYPE`: {"--source", "directory=/x"}, - `--source file,type=x: TYPE already sets "type"`: {"--source", "file,type=x"}, - `--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"--sink", "http,tls=1,tls.cert_file=c"}, - `--sink requires a value`: {"--sink="}, - `--format txt: pipeline "cli" already has one`: {"--format", "json", "--format", "txt"}, - `--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"--rate-limit", "rate=1,polcy=drop"}, + for want, pairs := range map[string][]string{ + `--sink http,port: missing "="`: {"sink", "http,port"}, + `--source directory=/x: missing TYPE`: {"source", "directory=/x"}, + `--source file,type=x: TYPE already sets "type"`: {"source", "file,type=x"}, + `--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http,tls=1,tls.cert_file=c"}, + `--sink requires a value`: {"sink", ""}, + `--format txt: pipeline "cli" already has one`: {"format", "json", "format", "txt"}, + `--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"rate-limit", "rate=1,polcy=drop"}, } { - if _, err := Load(args); err == nil || err.Error() != want { - t.Errorf("%q: err = %v, want %s", args, err, want) + if _, err := Load(Args{Specs: specs(pairs...)}); err == nil || err.Error() != want { + t.Errorf("%q: err = %v, want %s", pairs, err, want) } } } @@ -86,7 +94,7 @@ func TestPipelineSpecGrammar(t *testing.T) { // must not split into several, wider entries. func TestSpecValueIsOneListEntry(t *testing.T) { isolateConfig(t) - sink := loadPipelines(t, "--source", "null", "--sink", + sink := loadPipelines(t, "source", "null", "sink", `tcp,auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0] var opts TCPSinkOptions if err := Scan(sink.Config, &opts); err != nil { @@ -115,7 +123,7 @@ func TestEnvironmentPipeline(t *testing.T) { } { t.Setenv(name, value) } - m, err := Load(nil) + m, err := Load(Args{}) if err != nil { t.Fatal(err) } @@ -136,7 +144,7 @@ func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) { isolateConfig(t) t.Setenv("LOGWISP_SOURCE", "random") t.Setenv("LOGWISP_SINK", "console") - got := loadPipelines(t, "--source", "null", "--sink", "null") + got := loadPipelines(t, "source", "null", "sink", "null") if len(got) != 1 || len(got[0].PluginSources) != 1 || got[0].PluginSources[0].Type != "null" || len(got[0].PluginSinks) != 1 || got[0].PluginSinks[0].Type != "null" { t.Fatalf("environment specs mixed into command-line pipelines: %+v", got) @@ -147,7 +155,7 @@ func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) { func TestSpecPipelinesReplaceFilePipelines(t *testing.T) { isolateConfig(t) testutil.WriteFile(t, "file.toml", "[logging]\nlevel = \"warn\"\n[[pipelines]]\nname = \"file\"\n") - m, err := Load([]string{"-c", "file.toml", "--source", "null", "--sink", "null", "--status_reporter=false"}) + m, err := Load(Args{File: "file.toml", Specs: specs("source", "null", "sink", "null"), Overrides: []string{"--status_reporter=false"}}) if err != nil { t.Fatal(err) } @@ -165,12 +173,12 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) { // stdout, as a filter does; two pipelines cannot both read stdin. func TestSpecPipelinesDefaultToStdio(t *testing.T) { isolateConfig(t) - got := loadPipelines(t, "--filter", "include,patterns=ERROR") + got := loadPipelines(t, "filter", "include,patterns=ERROR") if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" || len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" { t.Fatalf("pipeline: %+v", p) } - _, err := Load([]string{"--pipeline", "a", "--sink", "null", "--pipeline", "b", "--sink", "null"}) + _, err := Load(Args{Specs: specs("pipeline", "a", "sink", "null", "pipeline", "b", "sink", "null")}) if err == nil || !strings.Contains(err.Error(), `console source already reads stdin in pipeline "a"`) { t.Fatalf("second stdin reader: %v", err) } @@ -181,7 +189,7 @@ func TestSpecPipelinesDefaultToStdio(t *testing.T) { // removal fails the reload instead of keeping the removed file's values. func TestReloadKeepsSpecPipelines(t *testing.T) { isolateConfig(t) - m, err := Load([]string{"--source", "random,special=true", "--sink", "null"}) + m, err := Load(Args{Specs: specs("source", "random,special=true", "sink", "null")}) if err != nil { t.Fatal(err) } diff --git a/internal/config/validate.go b/internal/config/validate.go index 3263266..e4af7f8 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -34,6 +34,10 @@ func ValidateConfig(cfg *Config) error { names[p.Name] = struct{}{} } + if err := lconfig.OneOf("auto", "always", "never")(cfg.Color); err != nil { + return fmt.Errorf("color: %w", err) + } + if err := validateLogConfig(cfg.Logging); err != nil { return fmt.Errorf("logging: %w", err) } diff --git a/internal/core/const.go b/internal/core/const.go index eaf8769..5938a91 100644 --- a/internal/core/const.go +++ b/internal/core/const.go @@ -43,6 +43,9 @@ const ( ShutdownTimeout = 10 * time.Second + // Bounds the flush of a network sink's queues at Stop + SinkFlushTimeout = 2 * time.Second + ConfigReloadTimeout = 30 * time.Second LoggerShutdownTimeout = 2 * time.Second diff --git a/internal/sink/console/console.go b/internal/sink/console/console.go index 96cf6b4..cc28b60 100644 --- a/internal/sink/console/console.go +++ b/internal/sink/console/console.go @@ -18,8 +18,10 @@ import ( "github.com/lixenwraith/logwisp/internal/session" "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/color" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" + "github.com/lixenwraith/terminal/inline" "golang.org/x/term" ) @@ -38,8 +40,9 @@ type ConsoleSink struct { session *session.Session // Configuration - config *config.ConsoleSinkOptions - escape bool + config *config.ConsoleSinkOptions + escape bool + painter *inline.Printer // paints level names; nil when color is off // Application input chan core.TransportEvent @@ -63,8 +66,23 @@ const ( DefaultConsoleTarget = "stdout" DefaultConsoleBufferSize = 1000 DefaultConsoleEscape = "auto" + DefaultConsoleColor = "auto" ) +// levelColors paints the first of a level's names in an entry, ANSI 16 so +// the terminal's theme picks the shades and a text console renders them. +// Names run longest first; cyan, not blue, stays readable on black. +var levelColors = map[string]struct { + names []string + style inline.Style +}{ + "ERROR": {[]string{"ERROR", "FATAL", "ERR"}, inline.FgANSI(color.ANSIRed).Bold()}, + "WARN": {[]string{"WARNING", "WARN"}, inline.FgANSI(color.ANSIYellow)}, + "INFO": {[]string{"INFO", "INF"}, inline.FgANSI(color.ANSIGreen)}, + "DEBUG": {[]string{"DEBUG", "DBG"}, inline.FgANSI(color.ANSICyan)}, + "TRACE": {[]string{"TRACE"}, inline.FgANSI(color.ANSIBrightBlack)}, +} + // NewConsoleSinkPlugin creates a console sink through plugin factory func NewConsoleSinkPlugin( id string, @@ -72,7 +90,7 @@ func NewConsoleSinkPlugin( logger *log.Logger, proxy *session.Proxy, ) (sink.Sink, error) { - opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape} + opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape, Color: DefaultConsoleColor} // Scan config map into struct if err := config.Scan(configMap, opts); err != nil { @@ -85,6 +103,9 @@ func NewConsoleSinkPlugin( if err := lconfig.OneOf("auto", "always", "never")(opts.Escape); err != nil { return nil, fmt.Errorf("escape: %w", err) } + if err := lconfig.OneOf("auto", "always", "never")(opts.Color); err != nil { + return nil, fmt.Errorf("color: %w", err) + } output := os.Stdout if opts.Target == "stderr" { @@ -95,17 +116,29 @@ func NewConsoleSinkPlugin( opts.BufferSize = DefaultConsoleBufferSize } + // inline decides auto: a terminal, NO_COLOR unset, TERM not dumb. Paint + // returns its text unchanged exactly when color is off. + printer := inline.New(output) + if opts.Color != "auto" { + printer.SetColor(opts.Color == "always") + } + var painter *inline.Printer + if printer.Paint("x", inline.Style{}) != "x" { + painter = printer + } + // Create and return plugin instance cs := &ConsoleSink{ - id: id, - proxy: proxy, - config: opts, - escape: opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())), - input: make(chan core.TransportEvent, opts.BufferSize), - output: output, - done: make(chan struct{}), - exited: make(chan struct{}), - logger: logger, + id: id, + proxy: proxy, + config: opts, + escape: opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())), + painter: painter, + input: make(chan core.TransportEvent, opts.BufferSize), + output: output, + done: make(chan struct{}), + exited: make(chan struct{}), + logger: logger, } cs.lastProcessed.Store(time.Time{}) @@ -124,6 +157,7 @@ func NewConsoleSinkPlugin( "instance_id", id, "target", opts.Target, "escape", cs.escape, + "color", painter != nil, ) return cs, nil @@ -187,6 +221,7 @@ func (cs *ConsoleSink) GetStats() sink.SinkStats { "target": cs.config.Target, "buffer_size": cs.config.BufferSize, "escape": cs.escape, + "color": cs.painter != nil, }, } } @@ -224,6 +259,9 @@ func (cs *ConsoleSink) write(event core.TransportEvent) { if cs.escape { payload = escapeControls(payload) } + if cs.painter != nil { + payload = paintLevel(cs.painter, payload, event.Entry.Level) + } if _, err := cs.output.Write(payload); err != nil { cs.logger.Error("msg", "Failed to write to console", "component", "console_sink", @@ -235,6 +273,42 @@ func (cs *ConsoleSink) write(event core.TransportEvent) { cs.lastProcessed.Store(time.Now()) } +// paintLevel colors the first whole-word, case-insensitive occurrence of a +// name of level, after escaping: the escape sequences are the sink's own. +func paintLevel(p *inline.Printer, payload []byte, level string) []byte { + lc, ok := levelColors[level] + if !ok { + return payload + } + for _, name := range lc.names { + for i := 0; i+len(name) <= len(payload); i++ { + j := i + len(name) + if !nameAt(payload[i:j], name) || i > 0 && word(payload[i-1]) || j < len(payload) && word(payload[j]) { + continue + } + out := make([]byte, 0, len(payload)+16) + out = append(out, payload[:i]...) + out = append(out, p.Paint(string(payload[i:j]), lc.style)...) + return append(out, payload[j:]...) + } + } + return payload +} + +// nameAt compares ASCII letters without case; name is upper case +func nameAt(b []byte, name string) bool { + for k := range len(name) { + if c := b[k]; c != name[k] && c != name[k]+'a'-'A' { + return false + } + } + return true +} + +func word(c byte) bool { + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_' +} + // escapeControls writes what a terminal would act on or reorder (C0 and C1 // controls but tab, DEL, bidi controls, line separators, invalid UTF-8) as // , keeping the final newline: a log line cannot drive the terminal. diff --git a/internal/sink/console/console_test.go b/internal/sink/console/console_test.go index 695022e..25c0005 100644 --- a/internal/sink/console/console_test.go +++ b/internal/sink/console/console_test.go @@ -1,6 +1,11 @@ package console -import "testing" +import ( + "io" + "testing" + + "github.com/lixenwraith/terminal/inline" +) // Everything a terminal would act on is escaped; tabs and the final newline, // which it only lays out, are kept. @@ -23,3 +28,22 @@ func TestEscapeControlsKeepsOnlyLayout(t *testing.T) { } } } + +// The entry's level is painted where its line first names it as a whole +// word, in any case, longest name first; other words and levels stay plain. +func TestPaintLevelColorsTheFirstWholeName(t *testing.T) { + p := inline.New(io.Discard) + p.SetColor(true) + red, yellow := "\x1b[0;1;31m", "\x1b[0;33m" + for _, c := range []struct{ in, level, want string }{ + {"2026 ERROR boom ERROR\n", "ERROR", "2026 " + red + "ERROR\x1b[0m boom ERROR\n"}, + {"[warning] disk\n", "WARN", "[" + yellow + "warning\x1b[0m] disk\n"}, + {"terror_x ERR:1\n", "ERROR", "terror_x " + red + "ERR\x1b[0m:1\n"}, + {"INFOS informal\n", "INFO", "INFOS informal\n"}, + {"ERROR without level\n", "", "ERROR without level\n"}, + } { + if got := string(paintLevel(p, []byte(c.in), c.level)); got != c.want { + t.Errorf("%q %s: got %q, want %q", c.in, c.level, got, c.want) + } + } +} diff --git a/internal/sink/http/http.go b/internal/sink/http/http.go index 187828a..01cd70d 100644 --- a/internal/sink/http/http.go +++ b/internal/sink/http/http.go @@ -43,7 +43,6 @@ const ( DefaultHTTPStreamPath = "/stream" DefaultHTTPStatusPath = "/status" HTTPReadHeaderTimeout = 10 * time.Second - HTTPShutdownTimeout = 2 * time.Second ) // HTTPSink streams log entries via Server-Sent Events @@ -80,8 +79,14 @@ type HTTPSink struct { auth *authz.Policy web map[string]http.Handler // proxy mode: GET /auth/... browser files - // Runtime + // Runtime. Stop closes flush; the broker then moves what remains of its + // input to tail and closes flushed; each stream writes its queue, tail + // and the disconnect event on its own, until flushBy. done chan struct{} + flush chan struct{} + flushed chan struct{} + flushBy time.Time + tail [][]byte // read only once flushed is closed stopOnce sync.Once wg sync.WaitGroup startTime time.Time @@ -145,7 +150,7 @@ func NewHTTPSinkPlugin( if opts.ClientBufferSize <= 0 { opts.ClientBufferSize = DefaultHTTPClientBufferSize } - tlsCfg, err := tlsx.Server(opts.TLS) + tlsCfg, err := tlsx.Server(opts.TLS, opts.Host) if err != nil { return nil, err } @@ -174,6 +179,8 @@ func NewHTTPSinkPlugin( network: network, input: make(chan core.TransportEvent, opts.BufferSize), done: make(chan struct{}), + flush: make(chan struct{}), + flushed: make(chan struct{}), logger: logger, clients: make(map[uint64]*sseClient), writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond, @@ -276,7 +283,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error { h.startTime = time.Now() h.wg.Add(1) - go h.brokerLoop(ctx) + go h.brokerLoop() serve := h.server.Serve if h.tlsConfig != nil { @@ -325,18 +332,21 @@ func (h *HTTPSink) Stop() { } // shutdown funnels ctx-cancel and Stop() teardown through a single path. -// done is closed first so SSE handlers exit and Shutdown can complete; -// Server.Close force-closes any handler stalled in a deadline-free write. +// Server.Shutdown refuses new streams and waits for the open ones, which end +// once they wrote what is queued, within sink.FlushBound; Server.Close cuts a +// stream stalled past it. func (h *HTTPSink) shutdown() { h.stopOnce.Do(func() { - close(h.done) if h.server != nil { - sctx, cancel := context.WithTimeout(context.Background(), HTTPShutdownTimeout) + h.flushBy = time.Now().Add(sink.FlushBound(h.writeTimeout)) + ctx, cancel := context.WithDeadline(context.Background(), h.flushBy) defer cancel() - if err := h.server.Shutdown(sctx); err != nil { + close(h.flush) + if err := h.server.Shutdown(ctx); err != nil { h.server.Close() } } + close(h.done) }) } @@ -356,19 +366,22 @@ func (h *HTTPSink) removeClient(id uint64) { } // brokerLoop fans out transport events to all client queues, non-blocking, -// and evicts clients whose sessions were idle-expired by the session manager -func (h *HTTPSink) brokerLoop(ctx context.Context) { +// and evicts clients whose sessions were idle-expired by the session manager, +// until flush, which shutdown closes on Stop and on context cancellation. +func (h *HTTPSink) brokerLoop() { defer h.wg.Done() for { select { - case <-ctx.Done(): + case <-h.flush: // first: a select with both ready picks at random + h.flushInput() return - case <-h.done: + default: + } + select { + case <-h.flush: + h.flushInput() return - case event, ok := <-h.input: - if !ok { - return - } + case event := <-h.input: h.totalProcessed.Add(1) h.lastProcessed.Store(time.Now()) @@ -395,6 +408,13 @@ func (h *HTTPSink) brokerLoop(ctx context.Context) { } } +// flushInput moves what remains of the input to tail for the streams +func (h *HTTPSink) flushInput() { + h.tail = sink.Drain(h.input) + h.totalProcessed.Add(uint64(len(h.tail))) + close(h.flushed) +} + // handleStream serves one client's SSE stream func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { if h.config.MaxConnections > 0 && h.activeClients.Load() >= h.config.MaxConnections { @@ -483,6 +503,13 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { idle := time.NewTicker(h.keepalive) defer idle.Stop() + send := func(payload []byte) bool { + if writeSSE(w, payload) != nil || rc.Flush() != nil { + return false + } + h.proxy.UpdateActivity(sess.ID) + return true + } clientGone := r.Context().Done() for { select { @@ -491,13 +518,9 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { return // broker evicted (stale session) } h.armWrite(rc) - if err := writeSSE(w, payload); err != nil { + if !send(payload) { return } - if err := rc.Flush(); err != nil { - return - } - h.proxy.UpdateActivity(sess.ID) case <-idle.C: h.armWrite(rc) if _, err := fmt.Fprint(w, ":\n\n"); err != nil { @@ -509,7 +532,20 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { h.proxy.UpdateActivity(sess.ID) case <-clientGone: return - case <-h.done: + case <-h.flushed: + // Nothing more will be queued: write the queue and the tail, then + // say why the stream ends + _ = rc.SetWriteDeadline(h.flushBy) + for len(c.send) > 0 { + if payload, ok := <-c.send; !ok || !send(payload) { + return + } + } + for _, payload := range h.tail { + if !send(payload) { + return + } + } fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n") rc.Flush() return diff --git a/internal/sink/http/http_test.go b/internal/sink/http/http_test.go index ac1deff..0d254a7 100644 --- a/internal/sink/http/http_test.go +++ b/internal/sink/http/http_test.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "fmt" "io" "maps" "net/http" @@ -319,3 +320,73 @@ func TestSSEFramesEveryLineAsData(t *testing.T) { t.Fatalf("framed %q, want %q", got, want) } } + +// Stop writes what is queued, in the sink and per stream, then the +// disconnect event, to a client that keeps reading, whatever another that +// stopped reading does; Stop returns at the flush bound. +func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) { + const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold + httpSink, _ := newTestHTTPSink(t, map[string]any{"buffer_size": int64(n), "client_buffer_size": int64(64), "write_timeout_ms": int64(500)}) + client, baseURL := serveTestHTTPSink(t, httpSink) + var scanners []*bufio.Scanner + for range 2 { + resp, err := client.Get(baseURL + "/stream") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + scanner := bufio.NewScanner(resp.Body) + scanner.Buffer(nil, 64*1024) + for scanner.Scan() && scanner.Text() != "event: connected" { + } + scanners = append(scanners, scanner) + } + for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) { + httpSink.clientsMu.Lock() + registered := len(httpSink.clients) + httpSink.clientsMu.Unlock() + if registered == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("streams never registered") + } + } + type result struct { + lines int + disconnect bool + } + done := make(chan result) + go func() { + var r result + for scanners[0].Scan() { + switch line := scanners[0].Text(); { + case strings.HasPrefix(line, "data: line "): + r.lines++ + case line == "event: disconnect": + r.disconnect = true + } + } + done <- r + }() + // Held, the lock keeps the input queued until Stop has begun its flush + httpSink.clientsMu.Lock() + padding := strings.Repeat("x", 8192) + for i := range n { + httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)} + } + stopped := make(chan time.Duration) + go func() { + start := time.Now() + httpSink.Stop() + stopped <- time.Since(start) + }() + time.Sleep(100 * time.Millisecond) + httpSink.clientsMu.Unlock() + if d := <-stopped; d > 1500*time.Millisecond { + t.Errorf("Stop took %v past a 500 ms flush bound", d) + } + if r := <-done; r.lines != n || !r.disconnect { + t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect) + } +} diff --git a/internal/sink/sink.go b/internal/sink/sink.go index 14fd4fd..2d7aa31 100644 --- a/internal/sink/sink.go +++ b/internal/sink/sink.go @@ -35,3 +35,27 @@ type SinkStats struct { LastProcessed time.Time Details map[string]any } + +// FlushBound is how long a network sink's Stop gives its clients to take +// what is queued: write_timeout_ms, at most core.SinkFlushTimeout, which a +// reload also waits out for a client that stopped reading. +func FlushBound(writeTimeout time.Duration) time.Duration { + if writeTimeout > 0 && writeTimeout < core.SinkFlushTimeout { + return writeTimeout + } + return core.SinkFlushTimeout +} + +// Drain takes the payloads queued in input without waiting: a network sink's +// last events, which each client then writes on its own. +func Drain(input <-chan core.TransportEvent) [][]byte { + var tail [][]byte + for { + select { + case event := <-input: + tail = append(tail, event.Payload) + default: + return tail + } + } +} diff --git a/internal/sink/tcp/tcp.go b/internal/sink/tcp/tcp.go index 87cec18..18758b4 100644 --- a/internal/sink/tcp/tcp.go +++ b/internal/sink/tcp/tcp.go @@ -7,6 +7,7 @@ import ( "fmt" "maps" "net" + "slices" "strconv" "sync" "sync/atomic" @@ -74,8 +75,15 @@ type TCPSink struct { // Authorization auth *authz.Policy - // Runtime + // Runtime. Stop closes flush; the broadcast loop then moves what remains + // of its input to tail and closes flushed; each writer writes its queue + // and tail on its own, until flushBy. done tears down what is left. done chan struct{} + flush chan struct{} + flushed chan struct{} + flushBy time.Time + tail [][]byte // read only once flushed is closed + flushing bool // under clientsMu: no client registers once set stopOnce sync.Once wg sync.WaitGroup startTime time.Time @@ -91,11 +99,13 @@ type TCPSink struct { // tcpClient is a registered connection's bounded send queue. // send is written by the broadcast loop (non-blocking) and drained by the -// writer goroutine; closed signals reader-detected disconnect. +// writer goroutine; closed signals reader-detected disconnect, exited the +// writer's end. type tcpClient struct { send chan []byte sessionID string closed chan struct{} + exited chan struct{} } // NewTCPSinkPlugin creates a tcp sink through plugin factory @@ -131,7 +141,7 @@ func NewTCPSinkPlugin( if opts.KeepAlivePeriodMS <= 0 { opts.KeepAlivePeriodMS = DefaultTCPKeepAlivePeriodMS } - tlsCfg, err := tlsx.Server(opts.TLS) + tlsCfg, err := tlsx.Server(opts.TLS, opts.Host) if err != nil { return nil, err } @@ -148,6 +158,8 @@ func NewTCPSinkPlugin( network: network, input: make(chan core.TransportEvent, opts.BufferSize), done: make(chan struct{}), + flush: make(chan struct{}), + flushed: make(chan struct{}), logger: logger, clients: make(map[uint64]*tcpClient), conns: make(map[net.Conn]struct{}), @@ -223,7 +235,7 @@ func (t *TCPSink) Start(ctx context.Context) error { t.wg.Add(2) go t.acceptLoop() - go t.broadcastLoop(ctx) + go t.broadcastLoop() // Pipeline context cancellation mirrors gnet engine stop: cease accepting // and tear down existing connections @@ -258,13 +270,27 @@ func (t *TCPSink) Stop() { "total_processed", t.totalProcessed.Load()) } -// shutdown funnels ctx-cancel and Stop() teardown through a single path +// shutdown funnels ctx-cancel and Stop() teardown through a single path. +// Clients first receive what is queued, within sink.FlushBound, so a finite +// input reaches them whole. func (t *TCPSink) shutdown() { t.stopOnce.Do(func() { - close(t.done) if t.listener != nil { t.listener.Close() // unblocks acceptLoop + t.flushBy = time.Now().Add(sink.FlushBound(t.writeTimeout)) + close(t.flush) + t.clientsMu.Lock() + t.flushing = true + clients := slices.Collect(maps.Values(t.clients)) + t.clientsMu.Unlock() + for _, c := range clients { + select { + case <-c.exited: + case <-time.After(time.Until(t.flushBy)): + } + } } + close(t.done) t.clientsMu.Lock() for conn := range t.conns { conn.Close() // unblocks handshakes and per-connection readers @@ -381,10 +407,17 @@ func (t *TCPSink) handleConn(conn net.Conn) { send: make(chan []byte, t.config.ClientBufferSize), sessionID: sess.ID, closed: make(chan struct{}), + exited: make(chan struct{}), } + defer close(c.exited) id := t.nextClientID.Add(1) t.clientsMu.Lock() + if t.flushing { + t.clientsMu.Unlock() + t.proxy.RemoveSession(sess.ID) + return + } t.clients[id] = c t.clientsMu.Unlock() @@ -422,23 +455,45 @@ func (t *TCPSink) handleConn(conn net.Conn) { // Writer: synchronous lib write with deadline. A failed write means // the kernel buffer stayed full for the full deadline - connection is - // dead or hopelessly stalled, so disconnect immediately (no gnet-style - // consecutive-error counter needed for transient async callback errors). + // dead or hopelessly stalled, so disconnect immediately. + write := func(data []byte, deadline time.Time) bool { + if !deadline.IsZero() { + conn.SetWriteDeadline(deadline) + } + if _, err := conn.Write(data); err != nil { + t.writeErrors.Add(1) + t.logger.Debug("msg", "Write failed, closing client", + "component", "tcp_sink", + "remote_addr", remote, + "error", err) + return false + } + t.proxy.UpdateActivity(sess.ID) + return true + } for { select { case data := <-c.send: + var deadline time.Time if t.writeTimeout > 0 { - conn.SetWriteDeadline(time.Now().Add(t.writeTimeout)) + deadline = time.Now().Add(t.writeTimeout) } - if _, err := conn.Write(data); err != nil { - t.writeErrors.Add(1) - t.logger.Debug("msg", "Write failed, closing client", - "component", "tcp_sink", - "remote_addr", remote, - "error", err) + if !write(data, deadline) { return } - t.proxy.UpdateActivity(sess.ID) + case <-t.flushed: + // Nothing more will be queued: write the queue, then the tail + for len(c.send) > 0 { + if !write(<-c.send, t.flushBy) { + return + } + } + for _, data := range t.tail { + if !write(data, t.flushBy) { + return + } + } + return case <-c.closed: return case <-t.done: @@ -447,19 +502,22 @@ func (t *TCPSink) handleConn(conn net.Conn) { } } -// broadcastLoop fans out transport events to all client queues, non-blocking -func (t *TCPSink) broadcastLoop(ctx context.Context) { +// broadcastLoop fans out transport events to all client queues until flush, +// which shutdown closes on Stop and on context cancellation alike. +func (t *TCPSink) broadcastLoop() { defer t.wg.Done() for { select { - case <-ctx.Done(): + case <-t.flush: // first: a select with both ready picks at random + t.flushInput() return - case <-t.done: + default: + } + select { + case <-t.flush: + t.flushInput() return - case event, ok := <-t.input: - if !ok { - return - } + case event := <-t.input: t.totalProcessed.Add(1) t.lastProcessed.Store(time.Now()) @@ -477,6 +535,13 @@ func (t *TCPSink) broadcastLoop(ctx context.Context) { } } +// flushInput moves what remains of the input to tail for the writers +func (t *TCPSink) flushInput() { + t.tail = sink.Drain(t.input) + t.totalProcessed.Add(uint64(len(t.tail))) + close(t.flushed) +} + // GetStats returns sink statistics func (t *TCPSink) GetStats() sink.SinkStats { lastProc, _ := t.lastProcessed.Load().(time.Time) diff --git a/internal/sink/tcp/tcp_test.go b/internal/sink/tcp/tcp_test.go index 1aa8712..d2861b6 100644 --- a/internal/sink/tcp/tcp_test.go +++ b/internal/sink/tcp/tcp_test.go @@ -1,11 +1,15 @@ package tcp import ( + "bufio" + "fmt" "net" "strconv" + "strings" "testing" "time" + "github.com/lixenwraith/logwisp/internal/core" "github.com/lixenwraith/logwisp/internal/session" "github.com/lixenwraith/logwisp/internal/testutil" @@ -57,3 +61,81 @@ func TestStopDoesNotWaitForSilentHandshake(t *testing.T) { t.Fatalf("Stop took %v with a silent peer in the handshake", d) } } + +// Stop writes what is queued, in the sink and per client, to a client that +// keeps reading, whatever another that stopped reading does: the end of a +// finite input reaches it whole, and Stop returns at the flush bound. +func TestStopWritesQueuedEventsToReadingClients(t *testing.T) { + const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold + probe, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := probe.Addr().String() + probe.Close() + _, port, _ := net.SplitHostPort(addr) + p, _ := strconv.ParseInt(port, 10, 64) + + manager := session.NewManager(time.Hour) + t.Cleanup(manager.Stop) + created, err := NewTCPSinkPlugin("out", map[string]any{ + "host": "127.0.0.1", "port": p, "buffer_size": int64(n), "client_buffer_size": int64(64), + "write_timeout_ms": int64(500), + }, log.NewLogger(), session.NewProxy(manager, "out")) + if err != nil { + t.Fatal(err) + } + sink := created.(*TCPSink) + if err := sink.Start(t.Context()); err != nil { + t.Fatal(err) + } + var conns []net.Conn + for range 2 { + conn, err := net.Dial("tcp4", addr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + conns = append(conns, conn) + } + for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) { + sink.clientsMu.Lock() + registered := len(sink.clients) + sink.clientsMu.Unlock() + if registered == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("clients never registered") + } + } + lines := make(chan int) + go func() { + count, scanner := 0, bufio.NewScanner(conns[0]) + scanner.Buffer(nil, 64*1024) + for scanner.Scan() { + count++ + } + lines <- count + }() + // Held, the lock keeps the input queued until Stop has begun its flush + sink.clientsMu.Lock() + padding := strings.Repeat("x", 8192) + for i := range n { + sink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)} + } + stopped := make(chan time.Duration) + go func() { + start := time.Now() + sink.Stop() + stopped <- time.Since(start) + }() + time.Sleep(100 * time.Millisecond) + sink.clientsMu.Unlock() + if d := <-stopped; d > 1500*time.Millisecond { + t.Errorf("Stop took %v past a 500 ms flush bound", d) + } + if got := <-lines; got != n { + t.Fatalf("reading client received %d of %d lines", got, n) + } +} diff --git a/internal/sink/tcpchain/tcpchain.go b/internal/sink/tcpchain/tcpchain.go index 0c4c8bb..72487c4 100644 --- a/internal/sink/tcpchain/tcpchain.go +++ b/internal/sink/tcpchain/tcpchain.go @@ -329,8 +329,9 @@ func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool { return false } t.failures++ - if errors.Is(err, authz.ErrRefused) { - // A refusal is configuration, not weather: show it by default + if errors.Is(err, authz.ErrRefused) || errors.As(err, new(*tls.CertificateVerificationError)) { + // A refusal, or a server failing its CA or pin, is + // configuration, not weather: show it by default t.logger.Warn("msg", "Chain connect refused", "component", "tcp_chain_sink", "target", t.addr, diff --git a/internal/source/httpchain/httpchain.go b/internal/source/httpchain/httpchain.go index 2850021..9c614fc 100644 --- a/internal/source/httpchain/httpchain.go +++ b/internal/source/httpchain/httpchain.go @@ -112,7 +112,7 @@ func NewHTTPChainSourcePlugin( if opts.ReadTimeoutMS <= 0 { opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS } - tlsCfg, err := tlsx.Server(opts.TLS) + tlsCfg, err := tlsx.Server(opts.TLS, opts.Host) if err != nil { return nil, err } diff --git a/internal/source/tcpchain/tcpchain.go b/internal/source/tcpchain/tcpchain.go index 1dcd7e9..0c1a44d 100644 --- a/internal/source/tcpchain/tcpchain.go +++ b/internal/source/tcpchain/tcpchain.go @@ -97,7 +97,7 @@ func NewTCPChainSourcePlugin( if opts.HelloTimeoutMS <= 0 { opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS } - tlsCfg, err := tlsx.Server(opts.TLS) + tlsCfg, err := tlsx.Server(opts.TLS, opts.Host) if err != nil { return nil, err } diff --git a/internal/testutil/fixtures.go b/internal/testutil/fixtures.go index 6bb41ef..52fdf8f 100644 --- a/internal/testutil/fixtures.go +++ b/internal/testutil/fixtures.go @@ -6,6 +6,7 @@ import ( "crypto/ecdsa" "crypto/elliptic" "crypto/rand" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -120,6 +121,40 @@ func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFil return writePEM(t, p.dir, name+".crt", "CERTIFICATE", der), writePEM(t, p.dir, name+".key", "PRIVATE KEY", keyDER) } +// Handshake runs a TLS handshake between the two configurations over +// loopback TCP, whose buffers let a failing side's alert go out without a +// reader, and returns the client's error, else the server's. +func Handshake(t testing.TB, server, client *tls.Config) error { + t.Helper() + ln, err := net.Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + serverErr := make(chan error, 1) + go func() { + conn, err := ln.Accept() + if err != nil { + serverErr <- err + return + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(5 * time.Second)) + serverErr <- tls.Server(conn, server).Handshake() + }() + conn, err := net.Dial("tcp4", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + conn.SetDeadline(time.Now().Add(5 * time.Second)) + err = tls.Client(conn, client).Handshake() + conn.Close() + if serr := <-serverErr; err == nil { + err = serr + } + return err +} + func writePEM(t testing.TB, dir, name, blockType string, der []byte) string { t.Helper() path := filepath.Join(dir, name) diff --git a/internal/tlsx/pki.go b/internal/tlsx/pki.go new file mode 100644 index 0000000..e35871f --- /dev/null +++ b/internal/tlsx/pki.go @@ -0,0 +1,231 @@ +package tlsx + +import ( + "bytes" + "cmp" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/pem" + "errors" + "fmt" + "math/big" + "net" + "os" + "slices" + "strings" + "sync" + "time" + "unicode" +) + +// Certificates are ECDSA P-256, for lw tls and for listeners without files. +// A CA signs leaves only; a leaf lasts at most 397 days, the longest browsers +// accept, and never past its issuer. +const ( + CAValidity = 10 * 365 * 24 * time.Hour + LeafValidity = 397 * 24 * time.Hour + pinPrefix = "sha256//" +) + +// processKey is the key of every generated listener certificate: reloads +// reissue certificates and keep it, so a pin holds until lw restarts. +var processKey = sync.OnceValues(NewKey) + +func NewKey() (*ecdsa.PrivateKey, error) { return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) } + +// NewCA makes a self-signed certificate authority that signs leaves only +func NewCA(name string, validity time.Duration) (*x509.Certificate, *ecdsa.PrivateKey, error) { + key, err := NewKey() + if err != nil { + return nil, nil, err + } + t, err := template(name, validity) + if err != nil { + return nil, nil, err + } + t.IsCA, t.MaxPathLenZero = true, true + t.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign + cert, err := sign(t, t, &key.PublicKey, key) + return cert, key, err +} + +// Leaf describes a certificate to issue. Hosts become DNS or IP SANs, which +// hostname verification reads; Name is the subject CN, which mtls reads. +type Leaf struct { + Name string + Hosts []string + Server, Client bool + Validity time.Duration +} + +// Issue signs a certificate for pub; a nil issuer makes it self-signed. +func (l Leaf) Issue(pub crypto.PublicKey, issuer *x509.Certificate, issuerKey crypto.Signer) (*x509.Certificate, error) { + t, err := template(l.Name, l.Validity) + if err != nil { + return nil, err + } + t.KeyUsage = x509.KeyUsageDigitalSignature + if l.Server { + t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageServerAuth) + } + if l.Client { + t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageClientAuth) + } + for _, h := range l.Hosts { + if ip := net.ParseIP(h); ip != nil { + t.IPAddresses = append(t.IPAddresses, ip) + } else { + t.DNSNames = append(t.DNSNames, h) + } + } + if issuer == nil { + return sign(t, t, pub, issuerKey) + } + if !time.Now().Before(issuer.NotAfter) { + return nil, fmt.Errorf("tls: issuer %q expired on %s", issuer.Subject.CommonName, issuer.NotAfter.UTC().Format(time.DateOnly)) + } + if t.NotAfter.After(issuer.NotAfter) { + t.NotAfter = issuer.NotAfter + } + return sign(t, issuer, pub, issuerKey) +} + +func template(name string, validity time.Duration) (*x509.Certificate, error) { + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, err + } + now := time.Now() + return &x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: name}, + NotBefore: now.Add(-5 * time.Minute), // peers' clocks drift + NotAfter: now.Add(validity), + BasicConstraintsValid: true, + }, nil +} + +func sign(t, parent *x509.Certificate, pub crypto.PublicKey, key crypto.Signer) (*x509.Certificate, error) { + der, err := x509.CreateCertificate(rand.Reader, t, parent, pub, key) + if err != nil { + return nil, fmt.Errorf("tls: sign certificate: %w", err) + } + return x509.ParseCertificate(der) +} + +// LoadIssuer reads a CA certificate and its key, as lw tls ca writes them +func LoadIssuer(certFile, keyFile string) (*x509.Certificate, crypto.Signer, error) { + pair, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, nil, fmt.Errorf("tls: load issuer: %w", err) + } + ca, err := x509.ParseCertificate(pair.Certificate[0]) // Leaf is nil under GODEBUG=x509keypairleaf=0 + if err != nil { + return nil, nil, fmt.Errorf("tls: load issuer: %w", err) + } + signer, ok := pair.PrivateKey.(crypto.Signer) + if !ok || !ca.IsCA || ca.KeyUsage&x509.KeyUsageCertSign == 0 { + return nil, nil, fmt.Errorf("tls: %s is no certificate authority", certFile) + } + return ca, signer, nil +} + +// generated issues a listener certificate on the process key, self-signed +// or from the issuer files, for hosts and the names this machine answers to. +func generated(issuerCert, issuerKey, host string, hosts []string) (tls.Certificate, error) { + key, err := processKey() + if err != nil { + return tls.Certificate{}, err + } + var issuer *x509.Certificate + var signer crypto.Signer = key + if issuerCert != "" { + if issuer, signer, err = LoadIssuer(issuerCert, issuerKey); err != nil { + return tls.Certificate{}, err + } + } + name, _ := os.Hostname() + all := slices.Clone(hosts) + if ip := net.ParseIP(host); host != "" && (ip == nil || !ip.IsUnspecified()) { + all = append(all, host) + } + if !strings.ContainsFunc(name, func(r rune) bool { return r > unicode.MaxASCII }) { + all = append(all, name) // a DNS SAN is ASCII; the CN keeps any name + } + all = append(all, "localhost", "127.0.0.1", "::1") + slices.Sort(all) + all = slices.Compact(slices.DeleteFunc(all, func(h string) bool { return h == "" })) + leaf, err := Leaf{Name: cmp.Or(name, "logwisp"), Hosts: all, Server: true, Validity: LeafValidity}.Issue(&key.PublicKey, issuer, signer) + if err != nil { + return tls.Certificate{}, err + } + chain := [][]byte{leaf.Raw} + if issuer != nil { + chain = append(chain, issuer.Raw) // an intermediate needs it to chain to its root + } + return tls.Certificate{Certificate: chain, PrivateKey: key, Leaf: leaf}, nil +} + +// PinSHA256 is the curl --pinnedpubkey form of a certificate's public key +func PinSHA256(c *x509.Certificate) string { return pin(c.RawSubjectPublicKeyInfo) } + +func pinOfKey(k *ecdsa.PrivateKey) string { + der, _ := x509.MarshalPKIXPublicKey(&k.PublicKey) // P-256 always marshals + return pin(der) +} + +func pin(spki []byte) string { + h := sha256.Sum256(spki) + return pinPrefix + base64.StdEncoding.EncodeToString(h[:]) +} + +// verifyPins replaces chain verification: the server's key must hash to one +// of the ';'-separated pins. Its name and validity are not checked. Go skips +// it on a resumed session, and no dialer keeps a ClientSessionCache. +func verifyPins(s string) (func([][]byte, [][]*x509.Certificate) error, error) { + var pins [][]byte + for p := range strings.SplitSeq(s, ";") { + b64, ok := strings.CutPrefix(strings.TrimSpace(p), pinPrefix) + h, err := base64.StdEncoding.DecodeString(b64) + if !ok || err != nil || len(h) != sha256.Size { + return nil, fmt.Errorf("tls: pin_sha256 %q: want %sBASE64 of a SHA-256, ';' between several", p, pinPrefix) + } + pins = append(pins, h) + } + return func(raw [][]byte, _ [][]*x509.Certificate) error { + if len(raw) == 0 { + return errors.New("tls: server sent no certificate") + } + c, err := x509.ParseCertificate(raw[0]) + if err != nil { + return err + } + h := sha256.Sum256(c.RawSubjectPublicKeyInfo) + if slices.ContainsFunc(pins, func(p []byte) bool { return bytes.Equal(p, h[:]) }) { + return nil + } + // The type of a failed chain check: callers treat both as a refusal + return &tls.CertificateVerificationError{UnverifiedCertificates: []*x509.Certificate{c}, + Err: fmt.Errorf("server key %s matches no tls.pin_sha256", PinSHA256(c))} + }, nil +} + +// EncodeCert and EncodeKey render PEM, the key as PKCS #8 +func EncodeCert(c *x509.Certificate) []byte { + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw}) +} + +func EncodeKey(k crypto.PrivateKey) ([]byte, error) { + der, err := x509.MarshalPKCS8PrivateKey(k) + if err != nil { + return nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} diff --git a/internal/tlsx/tlsx.go b/internal/tlsx/tlsx.go index fbcc0c8..1bae7b3 100644 --- a/internal/tlsx/tlsx.go +++ b/internal/tlsx/tlsx.go @@ -26,16 +26,36 @@ const ( expiryWindow = 30 * 24 * time.Hour ) -// Server builds the *tls.Config for listener plugins -// (tcp/http sinks, tcp_chain/http_chain sources). Returns (nil, nil) when disabled. -func Server(o *config.TLSOptions) (*tls.Config, error) { +// Server builds the *tls.Config for listener plugins (tcp/http sinks, +// tcp_chain/http_chain sources) from cert_file and key_file, or issues one on +// the process key: self-signed or from the issuer files, for host, hosts and +// this machine's names. Returns (nil, nil) when disabled. +func Server(o *config.TLSOptions, host string) (*tls.Config, error) { if o == nil || !o.Enabled { return nil, nil } - if o.CertFile == "" || o.KeyFile == "" { - return nil, fmt.Errorf("tls: cert_file and key_file are required for listeners") + files, issuer := o.CertFile != "" || o.KeyFile != "", o.IssuerCertFile != "" || o.IssuerKeyFile != "" + switch { + case o.PinSHA256 != "": + return nil, fmt.Errorf("tls: pin_sha256 applies to dialers") + case files && (issuer || o.SelfSigned) || issuer && o.SelfSigned: + return nil, fmt.Errorf("tls: set one of cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file") + case len(o.Hosts) > 0 && !issuer && !o.SelfSigned: + return nil, fmt.Errorf("tls: hosts applies to self_signed and issuer certificates") + case files && (o.CertFile == "" || o.KeyFile == ""): + return nil, fmt.Errorf("tls: cert_file and key_file must be set together") + case issuer && (o.IssuerCertFile == "" || o.IssuerKeyFile == ""): + return nil, fmt.Errorf("tls: issuer_cert_file and issuer_key_file must be set together") + case !files && !issuer && !o.SelfSigned: + return nil, fmt.Errorf("tls: listeners need cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file") + } + var cert tls.Certificate + var err error + if files { + cert, err = tls.LoadX509KeyPair(o.CertFile, o.KeyFile) + } else { + cert, err = generated(o.IssuerCertFile, o.IssuerKeyFile, host, o.Hosts) } - cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile) if err != nil { return nil, fmt.Errorf("tls: load keypair: %w", err) } @@ -64,11 +84,18 @@ func Server(o *config.TLSOptions) (*tls.Config, error) { // Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain // sinks). host seeds ServerName when no override is set; Go verifies IP SANs // when host is an address, without its zone, which names a local interface -// and no certificate carries. Returns (nil, nil) when disabled. +// and no certificate carries. pin_sha256 replaces chain and name checks. +// Returns (nil, nil) when disabled. func Client(o *config.TLSOptions, host string) (*tls.Config, error) { if o == nil || !o.Enabled { return nil, nil } + switch { + case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0: + return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file and hosts apply to listeners") + case o.PinSHA256 != "" && (o.CAFile != "" || o.InsecureSkipVerify): + return nil, fmt.Errorf("tls: pin_sha256 replaces ca_file and insecure_skip_verify: set one") + } mv, err := minVersion(o.MinVersion) if err != nil { return nil, err @@ -81,6 +108,12 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) { if cfg.ServerName == "" { cfg.ServerName, _, _ = strings.Cut(host, "%") } + if o.PinSHA256 != "" { + if cfg.VerifyPeerCertificate, err = verifyPins(o.PinSHA256); err != nil { + return nil, err + } + cfg.InsecureSkipVerify = true // the pin check above replaces it + } if o.CAFile != "" { pool, err := loadPool(o.CAFile) if err != nil { @@ -149,13 +182,22 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list if o == nil || !o.Enabled { return } + if key, err := processKey(); listener && o.SelfSigned && err == nil { + // Without a CA, a dialer verifies this listener by its pin only + l.Warn("msg", "Serving a self-signed certificate; it changes when lw restarts", + "component", component, + "instance_id", id, + "pin_sha256", pinOfKey(key), + "hint", "dialers set tls.pin_sha256; for a stable identity use issuer_cert_file or cert_file") + } var warnings []string if !listener && o.InsecureSkipVerify { warnings = append(warnings, "tls.insecure_skip_verify disables server verification: any peer can impersonate the server") } - files := []struct{ key, path string }{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}} + type file struct{ key, path string } + files := []file{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}} if listener { - files[1] = struct{ key, path string }{"client_ca_file", o.ClientCAFile} + files = []file{{"cert_file", o.CertFile}, {"client_ca_file", o.ClientCAFile}, {"issuer_cert_file", o.IssuerCertFile}} } now := time.Now() for _, f := range files { @@ -165,8 +207,10 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list } } } - if w := SecretFileWarning("tls.key_file", o.KeyFile); w != "" { - warnings = append(warnings, w) + for _, key := range []file{{"tls.key_file", o.KeyFile}, {"tls.issuer_key_file", o.IssuerKeyFile}} { + if w := SecretFileWarning(key.key, key.path); w != "" { + warnings = append(warnings, w) + } } for _, w := range warnings { l.Warn("msg", w, "component", component, "instance_id", id) diff --git a/internal/tlsx/tlsx_test.go b/internal/tlsx/tlsx_test.go index 0bb4b2c..57ff947 100644 --- a/internal/tlsx/tlsx_test.go +++ b/internal/tlsx/tlsx_test.go @@ -3,11 +3,15 @@ package tlsx import ( "crypto/x509" "crypto/x509/pkix" + "net" + "path/filepath" + "slices" "strings" "testing" "time" "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/testutil" ) // A dialer verifies an IPv6 target by its address: the zone names a local @@ -48,3 +52,104 @@ func TestExpiryWarning(t *testing.T) { } } } + +// A self-signed listener is verified by the pin of the process key, which a +// reload's reissued certificate keeps; any other pin, or none, fails. +func TestSelfSignedListenerIsVerifiedByItsPin(t *testing.T) { + o := &config.TLSOptions{Enabled: true, SelfSigned: true, Hosts: []string{"agg.example"}} + first, err := Server(o, "0.0.0.0") + if err != nil { + t.Fatal(err) + } + reloaded, err := Server(o, "0.0.0.0") + if err != nil { + t.Fatal(err) + } + leaf := reloaded.Certificates[0].Leaf + pin := PinSHA256(leaf) + if PinSHA256(first.Certificates[0].Leaf) != pin || !slices.Contains(leaf.DNSNames, "agg.example") || + !slices.Contains(leaf.DNSNames, "localhost") || slices.ContainsFunc(leaf.IPAddresses, net.IP.IsUnspecified) { + t.Fatalf("pin %s, names %v %v", pin, leaf.DNSNames, leaf.IPAddresses) + } + other := "sha256//" + strings.Repeat("A", 43) + "=" + for pins, want := range map[string]string{other + "; " + pin: "", other: "matches no tls.pin_sha256", "": "certificate"} { + client, err := Client(&config.TLSOptions{Enabled: true, PinSHA256: pins}, "agg.example") + if err != nil { + t.Fatal(err) + } + err = testutil.Handshake(t, reloaded, client) + if want == "" && err != nil || want != "" && (err == nil || !strings.Contains(err.Error(), want)) { + t.Errorf("pins %q: %v, want %q", pins, err, want) + } + } +} + +// An issuer-signed listener chains to the CA file, for its host, and expires +// with its issuer; a certificate that is no CA cannot issue. +func TestIssuedListenerCertificateChainsToTheIssuer(t *testing.T) { + dir := t.TempDir() + ca, key, err := NewCA("test CA", time.Hour) + if err != nil { + t.Fatal(err) + } + keyPEM, err := EncodeKey(key) + if err != nil { + t.Fatal(err) + } + caFile, keyFile := filepath.Join(dir, "ca.crt"), filepath.Join(dir, "ca.key") + testutil.WriteFile(t, caFile, string(EncodeCert(ca))) + testutil.WriteFile(t, keyFile, string(keyPEM)) + server, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: caFile, IssuerKeyFile: keyFile}, "127.0.0.1") + if err != nil { + t.Fatal(err) + } + if leaf := server.Certificates[0].Leaf; !leaf.NotAfter.Equal(ca.NotAfter) { + t.Fatalf("leaf expires %s, issuer %s", leaf.NotAfter, ca.NotAfter) + } + client, err := Client(&config.TLSOptions{Enabled: true, CAFile: caFile}, "127.0.0.1") + if err != nil { + t.Fatal(err) + } + if err := testutil.Handshake(t, server, client); err != nil { + t.Fatalf("issued chain: %v", err) + } + leafFile := filepath.Join(dir, "leaf.crt") + testutil.WriteFile(t, leafFile, string(EncodeCert(server.Certificates[0].Leaf))) + if _, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: leafFile, IssuerKeyFile: keyFile}, ""); err == nil { + t.Fatal("a leaf issued certificates") + } +} + +// Each role takes one source of trust and one certificate source; keys of the +// other role fail instead of doing nothing. +func TestTLSOptionConflicts(t *testing.T) { + pin := "sha256//" + strings.Repeat("A", 43) + "=" + for _, c := range []struct { + o config.TLSOptions + server bool + want string + }{ + {config.TLSOptions{SelfSigned: true, CertFile: "c", KeyFile: "k"}, true, "set one of"}, + {config.TLSOptions{SelfSigned: true, IssuerCertFile: "c"}, true, "set one of"}, + {config.TLSOptions{IssuerCertFile: "c"}, true, "must be set together"}, + {config.TLSOptions{CertFile: "c"}, true, "must be set together"}, + {config.TLSOptions{Hosts: []string{"a"}, CertFile: "c", KeyFile: "k"}, true, "hosts applies to"}, + {config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "applies to dialers"}, + {config.TLSOptions{}, true, "listeners need"}, + {config.TLSOptions{SelfSigned: true}, false, "apply to listeners"}, + {config.TLSOptions{PinSHA256: pin, CAFile: "ca"}, false, "set one"}, + {config.TLSOptions{PinSHA256: pin, InsecureSkipVerify: true}, false, "set one"}, + {config.TLSOptions{PinSHA256: "sha256//short"}, false, "want sha256//BASE64"}, + } { + c.o.Enabled = true + var err error + if c.server { + _, err = Server(&c.o, "") + } else { + _, err = Client(&c.o, "host") + } + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("%+v: %v, want %q", c.o, err, c.want) + } + } +} diff --git a/test/preset-test.sh b/test/preset-test.sh new file mode 100755 index 0000000..797b441 --- /dev/null +++ b/test/preset-test.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# logwisp preset test: colors, presets, --dump, lw tls, listeners that make +# their certificate (self-signed and pinned, or issued), and the http and tcp +# sinks writing a finite input whole. One-shot: both modes run the checks. +# Ports: 15890-15896 on 127.0.0.1. +# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only. + +set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +RUN=$E2E_DIR/run/preset +e2e_init "$@" + +section "Setup" +need curl openssl +ports_free 15890 15891 15892 15893 15894 15895 15896 +rm -rf "$RUN" +mkdir -p "$RUN/out" "$LOG" +cd "$RUN" || abort "no run directory" +# no configuration file or LOGWISP_ variable applies +for v in $(compgen -e LOGWISP_); do unset "$v"; done +export HOME=$RUN +lw() { "$BIN" "$@"; } +seq 1 300 | sed 's/^/edge line /' >"$RUN/edge.log" +info "runs lw in $(short "$RUN") with no configuration file" + +section "Color" +red=$'\e[0;1;31m' +check "--color paints the level name" "$(is "$(printf 'a ERROR b\n' | lw --color)" "a ${red}ERROR"$'\e[0m b')" +check "--color never, and auto on a pipe, leave it plain" \ + "$(is "$(printf 'a ERROR b\n' | lw --color never)$(printf 'a ERROR b\n' | lw)" "a ERROR ba ERROR b")" +check "LOGWISP_COLOR=always, a sink's own color=never wins" \ + "$(is "$(printf 'x WARN\n' | LOGWISP_COLOR=always lw --sink console,color=never)" "x WARN")" + +section "Presets" +seq 1 5000 >"$RUN/seq.txt" +lw --preset pipe <"$RUN/seq.txt" >"$RUN/pipe.out" +check "--preset pipe copies stdin line for line" "$(cmp -s "$RUN/seq.txt" "$RUN/pipe.out" && echo 1 || echo 0)" +out=$(timeout 3 "$BIN" --preset "tail,path=$RUN/edge.log,from=start" /dev/null | head -2 | tr '\n' '|') +check "--preset tail follows a file" "$(is "$out" "edge line 1|edge line 2|")" +lw --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15890,tls=self,hosts=logs.example" --color --dump >"$RUN/dump.toml" +lw -c "$RUN/dump.toml" --dump >"$RUN/dump2.toml" +check "--dump prints a file that reads back to the same dump" "$(cmp -s "$RUN/dump.toml" "$RUN/dump2.toml" && echo 1 || echo 0)" +check "lw --check builds the dumped preset" "$(lw --check -c "$RUN/dump.toml" 2>"$LOG/check.err" | grep -q 'configuration ok' && echo 1 || echo 0)" +lw preset edge -to 127.0.0.1:1 >/dev/null 2>"$LOG/edge-noauth.err" +rc=$? +check "an edge without credentials is a usage error ($rc)" "$( ((rc == 2)) && grep -q 'never sends unauthenticated' "$LOG/edge-noauth.err" && echo 1 || echo 0)" + +section "Certificates" +lw tls ca -dir "$RUN/pki" 2>"$LOG/tls.out" && + lw tls cert -ca-dir "$RUN/pki" -name logs.example -server -host 127.0.0.1,logs.example 2>>"$LOG/tls.out" +check "lw tls ca and cert issue a chain openssl verifies" \ + "$(openssl verify -CAfile "$RUN/pki/ca.crt" "$RUN/pki/logs.example.crt" 2>/dev/null | grep -q ': OK$' && echo 1 || echo 0)" +check "keys are private" "$(is "$(stat -c %a "$RUN/pki/ca.key" "$RUN/pki/logs.example.key" | tr '\n' ' ')" "600 600 ")" +lw tls ca -dir "$RUN/pki" 2>/dev/null +check "an existing CA is never replaced" "$(( $? == 1 ))" + +AUTH=$RUN +add_users "$RUN/users.toml" edge-01 +# pin DAEMON: the pin_sha256 a self-signed listener logged +pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; } +received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; } + +spawn agg-self "$BIN" --preset "aggregator,listen=127.0.0.1:15893,users=$RUN/users.toml,out=$RUN/out/self" +wait_port 15893 || abort "the self-signed aggregator did not listen" agg-self +PIN=$(pin agg-self) +check "a self-signed aggregator logs its pin" "$(is_set "$PIN")" +spawn edge-pin "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" +spawn edge-wrong "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=sha256//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,user=edge-01,password_file=$RUN/edge-01.pass" +wait_until 10 eval '(( $(received self) >= 300 ))' +check "an edge pinning it delivers every line over SCRAM ($(received self))" "$(( $(received self) == 300 ))" +wait_until 10 logged edge-wrong 'matches no tls.pin_sha256' +check "an edge with another pin is refused before it logs in" "$(logged edge-wrong 'matches no tls.pin_sha256' && echo 1 || echo 0)" + +spawn agg-issuer "$BIN" --preset "aggregator,listen=127.0.0.1:15894,transport=http,tls=issuer,issuer_cert=$RUN/pki/ca.crt,issuer_key=$RUN/pki/ca.key,users=$RUN/users.toml,out=$RUN/out/issuer" +wait_port 15894 || abort "the issuing aggregator did not listen" agg-issuer +spawn edge-ca "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" +wait_until 10 eval '(( $(received issuer) >= 300 ))' +check "an aggregator signing its own certificate is verified by the CA ($(received issuer))" "$(( $(received issuer) == 300 ))" + +spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15895,tls=self" +wait_port 15895 || abort "the serve preset did not listen" serve +SERVE_PIN=$(pin serve) +check "curl --pinnedpubkey reads a self-signed stream's status" \ + "$(curl -sk --pinnedpubkey "$SERVE_PIN" https://127.0.0.1:15895/status | grep -q '"tls":true' && echo 1 || echo 0)" +curl -sk --pinnedpubkey "$PIN" -o /dev/null https://127.0.0.1:15895/status +check "curl with another listener's pin is refused" "$(( $? != 0 ))" + +section "Flush at exit" +# feed SECONDS PORT: the lines, once a client had time to connect +feed() { wait_port "$2"; sleep "$1"; seq 1 20000 | sed 's/^/flush /'; } +queues='buffer_size=20000,client_buffer_size=20000' +feed 1 15891 | lw --sink "tcp,host=127.0.0.1,port=15891,$queues" 2>"$LOG/flush-tcp.err" & +fpid=$! +wait_port 15891 +n=$(tcp_read 15891 15 | grep -c '^flush ') +wait "$fpid" +check "a tcp client gets all 20000 lines of a finite input ($n)" "$((n == 20000))" +feed 1 15892 | lw --sink "http,host=127.0.0.1,port=15892,$queues" 2>"$LOG/flush-http.err" & +fpid=$! +wait_port 15892 +curl -sN --max-time 15 http://127.0.0.1:15892/stream >"$RUN/flush-http.out" +wait "$fpid" +n=$(grep -c '^data: flush ' "$RUN/flush-http.out") +check "an http stream gets all 20000 lines, then the disconnect event ($n)" \ + "$( ((n == 20000)) && tail -3 "$RUN/flush-http.out" | grep -q '^event: disconnect' && echo 1 || echo 0)" + +summary