v0.23.0 color, presets, certificates made at startup, cli.go, sink flush

AGENTS.md

@@ -76,7 +76,12 @@ change touches.
 - Plugin options decode through `config.Scan`, never `lconfig.ScanMap`: unknown
   keys must fail at any depth. Set defaults on the options struct before the
   call; `TestEveryPluginRejectsUnknownKeys` covers every registered plugin.
-- Network security has two seams: `tlsx` (TLS configs, startup warnings) and
+- `cmd/lw/cli.go` owns the command line: lw's own flags, and the commands
+  table whose groups (`auth`, `tls`, `preset`) share one flag mechanism;
+  `config.Load` takes the parsed `config.Args`. Presets are rows of one table
+  in `internal/config/preset.go`.
+- Network security has two seams: `tlsx` (TLS configs, certificates made at
+  startup or by `lw tls`, pins, startup warnings) and
   `authz`. Listeners admit only through `Admit`/`AuthorizeRequest`, dialers
   through `Greet`/`Prepare`; `Authorize` refuses under scram, so a plugin gated
   on it alone fails closed. Nil policies are valid: keep call sites branch-free.

README.md

@@ -22,6 +22,9 @@ streams, or downstream LogWisp nodes.
 - **A Unix filter out of the box**: with no configuration, `lw` copies stdin
   to stdout line for line and exits at the end of input; flags add filters,
   formats, sources and sinks
+- **Presets**: `tail`, `serve`, `edge` and `aggregator` build the common
+  pipelines from a few keys (`lw --preset tail,path=/var/log/app`);
+  `lw --dump` turns any command line into a configuration file
 - **Independent pipelines**, each `sources → flow → sinks`, running concurrently
   in one process
 - **Fan-in and fan-out**: many sources and many sinks per pipeline
@@ -39,7 +42,8 @@ streams, or downstream LogWisp nodes.
 
 ### Outputs
 
-`console` (control characters escaped on a terminal), `file` (rotating with
+`console` (level names in color and control characters escaped on a
+terminal), `file` (rotating with
 retention), `http` (Server-Sent Events plus a
 JSON status endpoint), `tcp` (broadcast server), `null`, and the chain
 forwarders `tcp_chain` and `http_chain`.
@@ -64,6 +68,9 @@ exponential backoff and jitter.
 - TLS 1.2/1.3 on every network source and sink, listener and dialer alike
 - Mutual TLS: listeners can require and verify client certificates; dialers can
   present a client identity
+- Certificates without a PKI: `lw tls` makes a CA and certificates, and a
+  listener can make its own at startup, self-signed (dialers pin its key with
+  `pin_sha256`) or signed by a CA
 - Authorization by certificate identity: an `auth` block admits named peers
   (exact or RE2) rather than everything the CA issued, gates the `http` sink's
   stream and status endpoints, and lets a dialer pin the server it talks to
@@ -109,8 +116,7 @@ authentication designs for the rationale and what is deliberately left out.
 - [Operations](doc/operations.md): running, monitoring, tuning,
   troubleshooting
 - [To Do](doc/todo.md): planned work in priority order: network access
-  control and the PROXY protocol, a hardening review of the config libraries,
+  control and the PROXY protocol, packaging
-  packaging
 
 A fully annotated configuration covering every option ships as
 [`config/logwisp.toml`](config/logwisp.toml).
@@ -135,6 +141,9 @@ journalctl -f | ./bin/lw --sink http,host=127.0.0.1,port=8080
 # tail a directory instead of stdin
 ./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \
     --format json,sanitizer_policy=json > all.json
+
+# the same with a preset; on a terminal the level names are in color
+./bin/lw --preset tail,path=/var/log/myapp
 ```
 
 As a service, a configuration file holds the pipelines:

cmd/lw/auth.go

@@ -28,15 +28,7 @@ import (
 
 const reloadHint = "send SIGHUP to logwisp to apply (auto_reload does not watch the credentials file)"
 
-// authCommand is one `lw auth` subcommand: define registers its flags
+var authCommands = []subcommand{
-// and returns the command, which runs once they are parsed and checked.
-type authCommand struct {
-	name, synopsis, summary string
-	required                []string
-	define                  func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error
-}
-
-var authCommands = []authCommand{
 	{"add-user", "-credentials FILE -user NAME [-password-file FILE] [-generate]",
 		"Add a user to a credentials file, or replace its password",
 		[]string{"credentials", "user"}, defineAddUser},
@@ -51,77 +43,6 @@ var authCommands = []authCommand{
 		[]string{"addr", "user", "password-file"}, defineStream},
 }
 
-// usageError is a command-line mistake: exit status 2 rather than 1
-type usageError string
-
-func (e usageError) Error() string { return string(e) }
-
-// runAuth runs `lw auth` and returns the exit status: 0 success or
-// help, 1 failure, 2 usage error.
-func runAuth(args []string, stdout, stderr io.Writer) int {
-	if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) {
-		printAuthUsage(stderr)
-		if len(args) == 0 {
-			return 2
-		}
-		return 0
-	}
-	i := slices.IndexFunc(authCommands, func(c authCommand) bool { return c.name == args[0] })
-	if i < 0 {
-		fmt.Fprintf(stderr, "lw auth: unknown command %q\n\n", args[0])
-		printAuthUsage(stderr)
-		return 2
-	}
-	c := authCommands[i]
-	fs := flag.NewFlagSet("lw auth "+c.name, flag.ContinueOnError)
-	fs.SetOutput(stderr)
-	fs.Usage = func() {
-		fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), c.synopsis, c.summary)
-		fs.PrintDefaults()
-	}
-	run := c.define(fs)
-	if err := fs.Parse(args[1:]); err != nil {
-		if errors.Is(err, flag.ErrHelp) {
-			return 0
-		}
-		return 2 // already reported by flag, with the usage
-	}
-	err := checkArgs(fs, c.required)
-	if err == nil {
-		err = run(stdout, stderr)
-	}
-	if err == nil {
-		return 0
-	}
-	fmt.Fprintf(stderr, "%s: %v\n", fs.Name(), err)
-	if _, ok := errors.AsType[usageError](err); ok {
-		fs.Usage()
-		return 2
-	}
-	return 1
-}
-
-func printAuthUsage(w io.Writer) {
-	fmt.Fprint(w, "Usage: lw auth <command> [flags]\n\n")
-	for _, c := range authCommands {
-		fmt.Fprintf(w, "  %-12s %s\n", c.name, c.summary)
-	}
-	fmt.Fprint(w, "\nRun lw auth <command> -h for its flags. Credential changes apply on SIGHUP.\n"+
-		"Exit status: 0 success, 1 failure, 2 usage error.\n")
-}
-
-func checkArgs(fs *flag.FlagSet, required []string) error {
-	if fs.NArg() > 0 {
-		return usageError(fmt.Sprintf("unexpected argument %q", fs.Arg(0)))
-	}
-	for _, name := range required {
-		if fs.Lookup(name).Value.String() == "" {
-			return usageError("-" + name + " is required")
-		}
-	}
-	return nil
-}
-
 func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 	path := fs.String("credentials", "", "credentials `file`, created when missing")
 	user := fs.String("user", "", "user `name`")
@@ -323,6 +244,7 @@ func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 func tlsFlags(fs *flag.FlagSet) *config.TLSOptions {
 	o := &config.TLSOptions{Enabled: true}
 	fs.StringVar(&o.CAFile, "ca-file", "", "CA `file` that verifies the server (default: system roots)")
+	fs.StringVar(&o.PinSHA256, "pin-sha256", "", "sha256//BASE64 `pin` of the server key, in place of -ca-file (a self_signed listener logs it)")
 	fs.StringVar(&o.ServerName, "server-name", "", "`name` the server certificate must carry (default: the host)")
 	fs.StringVar(&o.CertFile, "cert-file", "", "client certificate `file`, for listeners with tls.client_auth")
 	fs.StringVar(&o.KeyFile, "key-file", "", "client key `file`")

cmd/lw/auth_test.go

@@ -14,10 +14,7 @@ import (
 )
 
 func runAuthTest(t *testing.T, args ...string) (code int, stdout, stderr string) {
-	t.Helper()
+	return runCommand(t, "auth", args...)
-	var out, errOut bytes.Buffer
-	code = runAuth(args, &out, &errOut)
-	return code, out.String(), errOut.String()
 }
 
 // writeCredentials writes users sharing a cheap Argon2 profile. add-user

cmd/lw/bootstrap.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 
 	_ "github.com/lixenwraith/logwisp/internal/source/console"
@@ -27,6 +28,7 @@ import (
 
 	"github.com/lixenwraith/log"
 	"github.com/lixenwraith/log/sanitizer"
+	"github.com/lixenwraith/toml"
 )
 
 // bootstrapInitial handles initial service startup with status reporter
@@ -130,6 +132,29 @@ func checkConfig(cfg *config.Config) int {
 	return 0
 }
 
+// dumpConfig prints the effective configuration, flags and environment
+// resolved into it, as a file lw -c reads back.
+func dumpConfig(cfg *config.Config, w io.Writer) int {
+	cfg.Dump = false
+	// Snapshot gave each console sink the top-level color: keep it inherited
+	for _, p := range cfg.Pipelines {
+		for _, s := range p.PluginSinks {
+			if s.Type == "console" && s.Config["color"] == cfg.Color {
+				delete(s.Config, "color")
+			}
+		}
+	}
+	data, err := toml.Marshal(cfg)
+	if err == nil {
+		_, err = w.Write(data)
+	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		return 1
+	}
+	return 0
+}
+
 // initializeLogger sets up the global logger based on the application's configuration
 func initializeLogger(cfg *config.Config) error {
 	logger = log.NewLogger()

cmd/lw/cli.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"cmp"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"slices"
+	"strings"
+
+	"github.com/lixenwraith/logwisp/internal/config"
+
+	"github.com/lixenwraith/toml"
+)
+
+// command is a group of subcommands, recognized only as lw's first argument
+type command struct {
+	name, summary, footer string
+	subcommands           []subcommand
+}
+
+// subcommand is one `lw COMMAND NAME`: define registers its flags and
+// returns the work, which runs once they are parsed and checked.
+type subcommand struct {
+	name, synopsis, summary string
+	required                []string
+	define                  func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error
+}
+
+var commands = []command{
+	{"auth", "SCRAM users, bearer tokens, tcp sink viewing",
+		"Credential changes apply on SIGHUP.", authCommands},
+	{"tls", "A CA and certificates for TLS and mTLS",
+		"Files are never overwritten; keys are written mode 0600.", tlsCommands},
+	{"preset", "Show the pipeline a preset makes, as TOML",
+		"lw --preset NAME,key=value runs one; flags here are its keys.", presetCommands()},
+}
+
+// usageError is a command-line mistake: exit status 2 rather than 1
+type usageError string
+
+func (e usageError) Error() string { return string(e) }
+
+// run runs a subcommand and returns the exit status: 0 success or help,
+// 1 failure, 2 usage error.
+func (c *command) run(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) {
+		c.usage(stderr)
+		if len(args) == 0 {
+			return 2
+		}
+		return 0
+	}
+	i := slices.IndexFunc(c.subcommands, func(s subcommand) bool { return s.name == args[0] })
+	if i < 0 {
+		fmt.Fprintf(stderr, "lw %s: unknown command %q\n\n", c.name, args[0])
+		c.usage(stderr)
+		return 2
+	}
+	s := c.subcommands[i]
+	fs := flag.NewFlagSet("lw "+c.name+" "+s.name, flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	fs.Usage = func() {
+		fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), s.synopsis, s.summary)
+		fs.PrintDefaults()
+	}
+	work := s.define(fs)
+	if err := fs.Parse(args[1:]); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return 0
+		}
+		return 2 // already reported by flag, with the usage
+	}
+	err := checkArgs(fs, s.required)
+	if err == nil {
+		err = work(stdout, stderr)
+	}
+	if err == nil {
+		return 0
+	}
+	fmt.Fprintf(stderr, "%s: %v\n", fs.Name(), err)
+	if _, ok := errors.AsType[usageError](err); ok {
+		fs.Usage()
+		return 2
+	}
+	return 1
+}
+
+func (c *command) usage(w io.Writer) {
+	fmt.Fprintf(w, "Usage: lw %s <command> [flags]\n\n", c.name)
+	for _, s := range c.subcommands {
+		fmt.Fprintf(w, "  %-12s %s\n", s.name, s.summary)
+	}
+	fmt.Fprintf(w, "\nRun lw %s <command> -h for its flags. %s\n"+
+		"Exit status: 0 success, 1 failure, 2 usage error.\n", c.name, c.footer)
+}
+
+func checkArgs(fs *flag.FlagSet, required []string) error {
+	if fs.NArg() > 0 {
+		return usageError(fmt.Sprintf("unexpected argument %q", fs.Arg(0)))
+	}
+	for _, name := range required {
+		if fs.Lookup(name).Value.String() == "" {
+			return usageError("-" + name + " is required")
+		}
+	}
+	return nil
+}
+
+// invocation is a parsed command line: a command with its arguments, a help
+// request, or what config.Load reads.
+type invocation struct {
+	command *command
+	args    []string
+	help    bool
+	load    config.Args
+}
+
+// parseCommandLine reads lw's own flags; anything else is a --path=value
+// setting for lixenwraith/config, which reports what it does not know. A
+// pipeline flag takes the next argument unless it starts with '-'. A help
+// request wins over a malformed flag.
+func parseCommandLine(argv []string) (inv invocation, err error) {
+	defer func() {
+		if inv.help {
+			err = nil
+		}
+	}()
+	if len(argv) > 0 {
+		if i := slices.IndexFunc(commands, func(c command) bool { return c.name == argv[0] }); i >= 0 {
+			return invocation{command: &commands[i], args: argv[1:]}, nil
+		}
+		inv.help = argv[0] == "help"
+	}
+	specFlags := config.SpecFlags()
+	next := func(i *int) string {
+		if *i+1 < len(argv) && !strings.HasPrefix(argv[*i+1], "-") {
+			*i++
+			return argv[*i]
+		}
+		return ""
+	}
+	for i := 0; i < len(argv); i++ {
+		arg := argv[i]
+		name, value, inline := strings.Cut(arg, "=")
+		switch {
+		case arg == "--":
+			inv.load.Overrides = append(inv.load.Overrides, argv[i:]...)
+			return inv, err
+		case arg == "-h" || arg == "--help":
+			inv.help = true
+		case name == "-c" || name == "--config":
+			if !inline {
+				value = next(&i)
+			}
+			if value == "" {
+				err = cmp.Or(err, fmt.Errorf("%s requires a configuration file path", name))
+			}
+			inv.load.File = value
+		case arg == "--color":
+			// Bare, it means always: config takes no bare flag for a string key
+			inv.load.Overrides = append(inv.load.Overrides, "--color="+cmp.Or(next(&i), "always"))
+		case strings.HasPrefix(name, "--") && slices.Contains(specFlags, name[2:]):
+			if !inline {
+				value = next(&i)
+			}
+			inv.load.Specs = append(inv.load.Specs, config.Spec{Flag: name[2:], Value: value})
+		default:
+			inv.load.Overrides = append(inv.load.Overrides, arg)
+		}
+	}
+	return inv, err
+}
+
+// presetCommands makes each preset a subcommand whose flags are its keys
+func presetCommands() []subcommand {
+	var subs []subcommand
+	for _, p := range config.Presets() {
+		s := subcommand{name: p.Name, summary: p.Summary}
+		var synopsis []string
+		for _, k := range p.Params {
+			if k.Required {
+				s.required = append(s.required, k.Name)
+				synopsis = append(synopsis, "-"+k.Name+" VALUE")
+			}
+		}
+		s.synopsis = strings.Join(append(synopsis, "[-KEY VALUE...]"), " ")
+		s.define = func(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
+			values := map[string]*string{}
+			for _, k := range p.Params {
+				values[k.Name] = fs.String(k.Name, k.Default, k.Help)
+			}
+			return func(stdout, _ io.Writer) error {
+				set := map[string]string{}
+				for name, v := range values {
+					set[name] = *v
+				}
+				pipeline, err := config.ExpandPreset(p.Name, set)
+				if err != nil {
+					return usageError(err.Error())
+				}
+				data, err := toml.Marshal(map[string]any{"pipelines": []config.PipelineConfig{pipeline}})
+				if err == nil {
+					_, err = stdout.Write(data)
+				}
+				return err
+			}
+		}
+		subs = append(subs, s)
+	}
+	return subs
+}

cmd/lw/cli_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/lixenwraith/logwisp/internal/config"
+	"github.com/lixenwraith/logwisp/internal/testutil"
+	"github.com/lixenwraith/logwisp/internal/tlsx"
+)
+
+// lw's own flags: the last -c wins, a pipeline flag or --color takes the next
+// argument unless it starts with '-', -- ends lw's flags, and a command is
+// only the first argument. Everything else is a setting for config.
+func TestCommandLineGrammar(t *testing.T) {
+	for _, c := range []struct {
+		argv []string
+		want invocation
+	}{
+		{[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}},
+		{[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}},
+		{[]string{"--source=null", "--sink", "http,port=1", "--filter", "--sink"},
+			invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}, {Flag: "filter"}, {Flag: "sink"}}}}},
+		{[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}},
+		{[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}},
+		{[]string{"--quiet", "--", "-c", "x", "--source", "null"}, invocation{load: config.Args{Overrides: []string{"--quiet", "--", "-c", "x", "--source", "null"}}}},
+		{[]string{"--quiet", "-h"}, invocation{help: true, load: config.Args{Overrides: []string{"--quiet"}}}},
+		{[]string{"-c", "-h"}, invocation{help: true}},
+		{[]string{"help"}, invocation{help: true, load: config.Args{Overrides: []string{"help"}}}},
+		{[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}},
+		{[]string{"--quiet", "tls"}, invocation{load: config.Args{Overrides: []string{"--quiet", "tls"}}}},
+	} {
+		got, err := parseCommandLine(c.argv)
+		if err != nil || !reflect.DeepEqual(got, c.want) {
+			t.Errorf("%q:\n got %+v %v\nwant %+v", c.argv, got, err, c.want)
+		}
+	}
+	for _, argv := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}} {
+		if _, err := parseCommandLine(argv); err == nil {
+			t.Errorf("%q: missing path accepted", argv)
+		}
+	}
+}
+
+// --dump prints a file that lw -c reads back to the same configuration, its
+// console sinks still taking the top-level color
+func TestDumpReadsBack(t *testing.T) {
+	testLogger(t)
+	dir := t.TempDir()
+	empty := filepath.Join(dir, "empty.toml")
+	if err := os.WriteFile(empty, nil, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	args := []string{"--color", "--preset", "serve,listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example",
+		"--filter", `exclude,patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"}
+	cfg := loadTestConfig(t, empty, args...)
+	var dump bytes.Buffer
+	if code := dumpConfig(loadTestConfig(t, empty, args...), &dump); code != 0 {
+		t.Fatalf("dump exit %d", code)
+	}
+	path := filepath.Join(dir, "dump.toml")
+	if err := os.WriteFile(path, dump.Bytes(), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	again := loadTestConfig(t, path)
+	if !reflect.DeepEqual(again.Pipelines, cfg.Pipelines) || again.Color != "always" || again.Logging.Level != "debug" ||
+		strings.Count(dump.String(), "color = ") != 1 {
+		t.Fatalf("dump does not read back:\n%s\n got %+v\nwant %+v", dump.String(), again.Pipelines, cfg.Pipelines)
+	}
+}
+
+// Every preset expands to plugins that build, as lw --check builds them
+func TestEveryPresetBuilds(t *testing.T) {
+	testLogger(t)
+	dir := t.TempDir()
+	users, pass, empty := filepath.Join(dir, "users.toml"), filepath.Join(dir, "edge.pass"), filepath.Join(dir, "empty.toml")
+	writeCredentials(t, users, "edge-01")
+	for path, data := range map[string]string{pass: "password-edge-01\n", empty: ""} {
+		if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	pin := "sha256//" + strings.Repeat("A", 43) + "="
+	samples := map[string]string{
+		"pipe":       "pipe,format=txt",
+		"tail":       "tail,path=" + dir,
+		"serve":      "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users,
+		"edge":       "edge,to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass,
+		"aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out"),
+	}
+	for _, p := range config.Presets() {
+		sample, ok := samples[p.Name]
+		if !ok {
+			t.Errorf("preset %s has no sample here", p.Name)
+			continue
+		}
+		if code := checkConfig(loadTestConfig(t, empty, "--preset", sample)); code != 0 {
+			t.Errorf("%s: lw --check exit %d", sample, code)
+		}
+	}
+}
+
+// lw tls ca and lw tls cert write a chain a dialer verifies by the CA file,
+// keys private, and never replace a file
+func TestTLSCommandsIssueAVerifiableChain(t *testing.T) {
+	dir := t.TempDir()
+	steps := [][]string{
+		{"ca", "-dir", dir, "-name", "test CA"},
+		{"cert", "-ca-dir", dir, "-name", "agg", "-server", "-host", "127.0.0.1"},
+	}
+	for _, args := range steps {
+		if code, _, stderr := runCommand(t, "tls", args...); code != 0 {
+			t.Fatalf("%v: exit %d: %s", args, code, stderr)
+		}
+	}
+	for _, key := range []string{"ca.key", "agg.key"} {
+		if fi, err := os.Stat(filepath.Join(dir, key)); err != nil || fi.Mode().Perm() != 0o600 {
+			t.Fatalf("%s: %v %v", key, fi.Mode(), err)
+		}
+	}
+	server, err := tlsx.Server(&config.TLSOptions{Enabled: true, CertFile: filepath.Join(dir, "agg.crt"), KeyFile: filepath.Join(dir, "agg.key")}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	client, err := tlsx.Client(&config.TLSOptions{Enabled: true, CAFile: filepath.Join(dir, "ca.crt")}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := testutil.Handshake(t, server, client); err != nil {
+		t.Fatalf("issued chain: %v", err)
+	}
+	if code, _, stderr := runCommand(t, "tls", steps[0]...); code != 1 || !strings.Contains(stderr, "exists") {
+		t.Fatalf("second ca: exit %d: %s", code, stderr)
+	}
+}

cmd/lw/help.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"fmt"
-	"os"
+	"strings"
 
 	"github.com/lixenwraith/logwisp/internal/version"
 )
@@ -15,19 +15,21 @@ Usage:
                                 line for line, like cat; exits at end of input
   lw -c <file> [options]        Run the file's pipelines
   lw --check [options]          Build every pipeline and plugin, report, exit
+  lw --dump [options]           Print the effective configuration as TOML, exit
   lw help | -h | --help
   lw --version
 
-Subcommands:
+Commands (lw <command> -h lists its subcommands):
-  lw auth <command>             SCRAM credentials files, bearer tokens and tcp
+%s
-                                sink viewing; lw auth -h lists commands
-
 Scalar configuration keys are settable as flags using their TOML paths:
   --<path>=<value>              e.g. --logging.level=debug
 
 Common options:
   -c, --config <path>           Configuration file (default: ./logwisp.toml)
       --quiet                   Suppress console output
+      --color [<when>]          Level names in color on console sinks:
+                                auto (a terminal, NO_COLOR unset; default),
+                                always (bare --color) or never
       --status_reporter=<bool>  Periodic status logging (default: true with
                                 a configuration file)
       --auto_reload=<bool>      Config hot reload on file change (default: false)
@@ -39,6 +41,9 @@ at warn without one):
       --logging.file.directory=<path>
 
 Pipelines (replace the file's pipelines; see doc/cli.md):
+      --preset <name>[,k=v...]  Start a pipeline with a preset: pipe, tail,
+                                serve, edge, aggregator; lw preset <name> -h
+                                lists its keys, lw preset <name> prints it
       --pipeline <name>         Start a pipeline; earlier specs go to "cli"
       --source <spec>           TYPE[,key=value...], repeatable
       --sink <spec>             e.g. http,host=0.0.0.0,port=8080, repeatable
@@ -56,14 +61,19 @@ Examples:
   journalctl -f | lw --sink http,host=127.0.0.1,port=8080
                                                   Serve stdin as a live stream
   lw --source file,directory=/var/log/app,pattern='*.log' --format txt
+  lw --preset tail,path=/var/log/app              Follow a directory's files
+  lw --preset serve,path=/var/log/app,tls=self,users=users.toml
+                                                  HTTPS stream, SCRAM logins
+  lw --preset edge,path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\
+password_file=edge-01.pass                        Forward to an aggregator
 
 Environment:
   LOGWISP_<PATH>                Config path, '.' -> '_', uppercase
                                 e.g. LOGWISP_LOGGING_LEVEL=debug
   LOGWISP_CONFIG_FILE           Configuration file path
   LOGWISP_CONFIG_DIR            Configuration directory
-  LOGWISP_PIPELINE, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N], LOGWISP_FILTER[_N],
+  LOGWISP_PIPELINE, LOGWISP_PRESET, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N],
-  LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT
+  LOGWISP_FILTER[_N], LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT
                                 One pipeline, specs as the flags (_N adds
                                 more); ignored when a pipeline flag is given
 
@@ -77,22 +87,10 @@ Exit codes:
   2  configuration file not found
 `
 
-// handleHelp prints usage and exits if a help request is present in args
-func handleHelp(args []string) {
-	if len(args) > 0 && args[0] == "help" {
-		printHelp()
-	}
-	for _, arg := range args {
-		if arg == "--" {
-			break // end of flags
-		}
-		if arg == "-h" || arg == "--help" {
-			printHelp()
-		}
-	}
-}
-
 func printHelp() {
-	fmt.Printf(helpText, version.Short())
+	var list strings.Builder
-	os.Exit(0)
+	for _, c := range commands {
+		fmt.Fprintf(&list, "  lw %-26s %s\n", c.name+" <command>", c.summary)
+	}
+	fmt.Printf(helpText, version.Short(), list.String())
 }

cmd/lw/main.go

@@ -20,19 +20,23 @@ import (
 var logger *log.Logger
 
 func main() {
-	// Before handleHelp, so `lw auth <command> -h` prints the auth usage
+	inv, err := parseCommandLine(os.Args[1:])
-	if len(os.Args) > 1 && os.Args[1] == "auth" {
+	switch {
-		os.Exit(runAuth(os.Args[2:], os.Stdout, os.Stderr))
+	case err != nil:
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	case inv.command != nil:
+		os.Exit(inv.command.run(inv.args, os.Stdout, os.Stderr))
+	case inv.help:
+		printHelp()
+		os.Exit(0)
 	}
 
 	// --- 1. Initial setup ---
 	// Emulates nohup
 	signal.Ignore(syscall.SIGHUP)
 
-	// Before config parsing: the loader has no help flag
+	manager, err := config.Load(inv.load)
-	handleHelp(os.Args[1:])
-
-	manager, err := config.Load(os.Args[1:])
 	if err != nil {
 		if errors.Is(err, config.ErrConfigNotFound) {
 			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -57,6 +61,9 @@ func main() {
 	if cfg.Check {
 		os.Exit(checkConfig(cfg))
 	}
+	if cfg.Dump {
+		os.Exit(dumpConfig(cfg, os.Stdout))
+	}
 
 	if err := initializeLogger(cfg); err != nil {
 		FatalError(1, "Failed to initialize logger: %v\n", err)

cmd/lw/test_helpers_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"testing"
 
 	"github.com/lixenwraith/logwisp/internal/config"
@@ -19,7 +20,11 @@ func testLogger(t *testing.T) {
 func loadTestConfig(t *testing.T, path string, args ...string) *config.Config {
 	t.Helper()
 	testutil.ClearEnvPrefix(t, "LOGWISP_")
-	m, err := config.Load(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...))
+	inv, err := parseCommandLine(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := config.Load(inv.load)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -30,3 +35,15 @@ func loadTestConfig(t *testing.T, path string, args ...string) *config.Config {
 	}
 	return cfg
 }
+
+// runCommand runs `lw NAME ARGS...` and returns its exit status and output
+func runCommand(t *testing.T, name string, args ...string) (code int, stdout, stderr string) {
+	t.Helper()
+	inv, err := parseCommandLine(append([]string{name}, args...))
+	if err != nil || inv.command == nil {
+		t.Fatalf("lw %s: not a command: %v", name, err)
+	}
+	var out, errOut bytes.Buffer
+	code = inv.command.run(inv.args, &out, &errOut)
+	return code, out.String(), errOut.String()
+}

cmd/lw/tls.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"crypto"
+	"crypto/x509"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/lixenwraith/logwisp/internal/tlsx"
+)
+
+// maxDays keeps a validity well inside time.Duration, whose overflow would
+// date a certificate in the past
+const maxDays = 36500
+
+var tlsCommands = []subcommand{
+	{"ca", "-dir DIR [-name NAME] [-days N]",
+		"Create a certificate authority, DIR/ca.crt and DIR/ca.key, that signs certificates only",
+		[]string{"dir"}, defineCA},
+	{"cert", "-ca-dir DIR -name NAME [-host NAME,...] [-server] [-client] [-days N] [-out DIR]",
+		"Issue a certificate and key, OUT/NAME.crt and OUT/NAME.key, from DIR/ca.crt and DIR/ca.key",
+		[]string{"ca-dir", "name"}, defineCert},
+}
+
+func defineCA(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
+	dir := fs.String("dir", "", "`directory` to write ca.crt and ca.key in, created when missing")
+	name := fs.String("name", "logwisp CA", "subject common `name`")
+	days := fs.Int("days", int(tlsx.CAValidity/(24*time.Hour)), "validity in `days`")
+	return func(_, stderr io.Writer) error {
+		if *days < 1 || *days > maxDays {
+			return usageError(fmt.Sprintf("-days must be 1 to %d", maxDays))
+		}
+		cert, key, err := tlsx.NewCA(*name, time.Duration(*days)*24*time.Hour)
+		if err != nil {
+			return err
+		}
+		if err := writePair(*dir, "ca", cert, key); err != nil {
+			return err
+		}
+		fmt.Fprintf(stderr, "wrote %s and %s, valid until %s\n"+
+			"ca.crt verifies: dialers' tls.ca_file, listeners' tls.client_ca_file; with ca.key, it signs: lw tls cert, tls.issuer_cert_file\n",
+			filepath.Join(*dir, "ca.crt"), filepath.Join(*dir, "ca.key"), cert.NotAfter.UTC().Format(time.DateOnly))
+		return nil
+	}
+}
+
+func defineCert(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
+	caDir := fs.String("ca-dir", "", "`directory` holding ca.crt and ca.key (lw tls ca)")
+	name := fs.String("name", "", "subject common `name`, the mtls identity, and the file names")
+	hosts := fs.String("host", "", "DNS `names` and IP addresses, ',' between them (default with -server: -name)")
+	server := fs.Bool("server", false, "for listeners (TLS server authentication)")
+	client := fs.Bool("client", false, "for dialers presenting a certificate (TLS client authentication)")
+	days := fs.Int("days", int(tlsx.LeafValidity/(24*time.Hour)), "validity in `days`, at most the CA's")
+	out := fs.String("out", "", "`directory` to write in (default: -ca-dir)")
+	return func(_, stderr io.Writer) error {
+		switch {
+		case !*server && !*client:
+			return usageError("-server, -client or both is required")
+		case *days < 1 || *days > maxDays:
+			return usageError(fmt.Sprintf("-days must be 1 to %d", maxDays))
+		case strings.ContainsAny(*name, `/\`) || *name == "ca" || strings.HasPrefix(*name, "."):
+			return usageError(fmt.Sprintf("-name %q cannot name the files", *name))
+		}
+		issuer, signer, err := tlsx.LoadIssuer(filepath.Join(*caDir, "ca.crt"), filepath.Join(*caDir, "ca.key"))
+		if err != nil {
+			return err
+		}
+		leaf := tlsx.Leaf{Name: *name, Server: *server, Client: *client, Validity: time.Duration(*days) * 24 * time.Hour}
+		for h := range strings.SplitSeq(*hosts, ",") {
+			if h = strings.TrimSpace(h); h != "" {
+				leaf.Hosts = append(leaf.Hosts, h)
+			}
+		}
+		if len(leaf.Hosts) == 0 && *server {
+			leaf.Hosts = []string{*name}
+		}
+		key, err := tlsx.NewKey()
+		if err != nil {
+			return err
+		}
+		cert, err := leaf.Issue(&key.PublicKey, issuer, signer)
+		if err != nil {
+			return err
+		}
+		dir := *out
+		if dir == "" {
+			dir = *caDir
+		}
+		if err := writePair(dir, *name, cert, key); err != nil {
+			return err
+		}
+		fmt.Fprintf(stderr, "wrote %s and %s for %v, valid until %s, pin_sha256 %s\n",
+			filepath.Join(dir, *name+".crt"), filepath.Join(dir, *name+".key"),
+			append(leaf.Hosts, "CN="+*name), cert.NotAfter.UTC().Format(time.DateOnly), tlsx.PinSHA256(cert))
+		return nil
+	}
+}
+
+// writePair writes NAME.crt (0644) and NAME.key (0600) in dir, refusing to
+// replace either: a reissued key would silently break every pin and peer.
+func writePair(dir, name string, cert *x509.Certificate, key crypto.PrivateKey) error {
+	keyPEM, err := tlsx.EncodeKey(key)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return err
+	}
+	certPath, keyPath := filepath.Join(dir, name+".crt"), filepath.Join(dir, name+".key")
+	for _, p := range []string{certPath, keyPath} {
+		if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
+			return fmt.Errorf("%s exists; remove it to replace it", p)
+		}
+	}
+	if err := writeNew(keyPath, keyPEM, 0o600); err != nil {
+		return err
+	}
+	if err := writeNew(certPath, tlsx.EncodeCert(cert), 0o644); err != nil {
+		os.Remove(keyPath)
+		return err
+	}
+	return nil
+}
+
+func writeNew(path string, data []byte, mode os.FileMode) error {
+	f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(data)
+	if err == nil {
+		err = f.Sync()
+	}
+	if cerr := f.Close(); err == nil {
+		err = cerr
+	}
+	if err != nil {
+		os.Remove(path)
+	}
+	return err
+}

config/logwisp.toml

@@ -26,6 +26,7 @@
 quiet = false                           # No log of lw's own; pipeline output still flows
 status_reporter = true                  # Periodic status logging (30s, DEBUG level); false without a file
 auto_reload = false                     # Watch file; read at startup (signals always reread it)
+color = "auto"                          # auto|always|never: level names on console sinks; auto on a terminal, NO_COLOR unset
 
 ###############################################################################
 ### Logging (LogWisp's internal operational logging)
@@ -93,10 +94,12 @@ name = "default"
 ### TLS (shared shape; applies to every network source and sink)
 ###
 ### Listeners  (tcp/http sinks, tcp_chain/http_chain sources):
-###   cert_file + key_file are REQUIRED; client_auth + client_ca_file enable mTLS.
+###   cert_file + key_file, self_signed, or issuer_cert_file + issuer_key_file
+###   (exactly one); client_auth + client_ca_file enable mTLS.
 ### Dialers    (tcp_chain/http_chain sinks):
-###   ca_file + server_name verify the server; cert_file + key_file present a
+###   ca_file + server_name, or pin_sha256, verify the server; cert_file +
-###   client identity for mTLS.
+###   key_file present a client identity for mTLS.
+### `lw tls ca` / `lw tls cert` make a CA and certificates.
 ###
 ### enabled = false      Master switch
 ### cert_file = ""       Local certificate
@@ -105,7 +108,15 @@ name = "default"
 ### client_ca_file = ""  Listener: CA bundle verifying client certs
 ### ca_file = ""         Dialer: CA bundle verifying the server (empty = system)
 ### server_name = ""     Dialer: SNI / name to verify (empty = configured host)
+### self_signed = false  Listener: make a self-signed certificate at startup;
+###                      it logs the pin_sha256 dialers set (changes on restart)
+### issuer_cert_file = "" Listener: make one at startup signed by this CA ...
+### issuer_key_file = ""   ... and its key (lw tls ca writes both)
+### hosts = []           Listener: names/addresses of a made certificate, beyond
+###                      host, os.Hostname(), localhost, 127.0.0.1, ::1
 ### insecure_skip_verify = false   Dialer: disable verification (never in prod)
+### pin_sha256 = ""      Dialer: sha256//BASE64 of the server key, in place of
+###                      ca_file; ';' between several
 ### min_version = "1.3"  "1.2" or "1.3". No max_version, no cipher_suites.
 ###
 ### TLS alone is a CA membership check: ANY certificate the CA signed is
@@ -257,6 +268,7 @@ type = "console"
 target = "stdout"                       # stdout|stderr ("split" NOT supported)
 # buffer_size = 1000
 # escape = "auto"                       # auto|always|never: control characters as <hex>, auto on a terminal
+# color = "auto"                        # auto|always|never: level names in color (default: top-level color)
 
 ## File sink (rotating)
 # [[pipelines.plugin_sinks]]

doc/README.md

@@ -31,8 +31,7 @@ streams, or downstream LogWisp nodes.
   manual, [`lw.1`](lw.1)
 - [Operations](operations.md): running, monitoring, tuning, troubleshooting
 - [To Do](todo.md): planned work in priority order: network access control
-  and the PROXY protocol, a hardening review of the config libraries,
+  and the PROXY protocol, packaging
-  packaging
 
 A fully annotated configuration covering every option lives at
 [`config/logwisp.toml`](../config/logwisp.toml).

doc/chaining.md

@@ -26,7 +26,9 @@ nodes can filter and reformat as if the entries were local.
 
 Both chain sources can feed a single pipeline (fan-in) whose sinks then fan the
 merged stream out. `test/chain-test.sh` builds the two-independent-pipelines
-variant; `test/chain-aggregate-test.sh` builds the fan-in variant.
+variant; `test/chain-aggregate-test.sh` builds the fan-in variant. The `edge`
+and `aggregator` [presets](cli.md#presets) build an authenticated TLS link of
+either transport from a few keys.
 
 ## Node Identity
 

doc/cli.md

@@ -4,7 +4,11 @@
 lw [options] [< input]
 lw -c <file> [options]
 lw --check [options]
+lw --dump [options]
+lw --preset NAME[,key=value...] [options]
 lw auth <command> [flags]
+lw tls <command> [flags]
+lw preset <name> [flags]
 lw help | -h | --help
 lw --version
 ```
@@ -14,9 +18,10 @@ stdout, line for line, until the end of input ([Built-in
 Defaults](#built-in-defaults)). With a file it runs the file's pipelines as a
 service.
 
-`lw auth` manages SCRAM credentials and logs in to `scram` listeners; see
+A [preset](#presets) builds a common pipeline from a few keys. `lw auth`
-[below](#lw-auth). There is no certificate-generation command: use
+manages SCRAM credentials and logs in to `scram` listeners
-`openssl` or your PKI tooling — see [Security](security.md#enabling-mtls).
+([below](#lw-auth)); `lw tls` makes a certificate authority and certificates
+([below](#lw-tls)). A command is recognized only as the first argument.
 
 ## Options
 
@@ -36,6 +41,11 @@ pipelines have [their own flags](#pipelines):
   `./logwisp.toml` ([Configuration](configuration.md#file-location))
 - `--check`: build every pipeline and plugin, bind nothing, report and exit;
   see [Usage Patterns](#usage-patterns)
+- `--dump`: print the effective configuration, flags, environment and presets
+  resolved into it, as TOML that `-c` reads back, and exit
+- `--color [auto|always|never]`: level names in color on console sinks,
+  default `auto` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`); bare,
+  `always`. A console sink's own `color` wins ([console sink](sinks.md#console))
 - `--quiet`: silence lw's own log and notices, default `false`; pipeline
   output still flows
 - `--status_reporter=<bool>`: periodic status logging, default `true` with a
@@ -70,7 +80,10 @@ Pipeline flags define whole pipelines. They replace the configuration file's
 pipelines and the built-in default; every other key keeps its precedence.
 
 - `--pipeline NAME` starts a pipeline; specs before the first one go to a
-  pipeline named `cli`
+  pipeline named `cli`, or after the preset that starts them
+- `--preset SPEC` starts a pipeline with a [preset](#presets); later flags add
+  sources, sinks and filters to it, or replace its format, rate limit and
+  heartbeat
 - `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable
 - `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline
 - both `--flag SPEC` and `--flag=SPEC` work; `--` ends options
@@ -129,6 +142,61 @@ lw --pipeline app --source file,directory=/var/log/app \
 
 Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns).
 
+## Presets
+
+`--preset NAME,key=value,...` takes the [SPEC](#pipelines) syntax. A list
+key (`hosts`, `proxy`) takes several values, repeated (`hosts=a,hosts=b`) or
+`,`-separated; any other key takes one. `lw preset NAME [-key value ...]`
+prints the pipeline a preset expands to, ready for a configuration file;
+`lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An
+unknown key fails and lists the valid ones. A `path` is a file, a directory
+(its files) or a glob; a preset that takes one reads stdin without it, and
+`from` (`end`, or `start`) is where reading a file starts.
+
+- `pipe`: stdin to stdout, the built-in default
+  - `format` (`raw`)
+- `tail`: follow files to stdout, like `tail -F`
+  - `path` (required), `from`, `format` (`raw`)
+- `serve`: a live HTTP stream (SSE), an [http sink](sinks.md#http)
+  - `path`, `from`, `format` (`json`), `listen` (`127.0.0.1:8080`)
+  - `tls`: `off` (default), `self` (a self-signed certificate made at
+    startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or
+    `files` (`cert` and `key`); `hosts` adds names to a made certificate
+  - `users`: a credentials file; readers then log in with SCRAM
+  - `proxy`, `viewer=true`: the built-in login and viewer pages, which work
+    only behind the TLS-terminating proxies `proxy` names (a browser cannot
+    bind its login to the TLS channel); there is no standalone viewer
+- `edge`: forward to an aggregator over TLS (`tcp_chain`, or `http_chain`
+  with `transport=http`)
+  - `to` (required), `path`, `from`, `transport` (`tcp`), `node`
+  - verify the aggregator by `ca` (a CA file; default: system roots) or `pin`
+    (its `tls.pin_sha256`), and `server_name`
+  - authenticate by `user` and `password_file` (SCRAM), or a client `cert` and
+    `key` (mTLS): without either the preset refuses to run
+- `aggregator`: receive from edges over TLS, to stdout or files
+  - `listen` (`0.0.0.0:9000`), `transport` (`tcp`), `format` (`json`)
+  - `tls`: `self` (default), `issuer` or `files`, as for `serve`; never `off`
+  - authenticate by `users` (SCRAM) or `client_ca` (mTLS): one is required
+  - `out`: a directory for `aggregate*.log` files; default stdout
+
+```bash
+# follow a directory, level names in color on a terminal
+lw --preset tail,path=/var/log/app
+
+# serve it over HTTPS, self-signed, to SCRAM users; keep the result as a file
+lw --preset serve,path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml
+
+# an aggregator with a certificate made at startup, and an edge pinning it
+lw --preset aggregator,users=/etc/logwisp/users.toml,out=/var/log/edges
+lw --preset edge,path=/var/log/app,to=agg.example.org:9000,pin=sha256//BASE64,\
+user=edge-01,password_file=/etc/logwisp/edge-01.pass
+```
+
+A self-signed aggregator logs its pin at startup (`pin_sha256`, a warning, so
+it shows without a configuration file too). The pin holds across reloads and
+changes when lw restarts; a fleet signs with an issuer instead, and edges set
+`ca`. See [Security](security.md#certificates-made-at-startup).
+
 ## Environment Variables
 
 Configuration paths map to environment variables by replacing `.` with `_`,
@@ -156,6 +224,7 @@ One pipeline can come from the environment, with the [SPEC](#pipelines) syntax.
 Any pipeline flag on the command line makes LogWisp ignore all of them.
 
 - `LOGWISP_PIPELINE`: the pipeline's name, default `cli`
+- `LOGWISP_PRESET`: the [preset](#presets) that starts it
 - `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each
   - `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered
     one in numeric order; likewise `LOGWISP_SINK_N` and `LOGWISP_FILTER_N`
@@ -211,7 +280,7 @@ service untouched; see [Configuration](configuration.md#hot-reload).
 ## Exit Codes
 
 - `0`: the end of input, a clean shutdown (`SIGINT`, `SIGTERM`), `--version` /
-  `--help`, or a valid `--check`
+  `--help`, `--dump`, or a valid `--check`
 - `1`: general error: a configuration load or validation failure (`--check`
   included), a logger init failure, a service bootstrap failure
 - `2`: an explicitly requested configuration file is not found
@@ -239,8 +308,8 @@ lw < app.log > copy.log
   - A line over 1 MiB continues in the next entry; no byte is lost.
 - Nothing is dropped: a slow reader slows the reading of stdin.
 - On a terminal, control characters are written as `<hex>` (`ESC` becomes
-  `<1b>`); pipes and files get the bytes unchanged
+  `<1b>`) and level names are in color; pipes and files get the bytes
-  ([console sink](sinks.md#console)).
+  unchanged ([console sink](sinks.md#console)).
 - When a console source reads a terminal, lw says so once on stderr, unless
   `--quiet`: `lw: reading standard input; Ctrl-D ends it (lw --help for usage)`.
 
@@ -255,10 +324,9 @@ values from the file, the environment or flags win.
 finishes when all its sources have ended, and once every pipeline has
 finished lw shuts down gracefully and exits 0. Only a console source ends on
 its own: file and network sources run until a signal, and so does a pipeline
-holding one. Console and file sinks write what is queued before the
+holding one. Every sink writes what is queued before the exit; the `http`
-exit. The `http` and `tcp` sinks do not yet flush their clients' queues, so a
+and `tcp` sinks give each connected client `write_timeout_ms`, at most 2 s,
-connected client can miss the last entries of a finite input
+to take it.
-([To Do](todo.md)).
 
 ## Usage Patterns
 
@@ -363,8 +431,9 @@ Neither touches a running LogWisp: send `SIGHUP`, since `auto_reload` does not
 watch the credentials file.
 
 **`token`** and **`stream`** build the same TLS and SCRAM client as a chain
-sink. TLS flags: `-ca-file` (default: system roots), `-server-name` (default:
+sink. TLS flags: `-ca-file` (default: system roots) or `-pin-sha256` (the
-the host), and `-cert-file` / `-key-file` for a listener with `tls.client_auth`.
+`tls.pin_sha256` a self-signed listener logs), `-server-name` (default: the
+host), and `-cert-file` / `-key-file` for a listener with `tls.client_auth`.
 There is no flag to skip verification: an unverified server could relay the
 login. Redirects are not followed. `-unbound` logs in to an `http` sink behind
 a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's
@@ -403,3 +472,32 @@ curl --cacert ca.crt -H @<(printf 'Authorization: Bearer %s\n' \
 lw auth stream -addr HOST:PORT -user viewer -password-file viewer.pass \
   -ca-file ca.crt
 ```
+
+## `lw tls`
+
+Makes ECDSA P-256 certificates for `tls` blocks. It never replaces a file
+(remove one to replace it) and writes keys `0600`, certificates `0644`.
+
+```
+lw tls ca   -dir DIR [-name NAME] [-days 3650]
+lw tls cert -ca-dir DIR -name NAME [-host NAME,...] [-server] [-client]
+            [-days 397] [-out DIR]
+```
+
+- `ca`: `DIR/ca.crt` and `DIR/ca.key`, a CA that signs leaves only (path
+  length 0). `ca.crt` is what dialers set as `tls.ca_file` and listeners as
+  `tls.client_ca_file`; with `ca.key` it is a listener's
+  `tls.issuer_cert_file` and `tls.issuer_key_file`.
+- `cert`: `NAME.crt` and `NAME.key`, in `-out` (default: `-ca-dir`).
+  - `-server` for a listener: `-host` lists the DNS names and IP addresses it
+    carries, default `NAME`; `-client` for a dialer presenting a certificate;
+    both may be given.
+  - `NAME` is the subject CN, the identity `auth.type = "mtls"` matches.
+  - Valid 397 days (the longest browsers accept), never past the CA. It prints
+    the certificate's `pin_sha256`.
+
+```bash
+lw tls ca -dir /etc/logwisp/pki
+lw tls cert -ca-dir /etc/logwisp/pki -name agg.example.org -server
+lw tls cert -ca-dir /etc/logwisp/pki -name edge-01 -client
+```

doc/configuration.md

@@ -46,9 +46,12 @@ Top-level keys, each as type and default:
   emit a status report every 30 s at DEBUG level.
 - `auto_reload` (bool, `false`): watch the config file and reload pipelines on
   change.
+- `color` (string, `"auto"`): level names in color on console sinks, which may
+  set their own: `"auto"` (a terminal, `NO_COLOR` unset, `TERM` not `dumb`),
+  `"always"` or `"never"`; see the [console sink](sinks.md#console).
 
-`--version` prints version information and exits; it is not a persistent
+`--version`, `--check` and `--dump` print and exit; they are not persistent
-setting.
+settings.
 
 Note that `status_reporter` writes at DEBUG level, so it produces nothing unless
 `logging.level = "debug"`.
@@ -311,6 +314,7 @@ What a reload applies:
   - pipelines, sources, sinks
   - filters, formatters, rate limits, heartbeats
   - `status_reporter`
+  - `color`
 - Not reloaded:
   - `logging.*`: applied once at startup
   - `quiet`

doc/lw.1

@@ -1,4 +1,4 @@
-.Dd October 3, 2026
+.Dd October 4, 2026
 .Dt LW 1
 .Os
 .Sh NAME
@@ -6,10 +6,12 @@
 .Nd LogWisp log transport
 .Sh SYNOPSIS
 .Nm
-.Op Fl \-check
+.Op Fl \-check | Fl \-dump
 .Op Fl c Ar file
 .Op Fl \-quiet
+.Op Fl \-color Op Ar when
 .Op Fl \- Ns Ar key Ns = Ns Ar value ...
+.Op Fl \-preset Ar spec
 .Op Fl \-pipeline Ar name
 .Op Fl \-source Ar spec ...
 .Op Fl \-sink Ar spec ...
@@ -18,7 +20,7 @@
 .Op Fl \-rate\-limit Ar spec
 .Op Fl \-heartbeat Ar spec
 .Nm
-.Cm auth
+.Cm auth | tls | preset
 .Ar command
 .Op Ar flags
 .Nm
@@ -63,11 +65,13 @@ source runs until shutdown.
 When every pipeline has finished,
 .Nm
 shuts down gracefully and exits 0.
-Console and file sinks first write what they hold; the
+Every sink first writes what it holds; the
 .Cm http
 and
 .Cm tcp
-sinks do not yet, so a client can miss the last lines of a finite input.
+sinks give each client
+.Cm write_timeout_ms ,
+at most 2 seconds, to take it.
 Under a supervisor, standard input is usually
 .Pa /dev/null :
 a console source there ends at once.
@@ -88,6 +92,13 @@ option sets this:
 .Cm always
 or
 .Cm never .
+Its
+.Cm color
+option, by default the top-level
+.Cm color ,
+paints the level name in each line: debug cyan, info green, warn yellow,
+error bold red, trace grey, in the 16 colors every terminal and text console
+renders, with the terminal theme choosing the shades.
 .Pp
 .Nm
 stays in the foreground and logs to standard error; run a configuration as a
@@ -109,6 +120,22 @@ Set any scalar configuration key by its TOML path, such as
 A bare
 .Fl \- Ns Ar key
 sets a boolean to true.
+.It Fl \-color Op Ar when
+Color the level names console sinks write:
+.Cm auto
+.Pq the default: on a terminal, with Ev NO_COLOR unset and Ev TERM not Cm dumb ,
+.Cm always
+.Pq what a bare Fl \-color No means
+or
+.Cm never .
+A console sink's own
+.Cm color
+option takes precedence.
+The codes go into whatever the format makes, JSON included, so output another
+program parses keeps
+.Cm auto
+or
+.Cm never .
 .It Fl \-quiet
 Turn off the log of
 .Nm
@@ -151,6 +178,11 @@ or opening, reading or creating a log file; print
 and exit 0, or the error and exit 1, or 2 when a named configuration file is
 missing
 .Pq see Sx EXIT STATUS .
+.It Fl \-dump
+Print the effective configuration, with the options, environment and
+presets resolved into it, as a file
+.Fl c
+reads back, and exit.
 .It Fl \-version
 Print the version, commit and build time, and exit.
 .It Fl h , Fl \-help , Cm help
@@ -176,7 +208,12 @@ Start a new pipeline.
 Specifications before the first
 .Fl \-pipeline
 belong to a pipeline named
-.Dq cli .
+.Dq cli ,
+or after the preset that starts them.
+.It Fl \-preset Ar spec
+Start the pipeline with a preset, described in
+.Sx Presets ;
+later options add sources, sinks and filters to it, or replace its format.
 .It Fl \-source Ar spec , Fl \-sink Ar spec , Fl \-filter Ar spec
 Add a source, sink or filter; each is repeatable.
 .It Fl \-format Ar spec , Fl \-rate\-limit Ar spec , Fl \-heartbeat Ar spec
@@ -218,6 +255,125 @@ Any pipeline flag on the command line makes
 .Nm
 ignore the pipeline variables in
 .Sx ENVIRONMENT .
+.Ss Presets
+A preset builds a common pipeline from a few keys:
+.Fl \-preset Ar name Ns Op , Ns Ar key Ns = Ns Ar value ... ,
+in the syntax of the other specifications;
+.Nm
+.Cm preset Ar name Op Fl Ns Ar key Ar value ...
+prints the pipeline it expands to, and
+.Fl h
+there lists its keys and defaults.
+A
+.Ar path
+is a file, a directory, whose files are followed, or a glob; without one, a
+preset that takes it reads standard input.
+.Bl -tag -width Ds
+.It Cm pipe Op Cm format
+Standard input to standard output, the pipeline
+.Nm
+runs without a configuration.
+.It Cm tail path Op Cm from , format
+Follow files to standard output, like
+.Ql tail \-F .
+.It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer Oc
+Serve a live HTTP stream on
+.Cm listen
+.Pq default Li 127.0.0.1:8080 .
+.Cm tls
+is
+.Cm off ,
+.Cm self
+.Pq a self-signed certificate made at startup ,
+.Cm issuer
+.Pq one signed by Cm issuer_cert No and Cm issuer_key
+or
+.Cm files
+.Pq Cm cert No and Cm key ;
+.Cm users
+names a credentials file, so readers log in with SCRAM.
+The built-in login and viewer pages,
+.Cm viewer=true ,
+work only behind a TLS-terminating proxy named by
+.Cm proxy ,
+since a browser cannot bind its login to the TLS channel.
+.It Cm edge to Oo Cm path , transport , ca | pin , user , password_file , cert , key , node Oc
+Forward to an aggregator at
+.Cm to
+over TLS, by
+.Cm tcp_chain
+or, with
+.Cm transport=http ,
+.Cm http_chain .
+It refuses to run without credentials:
+.Cm user
+and
+.Cm password_file ,
+or a client
+.Cm cert
+and
+.Cm key .
+.It Cm aggregator Oo Cm listen , transport , tls , users , client_ca , out , format Oc
+Receive from edges on
+.Cm listen
+.Pq default Li 0.0.0.0:9000
+over TLS, never off
+.Pq default Cm tls=self ,
+authenticated by the
+.Cm users
+credentials file or the client certificates
+.Cm client_ca
+vouches for, and write to standard output or to files in
+.Cm out .
+.El
+.Ss Certificate commands
+.Nm
+.Cm tls
+makes ECDSA P-256 certificates; it never replaces a file, and writes keys
+with mode 0600.
+.Bl -tag -width Ds
+.It Cm ca Fl dir Ar dir Oo Fl name Ar name Oc Op Fl days Ar n
+Create a certificate authority,
+.Pa dir/ca.crt
+and
+.Pa dir/ca.key ,
+valid 3650 days.
+.It Cm cert Fl ca-dir Ar dir Fl name Ar name Oo Fl host Ar names Oc Oo Fl server Oc Oo Fl client Oc Oo Fl days Ar n Oc Op Fl out Ar dir
+Issue
+.Pa name.crt
+and
+.Pa name.key
+from that authority, valid 397 days and never past it.
+.Fl host
+lists the DNS names and addresses a server certificate carries
+.Pq default: Ar name ;
+.Ar name
+is also the subject, the identity
+.Cm auth.type=mtls
+checks.
+.El
+.Pp
+A listener needs no files:
+.Cm tls.self_signed=true ,
+or
+.Cm tls.issuer_cert_file
+with
+.Cm tls.issuer_key_file ,
+make its certificate at startup, for
+.Cm tls.hosts ,
+its
+.Cm host ,
+this machine's name, localhost and the loopback addresses.
+Its key lasts until
+.Nm
+exits, across reloads.
+A self-signed listener logs the
+.Cm tls.pin_sha256
+that dialers set in place of
+.Cm tls.ca_file ;
+it changes when
+.Nm
+restarts, so a fleet uses an issuer instead.
 .Ss Authentication commands
 .Nm
 .Cm auth
@@ -258,7 +414,9 @@ The
 .Ar tls-flags
 are
 .Fl ca-file Ar file
-.Pq default: the system roots ,
+.Pq default: the system roots
+or
+.Fl pin-sha256 Ar pin ,
 .Fl server-name Ar name ,
 and
 .Fl cert-file Ar file
@@ -290,6 +448,8 @@ The directory holding
 .Pa logwisp.toml .
 .It Ev LOGWISP_PIPELINE
 The name of the one pipeline the variables below define.
+.It Ev LOGWISP_PRESET
+The preset that starts it.
 All of these are ignored when a pipeline option is on the command line.
 .It Ev LOGWISP_SOURCE , LOGWISP_SINK , LOGWISP_FILTER
 Source, sink and filter specifications, in the syntax of the matching
@@ -299,6 +459,13 @@ counting from 1, adds more, as in
 .Ev LOGWISP_SINK_1 .
 .It Ev LOGWISP_FORMAT , LOGWISP_RATE_LIMIT , LOGWISP_HEARTBEAT
 The pipeline's format, rate limit and heartbeat specifications.
+.It Ev NO_COLOR , TERM
+Set to anything,
+.Ev NO_COLOR
+turns
+.Cm auto
+color off, as does
+.Ev TERM=dumb .
 .El
 .Pp
 Secrets never go in the environment: TLS keys, credentials files and
@@ -357,8 +524,11 @@ as it ends
 the shell reports status 141.
 .Pp
 .Nm
-.Cm auth
+.Cm auth ,
-exits 0 on success or help, 1 on failure, and 2 on a usage error.
+.Cm tls
+and
+.Cm preset
+exit 0 on success or help, 1 on failure, and 2 on a usage error.
 .Sh SIGNALS
 .Bl -tag -width "SIGINT, SIGTERM"
 .It Dv SIGINT , SIGTERM
@@ -405,6 +575,33 @@ $ lw \-\-source 'file,directory=/var/log/app,pattern=*.log,raw=true' \e
     > all.log
 .Ed
 .Pp
+Follow every file in a directory, with the level names in color on a
+terminal:
+.Bd -literal -offset indent
+$ lw \-\-preset tail,path=/var/log/app
+.Ed
+.Pp
+Print the configuration a set of options makes, to keep as a file:
+.Bd -literal -offset indent
+$ lw \-\-preset serve,path=/var/log/app,tls=self \-\-dump > serve.toml
+.Ed
+.Pp
+Ship a directory from an edge to an aggregator: on the aggregator, create
+the authority, its certificate and a user for the edge; copy
+.Pa ca.crt
+and the password file to the edge; then run both:
+.Bd -literal -offset indent
+# lw tls ca \-dir /etc/logwisp/pki
+# lw tls cert \-ca-dir /etc/logwisp/pki \-name agg.example.org \-server
+# lw auth add-user \-credentials /etc/logwisp/users.toml \e
+    \-user edge-01 \-password-file /root/edge-01.pass
+agg# lw \-\-preset aggregator,tls=files,users=/etc/logwisp/users.toml,\e
+cert=/etc/logwisp/pki/agg.example.org.crt,\e
+key=/etc/logwisp/pki/agg.example.org.key,out=/var/log/edges
+edge# lw \-\-preset edge,path=/var/log/app,to=agg.example.org:9000,\e
+ca=/etc/logwisp/pki/ca.crt,user=edge-01,password_file=/root/edge-01.pass
+.Ed
+.Pp
 Check a configuration without binding a port or touching a log file:
 .Bd -literal -offset indent
 $ lw \-\-check \-c /etc/logwisp/logwisp.toml

doc/networking.md

@@ -213,6 +213,9 @@ headers, and entry encoding.
   `client_ca_file` does not contain the issuer of the client certificate.
 - `certificate is not valid for any names` / SAN mismatch — the dialed `host`
   is not covered by the server certificate's SANs; set `server_name`.
+- `server key sha256//... matches no tls.pin_sha256` — the listener is not the
+  one pinned, or a self-signed listener restarted and made a new key: take the
+  pin from its startup warning, or move it to an issuer and `ca_file`.
 - `protocol version not supported` — one side is pinned to `min_version = "1.3"`
   and the other cannot negotiate it.
 - Listeners log handshake failures at WARN with the remote address (the `tcp`

doc/operations.md

@@ -17,8 +17,10 @@ lw exits on its own only at the end of input: once every pipeline has
 finished, a pipeline finishing when all its sources have ended. Only a console
 source ends, when stdin does; file and network sources run until a signal, and
 lw shuts down gracefully, exit 0, either way.
-Console and file sinks write what is queued before the exit; the `http` and
+Every sink writes what is queued before the exit; the `http` and `tcp` sinks
-`tcp` sinks do not yet flush their clients' queues ([To Do](todo.md)). See
+give each client `write_timeout_ms`, at most 2 s, to take it. A reload stops
+the old sinks the same way, so a client that stopped reading delays it by
+that much. See
 [CLI](cli.md#built-in-defaults).
 
 There is no built-in daemon mode. Run LogWisp in the foreground under a

doc/security.md

@@ -8,7 +8,11 @@ configure it, and — equally important — what it does not yet do.
 **Implemented:**
 
 - TLS 1.2 / 1.3 on all network sources and sinks.
-- Server certificate verification by dialers.
+- Server certificate verification by dialers, by a CA or by the pin of the
+  server's key (`pin_sha256`).
+- Certificates without files: listeners make theirs at startup, self-signed
+  or from an issuer CA, and `lw tls` makes a CA and certificates: see
+  [Certificates made at startup](#certificates-made-at-startup).
 - Mutual TLS (client certificate required and verified), at the transport
   layer.
 - Peer identity recorded per session.
@@ -26,16 +30,20 @@ configure it, and — equally important — what it does not yet do.
   [Startup Warnings](#startup-warnings).
 - Unknown configuration keys rejected: a typo in `tls`, `auth` or a table path
   fails startup.
-- Configuration input hardened (lixenwraith/config v0.2.2, reviewed with
+- Configuration input hardened (lixenwraith/config v0.2.2 and
-  adversarial tests and fuzzing like `auth`):
+  lixenwraith/toml v0.1.3, reviewed with adversarial tests and fuzzing like
+  `auth`):
+  - a document creates at most 65536 tables, so a small file cannot exhaust
+    memory, and unescaped control characters are refused in strings and
+    comments;
   - a string in the file is one list entry, so a filter pattern holding a
     comma is not split into patterns that match nothing;
   - the file is opened without blocking, so a FIFO swapped in for it cannot
     hang the watcher or a `SIGHUP` reload;
   - a permission change on the file is reported once and does not stop
     auto-reload;
-  - conversion errors name the key, never the value, so a secret placed in
+  - parse and conversion errors name the key, never the value, so a secret
-    the wrong key does not reach the log.
+    placed in the wrong key does not reach the log.
 
 **Not implemented:**
 
@@ -69,6 +77,11 @@ client_ca_file       = ""
 ca_file              = ""
 server_name          = ""
 insecure_skip_verify = false
+pin_sha256           = ""
+self_signed          = false
+issuer_cert_file     = ""
+issuer_key_file      = ""
+hosts                = []
 min_version          = "1.3"
 ```
 
@@ -76,11 +89,16 @@ Options by role, each with its default:
 
 - Both roles
   - `enabled` (`false`): master switch; when false the whole block is ignored.
-  - `cert_file`: local certificate; **required** for listeners, an optional
+  - `cert_file`: local certificate; a listener needs it, `self_signed` or the
-    client identity for dialers.
+    issuer files, a dialer may present it as its client identity.
   - `key_file`: private key for `cert_file`; set the two together.
   - `min_version` (`"1.3"`): `"1.2"` or `"1.3"`.
 - Listeners
+  - `self_signed` (`false`): make a self-signed certificate at startup.
+  - `issuer_cert_file`, `issuer_key_file`: make one at startup, signed by this
+    CA (`lw tls ca`).
+  - `hosts` (`[]`): names and addresses a made certificate carries, beyond the
+    listener's `host`, `os.Hostname()`, `localhost`, `127.0.0.1` and `::1`.
   - `client_auth` (`false`): require and verify a client certificate (mTLS).
   - `client_ca_file`: CA bundle that verifies client certificates; **required**
     when `client_auth` is true.
@@ -89,6 +107,9 @@ Options by role, each with its default:
   - `server_name` (the configured `host`): SNI and certificate name to verify
     against.
   - `insecure_skip_verify` (`false`): disable server verification.
+  - `pin_sha256`: `sha256//BASE64` of the server's public key, curl's
+    `--pinnedpubkey` form, `;` between several; it replaces the CA and name
+    checks.
 
 Listeners are the `tcp` and `http` sinks and the `tcp_chain` and `http_chain`
 sources; dialers are the `tcp_chain` and `http_chain` sinks.
@@ -102,7 +123,12 @@ sources; dialers are the `tcp_chain` and `http_chain` sinks.
 
 Misconfiguration fails at plugin construction, before the pipeline starts:
 
-- a listener with `enabled = true` and no `cert_file`/`key_file`
+- a listener with `enabled = true` and none of `cert_file`/`key_file`,
+  `self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one; an
+  issuer that is no CA (`lw tls ca` makes one)
+- `pin_sha256` with `ca_file` or `insecure_skip_verify`, or not of the
+  `sha256//BASE64` form; a listener key on a dialer, or `pin_sha256` on a
+  listener
 - `client_auth = true` with no `client_ca_file`
 - a dialer with only one of `cert_file` / `key_file`
 - a certificate or key that will not load, or a CA file containing no
@@ -298,20 +324,53 @@ Errors read like `auth: type "mtls" requires tls.client_auth`.
 Some settings work but are usually mistakes. Each plugin reports them at WARN
 when it is constructed, so startup and every reload repeat them:
 
-- a certificate in `cert_file`, `ca_file` or `client_ca_file` that has expired,
+- a certificate in `cert_file`, `ca_file`, `client_ca_file` or
-  is not yet valid, or expires within 30 days
+  `issuer_cert_file` that has expired, is not yet valid, or expires within 30
+  days
+- a self-signed listener, with the `pin_sha256` its dialers need
 - `insecure_skip_verify` on a dialer
 - an `allow_patterns` entry not anchored at both ends of every alternative
   (`^a|b$` admits `a…` and `…b`)
-- a `key_file`, `credentials_file` or `password_file` every local user can read
+- a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every
+  local user can read
   (once per path per process)
 
+## Certificates made at startup
+
+A listener needs no certificate files. With `self_signed = true`, or with
+`issuer_cert_file` and `issuer_key_file`, it makes an ECDSA P-256 key once per
+process and, at startup and every reload, a certificate for it valid 397 days
+(never past the issuer), for `hosts`, its `host` (unless a wildcard),
+`os.Hostname()`, `localhost`, `127.0.0.1` and `::1`.
+
+- **Self-signed.** No CA can vouch for it, so dialers pin its key:
+  `pin_sha256`, which the listener logs at WARN on every start. A reload
+  keeps the key and so the pin; a restart changes both, and every dialer must
+  be given the new pin. Suits one aggregator and a few edges, a test, or a
+  stream read by `curl --pinnedpubkey`.
+- **Issuer.** `lw tls ca` makes a CA once; the listener signs its own
+  certificate with it, and dialers verify it with `ca_file = ca.crt`, by name,
+  across restarts. The CA key then lives on the listener's host: give each
+  listener group its own CA, or issue certificates elsewhere with
+  `lw tls cert` and use `cert_file`.
+- **Pins.** A pin is the SHA-256 of the server's public key, not of its
+  certificate, so it survives reissue. `pin_sha256` accepts several, `;`
+  between them, for a rotation. It replaces chain, name and validity checks:
+  whoever holds the key is the server. `scram` dialers still bind every login
+  to that certificate.
+
+```bash
+lw tls ca -dir /etc/logwisp/pki                       # ca.crt, ca.key (0600)
+lw tls cert -ca-dir /etc/logwisp/pki -name agg.example.org -server
+lw tls cert -ca-dir /etc/logwisp/pki -name edge-01 -client
+```
+
 ## Enabling mTLS
 
 ### 1. Generate a CA and certificates
 
-LogWisp does not generate certificates; use `openssl`, `cfssl`, `step-cli`, or
+`lw tls` makes them ([above](#certificates-made-at-startup)); with `openssl`,
-your existing PKI.
+`cfssl`, `step-cli` or an existing PKI:
 
 ```bash
 # CA

doc/sinks.md

@@ -27,6 +27,7 @@ type = "console"
 target      = "stdout"
 buffer_size = 1000
 escape      = "auto"
+color       = "auto"
 ```
 
 Options, each as type and default:
@@ -36,6 +37,9 @@ Options, each as type and default:
 - `escape` (string, `auto`): write control characters as `<hex>` of their
   bytes, `ESC` as `<1b>`: `auto` when the output is a terminal, `always`, or
   `never`.
+- `color` (string, the top-level `color`, itself `auto`): paint level names:
+  `auto` on a terminal with `NO_COLOR` unset and `TERM` not `dumb`, `always`,
+  or `never`.
 
 > `split` is **not** a valid target for this sink and is rejected at startup.
 > Level-based splitting exists only for LogWisp's own application log
@@ -50,6 +54,14 @@ Payloads are written as the formatter made them, one record per line.
   characters, such as the joiners emoji and Indic scripts need, are kept.
   `never` keeps an application's own ANSI colours; under `auto`, pipes and
   files get the bytes unchanged.
+- Color: the first whole word in a line, in any case, that names the entry's
+  level is painted: `DEBUG`/`DBG` cyan (blue is hard to read on black),
+  `INFO`/`INF` green, `WARNING`/`WARN` yellow, `ERROR`/`ERR`/`FATAL` bold red,
+  `TRACE` grey. The colors are ANSI 16, so the terminal's theme picks the
+  shades and a Linux or BSD text console renders them; the codes are the
+  sink's own, written after escaping. A line whose source found no level
+  stays plain. The codes go into whatever the format makes, JSON included:
+  for output another program parses keep `auto` or `never`, not `always`.
 - Backpressure: the sink never drops. When its output is slow (a slow reader,
   a paused terminal, a stalled log collector) the pipeline waits for it, with
   its other sinks. A service whose stdout may stall writes to a `file` sink
@@ -205,10 +217,11 @@ Options, each as type and default:
 - Clients whose session has been idle-expired by the session manager are
   evicted by the broker. With the idle comment above, that reaches only a peer
   that has stopped accepting bytes on a sink configured `write_timeout_ms = 0`.
-- On shutdown, connected clients receive
+- On shutdown (a reload too), each stream writes its queue and what remains
-  `event: disconnect / data: {"reason":"server_shutdown"}`. Entries still
+  of the sink's input, then
-  queued for them are not flushed first, so a client can miss the last ones
+  `event: disconnect / data: {"reason":"server_shutdown"}`, on its own and
-  of a finite input ([To Do](todo.md)).
+  within `write_timeout_ms`, at most 2 s; a client still behind is cut then,
+  and one that stopped reading costs the others nothing.
 - HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1.
 
 **Status endpoint** returns service and version identity, host, port, TLS flag,
@@ -287,8 +300,8 @@ Options, each as type and default:
   whole timeout, so the client is disconnected immediately rather than retried.
 - A client whose send queue is full has that event dropped (`dropped_writes`)
   and stays connected.
-- On shutdown, entries still queued for clients are not flushed first, as
+- On shutdown, as with the `http` sink, each client receives what is queued,
-  with the `http` sink.
+  within `write_timeout_ms`, at most 2 s, before it is disconnected.
 - With TLS enabled the handshake runs under a 10 s bound *after* the
   `max_connections` check, so concurrent handshakes are bounded too.
 - With an `auth` block, authorization runs after that handshake and *before*

doc/todo.md

@@ -122,23 +122,7 @@ connection level, below TLS, so a refused peer costs no handshake.
   - Denied and allowed clients; per-client SCRAM budgets behind one proxy
     address.
 
-## 2. Flush network sinks at shutdown
+## 2. Packaging: AUR, FreeBSD ports, Debian
-
-At the end of a finite input lw shuts down, and the `http` and `tcp` sinks
-disconnect their clients without writing the entries still queued for them, so
-a client can miss the last ones. On `Stop`, each sink's broker should first move
-its input queue into the client queues, then let every client writer drain its
-queue within a bound (`write_timeout_ms`) before the disconnect frame. Verify
-with a finite stdin into each sink and a connected client counting lines.
-
-## 3. Bump `lixenwraith/toml`
-
-Its hardening commits (a table budget, TOML 1.1 escapes, control characters
-refused outside strings, bounded errors that never print a value) wait to merge
-in that repository. Then bump it here and drop this item; security.md's library
-note already covers it.
-
-## 4. Packaging: AUR, FreeBSD ports, Debian
 
 The foundation exists:
 - The `lw` binary name, free in Arch (official repositories and AUR), Ubuntu

go.mod

@@ -4,8 +4,10 @@ go 1.27.1
 
 require (
 	github.com/lixenwraith/auth v0.0.0-20261002191639-3ed0e995807c
+	github.com/lixenwraith/color v0.1.3
 	github.com/lixenwraith/config v0.2.2
 	github.com/lixenwraith/log v0.1.11
+	github.com/lixenwraith/terminal v0.1.9
 	github.com/lixenwraith/toml v0.1.3
 	golang.org/x/term v0.46.0
 )

go.sum

@@ -2,10 +2,14 @@ github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63Y
 github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
 github.com/lixenwraith/auth v0.0.0-20261002191639-3ed0e995807c h1:zwpVa85g267vo13UCzCao4lfVgUPaU79CmxPW9OdF8w=
 github.com/lixenwraith/auth v0.0.0-20261002191639-3ed0e995807c/go.mod h1:411JUqnbQgy9jgoFV+fH91TfVUEfzYA4dyPv7f4c264=
+github.com/lixenwraith/color v0.1.3 h1:vSuvtU4zh1mIZS/7UGHmMdsTr+a7eCjUYGbou7kzFn4=
+github.com/lixenwraith/color v0.1.3/go.mod h1:L/I3Vxp6pqslAIzWyZGgKCm2lyStHCA4vKaUoMUm7bU=
 github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZgOc=
 github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg=
 github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKhU=
 github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg=
+github.com/lixenwraith/terminal v0.1.9 h1:WPxLdqdzZoqtPgTb86oEwE66lF1+iPsJ/gpfo6h/rik=
+github.com/lixenwraith/terminal v0.1.9/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44=
 github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI=
 github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0=
 golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=

internal/authz/authz.go

@@ -129,8 +129,9 @@ func New(o *config.AuthOptions, tlsCfg *tls.Config, role Role, transport Transpo
 	if tlsCfg == nil && (o.Type != MethodSCRAM || len(o.TrustedProxies) == 0) {
 		return nil, fmt.Errorf("auth: type %q requires tls.enabled", o.Type)
 	}
-	if role == RoleDialer && tlsCfg != nil && tlsCfg.InsecureSkipVerify {
+	if role == RoleDialer && tlsCfg != nil && tlsCfg.InsecureSkipVerify && tlsCfg.VerifyPeerCertificate == nil {
-		// An unverified server makes identities claims and exposes credentials
+		// An unverified server makes identities claims and exposes credentials;
+		// tls.pin_sha256 verifies it in VerifyPeerCertificate instead
 		return nil, fmt.Errorf("auth: type %q cannot be used with tls.insecure_skip_verify", o.Type)
 	}
 

internal/authz/authz_test.go

@@ -167,10 +167,13 @@ func TestNewValidation(t *testing.T) {
 	}
 
 	// The rows' bases are valid: a dialer needs TLS but not client_auth, as it
-	// pins the server's identity
+	// pins the server's identity; tls.pin_sha256 verifies in place of the chain
+	pinned := &tls.Config{InsecureSkipVerify: true, VerifyPeerCertificate: func([][]byte, [][]*x509.Certificate) error { return nil }}
 	for _, o := range []*config.AuthOptions{{Type: MethodMTLS}, dialer(config.AuthOptions{})} {
-		if _, err := New(o, &tls.Config{}, RoleDialer, TCP); err != nil {
+		for _, c := range []*tls.Config{{}, pinned} {
-			t.Fatalf("dialer policy %s rejected: %v", o.Type, err)
+			if _, err := New(o, c, RoleDialer, TCP); err != nil {
+				t.Fatalf("dialer policy %s rejected: %v", o.Type, err)
+			}
 		}
 	}
 	if _, err := New(listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, RoleListener, HTTP); err != nil {

internal/config/config.go

@@ -7,8 +7,13 @@ type Config struct {
 	// Top-level flags for application control
 	ShowVersion bool `toml:"version"`
 	Check       bool `toml:"check"` // build every plugin, report, exit: lw --check
+	Dump        bool `toml:"dump"`  // print the effective configuration, exit: lw --dump
 	Quiet       bool `toml:"quiet"`
 
+	// Console sinks without their own color: "auto" (a terminal, NO_COLOR
+	// unset, TERM not dumb), "always", "never"
+	Color string `toml:"color"`
+
 	// Runtime behavior flags
 	StatusReporter   bool `toml:"status_reporter"`
 	ConfigAutoReload bool `toml:"auto_reload"`
@@ -85,7 +90,7 @@ type PipelineConfig struct {
 type FlowConfig struct {
 	Heartbeat *HeartbeatConfig `toml:"heartbeat"`
 	RateLimit *RateLimitConfig `toml:"rate_limit"`
-	Filters   []FilterConfig   `toml:"filters"`
+	Filters   []FilterConfig   `toml:"filters,omitempty"`
 	Format    *FormatConfig    `toml:"format"`
 }
 
@@ -105,9 +110,9 @@ type HeartbeatConfig struct {
 // FormatConfig is a polymorphic struct representing log entry formatting options
 type FormatConfig struct {
 	Type            string `toml:"type"` // "json", "txt", "raw"
-	Flags           int64  `toml:"flags"`
+	Flags           int64  `toml:"flags,omitempty"`
-	TimestampFormat string `toml:"timestamp_format"`
+	TimestampFormat string `toml:"timestamp_format,omitempty"`
-	SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell"
+	SanitizerPolicy string `toml:"sanitizer_policy,omitempty"` // "raw", "json", "txt", "shell"
 }
 
 // --- Rate Limit Options ---
@@ -245,6 +250,7 @@ type ConsoleSinkOptions struct {
 	Target     string `toml:"target"` // "stdout", "stderr"
 	BufferSize int64  `toml:"buffer_size"`
 	Escape     string `toml:"escape"` // control characters as <hex>: "auto" (on a terminal), "always", "never"
+	Color      string `toml:"color"`  // level names in color: "auto", "always", "never"; default: the top-level color
 }
 
 // FileSinkOptions defines settings for a file-based sink
@@ -366,9 +372,10 @@ type AuthOptions struct {
 // --- TLS Options ---
 
 // TLSOptions is one shape for both roles. Listeners present cert_file and
-// key_file and verify clients with client_auth and client_ca_file; dialers
+// key_file, or a certificate issued at startup (self_signed, or from the
-// verify the server with ca_file and server_name and may present
+// issuer files), and verify clients with client_auth and client_ca_file;
-// cert_file and key_file.
+// dialers verify the server with ca_file and server_name, or pin_sha256, and
+// may present cert_file and key_file.
 type TLSOptions struct {
 	Enabled bool `toml:"enabled"`
 
@@ -376,6 +383,14 @@ type TLSOptions struct {
 	CertFile string `toml:"cert_file"`
 	KeyFile  string `toml:"key_file"`
 
+	// Listeners without cert_file: a certificate on a key made at startup,
+	// self-signed or signed by the issuer CA, for hosts plus the host option,
+	// os.Hostname(), localhost and the loopback addresses
+	SelfSigned     bool     `toml:"self_signed"`
+	IssuerCertFile string   `toml:"issuer_cert_file"`
+	IssuerKeyFile  string   `toml:"issuer_key_file"`
+	Hosts          []string `toml:"hosts"`
+
 	// Listener-side peer verification (mTLS)
 	ClientAuth   bool   `toml:"client_auth"`
 	ClientCAFile string `toml:"client_ca_file"`
@@ -384,6 +399,7 @@ type TLSOptions struct {
 	CAFile             string `toml:"ca_file"`     // empty = system trust store
 	ServerName         string `toml:"server_name"` // default: config host
 	InsecureSkipVerify bool   `toml:"insecure_skip_verify"`
+	PinSHA256          string `toml:"pin_sha256"` // sha256//BASE64 of the server key, ';' between several
 
 	// Minimum protocol version: "1.2" | "1.3" (default "1.3")
 	MinVersion string `toml:"min_version"`

internal/config/loader.go

@@ -6,7 +6,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/lixenwraith/logwisp/internal/core"
 
@@ -26,14 +25,18 @@ type Manager struct {
 	specs    []pipelineSpec // command-line or environment pipelines, kept across reloads
 }
 
+// Args is a parsed command line; cmd/lw owns its grammar.
+type Args struct {
+	File      string   // -c: the configuration file, "" for the environment or discovery
+	Specs     []Spec   // pipeline flags, in order
+	Overrides []string // --path=value settings, for lixenwraith/config
+}
+
 // Load reads the startup sources and validates the initial configuration.
 // Watching starts only when Watch is called after successful service startup.
-func Load(args []string) (*Manager, error) {
+func Load(args Args) (*Manager, error) {
-	configPath, isExplicit, configArgs, err := resolveConfigPath(args)
+	configPath, isExplicit := resolveConfigPath(args.File)
-	if err != nil {
+	specs, err := cliSpecs(args.Specs)
-		return nil, err
-	}
-	specs, configArgs, err := takePipelineSpecs(configArgs)
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +56,7 @@ func Load(args []string) (*Manager, error) {
 	cfg, err := lconfig.NewBuilder().
 		WithTarget(initial).
 		WithEnvPrefix("LOGWISP_").
-		WithArgs(configArgs).
+		WithArgs(args.Overrides).
 		WithFile(configPath).
 		WithTypedValidator(func(cfg *Config) error {
 			effective := *cfg // the builder keeps cfg; only the copy is replaced
@@ -96,6 +99,17 @@ func (m *Manager) Snapshot() (*Config, error) {
 	if err := m.usePipelines(cfg); err != nil {
 		return nil, err
 	}
+	// The top-level color is each console sink's default; the maps are fresh
+	for _, p := range cfg.Pipelines {
+		for i, s := range p.PluginSinks {
+			if _, set := s.Config["color"]; s.Type == "console" && !set {
+				if s.Config == nil {
+					p.PluginSinks[i].Config = map[string]any{}
+				}
+				p.PluginSinks[i].Config["color"] = cfg.Color
+			}
+		}
+	}
 	if err := checkFileKeys(m.path); err != nil {
 		return nil, fmt.Errorf("config file %q: %w", m.path, err)
 	}
@@ -145,6 +159,7 @@ func (m *Manager) Close() { m.config.StopAutoUpdate() }
 func defaults() *Config {
 	return &Config{
 		StatusReporter: true,
+		Color:          "auto",
 		Logging: &LogConfig{
 			Output: "stderr",
 			Level:  "info",
@@ -174,51 +189,28 @@ func pipeDefault(name string) PipelineConfig {
 	}
 }
 
-// resolveConfigPath consumes file-selection flags before schema CLI parsing.
+// resolveConfigPath picks the file: -c, then LOGWISP_CONFIG_FILE and
-// The last selection wins, and -- terminates option handling.
+// LOGWISP_CONFIG_DIR, then the first of the user's and the working directory's
-func resolveConfigPath(args []string) (path string, isExplicit bool, remaining []string, err error) {
+// logwisp.toml. Only the discovered default may be missing.
-	remaining = make([]string, 0, len(args))
+func resolveConfigPath(file string) (path string, isExplicit bool) {
-	for i := 0; i < len(args); i++ {
+	if file != "" {
-		arg := args[i]
+		return file, true
-		if arg == "--" {
-			remaining = append(remaining, args[i:]...)
-			break
-		}
-		switch {
-		case arg == "-c" || arg == "--config":
-			if i+1 == len(args) || args[i+1] == "" || strings.HasPrefix(args[i+1], "-") {
-				return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg)
-			}
-			i++
-			path, isExplicit = args[i], true
-		case strings.HasPrefix(arg, "--config=") || strings.HasPrefix(arg, "-c="):
-			_, path, _ = strings.Cut(arg, "=")
-			if path == "" {
-				return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg)
-			}
-			isExplicit = true
-		default:
-			remaining = append(remaining, arg)
-		}
-	}
-	if isExplicit {
-		return path, true, remaining, nil
 	}
 	if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" {
 		path = configFile
 		if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
 			path = filepath.Join(configDir, configFile)
 		}
-		return path, true, remaining, nil
+		return path, true
 	}
 	if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
-		return filepath.Join(configDir, "logwisp.toml"), true, remaining, nil
+		return filepath.Join(configDir, "logwisp.toml"), true
 	}
 	if homeDir, err := os.UserHomeDir(); err == nil {
 		path = filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml")
 		if _, err := os.Stat(path); err == nil {
-			return path, false, remaining, nil
+			return path, false
 		}
 	}
-	return "logwisp.toml", false, remaining, nil
+	return "logwisp.toml", false
 }

internal/config/loader_test.go

@@ -12,29 +12,20 @@ import (
 	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
-func TestResolveConfigArguments(t *testing.T) {
+// -c wins over the environment, which wins over discovery; only the
+// discovered default is not explicit, so only it may be missing.
+func TestResolveConfigPath(t *testing.T) {
 	isolateConfig(t)
-	for _, args := range [][]string{{"-c", "chosen.toml"}, {"--config", "chosen.toml"}, {"--config=chosen.toml"}, {"-c=chosen.toml"}, {"-c", "old.toml", "--config=chosen.toml"}} {
+	if path, explicit := resolveConfigPath(""); path != "logwisp.toml" || explicit {
-		path, explicit, remaining, err := resolveConfigPath(append(args, "--quiet"))
+		t.Fatalf("discovered default: %q %v", path, explicit)
-		if err != nil || path != "chosen.toml" || !explicit || !reflect.DeepEqual(remaining, []string{"--quiet"}) {
-			t.Fatalf("%v: %q %v %v %v", args, path, explicit, remaining, err)
-		}
-	}
-	for _, args := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}} {
-		if _, _, _, err := resolveConfigPath(args); err == nil {
-			t.Fatalf("missing path accepted: %v", args)
-		}
-	}
-	args := []string{"--", "-c", "ignored.toml"}
-	path, explicit, remaining, err := resolveConfigPath(args)
-	if err != nil || explicit || path != "logwisp.toml" || !reflect.DeepEqual(remaining, args) {
-		t.Fatalf("terminator ignored: %q %v %v %v", path, explicit, remaining, err)
 	}
 	t.Setenv("LOGWISP_CONFIG_DIR", "configs")
 	t.Setenv("LOGWISP_CONFIG_FILE", "env.toml")
-	path, explicit, _, err = resolveConfigPath(nil)
+	if path, explicit := resolveConfigPath(""); path != filepath.Join("configs", "env.toml") || !explicit {
-	if err != nil || !explicit || path != filepath.Join("configs", "env.toml") {
+		t.Fatalf("environment path: %q %v", path, explicit)
-		t.Fatalf("environment path: %q %v %v", path, explicit, err)
+	}
+	if path, explicit := resolveConfigPath("chosen.toml"); path != "chosen.toml" || !explicit {
+		t.Fatalf("-c path: %q %v", path, explicit)
 	}
 }
 
@@ -43,7 +34,7 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
 	testutil.WriteFile(t, "selected.toml", "config_file = \"ignored.toml\"\n[logging]\nlevel = \"error\"\n")
 	t.Setenv("LOGGING_LEVEL", "invalid-bare-variable")
 	t.Setenv("LOGWISP_LOGGING_LEVEL", "debug")
-	m, err := Load([]string{"--config", "selected.toml", "--logging.level=warn", "--quiet"})
+	m, err := Load(Args{File: "selected.toml", Overrides: []string{"--logging.level=warn", "--quiet"}})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,7 +52,7 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
 	if _, aliased := second.Pipelines[0].PluginSources[0].Config["buffer_size"]; err != nil || second.Logging.Level != "warn" || aliased {
 		t.Fatalf("snapshot aliases another result: %+v %v", second, err)
 	}
-	env, err := Load([]string{"-c=selected.toml"})
+	env, err := Load(Args{File: "selected.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -74,19 +65,19 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
 
 func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) {
 	isolateConfig(t)
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
 	m.Close()
-	if _, err := Load([]string{"-c", "missing.toml"}); !errors.Is(err, ErrConfigNotFound) {
+	if _, err := Load(Args{File: "missing.toml"}); !errors.Is(err, ErrConfigNotFound) {
 		t.Fatalf("missing explicit path: %v", err)
 	}
-	if _, err := Load([]string{"--logging.file.max_size_mb=1.5"}); err == nil || errors.Is(err, ErrConfigNotFound) {
+	if _, err := Load(Args{Overrides: []string{"--logging.file.max_size_mb=1.5"}}); err == nil || errors.Is(err, ErrConfigNotFound) {
 		t.Fatalf("invalid CLI hidden by missing default: %v", err)
 	}
 	t.Setenv("LOGWISP_STATUS_REPORTER", "invalid")
-	if _, err := Load(nil); err == nil || errors.Is(err, ErrConfigNotFound) {
+	if _, err := Load(Args{}); err == nil || errors.Is(err, ErrConfigNotFound) {
 		t.Fatalf("invalid environment hidden by missing default: %v", err)
 	}
 }
@@ -94,7 +85,7 @@ func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) {
 func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
 	isolateConfig(t)
 	testutil.WriteFile(t, "reload.toml", "status_reporter = true\n")
-	m, err := Load([]string{"-c", "reload.toml"})
+	m, err := Load(Args{File: "reload.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -137,7 +128,7 @@ func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
 func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) {
 	isolateConfig(t)
 	testutil.WriteFile(t, "watched.toml", "auto_reload = true\n")
-	m, err := Load([]string{"-c", "watched.toml"})
+	m, err := Load(Args{File: "watched.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -181,7 +172,7 @@ server_name = "relay.example"
 type = "none"
 allow = ["one", "two"]
 `)
-	m, err := Load([]string{"-c", "plugins.toml"})
+	m, err := Load(Args{File: "plugins.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -221,7 +212,7 @@ type = "tcp"
 [pipelines.plugin_sinks.config]
 port = 9000
 `+table+"\n")
-		m, err := Load([]string{"-c", "plugins.toml"})
+		m, err := Load(Args{File: "plugins.toml"})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -257,7 +248,7 @@ type = "tcp"
 [pipelines.plugin_sinks.config]
 port = 9000
 `+body+"\n")
-		m, err := Load([]string{"-c", "typo.toml"})
+		m, err := Load(Args{File: "typo.toml"})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -280,7 +271,7 @@ func TestNoConfigFileMeansQuietLogging(t *testing.T) {
 		if c.file != "" {
 			testutil.WriteFile(t, "logwisp.toml", c.file)
 		}
-		m, err := Load(nil)
+		m, err := Load(Args{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -292,7 +283,7 @@ func TestNoConfigFileMeansQuietLogging(t *testing.T) {
 	}
 	t.Setenv("LOGWISP_LOGGING_LEVEL", "debug")
 	os.Remove("logwisp.toml")
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -309,7 +300,7 @@ func TestFileStringIsOneListEntry(t *testing.T) {
 	testutil.WriteFile(t, "logwisp.toml", "[[pipelines]]\nname = \"p\"\n"+
 		"[[pipelines.plugin_sources]]\nid = \"in\"\ntype = \"null\"\n[[pipelines.plugin_sinks]]\nid = \"out\"\ntype = \"null\"\n"+
 		"[[pipelines.flow.filters]]\ntype = \"exclude\"\npatterns = \"password=\\\\S{8,64}\"\n")
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -319,3 +310,25 @@ func TestFileStringIsOneListEntry(t *testing.T) {
 		t.Fatalf("patterns: %v %v", cfg.Pipelines[0].Flow.Filters, err)
 	}
 }
+
+// The top-level color is the default of every console sink, which may set
+// its own; other sinks do not take the key.
+func TestTopLevelColorIsEachConsoleSinksDefault(t *testing.T) {
+	isolateConfig(t)
+	m, err := Load(Args{Specs: specs("sink", "console", "sink", "console,color=never", "sink", "null"), Overrides: []string{"--color=always"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer m.Close()
+	cfg, err := m.Snapshot()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var colors []any
+	for _, s := range cfg.Pipelines[0].PluginSinks {
+		colors = append(colors, s.Config["color"])
+	}
+	if !reflect.DeepEqual(colors, []any{"always", "never", nil}) {
+		t.Fatalf("console sink colors: %v", colors)
+	}
+}

internal/config/preset.go

@@ -0,0 +1,332 @@
+package config
+
+import (
+	"cmp"
+	"fmt"
+	"maps"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Preset is a pipeline built from a few keys: --preset NAME,key=value starts
+// a pipeline with it. Later flags add sources, sinks and filters, or replace
+// its format, rate limit and heartbeat.
+type Preset struct {
+	Name, Summary string
+	Params        []PresetParam
+	build         func(p *PipelineConfig, v map[string]string) error
+}
+
+// PresetParam is one key. A list takes several values, ',' between them or
+// the key repeated; any other key takes one.
+type PresetParam struct {
+	Name, Default, Help string
+	Required, List      bool
+}
+
+// Keys several presets share
+var (
+	pathParam   = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"}
+	fromParam   = PresetParam{Name: "from", Default: "end", Help: "start|end: where reading a file starts"}
+	formatParam = func(def string) PresetParam { return PresetParam{Name: "format", Default: def, Help: "raw|txt|json"} }
+	tlsParams   = []PresetParam{
+		{Name: "cert", Help: "certificate file, for tls=files"},
+		{Name: "key", Help: "key file, for tls=files"},
+		{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"},
+		{Name: "issuer_key", Help: "CA key file, for tls=issuer"},
+		{Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"},
+	}
+	transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
+)
+
+var presets = []Preset{
+	{"pipe", "Standard input to standard output, line for line: lw without a file",
+		[]PresetParam{formatParam("raw")},
+		func(p *PipelineConfig, v map[string]string) error {
+			stdio := pipeDefault("")
+			p.PluginSources, p.PluginSinks = stdio.PluginSources, stdio.PluginSinks
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			return nil
+		}},
+	{"tail", "Follow files to standard output, like tail -F",
+		[]PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")},
+		func(p *PipelineConfig, v map[string]string) error {
+			p.PluginSources, p.PluginSinks = pathOrStdin(v), pipeDefault("").PluginSinks
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			return nil
+		}},
+	{"serve", "Serve files or standard input as a live HTTP stream (SSE)",
+		append([]PresetParam{pathParam, fromParam, formatParam("json"),
+			{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
+			{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
+			{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
+			{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
+			{Name: "viewer", Default: "false", Help: "true: the built-in login and viewer pages, behind proxy"},
+		}, tlsParams...),
+		func(p *PipelineConfig, v map[string]string) error {
+			p.PluginSources = pathOrStdin(v)
+			sink, err := listener(v, "off")
+			if err != nil {
+				return err
+			}
+			if v["users"] != "" {
+				sink["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
+			}
+			if v["proxy"] != "" {
+				auth, ok := sink["auth"].(map[string]any)
+				if !ok {
+					return fmt.Errorf("proxy needs users: browsers log in with SCRAM")
+				}
+				auth["trusted_proxies"] = list(v["proxy"])
+			}
+			switch v["viewer"] {
+			case "true":
+				sink["login_page"], sink["viewer_page"] = true, true
+			case "false":
+			default:
+				return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"])
+			}
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "http", Type: "http", Config: sink})
+			return nil
+		}},
+	{"edge", "Forward files or standard input to an aggregator over TLS, authenticated",
+		[]PresetParam{pathParam, fromParam,
+			{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"},
+			transportParam,
+			{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"},
+			{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
+			{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
+			{Name: "user", Help: "SCRAM user (lw auth add-user on the aggregator)"},
+			{Name: "password_file", Help: "file holding the SCRAM password"},
+			{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"},
+			{Name: "key", Help: "client key file"},
+			{Name: "node", Help: "origin label; default: this host's name"},
+		},
+		func(p *PipelineConfig, v map[string]string) error {
+			p.PluginSources = pathOrStdin(v)
+			typ, err := chainType(v["transport"])
+			if err != nil {
+				return err
+			}
+			host, port, err := splitAddr("to", v["to"])
+			if err != nil {
+				return err
+			}
+			tls := map[string]any{"enabled": true}
+			for key, option := range map[string]string{"ca": "ca_file", "pin": "pin_sha256", "server_name": "server_name", "cert": "cert_file", "key": "key_file"} {
+				if v[key] != "" {
+					tls[option] = v[key]
+				}
+			}
+			var auth map[string]any
+			switch {
+			case v["user"] != "" && v["password_file"] != "":
+				auth = map[string]any{"type": "scram", "username": v["user"], "password_file": v["password_file"]}
+			case v["user"] != "" || v["password_file"] != "":
+				return fmt.Errorf("user and password_file go together")
+			case v["cert"] != "" && v["key"] != "":
+				auth = map[string]any{"type": "mtls"}
+			default:
+				return fmt.Errorf("edge needs user and password_file, or cert and key: it never sends unauthenticated")
+			}
+			sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth}
+			if v["node"] != "" {
+				sink["node"] = v["node"]
+			}
+			p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "edge", Type: typ, Config: sink})
+			return nil
+		}},
+	{"aggregator", "Receive from edges over TLS, authenticated, to files or standard output",
+		append([]PresetParam{
+			{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"},
+			transportParam,
+			{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
+			{Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"},
+			{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
+			{Name: "out", Help: "directory for the received entries; default: standard output"},
+			formatParam("json"),
+		}, tlsParams...),
+		func(p *PipelineConfig, v map[string]string) error {
+			typ, err := chainType(v["transport"])
+			if err != nil {
+				return err
+			}
+			if v["tls"] == "off" {
+				return fmt.Errorf("tls=off: edges send credentials and logs, the aggregator always serves TLS")
+			}
+			src, err := listener(v, "self")
+			if err != nil {
+				return err
+			}
+			if v["client_ca"] != "" {
+				tls := src["tls"].(map[string]any)
+				tls["client_auth"], tls["client_ca_file"] = true, v["client_ca"]
+			}
+			switch {
+			case v["users"] != "":
+				src["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
+			case v["client_ca"] != "":
+				src["auth"] = map[string]any{"type": "mtls"}
+			default:
+				return fmt.Errorf("aggregator needs users or client_ca: it never receives unauthenticated")
+			}
+			p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src})
+			p.PluginSinks = pipeDefault("").PluginSinks
+			if v["out"] != "" {
+				p.PluginSinks = []PluginSinkConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": v["out"], "name": "aggregate"}}}
+			}
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			return nil
+		}},
+}
+
+// Presets lists the presets, for lw preset
+func Presets() []Preset { return slices.Clone(presets) }
+
+// ExpandPreset builds the pipeline a preset makes of values, as --preset does
+func ExpandPreset(name string, values map[string]string) (PipelineConfig, error) {
+	p := PipelineConfig{Flow: &FlowConfig{}}
+	each := map[string][]string{}
+	for k, v := range values {
+		each[k] = []string{v}
+	}
+	err := applyPreset(&p, name, each)
+	return p, err
+}
+
+// applyPreset fills defaults and checks keys before the preset's build
+func applyPreset(p *PipelineConfig, name string, values map[string][]string) error {
+	i := slices.IndexFunc(presets, func(r Preset) bool { return r.Name == name })
+	if i < 0 {
+		var names []string
+		for _, r := range presets {
+			names = append(names, r.Name)
+		}
+		return fmt.Errorf("no preset %q (valid: %s)", name, strings.Join(names, ", "))
+	}
+	r := presets[i]
+	v, keys := map[string]string{}, []string{}
+	for _, k := range r.Params {
+		v[k.Name] = k.Default
+		keys = append(keys, k.Name)
+	}
+	for k, each := range values {
+		j := slices.IndexFunc(r.Params, func(p PresetParam) bool { return p.Name == k })
+		switch {
+		case j < 0:
+			return fmt.Errorf("preset %s has no key %q (valid: %s)", name, k, strings.Join(keys, ", "))
+		case len(each) > 1 && !r.Params[j].List:
+			return fmt.Errorf("preset %s key %q takes one value", name, k)
+		}
+		v[k] = cmp.Or(strings.Join(each, ","), v[k])
+	}
+	for _, k := range r.Params {
+		if k.Required && v[k.Name] == "" {
+			return fmt.Errorf("preset %s needs %s", name, k.Name)
+		}
+	}
+	p.Name = cmp.Or(p.Name, name)
+	return r.build(p, v)
+}
+
+func pathOrStdin(v map[string]string) []PluginSourceConfig {
+	if v["path"] == "" {
+		return pipeDefault("").PluginSources
+	}
+	return []PluginSourceConfig{fileSource(v["path"], v["from"])}
+}
+
+// fileSource follows path: a directory's files, or the files a glob or a
+// file name matches in its directory.
+func fileSource(path, from string) PluginSourceConfig {
+	dir, pattern := path, "*"
+	if fi, err := os.Stat(path); err != nil || !fi.IsDir() {
+		dir, pattern = filepath.Dir(path), filepath.Base(path)
+	}
+	return PluginSourceConfig{ID: "file", Type: "file", Config: map[string]any{"directory": dir, "pattern": pattern, "from": from}}
+}
+
+// tlsModes is what each preset tls mode sets, and the preset keys it needs
+// by the tls option each fills; made marks a certificate made at startup,
+// which takes hosts.
+var tlsModes = map[string]struct {
+	set   map[string]any
+	needs map[string]string
+	made  bool
+}{
+	"off":    {},
+	"self":   {set: map[string]any{"self_signed": true}, made: true},
+	"issuer": {needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true},
+	"files":  {needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
+}
+
+// listener starts a listening plugin's options: host, port and tls, whose
+// mode decides which certificate keys apply.
+func listener(v map[string]string, def string) (map[string]any, error) {
+	host, port, err := splitAddr("listen", v["listen"])
+	if err != nil {
+		return nil, err
+	}
+	opts := map[string]any{"host": host, "port": port}
+	mode := cmp.Or(v["tls"], def)
+	m, ok := tlsModes[mode]
+	if !ok {
+		return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(slices.Sorted(maps.Keys(tlsModes)), ", "))
+	}
+	for _, other := range tlsModes {
+		for k := range other.needs {
+			if _, needed := m.needs[k]; needed && v[k] == "" {
+				return nil, fmt.Errorf("tls=%s needs %s", mode, k)
+			} else if !needed && v[k] != "" {
+				return nil, fmt.Errorf("%s does not apply to tls=%s", k, mode)
+			}
+		}
+	}
+	if v["hosts"] != "" && !m.made {
+		return nil, fmt.Errorf("hosts does not apply to tls=%s", mode)
+	}
+	if mode == "off" {
+		return opts, nil
+	}
+	tls := map[string]any{"enabled": true}
+	maps.Copy(tls, m.set)
+	for k, option := range m.needs {
+		tls[option] = v[k]
+	}
+	if v["hosts"] != "" {
+		tls["hosts"] = list(v["hosts"])
+	}
+	opts["tls"] = tls
+	return opts, nil
+}
+
+func splitAddr(key, addr string) (string, int64, error) {
+	host, p, err := net.SplitHostPort(addr)
+	port, perr := strconv.ParseUint(p, 10, 16)
+	if err != nil || perr != nil || port == 0 {
+		return "", 0, fmt.Errorf("%s %q: want HOST:PORT, an IPv6 address in brackets", key, addr)
+	}
+	return host, int64(port), nil
+}
+
+func chainType(transport string) (string, error) {
+	if transport != "tcp" && transport != "http" {
+		return "", fmt.Errorf("transport %q (valid: tcp, http)", transport)
+	}
+	return transport + "_chain", nil
+}
+
+func list(s string) []any {
+	var out []any
+	for e := range strings.SplitSeq(s, ",") {
+		if e = strings.TrimSpace(e); e != "" {
+			out = append(out, e)
+		}
+	}
+	return out
+}

internal/config/preset_test.go

@@ -0,0 +1,63 @@
+package config
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// A preset starts its pipeline and names it unless --pipeline did; later
+// sinks add to its own, a later format replaces its own.
+func TestPresetStartsAndNamesItsPipeline(t *testing.T) {
+	isolateConfig(t)
+	got := loadPipelines(t, "preset", "tail,path=/var/log/app.log", "sink", "null", "format", "json",
+		"pipeline", "mine", "preset", "pipe")
+	want := []PipelineConfig{{
+		Name:          "tail",
+		Flow:          &FlowConfig{Format: &FormatConfig{Type: "json"}},
+		PluginSources: []PluginSourceConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": "/var/log", "pattern": "app.log", "from": "end"}}},
+		PluginSinks: []PluginSinkConfig{
+			{ID: "stdout", Type: "console", Config: map[string]any{"color": "auto"}},
+			{ID: "null", Type: "null", Config: map[string]any{}},
+		},
+	}, {
+		Name:          "mine",
+		Flow:          &FlowConfig{Format: &FormatConfig{Type: "raw"}},
+		PluginSources: []PluginSourceConfig{{ID: "stdin", Type: "console", Config: map[string]any{}}},
+		PluginSinks:   []PluginSinkConfig{{ID: "stdout", Type: "console", Config: map[string]any{"color": "auto"}}},
+	}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
+	}
+	if _, err := Load(Args{Specs: specs("sink", "null", "preset", "pipe")}); err == nil || !strings.Contains(err.Error(), "must start its pipeline") {
+		t.Fatalf("preset after a sink: %v", err)
+	}
+}
+
+// Keys are checked against the preset's own, required ones enforced, and
+// the edge and aggregator refuse to run without authentication.
+func TestPresetKeysAreChecked(t *testing.T) {
+	isolateConfig(t)
+	for spec, want := range map[string]string{
+		"nope":                         `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`,
+		"tail,path=x,colour=1":         `preset tail has no key "colour" (valid: path, from, format)`,
+		"tail":                         "preset tail needs path",
+		"tail,path=x,format.type=json": "preset keys do not nest",
+		"tail,path=x,path=y":           `key "path" takes one value`,
+		"serve,listen=8080":            `listen "8080": want HOST:PORT`,
+		"serve,tls=self,cert=c":        "cert does not apply to tls=self",
+		"serve,tls=files,cert=c":       "tls=files needs key",
+		"serve,viewer=yes":             `viewer "yes"`,
+		"serve,proxy=127.0.0.1":        "proxy needs users",
+		"edge,to=agg:9000":             "it never sends unauthenticated",
+		"edge,to=agg:9000,user=u":      "user and password_file go together",
+		"edge,to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
+		"aggregator,users=u,tls=off":                            "the aggregator always serves TLS",
+		"aggregator":                                            "it never receives unauthenticated",
+	} {
+		_, err := Load(Args{Specs: specs("preset", spec)})
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("%s: err = %v, want %s", spec, err, want)
+		}
+	}
+}

internal/config/spec.go

@@ -16,11 +16,28 @@ type specKind struct {
 	flag  string
 	typed bool // SPEC starts with TYPE
 	many  bool // repeatable within a pipeline; the environment adds _1.._N
+	first bool // starts its pipeline, and names it by TYPE unless --pipeline did
 	apply func(p *PipelineConfig, typ string, opts map[string]any) error
 }
 
 var specKinds = []specKind{
-	{"source", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+		values := map[string][]string{}
+		for k, v := range opts {
+			switch v := v.(type) {
+			case string:
+				values[k] = []string{v}
+			case []any: // a repeated key
+				for _, e := range v {
+					values[k] = append(values[k], e.(string))
+				}
+			default:
+				return fmt.Errorf("preset keys do not nest: %q", k)
+			}
+		}
+		return applyPreset(p, typ, values)
+	}},
+	{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		ids := make([]string, len(p.PluginSources))
 		for i, s := range p.PluginSources {
 			ids[i] = s.ID
@@ -29,7 +46,7 @@ var specKinds = []specKind{
 		p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
 		return err
 	}},
-	{"sink", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		ids := make([]string, len(p.PluginSinks))
 		for i, s := range p.PluginSinks {
 			ids[i] = s.ID
@@ -38,24 +55,24 @@ var specKinds = []specKind{
 		p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
 		return err
 	}},
-	{"filter", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		opts["type"] = typ
 		f := FilterConfig{}
 		err := Scan(opts, &f)
 		p.Flow.Filters = append(p.Flow.Filters, f)
 		return err
 	}},
-	{"format", true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		opts["type"] = typ
 		p.Flow.Format = &FormatConfig{}
 		return Scan(opts, p.Flow.Format)
 	}},
 	// Naming a stage turns it on: the file defaults (pass, disabled) would not.
-	{"rate-limit", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
+	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
 		p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
 		return Scan(opts, p.Flow.RateLimit)
 	}},
-	{"heartbeat", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
+	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
 		p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
 		return Scan(opts, p.Flow.Heartbeat)
 	}},
@@ -68,33 +85,36 @@ type pipelineSpec struct {
 	value string
 }
 
-// takePipelineSpecs consumes the pipeline flags, which lixenwraith/config
+// Spec is one pipeline flag: its name without dashes, and its value.
-// would report as unrecognized.
+type Spec struct{ Flag, Value string }
-func takePipelineSpecs(args []string) (specs []pipelineSpec, rest []string, err error) {
+
-	for i := 0; i < len(args); i++ {
+// SpecFlags names the pipeline flags, for the command-line parser.
-		if args[i] == "--" {
+func SpecFlags() []string {
-			return specs, append(rest, args[i:]...), nil
+	flags := []string{"pipeline"}
-		}
+	for _, k := range specKinds {
-		name, value, inline := strings.Cut(args[i], "=")
+		flags = append(flags, k.flag)
+	}
+	return flags
+}
+
+// cliSpecs looks up the kind of each command-line spec.
+func cliSpecs(in []Spec) ([]pipelineSpec, error) {
+	var specs []pipelineSpec
+	for _, s := range in {
 		var kind *specKind
-		if name != "--pipeline" {
+		if s.Flag != "pipeline" {
-			k := slices.IndexFunc(specKinds, func(k specKind) bool { return "--"+k.flag == name })
+			k := slices.IndexFunc(specKinds, func(k specKind) bool { return k.flag == s.Flag })
 			if k < 0 {
-				rest = append(rest, args[i])
+				return nil, fmt.Errorf("--%s is no pipeline flag", s.Flag)
-				continue
 			}
 			kind = &specKinds[k]
 		}
-		if !inline && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") {
+		if s.Value == "" {
-			i++
+			return nil, fmt.Errorf("--%s requires a value", s.Flag)
-			value = args[i]
 		}
-		if value == "" {
+		specs = append(specs, pipelineSpec{"--" + s.Flag, kind, s.Value})
-			return nil, nil, fmt.Errorf("%s requires a value", name)
-		}
-		specs = append(specs, pipelineSpec{name, kind, value})
 	}
-	return specs, rest, nil
+	return specs, nil
 }
 
 // envPipelineSpecs reads the one-pipeline environment form. An empty variable
@@ -131,48 +151,58 @@ func envPipelineSpecs() []pipelineSpec {
 }
 
 // buildPipelines turns specs, in order, into pipelines; specs before the first
-// --pipeline go to one named "cli". A pipeline without a source reads stdin,
+// --pipeline go to one named "cli". Every call returns fresh maps.
-// one without a sink writes stdout. Every call returns fresh maps.
 func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
 	var pipelines []PipelineConfig
 	var seen map[*specKind]bool
 	for _, s := range specs {
-		if s.kind == nil || pipelines == nil {
+		if s.kind == nil {
-			name := "cli"
+			pipelines = append(pipelines, PipelineConfig{Name: s.value, Flow: &FlowConfig{}})
-			if s.kind == nil {
-				name = s.value
-			}
-			pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
 			seen = map[*specKind]bool{}
-			if s.kind == nil {
+			continue
-				continue
-			}
 		}
-		p := &pipelines[len(pipelines)-1]
-		if seen[s.kind] && !s.kind.many {
-			return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
-		}
-		seen[s.kind] = true
 		typ, opts, err := parseSpec(s.value, s.kind.typed)
 		if err != nil {
 			return nil, fmt.Errorf("%s %w", s.name, err)
 		}
+		if pipelines == nil {
+			name := "cli"
+			if s.kind.first {
+				name = typ
+			}
+			pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
+			seen = map[*specKind]bool{}
+		}
+		p := &pipelines[len(pipelines)-1]
+		switch {
+		case seen[s.kind] && !s.kind.many:
+			return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
+		case s.kind.first && len(seen) > 0:
+			return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value)
+		}
+		seen[s.kind] = true
 		if err := s.kind.apply(p, typ, opts); err != nil {
 			return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
 		}
 	}
 	for i := range pipelines {
-		pipe := pipeDefault(pipelines[i].Name)
+		useStdio(&pipelines[i])
-		if len(pipelines[i].PluginSources) == 0 {
-			pipelines[i].PluginSources = pipe.PluginSources
-		}
-		if len(pipelines[i].PluginSinks) == 0 {
-			pipelines[i].PluginSinks = pipe.PluginSinks
-		}
 	}
 	return pipelines, nil
 }
 
+// useStdio makes a pipeline without a source read stdin, and one without a
+// sink write stdout, as a filter does.
+func useStdio(p *PipelineConfig) {
+	pipe := pipeDefault(p.Name)
+	if len(p.PluginSources) == 0 {
+		p.PluginSources = pipe.PluginSources
+	}
+	if len(p.PluginSinks) == 0 {
+		p.PluginSinks = pipe.PluginSinks
+	}
+}
+
 // parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
 // a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
 func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {

internal/config/spec_test.go

@@ -10,9 +10,18 @@ import (
 	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
-func loadPipelines(t *testing.T, args ...string) []PipelineConfig {
+// specs pairs flags with values: specs("sink", "null") is --sink null
+func specs(pairs ...string) []Spec {
+	var out []Spec
+	for i := 0; i < len(pairs); i += 2 {
+		out = append(out, Spec{pairs[i], pairs[i+1]})
+	}
+	return out
+}
+
+func loadPipelines(t *testing.T, pairs ...string) []PipelineConfig {
 	t.Helper()
-	m, err := Load(args)
+	m, err := Load(Args{Specs: specs(pairs...)})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -27,15 +36,14 @@ func loadPipelines(t *testing.T, args ...string) []PipelineConfig {
 func TestPipelineSpecGrammar(t *testing.T) {
 	isolateConfig(t)
 	got := loadPipelines(t,
-		"--source", `file,directory=/var/log/a\,b,pattern=*.log`,
+		"source", `file,directory=/var/log/a\,b,pattern=*.log`,
-		`--source=file,id=app,directory=x\=y`,
+		"source", `file,id=app,directory=x\=y`,
-		"--source", `file,directory=c:\\logs\\`,
+		"source", `file,directory=c:\\logs\\`,
-		"--sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c",
+		"sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c",
-		"--filter", "include,patterns=ERROR,patterns=WARN",
+		"filter", "include,patterns=ERROR,patterns=WARN",
-		"--filter", `exclude,patterns=\d{1\,3}`,
+		"filter", `exclude,patterns=\d{1\,3}`,
-		"--format", `txt,timestamp_format=Jan 2\, 2006`,
+		"format", `txt,timestamp_format=Jan 2\, 2006`,
-		"--pipeline", "b", "--source", "null", "--sink", "null", "--sink", "null", "--heartbeat", "interval_ms=1000",
+		"pipeline", "b", "source", "null", "sink", "null", "sink", "null", "heartbeat", "interval_ms=1000")
-		"--", "--source", "random")
 	want := []PipelineConfig{{
 		Name: "cli",
 		Flow: &FlowConfig{
@@ -67,17 +75,17 @@ func TestPipelineSpecGrammar(t *testing.T) {
 	if !reflect.DeepEqual(got, want) {
 		t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
 	}
-	for want, args := range map[string][]string{
+	for want, pairs := range map[string][]string{
-		`--sink http,port: missing "="`:                                  {"--sink", "http,port"},
+		`--sink http,port: missing "="`:                                  {"sink", "http,port"},
-		`--source directory=/x: missing TYPE`:                            {"--source", "directory=/x"},
+		`--source directory=/x: missing TYPE`:                            {"source", "directory=/x"},
-		`--source file,type=x: TYPE already sets "type"`:                 {"--source", "file,type=x"},
+		`--source file,type=x: TYPE already sets "type"`:                 {"source", "file,type=x"},
-		`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"--sink", "http,tls=1,tls.cert_file=c"},
+		`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http,tls=1,tls.cert_file=c"},
-		`--sink requires a value`:                                        {"--sink="},
+		`--sink requires a value`:                                        {"sink", ""},
-		`--format txt: pipeline "cli" already has one`:                   {"--format", "json", "--format", "txt"},
+		`--format txt: pipeline "cli" already has one`:                   {"format", "json", "format", "txt"},
-		`--rate-limit rate=1,polcy=drop: unknown key "polcy"`:            {"--rate-limit", "rate=1,polcy=drop"},
+		`--rate-limit rate=1,polcy=drop: unknown key "polcy"`:            {"rate-limit", "rate=1,polcy=drop"},
 	} {
-		if _, err := Load(args); err == nil || err.Error() != want {
+		if _, err := Load(Args{Specs: specs(pairs...)}); err == nil || err.Error() != want {
-			t.Errorf("%q: err = %v, want %s", args, err, want)
+			t.Errorf("%q: err = %v, want %s", pairs, err, want)
 		}
 	}
 }
@@ -86,7 +94,7 @@ func TestPipelineSpecGrammar(t *testing.T) {
 // must not split into several, wider entries.
 func TestSpecValueIsOneListEntry(t *testing.T) {
 	isolateConfig(t)
-	sink := loadPipelines(t, "--source", "null", "--sink",
+	sink := loadPipelines(t, "source", "null", "sink",
 		`tcp,auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0]
 	var opts TCPSinkOptions
 	if err := Scan(sink.Config, &opts); err != nil {
@@ -115,7 +123,7 @@ func TestEnvironmentPipeline(t *testing.T) {
 	} {
 		t.Setenv(name, value)
 	}
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -136,7 +144,7 @@ func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) {
 	isolateConfig(t)
 	t.Setenv("LOGWISP_SOURCE", "random")
 	t.Setenv("LOGWISP_SINK", "console")
-	got := loadPipelines(t, "--source", "null", "--sink", "null")
+	got := loadPipelines(t, "source", "null", "sink", "null")
 	if len(got) != 1 || len(got[0].PluginSources) != 1 || got[0].PluginSources[0].Type != "null" ||
 		len(got[0].PluginSinks) != 1 || got[0].PluginSinks[0].Type != "null" {
 		t.Fatalf("environment specs mixed into command-line pipelines: %+v", got)
@@ -147,7 +155,7 @@ func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) {
 func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
 	isolateConfig(t)
 	testutil.WriteFile(t, "file.toml", "[logging]\nlevel = \"warn\"\n[[pipelines]]\nname = \"file\"\n")
-	m, err := Load([]string{"-c", "file.toml", "--source", "null", "--sink", "null", "--status_reporter=false"})
+	m, err := Load(Args{File: "file.toml", Specs: specs("source", "null", "sink", "null"), Overrides: []string{"--status_reporter=false"}})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -165,12 +173,12 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
 // stdout, as a filter does; two pipelines cannot both read stdin.
 func TestSpecPipelinesDefaultToStdio(t *testing.T) {
 	isolateConfig(t)
-	got := loadPipelines(t, "--filter", "include,patterns=ERROR")
+	got := loadPipelines(t, "filter", "include,patterns=ERROR")
 	if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" ||
 		len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" {
 		t.Fatalf("pipeline: %+v", p)
 	}
-	_, err := Load([]string{"--pipeline", "a", "--sink", "null", "--pipeline", "b", "--sink", "null"})
+	_, err := Load(Args{Specs: specs("pipeline", "a", "sink", "null", "pipeline", "b", "sink", "null")})
 	if err == nil || !strings.Contains(err.Error(), `console source already reads stdin in pipeline "a"`) {
 		t.Fatalf("second stdin reader: %v", err)
 	}
@@ -181,7 +189,7 @@ func TestSpecPipelinesDefaultToStdio(t *testing.T) {
 // removal fails the reload instead of keeping the removed file's values.
 func TestReloadKeepsSpecPipelines(t *testing.T) {
 	isolateConfig(t)
-	m, err := Load([]string{"--source", "random,special=true", "--sink", "null"})
+	m, err := Load(Args{Specs: specs("source", "random,special=true", "sink", "null")})
 	if err != nil {
 		t.Fatal(err)
 	}

internal/config/validate.go

@@ -34,6 +34,10 @@ func ValidateConfig(cfg *Config) error {
 		names[p.Name] = struct{}{}
 	}
 
+	if err := lconfig.OneOf("auto", "always", "never")(cfg.Color); err != nil {
+		return fmt.Errorf("color: %w", err)
+	}
+
 	if err := validateLogConfig(cfg.Logging); err != nil {
 		return fmt.Errorf("logging: %w", err)
 	}

internal/core/const.go

@@ -43,6 +43,9 @@ const (
 
 	ShutdownTimeout = 10 * time.Second
 
+	// Bounds the flush of a network sink's queues at Stop
+	SinkFlushTimeout = 2 * time.Second
+
 	ConfigReloadTimeout = 30 * time.Second
 
 	LoggerShutdownTimeout = 2 * time.Second

internal/sink/console/console.go

@@ -18,8 +18,10 @@ import (
 	"github.com/lixenwraith/logwisp/internal/session"
 	"github.com/lixenwraith/logwisp/internal/sink"
 
+	"github.com/lixenwraith/color"
 	lconfig "github.com/lixenwraith/config"
 	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/terminal/inline"
 	"golang.org/x/term"
 )
 
@@ -38,8 +40,9 @@ type ConsoleSink struct {
 	session *session.Session
 
 	// Configuration
-	config *config.ConsoleSinkOptions
+	config  *config.ConsoleSinkOptions
-	escape bool
+	escape  bool
+	painter *inline.Printer // paints level names; nil when color is off
 
 	// Application
 	input  chan core.TransportEvent
@@ -63,8 +66,23 @@ const (
 	DefaultConsoleTarget     = "stdout"
 	DefaultConsoleBufferSize = 1000
 	DefaultConsoleEscape     = "auto"
+	DefaultConsoleColor      = "auto"
 )
 
+// levelColors paints the first of a level's names in an entry, ANSI 16 so
+// the terminal's theme picks the shades and a text console renders them.
+// Names run longest first; cyan, not blue, stays readable on black.
+var levelColors = map[string]struct {
+	names []string
+	style inline.Style
+}{
+	"ERROR": {[]string{"ERROR", "FATAL", "ERR"}, inline.FgANSI(color.ANSIRed).Bold()},
+	"WARN":  {[]string{"WARNING", "WARN"}, inline.FgANSI(color.ANSIYellow)},
+	"INFO":  {[]string{"INFO", "INF"}, inline.FgANSI(color.ANSIGreen)},
+	"DEBUG": {[]string{"DEBUG", "DBG"}, inline.FgANSI(color.ANSICyan)},
+	"TRACE": {[]string{"TRACE"}, inline.FgANSI(color.ANSIBrightBlack)},
+}
+
 // NewConsoleSinkPlugin creates a console sink through plugin factory
 func NewConsoleSinkPlugin(
 	id string,
@@ -72,7 +90,7 @@ func NewConsoleSinkPlugin(
 	logger *log.Logger,
 	proxy *session.Proxy,
 ) (sink.Sink, error) {
-	opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape}
+	opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape, Color: DefaultConsoleColor}
 
 	// Scan config map into struct
 	if err := config.Scan(configMap, opts); err != nil {
@@ -85,6 +103,9 @@ func NewConsoleSinkPlugin(
 	if err := lconfig.OneOf("auto", "always", "never")(opts.Escape); err != nil {
 		return nil, fmt.Errorf("escape: %w", err)
 	}
+	if err := lconfig.OneOf("auto", "always", "never")(opts.Color); err != nil {
+		return nil, fmt.Errorf("color: %w", err)
+	}
 
 	output := os.Stdout
 	if opts.Target == "stderr" {
@@ -95,17 +116,29 @@ func NewConsoleSinkPlugin(
 		opts.BufferSize = DefaultConsoleBufferSize
 	}
 
+	// inline decides auto: a terminal, NO_COLOR unset, TERM not dumb. Paint
+	// returns its text unchanged exactly when color is off.
+	printer := inline.New(output)
+	if opts.Color != "auto" {
+		printer.SetColor(opts.Color == "always")
+	}
+	var painter *inline.Printer
+	if printer.Paint("x", inline.Style{}) != "x" {
+		painter = printer
+	}
+
 	// Create and return plugin instance
 	cs := &ConsoleSink{
-		id:     id,
+		id:      id,
-		proxy:  proxy,
+		proxy:   proxy,
-		config: opts,
+		config:  opts,
-		escape: opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())),
+		escape:  opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())),
-		input:  make(chan core.TransportEvent, opts.BufferSize),
+		painter: painter,
-		output: output,
+		input:   make(chan core.TransportEvent, opts.BufferSize),
-		done:   make(chan struct{}),
+		output:  output,
-		exited: make(chan struct{}),
+		done:    make(chan struct{}),
-		logger: logger,
+		exited:  make(chan struct{}),
+		logger:  logger,
 	}
 	cs.lastProcessed.Store(time.Time{})
 
@@ -124,6 +157,7 @@ func NewConsoleSinkPlugin(
 		"instance_id", id,
 		"target", opts.Target,
 		"escape", cs.escape,
+		"color", painter != nil,
 	)
 
 	return cs, nil
@@ -187,6 +221,7 @@ func (cs *ConsoleSink) GetStats() sink.SinkStats {
 			"target":      cs.config.Target,
 			"buffer_size": cs.config.BufferSize,
 			"escape":      cs.escape,
+			"color":       cs.painter != nil,
 		},
 	}
 }
@@ -224,6 +259,9 @@ func (cs *ConsoleSink) write(event core.TransportEvent) {
 	if cs.escape {
 		payload = escapeControls(payload)
 	}
+	if cs.painter != nil {
+		payload = paintLevel(cs.painter, payload, event.Entry.Level)
+	}
 	if _, err := cs.output.Write(payload); err != nil {
 		cs.logger.Error("msg", "Failed to write to console",
 			"component", "console_sink",
@@ -235,6 +273,42 @@ func (cs *ConsoleSink) write(event core.TransportEvent) {
 	cs.lastProcessed.Store(time.Now())
 }
 
+// paintLevel colors the first whole-word, case-insensitive occurrence of a
+// name of level, after escaping: the escape sequences are the sink's own.
+func paintLevel(p *inline.Printer, payload []byte, level string) []byte {
+	lc, ok := levelColors[level]
+	if !ok {
+		return payload
+	}
+	for _, name := range lc.names {
+		for i := 0; i+len(name) <= len(payload); i++ {
+			j := i + len(name)
+			if !nameAt(payload[i:j], name) || i > 0 && word(payload[i-1]) || j < len(payload) && word(payload[j]) {
+				continue
+			}
+			out := make([]byte, 0, len(payload)+16)
+			out = append(out, payload[:i]...)
+			out = append(out, p.Paint(string(payload[i:j]), lc.style)...)
+			return append(out, payload[j:]...)
+		}
+	}
+	return payload
+}
+
+// nameAt compares ASCII letters without case; name is upper case
+func nameAt(b []byte, name string) bool {
+	for k := range len(name) {
+		if c := b[k]; c != name[k] && c != name[k]+'a'-'A' {
+			return false
+		}
+	}
+	return true
+}
+
+func word(c byte) bool {
+	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'
+}
+
 // escapeControls writes what a terminal would act on or reorder (C0 and C1
 // controls but tab, DEL, bidi controls, line separators, invalid UTF-8) as
 // <hex>, keeping the final newline: a log line cannot drive the terminal.

internal/sink/console/console_test.go

@@ -1,6 +1,11 @@
 package console
 
-import "testing"
+import (
+	"io"
+	"testing"
+
+	"github.com/lixenwraith/terminal/inline"
+)
 
 // Everything a terminal would act on is escaped; tabs and the final newline,
 // which it only lays out, are kept.
@@ -23,3 +28,22 @@ func TestEscapeControlsKeepsOnlyLayout(t *testing.T) {
 		}
 	}
 }
+
+// The entry's level is painted where its line first names it as a whole
+// word, in any case, longest name first; other words and levels stay plain.
+func TestPaintLevelColorsTheFirstWholeName(t *testing.T) {
+	p := inline.New(io.Discard)
+	p.SetColor(true)
+	red, yellow := "\x1b[0;1;31m", "\x1b[0;33m"
+	for _, c := range []struct{ in, level, want string }{
+		{"2026 ERROR boom ERROR\n", "ERROR", "2026 " + red + "ERROR\x1b[0m boom ERROR\n"},
+		{"[warning] disk\n", "WARN", "[" + yellow + "warning\x1b[0m] disk\n"},
+		{"terror_x ERR:1\n", "ERROR", "terror_x " + red + "ERR\x1b[0m:1\n"},
+		{"INFOS informal\n", "INFO", "INFOS informal\n"},
+		{"ERROR without level\n", "", "ERROR without level\n"},
+	} {
+		if got := string(paintLevel(p, []byte(c.in), c.level)); got != c.want {
+			t.Errorf("%q %s: got %q, want %q", c.in, c.level, got, c.want)
+		}
+	}
+}

internal/sink/http/http.go

@@ -43,7 +43,6 @@ const (
 	DefaultHTTPStreamPath       = "/stream"
 	DefaultHTTPStatusPath       = "/status"
 	HTTPReadHeaderTimeout       = 10 * time.Second
-	HTTPShutdownTimeout         = 2 * time.Second
 )
 
 // HTTPSink streams log entries via Server-Sent Events
@@ -80,8 +79,14 @@ type HTTPSink struct {
 	auth *authz.Policy
 	web  map[string]http.Handler // proxy mode: GET /auth/... browser files
 
-	// Runtime
+	// Runtime. Stop closes flush; the broker then moves what remains of its
+	// input to tail and closes flushed; each stream writes its queue, tail
+	// and the disconnect event on its own, until flushBy.
 	done      chan struct{}
+	flush     chan struct{}
+	flushed   chan struct{}
+	flushBy   time.Time
+	tail      [][]byte // read only once flushed is closed
 	stopOnce  sync.Once
 	wg        sync.WaitGroup
 	startTime time.Time
@@ -145,7 +150,7 @@ func NewHTTPSinkPlugin(
 	if opts.ClientBufferSize <= 0 {
 		opts.ClientBufferSize = DefaultHTTPClientBufferSize
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -174,6 +179,8 @@ func NewHTTPSinkPlugin(
 		network:      network,
 		input:        make(chan core.TransportEvent, opts.BufferSize),
 		done:         make(chan struct{}),
+		flush:        make(chan struct{}),
+		flushed:      make(chan struct{}),
 		logger:       logger,
 		clients:      make(map[uint64]*sseClient),
 		writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
@@ -276,7 +283,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error {
 	h.startTime = time.Now()
 
 	h.wg.Add(1)
-	go h.brokerLoop(ctx)
+	go h.brokerLoop()
 
 	serve := h.server.Serve
 	if h.tlsConfig != nil {
@@ -325,18 +332,21 @@ func (h *HTTPSink) Stop() {
 }
 
 // shutdown funnels ctx-cancel and Stop() teardown through a single path.
-// done is closed first so SSE handlers exit and Shutdown can complete;
+// Server.Shutdown refuses new streams and waits for the open ones, which end
-// Server.Close force-closes any handler stalled in a deadline-free write.
+// once they wrote what is queued, within sink.FlushBound; Server.Close cuts a
+// stream stalled past it.
 func (h *HTTPSink) shutdown() {
 	h.stopOnce.Do(func() {
-		close(h.done)
 		if h.server != nil {
-			sctx, cancel := context.WithTimeout(context.Background(), HTTPShutdownTimeout)
+			h.flushBy = time.Now().Add(sink.FlushBound(h.writeTimeout))
+			ctx, cancel := context.WithDeadline(context.Background(), h.flushBy)
 			defer cancel()
-			if err := h.server.Shutdown(sctx); err != nil {
+			close(h.flush)
+			if err := h.server.Shutdown(ctx); err != nil {
 				h.server.Close()
 			}
 		}
+		close(h.done)
 	})
 }
 
@@ -356,19 +366,22 @@ func (h *HTTPSink) removeClient(id uint64) {
 }
 
 // brokerLoop fans out transport events to all client queues, non-blocking,
-// and evicts clients whose sessions were idle-expired by the session manager
+// and evicts clients whose sessions were idle-expired by the session manager,
-func (h *HTTPSink) brokerLoop(ctx context.Context) {
+// until flush, which shutdown closes on Stop and on context cancellation.
+func (h *HTTPSink) brokerLoop() {
 	defer h.wg.Done()
 	for {
 		select {
-		case <-ctx.Done():
+		case <-h.flush: // first: a select with both ready picks at random
+			h.flushInput()
 			return
-		case <-h.done:
+		default:
+		}
+		select {
+		case <-h.flush:
+			h.flushInput()
 			return
-		case event, ok := <-h.input:
+		case event := <-h.input:
-			if !ok {
-				return
-			}
 			h.totalProcessed.Add(1)
 			h.lastProcessed.Store(time.Now())
 
@@ -395,6 +408,13 @@ func (h *HTTPSink) brokerLoop(ctx context.Context) {
 	}
 }
 
+// flushInput moves what remains of the input to tail for the streams
+func (h *HTTPSink) flushInput() {
+	h.tail = sink.Drain(h.input)
+	h.totalProcessed.Add(uint64(len(h.tail)))
+	close(h.flushed)
+}
+
 // handleStream serves one client's SSE stream
 func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 	if h.config.MaxConnections > 0 && h.activeClients.Load() >= h.config.MaxConnections {
@@ -483,6 +503,13 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 	idle := time.NewTicker(h.keepalive)
 	defer idle.Stop()
 
+	send := func(payload []byte) bool {
+		if writeSSE(w, payload) != nil || rc.Flush() != nil {
+			return false
+		}
+		h.proxy.UpdateActivity(sess.ID)
+		return true
+	}
 	clientGone := r.Context().Done()
 	for {
 		select {
@@ -491,13 +518,9 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 				return // broker evicted (stale session)
 			}
 			h.armWrite(rc)
-			if err := writeSSE(w, payload); err != nil {
+			if !send(payload) {
 				return
 			}
-			if err := rc.Flush(); err != nil {
-				return
-			}
-			h.proxy.UpdateActivity(sess.ID)
 		case <-idle.C:
 			h.armWrite(rc)
 			if _, err := fmt.Fprint(w, ":\n\n"); err != nil {
@@ -509,7 +532,20 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 			h.proxy.UpdateActivity(sess.ID)
 		case <-clientGone:
 			return
-		case <-h.done:
+		case <-h.flushed:
+			// Nothing more will be queued: write the queue and the tail, then
+			// say why the stream ends
+			_ = rc.SetWriteDeadline(h.flushBy)
+			for len(c.send) > 0 {
+				if payload, ok := <-c.send; !ok || !send(payload) {
+					return
+				}
+			}
+			for _, payload := range h.tail {
+				if !send(payload) {
+					return
+				}
+			}
 			fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
 			rc.Flush()
 			return

internal/sink/http/http_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"fmt"
 	"io"
 	"maps"
 	"net/http"
@@ -319,3 +320,73 @@ func TestSSEFramesEveryLineAsData(t *testing.T) {
 		t.Fatalf("framed %q, want %q", got, want)
 	}
 }
+
+// Stop writes what is queued, in the sink and per stream, then the
+// disconnect event, to a client that keeps reading, whatever another that
+// stopped reading does; Stop returns at the flush bound.
+func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) {
+	const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold
+	httpSink, _ := newTestHTTPSink(t, map[string]any{"buffer_size": int64(n), "client_buffer_size": int64(64), "write_timeout_ms": int64(500)})
+	client, baseURL := serveTestHTTPSink(t, httpSink)
+	var scanners []*bufio.Scanner
+	for range 2 {
+		resp, err := client.Get(baseURL + "/stream")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { _ = resp.Body.Close() })
+		scanner := bufio.NewScanner(resp.Body)
+		scanner.Buffer(nil, 64*1024)
+		for scanner.Scan() && scanner.Text() != "event: connected" {
+		}
+		scanners = append(scanners, scanner)
+	}
+	for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+		httpSink.clientsMu.Lock()
+		registered := len(httpSink.clients)
+		httpSink.clientsMu.Unlock()
+		if registered == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("streams never registered")
+		}
+	}
+	type result struct {
+		lines      int
+		disconnect bool
+	}
+	done := make(chan result)
+	go func() {
+		var r result
+		for scanners[0].Scan() {
+			switch line := scanners[0].Text(); {
+			case strings.HasPrefix(line, "data: line "):
+				r.lines++
+			case line == "event: disconnect":
+				r.disconnect = true
+			}
+		}
+		done <- r
+	}()
+	// Held, the lock keeps the input queued until Stop has begun its flush
+	httpSink.clientsMu.Lock()
+	padding := strings.Repeat("x", 8192)
+	for i := range n {
+		httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)}
+	}
+	stopped := make(chan time.Duration)
+	go func() {
+		start := time.Now()
+		httpSink.Stop()
+		stopped <- time.Since(start)
+	}()
+	time.Sleep(100 * time.Millisecond)
+	httpSink.clientsMu.Unlock()
+	if d := <-stopped; d > 1500*time.Millisecond {
+		t.Errorf("Stop took %v past a 500 ms flush bound", d)
+	}
+	if r := <-done; r.lines != n || !r.disconnect {
+		t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect)
+	}
+}

internal/sink/sink.go

@@ -35,3 +35,27 @@ type SinkStats struct {
 	LastProcessed     time.Time
 	Details           map[string]any
 }
+
+// FlushBound is how long a network sink's Stop gives its clients to take
+// what is queued: write_timeout_ms, at most core.SinkFlushTimeout, which a
+// reload also waits out for a client that stopped reading.
+func FlushBound(writeTimeout time.Duration) time.Duration {
+	if writeTimeout > 0 && writeTimeout < core.SinkFlushTimeout {
+		return writeTimeout
+	}
+	return core.SinkFlushTimeout
+}
+
+// Drain takes the payloads queued in input without waiting: a network sink's
+// last events, which each client then writes on its own.
+func Drain(input <-chan core.TransportEvent) [][]byte {
+	var tail [][]byte
+	for {
+		select {
+		case event := <-input:
+			tail = append(tail, event.Payload)
+		default:
+			return tail
+		}
+	}
+}

internal/sink/tcp/tcp.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"maps"
 	"net"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -74,8 +75,15 @@ type TCPSink struct {
 	// Authorization
 	auth *authz.Policy
 
-	// Runtime
+	// Runtime. Stop closes flush; the broadcast loop then moves what remains
+	// of its input to tail and closes flushed; each writer writes its queue
+	// and tail on its own, until flushBy. done tears down what is left.
 	done      chan struct{}
+	flush     chan struct{}
+	flushed   chan struct{}
+	flushBy   time.Time
+	tail      [][]byte // read only once flushed is closed
+	flushing  bool     // under clientsMu: no client registers once set
 	stopOnce  sync.Once
 	wg        sync.WaitGroup
 	startTime time.Time
@@ -91,11 +99,13 @@ type TCPSink struct {
 
 // tcpClient is a registered connection's bounded send queue.
 // send is written by the broadcast loop (non-blocking) and drained by the
-// writer goroutine; closed signals reader-detected disconnect.
+// writer goroutine; closed signals reader-detected disconnect, exited the
+// writer's end.
 type tcpClient struct {
 	send      chan []byte
 	sessionID string
 	closed    chan struct{}
+	exited    chan struct{}
 }
 
 // NewTCPSinkPlugin creates a tcp sink through plugin factory
@@ -131,7 +141,7 @@ func NewTCPSinkPlugin(
 	if opts.KeepAlivePeriodMS <= 0 {
 		opts.KeepAlivePeriodMS = DefaultTCPKeepAlivePeriodMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -148,6 +158,8 @@ func NewTCPSinkPlugin(
 		network:      network,
 		input:        make(chan core.TransportEvent, opts.BufferSize),
 		done:         make(chan struct{}),
+		flush:        make(chan struct{}),
+		flushed:      make(chan struct{}),
 		logger:       logger,
 		clients:      make(map[uint64]*tcpClient),
 		conns:        make(map[net.Conn]struct{}),
@@ -223,7 +235,7 @@ func (t *TCPSink) Start(ctx context.Context) error {
 
 	t.wg.Add(2)
 	go t.acceptLoop()
-	go t.broadcastLoop(ctx)
+	go t.broadcastLoop()
 
 	// Pipeline context cancellation mirrors gnet engine stop: cease accepting
 	// and tear down existing connections
@@ -258,13 +270,27 @@ func (t *TCPSink) Stop() {
 		"total_processed", t.totalProcessed.Load())
 }
 
-// shutdown funnels ctx-cancel and Stop() teardown through a single path
+// shutdown funnels ctx-cancel and Stop() teardown through a single path.
+// Clients first receive what is queued, within sink.FlushBound, so a finite
+// input reaches them whole.
 func (t *TCPSink) shutdown() {
 	t.stopOnce.Do(func() {
-		close(t.done)
 		if t.listener != nil {
 			t.listener.Close() // unblocks acceptLoop
+			t.flushBy = time.Now().Add(sink.FlushBound(t.writeTimeout))
+			close(t.flush)
+			t.clientsMu.Lock()
+			t.flushing = true
+			clients := slices.Collect(maps.Values(t.clients))
+			t.clientsMu.Unlock()
+			for _, c := range clients {
+				select {
+				case <-c.exited:
+				case <-time.After(time.Until(t.flushBy)):
+				}
+			}
 		}
+		close(t.done)
 		t.clientsMu.Lock()
 		for conn := range t.conns {
 			conn.Close() // unblocks handshakes and per-connection readers
@@ -381,10 +407,17 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 		send:      make(chan []byte, t.config.ClientBufferSize),
 		sessionID: sess.ID,
 		closed:    make(chan struct{}),
+		exited:    make(chan struct{}),
 	}
+	defer close(c.exited)
 	id := t.nextClientID.Add(1)
 
 	t.clientsMu.Lock()
+	if t.flushing {
+		t.clientsMu.Unlock()
+		t.proxy.RemoveSession(sess.ID)
+		return
+	}
 	t.clients[id] = c
 	t.clientsMu.Unlock()
 
@@ -422,23 +455,45 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 
 	// Writer: synchronous lib write with deadline. A failed write means
 	// the kernel buffer stayed full for the full deadline - connection is
-	// dead or hopelessly stalled, so disconnect immediately (no gnet-style
+	// dead or hopelessly stalled, so disconnect immediately.
-	// consecutive-error counter needed for transient async callback errors).
+	write := func(data []byte, deadline time.Time) bool {
+		if !deadline.IsZero() {
+			conn.SetWriteDeadline(deadline)
+		}
+		if _, err := conn.Write(data); err != nil {
+			t.writeErrors.Add(1)
+			t.logger.Debug("msg", "Write failed, closing client",
+				"component", "tcp_sink",
+				"remote_addr", remote,
+				"error", err)
+			return false
+		}
+		t.proxy.UpdateActivity(sess.ID)
+		return true
+	}
 	for {
 		select {
 		case data := <-c.send:
+			var deadline time.Time
 			if t.writeTimeout > 0 {
-				conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
+				deadline = time.Now().Add(t.writeTimeout)
 			}
-			if _, err := conn.Write(data); err != nil {
+			if !write(data, deadline) {
-				t.writeErrors.Add(1)
-				t.logger.Debug("msg", "Write failed, closing client",
-					"component", "tcp_sink",
-					"remote_addr", remote,
-					"error", err)
 				return
 			}
-			t.proxy.UpdateActivity(sess.ID)
+		case <-t.flushed:
+			// Nothing more will be queued: write the queue, then the tail
+			for len(c.send) > 0 {
+				if !write(<-c.send, t.flushBy) {
+					return
+				}
+			}
+			for _, data := range t.tail {
+				if !write(data, t.flushBy) {
+					return
+				}
+			}
+			return
 		case <-c.closed:
 			return
 		case <-t.done:
@@ -447,19 +502,22 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 	}
 }
 
-// broadcastLoop fans out transport events to all client queues, non-blocking
+// broadcastLoop fans out transport events to all client queues until flush,
-func (t *TCPSink) broadcastLoop(ctx context.Context) {
+// which shutdown closes on Stop and on context cancellation alike.
+func (t *TCPSink) broadcastLoop() {
 	defer t.wg.Done()
 	for {
 		select {
-		case <-ctx.Done():
+		case <-t.flush: // first: a select with both ready picks at random
+			t.flushInput()
 			return
-		case <-t.done:
+		default:
+		}
+		select {
+		case <-t.flush:
+			t.flushInput()
 			return
-		case event, ok := <-t.input:
+		case event := <-t.input:
-			if !ok {
-				return
-			}
 			t.totalProcessed.Add(1)
 			t.lastProcessed.Store(time.Now())
 
@@ -477,6 +535,13 @@ func (t *TCPSink) broadcastLoop(ctx context.Context) {
 	}
 }
 
+// flushInput moves what remains of the input to tail for the writers
+func (t *TCPSink) flushInput() {
+	t.tail = sink.Drain(t.input)
+	t.totalProcessed.Add(uint64(len(t.tail)))
+	close(t.flushed)
+}
+
 // GetStats returns sink statistics
 func (t *TCPSink) GetStats() sink.SinkStats {
 	lastProc, _ := t.lastProcessed.Load().(time.Time)

internal/sink/tcp/tcp_test.go

@@ -1,11 +1,15 @@
 package tcp
 
 import (
+	"bufio"
+	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/lixenwraith/logwisp/internal/core"
 	"github.com/lixenwraith/logwisp/internal/session"
 	"github.com/lixenwraith/logwisp/internal/testutil"
 
@@ -57,3 +61,81 @@ func TestStopDoesNotWaitForSilentHandshake(t *testing.T) {
 		t.Fatalf("Stop took %v with a silent peer in the handshake", d)
 	}
 }
+
+// Stop writes what is queued, in the sink and per client, to a client that
+// keeps reading, whatever another that stopped reading does: the end of a
+// finite input reaches it whole, and Stop returns at the flush bound.
+func TestStopWritesQueuedEventsToReadingClients(t *testing.T) {
+	const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold
+	probe, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := probe.Addr().String()
+	probe.Close()
+	_, port, _ := net.SplitHostPort(addr)
+	p, _ := strconv.ParseInt(port, 10, 64)
+
+	manager := session.NewManager(time.Hour)
+	t.Cleanup(manager.Stop)
+	created, err := NewTCPSinkPlugin("out", map[string]any{
+		"host": "127.0.0.1", "port": p, "buffer_size": int64(n), "client_buffer_size": int64(64),
+		"write_timeout_ms": int64(500),
+	}, log.NewLogger(), session.NewProxy(manager, "out"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sink := created.(*TCPSink)
+	if err := sink.Start(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	var conns []net.Conn
+	for range 2 {
+		conn, err := net.Dial("tcp4", addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { conn.Close() })
+		conns = append(conns, conn)
+	}
+	for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+		sink.clientsMu.Lock()
+		registered := len(sink.clients)
+		sink.clientsMu.Unlock()
+		if registered == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("clients never registered")
+		}
+	}
+	lines := make(chan int)
+	go func() {
+		count, scanner := 0, bufio.NewScanner(conns[0])
+		scanner.Buffer(nil, 64*1024)
+		for scanner.Scan() {
+			count++
+		}
+		lines <- count
+	}()
+	// Held, the lock keeps the input queued until Stop has begun its flush
+	sink.clientsMu.Lock()
+	padding := strings.Repeat("x", 8192)
+	for i := range n {
+		sink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)}
+	}
+	stopped := make(chan time.Duration)
+	go func() {
+		start := time.Now()
+		sink.Stop()
+		stopped <- time.Since(start)
+	}()
+	time.Sleep(100 * time.Millisecond)
+	sink.clientsMu.Unlock()
+	if d := <-stopped; d > 1500*time.Millisecond {
+		t.Errorf("Stop took %v past a 500 ms flush bound", d)
+	}
+	if got := <-lines; got != n {
+		t.Fatalf("reading client received %d of %d lines", got, n)
+	}
+}

internal/sink/tcpchain/tcpchain.go

@@ -329,8 +329,9 @@ func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
 					return false
 				}
 				t.failures++
-				if errors.Is(err, authz.ErrRefused) {
+				if errors.Is(err, authz.ErrRefused) || errors.As(err, new(*tls.CertificateVerificationError)) {
-					// A refusal is configuration, not weather: show it by default
+					// A refusal, or a server failing its CA or pin, is
+					// configuration, not weather: show it by default
 					t.logger.Warn("msg", "Chain connect refused",
 						"component", "tcp_chain_sink",
 						"target", t.addr,

internal/source/httpchain/httpchain.go

@@ -112,7 +112,7 @@ func NewHTTPChainSourcePlugin(
 	if opts.ReadTimeoutMS <= 0 {
 		opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}

internal/source/tcpchain/tcpchain.go

@@ -97,7 +97,7 @@ func NewTCPChainSourcePlugin(
 	if opts.HelloTimeoutMS <= 0 {
 		opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}

internal/testutil/fixtures.go

@@ -6,6 +6,7 @@ import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
@@ -120,6 +121,40 @@ func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFil
 	return writePEM(t, p.dir, name+".crt", "CERTIFICATE", der), writePEM(t, p.dir, name+".key", "PRIVATE KEY", keyDER)
 }
 
+// Handshake runs a TLS handshake between the two configurations over
+// loopback TCP, whose buffers let a failing side's alert go out without a
+// reader, and returns the client's error, else the server's.
+func Handshake(t testing.TB, server, client *tls.Config) error {
+	t.Helper()
+	ln, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	serverErr := make(chan error, 1)
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		defer conn.Close()
+		conn.SetDeadline(time.Now().Add(5 * time.Second))
+		serverErr <- tls.Server(conn, server).Handshake()
+	}()
+	conn, err := net.Dial("tcp4", ln.Addr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn.SetDeadline(time.Now().Add(5 * time.Second))
+	err = tls.Client(conn, client).Handshake()
+	conn.Close()
+	if serr := <-serverErr; err == nil {
+		err = serr
+	}
+	return err
+}
+
 func writePEM(t testing.TB, dir, name, blockType string, der []byte) string {
 	t.Helper()
 	path := filepath.Join(dir, name)

internal/tlsx/pki.go

@@ -0,0 +1,231 @@
+package tlsx
+
+import (
+	"bytes"
+	"cmp"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+	"unicode"
+)
+
+// Certificates are ECDSA P-256, for lw tls and for listeners without files.
+// A CA signs leaves only; a leaf lasts at most 397 days, the longest browsers
+// accept, and never past its issuer.
+const (
+	CAValidity   = 10 * 365 * 24 * time.Hour
+	LeafValidity = 397 * 24 * time.Hour
+	pinPrefix    = "sha256//"
+)
+
+// processKey is the key of every generated listener certificate: reloads
+// reissue certificates and keep it, so a pin holds until lw restarts.
+var processKey = sync.OnceValues(NewKey)
+
+func NewKey() (*ecdsa.PrivateKey, error) { return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) }
+
+// NewCA makes a self-signed certificate authority that signs leaves only
+func NewCA(name string, validity time.Duration) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	key, err := NewKey()
+	if err != nil {
+		return nil, nil, err
+	}
+	t, err := template(name, validity)
+	if err != nil {
+		return nil, nil, err
+	}
+	t.IsCA, t.MaxPathLenZero = true, true
+	t.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
+	cert, err := sign(t, t, &key.PublicKey, key)
+	return cert, key, err
+}
+
+// Leaf describes a certificate to issue. Hosts become DNS or IP SANs, which
+// hostname verification reads; Name is the subject CN, which mtls reads.
+type Leaf struct {
+	Name           string
+	Hosts          []string
+	Server, Client bool
+	Validity       time.Duration
+}
+
+// Issue signs a certificate for pub; a nil issuer makes it self-signed.
+func (l Leaf) Issue(pub crypto.PublicKey, issuer *x509.Certificate, issuerKey crypto.Signer) (*x509.Certificate, error) {
+	t, err := template(l.Name, l.Validity)
+	if err != nil {
+		return nil, err
+	}
+	t.KeyUsage = x509.KeyUsageDigitalSignature
+	if l.Server {
+		t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
+	}
+	if l.Client {
+		t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
+	}
+	for _, h := range l.Hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			t.IPAddresses = append(t.IPAddresses, ip)
+		} else {
+			t.DNSNames = append(t.DNSNames, h)
+		}
+	}
+	if issuer == nil {
+		return sign(t, t, pub, issuerKey)
+	}
+	if !time.Now().Before(issuer.NotAfter) {
+		return nil, fmt.Errorf("tls: issuer %q expired on %s", issuer.Subject.CommonName, issuer.NotAfter.UTC().Format(time.DateOnly))
+	}
+	if t.NotAfter.After(issuer.NotAfter) {
+		t.NotAfter = issuer.NotAfter
+	}
+	return sign(t, issuer, pub, issuerKey)
+}
+
+func template(name string, validity time.Duration) (*x509.Certificate, error) {
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now()
+	return &x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: name},
+		NotBefore:             now.Add(-5 * time.Minute), // peers' clocks drift
+		NotAfter:              now.Add(validity),
+		BasicConstraintsValid: true,
+	}, nil
+}
+
+func sign(t, parent *x509.Certificate, pub crypto.PublicKey, key crypto.Signer) (*x509.Certificate, error) {
+	der, err := x509.CreateCertificate(rand.Reader, t, parent, pub, key)
+	if err != nil {
+		return nil, fmt.Errorf("tls: sign certificate: %w", err)
+	}
+	return x509.ParseCertificate(der)
+}
+
+// LoadIssuer reads a CA certificate and its key, as lw tls ca writes them
+func LoadIssuer(certFile, keyFile string) (*x509.Certificate, crypto.Signer, error) {
+	pair, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("tls: load issuer: %w", err)
+	}
+	ca, err := x509.ParseCertificate(pair.Certificate[0]) // Leaf is nil under GODEBUG=x509keypairleaf=0
+	if err != nil {
+		return nil, nil, fmt.Errorf("tls: load issuer: %w", err)
+	}
+	signer, ok := pair.PrivateKey.(crypto.Signer)
+	if !ok || !ca.IsCA || ca.KeyUsage&x509.KeyUsageCertSign == 0 {
+		return nil, nil, fmt.Errorf("tls: %s is no certificate authority", certFile)
+	}
+	return ca, signer, nil
+}
+
+// generated issues a listener certificate on the process key, self-signed
+// or from the issuer files, for hosts and the names this machine answers to.
+func generated(issuerCert, issuerKey, host string, hosts []string) (tls.Certificate, error) {
+	key, err := processKey()
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	var issuer *x509.Certificate
+	var signer crypto.Signer = key
+	if issuerCert != "" {
+		if issuer, signer, err = LoadIssuer(issuerCert, issuerKey); err != nil {
+			return tls.Certificate{}, err
+		}
+	}
+	name, _ := os.Hostname()
+	all := slices.Clone(hosts)
+	if ip := net.ParseIP(host); host != "" && (ip == nil || !ip.IsUnspecified()) {
+		all = append(all, host)
+	}
+	if !strings.ContainsFunc(name, func(r rune) bool { return r > unicode.MaxASCII }) {
+		all = append(all, name) // a DNS SAN is ASCII; the CN keeps any name
+	}
+	all = append(all, "localhost", "127.0.0.1", "::1")
+	slices.Sort(all)
+	all = slices.Compact(slices.DeleteFunc(all, func(h string) bool { return h == "" }))
+	leaf, err := Leaf{Name: cmp.Or(name, "logwisp"), Hosts: all, Server: true, Validity: LeafValidity}.Issue(&key.PublicKey, issuer, signer)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	chain := [][]byte{leaf.Raw}
+	if issuer != nil {
+		chain = append(chain, issuer.Raw) // an intermediate needs it to chain to its root
+	}
+	return tls.Certificate{Certificate: chain, PrivateKey: key, Leaf: leaf}, nil
+}
+
+// PinSHA256 is the curl --pinnedpubkey form of a certificate's public key
+func PinSHA256(c *x509.Certificate) string { return pin(c.RawSubjectPublicKeyInfo) }
+
+func pinOfKey(k *ecdsa.PrivateKey) string {
+	der, _ := x509.MarshalPKIXPublicKey(&k.PublicKey) // P-256 always marshals
+	return pin(der)
+}
+
+func pin(spki []byte) string {
+	h := sha256.Sum256(spki)
+	return pinPrefix + base64.StdEncoding.EncodeToString(h[:])
+}
+
+// verifyPins replaces chain verification: the server's key must hash to one
+// of the ';'-separated pins. Its name and validity are not checked. Go skips
+// it on a resumed session, and no dialer keeps a ClientSessionCache.
+func verifyPins(s string) (func([][]byte, [][]*x509.Certificate) error, error) {
+	var pins [][]byte
+	for p := range strings.SplitSeq(s, ";") {
+		b64, ok := strings.CutPrefix(strings.TrimSpace(p), pinPrefix)
+		h, err := base64.StdEncoding.DecodeString(b64)
+		if !ok || err != nil || len(h) != sha256.Size {
+			return nil, fmt.Errorf("tls: pin_sha256 %q: want %sBASE64 of a SHA-256, ';' between several", p, pinPrefix)
+		}
+		pins = append(pins, h)
+	}
+	return func(raw [][]byte, _ [][]*x509.Certificate) error {
+		if len(raw) == 0 {
+			return errors.New("tls: server sent no certificate")
+		}
+		c, err := x509.ParseCertificate(raw[0])
+		if err != nil {
+			return err
+		}
+		h := sha256.Sum256(c.RawSubjectPublicKeyInfo)
+		if slices.ContainsFunc(pins, func(p []byte) bool { return bytes.Equal(p, h[:]) }) {
+			return nil
+		}
+		// The type of a failed chain check: callers treat both as a refusal
+		return &tls.CertificateVerificationError{UnverifiedCertificates: []*x509.Certificate{c},
+			Err: fmt.Errorf("server key %s matches no tls.pin_sha256", PinSHA256(c))}
+	}, nil
+}
+
+// EncodeCert and EncodeKey render PEM, the key as PKCS #8
+func EncodeCert(c *x509.Certificate) []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw})
+}
+
+func EncodeKey(k crypto.PrivateKey) ([]byte, error) {
+	der, err := x509.MarshalPKCS8PrivateKey(k)
+	if err != nil {
+		return nil, err
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil
+}

internal/tlsx/tlsx.go

@@ -26,16 +26,36 @@ const (
 	expiryWindow = 30 * 24 * time.Hour
 )
 
-// Server builds the *tls.Config for listener plugins
+// Server builds the *tls.Config for listener plugins (tcp/http sinks,
-// (tcp/http sinks, tcp_chain/http_chain sources). Returns (nil, nil) when disabled.
+// tcp_chain/http_chain sources) from cert_file and key_file, or issues one on
-func Server(o *config.TLSOptions) (*tls.Config, error) {
+// the process key: self-signed or from the issuer files, for host, hosts and
+// this machine's names. Returns (nil, nil) when disabled.
+func Server(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if o == nil || !o.Enabled {
 		return nil, nil
 	}
-	if o.CertFile == "" || o.KeyFile == "" {
+	files, issuer := o.CertFile != "" || o.KeyFile != "", o.IssuerCertFile != "" || o.IssuerKeyFile != ""
-		return nil, fmt.Errorf("tls: cert_file and key_file are required for listeners")
+	switch {
+	case o.PinSHA256 != "":
+		return nil, fmt.Errorf("tls: pin_sha256 applies to dialers")
+	case files && (issuer || o.SelfSigned) || issuer && o.SelfSigned:
+		return nil, fmt.Errorf("tls: set one of cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
+	case len(o.Hosts) > 0 && !issuer && !o.SelfSigned:
+		return nil, fmt.Errorf("tls: hosts applies to self_signed and issuer certificates")
+	case files && (o.CertFile == "" || o.KeyFile == ""):
+		return nil, fmt.Errorf("tls: cert_file and key_file must be set together")
+	case issuer && (o.IssuerCertFile == "" || o.IssuerKeyFile == ""):
+		return nil, fmt.Errorf("tls: issuer_cert_file and issuer_key_file must be set together")
+	case !files && !issuer && !o.SelfSigned:
+		return nil, fmt.Errorf("tls: listeners need cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
+	}
+	var cert tls.Certificate
+	var err error
+	if files {
+		cert, err = tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
+	} else {
+		cert, err = generated(o.IssuerCertFile, o.IssuerKeyFile, host, o.Hosts)
 	}
-	cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
 	if err != nil {
 		return nil, fmt.Errorf("tls: load keypair: %w", err)
 	}
@@ -64,11 +84,18 @@ func Server(o *config.TLSOptions) (*tls.Config, error) {
 // Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain
 // sinks). host seeds ServerName when no override is set; Go verifies IP SANs
 // when host is an address, without its zone, which names a local interface
-// and no certificate carries. Returns (nil, nil) when disabled.
+// and no certificate carries. pin_sha256 replaces chain and name checks.
+// Returns (nil, nil) when disabled.
 func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if o == nil || !o.Enabled {
 		return nil, nil
 	}
+	switch {
+	case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0:
+		return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file and hosts apply to listeners")
+	case o.PinSHA256 != "" && (o.CAFile != "" || o.InsecureSkipVerify):
+		return nil, fmt.Errorf("tls: pin_sha256 replaces ca_file and insecure_skip_verify: set one")
+	}
 	mv, err := minVersion(o.MinVersion)
 	if err != nil {
 		return nil, err
@@ -81,6 +108,12 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if cfg.ServerName == "" {
 		cfg.ServerName, _, _ = strings.Cut(host, "%")
 	}
+	if o.PinSHA256 != "" {
+		if cfg.VerifyPeerCertificate, err = verifyPins(o.PinSHA256); err != nil {
+			return nil, err
+		}
+		cfg.InsecureSkipVerify = true // the pin check above replaces it
+	}
 	if o.CAFile != "" {
 		pool, err := loadPool(o.CAFile)
 		if err != nil {
@@ -149,13 +182,22 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list
 	if o == nil || !o.Enabled {
 		return
 	}
+	if key, err := processKey(); listener && o.SelfSigned && err == nil {
+		// Without a CA, a dialer verifies this listener by its pin only
+		l.Warn("msg", "Serving a self-signed certificate; it changes when lw restarts",
+			"component", component,
+			"instance_id", id,
+			"pin_sha256", pinOfKey(key),
+			"hint", "dialers set tls.pin_sha256; for a stable identity use issuer_cert_file or cert_file")
+	}
 	var warnings []string
 	if !listener && o.InsecureSkipVerify {
 		warnings = append(warnings, "tls.insecure_skip_verify disables server verification: any peer can impersonate the server")
 	}
-	files := []struct{ key, path string }{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}}
+	type file struct{ key, path string }
+	files := []file{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}}
 	if listener {
-		files[1] = struct{ key, path string }{"client_ca_file", o.ClientCAFile}
+		files = []file{{"cert_file", o.CertFile}, {"client_ca_file", o.ClientCAFile}, {"issuer_cert_file", o.IssuerCertFile}}
 	}
 	now := time.Now()
 	for _, f := range files {
@@ -165,8 +207,10 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list
 			}
 		}
 	}
-	if w := SecretFileWarning("tls.key_file", o.KeyFile); w != "" {
+	for _, key := range []file{{"tls.key_file", o.KeyFile}, {"tls.issuer_key_file", o.IssuerKeyFile}} {
-		warnings = append(warnings, w)
+		if w := SecretFileWarning(key.key, key.path); w != "" {
+			warnings = append(warnings, w)
+		}
 	}
 	for _, w := range warnings {
 		l.Warn("msg", w, "component", component, "instance_id", id)

internal/tlsx/tlsx_test.go

@@ -3,11 +3,15 @@ package tlsx
 import (
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"net"
+	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/lixenwraith/logwisp/internal/config"
+	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
 // A dialer verifies an IPv6 target by its address: the zone names a local
@@ -48,3 +52,104 @@ func TestExpiryWarning(t *testing.T) {
 		}
 	}
 }
+
+// A self-signed listener is verified by the pin of the process key, which a
+// reload's reissued certificate keeps; any other pin, or none, fails.
+func TestSelfSignedListenerIsVerifiedByItsPin(t *testing.T) {
+	o := &config.TLSOptions{Enabled: true, SelfSigned: true, Hosts: []string{"agg.example"}}
+	first, err := Server(o, "0.0.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	reloaded, err := Server(o, "0.0.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf := reloaded.Certificates[0].Leaf
+	pin := PinSHA256(leaf)
+	if PinSHA256(first.Certificates[0].Leaf) != pin || !slices.Contains(leaf.DNSNames, "agg.example") ||
+		!slices.Contains(leaf.DNSNames, "localhost") || slices.ContainsFunc(leaf.IPAddresses, net.IP.IsUnspecified) {
+		t.Fatalf("pin %s, names %v %v", pin, leaf.DNSNames, leaf.IPAddresses)
+	}
+	other := "sha256//" + strings.Repeat("A", 43) + "="
+	for pins, want := range map[string]string{other + "; " + pin: "", other: "matches no tls.pin_sha256", "": "certificate"} {
+		client, err := Client(&config.TLSOptions{Enabled: true, PinSHA256: pins}, "agg.example")
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = testutil.Handshake(t, reloaded, client)
+		if want == "" && err != nil || want != "" && (err == nil || !strings.Contains(err.Error(), want)) {
+			t.Errorf("pins %q: %v, want %q", pins, err, want)
+		}
+	}
+}
+
+// An issuer-signed listener chains to the CA file, for its host, and expires
+// with its issuer; a certificate that is no CA cannot issue.
+func TestIssuedListenerCertificateChainsToTheIssuer(t *testing.T) {
+	dir := t.TempDir()
+	ca, key, err := NewCA("test CA", time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyPEM, err := EncodeKey(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	caFile, keyFile := filepath.Join(dir, "ca.crt"), filepath.Join(dir, "ca.key")
+	testutil.WriteFile(t, caFile, string(EncodeCert(ca)))
+	testutil.WriteFile(t, keyFile, string(keyPEM))
+	server, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: caFile, IssuerKeyFile: keyFile}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if leaf := server.Certificates[0].Leaf; !leaf.NotAfter.Equal(ca.NotAfter) {
+		t.Fatalf("leaf expires %s, issuer %s", leaf.NotAfter, ca.NotAfter)
+	}
+	client, err := Client(&config.TLSOptions{Enabled: true, CAFile: caFile}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := testutil.Handshake(t, server, client); err != nil {
+		t.Fatalf("issued chain: %v", err)
+	}
+	leafFile := filepath.Join(dir, "leaf.crt")
+	testutil.WriteFile(t, leafFile, string(EncodeCert(server.Certificates[0].Leaf)))
+	if _, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: leafFile, IssuerKeyFile: keyFile}, ""); err == nil {
+		t.Fatal("a leaf issued certificates")
+	}
+}
+
+// Each role takes one source of trust and one certificate source; keys of the
+// other role fail instead of doing nothing.
+func TestTLSOptionConflicts(t *testing.T) {
+	pin := "sha256//" + strings.Repeat("A", 43) + "="
+	for _, c := range []struct {
+		o      config.TLSOptions
+		server bool
+		want   string
+	}{
+		{config.TLSOptions{SelfSigned: true, CertFile: "c", KeyFile: "k"}, true, "set one of"},
+		{config.TLSOptions{SelfSigned: true, IssuerCertFile: "c"}, true, "set one of"},
+		{config.TLSOptions{IssuerCertFile: "c"}, true, "must be set together"},
+		{config.TLSOptions{CertFile: "c"}, true, "must be set together"},
+		{config.TLSOptions{Hosts: []string{"a"}, CertFile: "c", KeyFile: "k"}, true, "hosts applies to"},
+		{config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "applies to dialers"},
+		{config.TLSOptions{}, true, "listeners need"},
+		{config.TLSOptions{SelfSigned: true}, false, "apply to listeners"},
+		{config.TLSOptions{PinSHA256: pin, CAFile: "ca"}, false, "set one"},
+		{config.TLSOptions{PinSHA256: pin, InsecureSkipVerify: true}, false, "set one"},
+		{config.TLSOptions{PinSHA256: "sha256//short"}, false, "want sha256//BASE64"},
+	} {
+		c.o.Enabled = true
+		var err error
+		if c.server {
+			_, err = Server(&c.o, "")
+		} else {
+			_, err = Client(&c.o, "host")
+		}
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%+v: %v, want %q", c.o, err, c.want)
+		}
+	}
+}

test/preset-test.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# logwisp preset test: colors, presets, --dump, lw tls, listeners that make
+# their certificate (self-signed and pinned, or issued), and the http and tcp
+# sinks writing a finite input whole. One-shot: both modes run the checks.
+# Ports: 15890-15896 on 127.0.0.1.
+# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only.
+
+set -u
+. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
+
+RUN=$E2E_DIR/run/preset
+e2e_init "$@"
+
+section "Setup"
+need curl openssl
+ports_free 15890 15891 15892 15893 15894 15895 15896
+rm -rf "$RUN"
+mkdir -p "$RUN/out" "$LOG"
+cd "$RUN" || abort "no run directory"
+# no configuration file or LOGWISP_ variable applies
+for v in $(compgen -e LOGWISP_); do unset "$v"; done
+export HOME=$RUN
+lw() { "$BIN" "$@"; }
+seq 1 300 | sed 's/^/edge line /' >"$RUN/edge.log"
+info "runs lw in $(short "$RUN") with no configuration file"
+
+section "Color"
+red=$'\e[0;1;31m'
+check "--color paints the level name" "$(is "$(printf 'a ERROR b\n' | lw --color)" "a ${red}ERROR"$'\e[0m b')"
+check "--color never, and auto on a pipe, leave it plain" \
+	"$(is "$(printf 'a ERROR b\n' | lw --color never)$(printf 'a ERROR b\n' | lw)" "a ERROR ba ERROR b")"
+check "LOGWISP_COLOR=always, a sink's own color=never wins" \
+	"$(is "$(printf 'x WARN\n' | LOGWISP_COLOR=always lw --sink console,color=never)" "x WARN")"
+
+section "Presets"
+seq 1 5000 >"$RUN/seq.txt"
+lw --preset pipe <"$RUN/seq.txt" >"$RUN/pipe.out"
+check "--preset pipe copies stdin line for line" "$(cmp -s "$RUN/seq.txt" "$RUN/pipe.out" && echo 1 || echo 0)"
+out=$(timeout 3 "$BIN" --preset "tail,path=$RUN/edge.log,from=start" </dev/null 2>/dev/null | head -2 | tr '\n' '|')
+check "--preset tail follows a file" "$(is "$out" "edge line 1|edge line 2|")"
+lw --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15890,tls=self,hosts=logs.example" --color --dump >"$RUN/dump.toml"
+lw -c "$RUN/dump.toml" --dump >"$RUN/dump2.toml"
+check "--dump prints a file that reads back to the same dump" "$(cmp -s "$RUN/dump.toml" "$RUN/dump2.toml" && echo 1 || echo 0)"
+check "lw --check builds the dumped preset" "$(lw --check -c "$RUN/dump.toml" 2>"$LOG/check.err" | grep -q 'configuration ok' && echo 1 || echo 0)"
+lw preset edge -to 127.0.0.1:1 >/dev/null 2>"$LOG/edge-noauth.err"
+rc=$?
+check "an edge without credentials is a usage error ($rc)" "$( ((rc == 2)) && grep -q 'never sends unauthenticated' "$LOG/edge-noauth.err" && echo 1 || echo 0)"
+
+section "Certificates"
+lw tls ca -dir "$RUN/pki" 2>"$LOG/tls.out" &&
+	lw tls cert -ca-dir "$RUN/pki" -name logs.example -server -host 127.0.0.1,logs.example 2>>"$LOG/tls.out"
+check "lw tls ca and cert issue a chain openssl verifies" \
+	"$(openssl verify -CAfile "$RUN/pki/ca.crt" "$RUN/pki/logs.example.crt" 2>/dev/null | grep -q ': OK$' && echo 1 || echo 0)"
+check "keys are private" "$(is "$(stat -c %a "$RUN/pki/ca.key" "$RUN/pki/logs.example.key" | tr '\n' ' ')" "600 600 ")"
+lw tls ca -dir "$RUN/pki" 2>/dev/null
+check "an existing CA is never replaced" "$(( $? == 1 ))"
+
+AUTH=$RUN
+add_users "$RUN/users.toml" edge-01
+# pin DAEMON: the pin_sha256 a self-signed listener logged
+pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; }
+received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; }
+
+spawn agg-self "$BIN" --preset "aggregator,listen=127.0.0.1:15893,users=$RUN/users.toml,out=$RUN/out/self"
+wait_port 15893 || abort "the self-signed aggregator did not listen" agg-self
+PIN=$(pin agg-self)
+check "a self-signed aggregator logs its pin" "$(is_set "$PIN")"
+spawn edge-pin "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass"
+spawn edge-wrong "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=sha256//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,user=edge-01,password_file=$RUN/edge-01.pass"
+wait_until 10 eval '(( $(received self) >= 300 ))'
+check "an edge pinning it delivers every line over SCRAM ($(received self))" "$(( $(received self) == 300 ))"
+wait_until 10 logged edge-wrong 'matches no tls.pin_sha256'
+check "an edge with another pin is refused before it logs in" "$(logged edge-wrong 'matches no tls.pin_sha256' && echo 1 || echo 0)"
+
+spawn agg-issuer "$BIN" --preset "aggregator,listen=127.0.0.1:15894,transport=http,tls=issuer,issuer_cert=$RUN/pki/ca.crt,issuer_key=$RUN/pki/ca.key,users=$RUN/users.toml,out=$RUN/out/issuer"
+wait_port 15894 || abort "the issuing aggregator did not listen" agg-issuer
+spawn edge-ca "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass"
+wait_until 10 eval '(( $(received issuer) >= 300 ))'
+check "an aggregator signing its own certificate is verified by the CA ($(received issuer))" "$(( $(received issuer) == 300 ))"
+
+spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15895,tls=self"
+wait_port 15895 || abort "the serve preset did not listen" serve
+SERVE_PIN=$(pin serve)
+check "curl --pinnedpubkey reads a self-signed stream's status" \
+	"$(curl -sk --pinnedpubkey "$SERVE_PIN" https://127.0.0.1:15895/status | grep -q '"tls":true' && echo 1 || echo 0)"
+curl -sk --pinnedpubkey "$PIN" -o /dev/null https://127.0.0.1:15895/status
+check "curl with another listener's pin is refused" "$(( $? != 0 ))"
+
+section "Flush at exit"
+# feed SECONDS PORT: the lines, once a client had time to connect
+feed() { wait_port "$2"; sleep "$1"; seq 1 20000 | sed 's/^/flush /'; }
+queues='buffer_size=20000,client_buffer_size=20000'
+feed 1 15891 | lw --sink "tcp,host=127.0.0.1,port=15891,$queues" 2>"$LOG/flush-tcp.err" &
+fpid=$!
+wait_port 15891
+n=$(tcp_read 15891 15 | grep -c '^flush ')
+wait "$fpid"
+check "a tcp client gets all 20000 lines of a finite input ($n)" "$((n == 20000))"
+feed 1 15892 | lw --sink "http,host=127.0.0.1,port=15892,$queues" 2>"$LOG/flush-http.err" &
+fpid=$!
+wait_port 15892
+curl -sN --max-time 15 http://127.0.0.1:15892/stream >"$RUN/flush-http.out"
+wait "$fpid"
+n=$(grep -c '^data: flush ' "$RUN/flush-http.out")
+check "an http stream gets all 20000 lines, then the disconnect event ($n)" \
+	"$( ((n == 20000)) && tail -3 "$RUN/flush-http.out" | grep -q '^event: disconnect' && echo 1 || echo 0)"
+
+summary