v0.23.0 color, presets, certificates made at startup, cli.go, sink flush

internal/authz/authz.go

@@ -129,8 +129,9 @@ func New(o *config.AuthOptions, tlsCfg *tls.Config, role Role, transport Transpo
 	if tlsCfg == nil && (o.Type != MethodSCRAM || len(o.TrustedProxies) == 0) {
 		return nil, fmt.Errorf("auth: type %q requires tls.enabled", o.Type)
 	}
-	if role == RoleDialer && tlsCfg != nil && tlsCfg.InsecureSkipVerify {
-		// An unverified server makes identities claims and exposes credentials
+	if role == RoleDialer && tlsCfg != nil && tlsCfg.InsecureSkipVerify && tlsCfg.VerifyPeerCertificate == nil {
+		// An unverified server makes identities claims and exposes credentials;
+		// tls.pin_sha256 verifies it in VerifyPeerCertificate instead
 		return nil, fmt.Errorf("auth: type %q cannot be used with tls.insecure_skip_verify", o.Type)
 	}
 

internal/authz/authz_test.go

@@ -167,10 +167,13 @@ func TestNewValidation(t *testing.T) {
 	}
 
 	// The rows' bases are valid: a dialer needs TLS but not client_auth, as it
-	// pins the server's identity
+	// pins the server's identity; tls.pin_sha256 verifies in place of the chain
+	pinned := &tls.Config{InsecureSkipVerify: true, VerifyPeerCertificate: func([][]byte, [][]*x509.Certificate) error { return nil }}
 	for _, o := range []*config.AuthOptions{{Type: MethodMTLS}, dialer(config.AuthOptions{})} {
-		if _, err := New(o, &tls.Config{}, RoleDialer, TCP); err != nil {
-			t.Fatalf("dialer policy %s rejected: %v", o.Type, err)
+		for _, c := range []*tls.Config{{}, pinned} {
+			if _, err := New(o, c, RoleDialer, TCP); err != nil {
+				t.Fatalf("dialer policy %s rejected: %v", o.Type, err)
+			}
 		}
 	}
 	if _, err := New(listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, RoleListener, HTTP); err != nil {

internal/config/config.go

@@ -7,8 +7,13 @@ type Config struct {
 	// Top-level flags for application control
 	ShowVersion bool `toml:"version"`
 	Check       bool `toml:"check"` // build every plugin, report, exit: lw --check
+	Dump        bool `toml:"dump"`  // print the effective configuration, exit: lw --dump
 	Quiet       bool `toml:"quiet"`
 
+	// Console sinks without their own color: "auto" (a terminal, NO_COLOR
+	// unset, TERM not dumb), "always", "never"
+	Color string `toml:"color"`
+
 	// Runtime behavior flags
 	StatusReporter   bool `toml:"status_reporter"`
 	ConfigAutoReload bool `toml:"auto_reload"`
@@ -85,7 +90,7 @@ type PipelineConfig struct {
 type FlowConfig struct {
 	Heartbeat *HeartbeatConfig `toml:"heartbeat"`
 	RateLimit *RateLimitConfig `toml:"rate_limit"`
-	Filters   []FilterConfig   `toml:"filters"`
+	Filters   []FilterConfig   `toml:"filters,omitempty"`
 	Format    *FormatConfig    `toml:"format"`
 }
 
@@ -105,9 +110,9 @@ type HeartbeatConfig struct {
 // FormatConfig is a polymorphic struct representing log entry formatting options
 type FormatConfig struct {
 	Type            string `toml:"type"` // "json", "txt", "raw"
-	Flags           int64  `toml:"flags"`
-	TimestampFormat string `toml:"timestamp_format"`
-	SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell"
+	Flags           int64  `toml:"flags,omitempty"`
+	TimestampFormat string `toml:"timestamp_format,omitempty"`
+	SanitizerPolicy string `toml:"sanitizer_policy,omitempty"` // "raw", "json", "txt", "shell"
 }
 
 // --- Rate Limit Options ---
@@ -245,6 +250,7 @@ type ConsoleSinkOptions struct {
 	Target     string `toml:"target"` // "stdout", "stderr"
 	BufferSize int64  `toml:"buffer_size"`
 	Escape     string `toml:"escape"` // control characters as <hex>: "auto" (on a terminal), "always", "never"
+	Color      string `toml:"color"`  // level names in color: "auto", "always", "never"; default: the top-level color
 }
 
 // FileSinkOptions defines settings for a file-based sink
@@ -366,9 +372,10 @@ type AuthOptions struct {
 // --- TLS Options ---
 
 // TLSOptions is one shape for both roles. Listeners present cert_file and
-// key_file and verify clients with client_auth and client_ca_file; dialers
-// verify the server with ca_file and server_name and may present
-// cert_file and key_file.
+// key_file, or a certificate issued at startup (self_signed, or from the
+// issuer files), and verify clients with client_auth and client_ca_file;
+// dialers verify the server with ca_file and server_name, or pin_sha256, and
+// may present cert_file and key_file.
 type TLSOptions struct {
 	Enabled bool `toml:"enabled"`
 
@@ -376,6 +383,14 @@ type TLSOptions struct {
 	CertFile string `toml:"cert_file"`
 	KeyFile  string `toml:"key_file"`
 
+	// Listeners without cert_file: a certificate on a key made at startup,
+	// self-signed or signed by the issuer CA, for hosts plus the host option,
+	// os.Hostname(), localhost and the loopback addresses
+	SelfSigned     bool     `toml:"self_signed"`
+	IssuerCertFile string   `toml:"issuer_cert_file"`
+	IssuerKeyFile  string   `toml:"issuer_key_file"`
+	Hosts          []string `toml:"hosts"`
+
 	// Listener-side peer verification (mTLS)
 	ClientAuth   bool   `toml:"client_auth"`
 	ClientCAFile string `toml:"client_ca_file"`
@@ -384,6 +399,7 @@ type TLSOptions struct {
 	CAFile             string `toml:"ca_file"`     // empty = system trust store
 	ServerName         string `toml:"server_name"` // default: config host
 	InsecureSkipVerify bool   `toml:"insecure_skip_verify"`
+	PinSHA256          string `toml:"pin_sha256"` // sha256//BASE64 of the server key, ';' between several
 
 	// Minimum protocol version: "1.2" | "1.3" (default "1.3")
 	MinVersion string `toml:"min_version"`

internal/config/loader.go

@@ -6,7 +6,6 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/lixenwraith/logwisp/internal/core"
 
@@ -26,14 +25,18 @@ type Manager struct {
 	specs    []pipelineSpec // command-line or environment pipelines, kept across reloads
 }
 
+// Args is a parsed command line; cmd/lw owns its grammar.
+type Args struct {
+	File      string   // -c: the configuration file, "" for the environment or discovery
+	Specs     []Spec   // pipeline flags, in order
+	Overrides []string // --path=value settings, for lixenwraith/config
+}
+
 // Load reads the startup sources and validates the initial configuration.
 // Watching starts only when Watch is called after successful service startup.
-func Load(args []string) (*Manager, error) {
-	configPath, isExplicit, configArgs, err := resolveConfigPath(args)
-	if err != nil {
-		return nil, err
-	}
-	specs, configArgs, err := takePipelineSpecs(configArgs)
+func Load(args Args) (*Manager, error) {
+	configPath, isExplicit := resolveConfigPath(args.File)
+	specs, err := cliSpecs(args.Specs)
 	if err != nil {
 		return nil, err
 	}
@@ -53,7 +56,7 @@ func Load(args []string) (*Manager, error) {
 	cfg, err := lconfig.NewBuilder().
 		WithTarget(initial).
 		WithEnvPrefix("LOGWISP_").
-		WithArgs(configArgs).
+		WithArgs(args.Overrides).
 		WithFile(configPath).
 		WithTypedValidator(func(cfg *Config) error {
 			effective := *cfg // the builder keeps cfg; only the copy is replaced
@@ -96,6 +99,17 @@ func (m *Manager) Snapshot() (*Config, error) {
 	if err := m.usePipelines(cfg); err != nil {
 		return nil, err
 	}
+	// The top-level color is each console sink's default; the maps are fresh
+	for _, p := range cfg.Pipelines {
+		for i, s := range p.PluginSinks {
+			if _, set := s.Config["color"]; s.Type == "console" && !set {
+				if s.Config == nil {
+					p.PluginSinks[i].Config = map[string]any{}
+				}
+				p.PluginSinks[i].Config["color"] = cfg.Color
+			}
+		}
+	}
 	if err := checkFileKeys(m.path); err != nil {
 		return nil, fmt.Errorf("config file %q: %w", m.path, err)
 	}
@@ -145,6 +159,7 @@ func (m *Manager) Close() { m.config.StopAutoUpdate() }
 func defaults() *Config {
 	return &Config{
 		StatusReporter: true,
+		Color:          "auto",
 		Logging: &LogConfig{
 			Output: "stderr",
 			Level:  "info",
@@ -174,51 +189,28 @@ func pipeDefault(name string) PipelineConfig {
 	}
 }
 
-// resolveConfigPath consumes file-selection flags before schema CLI parsing.
-// The last selection wins, and -- terminates option handling.
-func resolveConfigPath(args []string) (path string, isExplicit bool, remaining []string, err error) {
-	remaining = make([]string, 0, len(args))
-	for i := 0; i < len(args); i++ {
-		arg := args[i]
-		if arg == "--" {
-			remaining = append(remaining, args[i:]...)
-			break
-		}
-		switch {
-		case arg == "-c" || arg == "--config":
-			if i+1 == len(args) || args[i+1] == "" || strings.HasPrefix(args[i+1], "-") {
-				return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg)
-			}
-			i++
-			path, isExplicit = args[i], true
-		case strings.HasPrefix(arg, "--config=") || strings.HasPrefix(arg, "-c="):
-			_, path, _ = strings.Cut(arg, "=")
-			if path == "" {
-				return "", false, nil, fmt.Errorf("%s requires a configuration file path", arg)
-			}
-			isExplicit = true
-		default:
-			remaining = append(remaining, arg)
-		}
-	}
-	if isExplicit {
-		return path, true, remaining, nil
+// resolveConfigPath picks the file: -c, then LOGWISP_CONFIG_FILE and
+// LOGWISP_CONFIG_DIR, then the first of the user's and the working directory's
+// logwisp.toml. Only the discovered default may be missing.
+func resolveConfigPath(file string) (path string, isExplicit bool) {
+	if file != "" {
+		return file, true
 	}
 	if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" {
 		path = configFile
 		if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
 			path = filepath.Join(configDir, configFile)
 		}
-		return path, true, remaining, nil
+		return path, true
 	}
 	if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
-		return filepath.Join(configDir, "logwisp.toml"), true, remaining, nil
+		return filepath.Join(configDir, "logwisp.toml"), true
 	}
 	if homeDir, err := os.UserHomeDir(); err == nil {
 		path = filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml")
 		if _, err := os.Stat(path); err == nil {
-			return path, false, remaining, nil
+			return path, false
 		}
 	}
-	return "logwisp.toml", false, remaining, nil
+	return "logwisp.toml", false
 }

internal/config/loader_test.go

@@ -12,29 +12,20 @@ import (
 	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
-func TestResolveConfigArguments(t *testing.T) {
+// -c wins over the environment, which wins over discovery; only the
+// discovered default is not explicit, so only it may be missing.
+func TestResolveConfigPath(t *testing.T) {
 	isolateConfig(t)
-	for _, args := range [][]string{{"-c", "chosen.toml"}, {"--config", "chosen.toml"}, {"--config=chosen.toml"}, {"-c=chosen.toml"}, {"-c", "old.toml", "--config=chosen.toml"}} {
-		path, explicit, remaining, err := resolveConfigPath(append(args, "--quiet"))
-		if err != nil || path != "chosen.toml" || !explicit || !reflect.DeepEqual(remaining, []string{"--quiet"}) {
-			t.Fatalf("%v: %q %v %v %v", args, path, explicit, remaining, err)
-		}
-	}
-	for _, args := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}} {
-		if _, _, _, err := resolveConfigPath(args); err == nil {
-			t.Fatalf("missing path accepted: %v", args)
-		}
-	}
-	args := []string{"--", "-c", "ignored.toml"}
-	path, explicit, remaining, err := resolveConfigPath(args)
-	if err != nil || explicit || path != "logwisp.toml" || !reflect.DeepEqual(remaining, args) {
-		t.Fatalf("terminator ignored: %q %v %v %v", path, explicit, remaining, err)
+	if path, explicit := resolveConfigPath(""); path != "logwisp.toml" || explicit {
+		t.Fatalf("discovered default: %q %v", path, explicit)
 	}
 	t.Setenv("LOGWISP_CONFIG_DIR", "configs")
 	t.Setenv("LOGWISP_CONFIG_FILE", "env.toml")
-	path, explicit, _, err = resolveConfigPath(nil)
-	if err != nil || !explicit || path != filepath.Join("configs", "env.toml") {
-		t.Fatalf("environment path: %q %v %v", path, explicit, err)
+	if path, explicit := resolveConfigPath(""); path != filepath.Join("configs", "env.toml") || !explicit {
+		t.Fatalf("environment path: %q %v", path, explicit)
+	}
+	if path, explicit := resolveConfigPath("chosen.toml"); path != "chosen.toml" || !explicit {
+		t.Fatalf("-c path: %q %v", path, explicit)
 	}
 }
 
@@ -43,7 +34,7 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
 	testutil.WriteFile(t, "selected.toml", "config_file = \"ignored.toml\"\n[logging]\nlevel = \"error\"\n")
 	t.Setenv("LOGGING_LEVEL", "invalid-bare-variable")
 	t.Setenv("LOGWISP_LOGGING_LEVEL", "debug")
-	m, err := Load([]string{"--config", "selected.toml", "--logging.level=warn", "--quiet"})
+	m, err := Load(Args{File: "selected.toml", Overrides: []string{"--logging.level=warn", "--quiet"}})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -61,7 +52,7 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
 	if _, aliased := second.Pipelines[0].PluginSources[0].Config["buffer_size"]; err != nil || second.Logging.Level != "warn" || aliased {
 		t.Fatalf("snapshot aliases another result: %+v %v", second, err)
 	}
-	env, err := Load([]string{"-c=selected.toml"})
+	env, err := Load(Args{File: "selected.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -74,19 +65,19 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) {
 
 func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) {
 	isolateConfig(t)
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
 	m.Close()
-	if _, err := Load([]string{"-c", "missing.toml"}); !errors.Is(err, ErrConfigNotFound) {
+	if _, err := Load(Args{File: "missing.toml"}); !errors.Is(err, ErrConfigNotFound) {
 		t.Fatalf("missing explicit path: %v", err)
 	}
-	if _, err := Load([]string{"--logging.file.max_size_mb=1.5"}); err == nil || errors.Is(err, ErrConfigNotFound) {
+	if _, err := Load(Args{Overrides: []string{"--logging.file.max_size_mb=1.5"}}); err == nil || errors.Is(err, ErrConfigNotFound) {
 		t.Fatalf("invalid CLI hidden by missing default: %v", err)
 	}
 	t.Setenv("LOGWISP_STATUS_REPORTER", "invalid")
-	if _, err := Load(nil); err == nil || errors.Is(err, ErrConfigNotFound) {
+	if _, err := Load(Args{}); err == nil || errors.Is(err, ErrConfigNotFound) {
 		t.Fatalf("invalid environment hidden by missing default: %v", err)
 	}
 }
@@ -94,7 +85,7 @@ func TestMissingFilesDoNotHideInvalidOverrides(t *testing.T) {
 func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
 	isolateConfig(t)
 	testutil.WriteFile(t, "reload.toml", "status_reporter = true\n")
-	m, err := Load([]string{"-c", "reload.toml"})
+	m, err := Load(Args{File: "reload.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -137,7 +128,7 @@ func TestReloadReadsDiskValidatesAndPreservesOldSnapshots(t *testing.T) {
 func TestWatchStartsOnSubscriptionAndCloses(t *testing.T) {
 	isolateConfig(t)
 	testutil.WriteFile(t, "watched.toml", "auto_reload = true\n")
-	m, err := Load([]string{"-c", "watched.toml"})
+	m, err := Load(Args{File: "watched.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -181,7 +172,7 @@ server_name = "relay.example"
 type = "none"
 allow = ["one", "two"]
 `)
-	m, err := Load([]string{"-c", "plugins.toml"})
+	m, err := Load(Args{File: "plugins.toml"})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -221,7 +212,7 @@ type = "tcp"
 [pipelines.plugin_sinks.config]
 port = 9000
 `+table+"\n")
-		m, err := Load([]string{"-c", "plugins.toml"})
+		m, err := Load(Args{File: "plugins.toml"})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -257,7 +248,7 @@ type = "tcp"
 [pipelines.plugin_sinks.config]
 port = 9000
 `+body+"\n")
-		m, err := Load([]string{"-c", "typo.toml"})
+		m, err := Load(Args{File: "typo.toml"})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -280,7 +271,7 @@ func TestNoConfigFileMeansQuietLogging(t *testing.T) {
 		if c.file != "" {
 			testutil.WriteFile(t, "logwisp.toml", c.file)
 		}
-		m, err := Load(nil)
+		m, err := Load(Args{})
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -292,7 +283,7 @@ func TestNoConfigFileMeansQuietLogging(t *testing.T) {
 	}
 	t.Setenv("LOGWISP_LOGGING_LEVEL", "debug")
 	os.Remove("logwisp.toml")
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -309,7 +300,7 @@ func TestFileStringIsOneListEntry(t *testing.T) {
 	testutil.WriteFile(t, "logwisp.toml", "[[pipelines]]\nname = \"p\"\n"+
 		"[[pipelines.plugin_sources]]\nid = \"in\"\ntype = \"null\"\n[[pipelines.plugin_sinks]]\nid = \"out\"\ntype = \"null\"\n"+
 		"[[pipelines.flow.filters]]\ntype = \"exclude\"\npatterns = \"password=\\\\S{8,64}\"\n")
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -319,3 +310,25 @@ func TestFileStringIsOneListEntry(t *testing.T) {
 		t.Fatalf("patterns: %v %v", cfg.Pipelines[0].Flow.Filters, err)
 	}
 }
+
+// The top-level color is the default of every console sink, which may set
+// its own; other sinks do not take the key.
+func TestTopLevelColorIsEachConsoleSinksDefault(t *testing.T) {
+	isolateConfig(t)
+	m, err := Load(Args{Specs: specs("sink", "console", "sink", "console,color=never", "sink", "null"), Overrides: []string{"--color=always"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer m.Close()
+	cfg, err := m.Snapshot()
+	if err != nil {
+		t.Fatal(err)
+	}
+	var colors []any
+	for _, s := range cfg.Pipelines[0].PluginSinks {
+		colors = append(colors, s.Config["color"])
+	}
+	if !reflect.DeepEqual(colors, []any{"always", "never", nil}) {
+		t.Fatalf("console sink colors: %v", colors)
+	}
+}

internal/config/preset.go

@@ -0,0 +1,332 @@
+package config
+
+import (
+	"cmp"
+	"fmt"
+	"maps"
+	"net"
+	"os"
+	"path/filepath"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+// Preset is a pipeline built from a few keys: --preset NAME,key=value starts
+// a pipeline with it. Later flags add sources, sinks and filters, or replace
+// its format, rate limit and heartbeat.
+type Preset struct {
+	Name, Summary string
+	Params        []PresetParam
+	build         func(p *PipelineConfig, v map[string]string) error
+}
+
+// PresetParam is one key. A list takes several values, ',' between them or
+// the key repeated; any other key takes one.
+type PresetParam struct {
+	Name, Default, Help string
+	Required, List      bool
+}
+
+// Keys several presets share
+var (
+	pathParam   = PresetParam{Name: "path", Help: "file, directory (its files) or glob to follow; default: standard input"}
+	fromParam   = PresetParam{Name: "from", Default: "end", Help: "start|end: where reading a file starts"}
+	formatParam = func(def string) PresetParam { return PresetParam{Name: "format", Default: def, Help: "raw|txt|json"} }
+	tlsParams   = []PresetParam{
+		{Name: "cert", Help: "certificate file, for tls=files"},
+		{Name: "key", Help: "key file, for tls=files"},
+		{Name: "issuer_cert", Help: "CA certificate file (lw tls ca), for tls=issuer"},
+		{Name: "issuer_key", Help: "CA key file, for tls=issuer"},
+		{Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"},
+	}
+	transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
+)
+
+var presets = []Preset{
+	{"pipe", "Standard input to standard output, line for line: lw without a file",
+		[]PresetParam{formatParam("raw")},
+		func(p *PipelineConfig, v map[string]string) error {
+			stdio := pipeDefault("")
+			p.PluginSources, p.PluginSinks = stdio.PluginSources, stdio.PluginSinks
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			return nil
+		}},
+	{"tail", "Follow files to standard output, like tail -F",
+		[]PresetParam{{Name: "path", Required: true, Help: "file, directory (its files) or glob to follow"}, fromParam, formatParam("raw")},
+		func(p *PipelineConfig, v map[string]string) error {
+			p.PluginSources, p.PluginSinks = pathOrStdin(v), pipeDefault("").PluginSinks
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			return nil
+		}},
+	{"serve", "Serve files or standard input as a live HTTP stream (SSE)",
+		append([]PresetParam{pathParam, fromParam, formatParam("json"),
+			{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
+			{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
+			{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
+			{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
+			{Name: "viewer", Default: "false", Help: "true: the built-in login and viewer pages, behind proxy"},
+		}, tlsParams...),
+		func(p *PipelineConfig, v map[string]string) error {
+			p.PluginSources = pathOrStdin(v)
+			sink, err := listener(v, "off")
+			if err != nil {
+				return err
+			}
+			if v["users"] != "" {
+				sink["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
+			}
+			if v["proxy"] != "" {
+				auth, ok := sink["auth"].(map[string]any)
+				if !ok {
+					return fmt.Errorf("proxy needs users: browsers log in with SCRAM")
+				}
+				auth["trusted_proxies"] = list(v["proxy"])
+			}
+			switch v["viewer"] {
+			case "true":
+				sink["login_page"], sink["viewer_page"] = true, true
+			case "false":
+			default:
+				return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"])
+			}
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "http", Type: "http", Config: sink})
+			return nil
+		}},
+	{"edge", "Forward files or standard input to an aggregator over TLS, authenticated",
+		[]PresetParam{pathParam, fromParam,
+			{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"},
+			transportParam,
+			{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"},
+			{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
+			{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
+			{Name: "user", Help: "SCRAM user (lw auth add-user on the aggregator)"},
+			{Name: "password_file", Help: "file holding the SCRAM password"},
+			{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"},
+			{Name: "key", Help: "client key file"},
+			{Name: "node", Help: "origin label; default: this host's name"},
+		},
+		func(p *PipelineConfig, v map[string]string) error {
+			p.PluginSources = pathOrStdin(v)
+			typ, err := chainType(v["transport"])
+			if err != nil {
+				return err
+			}
+			host, port, err := splitAddr("to", v["to"])
+			if err != nil {
+				return err
+			}
+			tls := map[string]any{"enabled": true}
+			for key, option := range map[string]string{"ca": "ca_file", "pin": "pin_sha256", "server_name": "server_name", "cert": "cert_file", "key": "key_file"} {
+				if v[key] != "" {
+					tls[option] = v[key]
+				}
+			}
+			var auth map[string]any
+			switch {
+			case v["user"] != "" && v["password_file"] != "":
+				auth = map[string]any{"type": "scram", "username": v["user"], "password_file": v["password_file"]}
+			case v["user"] != "" || v["password_file"] != "":
+				return fmt.Errorf("user and password_file go together")
+			case v["cert"] != "" && v["key"] != "":
+				auth = map[string]any{"type": "mtls"}
+			default:
+				return fmt.Errorf("edge needs user and password_file, or cert and key: it never sends unauthenticated")
+			}
+			sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth}
+			if v["node"] != "" {
+				sink["node"] = v["node"]
+			}
+			p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "edge", Type: typ, Config: sink})
+			return nil
+		}},
+	{"aggregator", "Receive from edges over TLS, authenticated, to files or standard output",
+		append([]PresetParam{
+			{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"},
+			transportParam,
+			{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
+			{Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"},
+			{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
+			{Name: "out", Help: "directory for the received entries; default: standard output"},
+			formatParam("json"),
+		}, tlsParams...),
+		func(p *PipelineConfig, v map[string]string) error {
+			typ, err := chainType(v["transport"])
+			if err != nil {
+				return err
+			}
+			if v["tls"] == "off" {
+				return fmt.Errorf("tls=off: edges send credentials and logs, the aggregator always serves TLS")
+			}
+			src, err := listener(v, "self")
+			if err != nil {
+				return err
+			}
+			if v["client_ca"] != "" {
+				tls := src["tls"].(map[string]any)
+				tls["client_auth"], tls["client_ca_file"] = true, v["client_ca"]
+			}
+			switch {
+			case v["users"] != "":
+				src["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
+			case v["client_ca"] != "":
+				src["auth"] = map[string]any{"type": "mtls"}
+			default:
+				return fmt.Errorf("aggregator needs users or client_ca: it never receives unauthenticated")
+			}
+			p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src})
+			p.PluginSinks = pipeDefault("").PluginSinks
+			if v["out"] != "" {
+				p.PluginSinks = []PluginSinkConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": v["out"], "name": "aggregate"}}}
+			}
+			p.Flow.Format = &FormatConfig{Type: v["format"]}
+			return nil
+		}},
+}
+
+// Presets lists the presets, for lw preset
+func Presets() []Preset { return slices.Clone(presets) }
+
+// ExpandPreset builds the pipeline a preset makes of values, as --preset does
+func ExpandPreset(name string, values map[string]string) (PipelineConfig, error) {
+	p := PipelineConfig{Flow: &FlowConfig{}}
+	each := map[string][]string{}
+	for k, v := range values {
+		each[k] = []string{v}
+	}
+	err := applyPreset(&p, name, each)
+	return p, err
+}
+
+// applyPreset fills defaults and checks keys before the preset's build
+func applyPreset(p *PipelineConfig, name string, values map[string][]string) error {
+	i := slices.IndexFunc(presets, func(r Preset) bool { return r.Name == name })
+	if i < 0 {
+		var names []string
+		for _, r := range presets {
+			names = append(names, r.Name)
+		}
+		return fmt.Errorf("no preset %q (valid: %s)", name, strings.Join(names, ", "))
+	}
+	r := presets[i]
+	v, keys := map[string]string{}, []string{}
+	for _, k := range r.Params {
+		v[k.Name] = k.Default
+		keys = append(keys, k.Name)
+	}
+	for k, each := range values {
+		j := slices.IndexFunc(r.Params, func(p PresetParam) bool { return p.Name == k })
+		switch {
+		case j < 0:
+			return fmt.Errorf("preset %s has no key %q (valid: %s)", name, k, strings.Join(keys, ", "))
+		case len(each) > 1 && !r.Params[j].List:
+			return fmt.Errorf("preset %s key %q takes one value", name, k)
+		}
+		v[k] = cmp.Or(strings.Join(each, ","), v[k])
+	}
+	for _, k := range r.Params {
+		if k.Required && v[k.Name] == "" {
+			return fmt.Errorf("preset %s needs %s", name, k.Name)
+		}
+	}
+	p.Name = cmp.Or(p.Name, name)
+	return r.build(p, v)
+}
+
+func pathOrStdin(v map[string]string) []PluginSourceConfig {
+	if v["path"] == "" {
+		return pipeDefault("").PluginSources
+	}
+	return []PluginSourceConfig{fileSource(v["path"], v["from"])}
+}
+
+// fileSource follows path: a directory's files, or the files a glob or a
+// file name matches in its directory.
+func fileSource(path, from string) PluginSourceConfig {
+	dir, pattern := path, "*"
+	if fi, err := os.Stat(path); err != nil || !fi.IsDir() {
+		dir, pattern = filepath.Dir(path), filepath.Base(path)
+	}
+	return PluginSourceConfig{ID: "file", Type: "file", Config: map[string]any{"directory": dir, "pattern": pattern, "from": from}}
+}
+
+// tlsModes is what each preset tls mode sets, and the preset keys it needs
+// by the tls option each fills; made marks a certificate made at startup,
+// which takes hosts.
+var tlsModes = map[string]struct {
+	set   map[string]any
+	needs map[string]string
+	made  bool
+}{
+	"off":    {},
+	"self":   {set: map[string]any{"self_signed": true}, made: true},
+	"issuer": {needs: map[string]string{"issuer_cert": "issuer_cert_file", "issuer_key": "issuer_key_file"}, made: true},
+	"files":  {needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
+}
+
+// listener starts a listening plugin's options: host, port and tls, whose
+// mode decides which certificate keys apply.
+func listener(v map[string]string, def string) (map[string]any, error) {
+	host, port, err := splitAddr("listen", v["listen"])
+	if err != nil {
+		return nil, err
+	}
+	opts := map[string]any{"host": host, "port": port}
+	mode := cmp.Or(v["tls"], def)
+	m, ok := tlsModes[mode]
+	if !ok {
+		return nil, fmt.Errorf("tls %q (valid: %s)", mode, strings.Join(slices.Sorted(maps.Keys(tlsModes)), ", "))
+	}
+	for _, other := range tlsModes {
+		for k := range other.needs {
+			if _, needed := m.needs[k]; needed && v[k] == "" {
+				return nil, fmt.Errorf("tls=%s needs %s", mode, k)
+			} else if !needed && v[k] != "" {
+				return nil, fmt.Errorf("%s does not apply to tls=%s", k, mode)
+			}
+		}
+	}
+	if v["hosts"] != "" && !m.made {
+		return nil, fmt.Errorf("hosts does not apply to tls=%s", mode)
+	}
+	if mode == "off" {
+		return opts, nil
+	}
+	tls := map[string]any{"enabled": true}
+	maps.Copy(tls, m.set)
+	for k, option := range m.needs {
+		tls[option] = v[k]
+	}
+	if v["hosts"] != "" {
+		tls["hosts"] = list(v["hosts"])
+	}
+	opts["tls"] = tls
+	return opts, nil
+}
+
+func splitAddr(key, addr string) (string, int64, error) {
+	host, p, err := net.SplitHostPort(addr)
+	port, perr := strconv.ParseUint(p, 10, 16)
+	if err != nil || perr != nil || port == 0 {
+		return "", 0, fmt.Errorf("%s %q: want HOST:PORT, an IPv6 address in brackets", key, addr)
+	}
+	return host, int64(port), nil
+}
+
+func chainType(transport string) (string, error) {
+	if transport != "tcp" && transport != "http" {
+		return "", fmt.Errorf("transport %q (valid: tcp, http)", transport)
+	}
+	return transport + "_chain", nil
+}
+
+func list(s string) []any {
+	var out []any
+	for e := range strings.SplitSeq(s, ",") {
+		if e = strings.TrimSpace(e); e != "" {
+			out = append(out, e)
+		}
+	}
+	return out
+}

internal/config/preset_test.go

@@ -0,0 +1,63 @@
+package config
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// A preset starts its pipeline and names it unless --pipeline did; later
+// sinks add to its own, a later format replaces its own.
+func TestPresetStartsAndNamesItsPipeline(t *testing.T) {
+	isolateConfig(t)
+	got := loadPipelines(t, "preset", "tail,path=/var/log/app.log", "sink", "null", "format", "json",
+		"pipeline", "mine", "preset", "pipe")
+	want := []PipelineConfig{{
+		Name:          "tail",
+		Flow:          &FlowConfig{Format: &FormatConfig{Type: "json"}},
+		PluginSources: []PluginSourceConfig{{ID: "file", Type: "file", Config: map[string]any{"directory": "/var/log", "pattern": "app.log", "from": "end"}}},
+		PluginSinks: []PluginSinkConfig{
+			{ID: "stdout", Type: "console", Config: map[string]any{"color": "auto"}},
+			{ID: "null", Type: "null", Config: map[string]any{}},
+		},
+	}, {
+		Name:          "mine",
+		Flow:          &FlowConfig{Format: &FormatConfig{Type: "raw"}},
+		PluginSources: []PluginSourceConfig{{ID: "stdin", Type: "console", Config: map[string]any{}}},
+		PluginSinks:   []PluginSinkConfig{{ID: "stdout", Type: "console", Config: map[string]any{"color": "auto"}}},
+	}}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
+	}
+	if _, err := Load(Args{Specs: specs("sink", "null", "preset", "pipe")}); err == nil || !strings.Contains(err.Error(), "must start its pipeline") {
+		t.Fatalf("preset after a sink: %v", err)
+	}
+}
+
+// Keys are checked against the preset's own, required ones enforced, and
+// the edge and aggregator refuse to run without authentication.
+func TestPresetKeysAreChecked(t *testing.T) {
+	isolateConfig(t)
+	for spec, want := range map[string]string{
+		"nope":                         `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`,
+		"tail,path=x,colour=1":         `preset tail has no key "colour" (valid: path, from, format)`,
+		"tail":                         "preset tail needs path",
+		"tail,path=x,format.type=json": "preset keys do not nest",
+		"tail,path=x,path=y":           `key "path" takes one value`,
+		"serve,listen=8080":            `listen "8080": want HOST:PORT`,
+		"serve,tls=self,cert=c":        "cert does not apply to tls=self",
+		"serve,tls=files,cert=c":       "tls=files needs key",
+		"serve,viewer=yes":             `viewer "yes"`,
+		"serve,proxy=127.0.0.1":        "proxy needs users",
+		"edge,to=agg:9000":             "it never sends unauthenticated",
+		"edge,to=agg:9000,user=u":      "user and password_file go together",
+		"edge,to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
+		"aggregator,users=u,tls=off":                            "the aggregator always serves TLS",
+		"aggregator":                                            "it never receives unauthenticated",
+	} {
+		_, err := Load(Args{Specs: specs("preset", spec)})
+		if err == nil || !strings.Contains(err.Error(), want) {
+			t.Errorf("%s: err = %v, want %s", spec, err, want)
+		}
+	}
+}

internal/config/spec.go

@@ -16,11 +16,28 @@ type specKind struct {
 	flag  string
 	typed bool // SPEC starts with TYPE
 	many  bool // repeatable within a pipeline; the environment adds _1.._N
+	first bool // starts its pipeline, and names it by TYPE unless --pipeline did
 	apply func(p *PipelineConfig, typ string, opts map[string]any) error
 }
 
 var specKinds = []specKind{
-	{"source", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"preset", true, false, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+		values := map[string][]string{}
+		for k, v := range opts {
+			switch v := v.(type) {
+			case string:
+				values[k] = []string{v}
+			case []any: // a repeated key
+				for _, e := range v {
+					values[k] = append(values[k], e.(string))
+				}
+			default:
+				return fmt.Errorf("preset keys do not nest: %q", k)
+			}
+		}
+		return applyPreset(p, typ, values)
+	}},
+	{"source", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		ids := make([]string, len(p.PluginSources))
 		for i, s := range p.PluginSources {
 			ids[i] = s.ID
@@ -29,7 +46,7 @@ var specKinds = []specKind{
 		p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: id, Type: typ, Config: opts})
 		return err
 	}},
-	{"sink", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"sink", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		ids := make([]string, len(p.PluginSinks))
 		for i, s := range p.PluginSinks {
 			ids[i] = s.ID
@@ -38,24 +55,24 @@ var specKinds = []specKind{
 		p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: id, Type: typ, Config: opts})
 		return err
 	}},
-	{"filter", true, true, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"filter", true, true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		opts["type"] = typ
 		f := FilterConfig{}
 		err := Scan(opts, &f)
 		p.Flow.Filters = append(p.Flow.Filters, f)
 		return err
 	}},
-	{"format", true, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
+	{"format", true, false, false, func(p *PipelineConfig, typ string, opts map[string]any) error {
 		opts["type"] = typ
 		p.Flow.Format = &FormatConfig{}
 		return Scan(opts, p.Flow.Format)
 	}},
 	// Naming a stage turns it on: the file defaults (pass, disabled) would not.
-	{"rate-limit", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
+	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
 		p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
 		return Scan(opts, p.Flow.RateLimit)
 	}},
-	{"heartbeat", false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
+	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any) error {
 		p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
 		return Scan(opts, p.Flow.Heartbeat)
 	}},
@@ -68,33 +85,36 @@ type pipelineSpec struct {
 	value string
 }
 
-// takePipelineSpecs consumes the pipeline flags, which lixenwraith/config
-// would report as unrecognized.
-func takePipelineSpecs(args []string) (specs []pipelineSpec, rest []string, err error) {
-	for i := 0; i < len(args); i++ {
-		if args[i] == "--" {
-			return specs, append(rest, args[i:]...), nil
-		}
-		name, value, inline := strings.Cut(args[i], "=")
+// Spec is one pipeline flag: its name without dashes, and its value.
+type Spec struct{ Flag, Value string }
+
+// SpecFlags names the pipeline flags, for the command-line parser.
+func SpecFlags() []string {
+	flags := []string{"pipeline"}
+	for _, k := range specKinds {
+		flags = append(flags, k.flag)
+	}
+	return flags
+}
+
+// cliSpecs looks up the kind of each command-line spec.
+func cliSpecs(in []Spec) ([]pipelineSpec, error) {
+	var specs []pipelineSpec
+	for _, s := range in {
 		var kind *specKind
-		if name != "--pipeline" {
-			k := slices.IndexFunc(specKinds, func(k specKind) bool { return "--"+k.flag == name })
+		if s.Flag != "pipeline" {
+			k := slices.IndexFunc(specKinds, func(k specKind) bool { return k.flag == s.Flag })
 			if k < 0 {
-				rest = append(rest, args[i])
-				continue
+				return nil, fmt.Errorf("--%s is no pipeline flag", s.Flag)
 			}
 			kind = &specKinds[k]
 		}
-		if !inline && i+1 < len(args) && !strings.HasPrefix(args[i+1], "-") {
-			i++
-			value = args[i]
+		if s.Value == "" {
+			return nil, fmt.Errorf("--%s requires a value", s.Flag)
 		}
-		if value == "" {
-			return nil, nil, fmt.Errorf("%s requires a value", name)
-		}
-		specs = append(specs, pipelineSpec{name, kind, value})
+		specs = append(specs, pipelineSpec{"--" + s.Flag, kind, s.Value})
 	}
-	return specs, rest, nil
+	return specs, nil
 }
 
 // envPipelineSpecs reads the one-pipeline environment form. An empty variable
@@ -131,48 +151,58 @@ func envPipelineSpecs() []pipelineSpec {
 }
 
 // buildPipelines turns specs, in order, into pipelines; specs before the first
-// --pipeline go to one named "cli". A pipeline without a source reads stdin,
-// one without a sink writes stdout. Every call returns fresh maps.
+// --pipeline go to one named "cli". Every call returns fresh maps.
 func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) {
 	var pipelines []PipelineConfig
 	var seen map[*specKind]bool
 	for _, s := range specs {
-		if s.kind == nil || pipelines == nil {
-			name := "cli"
-			if s.kind == nil {
-				name = s.value
-			}
-			pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
+		if s.kind == nil {
+			pipelines = append(pipelines, PipelineConfig{Name: s.value, Flow: &FlowConfig{}})
 			seen = map[*specKind]bool{}
-			if s.kind == nil {
-				continue
-			}
+			continue
 		}
-		p := &pipelines[len(pipelines)-1]
-		if seen[s.kind] && !s.kind.many {
-			return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
-		}
-		seen[s.kind] = true
 		typ, opts, err := parseSpec(s.value, s.kind.typed)
 		if err != nil {
 			return nil, fmt.Errorf("%s %w", s.name, err)
 		}
+		if pipelines == nil {
+			name := "cli"
+			if s.kind.first {
+				name = typ
+			}
+			pipelines = append(pipelines, PipelineConfig{Name: name, Flow: &FlowConfig{}})
+			seen = map[*specKind]bool{}
+		}
+		p := &pipelines[len(pipelines)-1]
+		switch {
+		case seen[s.kind] && !s.kind.many:
+			return nil, fmt.Errorf("%s %s: pipeline %q already has one", s.name, s.value, p.Name)
+		case s.kind.first && len(seen) > 0:
+			return nil, fmt.Errorf("%s %s: must start its pipeline", s.name, s.value)
+		}
+		seen[s.kind] = true
 		if err := s.kind.apply(p, typ, opts); err != nil {
 			return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err)
 		}
 	}
 	for i := range pipelines {
-		pipe := pipeDefault(pipelines[i].Name)
-		if len(pipelines[i].PluginSources) == 0 {
-			pipelines[i].PluginSources = pipe.PluginSources
-		}
-		if len(pipelines[i].PluginSinks) == 0 {
-			pipelines[i].PluginSinks = pipe.PluginSinks
-		}
+		useStdio(&pipelines[i])
 	}
 	return pipelines, nil
 }
 
+// useStdio makes a pipeline without a source read stdin, and one without a
+// sink write stdout, as a filter does.
+func useStdio(p *PipelineConfig) {
+	pipe := pipeDefault(p.Name)
+	if len(p.PluginSources) == 0 {
+		p.PluginSources = pipe.PluginSources
+	}
+	if len(p.PluginSinks) == 0 {
+		p.PluginSinks = pipe.PluginSinks
+	}
+}
+
 // parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
 // a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
 func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {

internal/config/spec_test.go

@@ -10,9 +10,18 @@ import (
 	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
-func loadPipelines(t *testing.T, args ...string) []PipelineConfig {
+// specs pairs flags with values: specs("sink", "null") is --sink null
+func specs(pairs ...string) []Spec {
+	var out []Spec
+	for i := 0; i < len(pairs); i += 2 {
+		out = append(out, Spec{pairs[i], pairs[i+1]})
+	}
+	return out
+}
+
+func loadPipelines(t *testing.T, pairs ...string) []PipelineConfig {
 	t.Helper()
-	m, err := Load(args)
+	m, err := Load(Args{Specs: specs(pairs...)})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -27,15 +36,14 @@ func loadPipelines(t *testing.T, args ...string) []PipelineConfig {
 func TestPipelineSpecGrammar(t *testing.T) {
 	isolateConfig(t)
 	got := loadPipelines(t,
-		"--source", `file,directory=/var/log/a\,b,pattern=*.log`,
-		`--source=file,id=app,directory=x\=y`,
-		"--source", `file,directory=c:\\logs\\`,
-		"--sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c",
-		"--filter", "include,patterns=ERROR,patterns=WARN",
-		"--filter", `exclude,patterns=\d{1\,3}`,
-		"--format", `txt,timestamp_format=Jan 2\, 2006`,
-		"--pipeline", "b", "--source", "null", "--sink", "null", "--sink", "null", "--heartbeat", "interval_ms=1000",
-		"--", "--source", "random")
+		"source", `file,directory=/var/log/a\,b,pattern=*.log`,
+		"source", `file,id=app,directory=x\=y`,
+		"source", `file,directory=c:\\logs\\`,
+		"sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.cert_file=c",
+		"filter", "include,patterns=ERROR,patterns=WARN",
+		"filter", `exclude,patterns=\d{1\,3}`,
+		"format", `txt,timestamp_format=Jan 2\, 2006`,
+		"pipeline", "b", "source", "null", "sink", "null", "sink", "null", "heartbeat", "interval_ms=1000")
 	want := []PipelineConfig{{
 		Name: "cli",
 		Flow: &FlowConfig{
@@ -67,17 +75,17 @@ func TestPipelineSpecGrammar(t *testing.T) {
 	if !reflect.DeepEqual(got, want) {
 		t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
 	}
-	for want, args := range map[string][]string{
-		`--sink http,port: missing "="`:                                  {"--sink", "http,port"},
-		`--source directory=/x: missing TYPE`:                            {"--source", "directory=/x"},
-		`--source file,type=x: TYPE already sets "type"`:                 {"--source", "file,type=x"},
-		`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"--sink", "http,tls=1,tls.cert_file=c"},
-		`--sink requires a value`:                                        {"--sink="},
-		`--format txt: pipeline "cli" already has one`:                   {"--format", "json", "--format", "txt"},
-		`--rate-limit rate=1,polcy=drop: unknown key "polcy"`:            {"--rate-limit", "rate=1,polcy=drop"},
+	for want, pairs := range map[string][]string{
+		`--sink http,port: missing "="`:                                  {"sink", "http,port"},
+		`--source directory=/x: missing TYPE`:                            {"source", "directory=/x"},
+		`--source file,type=x: TYPE already sets "type"`:                 {"source", "file,type=x"},
+		`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http,tls=1,tls.cert_file=c"},
+		`--sink requires a value`:                                        {"sink", ""},
+		`--format txt: pipeline "cli" already has one`:                   {"format", "json", "format", "txt"},
+		`--rate-limit rate=1,polcy=drop: unknown key "polcy"`:            {"rate-limit", "rate=1,polcy=drop"},
 	} {
-		if _, err := Load(args); err == nil || err.Error() != want {
-			t.Errorf("%q: err = %v, want %s", args, err, want)
+		if _, err := Load(Args{Specs: specs(pairs...)}); err == nil || err.Error() != want {
+			t.Errorf("%q: err = %v, want %s", pairs, err, want)
 		}
 	}
 }
@@ -86,7 +94,7 @@ func TestPipelineSpecGrammar(t *testing.T) {
 // must not split into several, wider entries.
 func TestSpecValueIsOneListEntry(t *testing.T) {
 	isolateConfig(t)
-	sink := loadPipelines(t, "--source", "null", "--sink",
+	sink := loadPipelines(t, "source", "null", "sink",
 		`tcp,auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0]
 	var opts TCPSinkOptions
 	if err := Scan(sink.Config, &opts); err != nil {
@@ -115,7 +123,7 @@ func TestEnvironmentPipeline(t *testing.T) {
 	} {
 		t.Setenv(name, value)
 	}
-	m, err := Load(nil)
+	m, err := Load(Args{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -136,7 +144,7 @@ func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) {
 	isolateConfig(t)
 	t.Setenv("LOGWISP_SOURCE", "random")
 	t.Setenv("LOGWISP_SINK", "console")
-	got := loadPipelines(t, "--source", "null", "--sink", "null")
+	got := loadPipelines(t, "source", "null", "sink", "null")
 	if len(got) != 1 || len(got[0].PluginSources) != 1 || got[0].PluginSources[0].Type != "null" ||
 		len(got[0].PluginSinks) != 1 || got[0].PluginSinks[0].Type != "null" {
 		t.Fatalf("environment specs mixed into command-line pipelines: %+v", got)
@@ -147,7 +155,7 @@ func TestCommandLinePipelinesIgnoreEnvironment(t *testing.T) {
 func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
 	isolateConfig(t)
 	testutil.WriteFile(t, "file.toml", "[logging]\nlevel = \"warn\"\n[[pipelines]]\nname = \"file\"\n")
-	m, err := Load([]string{"-c", "file.toml", "--source", "null", "--sink", "null", "--status_reporter=false"})
+	m, err := Load(Args{File: "file.toml", Specs: specs("source", "null", "sink", "null"), Overrides: []string{"--status_reporter=false"}})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -165,12 +173,12 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
 // stdout, as a filter does; two pipelines cannot both read stdin.
 func TestSpecPipelinesDefaultToStdio(t *testing.T) {
 	isolateConfig(t)
-	got := loadPipelines(t, "--filter", "include,patterns=ERROR")
+	got := loadPipelines(t, "filter", "include,patterns=ERROR")
 	if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" ||
 		len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" {
 		t.Fatalf("pipeline: %+v", p)
 	}
-	_, err := Load([]string{"--pipeline", "a", "--sink", "null", "--pipeline", "b", "--sink", "null"})
+	_, err := Load(Args{Specs: specs("pipeline", "a", "sink", "null", "pipeline", "b", "sink", "null")})
 	if err == nil || !strings.Contains(err.Error(), `console source already reads stdin in pipeline "a"`) {
 		t.Fatalf("second stdin reader: %v", err)
 	}
@@ -181,7 +189,7 @@ func TestSpecPipelinesDefaultToStdio(t *testing.T) {
 // removal fails the reload instead of keeping the removed file's values.
 func TestReloadKeepsSpecPipelines(t *testing.T) {
 	isolateConfig(t)
-	m, err := Load([]string{"--source", "random,special=true", "--sink", "null"})
+	m, err := Load(Args{Specs: specs("source", "random,special=true", "sink", "null")})
 	if err != nil {
 		t.Fatal(err)
 	}

internal/config/validate.go

@@ -34,6 +34,10 @@ func ValidateConfig(cfg *Config) error {
 		names[p.Name] = struct{}{}
 	}
 
+	if err := lconfig.OneOf("auto", "always", "never")(cfg.Color); err != nil {
+		return fmt.Errorf("color: %w", err)
+	}
+
 	if err := validateLogConfig(cfg.Logging); err != nil {
 		return fmt.Errorf("logging: %w", err)
 	}

internal/core/const.go

@@ -43,6 +43,9 @@ const (
 
 	ShutdownTimeout = 10 * time.Second
 
+	// Bounds the flush of a network sink's queues at Stop
+	SinkFlushTimeout = 2 * time.Second
+
 	ConfigReloadTimeout = 30 * time.Second
 
 	LoggerShutdownTimeout = 2 * time.Second

internal/sink/console/console.go

@@ -18,8 +18,10 @@ import (
 	"github.com/lixenwraith/logwisp/internal/session"
 	"github.com/lixenwraith/logwisp/internal/sink"
 
+	"github.com/lixenwraith/color"
 	lconfig "github.com/lixenwraith/config"
 	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/terminal/inline"
 	"golang.org/x/term"
 )
 
@@ -38,8 +40,9 @@ type ConsoleSink struct {
 	session *session.Session
 
 	// Configuration
-	config *config.ConsoleSinkOptions
-	escape bool
+	config  *config.ConsoleSinkOptions
+	escape  bool
+	painter *inline.Printer // paints level names; nil when color is off
 
 	// Application
 	input  chan core.TransportEvent
@@ -63,8 +66,23 @@ const (
 	DefaultConsoleTarget     = "stdout"
 	DefaultConsoleBufferSize = 1000
 	DefaultConsoleEscape     = "auto"
+	DefaultConsoleColor      = "auto"
 )
 
+// levelColors paints the first of a level's names in an entry, ANSI 16 so
+// the terminal's theme picks the shades and a text console renders them.
+// Names run longest first; cyan, not blue, stays readable on black.
+var levelColors = map[string]struct {
+	names []string
+	style inline.Style
+}{
+	"ERROR": {[]string{"ERROR", "FATAL", "ERR"}, inline.FgANSI(color.ANSIRed).Bold()},
+	"WARN":  {[]string{"WARNING", "WARN"}, inline.FgANSI(color.ANSIYellow)},
+	"INFO":  {[]string{"INFO", "INF"}, inline.FgANSI(color.ANSIGreen)},
+	"DEBUG": {[]string{"DEBUG", "DBG"}, inline.FgANSI(color.ANSICyan)},
+	"TRACE": {[]string{"TRACE"}, inline.FgANSI(color.ANSIBrightBlack)},
+}
+
 // NewConsoleSinkPlugin creates a console sink through plugin factory
 func NewConsoleSinkPlugin(
 	id string,
@@ -72,7 +90,7 @@ func NewConsoleSinkPlugin(
 	logger *log.Logger,
 	proxy *session.Proxy,
 ) (sink.Sink, error) {
-	opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape}
+	opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape, Color: DefaultConsoleColor}
 
 	// Scan config map into struct
 	if err := config.Scan(configMap, opts); err != nil {
@@ -85,6 +103,9 @@ func NewConsoleSinkPlugin(
 	if err := lconfig.OneOf("auto", "always", "never")(opts.Escape); err != nil {
 		return nil, fmt.Errorf("escape: %w", err)
 	}
+	if err := lconfig.OneOf("auto", "always", "never")(opts.Color); err != nil {
+		return nil, fmt.Errorf("color: %w", err)
+	}
 
 	output := os.Stdout
 	if opts.Target == "stderr" {
@@ -95,17 +116,29 @@ func NewConsoleSinkPlugin(
 		opts.BufferSize = DefaultConsoleBufferSize
 	}
 
+	// inline decides auto: a terminal, NO_COLOR unset, TERM not dumb. Paint
+	// returns its text unchanged exactly when color is off.
+	printer := inline.New(output)
+	if opts.Color != "auto" {
+		printer.SetColor(opts.Color == "always")
+	}
+	var painter *inline.Printer
+	if printer.Paint("x", inline.Style{}) != "x" {
+		painter = printer
+	}
+
 	// Create and return plugin instance
 	cs := &ConsoleSink{
-		id:     id,
-		proxy:  proxy,
-		config: opts,
-		escape: opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())),
-		input:  make(chan core.TransportEvent, opts.BufferSize),
-		output: output,
-		done:   make(chan struct{}),
-		exited: make(chan struct{}),
-		logger: logger,
+		id:      id,
+		proxy:   proxy,
+		config:  opts,
+		escape:  opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())),
+		painter: painter,
+		input:   make(chan core.TransportEvent, opts.BufferSize),
+		output:  output,
+		done:    make(chan struct{}),
+		exited:  make(chan struct{}),
+		logger:  logger,
 	}
 	cs.lastProcessed.Store(time.Time{})
 
@@ -124,6 +157,7 @@ func NewConsoleSinkPlugin(
 		"instance_id", id,
 		"target", opts.Target,
 		"escape", cs.escape,
+		"color", painter != nil,
 	)
 
 	return cs, nil
@@ -187,6 +221,7 @@ func (cs *ConsoleSink) GetStats() sink.SinkStats {
 			"target":      cs.config.Target,
 			"buffer_size": cs.config.BufferSize,
 			"escape":      cs.escape,
+			"color":       cs.painter != nil,
 		},
 	}
 }
@@ -224,6 +259,9 @@ func (cs *ConsoleSink) write(event core.TransportEvent) {
 	if cs.escape {
 		payload = escapeControls(payload)
 	}
+	if cs.painter != nil {
+		payload = paintLevel(cs.painter, payload, event.Entry.Level)
+	}
 	if _, err := cs.output.Write(payload); err != nil {
 		cs.logger.Error("msg", "Failed to write to console",
 			"component", "console_sink",
@@ -235,6 +273,42 @@ func (cs *ConsoleSink) write(event core.TransportEvent) {
 	cs.lastProcessed.Store(time.Now())
 }
 
+// paintLevel colors the first whole-word, case-insensitive occurrence of a
+// name of level, after escaping: the escape sequences are the sink's own.
+func paintLevel(p *inline.Printer, payload []byte, level string) []byte {
+	lc, ok := levelColors[level]
+	if !ok {
+		return payload
+	}
+	for _, name := range lc.names {
+		for i := 0; i+len(name) <= len(payload); i++ {
+			j := i + len(name)
+			if !nameAt(payload[i:j], name) || i > 0 && word(payload[i-1]) || j < len(payload) && word(payload[j]) {
+				continue
+			}
+			out := make([]byte, 0, len(payload)+16)
+			out = append(out, payload[:i]...)
+			out = append(out, p.Paint(string(payload[i:j]), lc.style)...)
+			return append(out, payload[j:]...)
+		}
+	}
+	return payload
+}
+
+// nameAt compares ASCII letters without case; name is upper case
+func nameAt(b []byte, name string) bool {
+	for k := range len(name) {
+		if c := b[k]; c != name[k] && c != name[k]+'a'-'A' {
+			return false
+		}
+	}
+	return true
+}
+
+func word(c byte) bool {
+	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'
+}
+
 // escapeControls writes what a terminal would act on or reorder (C0 and C1
 // controls but tab, DEL, bidi controls, line separators, invalid UTF-8) as
 // <hex>, keeping the final newline: a log line cannot drive the terminal.

internal/sink/console/console_test.go

@@ -1,6 +1,11 @@
 package console
 
-import "testing"
+import (
+	"io"
+	"testing"
+
+	"github.com/lixenwraith/terminal/inline"
+)
 
 // Everything a terminal would act on is escaped; tabs and the final newline,
 // which it only lays out, are kept.
@@ -23,3 +28,22 @@ func TestEscapeControlsKeepsOnlyLayout(t *testing.T) {
 		}
 	}
 }
+
+// The entry's level is painted where its line first names it as a whole
+// word, in any case, longest name first; other words and levels stay plain.
+func TestPaintLevelColorsTheFirstWholeName(t *testing.T) {
+	p := inline.New(io.Discard)
+	p.SetColor(true)
+	red, yellow := "\x1b[0;1;31m", "\x1b[0;33m"
+	for _, c := range []struct{ in, level, want string }{
+		{"2026 ERROR boom ERROR\n", "ERROR", "2026 " + red + "ERROR\x1b[0m boom ERROR\n"},
+		{"[warning] disk\n", "WARN", "[" + yellow + "warning\x1b[0m] disk\n"},
+		{"terror_x ERR:1\n", "ERROR", "terror_x " + red + "ERR\x1b[0m:1\n"},
+		{"INFOS informal\n", "INFO", "INFOS informal\n"},
+		{"ERROR without level\n", "", "ERROR without level\n"},
+	} {
+		if got := string(paintLevel(p, []byte(c.in), c.level)); got != c.want {
+			t.Errorf("%q %s: got %q, want %q", c.in, c.level, got, c.want)
+		}
+	}
+}

internal/sink/http/http.go

@@ -43,7 +43,6 @@ const (
 	DefaultHTTPStreamPath       = "/stream"
 	DefaultHTTPStatusPath       = "/status"
 	HTTPReadHeaderTimeout       = 10 * time.Second
-	HTTPShutdownTimeout         = 2 * time.Second
 )
 
 // HTTPSink streams log entries via Server-Sent Events
@@ -80,8 +79,14 @@ type HTTPSink struct {
 	auth *authz.Policy
 	web  map[string]http.Handler // proxy mode: GET /auth/... browser files
 
-	// Runtime
+	// Runtime. Stop closes flush; the broker then moves what remains of its
+	// input to tail and closes flushed; each stream writes its queue, tail
+	// and the disconnect event on its own, until flushBy.
 	done      chan struct{}
+	flush     chan struct{}
+	flushed   chan struct{}
+	flushBy   time.Time
+	tail      [][]byte // read only once flushed is closed
 	stopOnce  sync.Once
 	wg        sync.WaitGroup
 	startTime time.Time
@@ -145,7 +150,7 @@ func NewHTTPSinkPlugin(
 	if opts.ClientBufferSize <= 0 {
 		opts.ClientBufferSize = DefaultHTTPClientBufferSize
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -174,6 +179,8 @@ func NewHTTPSinkPlugin(
 		network:      network,
 		input:        make(chan core.TransportEvent, opts.BufferSize),
 		done:         make(chan struct{}),
+		flush:        make(chan struct{}),
+		flushed:      make(chan struct{}),
 		logger:       logger,
 		clients:      make(map[uint64]*sseClient),
 		writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
@@ -276,7 +283,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error {
 	h.startTime = time.Now()
 
 	h.wg.Add(1)
-	go h.brokerLoop(ctx)
+	go h.brokerLoop()
 
 	serve := h.server.Serve
 	if h.tlsConfig != nil {
@@ -325,18 +332,21 @@ func (h *HTTPSink) Stop() {
 }
 
 // shutdown funnels ctx-cancel and Stop() teardown through a single path.
-// done is closed first so SSE handlers exit and Shutdown can complete;
-// Server.Close force-closes any handler stalled in a deadline-free write.
+// Server.Shutdown refuses new streams and waits for the open ones, which end
+// once they wrote what is queued, within sink.FlushBound; Server.Close cuts a
+// stream stalled past it.
 func (h *HTTPSink) shutdown() {
 	h.stopOnce.Do(func() {
-		close(h.done)
 		if h.server != nil {
-			sctx, cancel := context.WithTimeout(context.Background(), HTTPShutdownTimeout)
+			h.flushBy = time.Now().Add(sink.FlushBound(h.writeTimeout))
+			ctx, cancel := context.WithDeadline(context.Background(), h.flushBy)
 			defer cancel()
-			if err := h.server.Shutdown(sctx); err != nil {
+			close(h.flush)
+			if err := h.server.Shutdown(ctx); err != nil {
 				h.server.Close()
 			}
 		}
+		close(h.done)
 	})
 }
 
@@ -356,19 +366,22 @@ func (h *HTTPSink) removeClient(id uint64) {
 }
 
 // brokerLoop fans out transport events to all client queues, non-blocking,
-// and evicts clients whose sessions were idle-expired by the session manager
-func (h *HTTPSink) brokerLoop(ctx context.Context) {
+// and evicts clients whose sessions were idle-expired by the session manager,
+// until flush, which shutdown closes on Stop and on context cancellation.
+func (h *HTTPSink) brokerLoop() {
 	defer h.wg.Done()
 	for {
 		select {
-		case <-ctx.Done():
+		case <-h.flush: // first: a select with both ready picks at random
+			h.flushInput()
 			return
-		case <-h.done:
+		default:
+		}
+		select {
+		case <-h.flush:
+			h.flushInput()
 			return
-		case event, ok := <-h.input:
-			if !ok {
-				return
-			}
+		case event := <-h.input:
 			h.totalProcessed.Add(1)
 			h.lastProcessed.Store(time.Now())
 
@@ -395,6 +408,13 @@ func (h *HTTPSink) brokerLoop(ctx context.Context) {
 	}
 }
 
+// flushInput moves what remains of the input to tail for the streams
+func (h *HTTPSink) flushInput() {
+	h.tail = sink.Drain(h.input)
+	h.totalProcessed.Add(uint64(len(h.tail)))
+	close(h.flushed)
+}
+
 // handleStream serves one client's SSE stream
 func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 	if h.config.MaxConnections > 0 && h.activeClients.Load() >= h.config.MaxConnections {
@@ -483,6 +503,13 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 	idle := time.NewTicker(h.keepalive)
 	defer idle.Stop()
 
+	send := func(payload []byte) bool {
+		if writeSSE(w, payload) != nil || rc.Flush() != nil {
+			return false
+		}
+		h.proxy.UpdateActivity(sess.ID)
+		return true
+	}
 	clientGone := r.Context().Done()
 	for {
 		select {
@@ -491,13 +518,9 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 				return // broker evicted (stale session)
 			}
 			h.armWrite(rc)
-			if err := writeSSE(w, payload); err != nil {
+			if !send(payload) {
 				return
 			}
-			if err := rc.Flush(); err != nil {
-				return
-			}
-			h.proxy.UpdateActivity(sess.ID)
 		case <-idle.C:
 			h.armWrite(rc)
 			if _, err := fmt.Fprint(w, ":\n\n"); err != nil {
@@ -509,7 +532,20 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 			h.proxy.UpdateActivity(sess.ID)
 		case <-clientGone:
 			return
-		case <-h.done:
+		case <-h.flushed:
+			// Nothing more will be queued: write the queue and the tail, then
+			// say why the stream ends
+			_ = rc.SetWriteDeadline(h.flushBy)
+			for len(c.send) > 0 {
+				if payload, ok := <-c.send; !ok || !send(payload) {
+					return
+				}
+			}
+			for _, payload := range h.tail {
+				if !send(payload) {
+					return
+				}
+			}
 			fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
 			rc.Flush()
 			return

internal/sink/http/http_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"fmt"
 	"io"
 	"maps"
 	"net/http"
@@ -319,3 +320,73 @@ func TestSSEFramesEveryLineAsData(t *testing.T) {
 		t.Fatalf("framed %q, want %q", got, want)
 	}
 }
+
+// Stop writes what is queued, in the sink and per stream, then the
+// disconnect event, to a client that keeps reading, whatever another that
+// stopped reading does; Stop returns at the flush bound.
+func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) {
+	const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold
+	httpSink, _ := newTestHTTPSink(t, map[string]any{"buffer_size": int64(n), "client_buffer_size": int64(64), "write_timeout_ms": int64(500)})
+	client, baseURL := serveTestHTTPSink(t, httpSink)
+	var scanners []*bufio.Scanner
+	for range 2 {
+		resp, err := client.Get(baseURL + "/stream")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { _ = resp.Body.Close() })
+		scanner := bufio.NewScanner(resp.Body)
+		scanner.Buffer(nil, 64*1024)
+		for scanner.Scan() && scanner.Text() != "event: connected" {
+		}
+		scanners = append(scanners, scanner)
+	}
+	for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+		httpSink.clientsMu.Lock()
+		registered := len(httpSink.clients)
+		httpSink.clientsMu.Unlock()
+		if registered == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("streams never registered")
+		}
+	}
+	type result struct {
+		lines      int
+		disconnect bool
+	}
+	done := make(chan result)
+	go func() {
+		var r result
+		for scanners[0].Scan() {
+			switch line := scanners[0].Text(); {
+			case strings.HasPrefix(line, "data: line "):
+				r.lines++
+			case line == "event: disconnect":
+				r.disconnect = true
+			}
+		}
+		done <- r
+	}()
+	// Held, the lock keeps the input queued until Stop has begun its flush
+	httpSink.clientsMu.Lock()
+	padding := strings.Repeat("x", 8192)
+	for i := range n {
+		httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)}
+	}
+	stopped := make(chan time.Duration)
+	go func() {
+		start := time.Now()
+		httpSink.Stop()
+		stopped <- time.Since(start)
+	}()
+	time.Sleep(100 * time.Millisecond)
+	httpSink.clientsMu.Unlock()
+	if d := <-stopped; d > 1500*time.Millisecond {
+		t.Errorf("Stop took %v past a 500 ms flush bound", d)
+	}
+	if r := <-done; r.lines != n || !r.disconnect {
+		t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect)
+	}
+}

internal/sink/sink.go

@@ -35,3 +35,27 @@ type SinkStats struct {
 	LastProcessed     time.Time
 	Details           map[string]any
 }
+
+// FlushBound is how long a network sink's Stop gives its clients to take
+// what is queued: write_timeout_ms, at most core.SinkFlushTimeout, which a
+// reload also waits out for a client that stopped reading.
+func FlushBound(writeTimeout time.Duration) time.Duration {
+	if writeTimeout > 0 && writeTimeout < core.SinkFlushTimeout {
+		return writeTimeout
+	}
+	return core.SinkFlushTimeout
+}
+
+// Drain takes the payloads queued in input without waiting: a network sink's
+// last events, which each client then writes on its own.
+func Drain(input <-chan core.TransportEvent) [][]byte {
+	var tail [][]byte
+	for {
+		select {
+		case event := <-input:
+			tail = append(tail, event.Payload)
+		default:
+			return tail
+		}
+	}
+}

internal/sink/tcp/tcp.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"maps"
 	"net"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -74,8 +75,15 @@ type TCPSink struct {
 	// Authorization
 	auth *authz.Policy
 
-	// Runtime
+	// Runtime. Stop closes flush; the broadcast loop then moves what remains
+	// of its input to tail and closes flushed; each writer writes its queue
+	// and tail on its own, until flushBy. done tears down what is left.
 	done      chan struct{}
+	flush     chan struct{}
+	flushed   chan struct{}
+	flushBy   time.Time
+	tail      [][]byte // read only once flushed is closed
+	flushing  bool     // under clientsMu: no client registers once set
 	stopOnce  sync.Once
 	wg        sync.WaitGroup
 	startTime time.Time
@@ -91,11 +99,13 @@ type TCPSink struct {
 
 // tcpClient is a registered connection's bounded send queue.
 // send is written by the broadcast loop (non-blocking) and drained by the
-// writer goroutine; closed signals reader-detected disconnect.
+// writer goroutine; closed signals reader-detected disconnect, exited the
+// writer's end.
 type tcpClient struct {
 	send      chan []byte
 	sessionID string
 	closed    chan struct{}
+	exited    chan struct{}
 }
 
 // NewTCPSinkPlugin creates a tcp sink through plugin factory
@@ -131,7 +141,7 @@ func NewTCPSinkPlugin(
 	if opts.KeepAlivePeriodMS <= 0 {
 		opts.KeepAlivePeriodMS = DefaultTCPKeepAlivePeriodMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -148,6 +158,8 @@ func NewTCPSinkPlugin(
 		network:      network,
 		input:        make(chan core.TransportEvent, opts.BufferSize),
 		done:         make(chan struct{}),
+		flush:        make(chan struct{}),
+		flushed:      make(chan struct{}),
 		logger:       logger,
 		clients:      make(map[uint64]*tcpClient),
 		conns:        make(map[net.Conn]struct{}),
@@ -223,7 +235,7 @@ func (t *TCPSink) Start(ctx context.Context) error {
 
 	t.wg.Add(2)
 	go t.acceptLoop()
-	go t.broadcastLoop(ctx)
+	go t.broadcastLoop()
 
 	// Pipeline context cancellation mirrors gnet engine stop: cease accepting
 	// and tear down existing connections
@@ -258,13 +270,27 @@ func (t *TCPSink) Stop() {
 		"total_processed", t.totalProcessed.Load())
 }
 
-// shutdown funnels ctx-cancel and Stop() teardown through a single path
+// shutdown funnels ctx-cancel and Stop() teardown through a single path.
+// Clients first receive what is queued, within sink.FlushBound, so a finite
+// input reaches them whole.
 func (t *TCPSink) shutdown() {
 	t.stopOnce.Do(func() {
-		close(t.done)
 		if t.listener != nil {
 			t.listener.Close() // unblocks acceptLoop
+			t.flushBy = time.Now().Add(sink.FlushBound(t.writeTimeout))
+			close(t.flush)
+			t.clientsMu.Lock()
+			t.flushing = true
+			clients := slices.Collect(maps.Values(t.clients))
+			t.clientsMu.Unlock()
+			for _, c := range clients {
+				select {
+				case <-c.exited:
+				case <-time.After(time.Until(t.flushBy)):
+				}
+			}
 		}
+		close(t.done)
 		t.clientsMu.Lock()
 		for conn := range t.conns {
 			conn.Close() // unblocks handshakes and per-connection readers
@@ -381,10 +407,17 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 		send:      make(chan []byte, t.config.ClientBufferSize),
 		sessionID: sess.ID,
 		closed:    make(chan struct{}),
+		exited:    make(chan struct{}),
 	}
+	defer close(c.exited)
 	id := t.nextClientID.Add(1)
 
 	t.clientsMu.Lock()
+	if t.flushing {
+		t.clientsMu.Unlock()
+		t.proxy.RemoveSession(sess.ID)
+		return
+	}
 	t.clients[id] = c
 	t.clientsMu.Unlock()
 
@@ -422,23 +455,45 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 
 	// Writer: synchronous lib write with deadline. A failed write means
 	// the kernel buffer stayed full for the full deadline - connection is
-	// dead or hopelessly stalled, so disconnect immediately (no gnet-style
-	// consecutive-error counter needed for transient async callback errors).
+	// dead or hopelessly stalled, so disconnect immediately.
+	write := func(data []byte, deadline time.Time) bool {
+		if !deadline.IsZero() {
+			conn.SetWriteDeadline(deadline)
+		}
+		if _, err := conn.Write(data); err != nil {
+			t.writeErrors.Add(1)
+			t.logger.Debug("msg", "Write failed, closing client",
+				"component", "tcp_sink",
+				"remote_addr", remote,
+				"error", err)
+			return false
+		}
+		t.proxy.UpdateActivity(sess.ID)
+		return true
+	}
 	for {
 		select {
 		case data := <-c.send:
+			var deadline time.Time
 			if t.writeTimeout > 0 {
-				conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
+				deadline = time.Now().Add(t.writeTimeout)
 			}
-			if _, err := conn.Write(data); err != nil {
-				t.writeErrors.Add(1)
-				t.logger.Debug("msg", "Write failed, closing client",
-					"component", "tcp_sink",
-					"remote_addr", remote,
-					"error", err)
+			if !write(data, deadline) {
 				return
 			}
-			t.proxy.UpdateActivity(sess.ID)
+		case <-t.flushed:
+			// Nothing more will be queued: write the queue, then the tail
+			for len(c.send) > 0 {
+				if !write(<-c.send, t.flushBy) {
+					return
+				}
+			}
+			for _, data := range t.tail {
+				if !write(data, t.flushBy) {
+					return
+				}
+			}
+			return
 		case <-c.closed:
 			return
 		case <-t.done:
@@ -447,19 +502,22 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 	}
 }
 
-// broadcastLoop fans out transport events to all client queues, non-blocking
-func (t *TCPSink) broadcastLoop(ctx context.Context) {
+// broadcastLoop fans out transport events to all client queues until flush,
+// which shutdown closes on Stop and on context cancellation alike.
+func (t *TCPSink) broadcastLoop() {
 	defer t.wg.Done()
 	for {
 		select {
-		case <-ctx.Done():
+		case <-t.flush: // first: a select with both ready picks at random
+			t.flushInput()
 			return
-		case <-t.done:
+		default:
+		}
+		select {
+		case <-t.flush:
+			t.flushInput()
 			return
-		case event, ok := <-t.input:
-			if !ok {
-				return
-			}
+		case event := <-t.input:
 			t.totalProcessed.Add(1)
 			t.lastProcessed.Store(time.Now())
 
@@ -477,6 +535,13 @@ func (t *TCPSink) broadcastLoop(ctx context.Context) {
 	}
 }
 
+// flushInput moves what remains of the input to tail for the writers
+func (t *TCPSink) flushInput() {
+	t.tail = sink.Drain(t.input)
+	t.totalProcessed.Add(uint64(len(t.tail)))
+	close(t.flushed)
+}
+
 // GetStats returns sink statistics
 func (t *TCPSink) GetStats() sink.SinkStats {
 	lastProc, _ := t.lastProcessed.Load().(time.Time)

internal/sink/tcp/tcp_test.go

@@ -1,11 +1,15 @@
 package tcp
 
 import (
+	"bufio"
+	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/lixenwraith/logwisp/internal/core"
 	"github.com/lixenwraith/logwisp/internal/session"
 	"github.com/lixenwraith/logwisp/internal/testutil"
 
@@ -57,3 +61,81 @@ func TestStopDoesNotWaitForSilentHandshake(t *testing.T) {
 		t.Fatalf("Stop took %v with a silent peer in the handshake", d)
 	}
 }
+
+// Stop writes what is queued, in the sink and per client, to a client that
+// keeps reading, whatever another that stopped reading does: the end of a
+// finite input reaches it whole, and Stop returns at the flush bound.
+func TestStopWritesQueuedEventsToReadingClients(t *testing.T) {
+	const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold
+	probe, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := probe.Addr().String()
+	probe.Close()
+	_, port, _ := net.SplitHostPort(addr)
+	p, _ := strconv.ParseInt(port, 10, 64)
+
+	manager := session.NewManager(time.Hour)
+	t.Cleanup(manager.Stop)
+	created, err := NewTCPSinkPlugin("out", map[string]any{
+		"host": "127.0.0.1", "port": p, "buffer_size": int64(n), "client_buffer_size": int64(64),
+		"write_timeout_ms": int64(500),
+	}, log.NewLogger(), session.NewProxy(manager, "out"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sink := created.(*TCPSink)
+	if err := sink.Start(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	var conns []net.Conn
+	for range 2 {
+		conn, err := net.Dial("tcp4", addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { conn.Close() })
+		conns = append(conns, conn)
+	}
+	for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+		sink.clientsMu.Lock()
+		registered := len(sink.clients)
+		sink.clientsMu.Unlock()
+		if registered == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("clients never registered")
+		}
+	}
+	lines := make(chan int)
+	go func() {
+		count, scanner := 0, bufio.NewScanner(conns[0])
+		scanner.Buffer(nil, 64*1024)
+		for scanner.Scan() {
+			count++
+		}
+		lines <- count
+	}()
+	// Held, the lock keeps the input queued until Stop has begun its flush
+	sink.clientsMu.Lock()
+	padding := strings.Repeat("x", 8192)
+	for i := range n {
+		sink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)}
+	}
+	stopped := make(chan time.Duration)
+	go func() {
+		start := time.Now()
+		sink.Stop()
+		stopped <- time.Since(start)
+	}()
+	time.Sleep(100 * time.Millisecond)
+	sink.clientsMu.Unlock()
+	if d := <-stopped; d > 1500*time.Millisecond {
+		t.Errorf("Stop took %v past a 500 ms flush bound", d)
+	}
+	if got := <-lines; got != n {
+		t.Fatalf("reading client received %d of %d lines", got, n)
+	}
+}

internal/sink/tcpchain/tcpchain.go

@@ -329,8 +329,9 @@ func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
 					return false
 				}
 				t.failures++
-				if errors.Is(err, authz.ErrRefused) {
-					// A refusal is configuration, not weather: show it by default
+				if errors.Is(err, authz.ErrRefused) || errors.As(err, new(*tls.CertificateVerificationError)) {
+					// A refusal, or a server failing its CA or pin, is
+					// configuration, not weather: show it by default
 					t.logger.Warn("msg", "Chain connect refused",
 						"component", "tcp_chain_sink",
 						"target", t.addr,

internal/source/httpchain/httpchain.go

@@ -112,7 +112,7 @@ func NewHTTPChainSourcePlugin(
 	if opts.ReadTimeoutMS <= 0 {
 		opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}

internal/source/tcpchain/tcpchain.go

@@ -97,7 +97,7 @@ func NewTCPChainSourcePlugin(
 	if opts.HelloTimeoutMS <= 0 {
 		opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}

internal/testutil/fixtures.go

@@ -6,6 +6,7 @@ import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/tls"
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/pem"
@@ -120,6 +121,40 @@ func (p *PKI) Leaf(t testing.TB, name, cn string, client bool) (certFile, keyFil
 	return writePEM(t, p.dir, name+".crt", "CERTIFICATE", der), writePEM(t, p.dir, name+".key", "PRIVATE KEY", keyDER)
 }
 
+// Handshake runs a TLS handshake between the two configurations over
+// loopback TCP, whose buffers let a failing side's alert go out without a
+// reader, and returns the client's error, else the server's.
+func Handshake(t testing.TB, server, client *tls.Config) error {
+	t.Helper()
+	ln, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	serverErr := make(chan error, 1)
+	go func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			serverErr <- err
+			return
+		}
+		defer conn.Close()
+		conn.SetDeadline(time.Now().Add(5 * time.Second))
+		serverErr <- tls.Server(conn, server).Handshake()
+	}()
+	conn, err := net.Dial("tcp4", ln.Addr().String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	conn.SetDeadline(time.Now().Add(5 * time.Second))
+	err = tls.Client(conn, client).Handshake()
+	conn.Close()
+	if serr := <-serverErr; err == nil {
+		err = serr
+	}
+	return err
+}
+
 func writePEM(t testing.TB, dir, name, blockType string, der []byte) string {
 	t.Helper()
 	path := filepath.Join(dir, name)

internal/tlsx/pki.go

@@ -0,0 +1,231 @@
+package tlsx
+
+import (
+	"bytes"
+	"cmp"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"slices"
+	"strings"
+	"sync"
+	"time"
+	"unicode"
+)
+
+// Certificates are ECDSA P-256, for lw tls and for listeners without files.
+// A CA signs leaves only; a leaf lasts at most 397 days, the longest browsers
+// accept, and never past its issuer.
+const (
+	CAValidity   = 10 * 365 * 24 * time.Hour
+	LeafValidity = 397 * 24 * time.Hour
+	pinPrefix    = "sha256//"
+)
+
+// processKey is the key of every generated listener certificate: reloads
+// reissue certificates and keep it, so a pin holds until lw restarts.
+var processKey = sync.OnceValues(NewKey)
+
+func NewKey() (*ecdsa.PrivateKey, error) { return ecdsa.GenerateKey(elliptic.P256(), rand.Reader) }
+
+// NewCA makes a self-signed certificate authority that signs leaves only
+func NewCA(name string, validity time.Duration) (*x509.Certificate, *ecdsa.PrivateKey, error) {
+	key, err := NewKey()
+	if err != nil {
+		return nil, nil, err
+	}
+	t, err := template(name, validity)
+	if err != nil {
+		return nil, nil, err
+	}
+	t.IsCA, t.MaxPathLenZero = true, true
+	t.KeyUsage = x509.KeyUsageCertSign | x509.KeyUsageCRLSign
+	cert, err := sign(t, t, &key.PublicKey, key)
+	return cert, key, err
+}
+
+// Leaf describes a certificate to issue. Hosts become DNS or IP SANs, which
+// hostname verification reads; Name is the subject CN, which mtls reads.
+type Leaf struct {
+	Name           string
+	Hosts          []string
+	Server, Client bool
+	Validity       time.Duration
+}
+
+// Issue signs a certificate for pub; a nil issuer makes it self-signed.
+func (l Leaf) Issue(pub crypto.PublicKey, issuer *x509.Certificate, issuerKey crypto.Signer) (*x509.Certificate, error) {
+	t, err := template(l.Name, l.Validity)
+	if err != nil {
+		return nil, err
+	}
+	t.KeyUsage = x509.KeyUsageDigitalSignature
+	if l.Server {
+		t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageServerAuth)
+	}
+	if l.Client {
+		t.ExtKeyUsage = append(t.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
+	}
+	for _, h := range l.Hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			t.IPAddresses = append(t.IPAddresses, ip)
+		} else {
+			t.DNSNames = append(t.DNSNames, h)
+		}
+	}
+	if issuer == nil {
+		return sign(t, t, pub, issuerKey)
+	}
+	if !time.Now().Before(issuer.NotAfter) {
+		return nil, fmt.Errorf("tls: issuer %q expired on %s", issuer.Subject.CommonName, issuer.NotAfter.UTC().Format(time.DateOnly))
+	}
+	if t.NotAfter.After(issuer.NotAfter) {
+		t.NotAfter = issuer.NotAfter
+	}
+	return sign(t, issuer, pub, issuerKey)
+}
+
+func template(name string, validity time.Duration) (*x509.Certificate, error) {
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now()
+	return &x509.Certificate{
+		SerialNumber:          serial,
+		Subject:               pkix.Name{CommonName: name},
+		NotBefore:             now.Add(-5 * time.Minute), // peers' clocks drift
+		NotAfter:              now.Add(validity),
+		BasicConstraintsValid: true,
+	}, nil
+}
+
+func sign(t, parent *x509.Certificate, pub crypto.PublicKey, key crypto.Signer) (*x509.Certificate, error) {
+	der, err := x509.CreateCertificate(rand.Reader, t, parent, pub, key)
+	if err != nil {
+		return nil, fmt.Errorf("tls: sign certificate: %w", err)
+	}
+	return x509.ParseCertificate(der)
+}
+
+// LoadIssuer reads a CA certificate and its key, as lw tls ca writes them
+func LoadIssuer(certFile, keyFile string) (*x509.Certificate, crypto.Signer, error) {
+	pair, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("tls: load issuer: %w", err)
+	}
+	ca, err := x509.ParseCertificate(pair.Certificate[0]) // Leaf is nil under GODEBUG=x509keypairleaf=0
+	if err != nil {
+		return nil, nil, fmt.Errorf("tls: load issuer: %w", err)
+	}
+	signer, ok := pair.PrivateKey.(crypto.Signer)
+	if !ok || !ca.IsCA || ca.KeyUsage&x509.KeyUsageCertSign == 0 {
+		return nil, nil, fmt.Errorf("tls: %s is no certificate authority", certFile)
+	}
+	return ca, signer, nil
+}
+
+// generated issues a listener certificate on the process key, self-signed
+// or from the issuer files, for hosts and the names this machine answers to.
+func generated(issuerCert, issuerKey, host string, hosts []string) (tls.Certificate, error) {
+	key, err := processKey()
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	var issuer *x509.Certificate
+	var signer crypto.Signer = key
+	if issuerCert != "" {
+		if issuer, signer, err = LoadIssuer(issuerCert, issuerKey); err != nil {
+			return tls.Certificate{}, err
+		}
+	}
+	name, _ := os.Hostname()
+	all := slices.Clone(hosts)
+	if ip := net.ParseIP(host); host != "" && (ip == nil || !ip.IsUnspecified()) {
+		all = append(all, host)
+	}
+	if !strings.ContainsFunc(name, func(r rune) bool { return r > unicode.MaxASCII }) {
+		all = append(all, name) // a DNS SAN is ASCII; the CN keeps any name
+	}
+	all = append(all, "localhost", "127.0.0.1", "::1")
+	slices.Sort(all)
+	all = slices.Compact(slices.DeleteFunc(all, func(h string) bool { return h == "" }))
+	leaf, err := Leaf{Name: cmp.Or(name, "logwisp"), Hosts: all, Server: true, Validity: LeafValidity}.Issue(&key.PublicKey, issuer, signer)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	chain := [][]byte{leaf.Raw}
+	if issuer != nil {
+		chain = append(chain, issuer.Raw) // an intermediate needs it to chain to its root
+	}
+	return tls.Certificate{Certificate: chain, PrivateKey: key, Leaf: leaf}, nil
+}
+
+// PinSHA256 is the curl --pinnedpubkey form of a certificate's public key
+func PinSHA256(c *x509.Certificate) string { return pin(c.RawSubjectPublicKeyInfo) }
+
+func pinOfKey(k *ecdsa.PrivateKey) string {
+	der, _ := x509.MarshalPKIXPublicKey(&k.PublicKey) // P-256 always marshals
+	return pin(der)
+}
+
+func pin(spki []byte) string {
+	h := sha256.Sum256(spki)
+	return pinPrefix + base64.StdEncoding.EncodeToString(h[:])
+}
+
+// verifyPins replaces chain verification: the server's key must hash to one
+// of the ';'-separated pins. Its name and validity are not checked. Go skips
+// it on a resumed session, and no dialer keeps a ClientSessionCache.
+func verifyPins(s string) (func([][]byte, [][]*x509.Certificate) error, error) {
+	var pins [][]byte
+	for p := range strings.SplitSeq(s, ";") {
+		b64, ok := strings.CutPrefix(strings.TrimSpace(p), pinPrefix)
+		h, err := base64.StdEncoding.DecodeString(b64)
+		if !ok || err != nil || len(h) != sha256.Size {
+			return nil, fmt.Errorf("tls: pin_sha256 %q: want %sBASE64 of a SHA-256, ';' between several", p, pinPrefix)
+		}
+		pins = append(pins, h)
+	}
+	return func(raw [][]byte, _ [][]*x509.Certificate) error {
+		if len(raw) == 0 {
+			return errors.New("tls: server sent no certificate")
+		}
+		c, err := x509.ParseCertificate(raw[0])
+		if err != nil {
+			return err
+		}
+		h := sha256.Sum256(c.RawSubjectPublicKeyInfo)
+		if slices.ContainsFunc(pins, func(p []byte) bool { return bytes.Equal(p, h[:]) }) {
+			return nil
+		}
+		// The type of a failed chain check: callers treat both as a refusal
+		return &tls.CertificateVerificationError{UnverifiedCertificates: []*x509.Certificate{c},
+			Err: fmt.Errorf("server key %s matches no tls.pin_sha256", PinSHA256(c))}
+	}, nil
+}
+
+// EncodeCert and EncodeKey render PEM, the key as PKCS #8
+func EncodeCert(c *x509.Certificate) []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.Raw})
+}
+
+func EncodeKey(k crypto.PrivateKey) ([]byte, error) {
+	der, err := x509.MarshalPKCS8PrivateKey(k)
+	if err != nil {
+		return nil, err
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil
+}

internal/tlsx/tlsx.go

@@ -26,16 +26,36 @@ const (
 	expiryWindow = 30 * 24 * time.Hour
 )
 
-// Server builds the *tls.Config for listener plugins
-// (tcp/http sinks, tcp_chain/http_chain sources). Returns (nil, nil) when disabled.
-func Server(o *config.TLSOptions) (*tls.Config, error) {
+// Server builds the *tls.Config for listener plugins (tcp/http sinks,
+// tcp_chain/http_chain sources) from cert_file and key_file, or issues one on
+// the process key: self-signed or from the issuer files, for host, hosts and
+// this machine's names. Returns (nil, nil) when disabled.
+func Server(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if o == nil || !o.Enabled {
 		return nil, nil
 	}
-	if o.CertFile == "" || o.KeyFile == "" {
-		return nil, fmt.Errorf("tls: cert_file and key_file are required for listeners")
+	files, issuer := o.CertFile != "" || o.KeyFile != "", o.IssuerCertFile != "" || o.IssuerKeyFile != ""
+	switch {
+	case o.PinSHA256 != "":
+		return nil, fmt.Errorf("tls: pin_sha256 applies to dialers")
+	case files && (issuer || o.SelfSigned) || issuer && o.SelfSigned:
+		return nil, fmt.Errorf("tls: set one of cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
+	case len(o.Hosts) > 0 && !issuer && !o.SelfSigned:
+		return nil, fmt.Errorf("tls: hosts applies to self_signed and issuer certificates")
+	case files && (o.CertFile == "" || o.KeyFile == ""):
+		return nil, fmt.Errorf("tls: cert_file and key_file must be set together")
+	case issuer && (o.IssuerCertFile == "" || o.IssuerKeyFile == ""):
+		return nil, fmt.Errorf("tls: issuer_cert_file and issuer_key_file must be set together")
+	case !files && !issuer && !o.SelfSigned:
+		return nil, fmt.Errorf("tls: listeners need cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
+	}
+	var cert tls.Certificate
+	var err error
+	if files {
+		cert, err = tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
+	} else {
+		cert, err = generated(o.IssuerCertFile, o.IssuerKeyFile, host, o.Hosts)
 	}
-	cert, err := tls.LoadX509KeyPair(o.CertFile, o.KeyFile)
 	if err != nil {
 		return nil, fmt.Errorf("tls: load keypair: %w", err)
 	}
@@ -64,11 +84,18 @@ func Server(o *config.TLSOptions) (*tls.Config, error) {
 // Client builds the *tls.Config for dialer plugins (tcp_chain/http_chain
 // sinks). host seeds ServerName when no override is set; Go verifies IP SANs
 // when host is an address, without its zone, which names a local interface
-// and no certificate carries. Returns (nil, nil) when disabled.
+// and no certificate carries. pin_sha256 replaces chain and name checks.
+// Returns (nil, nil) when disabled.
 func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if o == nil || !o.Enabled {
 		return nil, nil
 	}
+	switch {
+	case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0:
+		return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file and hosts apply to listeners")
+	case o.PinSHA256 != "" && (o.CAFile != "" || o.InsecureSkipVerify):
+		return nil, fmt.Errorf("tls: pin_sha256 replaces ca_file and insecure_skip_verify: set one")
+	}
 	mv, err := minVersion(o.MinVersion)
 	if err != nil {
 		return nil, err
@@ -81,6 +108,12 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 	if cfg.ServerName == "" {
 		cfg.ServerName, _, _ = strings.Cut(host, "%")
 	}
+	if o.PinSHA256 != "" {
+		if cfg.VerifyPeerCertificate, err = verifyPins(o.PinSHA256); err != nil {
+			return nil, err
+		}
+		cfg.InsecureSkipVerify = true // the pin check above replaces it
+	}
 	if o.CAFile != "" {
 		pool, err := loadPool(o.CAFile)
 		if err != nil {
@@ -149,13 +182,22 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list
 	if o == nil || !o.Enabled {
 		return
 	}
+	if key, err := processKey(); listener && o.SelfSigned && err == nil {
+		// Without a CA, a dialer verifies this listener by its pin only
+		l.Warn("msg", "Serving a self-signed certificate; it changes when lw restarts",
+			"component", component,
+			"instance_id", id,
+			"pin_sha256", pinOfKey(key),
+			"hint", "dialers set tls.pin_sha256; for a stable identity use issuer_cert_file or cert_file")
+	}
 	var warnings []string
 	if !listener && o.InsecureSkipVerify {
 		warnings = append(warnings, "tls.insecure_skip_verify disables server verification: any peer can impersonate the server")
 	}
-	files := []struct{ key, path string }{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}}
+	type file struct{ key, path string }
+	files := []file{{"cert_file", o.CertFile}, {"ca_file", o.CAFile}}
 	if listener {
-		files[1] = struct{ key, path string }{"client_ca_file", o.ClientCAFile}
+		files = []file{{"cert_file", o.CertFile}, {"client_ca_file", o.ClientCAFile}, {"issuer_cert_file", o.IssuerCertFile}}
 	}
 	now := time.Now()
 	for _, f := range files {
@@ -165,8 +207,10 @@ func LogWarnings(l *log.Logger, component, id string, o *config.TLSOptions, list
 			}
 		}
 	}
-	if w := SecretFileWarning("tls.key_file", o.KeyFile); w != "" {
-		warnings = append(warnings, w)
+	for _, key := range []file{{"tls.key_file", o.KeyFile}, {"tls.issuer_key_file", o.IssuerKeyFile}} {
+		if w := SecretFileWarning(key.key, key.path); w != "" {
+			warnings = append(warnings, w)
+		}
 	}
 	for _, w := range warnings {
 		l.Warn("msg", w, "component", component, "instance_id", id)

internal/tlsx/tlsx_test.go

@@ -3,11 +3,15 @@ package tlsx
 import (
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"net"
+	"path/filepath"
+	"slices"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/lixenwraith/logwisp/internal/config"
+	"github.com/lixenwraith/logwisp/internal/testutil"
 )
 
 // A dialer verifies an IPv6 target by its address: the zone names a local
@@ -48,3 +52,104 @@ func TestExpiryWarning(t *testing.T) {
 		}
 	}
 }
+
+// A self-signed listener is verified by the pin of the process key, which a
+// reload's reissued certificate keeps; any other pin, or none, fails.
+func TestSelfSignedListenerIsVerifiedByItsPin(t *testing.T) {
+	o := &config.TLSOptions{Enabled: true, SelfSigned: true, Hosts: []string{"agg.example"}}
+	first, err := Server(o, "0.0.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	reloaded, err := Server(o, "0.0.0.0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf := reloaded.Certificates[0].Leaf
+	pin := PinSHA256(leaf)
+	if PinSHA256(first.Certificates[0].Leaf) != pin || !slices.Contains(leaf.DNSNames, "agg.example") ||
+		!slices.Contains(leaf.DNSNames, "localhost") || slices.ContainsFunc(leaf.IPAddresses, net.IP.IsUnspecified) {
+		t.Fatalf("pin %s, names %v %v", pin, leaf.DNSNames, leaf.IPAddresses)
+	}
+	other := "sha256//" + strings.Repeat("A", 43) + "="
+	for pins, want := range map[string]string{other + "; " + pin: "", other: "matches no tls.pin_sha256", "": "certificate"} {
+		client, err := Client(&config.TLSOptions{Enabled: true, PinSHA256: pins}, "agg.example")
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = testutil.Handshake(t, reloaded, client)
+		if want == "" && err != nil || want != "" && (err == nil || !strings.Contains(err.Error(), want)) {
+			t.Errorf("pins %q: %v, want %q", pins, err, want)
+		}
+	}
+}
+
+// An issuer-signed listener chains to the CA file, for its host, and expires
+// with its issuer; a certificate that is no CA cannot issue.
+func TestIssuedListenerCertificateChainsToTheIssuer(t *testing.T) {
+	dir := t.TempDir()
+	ca, key, err := NewCA("test CA", time.Hour)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyPEM, err := EncodeKey(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	caFile, keyFile := filepath.Join(dir, "ca.crt"), filepath.Join(dir, "ca.key")
+	testutil.WriteFile(t, caFile, string(EncodeCert(ca)))
+	testutil.WriteFile(t, keyFile, string(keyPEM))
+	server, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: caFile, IssuerKeyFile: keyFile}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if leaf := server.Certificates[0].Leaf; !leaf.NotAfter.Equal(ca.NotAfter) {
+		t.Fatalf("leaf expires %s, issuer %s", leaf.NotAfter, ca.NotAfter)
+	}
+	client, err := Client(&config.TLSOptions{Enabled: true, CAFile: caFile}, "127.0.0.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := testutil.Handshake(t, server, client); err != nil {
+		t.Fatalf("issued chain: %v", err)
+	}
+	leafFile := filepath.Join(dir, "leaf.crt")
+	testutil.WriteFile(t, leafFile, string(EncodeCert(server.Certificates[0].Leaf)))
+	if _, err := Server(&config.TLSOptions{Enabled: true, IssuerCertFile: leafFile, IssuerKeyFile: keyFile}, ""); err == nil {
+		t.Fatal("a leaf issued certificates")
+	}
+}
+
+// Each role takes one source of trust and one certificate source; keys of the
+// other role fail instead of doing nothing.
+func TestTLSOptionConflicts(t *testing.T) {
+	pin := "sha256//" + strings.Repeat("A", 43) + "="
+	for _, c := range []struct {
+		o      config.TLSOptions
+		server bool
+		want   string
+	}{
+		{config.TLSOptions{SelfSigned: true, CertFile: "c", KeyFile: "k"}, true, "set one of"},
+		{config.TLSOptions{SelfSigned: true, IssuerCertFile: "c"}, true, "set one of"},
+		{config.TLSOptions{IssuerCertFile: "c"}, true, "must be set together"},
+		{config.TLSOptions{CertFile: "c"}, true, "must be set together"},
+		{config.TLSOptions{Hosts: []string{"a"}, CertFile: "c", KeyFile: "k"}, true, "hosts applies to"},
+		{config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "applies to dialers"},
+		{config.TLSOptions{}, true, "listeners need"},
+		{config.TLSOptions{SelfSigned: true}, false, "apply to listeners"},
+		{config.TLSOptions{PinSHA256: pin, CAFile: "ca"}, false, "set one"},
+		{config.TLSOptions{PinSHA256: pin, InsecureSkipVerify: true}, false, "set one"},
+		{config.TLSOptions{PinSHA256: "sha256//short"}, false, "want sha256//BASE64"},
+	} {
+		c.o.Enabled = true
+		var err error
+		if c.server {
+			_, err = Server(&c.o, "")
+		} else {
+			_, err = Client(&c.o, "host")
+		}
+		if err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%+v: %v, want %q", c.o, err, c.want)
+		}
+	}
+}