v0.31.0 refinements

internal/config/config.go

@@ -103,8 +103,8 @@ const (
 
 // RateLimitConfig defines the configuration for pipeline-level rate limiting
 type RateLimitConfig struct {
-	Rate              float64 `toml:"rate" lw:"min=0,zero=off" help:"entries a second"`
-	Burst             float64 `toml:"burst" lw:"min=0,zero=the rate" help:"entries at once"`
+	EntriesPerSecond  float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second the limit lets through"`
+	BurstEntries      float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries let through at once before the rate applies"`
 	Policy            string  `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop entries over the rate, or pass them"`
 	MaxEntrySizeBytes int64   `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries"`
 }
@@ -262,8 +262,8 @@ type HTTPSinkOptions struct {
 	WriteTimeoutMS   int64        `toml:"write_timeout_ms" lw:"min=0,zero=none" help:"milliseconds an event write may take"`
 	MaxConnections   int64        `toml:"max_connections" lw:"min=0,zero=unlimited" help:"clients at once"`
 	ReplayLines      int64        `toml:"replay_lines" lw:"min=0,zero=none" help:"recent entries a new stream gets first"`
-	LoginPage        bool         `toml:"login_page" help:"serve the login page; scram behind auth.trusted_proxies"`
-	ViewerPage       bool         `toml:"viewer_page" help:"serve the viewer to users logged in; needs login_page"`
+	LoginPage        bool         `toml:"login_page" help:"scram behind auth.trusted_proxies only: a login page at /auth/login"`
+	ViewerPage       bool         `toml:"viewer_page" help:"scram behind auth.trusted_proxies only: the viewer after a login; without scram it is always at /"`
 	Auth             *AuthOptions `toml:"auth"`
 	ACL              *ACLOptions  `toml:"acl"`
 }
@@ -307,9 +307,9 @@ type HTTPChainSinkOptions struct {
 
 // AuthOptions selects how a network plugin authenticates its peer. It sits
 // beside `tls`: TLS answers "is this channel private", auth answers "may this
-// peer do this". Listeners verify (mtls: certificate identity; scram: password
-// via credentials_file); dialers prove themselves (scram) or pin the server
-// (mtls). AuthOptions.Check holds the rules of each side.
+// peer do this". Listeners verify (mtls: certificate identity; scram: the
+// credentials_file's users, or one username and password_file); dialers prove
+// themselves (scram) or pin the server (mtls). Check holds each side's rules.
 type AuthOptions struct {
 	Type            string   `toml:"type" default:"none" lw:"enum=none|mtls|scram" help:"how peers authenticate"`
 	Identity        string   `toml:"identity" lw:"enum=cn|san_dns|san_uri|san_email,zero=cn under mtls; no binding under scram" help:"certificate field naming the peer; under scram it must equal the user"`
@@ -321,8 +321,8 @@ type AuthOptions struct {
 	// leaves room for renewal ahead of expiry
 	TokenLifetimeMS int64    `toml:"token_lifetime_ms" lw:"listener,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
 	TrustedProxies  []string `toml:"trusted_proxies" lw:"listener,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"`
-	Username        string   `toml:"username" lw:"dialer" help:"scram: the user this dialer logs in as"`
-	PasswordFile    string   `toml:"password_file" lw:"dialer,hint=file" help:"scram: file holding the user's password"`
+	Username        string   `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"`
+	PasswordFile    string   `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"`
 }
 
 // ACLOptions is a listener's address rules, entries being addresses or CIDRs:

internal/config/loader_test.go

@@ -340,7 +340,7 @@ func TestFileStringIsOneListEntry(t *testing.T) {
 // its own; other sinks do not take the key.
 func TestTopLevelColorIsEachConsoleSinksDefault(t *testing.T) {
 	isolateConfig(t)
-	m, err := Load(Args{Specs: specs("sink", "console", "sink", "console,color=never", "sink", "null"), Overrides: []string{"--color=always"}})
+	m, err := Load(Args{Specs: specs("sink", "console", "sink", "console:color=never", "sink", "null"), Overrides: []string{"--color=always"}})
 	if err != nil {
 		t.Fatal(err)
 	}

internal/config/preset.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 )
 
-// Preset is a pipeline built from a few keys: --preset NAME,key=value starts
+// Preset is a pipeline built from a few keys: --preset NAME:key=value starts
 // a pipeline with it. Later flags add sources, sinks and filters, or replace
 // its format, rate limit and heartbeat.
 type Preset struct {
@@ -64,6 +64,10 @@ var (
 		{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"},
 	}
 	transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
+	userParams     = []PresetParam{
+		{Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"},
+		{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"},
+	}
 )
 
 var presets = []Preset{
@@ -88,8 +92,9 @@ var presets = []Preset{
 			{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
 			{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
 			{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
+		}, userParams, []PresetParam{
 			{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
-			{Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"},
+			{Name: "viewer", Default: "false", Help: "true: the login page and viewer, behind proxy; without user or users the viewer is always on"},
 		}, aclParams, tlsParams),
 		func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
 			sources, err := pathOrStdin(v, isDir)
@@ -101,14 +106,18 @@ var presets = []Preset{
 			if err != nil {
 				return err
 			}
-			if v["users"] != "" {
-				sink["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
+			auth, err := scramAuth(v)
+			switch {
+			case err != nil:
+				return err
+			case auth != nil && v["tls"] == "off" && v["proxy"] == "":
+				return fmt.Errorf("user and users need tls (self, issuer or files), or proxy")
+			case auth != nil:
+				sink["auth"] = auth
+			case v["proxy"] != "":
+				return fmt.Errorf("proxy needs user or users: browsers log in with SCRAM")
 			}
 			if v["proxy"] != "" {
-				auth, ok := sink["auth"].(map[string]any)
-				if !ok {
-					return fmt.Errorf("proxy needs users: browsers log in with SCRAM")
-				}
 				auth["trusted_proxies"] = list(v["proxy"])
 			}
 			switch v["viewer"] {
@@ -127,18 +136,17 @@ var presets = []Preset{
 			return nil
 		}},
 	{"edge", "Forward files or standard input to an aggregator over TLS, authenticated",
-		[]PresetParam{pathParam, fromParam,
+		slices.Concat([]PresetParam{pathParam, fromParam,
 			{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"},
 			transportParam,
 			{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"},
 			{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
 			{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
-			{Name: "user", Help: "SCRAM user (lw auth add-user on the aggregator)"},
-			{Name: "password_file", Help: "file holding the SCRAM password"},
+		}, userParams, []PresetParam{
 			{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"},
 			{Name: "key", Help: "client key file"},
 			{Name: "node", Help: "origin label; default: this host's name"},
-		},
+		}),
 		func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
 			sources, err := pathOrStdin(v, isDir)
 			if err != nil {
@@ -159,16 +167,15 @@ var presets = []Preset{
 					tls[option] = v[key]
 				}
 			}
-			var auth map[string]any
+			auth, err := scramAuth(v)
 			switch {
-			case v["user"] != "" && v["password_file"] != "":
-				auth = map[string]any{"type": "scram", "username": v["user"], "password_file": v["password_file"]}
-			case v["user"] != "" || v["password_file"] != "":
-				return fmt.Errorf("user and password_file go together")
+			case err != nil:
+				return err
+			case auth != nil:
 			case v["cert"] != "" && v["key"] != "":
 				auth = map[string]any{"type": "mtls"}
 			default:
-				return fmt.Errorf("edge needs user and password_file, or cert and key: it never sends unauthenticated")
+				return fmt.Errorf("edge needs user, or cert and key: it never sends unauthenticated")
 			}
 			sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth}
 			if v["node"] != "" {
@@ -183,6 +190,7 @@ var presets = []Preset{
 			transportParam,
 			{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
 			{Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"},
+		}, userParams, []PresetParam{
 			{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
 			{Name: "out", Help: "directory for the received entries; default: standard output"},
 			formatParam("json"),
@@ -203,13 +211,16 @@ var presets = []Preset{
 				tls := src["tls"].(map[string]any)
 				tls["client_auth"], tls["client_ca_file"] = true, v["client_ca"]
 			}
+			auth, err := scramAuth(v)
 			switch {
-			case v["users"] != "":
-				src["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
+			case err != nil:
+				return err
+			case auth != nil:
+				src["auth"] = auth
 			case v["client_ca"] != "":
 				src["auth"] = map[string]any{"type": "mtls"}
 			default:
-				return fmt.Errorf("aggregator needs users or client_ca: it never receives unauthenticated")
+				return fmt.Errorf("aggregator needs user, users or client_ca: it never receives unauthenticated")
 			}
 			p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src})
 			p.PluginSinks = pipeDefault("").PluginSinks
@@ -356,6 +367,22 @@ func listener(v map[string]string, def string) (map[string]any, error) {
 	return opts, nil
 }
 
+// scramAuth is the scram table of user, whose password the terminal asks for
+// unless password_file names it, or of a listener's users file; nil without
+func scramAuth(v map[string]string) (map[string]any, error) {
+	switch {
+	case v["user"] != "" && v["users"] != "":
+		return nil, fmt.Errorf("user or users, not both")
+	case v["user"] != "":
+		return map[string]any{"type": "scram", "username": v["user"], "password_file": cmp.Or(v["password_file"], "/dev/tty")}, nil
+	case v["password_file"] != "":
+		return nil, fmt.Errorf("password_file needs user")
+	case v["users"] != "":
+		return map[string]any{"type": "scram", "credentials_file": v["users"]}, nil
+	}
+	return nil, nil
+}
+
 func splitAddr(key, addr string) (string, int64, error) {
 	host, p, err := net.SplitHostPort(addr)
 	port, perr := strconv.ParseUint(p, 10, 16)

internal/config/preset_test.go

@@ -11,7 +11,7 @@ import (
 // sinks add to its own, a later format replaces its own.
 func TestPresetStartsAndNamesItsPipeline(t *testing.T) {
 	isolateConfig(t)
-	got := loadPipelines(t, "preset", "tail,path=/var/log/app.log", "sink", "null", "format", "json",
+	got := loadPipelines(t, "preset", "tail:path=/var/log/app.log", "sink", "null", "format", "json",
 		"pipeline", "mine", "preset", "pipe")
 	want := []PipelineConfig{{
 		Name:          "tail",
@@ -40,21 +40,24 @@ func TestPresetStartsAndNamesItsPipeline(t *testing.T) {
 func TestPresetKeysAreChecked(t *testing.T) {
 	isolateConfig(t)
 	for spec, want := range map[string]string{
-		"nope":                         `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`,
-		"tail,path=x,colour=1":         `preset tail has no key "colour" (valid: path, from, format)`,
-		"tail":                         "preset tail needs path",
-		"tail,path=x,format.type=json": "preset keys do not nest",
-		"tail,path=x,path=y":           `key "path" takes one value`,
-		"serve,listen=8080":            `listen "8080": want HOST:PORT`,
-		"serve,tls=self,cert=c":        "cert does not apply to tls=self",
-		"serve,tls=files,cert=c":       "tls=files needs key",
-		"serve,viewer=yes":             `viewer "yes"`,
-		"serve,proxy=127.0.0.1":        "proxy needs users",
-		"serve,viewer=true":            "viewer=true needs proxy",
-		"edge,to=agg:9000":             "it never sends unauthenticated",
-		"edge,to=agg:9000,user=u":      "user and password_file go together",
-		"edge,to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
-		"aggregator,users=u,tls=off":                            "the aggregator always serves TLS",
+		"nope":                             `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`,
+		"tail:path=x,colour=1":             `preset tail has no key "colour" (valid: path, from, format)`,
+		"tail":                             "preset tail needs path",
+		"tail:path=x,format.type=json":     "preset keys do not nest",
+		"tail:path=x,path=y":               `key "path" takes one value`,
+		"serve:listen=8080":                `listen "8080": want HOST:PORT`,
+		"serve:tls=self,cert=c":            "cert does not apply to tls=self",
+		"serve:tls=files,cert=c":           "tls=files needs key",
+		"serve:viewer=yes":                 `viewer "yes"`,
+		"serve:proxy=127.0.0.1":            "proxy needs user or users",
+		"serve:viewer=true":                "viewer=true needs proxy",
+		"serve:user=u":                     "user and users need tls (self, issuer or files), or proxy",
+		"serve:tls=self,user=u,users=f":    "user or users, not both",
+		"edge:to=agg:9000":                 "it never sends unauthenticated",
+		"edge:to=agg:9000,password_file=p": "password_file needs user",
+		"edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
+		"aggregator:users=u,tls=off":                            "the aggregator always serves TLS",
+		"aggregator:user=u,users=f":                             "user or users, not both",
 		"aggregator":                                            "it never receives unauthenticated",
 	} {
 		_, err := Load(Args{Specs: specs("preset", spec)})
@@ -64,6 +67,17 @@ func TestPresetKeysAreChecked(t *testing.T) {
 	}
 }
 
+// A preset user without password_file is asked for on the terminal
+func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
+	p, err := ExpandPreset("edge", map[string]string{"to": "agg:9000", "user": "u"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if auth := p.PluginSinks[0].Config["auth"].(map[string]any); auth["password_file"] != "/dev/tty" {
+		t.Fatalf("auth = %v, want password_file /dev/tty", auth)
+	}
+}
+
 // A preset path is what isDir answers, a directory named with brackets too;
 // without isDir, off the target host, it is a directory when it ends in '/',
 // a pattern when it holds a glob, and otherwise ErrPathKind asks

internal/config/schema_test.go

@@ -117,8 +117,8 @@ func TestValidateConfigNamesThePath(t *testing.T) {
 			c.Pipelines[0].PluginSinks[0] = PluginSinkConfig{ID: "up", Type: "tcp_chain", Config: map[string]any{"host": "h", "port": int64(1),
 				"tls": map[string]any{"enabled": true, "insecure_skip_verify": true}, "auth": map[string]any{"type": "mtls"}}}
 		},
-		"pipelines[0].flow.rate_limit.rate: must be a finite number": func(c *Config) {
-			c.Pipelines[0].Flow.RateLimit = &RateLimitConfig{Rate: math.Inf(1)}
+		"pipelines[0].flow.rate_limit.entries_per_second: must be a finite number": func(c *Config) {
+			c.Pipelines[0].Flow.RateLimit = &RateLimitConfig{EntriesPerSecond: math.Inf(1)}
 		},
 		"color: must be one of auto, always, never, got \"\"": func(c *Config) {
 			c.Color = ""

internal/config/spec.go

@@ -93,10 +93,9 @@ var specKinds = []specKind{
 		Fill(&format)
 		return []string{typedSpec(&format)}
 	}},
-	// Naming a stage turns it on: the file defaults (pass, disabled) would not.
 	{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
-		p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
-		return Scan(opts, p.Flow.RateLimit)
+		p.Flow.RateLimit = &RateLimitConfig{}
+		return Scan(StageOn("rate_limit", opts), p.Flow.RateLimit)
 	}, func(p *PipelineConfig) []string {
 		if p.Flow.RateLimit == nil {
 			return nil
@@ -106,8 +105,8 @@ var specKinds = []specKind{
 		return []string{formatSpec("", Values(&rate))}
 	}},
 	{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
-		p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
-		return Scan(opts, p.Flow.Heartbeat)
+		p.Flow.Heartbeat = &HeartbeatConfig{}
+		return Scan(StageOn("heartbeat", opts), p.Flow.Heartbeat)
 	}, func(p *PipelineConfig) []string {
 		if p.Flow.Heartbeat == nil {
 			return nil
@@ -127,6 +126,18 @@ type pipelineSpec struct {
 	value string
 }
 
+// stageOn are the values naming a flow stage sets, in a spec or an engine
+// edit: the file's defaults (pass, disabled) would leave it doing nothing
+var stageOn = map[string]map[string]any{"rate_limit": {"policy": "drop"}, "heartbeat": {"enabled": true}}
+
+// StageOn returns opts over the values that turn a flow stage on
+func StageOn(stage string, opts map[string]any) map[string]any {
+	on := map[string]any{}
+	maps.Copy(on, stageOn[stage])
+	maps.Copy(on, opts)
+	return on
+}
+
 // Spec is one pipeline flag: its name without dashes, and its value.
 type Spec struct{ Flag, Value string }
 
@@ -282,13 +293,10 @@ func typedSpec(stage any) string {
 	return formatSpec(typ, m)
 }
 
-// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a
-// repeated key
+// formatSpec writes TYPE[:key=value,...] or key=value,...: tables as dotted
+// keys, lists as a repeated key
 func formatSpec(typ string, tables ...map[string]any) string {
 	var parts []string
-	if typ != "" {
-		parts = append(parts, escape(typ))
-	}
 	var add func(prefix string, m map[string]any)
 	add = func(prefix string, m map[string]any) {
 		for _, k := range slices.Sorted(maps.Keys(m)) {
@@ -307,7 +315,11 @@ func formatSpec(typ string, tables ...map[string]any) string {
 	for _, m := range tables {
 		add("", m)
 	}
-	return strings.Join(parts, ",")
+	opts := strings.Join(parts, ",")
+	if typ == "" || opts == "" {
+		return typ + opts
+	}
+	return typ + ":" + opts
 }
 
 func text(v any) string {
@@ -365,16 +377,20 @@ func useStdio(p *PipelineConfig) {
 	}
 }
 
-// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
-// a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
+// parseSpec reads TYPE[:key=value,...] or key=value,...: TYPE ends at the
+// first ':', dotted keys nest, a repeated key makes a list, and a backslash
+// escapes ',', '=' and '\'. Errors name the part.
 func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {
 	rest, more := spec, true
 	if typed {
-		typ, rest, more = cutUnescaped(spec, ',')
-		if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq {
+		typ, rest, _ = strings.Cut(spec, ":")
+		more = rest != ""
+		if head, _, comma := cutUnescaped(typ, ','); comma && !strings.Contains(head, "=") {
+			return "", nil, fmt.Errorf("%s: TYPE ends at ':', as in %s:%s", spec, head, spec[len(head)+1:])
+		}
+		if typ == "" || strings.ContainsAny(typ, "=,\\") {
 			return "", nil, fmt.Errorf("%s: missing TYPE", spec)
 		}
-		typ = unescape(typ)
 	}
 	opts = map[string]any{}
 	for more {
@@ -390,7 +406,7 @@ func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err er
 			err = SetOption(opts, key, unescape(value))
 		}
 		if err != nil {
-			return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
+			return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+":"+part, ":"), err)
 		}
 	}
 	return typ, opts, nil

internal/config/spec_test.go

@@ -36,13 +36,13 @@ func loadPipelines(t *testing.T, pairs ...string) []PipelineConfig {
 func TestPipelineSpecGrammar(t *testing.T) {
 	isolateConfig(t)
 	got := loadPipelines(t,
-		"source", `file,directory=/var/log/a\,b,pattern=*.log`,
-		"source", `file,id=app,directory=x\=y`,
-		"source", `file,directory=c:\\logs\\`,
-		"sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.enabled=true,tls.cert_file=c,tls.key_file=k",
-		"filter", "include,patterns=ERROR,patterns=WARN",
-		"filter", `exclude,patterns=\d{1\,3}`,
-		"format", `txt,timestamp_format=Jan 2\, 2006`,
+		"source", `file:directory=/var/log/a\,b,pattern=*.log`,
+		"source", `file:id=app,directory=x\=y`,
+		"source", `file:directory=c:\\logs\\`,
+		"sink", "http:port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.enabled=true,tls.cert_file=c,tls.key_file=k",
+		"filter", "include:patterns=ERROR,patterns=WARN",
+		"filter", `exclude:patterns=\d{1\,3}`,
+		"format", `txt:timestamp_format=Jan 2\, 2006`,
 		"pipeline", "b", "source", "null", "sink", "null", "sink", "null", "heartbeat", "interval_ms=1000")
 	want := []PipelineConfig{{
 		Name: "cli",
@@ -76,13 +76,14 @@ func TestPipelineSpecGrammar(t *testing.T) {
 		t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
 	}
 	for want, pairs := range map[string][]string{
-		`--sink http,port: missing "="`:                                  {"sink", "http,port"},
-		`--source directory=/x: missing TYPE`:                            {"source", "directory=/x"},
-		`--source file,type=x: TYPE already sets "type"`:                 {"source", "file,type=x"},
-		`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http,tls=1,tls.cert_file=c"},
-		`--sink requires a value`:                                        {"sink", ""},
-		`--format txt: pipeline "cli" already has one`:                   {"format", "json", "format", "txt"},
-		`--rate-limit rate=1,polcy=drop: unknown key "polcy"`:            {"rate-limit", "rate=1,polcy=drop"},
+		`--sink http:port: missing "="`:                                         {"sink", "http:port"},
+		`--source file,directory=/x: TYPE ends at ':', as in file:directory=/x`: {"source", "file,directory=/x"},
+		`--source directory=/x: missing TYPE`:                                   {"source", "directory=/x"},
+		`--source file:type=x: TYPE already sets "type"`:                        {"source", "file:type=x"},
+		`--sink http:tls.cert_file=c: "tls" is both a value and a table`:        {"sink", "http:tls=1,tls.cert_file=c"},
+		`--sink requires a value`:                                               {"sink", ""},
+		`--format txt: pipeline "cli" already has one`:                          {"format", "json", "format", "txt"},
+		`--rate-limit entries_per_second=1,polcy=drop: unknown key "polcy"`:     {"rate-limit", "entries_per_second=1,polcy=drop"},
 	} {
 		if _, err := Load(Args{Specs: specs(pairs...)}); err == nil || err.Error() != want {
 			t.Errorf("%q: err = %v, want %s", pairs, err, want)
@@ -95,7 +96,7 @@ func TestPipelineSpecGrammar(t *testing.T) {
 func TestSpecValueIsOneListEntry(t *testing.T) {
 	isolateConfig(t)
 	sink := loadPipelines(t, "source", "null", "sink",
-		`tcp,port=9000,tls.enabled=true,tls.self_signed=true,tls.client_auth=true,tls.client_ca_file=ca,`+
+		`tcp:port=9000,tls.enabled=true,tls.self_signed=true,tls.client_auth=true,tls.client_ca_file=ca,`+
 			`auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0]
 	var opts TCPSinkOptions
 	if err := Scan(sink.Config, &opts); err != nil {
@@ -114,12 +115,12 @@ func TestEnvironmentPipeline(t *testing.T) {
 	for name, value := range map[string]string{
 		"LOGWISP_PIPELINE":      "edge",
 		"LOGWISP_SOURCE":        "random",
-		"LOGWISP_SOURCE_10":     "null,id=ten",
-		"LOGWISP_SOURCE_2":      "null,id=two",
+		"LOGWISP_SOURCE_10":     "null:id=ten",
+		"LOGWISP_SOURCE_2":      "null:id=two",
 		"LOGWISP_SINK_1":        "console",
 		"LOGWISP_FILTER":        "",
 		"LOGWISP_FORMAT_1":      "raw",
-		"LOGWISP_RATE_LIMIT":    "rate=10",
+		"LOGWISP_RATE_LIMIT":    "entries_per_second=10",
 		"LOGWISP_LOGGING_LEVEL": "debug",
 	} {
 		t.Setenv(name, value)
@@ -136,7 +137,7 @@ func TestEnvironmentPipeline(t *testing.T) {
 	p := cfg.Pipelines
 	if len(p) != 1 || p[0].Name != "edge" || len(p[0].PluginSources) != 3 || p[0].PluginSources[1].ID != "two" ||
 		p[0].PluginSources[2].ID != "ten" || len(p[0].PluginSinks) != 1 || p[0].Flow.Filters != nil || p[0].Flow.Format != nil ||
-		*p[0].Flow.RateLimit != (RateLimitConfig{Rate: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" {
+		*p[0].Flow.RateLimit != (RateLimitConfig{EntriesPerSecond: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" {
 		t.Fatalf("environment pipeline: %+v %+v", p, cfg.Logging)
 	}
 }
@@ -174,7 +175,7 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
 // stdout, as a filter does; two pipelines cannot both read stdin.
 func TestSpecPipelinesDefaultToStdio(t *testing.T) {
 	isolateConfig(t)
-	got := loadPipelines(t, "filter", "include,patterns=ERROR")
+	got := loadPipelines(t, "filter", "include:patterns=ERROR")
 	if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" ||
 		len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" {
 		t.Fatalf("pipeline: %+v", p)
@@ -190,7 +191,7 @@ func TestSpecPipelinesDefaultToStdio(t *testing.T) {
 // removal fails the reload instead of keeping the removed file's values.
 func TestReloadKeepsSpecPipelines(t *testing.T) {
 	isolateConfig(t)
-	m, err := Load(Args{Specs: specs("source", "random,special=true", "sink", "null")})
+	m, err := Load(Args{Specs: specs("source", "random:special=true", "sink", "null")})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -217,11 +218,11 @@ func TestReloadKeepsSpecPipelines(t *testing.T) {
 // Written specs build the pipelines a file would: each flow stage with its
 // defaults, so the rate limit keeps its file policy and the heartbeat its off
 func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) {
-	p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{Rate: 5}, Heartbeat: &HeartbeatConfig{}},
+	p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5}, Heartbeat: &HeartbeatConfig{}},
 		PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}},
 		PluginSinks:   []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}}
 	got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil)
-	p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{Rate: 5, Policy: "pass"},
+	p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"},
 		Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}}
 	if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) {
 		t.Fatalf("got %+v %v\nwant %+v", got, err, p)

internal/config/validate.go

@@ -296,7 +296,7 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
 		}
 		return nil
 	case len(o.Allow) > 0 || len(o.AllowPatterns) > 0:
-		return fmt.Errorf("auth: allow and allow_patterns apply only to type %q; the credentials file is the allow list", "mtls")
+		return fmt.Errorf("auth: allow and allow_patterns apply only to type %q; the users are the allow list", "mtls")
 	case side == Dialer && (o.CredentialsFile != "" || o.TokenLifetimeMS != 0 || len(o.TrustedProxies) > 0):
 		return errors.New("auth: credentials_file, token_lifetime_ms and trusted_proxies apply only to listeners")
 	case side == Dialer && o.Identity != "":
@@ -305,10 +305,12 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
 		return fmt.Errorf("auth: type %q on a dialer requires username and password_file", "scram")
 	case side == Dialer:
 		return nil
-	case o.Username != "" || o.PasswordFile != "":
-		return errors.New("auth: username and password_file apply only to dialers")
-	case o.CredentialsFile == "":
-		return fmt.Errorf("auth: type %q requires credentials_file", "scram")
+	case o.CredentialsFile != "" && (o.Username != "" || o.PasswordFile != ""):
+		return errors.New("auth: credentials_file, or username and password_file: not both")
+	case (o.Username == "") != (o.PasswordFile == ""):
+		return errors.New("auth: username and password_file go together")
+	case o.CredentialsFile == "" && o.Username == "":
+		return fmt.Errorf("auth: type %q requires credentials_file, or username and password_file", "scram")
 	case o.TokenLifetimeMS > 0 && !http:
 		return errors.New("auth: token_lifetime_ms applies only to HTTP listeners")
 	case len(o.TrustedProxies) > 0 && (side != Listener || !http):
@@ -425,6 +427,10 @@ func and(keys []string) string {
 	return strings.Join(keys[:len(keys)-1], ", ") + " and " + keys[len(keys)-1]
 }
 
+// pagesBehind is why a login or viewer page needs a proxy: only there do
+// browsers log in, and elsewhere the viewer needs no page
+const pagesBehind = "only for scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls, the viewer is always at /"
+
 // Check keeps a stream and a status path ServeMux and a URL take literally,
 // apart, and off the authentication paths; a login page needs browsers that
 // log in, behind a proxy
@@ -442,10 +448,12 @@ func (o *HTTPSinkOptions) Check() error {
 	switch {
 	case o.StreamPath == o.StatusPath:
 		return fmt.Errorf("stream_path and status_path must differ")
-	case (o.LoginPage || o.ViewerPage) && !behind:
-		return errors.New("login_page and viewer_page apply to scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls the viewer is always served")
+	case o.LoginPage && !behind:
+		return &KeyError{"login_page", errors.New(pagesBehind)}
+	case o.ViewerPage && !behind:
+		return &KeyError{"viewer_page", errors.New(pagesBehind)}
 	case o.ViewerPage && !o.LoginPage:
-		return errors.New("viewer_page needs login_page, where it sends a signed-out viewer")
+		return &KeyError{"viewer_page", errors.New("needs login_page, where it sends a signed-out viewer")}
 	}
 	return nil
 }