diff --git a/README.md b/README.md index 4f6d6bf..95bf534 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ streams, or downstream LogWisp nodes. to stdout line for line and exits at the end of input; flags add filters, formats, sources and sinks - **Presets**: `tail`, `serve`, `edge` and `aggregator` build the common - pipelines from a few keys (`lw --preset tail,path=/var/log/app`); + pipelines from a few keys (`lw --preset tail:path=/var/log/app`); `lw --dump` turns any command line into a configuration file - **Independent pipelines**, each `sources → flow → sinks`, running concurrently in one process @@ -132,18 +132,18 @@ one without `--sink` writes stdout: ```bash ./bin/lw < app.log > copy.log -tail -F app.log | ./bin/lw --filter include,patterns=ERROR,patterns=WARN +tail -F app.log | ./bin/lw --filter include:patterns=ERROR,patterns=WARN # stdin as a live SSE stream, with a browser viewer at http://127.0.0.1:8080/; # the http sink binds 0.0.0.0 unless told -journalctl -f | ./bin/lw --sink http,host=127.0.0.1,port=8080 +journalctl -f | ./bin/lw --sink http:host=127.0.0.1,port=8080 # tail a directory instead of stdin -./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \ - --format json,sanitizer_policy=json > all.json +./bin/lw --source 'file:directory=/var/log/myapp,pattern=*.log' \ + --format json:sanitizer_policy=json > all.json # the same with a preset; on a terminal the level names are in color -./bin/lw --preset tail,path=/var/log/myapp +./bin/lw --preset tail:path=/var/log/myapp ``` As a service, a configuration file holds the pipelines: diff --git a/cmd/lw/auth.go b/cmd/lw/auth.go index 27bf9de..c1fd690 100644 --- a/cmd/lw/auth.go +++ b/cmd/lw/auth.go @@ -35,12 +35,12 @@ var authCommands = []subcommand{ {"remove-user", "--credentials FILE -u NAME", "Remove a user from a credentials file", []string{"credentials", "user"}, defineRemoveUser}, - {"token", "--url https://HOST:PORT -u NAME --password-file FILE [--unbound] [TLS flags]", + {"token", "--url https://HOST:PORT -u NAME [--password-file FILE] [--unbound] [TLS flags]", "Log in to an http sink or http_chain source and print a bearer token", - []string{"url", "user", "password-file"}, defineToken}, - {"stream", "--addr HOST:PORT -u NAME --password-file FILE [TLS flags]", + []string{"url", "user"}, defineToken}, + {"stream", "--addr HOST:PORT -u NAME [--password-file FILE] [TLS flags]", "Log in to a tcp sink and copy its stream to stdout until interrupted", - []string{"addr", "user", "password-file"}, defineStream}, + []string{"addr", "user"}, defineStream}, } func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { @@ -62,7 +62,7 @@ func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { var password string if *passwordFile != "" && !*generate { - password, err = authz.ReadPassword(*passwordFile) + password, err = authz.ReadPassword(*passwordFile, *user, false) switch { case err == nil && len(password) < 8: return fmt.Errorf("the password in %s is shorter than 8 bytes", *passwordFile) @@ -155,7 +155,7 @@ func defineRemoveUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT or https://[IPV6]:PORT (with --unbound, a proxy's mount URL may carry a path)") user := fs.String("user", "", "user `name`") - passwordFile := fs.String("password-file", "", "`file` holding the password") + passwordFile := fs.String("password-file", "/dev/tty", "`file` holding the password; a terminal asks") unbound := fs.Bool("unbound", false, "log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)") tlsOpts := tlsFlags(fs) return func(stdout, _ io.Writer) error { @@ -205,7 +205,7 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error { addr := fs.String("addr", "", "tcp sink address, `HOST:PORT` or [IPV6]:PORT") user := fs.String("user", "", "user `name`") - passwordFile := fs.String("password-file", "", "`file` holding the password") + passwordFile := fs.String("password-file", "/dev/tty", "`file` holding the password; a terminal asks") tlsOpts := tlsFlags(fs) return func(stdout, _ io.Writer) error { host, _, err := net.SplitHostPort(*addr) diff --git a/cmd/lw/auth_test.go b/cmd/lw/auth_test.go index 0a0287d..babf510 100644 --- a/cmd/lw/auth_test.go +++ b/cmd/lw/auth_test.go @@ -63,7 +63,7 @@ func TestAddUserCreatesPrivateFilesWithAMatchingVerifier(t *testing.T) { if err != nil { t.Fatal(err) } - password, err := authz.ReadPassword(pass) + password, err := authz.ReadPassword(pass, "edge-01", false) if err != nil { t.Fatal(err) } diff --git a/cmd/lw/bootstrap.go b/cmd/lw/bootstrap.go index f0e20cd..27abf92 100644 --- a/cmd/lw/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -22,6 +22,7 @@ import ( _ "github.com/lixenwraith/logwisp/internal/sink/tcp" _ "github.com/lixenwraith/logwisp/internal/sink/tcpchain" + "github.com/lixenwraith/logwisp/internal/authz" "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/logwisp/internal/service" "github.com/lixenwraith/logwisp/internal/version" @@ -37,6 +38,7 @@ func bootstrapInitial(ctx context.Context, cfg *config.Config) (*service.Service if err != nil { return nil, nil, fmt.Errorf("failed to bootstrap service: %w", err) } + authz.EndStartup() // a reload reads no descriptor, pipe or terminal if err := svc.Start(); err != nil { svc.Shutdown() diff --git a/cmd/lw/cli.go b/cmd/lw/cli.go index b46fdcd..0e5c3b8 100644 --- a/cmd/lw/cli.go +++ b/cmd/lw/cli.go @@ -40,7 +40,7 @@ var commands = []command{ {"tls", "A CA and certificates for TLS and mTLS", "Files are never overwritten; keys are written mode 0600.", tlsCommands}, {"preset", "Show the pipeline a preset makes, as TOML", - "lw --preset NAME,key=value runs one; flags here are its keys.", presetCommands()}, + "lw --preset NAME:key=value runs one; flags here are its keys.", presetCommands()}, {"config", "Place the annotated default configuration", "Files are never overwritten; lw --dump prints the effective configuration.", configCommands}, } diff --git a/cmd/lw/cli_test.go b/cmd/lw/cli_test.go index d03969a..410d92a 100644 --- a/cmd/lw/cli_test.go +++ b/cmd/lw/cli_test.go @@ -31,9 +31,9 @@ func TestCommandLineGrammar(t *testing.T) { }{ {[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}}, {[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}}, - {[]string{"--source=null", "--sink", "http,port=1"}, - invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}}}}}, - {[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}}, + {[]string{"--source=null", "--sink", "http:port=1"}, + invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http:port=1"}}}}}, + {[]string{"--preset", "tail:path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail:path=x"}, {Flag: "pipeline", Value: "b"}}}}}, {[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}}, {[]string{"--quiet", "--"}, invocation{load: config.Args{Overrides: []string{"--quiet"}}}}, {[]string{"--quiet", "-h"}, invocation{help: true, load: config.Args{Overrides: []string{"--quiet"}}}}, @@ -41,9 +41,9 @@ func TestCommandLineGrammar(t *testing.T) { {[]string{"help"}, invocation{help: true}}, {[]string{"-T", "-c", "a.toml"}, invocation{on: map[string]bool{"tui": true}, load: config.Args{File: "a.toml"}}}, {[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}}, - {[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"}, + {[]string{"-q", "-t", "-V", "-p", "tail:path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"}, invocation{on: map[string]bool{"check": true, "version": true, "dump": true}, - load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}}, + load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail:path=x"}}, Overrides: []string{"--quiet", "--logging.level=debug", "--logging.file.retention_hours=-1"}}}}, } { got, err := parseCommandLine(c.argv) @@ -173,8 +173,8 @@ func TestDumpReadsBack(t *testing.T) { if err := os.WriteFile(empty, nil, 0o600); err != nil { t.Fatal(err) } - args := []string{"--color", "--preset", "serve,listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example", - "--filter", `exclude,patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"} + args := []string{"--color", "--preset", "serve:listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example", + "--filter", `exclude:patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"} cfg := loadTestConfig(t, empty, args...) var dump bytes.Buffer if code := dumpConfig(loadTestConfig(t, empty, args...), &dump); code != 0 { @@ -225,7 +225,7 @@ func TestEveryFormLoadsTheSamePipelines(t *testing.T) { written := filepath.Join(dir, "written.toml") if err := os.WriteFile(written, []byte(`[[pipelines]] name = "written" -flow.rate_limit.rate = 5 +flow.rate_limit.entries_per_second = 5 plugin_sources = [{id = "in", type = "null"}] plugin_sinks = [{id = "out", type = "http", config = {port = 15844, acl = {requests_per_second_per_client = 2}}}] `), 0o600); err != nil { @@ -321,11 +321,11 @@ func TestEveryPresetBuilds(t *testing.T) { } pin := "sha256//" + strings.Repeat("A", 43) + "=" samples := map[string]string{ - "pipe": "pipe,format=txt", - "tail": "tail,path=" + dir, - "serve": "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2", - "edge": "edge,to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass, - "aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1", + "pipe": "pipe:format=txt", + "tail": "tail:path=" + dir, + "serve": "serve:path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2", + "edge": "edge:to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass, + "aggregator": "aggregator:listen=127.0.0.1:15843,user=edge-01,password_file=" + pass + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1", } for _, p := range config.Presets() { sample, ok := samples[p.Name] diff --git a/cmd/lw/help.go b/cmd/lw/help.go index bc20390..a506cd0 100644 --- a/cmd/lw/help.go +++ b/cmd/lw/help.go @@ -81,7 +81,7 @@ key alone means true: chooses the console Pipelines (they replace the configuration file's pipelines): - -p, --preset NAME[,KEY=VALUE...] + -p, --preset NAME[:KEY=VALUE,...] Start a pipeline with a preset: pipe, tail, serve, edge, aggregator; lw preset NAME -h lists its keys, lw preset NAME prints it @@ -89,27 +89,32 @@ Pipelines (they replace the configuration file's pipelines): pipeline options after it configure; those before any --pipeline make one named after its --preset, or "cli" without one - --source SPEC TYPE[,KEY=VALUE...], repeatable - --sink SPEC e.g. http,host=0.0.0.0,port=8080, repeatable - --filter SPEC include|exclude,patterns=RE, repeatable - --format SPEC json|txt|raw[,KEY=VALUE...] - --rate-limit SPEC rate=N[,burst=N,policy=drop|pass] + --source SPEC TYPE[:KEY=VALUE,...], repeatable + --sink SPEC e.g. http:host=0.0.0.0,port=8080, repeatable + --filter SPEC include|exclude:patterns=RE, repeatable + --format SPEC json|txt|raw[:KEY=VALUE,...] + --rate-limit SPEC entries_per_second=N[,burst_entries=N, + policy=drop|pass] --heartbeat SPEC interval_ms=N[,include_stats=true,...] A pipeline without --source reads stdin, one without --sink writes stdout. - Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and - '\' escapes ',' '=' '\' in values. + TYPE ends at the first ':'. Keys nest with '.' (tls.cert_file=...), a + repeated key makes a list, and '\' escapes ',' '=' '\' in values. Examples: lw < app.log > copy.log Copy, line for line - tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN - journalctl -f | lw --sink http,host=127.0.0.1,port=8080 + tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN + journalctl -f | lw --sink http:host=127.0.0.1,port=8080 Serve stdin live; browse http://127.0.0.1:8080/ - lw --source file,directory=/var/log/app,pattern='*.log' --format txt - lw -p tail,path=/var/log/app Follow a directory's files - lw --preset serve,path=/var/log/app,tls=self,users=users.toml + lw --source file:directory=/var/log/app,pattern='*.log' --format txt + lw -p tail:path=/var/log/app Follow a directory's files + lw --preset serve:path=/var/log/app,tls=self,users=users.toml HTTPS stream, SCRAM logins - lw --preset edge,path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\ + lw --preset edge:path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\ password_file=edge-01.pass Forward to an aggregator + lw -p aggregator:user=pipe,format=raw A temporary pipe's end: Enter + at the prompt draws a password + cmd | lw -p edge:to=HOST:9000,pin=PIN,user=pipe + Its source, asked the password Environment: LOGWISP_ A configuration key, '.' as '_', uppercase: diff --git a/cmd/lw/reload_test.go b/cmd/lw/reload_test.go index eada4e0..c9daa3f 100644 --- a/cmd/lw/reload_test.go +++ b/cmd/lw/reload_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "github.com/lixenwraith/logwisp/internal/authz" "github.com/lixenwraith/logwisp/internal/plugin" "github.com/lixenwraith/logwisp/internal/session" "github.com/lixenwraith/logwisp/internal/testutil" @@ -60,6 +61,25 @@ func TestShippedConfigurationBuildsWithCheckedPluginDecoder(t *testing.T) { svc.Shutdown() // A later reload failure or process exit can close it again. } +// Once the first service is built a reload reads no descriptor, pipe or +// terminal, which a console source may be reading +func TestStartupEndsWithTheFirstService(t *testing.T) { + testLogger(t) + path := filepath.Join(t.TempDir(), "empty.toml") + testutil.WriteFile(t, path, "") + svc, cancel, err := bootstrapInitial(context.Background(), loadTestConfig(t, path, "--source", "null", "--sink", "null")) + if err != nil { + t.Fatal(err) + } + defer svc.Shutdown() + if cancel != nil { + defer cancel() + } + if _, err := authz.ReadPassword(os.DevNull, "late", false); err == nil || !strings.Contains(err.Error(), "only at startup") { + t.Fatalf("a device after the first service: %v", err) + } +} + func TestInvalidReloadLeavesStatusReporterRunning(t *testing.T) { testLogger(t) path := filepath.Join(t.TempDir(), "config.toml") @@ -124,8 +144,8 @@ func TestCheckBuildsWithoutStarting(t *testing.T) { dir := t.TempDir() path, out := filepath.Join(dir, "empty.toml"), filepath.Join(dir, "out") testutil.WriteFile(t, path, "") - if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15862", - "--sink", "file,name=check,directory="+out)); code != 0 { + if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http:host=127.0.0.1,port=15862", + "--sink", "file:name=check,directory="+out)); code != 0 { t.Fatalf("valid configuration: exit %d", code) } ln, err := net.Listen("tcp4", "127.0.0.1:15862") @@ -138,7 +158,7 @@ func TestCheckBuildsWithoutStarting(t *testing.T) { } missing := filepath.Join(dir, "missing") if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", - "http,port=15862,tls.enabled=true,tls.cert_file="+missing+",tls.key_file="+missing)); code != 1 { + "http:port=15862,tls.enabled=true,tls.cert_file="+missing+",tls.key_file="+missing)); code != 1 { t.Fatalf("listener TLS without its certificate file: exit %d, want 1", code) } } diff --git a/config/logwisp.toml b/config/logwisp.toml index ad67033..58a9202 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -64,10 +64,10 @@ name = "default" ###============================================================================ ## policy="pass" short-circuits the size check too: enforcing a size cap needs -## rate > 0 AND policy = "drop". +## entries_per_second > 0 AND policy = "drop". # [pipelines.flow.rate_limit] -# rate = 0.0 # Entries/second (0 = limiter disabled) -# burst = 0.0 # Burst capacity (defaults to rate) +# entries_per_second = 0.0 # 0 = limiter disabled +# burst_entries = 0.0 # Bucket capacity (0 = entries_per_second) # policy = "pass" # pass|drop # max_entry_size_bytes = 0 # 0 = unlimited @@ -131,7 +131,8 @@ name = "default" ### On the http sink the block gates stream_path AND status_path. ### Listeners (tcp/http sinks, tcp_chain/http_chain sources): ### mtls authorizes the client certificate; REQUIRES tls.client_auth. -### scram admits the users in credentials_file (`lw auth add-user`). +### scram admits the users in credentials_file (`lw auth add-user`), or +### the one username whose password is in password_file. ### Dialers (tcp_chain/http_chain sinks): ### mtls pins the server identity. ### scram logs in as username with the password in password_file. @@ -153,8 +154,10 @@ name = "default" ### Overrides trust_node. See doc/security.md. ### credentials_file = "" scram listeners: verifiers; read at (re)load, not watched ### token_lifetime_ms = 0 scram, http sink and http_chain source: 10 s to 24 h, 0 = 15 minutes -### username = "" scram dialers -### password_file = "" scram dialers: one trailing line break is trimmed +### username = "" scram dialers, or a listener's one user +### password_file = "" scram, with username: one trailing line break is +### trimmed; /dev/fd/N or a pipe is read once and kept +### for reloads, /dev/tty asks once ### trusted_proxies = [] scram http sink: TLS-terminating reverse proxies ### (addresses or CIDRs); browsers log in through them ###============================================================================ diff --git a/deploy/package/completion/_lw b/deploy/package/completion/_lw index 24c76fa..ac7555b 100644 --- a/deploy/package/completion/_lw +++ b/deploy/package/completion/_lw @@ -31,7 +31,7 @@ auth:*) '--ca-file=[CA file that verifies the server (default: system roots)]:value:_default' \ '--cert-file=[client certificate file, for listeners with tls.client_auth]:value:_default' \ '--key-file=[client key file]:value:_default' \ - '--password-file=[file holding the password]:value:_default' \ + '--password-file=[file holding the password; a terminal asks]:value:_default' \ '--pin-sha256=[sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)]:value:_default' \ '--server-name=[name the server certificate must carry (default: the host)]:value:_default' \ '--unbound[log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)]' \ @@ -43,7 +43,7 @@ auth:*) '--ca-file=[CA file that verifies the server (default: system roots)]:value:_default' \ '--cert-file=[client certificate file, for listeners with tls.client_auth]:value:_default' \ '--key-file=[client key file]:value:_default' \ - '--password-file=[file holding the password]:value:_default' \ + '--password-file=[file holding the password; a terminal asks]:value:_default' \ '--pin-sha256=[sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)]:value:_default' \ '--server-name=[name the server certificate must carry (default: the host)]:value:_default' \ '(-u --user)'{-u=,--user=}'[user name]:value:_default' \ @@ -112,11 +112,13 @@ preset:*) '--issuer_key=[CA key file, for tls=issuer]:value:_default' \ '--key=[key file, for tls=files]:value:_default' \ '--listen=[HOST:PORT; [IPV6\]:PORT]:value:_default' \ + '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ '--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \ '--tls=[off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \ + '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \ - '--viewer=[true: the login page and viewer for users, behind proxy; without users the viewer is always on]:value:_default' \ + '--viewer=[true: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; edge) _arguments -S \ '--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \ @@ -124,13 +126,13 @@ preset:*) '--from=[start|end: where reading a file starts]:value:_default' \ '--key=[client key file]:value:_default' \ '--node=[origin label; default: this host'\''s name]:value:_default' \ - '--password_file=[file holding the SCRAM password]:value:_default' \ + '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \ '--pin=[sha256//BASE64 of the aggregator'\''s key, which it logs with tls=self]:value:_default' \ '--server_name=[name the aggregator'\''s certificate carries; default: to'\''s host]:value:_default' \ '--to=[the aggregator'\''s HOST:PORT; [IPV6\]:PORT]:value:_default' \ '--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ - '(-u --user)'{-u=,--user=}'[SCRAM user (lw auth add-user on the aggregator)]:value:_default' \ + '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; aggregator) _arguments -S \ '--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \ @@ -144,8 +146,10 @@ preset:*) '--key=[key file, for tls=files]:value:_default' \ '--listen=[HOST:PORT; [IPV6\]:PORT; [::\] for IPv6 only]:value:_default' \ '--out=[directory for the received entries; default: standard output]:value:_default' \ + '--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \ '--tls=[self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \ '--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \ + '(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \ '--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \ '(-h --help)'{-h,--help}'[print the flags]' ;; esac diff --git a/deploy/package/completion/lw.bash b/deploy/package/completion/lw.bash index 00822b2..06cf15b 100644 --- a/deploy/package/completion/lw.bash +++ b/deploy/package/completion/lw.bash @@ -66,9 +66,9 @@ _lw() { words='--format --from --path --help -h' ;; serve) case $prev in - --allow|-allow|--cert|-cert|--deny|-deny|--format|-format|--from|-from|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--path|-path|--proxy|-proxy|--tls|-tls|--users|-users|--viewer|-viewer) COMPREPLY=(); return ;; + --allow|-allow|--cert|-cert|--deny|-deny|--format|-format|--from|-from|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--password_file|-password_file|--path|-path|--proxy|-proxy|--tls|-tls|--user|-user|-u|--users|-users|--viewer|-viewer) COMPREPLY=(); return ;; esac - words='--allow --cert --deny --format --from --hosts --issuer_cert --issuer_key --key --listen --path --proxy --tls --users --viewer --help -h' ;; + words='--allow --cert --deny --format --from --hosts --issuer_cert --issuer_key --key --listen --password_file --path --proxy --tls --user -u --users --viewer --help -h' ;; edge) case $prev in --ca|-ca|--cert|-cert|--from|-from|--key|-key|--node|-node|--password_file|-password_file|--path|-path|--pin|-pin|--server_name|-server_name|--to|-to|--transport|-transport|--user|-user|-u) COMPREPLY=(); return ;; @@ -76,9 +76,9 @@ _lw() { words='--ca --cert --from --key --node --password_file --path --pin --server_name --to --transport --user -u --help -h' ;; aggregator) case $prev in - --allow|-allow|--cert|-cert|--client_ca|-client_ca|--deny|-deny|--format|-format|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--out|-out|--tls|-tls|--transport|-transport|--users|-users) COMPREPLY=(); return ;; + --allow|-allow|--cert|-cert|--client_ca|-client_ca|--deny|-deny|--format|-format|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--out|-out|--password_file|-password_file|--tls|-tls|--transport|-transport|--user|-user|-u|--users|-users) COMPREPLY=(); return ;; esac - words='--allow --cert --client_ca --deny --format --hosts --issuer_cert --issuer_key --key --listen --out --tls --transport --users --help -h' ;; + words='--allow --cert --client_ca --deny --format --hosts --issuer_cert --issuer_key --key --listen --out --password_file --tls --transport --user -u --users --help -h' ;; esac ;; config:2) words='init' ;; config:*) diff --git a/deploy/package/completion/lw.fish b/deploy/package/completion/lw.fish index 84b5d2a..61d7ff7 100644 --- a/deploy/package/completion/lw.fish +++ b/deploy/package/completion/lw.fish @@ -59,7 +59,7 @@ complete -c lw -n '__lw_in auth remove-user' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in auth token' -l ca-file -r -F -d 'CA file that verifies the server (default: system roots)' complete -c lw -n '__lw_in auth token' -l cert-file -r -F -d 'client certificate file, for listeners with tls.client_auth' complete -c lw -n '__lw_in auth token' -l key-file -r -F -d 'client key file' -complete -c lw -n '__lw_in auth token' -l password-file -r -F -d 'file holding the password' +complete -c lw -n '__lw_in auth token' -l password-file -r -F -d 'file holding the password; a terminal asks' complete -c lw -n '__lw_in auth token' -l pin-sha256 -r -F -d 'sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)' complete -c lw -n '__lw_in auth token' -l server-name -r -F -d 'name the server certificate must carry (default: the host)' complete -c lw -n '__lw_in auth token' -l unbound -d 'log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)' @@ -70,7 +70,7 @@ complete -c lw -n '__lw_in auth stream' -l addr -r -F -d 'tcp sink address, HOST complete -c lw -n '__lw_in auth stream' -l ca-file -r -F -d 'CA file that verifies the server (default: system roots)' complete -c lw -n '__lw_in auth stream' -l cert-file -r -F -d 'client certificate file, for listeners with tls.client_auth' complete -c lw -n '__lw_in auth stream' -l key-file -r -F -d 'client key file' -complete -c lw -n '__lw_in auth stream' -l password-file -r -F -d 'file holding the password' +complete -c lw -n '__lw_in auth stream' -l password-file -r -F -d 'file holding the password; a terminal asks' complete -c lw -n '__lw_in auth stream' -l pin-sha256 -r -F -d 'sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)' complete -c lw -n '__lw_in auth stream' -l server-name -r -F -d 'name the server certificate must carry (default: the host)' complete -c lw -n '__lw_in auth stream' -l user -o u -r -F -d 'user name' @@ -112,24 +112,26 @@ complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate complete -c lw -n '__lw_in preset serve' -l issuer_key -r -F -d 'CA key file, for tls=issuer' complete -c lw -n '__lw_in preset serve' -l key -r -F -d 'key file, for tls=files' complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT' +complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them' complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files' +complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM' -complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true: the login page and viewer for users, behind proxy; without users the viewer is always on' +complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true: the login page and viewer, behind proxy; without user or users the viewer is always on' complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots' complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca' complete -c lw -n '__lw_in preset edge' -l from -r -F -d 'start|end: where reading a file starts' complete -c lw -n '__lw_in preset edge' -l key -r -F -d 'client key file' complete -c lw -n '__lw_in preset edge' -l node -r -F -d 'origin label; default: this host\'s name' -complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding the SCRAM password' +complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset edge' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input' complete -c lw -n '__lw_in preset edge' -l pin -r -F -d 'sha256//BASE64 of the aggregator\'s key, which it logs with tls=self' complete -c lw -n '__lw_in preset edge' -l server_name -r -F -d 'name the aggregator\'s certificate carries; default: to\'s host' complete -c lw -n '__lw_in preset edge' -l to -r -F -d 'the aggregator\'s HOST:PORT; [IPV6]:PORT' complete -c lw -n '__lw_in preset edge' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' -complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user (lw auth add-user on the aggregator)' +complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset edge' -l help -o h -d 'print the flags' complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them' complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files' @@ -142,8 +144,10 @@ complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key fil complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls=files' complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT; [::] for IPv6 only' complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output' +complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks' complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files' complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain' +complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users' complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM' complete -c lw -n '__lw_in preset aggregator' -l help -o h -d 'print the flags' diff --git a/doc/README.md b/doc/README.md index c244e53..ce9de30 100644 --- a/doc/README.md +++ b/doc/README.md @@ -90,7 +90,7 @@ stdout: ```bash lw < app.log > copy.log -tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN +tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN ``` As a service, a file holds the pipelines: diff --git a/doc/chaining.md b/doc/chaining.md index 6db10d0..4de85bc 100644 --- a/doc/chaining.md +++ b/doc/chaining.md @@ -255,8 +255,9 @@ is what their certificates say, regardless of the `node` each edge configured. `test/mtls-chain-test.sh` builds exactly this shape against a throwaway PKI. To use passwords instead of client certificates, replace both `auth` blocks: -`type = "scram"` with `username` and `password_file` on the edge, and with a -`credentials_file` on the relay. `client_auth` and the edge's certificate become +`type = "scram"` with `username` and `password_file` on the edge, and on the +relay a `credentials_file`, or one `username` with its `password_file`. +`client_auth` and the edge's certificate become optional, and entries are labelled with the username. See [Password Authentication](security.md#password-authentication-scram). diff --git a/doc/cli.md b/doc/cli.md index 2d17b1d..8f0715d 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -3,7 +3,7 @@ ``` lw [options] [< input] lw -c|--config FILE [options] -lw -p|--preset NAME[,KEY=VALUE...] [options] +lw -p|--preset NAME[:KEY=VALUE,...] [options] lw -t|--check [options] lw --dump [options] lw -T|--tui [options] @@ -45,7 +45,7 @@ first argument. tail`, `lw -t FILE`) are usage errors, named on stderr. - lw's multiword options join words with `-` (`--rate-limit`, `--password-file`); configuration keys, spec keys and preset keys keep - TOML's `_` and `.` (`--status_reporter`, `--sink http,tls.cert_file=F`, + TOML's `_` and `.` (`--status_reporter`, `--sink http:tls.cert_file=F`, `lw preset edge --password_file F`). - The commands also take a long option after one dash (`-user NAME`). @@ -121,14 +121,15 @@ pipelines and the built-in default; every other key keeps its precedence. - stdin has one reader, so the whole configuration holds at most one console source: `pipeline "b": a console source already reads stdin in pipeline "a"` -A SPEC is a comma-separated list: +A SPEC is `TYPE:key=value,...`, or `key=value,...` without a TYPE: -- source, sink, filter and format specs start with a TYPE +- source, sink, filter and format specs start with a TYPE, which ends at the + first `:`; a TYPE alone (`--sink null`, `--format txt`) takes the defaults - the plugin type: [Sources](sources.md), [Sinks](sinks.md) - `include` or `exclude`: [Filters](filters.md) - `json`, `txt` or `raw`: [Formatters](formatters.md) - rate-limit and heartbeat specs have no TYPE -- the rest are `key=value` pairs with the TOML keys of the plugin's `config` +- the pairs are comma-separated, with the TOML keys of the plugin's `config` table or of the [flow stage](configuration.md#flow-stages) - a dotted key reaches a nested table: `tls.cert_file=...`, `auth.type=scram` - a repeated key makes a list: `patterns=ERROR,patterns=WARN` @@ -136,7 +137,8 @@ A SPEC is a comma-separated list: is one regex, `auth.allow=a,auth.allow=b` two entries - values convert to the option's type: `port=8080`, `raw=true` - `\` escapes `,`, `=` and `\` in a value; any other backslash stays, so regex - escapes such as `\d` pass unchanged + escapes such as `\d` pass unchanged; a `:` in a value needs no escape + (`host=::1`, `listen=127.0.0.1:8080`) - `id=NAME` names a source or sink; the default is its TYPE, then `TYPE_2`, `TYPE_3`, ... - naming a stage turns it on: `--rate-limit` defaults `policy` to `drop`, @@ -145,8 +147,9 @@ A SPEC is a comma-separated list: Errors name the flag and the offending part, and nothing starts: ``` ---sink http,port: missing "=" ---rate-limit rate=100,polcy=drop: unknown key "polcy" +--sink http:port: missing "=" +--rate-limit entries_per_second=100,polcy=drop: unknown key "polcy" +--source file,directory=/x: TYPE ends at ':', as in file:directory=/x ``` A misspelled plugin key fails when the plugin is built, before any listener @@ -155,35 +158,39 @@ opens: `failed to create sink http: ... unknown key "tls.enabeld"`. ```bash # tail a directory, serve it over SSE on every interface; a browser at # http://HOST:8080/ shows it -lw --source 'file,directory=/var/log/app,pattern=*.log' \ - --sink http,host=0.0.0.0,port=8080 +lw --source 'file:directory=/var/log/app,pattern=*.log' \ + --sink http:host=0.0.0.0,port=8080 # errors and warnings as JSON, over TLS with SCRAM logins -lw --source file,directory=/var/log/app \ - --filter include,patterns=ERROR,patterns=WARN --format json \ - --sink "http,port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\ +lw --source file:directory=/var/log/app \ + --filter include:patterns=ERROR,patterns=WARN --format json \ + --sink "http:port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\ tls.enabled=true,tls.cert_file=/etc/logwisp/server.crt,tls.key_file=/etc/logwisp/server.key" # two pipelines: app writes stdout, relay a file -lw --pipeline app --source file,directory=/var/log/app \ - --pipeline relay --source tcp_chain,port=9000 \ - --sink file,directory=/var/log/relay,name=relay +lw --pipeline app --source file:directory=/var/log/app \ + --pipeline relay --source tcp_chain:port=9000 \ + --sink file:directory=/var/log/relay,name=relay ``` Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns). ## Presets -`--preset NAME,key=value,...` takes the [SPEC](#pipelines) syntax. A list -key (`hosts`, `proxy`, `allow`, `deny`) takes several values, repeated -(`hosts=a,hosts=b`) or `,`-separated; any other key takes one. +`--preset NAME:key=value,...` takes the [SPEC](#pipelines) syntax. A list +key (`hosts`, `proxy`, `allow`, `deny`) takes several values: repeated +(`hosts=a,hosts=b`) or `\,`-separated in a SPEC, `,`-separated in `lw preset` +flags and the TUI; any other key takes one. `lw preset NAME [--key value ...]` prints the pipeline a preset expands to, ready for a configuration file; its flags are the keys, spelled as in the SPEC (`--password_file`), and `-u` is `--user`; `lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An unknown key fails and lists the valid ones. A `path` is a file, a directory (its files) or a glob; a preset that takes one reads stdin without it, and -`from` (`end`, or `start`) is where reading a file starts. +`from` (`end`, or `start`) is where reading a file starts. `user` names one +SCRAM user, an edge's login or a listener's only user in place of `users`; +`password_file` holds its password and defaults to `/dev/tty`, which asks +([password sources](security.md#one-user-without-a-credentials-file)). - `pipe`: stdin to stdout, the built-in default - `format` (`raw`) @@ -196,41 +203,49 @@ unknown key fails and lists the valid ones. A `path` is a file, a directory - `tls`: `off` (default), `self` (a self-signed certificate made at startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or `files` (`cert` and `key`); `hosts` adds names to a made certificate - - without `users` the viewer needs no login, over plain http too + - without `user` or `users` the viewer needs no login, over plain http too - `allow` and `deny`: addresses or CIDRs, the sink's [acl](security.md#the-acl-block) - - `users`: a credentials file; readers then log in with SCRAM, and browsers - need `proxy` and `viewer=true`: the login page and viewer behind the - TLS-terminating proxies `proxy` names, since a browser cannot bind its - login to the TLS channel + - `users` (a credentials file) or `user`: readers then log in with SCRAM, + which needs `tls` or `proxy`; browsers need `proxy` and `viewer=true`: the + login page and viewer behind the TLS-terminating proxies `proxy` names, + since a browser cannot bind its login to the TLS channel - `edge`: forward to an aggregator over TLS (`tcp_chain`, or `http_chain` with `transport=http`) - `to` (required), `path`, `from`, `transport` (`tcp`), `node` - verify the aggregator by `ca` (a CA file; default: system roots) or `pin` (its `tls.pin_sha256`), and `server_name` - - authenticate by `user` and `password_file` (SCRAM), or a client `cert` and - `key` (mTLS): without either the preset refuses to run + - authenticate by `user` (SCRAM), or a client `cert` and `key` (mTLS): + without either the preset refuses to run - `aggregator`: receive from edges over TLS, to stdout or files - `listen` (`0.0.0.0:9000`), `transport` (`tcp`), `format` (`json`) - `tls`: `self` (default), `issuer` or `files`, as for `serve`; never `off` - - authenticate by `users` (SCRAM) or `client_ca` (mTLS): one is required + - authenticate by `users` or `user` (SCRAM), or `client_ca` (mTLS): one is + required - `allow` and `deny`, as for `serve` - `out`: a directory for `aggregate*.log` files; default stdout ```bash # follow a directory, level names in color on a terminal -lw --preset tail,path=/var/log/app +lw --preset tail:path=/var/log/app # serve it to a browser at http://127.0.0.1:8080/ -lw --preset serve,path=/var/log/app +lw --preset serve:path=/var/log/app # serve it over HTTPS, self-signed, to SCRAM users; keep the result as a file -lw --preset serve,path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml +lw --preset serve:path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml # an aggregator with a certificate made at startup, and an edge pinning it -lw --preset aggregator,users=/etc/logwisp/users.toml,out=/var/log/edges -lw --preset edge,path=/var/log/app,to=agg.example.org:9000,pin=sha256//BASE64,\ +lw --preset aggregator:users=/etc/logwisp/users.toml,out=/var/log/edges +lw --preset edge:path=/var/log/app,to=agg.example.org:9000,pin=sha256//BASE64,\ user=edge-01,password_file=/etc/logwisp/edge-01.pass + +# a temporary pipe, nothing on disk: Enter at the receiver's prompt draws the +# password, typed at the sender's; or both read it from a descriptor +lw -p aggregator:listen=0.0.0.0:9000,user=pipe,format=raw,allow=SENDER/32 +cmd | lw -p edge:to=RECEIVER:9000,pin=sha256//BASE64,user=pipe +cmd | lw -p edge:to=RECEIVER:9000,pin=sha256//BASE64,user=pipe,\ +password_file=/dev/fd/3 3< <(pass show lw/pipe) ``` A self-signed aggregator logs its pin at startup (`pin_sha256`, a warning, so @@ -278,9 +293,9 @@ A container needs no configuration file: ```bash docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \ -e LOGWISP_PIPELINE=app \ - -e LOGWISP_SOURCE='file,directory=/logs,pattern=*.log' \ - -e LOGWISP_FILTER='exclude,patterns=DEBUG' \ - -e LOGWISP_SINK='http,host=0.0.0.0,port=8080' \ + -e LOGWISP_SOURCE='file:directory=/logs,pattern=*.log' \ + -e LOGWISP_FILTER='exclude:patterns=DEBUG' \ + -e LOGWISP_SINK='http:host=0.0.0.0,port=8080' \ -e LOGWISP_SINK_1=console \ -e LOGWISP_LOGGING_LEVEL=info \ logwisp @@ -382,13 +397,13 @@ sink gives its clients or its downstream its `write_timeout_ms` lw < app.log > copy.log # errors and warnings only -tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN +tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN # stdin as a live SSE stream; the http and tcp sinks bind 0.0.0.0 by default -journalctl -f | lw --sink http,host=127.0.0.1,port=8080 +journalctl -f | lw --sink http:host=127.0.0.1,port=8080 # a directory's files to one file, through stdout -lw --source file,directory=/var/log/app,pattern='*.log' > all.log +lw --source file:directory=/var/log/app,pattern='*.log' > all.log ``` A console sink never drops: when its reader is slow, the pipeline waits, and @@ -411,7 +426,7 @@ prints `configuration ok: N pipeline(s)` and exits 0, or the error and exits 1 ```bash lw --check -c /etc/logwisp/logwisp.toml -lw --check --source file,directory=/var/log/app --sink http,port=8080 +lw --check --source file:directory=/var/log/app --sink http:port=8080 ``` **Production** @@ -443,21 +458,30 @@ WebAssembly. pipeline flags, `LOGWISP_SOURCE`, a discovered file); with none but the built-in pipe, the preset menu (or `empty`). A pasted `lw` command line replaces the pipelines, in the preset menu too. -- Screen: SOURCES, FLOW and SINKS side by side from 64 columns, stacked - below; each source and sink a tinted bar with its type, whether it listens - or dials, and a summary; the four flow stages in their order in a dashed - box, the ones off dimmed. The inspector, the chosen part's options, sits - below the canvas, and beside it from 110 columns. -- Keys: arrows or `hjkl` move; Enter inspects, then edits a value (empty - returns it to its default); Tab sets it and goes to the next; Esc back; `a` adds a source, - sink or filter; `d` deletes, or returns a value to its default; Space turns - a flow stage or a switch on or off; `J`/`K` move a filter; `[` `]` switch - pipelines; `p` presets; `c` shows the first problem; `o` output; `?` keys; - `q` quits. -- Output, `o`: run the pipelines with the other options given (so `--check` - and `--dump` apply to them, and reloads keep them), or print one of the - command line, the environment variables (one pipeline) and the file's - pipelines on standard output once the screen closes. +- Screen: the pipeline bar on top; SOURCES, FLOW and SINKS side by side + from 64 columns, stacked below; each source and sink a tinted bar with its + type, whether it listens or dials, and a summary; the four flow stages in + their order in a dashed box, the ones off dimmed. The inspector, the + chosen part's options with radio buttons for few choices, sits below the + canvas, and beside it from 110 columns. Narrow panes stack labels, wrap + text and scroll dialogs; the status line wraps to three rows. +- Keys: arrows or `hjkl` move; Enter inspects a part, opening on its first + required key, then edits a value (empty returns it to its default), opens + a list's choices or adds on an add row; in a value, Enter sets it, and Tab + or Up/Down set it and move. Left/Right step a choice or set a switch; + Space flips it; Esc goes back. `a` adds a source, sink or filter; `c` + changes a source's or sink's type in its place; `d` deletes, or returns a + value to its default; Space turns a flow stage on or off; `J`/`K` move a + filter; `[` `]` switch pipelines; `p` presets; `!` shows the first + problem; `r` runs; `o` output; `?` keys; `q` quits. +- Pipelines: Up from a column's top row reaches the bar, where `[` `]` or + Left/Right switch, `a` adds one (empty or from a preset), Enter renames and + `d` deletes, asking first when it has parts. +- Output, `o`, each picked by its letter: `r` runs the pipelines with the + other options given (so `--check` and `--dump` apply to them, and reloads + keep them), or `c`, `e` and `f` print the command line, the environment + variables (one pipeline) or the file's pipelines on standard output once + the screen closes. - The screen draws on the controlling terminal: `lw -T > pipelines.toml` and `cmd | lw -T` keep standard input and output as data. lw never writes the file it starts from, but the shell empties a redirect's target before lw @@ -476,9 +500,9 @@ them as a viewer. See ``` lw auth add-user --credentials FILE -u NAME [--password-file FILE] [--generate] lw auth remove-user --credentials FILE -u NAME -lw auth token --url https://HOST:PORT[/PATH] -u NAME --password-file FILE +lw auth token --url https://HOST:PORT[/PATH] -u NAME [--password-file FILE] [--unbound] [TLS flags] -lw auth stream --addr HOST:PORT -u NAME --password-file FILE [TLS flags] +lw auth stream --addr HOST:PORT -u NAME [--password-file FILE] [TLS flags] ``` `-u` is short for `--user`. @@ -496,7 +520,7 @@ success, `1` on failure and `2` on a usage error. **`add-user`** creates the credentials file, with a fresh `decoy_key`, when it does not exist or is empty (create it empty first to choose its owner). The password comes from `--password-file` when that file exists (at least 8 bytes; -one trailing line break is trimmed). Otherwise a random +one trailing line break is trimmed; `/dev/tty` asks). Otherwise a random 26-character password (130 bits) is generated and written to `--password-file`, or printed once to stdout when there is none. Replacing an existing user's password needs an existing `--password-file` or `--generate`, so a mistyped path @@ -512,11 +536,13 @@ Neither touches a running LogWisp: send `SIGHUP`, since `auto_reload` does not watch the credentials file. **`token`** and **`stream`** build the same TLS and SCRAM client as a chain -sink. TLS flags: `--ca-file` (default: system roots) or `--pin-sha256` (the -`tls.pin_sha256` a self-signed listener logs), `--server-name` (default: the -host), and `--cert-file` / `--key-file` for a listener with `tls.client_auth`. -There is no flag to skip verification: an unverified server could relay the -login. Redirects are not followed. `--unbound` logs in to an `http` sink behind +sink. `--password-file` defaults to `/dev/tty`: the password is asked without +echo; a file, `/dev/fd/N` or a pipe is read as by a +[plugin](security.md#one-user-without-a-credentials-file). TLS flags: +`--ca-file` (default: system roots) or `--pin-sha256` (the `tls.pin_sha256` a +self-signed listener logs), `--server-name` (default: the host), and +`--cert-file` / `--key-file` for a listener with `tls.client_auth`. There is no +flag to skip verification: an unverified server could relay the login. Redirects are not followed. `--unbound` logs in to an `http` sink behind a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's certificate across the two requests; only then may `--url` carry the path the proxy mounts LogWisp at. `stream` exits `0` on `SIGINT` or `SIGTERM`, and `1` @@ -549,9 +575,8 @@ curl --cacert ca.crt -H @<(printf 'Authorization: Bearer %s\n' \ "$(lw auth token --url https://HOST:PORT --user viewer \ --password-file viewer.pass --ca-file ca.crt)") https://HOST:PORT/status -# follow a tcp sink -lw auth stream --addr HOST:PORT --user viewer --password-file viewer.pass \ - --ca-file ca.crt +# follow a tcp sink, asked for the password +lw auth stream --addr HOST:PORT --user viewer --pin-sha256 sha256//BASE64 ``` ## `lw tls` diff --git a/doc/configuration.md b/doc/configuration.md index 45fb69f..c87335e 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -111,8 +111,8 @@ name = "app" # required, unique across pipelines # --- flow: everything between sources and sinks --- [pipelines.flow.rate_limit] -rate = 1000.0 -burst = 2000.0 +entries_per_second = 1000.0 +burst_entries = 2000.0 policy = "drop" max_entry_size_bytes = 65536 @@ -168,19 +168,19 @@ Every stage is optional: Options, each as type and default: -- `rate` (float, `0`): entries per second; `<= 0` disables the limiter - entirely. -- `burst` (float, `rate`): token bucket capacity. +- `entries_per_second` (float, `0`): `<= 0` disables the limiter entirely. +- `burst_entries` (float, `entries_per_second`): token bucket capacity. - `policy` (string, `pass`): `pass` allows everything through, `drop` discards over-limit entries. - `max_entry_size_bytes` (int, `0`): per-entry byte cap; `0` = unlimited. Two behaviours are easy to trip over: -- The limiter is constructed only when `rate > 0`. With `rate = 0`, +- The limiter is constructed only when `entries_per_second > 0`. With `0`, `max_entry_size_bytes` is never enforced. - `policy = "pass"` short-circuits the whole check, including the size cap. - To enforce a size cap you need `rate > 0` **and** `policy = "drop"`. + To enforce a size cap you need `entries_per_second > 0` **and** + `policy = "drop"`. #### Heartbeat @@ -258,7 +258,7 @@ Startup validation is intentionally split. "pipelines[0].plugin_sinks[0].confg"`. A misspelled table path would otherwise drop the whole table it heads. A top-level `config_file` is ignored - pipeline [specs](cli.md#pipelines): syntax, and unknown flow-stage keys such - as `--rate-limit rate=1,polcy=drop` + as `--rate-limit entries_per_second=1,polcy=drop` - at least one pipeline - unique, non-empty pipeline names - at least one source and one sink per pipeline diff --git a/doc/installation.md b/doc/installation.md index c580267..2accceb 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -317,7 +317,7 @@ docker run -d --name logwisp \ ```bash -e LOGWISP_PIPELINE=relay \ - -e LOGWISP_SOURCE=tcp_chain,port=9000,tls.enabled=true,tls.cert_file=/run/secrets/tls.crt,tls.key_file=/run/secrets/tls.key,auth.type=scram,auth.credentials_file=/run/secrets/users.toml \ + -e LOGWISP_SOURCE=tcp_chain:port=9000,tls.enabled=true,tls.cert_file=/run/secrets/tls.crt,tls.key_file=/run/secrets/tls.key,auth.type=scram,auth.credentials_file=/run/secrets/users.toml \ -e LOGWISP_SINK=console ``` diff --git a/doc/lw.1 b/doc/lw.1 index 69d895a..5676b97 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -342,17 +342,20 @@ Set the pipeline's formatter, rate limit and heartbeat. .Pp A source, sink, filter or format .Ar spec -is a type followed by comma-separated +is a type, then a colon and comma-separated .Ar key Ns = Ns Ar value -pairs; a rate limit or heartbeat +pairs; the first +.Ql \&: +ends the type, and a type alone takes the defaults. +A rate limit or heartbeat .Ar spec has the pairs only: .Bd -literal -offset indent -\-\-source file,directory=/var/log/app,pattern=*.log -\-\-sink http,host=127.0.0.1,port=8080,id=web -\-\-filter include,patterns=ERROR,patterns=WARN +\-\-source file:directory=/var/log/app,pattern=*.log +\-\-sink http:host=127.0.0.1,port=8080,id=web +\-\-filter include:patterns=ERROR,patterns=WARN \-\-format json -\-\-rate\-limit rate=100,burst=200,policy=drop +\-\-rate\-limit entries_per_second=100,burst_entries=200,policy=drop .Ed .Pp Keys are dotted paths into the plugin's options, such as @@ -366,6 +369,8 @@ and inside a value; quote such a .Ar spec so the shell keeps the backslash. +A colon in a value needs no escape, as in +.Cm host=::1 . .Cm id= Ns Ar name names a source or sink; the default is its type, then .Ar type Ns _2 , @@ -377,7 +382,7 @@ ignore the pipeline variables in .Sx ENVIRONMENT . .Ss Presets A preset builds a common pipeline from a few keys: -.Fl \-preset Ar name Ns Op , Ns Ar key Ns = Ns Ar value ... , +.Fl \-preset Ar name Ns Op : Ns Ar key Ns = Ns Ar value ... , in the syntax of the other specifications; .Nm .Cm preset Ar name Op Fl \- Ns Ar key Ar value ... @@ -392,6 +397,16 @@ A .Ar path is a file, a directory, whose files are followed, or a glob; without one, a preset that takes it reads standard input. +.Cm user +names one SCRAM user, an edge's login or a listener's only user in place of +.Cm users ; +.Cm password_file +holds its password, read once at startup when it is +.Pa /dev/fd/ Ns Ar N +or a pipe, and defaults to +.Pa /dev/tty , +which asks at startup; on a listener, Enter alone generates a password and +shows it there. .Bl -tag -width Ds .It Cm pipe Op Cm format Standard input to standard output, the pipeline @@ -400,7 +415,7 @@ runs without a configuration. .It Cm tail path Op Cm from , format Follow files to standard output, like .Ql tail \-F . -.It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer , allow , deny Oc +.It Cm serve Oo Cm path , listen , format , tls , users , user , password_file , proxy , viewer , allow , deny Oc Serve a live HTTP stream on .Cm listen .Pq default Li 127.0.0.1:8080 , @@ -416,9 +431,13 @@ or .Cm files .Pq Cm cert No and Cm key ; .Cm users -names a credentials file, so readers log in with SCRAM. -A browser at the root gets a live viewer; with -.Cm users , +names a credentials file, or +.Cm user +one user, so readers log in with SCRAM, which needs +.Cm tls +or +.Cm proxy . +A browser at the root gets a live viewer; with a user, only behind a TLS-terminating proxy named by .Cm proxy , with @@ -437,14 +456,12 @@ or, with .Cm transport=http , .Cm http_chain . It refuses to run without credentials: -.Cm user -and -.Cm password_file , +.Cm user , or a client .Cm cert and .Cm key . -.It Cm aggregator Oo Cm listen , transport , tls , users , client_ca , out , format , allow , deny Oc +.It Cm aggregator Oo Cm listen , transport , tls , users , user , password_file , client_ca , out , format , allow , deny Oc Receive from edges on .Cm listen .Pq default Li 0.0.0.0:9000 @@ -452,7 +469,9 @@ over TLS, never off .Pq default Cm tls=self , authenticated by the .Cm users -credentials file or the client certificates +credentials file, the one +.Cm user , +or the client certificates .Cm client_ca vouches for, and write to standard output or to files in .Cm out ; @@ -534,7 +553,7 @@ one keeps its owner and mode. .It Cm remove-user Fl \-credentials Ar file Fl u Ar name Remove a user. The last user cannot be removed. -.It Cm token Fl \-url Ar https://host:port Fl u Ar name Fl \-password\-file Ar file Oo Ar tls-flags Oc Op Fl \-unbound +.It Cm token Fl \-url Ar https://host:port Fl u Ar name Oo Fl \-password\-file Ar file Oc Oo Ar tls-flags Oc Op Fl \-unbound Log in to an .Cm http sink or @@ -544,12 +563,21 @@ source and print a bearer token. logs in without channel binding, to an .Cm http sink behind a TLS-terminating proxy; the URL may then carry the proxy's path. -.It Cm stream Fl \-addr Ar host:port Fl u Ar name Fl \-password\-file Ar file Op Ar tls-flags +.It Cm stream Fl \-addr Ar host:port Fl u Ar name Oo Fl \-password\-file Ar file Oc Op Ar tls-flags Log in to a .Cm tcp sink and copy its stream to standard output until interrupted. .El .Pp +For +.Cm token +and +.Cm stream , +.Fl \-password\-file +defaults to +.Pa /dev/tty , +where the password is asked without echo. +.Pp The .Ar tls-flags are @@ -627,7 +655,8 @@ is looked for. .Pp Secrets never go in the environment: TLS keys, credentials files and password files are named by path -.Pq Cm tls.key_file , auth.credentials_file , auth.password_file . +.Pq Cm tls.key_file , auth.credentials_file , auth.password_file , +or asked on the terminal. .Sh FILES Without .Fl c @@ -728,7 +757,7 @@ $ lw < app.log > copy.log .Pp Follow a log and keep only errors and warnings: .Bd -literal -offset indent -$ tail \-F app.log | lw \-\-filter include,patterns=ERROR,patterns=WARN +$ tail \-F app.log | lw \-\-filter include:patterns=ERROR,patterns=WARN .Ed .Pp Prefix each line with the time, the level found in it and the source: @@ -743,7 +772,7 @@ sink listens on every address by default, and read it from another terminal as server-sent events; a client gets the lines that arrive while it is connected: .Bd -literal -offset indent -$ journalctl \-f | lw \-\-sink http,host=127.0.0.1,port=8080 +$ journalctl \-f | lw \-\-sink http:host=127.0.0.1,port=8080 $ curl \-N http://127.0.0.1:8080/stream .Ed .Pp @@ -753,19 +782,19 @@ file in a directory, as .Ql tail \-F does, and write the new lines unparsed to one file: .Bd -literal -offset indent -$ lw \-\-source 'file,directory=/var/log/app,pattern=*.log,raw=true' \e +$ lw \-\-source 'file:directory=/var/log/app,pattern=*.log,raw=true' \e > all.log .Ed .Pp Follow every file in a directory, with the level names in color on a terminal: .Bd -literal -offset indent -$ lw \-\-preset tail,path=/var/log/app +$ lw \-\-preset tail:path=/var/log/app .Ed .Pp Print the configuration a set of options makes, to keep as a file: .Bd -literal -offset indent -$ lw \-\-preset serve,path=/var/log/app,tls=self \-\-dump > serve.toml +$ lw \-\-preset serve:path=/var/log/app,tls=self \-\-dump > serve.toml .Ed .Pp Ship a directory from an edge to an aggregator: on the aggregator, create @@ -777,13 +806,22 @@ and the password file to the edge; then run both: # lw tls cert \-\-ca-dir /etc/logwisp/pki \-\-name agg.example.org \-\-server # lw auth add-user \-\-credentials /etc/logwisp/users.toml \e \-\-user edge-01 \-\-password-file /root/edge-01.pass -agg# lw \-\-preset aggregator,tls=files,users=/etc/logwisp/users.toml,\e +agg# lw \-\-preset aggregator:tls=files,users=/etc/logwisp/users.toml,\e cert=/etc/logwisp/pki/agg.example.org.crt,\e key=/etc/logwisp/pki/agg.example.org.key,out=/var/log/edges -edge# lw \-\-preset edge,path=/var/log/app,to=agg.example.org:9000,\e +edge# lw \-\-preset edge:path=/var/log/app,to=agg.example.org:9000,\e ca=/etc/logwisp/pki/ca.crt,user=edge-01,password_file=/root/edge-01.pass .Ed .Pp +A temporary pipe with nothing on disk: the receiver makes a self-signed +certificate, logs its pin, and generates the password when Enter alone +answers its prompt; the sender pins the certificate and is asked for the +password: +.Bd -literal -offset indent +recv$ lw \-p aggregator:listen=0.0.0.0:9000,user=pipe,format=raw +send$ cmd | lw \-p edge:to=RECEIVER:9000,pin=sha256//BASE64,user=pipe +.Ed +.Pp Write the annotated configuration to the file .Nm reads without diff --git a/doc/operations.md b/doc/operations.md index c6fe3f2..aceec07 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -7,7 +7,7 @@ Running, monitoring, and maintaining LogWisp. ```bash # a filter: stdin to stdout, exits 0 at the end of input lw < app.log > copy.log -tail -F app.log | lw --filter include,patterns=ERROR +tail -F app.log | lw --filter include:patterns=ERROR # a service: the file's pipelines, until SIGINT or SIGTERM lw -c /etc/logwisp/logwisp.toml @@ -249,16 +249,16 @@ client_buffer_size = 1024 ```toml [pipelines.flow.rate_limit] -rate = 1000.0 -burst = 2000.0 +entries_per_second = 1000.0 +burst_entries = 2000.0 policy = "drop" max_entry_size_bytes = 65536 ``` Two behaviours to keep in mind: the limiter does not exist at all when -`rate <= 0`, and `policy = "pass"` short-circuits the size cap as well as the -rate check. Enforcing `max_entry_size_bytes` therefore requires `rate > 0` and -`policy = "drop"`. +`entries_per_second <= 0`, and `policy = "pass"` short-circuits the size cap as +well as the rate check. Enforcing `max_entry_size_bytes` therefore requires +`entries_per_second > 0` and `policy = "drop"`. ### Formatting diff --git a/doc/scram-auth-plan.md b/doc/scram-auth-plan.md index 07875b6..634cc5b 100644 --- a/doc/scram-auth-plan.md +++ b/doc/scram-auth-plan.md @@ -15,8 +15,10 @@ next to the existing certificate method, without weakening what mTLS gives. - Password authentication for chain links (`tcp_chain`, `http_chain`: sources verify, sinks present) and for viewers of the `tcp` and `http` sinks. -- No password or password-equivalent on the wire, no KDF work on listeners, and - no credential relay through a TLS-terminating MITM. +- No password or password-equivalent on the wire, no KDF work per login on + listeners, and no credential relay through a TLS-terminating MITM. +- A temporary pipe needs no file: one listener user from a descriptor, a pipe + or the terminal, with a certificate made at startup. - Default stays `auth.type = "none"`; existing configurations are unaffected. - Config in TOML, credential management and viewer clients in a `lw auth` CLI. @@ -66,7 +68,7 @@ curl by the CLI is unpinned. # listener (tcp_chain/http_chain source, tcp/http sink) [pipelines.plugin_sources.config.auth] type = "scram" -credentials_file = "/etc/logwisp/users.toml" +credentials_file = "/etc/logwisp/users.toml" # or username + password_file token_lifetime_ms = 900000 # http sink and http_chain source only node_binding = "force" # chain sources; binds to the username @@ -81,12 +83,20 @@ Validation at construction: - `scram` requires TLS, except on an `http` sink in proxy mode; a dialer also forbids `insecure_skip_verify`. -- Listeners require `credentials_file` and reject `allow`/`allow_patterns` (the - credentials file is the allow list), `username` and `password_file`. Use one - file per listener when listeners admit different users. -- Dialers require `username` + `password_file` (1–1024 bytes after trimming one - line break), and reject `credentials_file`, `token_lifetime_ms`, `allow*` and - `identity` (server pinning stays an mTLS feature). +- Listeners require `credentials_file`, or `username` and `password_file` + (one user, its verifier derived in memory at construction), not both, and + reject `allow`/`allow_patterns`: the users are the allow list. Use one file + per listener when listeners admit different users. +- Dialers require `username` + `password_file`, and reject `credentials_file`, + `token_lifetime_ms`, `allow*` and `identity` (server pinning stays an mTLS + feature). +- A password is 1–1024 bytes after trimming one line break, 8 or more on a + listener. A regular file is read at every construction; `/dev/fd/N` (the + inherited descriptor, then closed), a pipe, a FIFO or another device once, at + startup, and kept for reloads; a terminal is asked once, at startup, without + echo, and on a listener Enter alone generates a password shown only there. + The open file decides, not stat: FreeBSD stats `/dev/stdin` as a device + whatever it holds. No terminal fails closed. - `identity` on a listener requires `tls.client_auth` and **binds the certificate to the user**: the certificate's identity field must equal the SCRAM username, so a peer needs its own certificate and its own password (PostgreSQL's @@ -207,8 +217,8 @@ closed. ``` lw auth add-user --credentials F --user U [--password-file P] [--generate] lw auth remove-user --credentials F --user U -lw auth token --url https://host:port[/path] --user U --password-file P [--unbound] [TLS flags] -lw auth stream --addr host:port --user U --password-file P [TLS flags] +lw auth token --url https://host:port[/path] --user U [--password-file P] [--unbound] [TLS flags] +lw auth stream --addr host:port --user U [--password-file P] [TLS flags] ``` - `add-user` takes the password from `--password-file` when it exists (at least @@ -224,7 +234,7 @@ lw auth stream --addr host:port --user U --password-file P [TLS flags] - `token` prints a bearer token for curl (`-H @<(...)` keeps it out of argv); `stream` authenticates to a `tcp` sink and copies the stream to stdout until interrupted. `--url` takes a path only with `--unbound`, and redirects are not - followed. + followed. `--password-file` defaults to `/dev/tty`, which asks. - TLS flags: `--ca-file` or `--pin-sha256`, `--server-name`, `--cert-file`, `--key-file`. There is no insecure flag. - Exit status: 0 success, 1 failure, 2 usage error. @@ -340,7 +350,11 @@ a challenge below its Argon2 floor; the HTTP exchange on a TLS test server (token accepted, garbage and foreign tokens `401`, forged final yields no token, a different certificate after the challenge refused before the proof is sent, renewal ahead of expiry); `Authorize` failing closed under SCRAM; password -trimming; refused hellos freeing their slot; unbound logins pinned too. Proxy +trimming; a one-user listener admitting its password, refusing another and an +unknown user alike; a descriptor's password surviving a reload, closed and +refused to a second user; a descriptor lw opened itself never read; refused +hellos freeing their slot; unbound logins pinned too. In `tlsx`, terminals, +pipes and descriptors never warned as world-readable. Proxy mode: trusted peers, https and the client hop; throttling on the forwarded client and IPv6 /64; an unbound browser login with the cookie's attributes, the cookie opening endpoints (beside Basic auth) until logout revokes it, cookie @@ -371,7 +385,12 @@ viewer's own form, not a loop, its token kept out of storage and the URL; a profile blocking every cookie signing in on the viewer, its proofs asking for no cookie, streaming with a bearer, sign-out revoking the token, and a reload's `401` asking again; `token --unbound` and curl through the proxy; direct peers -and plaintext-forwarded requests `403`. The existing scripts keep passing. +and plaintext-forwarded requests `403`. `test/preset-test.sh --auto` (ports +15897-15898): an aggregator and an edge given one password on `/dev/fd/3` +deliver, another password is refused, both survive `SIGHUP` and deliver again, +and the password is in no file, argv or environment; under `script(1)` a +listener's generated password shows on its terminal and not in its log, and +an edge and `lw auth token` log in with it. The existing scripts keep passing. ## Not Implemented diff --git a/doc/security.md b/doc/security.md index 9e0347c..0e965b0 100644 --- a/doc/security.md +++ b/doc/security.md @@ -159,8 +159,8 @@ allow_patterns = [] # mtls node_binding = "force" # chain sources only credentials_file = "" # scram listeners token_lifetime_ms = 0 # scram, HTTP listeners; 10 s to 24 h, 0 = 15 minutes -username = "" # scram dialers -password_file = "" # scram dialers +username = "" # scram dialers, or a listener's one user +password_file = "" # scram, with username trusted_proxies = [] # scram http sink behind a TLS-terminating proxy ``` @@ -181,7 +181,9 @@ Options by where they apply, each with its default: them yourself. - `scram` listeners - `credentials_file`: verifiers written by - [`lw auth add-user`](cli.md#lw-auth). + [`lw auth add-user`](cli.md#lw-auth); or, in its place, `username` and + `password_file`, see + [One user without a credentials file](#one-user-without-a-credentials-file). - `token_lifetime_ms` (15 minutes): bearer token lifetime, 10 s to 24 h; the `http` sink and `http_chain` source only. - `trusted_proxies` (`[]`): addresses or CIDRs of the reverse proxies that @@ -190,7 +192,7 @@ Options by where they apply, each with its default: - `scram` dialers - `username`: user to log in as. - `password_file`: file holding the password; one trailing line break is - trimmed. + trimmed. `/dev/fd/N`, a pipe or a terminal are read once, as on listeners. **Roles by plugin**, and what the block decides: @@ -306,9 +308,10 @@ Misconfiguration fails at plugin construction, before the pipeline starts: type forgotten - a key of the other method: `allow` or `allow_patterns` under `scram`, a `scram` key under `mtls` -- a `scram` listener without a loadable `credentials_file` (see - [Credentials file](#credentials-file)), with `username` or `password_file`, - with a `token_lifetime_ms` outside 10 s to 24 h or on a TCP listener, or with +- a `scram` listener with neither a loadable `credentials_file` (see + [Credentials file](#credentials-file)) nor `username` and `password_file`, + or with both, with only one of `username` and `password_file`, with a + `token_lifetime_ms` outside 10 s to 24 h or on a TCP listener, or with `identity` but no `tls.client_auth` - a `scram` dialer without `username` and `password_file`, with `credentials_file`, `token_lifetime_ms`, `trusted_proxies` or `identity` @@ -435,8 +438,8 @@ when it is constructed, so startup and every reload repeat them: - an `allow_patterns` entry not anchored at both ends of every alternative (`^a|b$` admits `a…` and `…b`) - a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every - local user can read - (once per path per process) + local user can read (once per path per process); terminals, pipes and + descriptors hold nothing at rest and are not reported - an `acl` entry with host bits set (`10.0.0.1/8`), naming the network it matches; an `acl.allow` entry of a whole family (`0.0.0.0/0`, `::/0`); a wildcard listener whose `acl` has `deny` but no `allow`; an `acl.proxy_from` @@ -477,6 +480,18 @@ lw tls cert --ca-dir /etc/logwisp/pki --name agg.example.org --server lw tls cert --ca-dir /etc/logwisp/pki --name edge-01 --client ``` +A temporary pipe needs no file at all: a self-signed certificate and +[one user](#one-user-without-a-credentials-file) whose password is typed, or +generated, at startup. Both last until lw exits. + +```bash +# receiver: Enter at the prompt generates the password; copy it, and the +# pin_sha256 of the WARN line (-q would hide it) +lw -p aggregator:listen=0.0.0.0:9000,tls=self,user=pipe,format=raw,allow=SENDER/32 +# sender: asked for the password +cmd | lw -p edge:to=RECEIVER:9000,pin=sha256//PIN,user=pipe +``` + ## Enabling mTLS ### 1. Generate a CA and certificates @@ -702,6 +717,50 @@ only when the plugin is built and `auto_reload` does not watch it: send `SIGHUP` after every change. Use one file per listener when listeners admit different users. +### One user without a credentials file + +`username` and `password_file` on a listener replace `credentials_file`: one +user whose verifier is derived in memory when the plugin is built (one +Argon2id at startup and each reload, never per login), with a decoy key drawn +once per process. Nothing is written. It suits a temporary pipe or a single +edge. `password_file` is read by what it names, on either side: + +- A regular file: at every build, so `SIGHUP` rotates it. A systemd + `LoadCredential=` file sits on tmpfs. +- `/dev/fd/N`, a pipe, a FIFO or another device: once, at startup, then kept + in memory for reloads; restart lw to change it. A reload naming a new one + fails and the running pipelines stay: a console source may be reading the + same input, and the read would hold the reload and its signals. + - `/dev/fd/N` is the descriptor lw was given (`3< <(pass show lw/pipe)`), + read and closed; it needs no fdescfs on FreeBSD. A descriptor lw opened + itself is refused. + - One such source holds one user's password. `/dev/stdin` would take the + console source's input. +- A terminal: `/dev/tty`, the presets' default with `user`, asked once, at + startup, without echo. lw asks whatever file is a terminal once opened, so + `/dev/stdin` holding a pipe is read on FreeBSD too, where stat calls it a + device. + - On a listener, Enter alone generates a 130-bit password and shows it once + on that terminal, never in the log or on stderr: logs persist, are + forwarded, and stderr is often a file. + - With no terminal (systemd, cron) the plugin fails. `lw ... &` stops on + `SIGTTIN` at the prompt. + +Trade-offs: + +- The password lives as long as the process, as dialers' passwords do: + `tcp_chain` logs in again on every reconnect and `http_chain` at every token + renewal, so it cannot be single-use. +- A generated password stays in terminal scrollback, tmux or screen logs, and + the clipboard that carries it. A typed one may be as short as 8 bytes; on an + exposed listener prefer the generated one and `acl.allow`. +- It never goes in argv or the environment: `/proc/PID/cmdline` is + world-readable, and `--dump` keeps only the path. +- Each build draws a new salt, which, like a rotation, shows a prober that the + name exists. +- Older bash and zsh back a here-string (`<<<`) with a temporary file; prefer + `3< <(...)`. + ### Rollout, rotation and revocation - **Rollout.** Turning `scram` on cuts off every dialer of that listener that diff --git a/doc/sinks.md b/doc/sinks.md index 0671c79..9279b61 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -173,11 +173,12 @@ Options, each as type and default: - `replay_lines` (int, `0`; `1000` from the `serve` preset): the last entries a new stream gets after its `connected` event, before the live ones, so a browser opened late sees a backlog; `0` = none. -- `login_page` (bool, `false`): `scram` in proxy mode only (an error - elsewhere): the browser login page at `/auth/login`. -- `viewer_page` (bool, `false`): `scram` in proxy mode only (an error - elsewhere): the live viewer at `/auth/view`; needs `login_page`. Without - auth, or under `mtls`, the viewer is always served. +- `login_page` (bool, `false`): only for `scram` behind + `auth.trusted_proxies`, where browsers log in (an error elsewhere): the + login page at `/auth/login`. +- `viewer_page` (bool, `false`): the same, the live viewer at `/auth/view` + after a login; needs `login_page`. Without auth, or under `mtls`, the + viewer is always at `/`, with neither key. - `tls` (table): listener TLS; see [Security](security.md). - `auth` (table): client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block). diff --git a/doc/sources.md b/doc/sources.md index 74cc98f..b38512a 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -227,7 +227,8 @@ Options, each as type and default: [Security](security.md#the-acl-block). For passwords instead of certificates, the `auth` block takes `type = "scram"` -and a `credentials_file`, and `client_auth` becomes optional; see +and a `credentials_file`, or one `username` with its `password_file`, and +`client_auth` becomes optional; see [Password Authentication](security.md#password-authentication-scram). **Behaviour** diff --git a/doc/todo.md b/doc/todo.md index 4b06732..43695be 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -102,7 +102,9 @@ What exists: auth and acl tables, sources, sinks and presets, as one JSON document. - `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions, layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal, - ConfirmDialog, scrolling, mouse. `github.com/lixenwraith/color`: RGB, + ConfirmDialog, scrolling, mouse; themed Radio, Field, TextInput, + OptionList and menus, Frame, Rule, Wires and Window, which `lw --tui` + draws with. `github.com/lixenwraith/color`: RGB, blending, RGBTo256, RGBTo16. Steps: @@ -112,7 +114,9 @@ Steps: 4. `lixenwraith/terminal` v0.2.0: done: 16 and no color, the terminal's default colors, `/dev/tty`, bracketed paste, themed form controls. Still optional, in `color`: export the 16 xterm reference shades (RGBTo16 maps - the site's hues badly, so the 16-color tier is chosen by hand). + the site's hues badly, so the 16-color tier is chosen by hand). v0.3.0: + radio buttons, narrow form fields and option lists, menus, frames, rules, + wires and windows, moved from `lw --tui`. 5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)). 6. WebAssembly: done. `make wasm` builds `bin/lwconf.wasm`, version stamped, beside the toolchain's `wasm_exec.js`, and prints both checksums: about diff --git a/go.mod b/go.mod index cd43cb8..5f9505f 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/lixenwraith/logwisp -go 1.27.1 +go 1.27.2 require ( github.com/lixenwraith/auth v0.5.1 github.com/lixenwraith/color v0.1.3 github.com/lixenwraith/config v0.2.2 github.com/lixenwraith/log v0.1.11 - github.com/lixenwraith/terminal v0.2.0 + github.com/lixenwraith/terminal v0.2.1 github.com/lixenwraith/toml v0.1.3 golang.org/x/term v0.47.0 ) diff --git a/go.sum b/go.sum index 109d5e1..ddaef54 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,10 @@ github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZ github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg= github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKhU= github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg= -github.com/lixenwraith/terminal v0.2.0 h1:Xwsg8LoAeV7E//JLIFmEBTTV9e03jpze3EUAOdqS6P8= -github.com/lixenwraith/terminal v0.2.0/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44= +github.com/lixenwraith/terminal v0.2.1-0.20261010134811-bbb83d6c20df h1:HPfMYYSfNva8cMlgyTtZntu3oUwNKXW48rCwpmTEPoU= +github.com/lixenwraith/terminal v0.2.1-0.20261010134811-bbb83d6c20df/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44= +github.com/lixenwraith/terminal v0.2.1 h1:KsMfmxrRicBFWMdXJIEJ3cpLWVw6IoNF8GwgsxbD3kQ= +github.com/lixenwraith/terminal v0.2.1/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY= github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI= github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0= golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= diff --git a/internal/authz/authz.go b/internal/authz/authz.go index 984cb93..3c9babe 100644 --- a/internal/authz/authz.go +++ b/internal/authz/authz.go @@ -448,7 +448,7 @@ func (p *Policy) Enabled() bool { return p != nil } func (p *Policy) NeedsLogin() bool { return p != nil && p.listener != nil } // Unrestricted reports whether an mtls policy admits any identity the CA -// vouches for. Under scram the credentials file is the allow list. +// vouches for. Under scram the users are the allow list. func (p *Policy) Unrestricted() bool { return p != nil && p.method == MethodMTLS && len(p.allow) == 0 && len(p.patterns) == 0 } diff --git a/internal/authz/authz_test.go b/internal/authz/authz_test.go index 3fc2f86..36d1ca9 100644 --- a/internal/authz/authz_test.go +++ b/internal/authz/authz_test.go @@ -139,9 +139,10 @@ func TestNewValidation(t *testing.T) { {"dialer skips verify", &config.AuthOptions{Type: MethodMTLS}, &tls.Config{InsecureSkipVerify: true}, config.Dialer, TCP, "insecure_skip_verify"}, {"mtls with a password", &config.AuthOptions{Type: MethodMTLS, PasswordFile: pw}, &tls.Config{}, config.Dialer, TCP, "password_file applies only"}, {"scram without tls", listener(config.AuthOptions{}), nil, config.Listener, TCP, "requires tls.enabled"}, - {"scram without credentials", scram(config.AuthOptions{}), f.serverTLS, config.Listener, TCP, "requires credentials_file"}, - {"scram with an allow list", listener(config.AuthOptions{Allow: []string{"edge-01"}}), f.serverTLS, config.Listener, TCP, "credentials file is the allow list"}, - {"scram listener with a password", listener(config.AuthOptions{Username: "edge-01"}), f.serverTLS, config.Listener, TCP, "only to dialers"}, + {"scram without users", scram(config.AuthOptions{}), f.serverTLS, config.Listener, TCP, "requires credentials_file, or username and password_file"}, + {"scram with an allow list", listener(config.AuthOptions{Allow: []string{"edge-01"}}), f.serverTLS, config.Listener, TCP, "the users are the allow list"}, + {"credentials and a user", listener(config.AuthOptions{Username: "edge-01"}), f.serverTLS, config.Listener, TCP, "not both"}, + {"user without password_file", scram(config.AuthOptions{Username: "edge-01"}), f.serverTLS, config.Listener, TCP, "go together"}, {"scram certificate binding without client_auth", listener(config.AuthOptions{Identity: "cn"}), f.serverTLS, config.Listener, TCP, "requires tls.client_auth"}, {"token lifetime on tcp", listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, config.Listener, TCP, "only to HTTP listeners"}, {"token lifetime too short", listener(config.AuthOptions{TokenLifetimeMS: 900}), f.serverTLS, config.Listener, HTTP, "must be from 10000 to 86400000"}, @@ -176,8 +177,10 @@ func TestNewValidation(t *testing.T) { } } } - if _, err := New(listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, nil, config.Listener, HTTP); err != nil { - t.Fatalf("scram listener rejected: %v", err) + for _, o := range []*config.AuthOptions{listener(config.AuthOptions{TokenLifetimeMS: 60000}), scram(config.AuthOptions{Username: "edge-01", PasswordFile: pw})} { + if _, err := New(o, f.serverTLS, nil, config.Listener, HTTP); err != nil { + t.Fatalf("scram listener rejected: %v", err) + } } // Behind proxies TLS ends at the proxy, so the hop may be plaintext if _, err := New(listener(config.AuthOptions{TrustedProxies: proxies}), nil, nil, config.Listener, HTTP); err != nil { diff --git a/internal/authz/scram.go b/internal/authz/scram.go index 62bdd74..b132036 100644 --- a/internal/authz/scram.go +++ b/internal/authz/scram.go @@ -16,8 +16,11 @@ import ( "net" "net/http" "os" + "strconv" "strings" + "sync" "sync/atomic" + "syscall" "time" "github.com/lixenwraith/logwisp/internal/chain" @@ -27,6 +30,7 @@ import ( "github.com/lixenwraith/auth" "github.com/lixenwraith/toml" + "golang.org/x/term" ) const ( @@ -162,10 +166,87 @@ func (c *Credentials) Marshal() ([]byte, error) { return append([]byte(header), body...), nil } -// ReadPassword reads a password file, trimming one trailing line break so a -// file from an editor and one from `lw auth add-user` agree. -func ReadPassword(path string) (string, error) { - data, err := os.ReadFile(path) +// processDecoy is the decoy key of listeners without a credentials file: +// one per process, so reloads keep unknown-user challenges stable +var processDecoy = sync.OnceValue(func() []byte { key := make([]byte, 32); rand.Read(key); return key }) + +// oneUser is a listener's only user, derived in memory at the dialers' floor +// cost, the default outside tests +func oneUser(user, path string) (*Credentials, error) { + pw, err := ReadPassword(path, user, true) + if err != nil { + return nil, err + } + cred, err := auth.NewCredential(user, pw, auth.WithTime(minArgonTime), auth.WithMemory(minArgonMemory)) + if err != nil { + return nil, fmt.Errorf("auth: user %q: %w", user, err) + } + return &Credentials{DecoyKey: processDecoy(), Users: []*auth.Credential{cred}}, nil +} + +// readOnce keeps what a reload cannot read again, by path and user, and +// whether a terminal was asked: one may be asked for several users +var readOnce = struct { + sync.Mutex + m map[[2]string]asked +}{m: map[[2]string]asked{}} + +type asked struct { + pw string + tty bool +} + +// startupOver is set once lw's first service is built +var startupOver atomic.Bool + +// EndStartup stops reading descriptors, pipes, FIFOs and terminals: at a +// reload a console source may be reading the same one, and a read would hold +// the reload and its signals. What was read before stays for reloads. +func EndStartup() { startupOver.Store(true) } + +// ReadPassword reads user's password from path, trimming one trailing line +// break so a file from an editor and one from `lw auth add-user` agree. A +// regular file is read each time, for rotation by reload; /dev/fd/N, a pipe, +// a FIFO or another device once, before EndStartup; a terminal is asked once, +// where generate lets Enter alone draw the password. +func ReadPassword(path, user string, generate bool) (string, error) { + fd, inherited := descriptor(path) + if fi, err := os.Stat(path); !inherited && (err != nil || fi.Mode().IsRegular()) { + data, err := os.ReadFile(path) + return checkPassword(path, data, err) + } + readOnce.Lock() + defer readOnce.Unlock() + key := [2]string{path, user} + if a, ok := readOnce.m[key]; ok { + return a.pw, nil + } + if startupOver.Load() { + return "", fmt.Errorf("auth: password_file %s is read only at startup; name a regular file or restart lw", path) + } + for k, a := range readOnce.m { + if k[0] == path && !a.tty { + return "", fmt.Errorf("auth: password_file %s was read for user %q and holds one password", path, k[1]) + } + } + var ( + data []byte + tty bool + err error + ) + if inherited { + data, err = readDescriptor(fd, path) + } else { + data, tty, err = readDevice(path, user, generate) + } + pw, err := checkPassword(path, data, err) + if err == nil { + readOnce.m[key] = asked{pw, tty} + } + return pw, err +} + +func checkPassword(path string, data []byte, err error) (string, error) { if err != nil { return "", fmt.Errorf("auth: password_file: %w", err) } @@ -176,6 +257,70 @@ func ReadPassword(path string) (string, error) { return pw, nil } +// descriptor is N of /dev/fd/N, which ReadPassword closes: the path then +// names another file. 0-2 stay open, or stdio would go to the next file. +func descriptor(path string) (int, bool) { + n, ok := strings.CutPrefix(path, "/dev/fd/") + fd, err := strconv.Atoi(n) + return fd, ok && err == nil && fd > 2 +} + +// readDescriptor reads an inherited descriptor to its end and closes it, +// which needs no fdescfs on FreeBSD. Go opens its own close-on-exec, so a +// descriptor with the flag is lw's: reading or closing it would break lw. +func readDescriptor(fd int, path string) ([]byte, error) { + flags, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFD, 0) + if errno != 0 || flags&syscall.FD_CLOEXEC != 0 { + return nil, fmt.Errorf("%s was not passed to lw: redirect it, as in %d= 0 && j != i { + return fmt.Errorf("pipeline %q: a name no other pipeline has", name) + } + return nil +} + func (c *Composition) RemovePipeline(i int) error { if _, err := c.pipeline(i); err != nil { return err @@ -233,6 +254,52 @@ func (c *Composition) Add(pi int, role, typ string) (Node, error) { return n, nil } +// Retype gives a source or sink another type of its role, in its place: a +// chosen id stays and an automatic one follows the type. Options of the same +// name and kind carry over to a type on the same network side, where they +// mean the same; across sides host, port, tls and auth do not. +func (c *Composition) Retype(pi int, n Node, typ string) (Node, error) { + p, err := c.pipeline(pi) + if err != nil { + return Node{}, err + } + to, ok := config.LookupPlugin(n.Role, typ) + if !ok { + return Node{}, fmt.Errorf("unknown %s type %q", n.Role, typ) + } + var id, old *string + var opts *map[string]any + var ids []string + switch i := index(p, n); { + case i >= 0 && n.Role == "source": + id, old, opts = &p.PluginSources[i].ID, &p.PluginSources[i].Type, &p.PluginSources[i].Config + for _, s := range p.PluginSources { + ids = append(ids, s.ID) + } + case i >= 0 && n.Role == "sink": + id, old, opts = &p.PluginSinks[i].ID, &p.PluginSinks[i].Type, &p.PluginSinks[i].Config + for _, s := range p.PluginSinks { + ids = append(ids, s.ID) + } + default: + return Node{}, n.missing() + } + from, _ := config.LookupPlugin(n.Role, *old) + kept := map[string]any{} + for _, k := range from.Keys() { + same := func(t config.Key) bool { return t.Name == k.Name && t.Kind == k.Kind } + if v, set := (*opts)[k.Name]; set && from.Side == to.Side && slices.ContainsFunc(to.Keys(), same) { + kept[k.Name] = v + } + } + suffix, auto := strings.CutPrefix(*id, *old) + if n, err := strconv.Atoi(strings.TrimPrefix(suffix, "_")); auto && (suffix == "" || suffix[0] == '_' && err == nil && n > 1) { + *id = config.FreeID(typ, slices.DeleteFunc(ids, func(s string) bool { return s == *id })) + } + *old, *opts = typ, kept + return Node{Role: n.Role, ID: *id}, nil +} + // Remove deletes a source, sink or filter, or turns a flow stage off func (c *Composition) Remove(pi int, n Node) error { p, err := c.pipeline(pi) @@ -353,22 +420,22 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er return plugin(p.PluginSinks[i].Type, &p.PluginSinks[i].Config) case n.Role == "filters": filter := &p.Flow.Filters[i] - if err := editStage(reflect.ValueOf(&filter).Elem(), change); err != nil { + if err := editStage(reflect.ValueOf(&filter).Elem(), "", change); err != nil { return err } p.Flow.Filters[i] = *filter return nil } if stage, ok := stages(p.Flow)[n.Role]; ok { - return editStage(stage, change) + return editStage(stage, n.Role, change) } return n.missing() } // editStage edits a stage's keys as a spec sets them; the table they scan into -// replaces the stage, which turns on -func editStage(stage reflect.Value, change func(map[string]any) error) error { - m := map[string]any{} +// replaces the stage, which turns on as a spec turns it on +func editStage(stage reflect.Value, key string, change func(map[string]any) error) error { + m := config.StageOn(key, nil) if !stage.IsNil() { m = config.Values(stage.Interface()) } diff --git a/internal/compose/compose_test.go b/internal/compose/compose_test.go index 8894077..a86bcf2 100644 --- a/internal/compose/compose_test.go +++ b/internal/compose/compose_test.go @@ -1,6 +1,7 @@ package compose import ( + "fmt" "os" "os/exec" "reflect" @@ -64,7 +65,7 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) { _, err = c.Add(0, "filters", "include") must(err) must(c.MoveFilter(0, 1, 0)) - must(c.Set(0, Node{Role: "rate_limit"}, "rate", "100")) + must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100")) must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000")) must(c.Set(0, Node{Role: "heartbeat"}, "enabled", "true")) must(c.AddPipeline("-b")) @@ -190,3 +191,71 @@ func TestUnnamedPartsTakeFreeIDs(t *testing.T) { t.Fatalf("%v\n%s", err, line) } } + +// Retype keeps the part's place and a chosen id, renames an automatic one, +// and carries the options of the same name and kind only within a side +func TestRetypeKeepsThePlace(t *testing.T) { + c := &Composition{} + if err := c.AddPipeline("p"); err != nil { + t.Fatal(err) + } + for _, typ := range []string{"tcp", "http", "tcp"} { + if _, err := c.Add(0, "sink", typ); err != nil { + t.Fatal(err) + } + } + tcp2, chosen := Node{Role: "sink", ID: "tcp_2"}, Node{Role: "sink", ID: "web"} + c.Pipelines[0].PluginSinks[1].ID = chosen.ID + for _, err := range []error{c.Set(0, tcp2, "port", "9000"), c.Set(0, tcp2, "keep_alive", "false"), c.Set(0, chosen, "port", "80")} { + if err != nil { + t.Fatal(err) + } + } + got, err := c.Retype(0, tcp2, "http") + if err != nil || got.ID != "http" { + t.Fatalf("%+v %v", got, err) + } + if _, err := c.Retype(0, chosen, "tcp_chain"); err != nil { + t.Fatal(err) + } + var parts []string + for _, s := range c.Pipelines[0].PluginSinks { + parts = append(parts, fmt.Sprintf("%s=%s %v", s.ID, s.Type, s.Config)) + } + if want := []string{"tcp=tcp map[]", "web=tcp_chain map[]", "http=http map[port:9000]"}; !slices.Equal(parts, want) { + t.Fatalf("%q, want %q", parts, want) + } +} + +// A pipeline takes a name no other has, renamed as when added +func TestPipelineNamesAreUnique(t *testing.T) { + c := &Composition{} + for _, name := range []string{"a", "b"} { + if err := c.AddPipeline(name); err != nil { + t.Fatal(err) + } + } + if c.AddPipeline("a") == nil || c.RenamePipeline(1, "a") == nil || c.RenamePipeline(1, "") == nil { + t.Fatal("took a name another pipeline has") + } + if err := c.RenamePipeline(1, "b"); err != nil || c.RenamePipeline(1, "c") != nil || c.Pipelines[1].Name != "c" { + t.Fatalf("%v, %+v", err, c.Pipelines) + } +} + +// A stage an edit turns on does what its spec would: a rate limit drops what +// passes its rate, a heartbeat beats +func TestAStageTurnsOnAsItsSpecDoes(t *testing.T) { + c := &Composition{} + if err := c.AddPipeline("p"); err != nil { + t.Fatal(err) + } + for _, err := range []error{c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500")} { + if err != nil { + t.Fatal(err) + } + } + if f := c.Pipelines[0].Flow; f.RateLimit.Policy != "drop" || !f.Heartbeat.Enabled { + t.Fatalf("%+v %+v", f.RateLimit, f.Heartbeat) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 32e4db1..0bb5208 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -103,8 +103,8 @@ const ( // RateLimitConfig defines the configuration for pipeline-level rate limiting type RateLimitConfig struct { - Rate float64 `toml:"rate" lw:"min=0,zero=off" help:"entries a second"` - Burst float64 `toml:"burst" lw:"min=0,zero=the rate" help:"entries at once"` + EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second the limit lets through"` + BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries let through at once before the rate applies"` Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop entries over the rate, or pass them"` MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries"` } @@ -262,8 +262,8 @@ type HTTPSinkOptions struct { WriteTimeoutMS int64 `toml:"write_timeout_ms" lw:"min=0,zero=none" help:"milliseconds an event write may take"` MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"clients at once"` ReplayLines int64 `toml:"replay_lines" lw:"min=0,zero=none" help:"recent entries a new stream gets first"` - LoginPage bool `toml:"login_page" help:"serve the login page; scram behind auth.trusted_proxies"` - ViewerPage bool `toml:"viewer_page" help:"serve the viewer to users logged in; needs login_page"` + LoginPage bool `toml:"login_page" help:"scram behind auth.trusted_proxies only: a login page at /auth/login"` + ViewerPage bool `toml:"viewer_page" help:"scram behind auth.trusted_proxies only: the viewer after a login; without scram it is always at /"` Auth *AuthOptions `toml:"auth"` ACL *ACLOptions `toml:"acl"` } @@ -307,9 +307,9 @@ type HTTPChainSinkOptions struct { // AuthOptions selects how a network plugin authenticates its peer. It sits // beside `tls`: TLS answers "is this channel private", auth answers "may this -// peer do this". Listeners verify (mtls: certificate identity; scram: password -// via credentials_file); dialers prove themselves (scram) or pin the server -// (mtls). AuthOptions.Check holds the rules of each side. +// peer do this". Listeners verify (mtls: certificate identity; scram: the +// credentials_file's users, or one username and password_file); dialers prove +// themselves (scram) or pin the server (mtls). Check holds each side's rules. type AuthOptions struct { Type string `toml:"type" default:"none" lw:"enum=none|mtls|scram" help:"how peers authenticate"` Identity string `toml:"identity" lw:"enum=cn|san_dns|san_uri|san_email,zero=cn under mtls; no binding under scram" help:"certificate field naming the peer; under scram it must equal the user"` @@ -321,8 +321,8 @@ type AuthOptions struct { // leaves room for renewal ahead of expiry TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"` TrustedProxies []string `toml:"trusted_proxies" lw:"listener,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"` - Username string `toml:"username" lw:"dialer" help:"scram: the user this dialer logs in as"` - PasswordFile string `toml:"password_file" lw:"dialer,hint=file" help:"scram: file holding the user's password"` + Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"` + PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"` } // ACLOptions is a listener's address rules, entries being addresses or CIDRs: diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go index d37941c..2c1b167 100644 --- a/internal/config/loader_test.go +++ b/internal/config/loader_test.go @@ -340,7 +340,7 @@ func TestFileStringIsOneListEntry(t *testing.T) { // its own; other sinks do not take the key. func TestTopLevelColorIsEachConsoleSinksDefault(t *testing.T) { isolateConfig(t) - m, err := Load(Args{Specs: specs("sink", "console", "sink", "console,color=never", "sink", "null"), Overrides: []string{"--color=always"}}) + m, err := Load(Args{Specs: specs("sink", "console", "sink", "console:color=never", "sink", "null"), Overrides: []string{"--color=always"}}) if err != nil { t.Fatal(err) } diff --git a/internal/config/preset.go b/internal/config/preset.go index b704d68..39fb52e 100644 --- a/internal/config/preset.go +++ b/internal/config/preset.go @@ -13,7 +13,7 @@ import ( "strings" ) -// Preset is a pipeline built from a few keys: --preset NAME,key=value starts +// Preset is a pipeline built from a few keys: --preset NAME:key=value starts // a pipeline with it. Later flags add sources, sinks and filters, or replace // its format, rate limit and heartbeat. type Preset struct { @@ -64,6 +64,10 @@ var ( {Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"}, } transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"} + userParams = []PresetParam{ + {Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"}, + {Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"}, + } ) var presets = []Preset{ @@ -88,8 +92,9 @@ var presets = []Preset{ {Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"}, {Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"}, {Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"}, + }, userParams, []PresetParam{ {Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"}, - {Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"}, + {Name: "viewer", Default: "false", Help: "true: the login page and viewer, behind proxy; without user or users the viewer is always on"}, }, aclParams, tlsParams), func(p *PipelineConfig, v map[string]string, isDir IsDir) error { sources, err := pathOrStdin(v, isDir) @@ -101,14 +106,18 @@ var presets = []Preset{ if err != nil { return err } - if v["users"] != "" { - sink["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]} + auth, err := scramAuth(v) + switch { + case err != nil: + return err + case auth != nil && v["tls"] == "off" && v["proxy"] == "": + return fmt.Errorf("user and users need tls (self, issuer or files), or proxy") + case auth != nil: + sink["auth"] = auth + case v["proxy"] != "": + return fmt.Errorf("proxy needs user or users: browsers log in with SCRAM") } if v["proxy"] != "" { - auth, ok := sink["auth"].(map[string]any) - if !ok { - return fmt.Errorf("proxy needs users: browsers log in with SCRAM") - } auth["trusted_proxies"] = list(v["proxy"]) } switch v["viewer"] { @@ -127,18 +136,17 @@ var presets = []Preset{ return nil }}, {"edge", "Forward files or standard input to an aggregator over TLS, authenticated", - []PresetParam{pathParam, fromParam, + slices.Concat([]PresetParam{pathParam, fromParam, {Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"}, transportParam, {Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"}, {Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"}, {Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"}, - {Name: "user", Help: "SCRAM user (lw auth add-user on the aggregator)"}, - {Name: "password_file", Help: "file holding the SCRAM password"}, + }, userParams, []PresetParam{ {Name: "cert", Help: "client certificate file, for an aggregator with client_ca"}, {Name: "key", Help: "client key file"}, {Name: "node", Help: "origin label; default: this host's name"}, - }, + }), func(p *PipelineConfig, v map[string]string, isDir IsDir) error { sources, err := pathOrStdin(v, isDir) if err != nil { @@ -159,16 +167,15 @@ var presets = []Preset{ tls[option] = v[key] } } - var auth map[string]any + auth, err := scramAuth(v) switch { - case v["user"] != "" && v["password_file"] != "": - auth = map[string]any{"type": "scram", "username": v["user"], "password_file": v["password_file"]} - case v["user"] != "" || v["password_file"] != "": - return fmt.Errorf("user and password_file go together") + case err != nil: + return err + case auth != nil: case v["cert"] != "" && v["key"] != "": auth = map[string]any{"type": "mtls"} default: - return fmt.Errorf("edge needs user and password_file, or cert and key: it never sends unauthenticated") + return fmt.Errorf("edge needs user, or cert and key: it never sends unauthenticated") } sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth} if v["node"] != "" { @@ -183,6 +190,7 @@ var presets = []Preset{ transportParam, {Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"}, {Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"}, + }, userParams, []PresetParam{ {Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"}, {Name: "out", Help: "directory for the received entries; default: standard output"}, formatParam("json"), @@ -203,13 +211,16 @@ var presets = []Preset{ tls := src["tls"].(map[string]any) tls["client_auth"], tls["client_ca_file"] = true, v["client_ca"] } + auth, err := scramAuth(v) switch { - case v["users"] != "": - src["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]} + case err != nil: + return err + case auth != nil: + src["auth"] = auth case v["client_ca"] != "": src["auth"] = map[string]any{"type": "mtls"} default: - return fmt.Errorf("aggregator needs users or client_ca: it never receives unauthenticated") + return fmt.Errorf("aggregator needs user, users or client_ca: it never receives unauthenticated") } p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src}) p.PluginSinks = pipeDefault("").PluginSinks @@ -356,6 +367,22 @@ func listener(v map[string]string, def string) (map[string]any, error) { return opts, nil } +// scramAuth is the scram table of user, whose password the terminal asks for +// unless password_file names it, or of a listener's users file; nil without +func scramAuth(v map[string]string) (map[string]any, error) { + switch { + case v["user"] != "" && v["users"] != "": + return nil, fmt.Errorf("user or users, not both") + case v["user"] != "": + return map[string]any{"type": "scram", "username": v["user"], "password_file": cmp.Or(v["password_file"], "/dev/tty")}, nil + case v["password_file"] != "": + return nil, fmt.Errorf("password_file needs user") + case v["users"] != "": + return map[string]any{"type": "scram", "credentials_file": v["users"]}, nil + } + return nil, nil +} + func splitAddr(key, addr string) (string, int64, error) { host, p, err := net.SplitHostPort(addr) port, perr := strconv.ParseUint(p, 10, 16) diff --git a/internal/config/preset_test.go b/internal/config/preset_test.go index 8e04a9d..72ed093 100644 --- a/internal/config/preset_test.go +++ b/internal/config/preset_test.go @@ -11,7 +11,7 @@ import ( // sinks add to its own, a later format replaces its own. func TestPresetStartsAndNamesItsPipeline(t *testing.T) { isolateConfig(t) - got := loadPipelines(t, "preset", "tail,path=/var/log/app.log", "sink", "null", "format", "json", + got := loadPipelines(t, "preset", "tail:path=/var/log/app.log", "sink", "null", "format", "json", "pipeline", "mine", "preset", "pipe") want := []PipelineConfig{{ Name: "tail", @@ -40,21 +40,24 @@ func TestPresetStartsAndNamesItsPipeline(t *testing.T) { func TestPresetKeysAreChecked(t *testing.T) { isolateConfig(t) for spec, want := range map[string]string{ - "nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`, - "tail,path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`, - "tail": "preset tail needs path", - "tail,path=x,format.type=json": "preset keys do not nest", - "tail,path=x,path=y": `key "path" takes one value`, - "serve,listen=8080": `listen "8080": want HOST:PORT`, - "serve,tls=self,cert=c": "cert does not apply to tls=self", - "serve,tls=files,cert=c": "tls=files needs key", - "serve,viewer=yes": `viewer "yes"`, - "serve,proxy=127.0.0.1": "proxy needs users", - "serve,viewer=true": "viewer=true needs proxy", - "edge,to=agg:9000": "it never sends unauthenticated", - "edge,to=agg:9000,user=u": "user and password_file go together", - "edge,to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`, - "aggregator,users=u,tls=off": "the aggregator always serves TLS", + "nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`, + "tail:path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`, + "tail": "preset tail needs path", + "tail:path=x,format.type=json": "preset keys do not nest", + "tail:path=x,path=y": `key "path" takes one value`, + "serve:listen=8080": `listen "8080": want HOST:PORT`, + "serve:tls=self,cert=c": "cert does not apply to tls=self", + "serve:tls=files,cert=c": "tls=files needs key", + "serve:viewer=yes": `viewer "yes"`, + "serve:proxy=127.0.0.1": "proxy needs user or users", + "serve:viewer=true": "viewer=true needs proxy", + "serve:user=u": "user and users need tls (self, issuer or files), or proxy", + "serve:tls=self,user=u,users=f": "user or users, not both", + "edge:to=agg:9000": "it never sends unauthenticated", + "edge:to=agg:9000,password_file=p": "password_file needs user", + "edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`, + "aggregator:users=u,tls=off": "the aggregator always serves TLS", + "aggregator:user=u,users=f": "user or users, not both", "aggregator": "it never receives unauthenticated", } { _, err := Load(Args{Specs: specs("preset", spec)}) @@ -64,6 +67,17 @@ func TestPresetKeysAreChecked(t *testing.T) { } } +// A preset user without password_file is asked for on the terminal +func TestPresetUserWithoutPasswordFileAsks(t *testing.T) { + p, err := ExpandPreset("edge", map[string]string{"to": "agg:9000", "user": "u"}, nil) + if err != nil { + t.Fatal(err) + } + if auth := p.PluginSinks[0].Config["auth"].(map[string]any); auth["password_file"] != "/dev/tty" { + t.Fatalf("auth = %v, want password_file /dev/tty", auth) + } +} + // A preset path is what isDir answers, a directory named with brackets too; // without isDir, off the target host, it is a directory when it ends in '/', // a pattern when it holds a glob, and otherwise ErrPathKind asks diff --git a/internal/config/schema_test.go b/internal/config/schema_test.go index 3ac0479..5175f3b 100644 --- a/internal/config/schema_test.go +++ b/internal/config/schema_test.go @@ -117,8 +117,8 @@ func TestValidateConfigNamesThePath(t *testing.T) { c.Pipelines[0].PluginSinks[0] = PluginSinkConfig{ID: "up", Type: "tcp_chain", Config: map[string]any{"host": "h", "port": int64(1), "tls": map[string]any{"enabled": true, "insecure_skip_verify": true}, "auth": map[string]any{"type": "mtls"}}} }, - "pipelines[0].flow.rate_limit.rate: must be a finite number": func(c *Config) { - c.Pipelines[0].Flow.RateLimit = &RateLimitConfig{Rate: math.Inf(1)} + "pipelines[0].flow.rate_limit.entries_per_second: must be a finite number": func(c *Config) { + c.Pipelines[0].Flow.RateLimit = &RateLimitConfig{EntriesPerSecond: math.Inf(1)} }, "color: must be one of auto, always, never, got \"\"": func(c *Config) { c.Color = "" diff --git a/internal/config/spec.go b/internal/config/spec.go index 0dd9cd8..7e0cb95 100644 --- a/internal/config/spec.go +++ b/internal/config/spec.go @@ -93,10 +93,9 @@ var specKinds = []specKind{ Fill(&format) return []string{typedSpec(&format)} }}, - // Naming a stage turns it on: the file defaults (pass, disabled) would not. {"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error { - p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"} - return Scan(opts, p.Flow.RateLimit) + p.Flow.RateLimit = &RateLimitConfig{} + return Scan(StageOn("rate_limit", opts), p.Flow.RateLimit) }, func(p *PipelineConfig) []string { if p.Flow.RateLimit == nil { return nil @@ -106,8 +105,8 @@ var specKinds = []specKind{ return []string{formatSpec("", Values(&rate))} }}, {"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error { - p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true} - return Scan(opts, p.Flow.Heartbeat) + p.Flow.Heartbeat = &HeartbeatConfig{} + return Scan(StageOn("heartbeat", opts), p.Flow.Heartbeat) }, func(p *PipelineConfig) []string { if p.Flow.Heartbeat == nil { return nil @@ -127,6 +126,18 @@ type pipelineSpec struct { value string } +// stageOn are the values naming a flow stage sets, in a spec or an engine +// edit: the file's defaults (pass, disabled) would leave it doing nothing +var stageOn = map[string]map[string]any{"rate_limit": {"policy": "drop"}, "heartbeat": {"enabled": true}} + +// StageOn returns opts over the values that turn a flow stage on +func StageOn(stage string, opts map[string]any) map[string]any { + on := map[string]any{} + maps.Copy(on, stageOn[stage]) + maps.Copy(on, opts) + return on +} + // Spec is one pipeline flag: its name without dashes, and its value. type Spec struct{ Flag, Value string } @@ -282,13 +293,10 @@ func typedSpec(stage any) string { return formatSpec(typ, m) } -// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a -// repeated key +// formatSpec writes TYPE[:key=value,...] or key=value,...: tables as dotted +// keys, lists as a repeated key func formatSpec(typ string, tables ...map[string]any) string { var parts []string - if typ != "" { - parts = append(parts, escape(typ)) - } var add func(prefix string, m map[string]any) add = func(prefix string, m map[string]any) { for _, k := range slices.Sorted(maps.Keys(m)) { @@ -307,7 +315,11 @@ func formatSpec(typ string, tables ...map[string]any) string { for _, m := range tables { add("", m) } - return strings.Join(parts, ",") + opts := strings.Join(parts, ",") + if typ == "" || opts == "" { + return typ + opts + } + return typ + ":" + opts } func text(v any) string { @@ -365,16 +377,20 @@ func useStdio(p *PipelineConfig) { } } -// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes -// a list, and a backslash escapes ',', '=' and '\'. Errors name the part. +// parseSpec reads TYPE[:key=value,...] or key=value,...: TYPE ends at the +// first ':', dotted keys nest, a repeated key makes a list, and a backslash +// escapes ',', '=' and '\'. Errors name the part. func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) { rest, more := spec, true if typed { - typ, rest, more = cutUnescaped(spec, ',') - if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq { + typ, rest, _ = strings.Cut(spec, ":") + more = rest != "" + if head, _, comma := cutUnescaped(typ, ','); comma && !strings.Contains(head, "=") { + return "", nil, fmt.Errorf("%s: TYPE ends at ':', as in %s:%s", spec, head, spec[len(head)+1:]) + } + if typ == "" || strings.ContainsAny(typ, "=,\\") { return "", nil, fmt.Errorf("%s: missing TYPE", spec) } - typ = unescape(typ) } opts = map[string]any{} for more { @@ -390,7 +406,7 @@ func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err er err = SetOption(opts, key, unescape(value)) } if err != nil { - return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err) + return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+":"+part, ":"), err) } } return typ, opts, nil diff --git a/internal/config/spec_test.go b/internal/config/spec_test.go index a0b1082..5dcaf12 100644 --- a/internal/config/spec_test.go +++ b/internal/config/spec_test.go @@ -36,13 +36,13 @@ func loadPipelines(t *testing.T, pairs ...string) []PipelineConfig { func TestPipelineSpecGrammar(t *testing.T) { isolateConfig(t) got := loadPipelines(t, - "source", `file,directory=/var/log/a\,b,pattern=*.log`, - "source", `file,id=app,directory=x\=y`, - "source", `file,directory=c:\\logs\\`, - "sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.enabled=true,tls.cert_file=c,tls.key_file=k", - "filter", "include,patterns=ERROR,patterns=WARN", - "filter", `exclude,patterns=\d{1\,3}`, - "format", `txt,timestamp_format=Jan 2\, 2006`, + "source", `file:directory=/var/log/a\,b,pattern=*.log`, + "source", `file:id=app,directory=x\=y`, + "source", `file:directory=c:\\logs\\`, + "sink", "http:port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.enabled=true,tls.cert_file=c,tls.key_file=k", + "filter", "include:patterns=ERROR,patterns=WARN", + "filter", `exclude:patterns=\d{1\,3}`, + "format", `txt:timestamp_format=Jan 2\, 2006`, "pipeline", "b", "source", "null", "sink", "null", "sink", "null", "heartbeat", "interval_ms=1000") want := []PipelineConfig{{ Name: "cli", @@ -76,13 +76,14 @@ func TestPipelineSpecGrammar(t *testing.T) { t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want) } for want, pairs := range map[string][]string{ - `--sink http,port: missing "="`: {"sink", "http,port"}, - `--source directory=/x: missing TYPE`: {"source", "directory=/x"}, - `--source file,type=x: TYPE already sets "type"`: {"source", "file,type=x"}, - `--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http,tls=1,tls.cert_file=c"}, - `--sink requires a value`: {"sink", ""}, - `--format txt: pipeline "cli" already has one`: {"format", "json", "format", "txt"}, - `--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"rate-limit", "rate=1,polcy=drop"}, + `--sink http:port: missing "="`: {"sink", "http:port"}, + `--source file,directory=/x: TYPE ends at ':', as in file:directory=/x`: {"source", "file,directory=/x"}, + `--source directory=/x: missing TYPE`: {"source", "directory=/x"}, + `--source file:type=x: TYPE already sets "type"`: {"source", "file:type=x"}, + `--sink http:tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http:tls=1,tls.cert_file=c"}, + `--sink requires a value`: {"sink", ""}, + `--format txt: pipeline "cli" already has one`: {"format", "json", "format", "txt"}, + `--rate-limit entries_per_second=1,polcy=drop: unknown key "polcy"`: {"rate-limit", "entries_per_second=1,polcy=drop"}, } { if _, err := Load(Args{Specs: specs(pairs...)}); err == nil || err.Error() != want { t.Errorf("%q: err = %v, want %s", pairs, err, want) @@ -95,7 +96,7 @@ func TestPipelineSpecGrammar(t *testing.T) { func TestSpecValueIsOneListEntry(t *testing.T) { isolateConfig(t) sink := loadPipelines(t, "source", "null", "sink", - `tcp,port=9000,tls.enabled=true,tls.self_signed=true,tls.client_auth=true,tls.client_ca_file=ca,`+ + `tcp:port=9000,tls.enabled=true,tls.self_signed=true,tls.client_auth=true,tls.client_ca_file=ca,`+ `auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0] var opts TCPSinkOptions if err := Scan(sink.Config, &opts); err != nil { @@ -114,12 +115,12 @@ func TestEnvironmentPipeline(t *testing.T) { for name, value := range map[string]string{ "LOGWISP_PIPELINE": "edge", "LOGWISP_SOURCE": "random", - "LOGWISP_SOURCE_10": "null,id=ten", - "LOGWISP_SOURCE_2": "null,id=two", + "LOGWISP_SOURCE_10": "null:id=ten", + "LOGWISP_SOURCE_2": "null:id=two", "LOGWISP_SINK_1": "console", "LOGWISP_FILTER": "", "LOGWISP_FORMAT_1": "raw", - "LOGWISP_RATE_LIMIT": "rate=10", + "LOGWISP_RATE_LIMIT": "entries_per_second=10", "LOGWISP_LOGGING_LEVEL": "debug", } { t.Setenv(name, value) @@ -136,7 +137,7 @@ func TestEnvironmentPipeline(t *testing.T) { p := cfg.Pipelines if len(p) != 1 || p[0].Name != "edge" || len(p[0].PluginSources) != 3 || p[0].PluginSources[1].ID != "two" || p[0].PluginSources[2].ID != "ten" || len(p[0].PluginSinks) != 1 || p[0].Flow.Filters != nil || p[0].Flow.Format != nil || - *p[0].Flow.RateLimit != (RateLimitConfig{Rate: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" { + *p[0].Flow.RateLimit != (RateLimitConfig{EntriesPerSecond: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" { t.Fatalf("environment pipeline: %+v %+v", p, cfg.Logging) } } @@ -174,7 +175,7 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) { // stdout, as a filter does; two pipelines cannot both read stdin. func TestSpecPipelinesDefaultToStdio(t *testing.T) { isolateConfig(t) - got := loadPipelines(t, "filter", "include,patterns=ERROR") + got := loadPipelines(t, "filter", "include:patterns=ERROR") if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" || len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" { t.Fatalf("pipeline: %+v", p) @@ -190,7 +191,7 @@ func TestSpecPipelinesDefaultToStdio(t *testing.T) { // removal fails the reload instead of keeping the removed file's values. func TestReloadKeepsSpecPipelines(t *testing.T) { isolateConfig(t) - m, err := Load(Args{Specs: specs("source", "random,special=true", "sink", "null")}) + m, err := Load(Args{Specs: specs("source", "random:special=true", "sink", "null")}) if err != nil { t.Fatal(err) } @@ -217,11 +218,11 @@ func TestReloadKeepsSpecPipelines(t *testing.T) { // Written specs build the pipelines a file would: each flow stage with its // defaults, so the rate limit keeps its file policy and the heartbeat its off func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) { - p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{Rate: 5}, Heartbeat: &HeartbeatConfig{}}, + p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5}, Heartbeat: &HeartbeatConfig{}}, PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}}, PluginSinks: []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}} got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil) - p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{Rate: 5, Policy: "pass"}, + p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"}, Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}} if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) { t.Fatalf("got %+v %v\nwant %+v", got, err, p) diff --git a/internal/config/validate.go b/internal/config/validate.go index 726c754..fdca93f 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -296,7 +296,7 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error { } return nil case len(o.Allow) > 0 || len(o.AllowPatterns) > 0: - return fmt.Errorf("auth: allow and allow_patterns apply only to type %q; the credentials file is the allow list", "mtls") + return fmt.Errorf("auth: allow and allow_patterns apply only to type %q; the users are the allow list", "mtls") case side == Dialer && (o.CredentialsFile != "" || o.TokenLifetimeMS != 0 || len(o.TrustedProxies) > 0): return errors.New("auth: credentials_file, token_lifetime_ms and trusted_proxies apply only to listeners") case side == Dialer && o.Identity != "": @@ -305,10 +305,12 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error { return fmt.Errorf("auth: type %q on a dialer requires username and password_file", "scram") case side == Dialer: return nil - case o.Username != "" || o.PasswordFile != "": - return errors.New("auth: username and password_file apply only to dialers") - case o.CredentialsFile == "": - return fmt.Errorf("auth: type %q requires credentials_file", "scram") + case o.CredentialsFile != "" && (o.Username != "" || o.PasswordFile != ""): + return errors.New("auth: credentials_file, or username and password_file: not both") + case (o.Username == "") != (o.PasswordFile == ""): + return errors.New("auth: username and password_file go together") + case o.CredentialsFile == "" && o.Username == "": + return fmt.Errorf("auth: type %q requires credentials_file, or username and password_file", "scram") case o.TokenLifetimeMS > 0 && !http: return errors.New("auth: token_lifetime_ms applies only to HTTP listeners") case len(o.TrustedProxies) > 0 && (side != Listener || !http): @@ -425,6 +427,10 @@ func and(keys []string) string { return strings.Join(keys[:len(keys)-1], ", ") + " and " + keys[len(keys)-1] } +// pagesBehind is why a login or viewer page needs a proxy: only there do +// browsers log in, and elsewhere the viewer needs no page +const pagesBehind = "only for scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls, the viewer is always at /" + // Check keeps a stream and a status path ServeMux and a URL take literally, // apart, and off the authentication paths; a login page needs browsers that // log in, behind a proxy @@ -442,10 +448,12 @@ func (o *HTTPSinkOptions) Check() error { switch { case o.StreamPath == o.StatusPath: return fmt.Errorf("stream_path and status_path must differ") - case (o.LoginPage || o.ViewerPage) && !behind: - return errors.New("login_page and viewer_page apply to scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls the viewer is always served") + case o.LoginPage && !behind: + return &KeyError{"login_page", errors.New(pagesBehind)} + case o.ViewerPage && !behind: + return &KeyError{"viewer_page", errors.New(pagesBehind)} case o.ViewerPage && !o.LoginPage: - return errors.New("viewer_page needs login_page, where it sends a signed-out viewer") + return &KeyError{"viewer_page", errors.New("needs login_page, where it sends a signed-out viewer")} } return nil } diff --git a/internal/flow/ratelimiter.go b/internal/flow/ratelimiter.go index b2e496c..a79bbfc 100644 --- a/internal/flow/ratelimiter.go +++ b/internal/flow/ratelimiter.go @@ -27,12 +27,12 @@ func NewRateLimiter(cfg config.RateLimitConfig, logger *log.Logger) (*RateLimite if err := config.Settle(&cfg); err != nil { return nil, err } - if cfg.Rate == 0 { + if cfg.EntriesPerSecond == 0 { return nil, nil // off } - burst := cfg.Burst + burst := cfg.BurstEntries if burst == 0 { - burst = cfg.Rate + burst = cfg.EntriesPerSecond } policy := config.PolicyPass if cfg.Policy == "drop" { @@ -40,7 +40,7 @@ func NewRateLimiter(cfg config.RateLimitConfig, logger *log.Logger) (*RateLimite } l := &RateLimiter{ - bucket: tokenbucket.New(burst, cfg.Rate), + bucket: tokenbucket.New(burst, cfg.EntriesPerSecond), policy: policy, logger: logger, maxEntrySizeBytes: cfg.MaxEntrySizeBytes, @@ -85,8 +85,8 @@ func (l *RateLimiter) GetStats() map[string]any { stats := map[string]any{ "enabled": true, - "rate": l.bucket.Rate(), - "burst": l.bucket.Capacity(), + "entries_per_second": l.bucket.Rate(), + "burst_entries": l.bucket.Capacity(), "dropped_total": l.droppedCount.Load(), "dropped_by_size_total": l.droppedBySizeCount.Load(), "policy": policyString(l.policy), diff --git a/internal/tlsx/tlsx.go b/internal/tlsx/tlsx.go index 4ec645a..2f1b33d 100644 --- a/internal/tlsx/tlsx.go +++ b/internal/tlsx/tlsx.go @@ -240,13 +240,14 @@ var warnedSecrets sync.Map // SecretFileWarning describes a secret file every local user can read, once // per path per process. Group access is left alone: container secret mounts -// commonly grant it on purpose. +// commonly grant it on purpose. Terminals, pipes and descriptors hold no +// secret at rest; lw closes /dev/fd/N after reading, and N then names another. func SecretFileWarning(key, path string) string { - if path == "" { + if path == "" || strings.HasPrefix(path, "/dev/fd/") { return "" } fi, err := os.Stat(path) - if err != nil || fi.Mode().Perm()&0o004 == 0 { + if err != nil || !fi.Mode().IsRegular() || fi.Mode().Perm()&0o004 == 0 { return "" } if _, seen := warnedSecrets.LoadOrStore(path, struct{}{}); seen { diff --git a/internal/tlsx/tlsx_test.go b/internal/tlsx/tlsx_test.go index 9279dfa..d193e07 100644 --- a/internal/tlsx/tlsx_test.go +++ b/internal/tlsx/tlsx_test.go @@ -4,10 +4,13 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "fmt" "net" + "os" "path/filepath" "slices" "strings" + "syscall" "testing" "time" @@ -200,3 +203,33 @@ func TestTLSOptionConflicts(t *testing.T) { } } } + +// Terminals, pipes and descriptors hold no secret at rest: /dev/tty, mode +// 0666 like /dev/null, is not warned about, nor /dev/fd/N whatever it names, +// while a world-readable file is. +func TestSecretFileWarningSkipsTerminalsAndPipes(t *testing.T) { + dir := t.TempDir() + fifo, file := filepath.Join(dir, "fifo"), filepath.Join(dir, "pw") + if err := syscall.Mkfifo(fifo, 0o600); err != nil { + t.Fatal(err) + } + testutil.WriteFile(t, file, "secret\n") + for _, path := range []string{fifo, file} { + if err := os.Chmod(path, 0o644); err != nil { // past the umask + t.Fatal(err) + } + } + held, err := os.Open(file) + if err != nil { + t.Fatal(err) + } + defer held.Close() + for _, path := range []string{os.DevNull, fifo, fmt.Sprintf("/dev/fd/%d", held.Fd())} { + if w := SecretFileWarning("auth.password_file", path); w != "" { + t.Errorf("%s: %s", path, w) + } + } + if SecretFileWarning("auth.password_file", file) == "" { + t.Error("a world-readable file was not warned about") + } +} diff --git a/internal/tui/canvas.go b/internal/tui/canvas.go index 04fa497..c51f088 100644 --- a/internal/tui/canvas.go +++ b/internal/tui/canvas.go @@ -10,7 +10,6 @@ import ( "github.com/lixenwraith/logwisp/internal/compose" "github.com/lixenwraith/logwisp/internal/config" - "github.com/lixenwraith/terminal" ui "github.com/lixenwraith/terminal/tui" ) @@ -92,22 +91,9 @@ func (a *app) layout(w int) layout { func (a *app) drawCanvas(r ui.Region) { l := a.layout(r.W) chosen := l.nodes[a.col][min(a.row[a.col], len(l.nodes[a.col])-1)] - if chosen.y < a.scroll { - a.scroll = chosen.y - } - if tall := 2 - a.col%2; chosen.y+tall > a.scroll+r.H { - a.scroll = chosen.y + tall - r.H - } - a.scroll = max(0, min(a.scroll, l.h-r.H)) - - // Drawn whole off screen, then the rows in view copied - cells := make([]terminal.Cell, r.W*l.h) - full := ui.NewRegion(cells, r.W, 0, 0, r.W, l.h) - full.FillStyle(a.th.Text) - a.drawParts(full, l) - for y := range min(r.H, l.h-a.scroll) { - copy(r.Cells[(r.Y+y)*r.TotalW+r.X:][:r.W], cells[(a.scroll+y)*r.W:][:r.W]) - } + a.scroll.SetDimensions(l.h, r.H) + a.scroll.EnsureRange(chosen.y, 2-a.col%2) + r.Window(l.h, &a.scroll, a.th.Text, func(full ui.Region) { a.drawParts(full, l) }) } func (a *app) drawParts(r ui.Region, l layout) { @@ -118,7 +104,7 @@ func (a *app) drawParts(r ui.Region, l layout) { r.TextStyled(t.x, t.y, string(f.dot)+" "+c.title, tints[col].On(th.Text)) } a.drawBox(r, l) - g := wires{w: r.W, arms: make([]uint8, r.W*l.h)} + g := ui.NewWires(r.W, l.h) for _, col := range []int{sourcesCol, sinksCol} { nodes := a.nodes(col) for i, s := range l.nodes[col] { @@ -130,7 +116,7 @@ func (a *app) drawParts(r ui.Region, l layout) { a.drawNode(r, s, nodes[i], tints[col], chosen) } if l.wide && len(nodes) > 0 { - g.join(l, col) + join(g, l, col) } } if !l.wide { @@ -139,11 +125,7 @@ func (a *app) drawParts(r ui.Region, l layout) { } return } - for i, arms := range g.arms { - if arms != 0 { - r.TextStyled(i%r.W, i/r.W, string(f.wire[arms]), th.Border) - } - } + r.DrawWires(g, f.Line, th.Border) if len(a.nodes(sourcesCol)) > 0 { r.TextStyled(l.box.x-1, l.entry, string(f.arrow), th.Border) } @@ -152,30 +134,9 @@ func (a *app) drawParts(r ui.Region, l layout) { } } -// wires are the canvas's cells by the arms they have: up 1, right 2, down 4, -// left 8, so every junction draws from what meets there -type wires struct { - w int - arms []uint8 -} - -func (g wires) h(y, x0, x1 int) { - for x := x0; x < x1; x++ { - g.arms[y*g.w+x] |= 2 - g.arms[y*g.w+x+1] |= 8 - } -} - -func (g wires) v(x, y0, y1 int) { - for y := y0; y < y1; y++ { - g.arms[y*g.w+x] |= 4 - g.arms[(y+1)*g.w+x] |= 1 - } -} - // join wires a column's nodes to a trunk beside the flow box, and the trunk // to the box's side -func (g wires) join(l layout, col int) { +func join(g ui.Wires, l layout, col int) { ys := []int{l.entry} if col == sinksCol { ys[0] = l.exit @@ -186,18 +147,18 @@ func (g wires) join(l layout, col int) { if col == sourcesCol { trunk := l.box.x - gutter + 1 for _, s := range l.nodes[col] { - g.h(s.y, trunk-1, trunk) + g.H(s.y, trunk-1, trunk) } - g.v(trunk, slices.Min(ys), slices.Max(ys)) - g.h(l.entry, trunk, l.box.x-1) + g.V(trunk, slices.Min(ys), slices.Max(ys)) + g.H(l.entry, trunk, l.box.x-1) return } right := l.box.x + l.box.w trunk := right + 2 - g.h(l.exit, right, trunk) - g.v(trunk, slices.Min(ys), slices.Max(ys)) + g.H(l.exit, right, trunk) + g.V(trunk, slices.Min(ys), slices.Max(ys)) for _, s := range l.nodes[col] { - g.h(s.y, trunk, s.x-1) + g.H(s.y, trunk, s.x-1) } } @@ -245,14 +206,7 @@ func (a *app) drawEmpty(r ui.Region, s spot, role string, chosen bool) { func (a *app) drawBox(r ui.Region, l layout) { th, f := a.th, a.font b := l.box - edge := []rune(f.box) - across := strings.Repeat(string(edge[4]), max(0, b.w-2)) - r.TextStyled(b.x, b.y, string(edge[0])+across+string(edge[1]), th.Border) - r.TextStyled(b.x, b.y+l.boxH-1, string(edge[2])+across+string(edge[3]), th.Border) - for y := b.y + 1; y < b.y+l.boxH-1; y++ { - r.TextStyled(b.x, y, string(edge[5]), th.Border) - r.TextStyled(b.x+b.w-1, y, string(edge[5]), th.Border) - } + r.Sub(b.x, b.y, b.w, l.boxH).BoxStyle(f.box, th.Border) for i, n := range a.nodes(flowCol) { s := l.nodes[flowCol][i] chosen := a.col == flowCol && a.row[flowCol] == i @@ -345,11 +299,14 @@ func show(v any) string { // key, as config's key paths write them var problemPath = regexp.MustCompile(`^pipelines?\[(\d+)\](?:\.(plugin_sources|plugin_sinks)\[([^\]]*)\](?:\.config\.?([\w.]*))?|\.flow\.(\w+)(?:\[(\d+)\])?\.?([\w.]*))?`) -func locate(err error) (pi int, n compose.Node, key string, ok bool) { - m := problemPath.FindStringSubmatch(err.Error()) - if m == nil { - return 0, n, "", false +// locate reads a problem's part, and its text past the path +func locate(err error) (pi int, n compose.Node, key, text string, ok bool) { + at := problemPath.FindStringSubmatchIndex(err.Error()) + if at == nil { + return 0, n, "", err.Error(), false } + m := problemPath.FindStringSubmatch(err.Error()) + text = strings.TrimPrefix(err.Error()[at[1]:], ": ") pi, _ = strconv.Atoi(m[1]) switch { case m[2] != "": @@ -362,7 +319,7 @@ func locate(err error) (pi int, n compose.Node, key string, ok bool) { } key = m[7] } - return pi, n, key, true + return pi, n, key, text, true } // faulty reports whether the first problem names this node, or a filter @@ -371,6 +328,28 @@ func (a *app) faulty(n compose.Node) bool { if a.problem == nil { return false } - pi, m, _, ok := locate(a.problem) + pi, m, _, _, ok := locate(a.problem) return ok && pi == a.pi && m.Role == n.Role && m.ID == n.ID } + +// where names a problem's part as the screen shows it: its pipeline when +// another is shown, the part, and its key +func (a *app) where(pi int, n compose.Node, key string) string { + var out []string + if pi != a.pi && pi < len(a.comp.Pipelines) { + out = append(out, "pipeline "+a.comp.Pipelines[pi].Name) + } + switch n.Role { + case "": + case "source", "sink": + out = append(out, n.Role+" "+n.ID) + case "filters": + out = append(out, fmt.Sprintf("filter %d", n.Index+1)) + default: + out = append(out, n.Role) + } + if key != "" { + out = append(out, key) + } + return strings.Join(out, " · ") +} diff --git a/internal/tui/dialog.go b/internal/tui/dialog.go index d1e5c74..6b93400 100644 --- a/internal/tui/dialog.go +++ b/internal/tui/dialog.go @@ -1,6 +1,7 @@ package tui import ( + "fmt" "strings" "github.com/lixenwraith/logwisp/internal/compose" @@ -16,23 +17,19 @@ type dialog interface { handle(a *app, ev terminal.Event) } -// frame draws a dialog's box, w by h at most, centered in r with its title, -// and returns the inside +// frame draws a dialog's box around an inside w by h, centered in r and +// clipped to it, with its title, and returns the inside; sized to what it +// holds, it ends on its last line func (a *app) frame(r ui.Region, w, h int, title string) ui.Region { - b := ui.Center(r, min(w, r.W), min(h, r.H)) - b.FillStyle(a.th.Text) - wire := a.font.wire - across := strings.Repeat(string(wire[10]), max(0, b.W-2)) - b.TextStyled(0, 0, string(wire[6])+across+string(wire[12]), a.th.Border) - b.TextStyled(0, b.H-1, string(wire[3])+across+string(wire[9]), a.th.Border) - for y := 1; y < b.H-1; y++ { - b.TextStyled(0, y, string(wire[5]), a.th.Border) - b.TextStyled(b.W-1, y, string(wire[5]), a.th.Border) - } - b.TextStyled(2, 0, " "+ui.Truncate(title, b.W-6)+" ", a.th.Accent) - return b.Sub(2, 1, b.W-4, b.H-2) + return ui.Center(r, min(w+4, r.W), min(h+2, r.H)).Frame(title, a.th) } +// inside is a dialog's widest inside on a screen w wide; one narrower than +// stacked puts labels and keys above what they name +func inside(w int) int { return max(1, min(72, w-4)) } + +const stacked = 32 + // chooser picks one option from a filtered list type chooser struct { title string @@ -43,8 +40,8 @@ type chooser struct { } func (c *chooser) draw(a *app, r ui.Region) { - in := a.frame(r, 76, len(c.list.Options)+4, c.title) - in.OptionList(c.list, a.th) + w := inside(r.W) + a.frame(r, w, c.list.Rows(w), c.title).OptionList(c.list, a.th) } func (c *chooser) handle(a *app, ev terminal.Event) { @@ -64,6 +61,11 @@ func (c *chooser) handle(a *app, ev terminal.Event) { a.dialog = nil c.pick(a, i) } + case ev.Key == terminal.KeyRune && c.list.Menu: + if i, ok := c.list.Pick(ev.Rune); ok { + a.dialog = nil + c.pick(a, i) + } default: c.list.HandleKey(ev.Key, ev.Rune, ev.Modifiers) } @@ -95,56 +97,104 @@ func (a *app) addMenu() dialog { return } a.choose(n) - a.inspect, a.insp.cursor = true, 0 - for j, l := range a.lines() { - if l.group != "" && l.node == n { - a.insp.cursor = j - } - } + a.inspectFirst(n) }} } -// presetMenu starts the pipeline over from a preset, or the pipelines from a -// pasted command line; on a bare start, Esc starts empty -func (a *app) presetMenu() dialog { - presets := config.Presets() +// typeMenu changes a source's or sink's type, in its place +func (a *app) typeMenu(n compose.Node) dialog { + p, _ := a.plugin(n) var options []ui.Option - for _, p := range presets { - options = append(options, ui.Option{Name: p.Name, Hint: p.Summary}) + for _, q := range config.Plugins() { + if q.Role == n.Role && q.Type != p.Type { + options = append(options, ui.Option{Name: q.Type, Hint: q.Summary}) + } } - options = append(options, ui.Option{Name: "empty", Hint: "no sources or sinks: add your own"}) + return &chooser{title: "Change " + p.Type, list: ui.NewOptionListState(options), pick: func(a *app, i int) { + m, err := a.comp.Retype(a.pi, n, options[i].Name) + if a.edit(err) { + a.choose(m) + a.inspectFirst(m) + } + }} +} + +// enumMenu sets an enum key to the option chosen, the cursor on its value +func (a *app) enumMenu(l line, cur any) dialog { + var options []ui.Option + for _, e := range l.key.Enum { + hint := "" + if e == l.key.Default { + hint = "the default" + } + options = append(options, ui.Option{Name: e, Hint: hint}) + } + list := ui.NewOptionListState(options) + list.Cursor = max(0, choice(l.key, cur)) + return &chooser{title: label(l), list: list, pick: func(a *app, i int) { + a.edit(a.comp.Set(a.pi, l.node, l.key.Name, options[i].Name)) + }} +} + +// presetMenu replaces the pipeline shown from a preset, or the pipelines +// from a pasted command line; on a bare start, Esc starts empty +func (a *app) presetMenu() dialog { title := "Start from a preset, or paste an lw command line" if len(a.comp.Pipelines) > 0 { title = "Replace pipeline " + a.pipeline().Name + " with a preset" } - empty := func(a *app) { - a.use(config.PipelineConfig{Name: "default", Flow: &config.FlowConfig{}}) - } - menu := &chooser{title: title, list: ui.NewOptionListState(options)} + menu := a.pipelineMenu(title, (*app).use) menu.paste = func(a *app, text string) { a.dialog = nil if a.paste(text); len(a.comp.Pipelines) == 0 { a.dialog = menu // refused: the status says why } } - menu.pick = func(a *app, i int) { - switch { - case i == len(presets): - empty(a) - case len(presets[i].Params) == 0: - (&presetForm{preset: presets[i]}).apply(a) - default: - a.dialog = newPresetForm(presets[i]) - } - } menu.back = func(a *app) { if len(a.comp.Pipelines) == 0 { - empty(a) + a.use(empty()) } } return menu } +// pipelineMenu offers the presets and an empty pipeline, which place puts +// on screen +func (a *app) pipelineMenu(title string, place func(a *app, p config.PipelineConfig)) *chooser { + presets := config.Presets() + var options []ui.Option + for _, p := range presets { + options = append(options, ui.Option{Name: p.Name, Hint: p.Summary}) + } + options = append(options, ui.Option{Name: "empty", Hint: "no sources or sinks: add your own"}) + menu := &chooser{title: title, list: ui.NewOptionListState(options)} + menu.pick = func(a *app, i int) { + if i == len(presets) { + place(a, empty()) + return + } + p := presets[i] + f := &form{title: "preset " + p.Name, summary: p.Summary, params: p.Params, back: func(a *app) { a.dialog = menu }, + apply: func(a *app, values map[string]string) error { + c, err := compose.FromPreset(p.Name, values, config.HostIsDir) + if err == nil { + place(a, c.Pipelines[0]) + } + return err + }} + if f.fill(nil); len(p.Params) == 0 { + f.submit(a) + return + } + a.dialog = f + } + return menu +} + +func empty() config.PipelineConfig { + return config.PipelineConfig{Name: "default", Flow: &config.FlowConfig{}} +} + // use puts a pipeline in place of the one shown, under its name, asking // first when that one has sources or sinks; the first pipeline keeps its own, // and its parts are work quit asks about @@ -167,51 +217,102 @@ func (a *app) use(p config.PipelineConfig) { a.dialog = &confirm{question: "Replace pipeline " + old.Name + "?", yes: replace} } -// presetForm takes a preset's keys, its defaults as placeholders -type presetForm struct { - preset config.Preset - fields []*ui.TextFieldState - focus ui.Focus - scroll int // the first field shown - err string +// append adds a pipeline after the others, under its name or the first free +// one after it, and shows it +func (a *app) append(p config.PipelineConfig) { + var names []string + for _, q := range a.comp.Pipelines { + names = append(names, q.Name) + } + p.Name = config.FreeID(p.Name, names) + a.comp.Pipelines = append(a.comp.Pipelines, p) + a.pi, a.row, a.bar, a.changed = len(a.comp.Pipelines)-1, [3]int{}, false, true + a.check() } -func newPresetForm(p config.Preset) *presetForm { - f := &presetForm{preset: p, focus: ui.Focus{Len: len(p.Params)}} - for range p.Params { - f.fields = append(f.fields, ui.NewTextFieldState("")) - } - return f +// form takes a few keys, their defaults as placeholders, then applies them; +// a refusal stays in the form +type form struct { + title, summary string + params []config.PresetParam + fields []*ui.TextFieldState + focus ui.Focus + scroll ui.ViewportScroll // the fields' rows + err string + apply func(a *app, values map[string]string) error + back func(a *app) // on Esc, after the form closes } -func (f *presetForm) draw(a *app, r ui.Region) { - th := a.th - in := a.frame(r, 76, len(f.fields)+6, "preset "+f.preset.Name) - in.TextStyled(0, 0, ui.Truncate(f.preset.Summary, in.W), th.Muted) - rows := max(1, in.H-3) // between the summary and the note - f.scroll = max(0, min(f.scroll, f.focus.Index, len(f.fields)-rows), f.focus.Index-rows+1) - for i := f.scroll; i < len(f.fields) && i-f.scroll < rows; i++ { - p := f.preset.Params[i] - v, _ := in.Field(2+i-f.scroll, 14, ui.Field{Label: p.Name, Required: p.Required}, i == f.focus.Index, th) - v.TextInput(f.fields[i], p.Default, i == f.focus.Index, th) +// fill gives the fields their first values +func (f *form) fill(values map[string]string) { + f.focus, f.fields = ui.Focus{Len: len(f.params)}, nil + for _, p := range f.params { + f.fields = append(f.fields, ui.NewTextFieldState(values[p.Name])) } - note, style := f.preset.Params[f.focus.Index].Help, th.Muted +} + +// draw lays the form out at the screen's width: the summary, the fields, +// their labels beside them or, when stacked, above, then the focused +// field's help or the refusal, in rows kept for the longest of them +func (f *form) draw(a *app, r ui.Region) { + th, w := a.th, inside(r.W) + labelW, tall := 0, 2 + if w >= stacked { + for _, p := range f.params { + labelW = max(labelW, ui.RuneLen(p.Name)+2) + } + tall = 1 + } + summary, noteRows := wrap(f.summary, w, 2), len(wrap(f.err, w, 3)) + for _, p := range f.params { + noteRows = max(noteRows, len(wrap(p.Help, w, 3))) + } + top, bottom := len(summary)+min(1, len(summary)), noteRows+min(1, noteRows) + in := a.frame(r, w, top+len(f.fields)*tall+bottom, f.title) + for i, l := range summary { + in.TextStyled(0, i, l, th.Muted) + } + view := in.Sub(0, top, in.W, max(1, in.H-top-bottom)) + f.scroll.SetDimensions(len(f.fields)*tall, view.H) + f.scroll.EnsureRange(f.focus.Index*tall, tall) + view.Window(len(f.fields)*tall, &f.scroll, th.Text, func(all ui.Region) { + for i, p := range f.params { + v, _ := all.Field(i*tall, labelW, ui.Field{Label: p.Name, Required: p.Required}, i == f.focus.Index, th) + v.TextInput(f.fields[i], p.Default, i == f.focus.Index, th) + } + }) + note, style := f.params[f.focus.Index].Help, th.Muted if f.err != "" { note, style = f.err, th.Error } - in.TextStyled(0, in.H-1, ui.Truncate(note, in.W), style) + lines := wrap(note, in.W, noteRows) // on the last rows, the box ending on text + for i, l := range lines { + in.TextStyled(0, in.H-len(lines)+i, l, style) + } } -func (f *presetForm) handle(a *app, ev terminal.Event) { +// wrap is s wrapped to w, at most n lines, the last cut with '…'; none when +// s is empty +func wrap(s string, w, n int) []string { + if s == "" { + return nil + } + return clip(ui.WrapText(s, w), n) +} + +func (f *form) handle(a *app, ev terminal.Event) { field := f.fields[f.focus.Index] switch { case ev.Type == terminal.EventPaste: field.Paste(ev.Text) case ev.Type != terminal.EventKey: case is(ev, terminal.KeyEscape, ""): - a.dialog = a.presetMenu() + a.dialog = nil + if f.back != nil { + f.back(a) + } case is(ev, terminal.KeyEnter, ""): - f.apply(a) + f.submit(a) case is(ev, terminal.KeyUp, ""): f.focus.Index = max(0, f.focus.Index-1) case is(ev, terminal.KeyDown, ""): @@ -222,39 +323,33 @@ func (f *presetForm) handle(a *app, ev terminal.Event) { } } -// apply builds the preset's pipeline, on this host: a path is a directory -// when one is there -func (f *presetForm) apply(a *app) { +// submit applies the values typed, closing the form unless they are refused +func (f *form) submit(a *app) { values := map[string]string{} - for i, p := range f.preset.Params { + for i, p := range f.params { if v := f.fields[i].Value(); v != "" { values[p.Name] = v } } - c, err := compose.FromPreset(f.preset.Name, values, config.HostIsDir) - if err != nil { - f.err = err.Error() - return - } a.dialog = nil - a.use(c.Pipelines[0]) + if err := f.apply(a, values); err != nil { + f.err, a.dialog = err.Error(), f + } } // outputMenu is what o offers: run, or print one of the forms func (a *app) outputMenu() dialog { forms := compose.Forms() - options := []ui.Option{{Name: "run", Hint: "start the pipelines now, with lw's other settings"}} + options := []ui.Option{{Name: "run", Key: 'r', Hint: "start the pipelines now, with lw's other settings"}} for _, f := range forms { - options = append(options, ui.Option{Name: f.Name, Hint: "print " + f.Hint}) + options = append(options, ui.Option{Name: f.Name, Key: rune(f.Name[0]), Hint: "print " + f.Hint}) } - return &chooser{title: "Output, once the screen closes", list: ui.NewOptionListState(options), + list := ui.NewOptionListState(options) + list.Menu = true + return &chooser{title: "Output, once the screen closes", list: list, pick: func(a *app, i int) { if i == 0 { - if a.problem != nil { - a.status = "not valid yet: c shows the problem" - return - } - a.result, a.done = Result{Exit: Start, Pipelines: a.comp.Pipelines}, true + a.run() return } text, err := forms[i-1].Write(a.comp) @@ -266,88 +361,120 @@ func (a *app) outputMenu() dialog { }} } -// notice shows lines until any key +// notice shows text, wrapped to the screen, until a key other than one +// that scrolls it type notice struct { title string - lines []string + body func(w int) []string // the lines at a width + view ui.ViewportScroll +} + +// textNotice is a notice of plain text, its paragraphs split at '\n' +func textNotice(title, body string) *notice { + return ¬ice{title: title, body: func(w int) []string { return ui.WrapText(body, w) }} } func (n *notice) draw(a *app, r ui.Region) { - w, lines := ui.RuneLen(n.title)+8, fit(n.lines, r.W) + lines, w := n.body(inside(r.W)), ui.RuneLen(n.title)+2 for _, l := range lines { - w = max(w, ui.RuneLen(l)+6) + w = max(w, ui.RuneLen(l)) } - in := a.frame(r, w, len(lines)+4, n.title) - for i, l := range lines { - in.TextStyled(0, 1+i, l, a.th.Text) + in := a.frame(r, min(w, inside(r.W)), len(lines), n.title) + rows := in.H + if len(lines) > in.H { // the last row says where the view is + rows = max(1, in.H-1) } -} - -// fit wraps lines to a dialog's inside on a screen w wide -func fit(lines []string, w int) []string { - var out []string - for _, l := range lines { - out = append(out, ui.WrapText(l, max(1, w-6))...) + n.view.SetDimensions(len(lines), rows) + for y := range min(rows, len(lines)-n.view.Offset) { + in.TextStyled(0, y, lines[n.view.Offset+y], a.th.Text) + } + if rows < in.H { + where := fmt.Sprintf("j k: %d-%d of %d", n.view.Offset+1, n.view.Offset+rows, len(lines)) + in.TextStyled(0, rows, ui.Truncate(where, in.W), a.th.Muted) } - return out } func (n *notice) handle(a *app, ev terminal.Event) { - if ev.Type == terminal.EventKey { + switch { + case ev.Type != terminal.EventKey: + case is(ev, terminal.KeyUp, "k"): + n.view.ScrollBy(-1) + case is(ev, terminal.KeyDown, "j"): + n.view.ScrollBy(1) + case is(ev, terminal.KeyPageUp, ""): + n.view.PageUp() + case is(ev, terminal.KeyPageDown, ""): + n.view.PageDown() + default: a.dialog = nil } } -// confirm asks a yes or no question +// confirm asks a yes or no question: y or n answers, as does Enter on the +// answer the arrows chose; Esc is no type confirm struct { question string yes func(a *app) + state ui.ConfirmState } func (c *confirm) draw(a *app, r ui.Region) { - lines := fit([]string{c.question}, r.W) - w := 0 + lines := ui.WrapText(c.question, inside(r.W)) + w := 12 // the answers for _, l := range lines { - w = max(w, ui.RuneLen(l)+6) + w = max(w, ui.RuneLen(l)) } - in := a.frame(r, w, len(lines)+5, "Confirm") + in := a.frame(r, w, len(lines)+1, "Confirm") for i, l := range lines { - in.TextStyled(0, 1+i, l, a.th.Text) + in.TextStyled(0, i, l, a.th.Text) } - in.TextStyled(0, 1+len(lines), "y yes n no", a.th.Muted) + in.Radio(0, in.H-1, []string{"yes", "no"}, map[bool]int{true: 0, false: 1}[c.state.FocusYes], a.th) } func (c *confirm) handle(a *app, ev terminal.Event) { - switch { - case is(ev, terminal.KeyNone, "yY"): - a.dialog = nil + if ev.Type != terminal.EventKey || !c.state.HandleKey(ev.Key, ev.Rune) { + return + } + if a.dialog = nil; c.state.Result == ui.ConfirmYes { c.yes(a) - case is(ev, terminal.KeyEscape, "nNq"): - a.dialog = nil } } -// help lists the keys +// help lists the keys, each beside its description or, narrow, above it func help(f font) dialog { keys := [][2]string{ - {"arrows, hjkl", "move"}, - {f.enter, "inspect the chosen part; edit a value"}, + {"arrows, hjkl", "move; up from the top row to the pipeline bar"}, + {f.enter, "inspect a part; edit a value; open a list; add"}, {"tab", "next value"}, {"esc", "back"}, - {"a", "add a source, sink or filter"}, + {"a", "add a source, sink or filter; on the bar, a pipeline"}, + {"c", "change a source's or sink's type"}, {"d", "delete it; a value back to its default"}, {f.space, "a flow stage or a switch on or off"}, {"J K", "move a filter down or up"}, {"[ ]", "previous or next pipeline"}, {"p", "start over from a preset"}, + {"r", "run the pipelines"}, {"o", "run, or print a command line, environment or file"}, - {"c", "the first problem"}, + {"!", "the first problem"}, {"paste", "an lw command line replaces the pipelines"}, {"q", "quit"}, } - n := ¬ice{title: "Keys"} - for _, k := range keys { - n.lines = append(n.lines, ui.PadRight(k[0], 14)+k[1]) - } - return n + return ¬ice{title: "Keys", body: func(w int) []string { + var lines []string + for _, k := range keys { + if w < stacked { + lines = append(lines, k[0]) + for _, l := range ui.WrapText(k[1], w-2) { + lines = append(lines, " "+l) + } + continue + } + key := k[0] + for _, l := range ui.WrapText(k[1], w-14) { + lines, key = append(lines, ui.PadRight(key, 14)+l), "" + } + } + return lines + }} } diff --git a/internal/tui/inspector.go b/internal/tui/inspector.go index 504701f..db8caf9 100644 --- a/internal/tui/inspector.go +++ b/internal/tui/inspector.go @@ -21,15 +21,21 @@ type inspector struct { field *ui.TextFieldState // the value being typed, nil while none } -// line is one inspector row: a key, a group that folds, or a list's element +// line is one inspector row: a key, a group that folds, a list's element, or +// the row that adds an element or a filter type line struct { node compose.Node - key config.Key // its Name is the dotted path in the node's options + key config.Key // its Name is the dotted path in the node's options, "" on a filter's rows depth int group string // a group's title, "" for a value item int // a list element's index, the list's length on the row that adds one; -1 elsewhere } +// adds reports whether a row adds: a list's last row, or the filters' +func (a *app) adds(l line) bool { + return l.item >= 0 && (l.key.Name == "" || l.item == len(a.items(l.node, l.key.Name))) +} + func (a *app) foldKey(n compose.Node, path string) string { return fmt.Sprintf("%d/%s/%s/%d/%s", a.pi, n.Role, n.ID, n.Index, path) } @@ -44,14 +50,15 @@ func (a *app) lines() []line { return a.keyLines(nil, n, a.keys(n), "", 0) } var out []line - for i, f := range a.pipeline().Flow.Filters { + filters := a.pipeline().Flow.Filters + for i, f := range filters { fn := compose.Node{Role: "filters", Index: i} out = append(out, line{node: fn, group: fmt.Sprintf("%d %s", i+1, f.Type), item: -1}) if a.insp.open[a.foldKey(n, strconv.Itoa(i))] { out = a.keyLines(out, fn, a.keys(n), "", 1) } } - return out + return append(out, line{node: compose.Node{Role: "filters", Index: len(filters)}, item: len(filters)}) } func (a *app) keyLines(out []line, n compose.Node, keys []config.Key, prefix string, depth int) []line { @@ -120,31 +127,31 @@ func (a *app) drawInspector(r ui.Region) { if a.inspect { hint = f.enter + " edit esc back" } - r.TextStyled(0, 0, strings.Repeat(string(f.rule), r.W), th.Border) - r.TextStyled(2, 0, " "+ui.Truncate(title, r.W-ui.RuneLen(hint)-10)+" ", th.Text) - r.TextStyled(r.W-ui.RuneLen(hint)-3, 0, " "+hint+" ", th.Muted) + r.Rule(0, title, hint, th) lines := a.lines() switch { case !ok: r.TextStyled(2, 2, "a adds a "+columns[a.col].role, th.Muted) return - case len(lines) == 0 && n.Role == "filters": - r.TextStyled(2, 2, "no filters: a adds one", th.Muted) - return case len(lines) == 0: r.TextStyled(2, 2, "no options", th.Muted) return } a.insp.cursor = max(0, min(a.insp.cursor, len(lines)-1)) - rows := r.H - 2 + var help []string + style := th.Muted + if a.inspect { + var text string + text, style = a.help(lines[a.insp.cursor]) + help = clip(ui.WrapText(text, r.W-4), 2) + } + rows := r.H - 1 - len(help) cols, perCol := 1, rows if half := (len(lines) + 1) / 2; r.W >= 80 && half <= rows { cols, perCol = 2, half } - if cols == 1 { - a.insp.scroll = max(0, min(a.insp.scroll, a.insp.cursor, len(lines)-rows)) - a.insp.scroll = max(a.insp.scroll, a.insp.cursor-rows+1) - } else { + a.insp.scroll = ui.AdjustScroll(a.insp.cursor, ui.ClampScroll(a.insp.scroll, rows, len(lines)), rows, len(lines)) + if cols == 2 { a.insp.scroll = 0 } colW, labelW := r.W/cols, 0 @@ -157,9 +164,8 @@ func (a *app) drawInspector(r ui.Region) { row := r.Sub(at/perCol*colW, 1+at%perCol, colW-1, 1) a.drawLine(row, lines[i], labelW, a.inspect && i == a.insp.cursor) } - if a.inspect && rows > 0 { - help, style := a.help(lines[a.insp.cursor]) - r.TextStyled(2, r.H-1, ui.Truncate(help, r.W-4), style) + for i, h := range help { + r.TextStyled(2, r.H-len(help)+i, h, style) } } @@ -177,12 +183,15 @@ func (a *app) nodeTitle(n compose.Node) string { // help is the focused row's problem, or its key's help and default func (a *app) help(l line) (string, ui.Style) { if a.problem != nil { - if pi, n, key, ok := locate(a.problem); ok && pi == a.pi && n == l.node && key == l.key.Name { - return a.problem.Error(), a.th.Error + if pi, n, key, text, ok := locate(a.problem); ok && pi == a.pi && n == l.node && key == l.key.Name { + return text, a.th.Error } } - if l.group != "" && l.key.Name == "" { - return "J and K move this filter; d deletes it", a.th.Muted + switch { + case l.group != "" && l.key.Name == "": + return "J and K move this filter; c changes its type; d deletes it", a.th.Muted + case a.adds(l) && l.key.Name == "": + return "adds a filter: include passes only the entries that match, exclude drops them", a.th.Muted } help := l.key.Help if d := placeholder(l.key); d != "" && l.group == "" { @@ -217,12 +226,12 @@ func (a *app) drawLine(r ui.Region, l line, labelW int, focused bool) { switch { case focused && a.insp.field != nil: v.TextInput(a.insp.field, placeholder(k), true, th) - case l.item == len(items): + case a.adds(l): v.TextStyled(0, 0, "+ add", th.Muted) case l.item >= 0: v.TextStyled(0, 0, ui.Truncate(items[l.item], v.W), th.Text) case len(k.Enum) > 0: - v.Choice(0, 0, k.Enum, choice(k, cur), th) + v.Radio(0, 0, k.Enum, choice(k, cur), th) case k.Kind == "bool": v.Toggle(0, 0, cur == true || cur == nil && k.Default == "true", th) case cur == nil: @@ -289,7 +298,8 @@ func (a *app) inspectKey(ev terminal.Event) bool { k := l.key cur := a.value(l.node, k.Name) focus := ui.Focus{Index: a.insp.cursor, Len: len(lines)} - toggle := is(ev, terminal.KeyEnter, " ") || is(ev, terminal.KeySpace, "") + enter, space := is(ev, terminal.KeyEnter, ""), is(ev, terminal.KeySpace, " ") + left, right := is(ev, terminal.KeyLeft, "h"), is(ev, terminal.KeyRight, "l") switch { case focus.HandleKey(ev.Key, ev.Modifiers): a.insp.cursor = focus.Index @@ -297,31 +307,35 @@ func (a *app) inspectKey(ev terminal.Event) bool { a.insp.cursor = max(0, a.insp.cursor-1) case is(ev, terminal.KeyDown, "j"): a.insp.cursor = min(len(lines)-1, a.insp.cursor+1) - case l.group != "" && (toggle || is(ev, terminal.KeyRight, "l") || is(ev, terminal.KeyLeft, "h")): + case a.adds(l) && k.Name == "" && enter: + a.dialog = a.addMenu() + case l.group != "" && (enter || space || left || right): fold := a.foldKey(l.node, k.Name) if k.Name == "" { fold = a.foldKey(compose.Node{Role: "filters"}, strconv.Itoa(l.node.Index)) } - a.insp.open[fold] = toggle && !a.insp.open[fold] || is(ev, terminal.KeyRight, "l") - case l.group != "" && l.key.Name == "" && is(ev, terminal.KeyNone, "JK"): + a.insp.open[fold] = (enter || space) && !a.insp.open[fold] || right + case l.group != "" && k.Name == "" && is(ev, terminal.KeyNone, "JK"): a.moveFilter(l.node.Index, map[rune]int{'J': 1, 'K': -1}[ev.Rune]) case is(ev, terminal.KeyNone, "d"): a.clear(l) - case len(k.Enum) > 0 && l.item < 0 && (toggle || is(ev, terminal.KeyLeft, "h") || is(ev, terminal.KeyRight, "l")): + case len(k.Enum) > 0 && l.item < 0 && enter: + a.dialog = a.enumMenu(l, cur) + case len(k.Enum) > 0 && l.item < 0 && (space || left || right): i := choice(k, cur) switch next, moved := ui.StepChoice(ev.Key, ev.Rune, i, len(k.Enum)); { case moved: i = next - case !toggle: + case !space: return true // an arrow at either end default: i = (i + 1) % len(k.Enum) } a.edit(a.comp.Set(a.pi, l.node, k.Name, k.Enum[i])) - case k.Kind == "bool" && l.item < 0 && toggle: + case k.Kind == "bool" && l.item < 0 && (enter || space || left || right): on := cur == true || cur == nil && k.Default == "true" - a.edit(a.comp.Set(a.pi, l.node, k.Name, strconv.FormatBool(!on))) - case is(ev, terminal.KeyEnter, ""): + a.edit(a.comp.Set(a.pi, l.node, k.Name, strconv.FormatBool(right || !left && !on))) + case enter: text := show(cmp.Or(cur, any(""))) if items := a.items(l.node, k.Name); l.item >= 0 { text = "" @@ -337,13 +351,14 @@ func (a *app) inspectKey(ev terminal.Event) bool { return true } -// typeKey edits the value being typed: Enter sets it, Tab sets it and moves -// on, Esc drops it +// typeKey edits the value being typed: Enter sets it, Tab and the arrows up +// and down set it and move on, Esc drops it func (a *app) typeKey(ev terminal.Event, l line) { + up, down := is(ev, terminal.KeyUp, ""), is(ev, terminal.KeyDown, "") switch { case is(ev, terminal.KeyEscape, ""): a.insp.field = nil - case is(ev, terminal.KeyEnter, "") || is(ev, terminal.KeyTab, "") || is(ev, terminal.KeyBacktab, ""): + case is(ev, terminal.KeyEnter, "") || is(ev, terminal.KeyTab, "") || is(ev, terminal.KeyBacktab, "") || up || down: value := a.insp.field.Value() if a.commit(l, value) { a.insp.field = nil @@ -351,7 +366,12 @@ func (a *app) typeKey(ev terminal.Event, l line) { a.insp.cursor++ // stays on the row that adds one } focus := ui.Focus{Index: a.insp.cursor, Len: len(a.lines())} - if focus.HandleKey(ev.Key, ev.Modifiers) { + switch { + case up: + a.insp.cursor = max(0, a.insp.cursor-1) + case down: + a.insp.cursor = min(focus.Len-1, a.insp.cursor+1) + case focus.HandleKey(ev.Key, ev.Modifiers): a.insp.cursor = focus.Index } } @@ -390,6 +410,7 @@ func (a *app) commit(l line, value string) bool { // value, which returns to its default func (a *app) clear(l line) { switch { + case a.adds(l): case l.group != "" && l.key.Name == "": a.edit(a.comp.Remove(a.pi, l.node)) case l.item >= 0: diff --git a/internal/tui/theme.go b/internal/tui/theme.go index 1a725ed..e1cda1b 100644 --- a/internal/tui/theme.go +++ b/internal/tui/theme.go @@ -55,14 +55,14 @@ var looks = map[terminal.ColorMode]look{ Source: ui.MonoTheme.Text, Flow: ui.MonoTheme.Text, Sink: ui.MonoTheme.Text}, } -// font is the marks one character set draws: the widgets' and the canvas's +// font is the marks one character set draws: the widgets' and the canvas's; +// Glyphs.Line draws frames, rules and wires type font struct { ui.Glyphs - wire []rune // a wire cell by its arms: up 1, right 2, down 4, left 8 - box []rune // the flow box: top left, top right, bottom left, bottom right, across, down - arrow, down rune // into the flow box and a sink; between stacked parts - bar, barOn rune // a node, and the chosen node - dot, rule rune // before a column's title; the inspector's rule + box ui.LineType // the flow box + arrow, down rune // into the flow box and a sink; between stacked parts + bar, barOn rune // a node, and the chosen node + dot rune // before a column's title valid string invalid string enter string @@ -85,13 +85,13 @@ func (f font) fold(r rune) rune { // fonts: Unicode, a text console's CP437, and ASCII outside UTF-8 var ( - unicodeFont = font{Glyphs: ui.GlyphsUnicode, wire: []rune(" │─└││┌├─┘─┴┐┤┬┼"), box: []rune("╭╮╰╯╌╎"), - arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '●', rule: '─', valid: "✓", invalid: "✗", enter: "⏎", space: "␣"} - cp437Font = font{Glyphs: ui.GlyphsCP437, wire: []rune(" │─└││┌├─┘─┴┐┤┬┼"), box: []rune("┌┐└┘─│"), - arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '•', rule: '─', valid: "√", invalid: "x", enter: "enter", space: "space", + unicodeFont = font{Glyphs: ui.GlyphsUnicode, box: ui.LineDashed, + arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '●', valid: "✓", invalid: "✗", enter: "⏎", space: "␣"} + cp437Font = font{Glyphs: ui.GlyphsCP437, box: ui.LineSingle, + arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '•', valid: "√", invalid: "x", enter: "enter", space: "space", folds: map[rune]rune{'…': '.'}} - asciiFont = font{Glyphs: ui.GlyphsASCII, wire: []rune(" |-+||++-+-+++++"), box: []rune("++++-|"), - arrow: '>', down: 'v', bar: '|', barOn: '#', dot: '*', rule: '-', valid: "ok", invalid: "x", enter: "enter", space: "space", + asciiFont = font{Glyphs: ui.GlyphsASCII, box: ui.LineASCII, + arrow: '>', down: 'v', bar: '|', barOn: '#', dot: '*', valid: "ok", invalid: "x", enter: "enter", space: "space", folds: map[rune]rune{'…': '.', '·': '-'}, ascii: true} ) diff --git a/internal/tui/tui.go b/internal/tui/tui.go index e9a5946..bf4fe52 100644 --- a/internal/tui/tui.go +++ b/internal/tui/tui.go @@ -7,6 +7,7 @@ package tui import ( "errors" "fmt" + "slices" "strings" "github.com/lixenwraith/logwisp/internal/compose" @@ -85,11 +86,12 @@ type app struct { th ui.Theme font font tables map[string][]config.Key - pi int // the pipeline shown - col int // the chosen column - row [3]int // the chosen node of each column - scroll int // the canvas's first row - inspect bool // keys go to the inspector + pi int // the pipeline shown + col int // the chosen column + row [3]int // the chosen node of each column + scroll ui.ViewportScroll // the canvas's rows + bar bool // keys go to the pipeline bar + inspect bool // keys go to the inspector insp inspector dialog dialog // takes every key while open status string // the last edit's refusal @@ -217,6 +219,8 @@ func (a *app) handle(ev terminal.Event) { case ev.Type == terminal.EventPaste: a.paste(ev.Text) case ev.Type != terminal.EventKey: + case a.bar: + a.barKey(ev) case !a.inspect: a.canvasKey(ev) case !a.inspectKey(ev): @@ -238,13 +242,17 @@ func (a *app) canvasKey(ev terminal.Event) { a.col = min(2, a.col+1) case is(ev, terminal.KeyTab, ""): a.col = (a.col + 1) % 3 + case is(ev, terminal.KeyUp, "k") && a.row[a.col] == 0: + a.bar = true case is(ev, terminal.KeyUp, "k"): - a.row[a.col] = max(0, a.row[a.col]-1) + a.row[a.col]-- case is(ev, terminal.KeyDown, "j"): a.row[a.col]++ a.node() - case is(ev, terminal.KeyEnter, "") && ok: - a.inspect, a.insp.cursor = true, 0 + case is(ev, terminal.KeyEnter, "") && (!ok || n.Role == "filters" && len(a.pipeline().Flow.Filters) == 0): + a.dialog = a.addMenu() // an empty column, or no filters, as a + case is(ev, terminal.KeyEnter, ""): + a.inspectFirst(n) case is(ev, terminal.KeyNone, "a"): a.dialog = a.addMenu() case is(ev, terminal.KeyNone, "d") && ok && n.Role == "filters": @@ -269,8 +277,12 @@ func (a *app) globalKey(ev terminal.Event) { a.dialog = a.outputMenu() case is(ev, terminal.KeyNone, "?"): a.dialog = help(a.font) - case is(ev, terminal.KeyNone, "c"): + case is(ev, terminal.KeyNone, "!"): a.showProblem() + case is(ev, terminal.KeyNone, "c"): + a.changeType() + case is(ev, terminal.KeyNone, "r"): + a.run() case is(ev, terminal.KeyNone, "]") && n > 0: a.pi, a.row, a.inspect = (a.pi+1)%n, [3]int{}, false case is(ev, terminal.KeyNone, "[") && n > 0: @@ -280,6 +292,76 @@ func (a *app) globalKey(ev terminal.Event) { } } +// barKey takes the keys on the pipeline bar: switch, add, rename, delete +func (a *app) barKey(ev terminal.Event) { + n := len(a.comp.Pipelines) + switch p := a.pipeline(); { + case is(ev, terminal.KeyLeft, "h["): + a.pi, a.row = (a.pi+n-1)%n, [3]int{} + case is(ev, terminal.KeyRight, "l]"): + a.pi, a.row = (a.pi+1)%n, [3]int{} + case is(ev, terminal.KeyDown, "j") || is(ev, terminal.KeyEscape, ""): + a.bar = false + case is(ev, terminal.KeyNone, "a"): + a.dialog = a.pipelineMenu("Add a pipeline", (*app).append) + case is(ev, terminal.KeyEnter, ""): + f := &form{title: "Rename pipeline " + p.Name, params: []config.PresetParam{{Name: "name", Required: true, Help: "a name no other pipeline has"}}, + apply: func(a *app, v map[string]string) error { + err := a.comp.RenamePipeline(a.pi, v["name"]) + if err == nil { + a.edit(nil) + } + return err + }} + f.fill(map[string]string{"name": p.Name}) + a.dialog = f + case is(ev, terminal.KeyNone, "d"): + remove := func(a *app) { + a.edit(a.comp.RemovePipeline(a.pi)) + a.pi, a.row, a.insp.open = max(0, a.pi-1), [3]int{}, map[string]bool{} // folds are keyed by index + if len(a.comp.Pipelines) == 0 { + a.bar, a.dialog = false, a.presetMenu() + } + } + if len(p.PluginSources)+len(p.PluginSinks) == 0 { + remove(a) + return + } + a.dialog = &confirm{question: "Delete pipeline " + p.Name + "?", yes: remove} + default: + a.globalKey(ev) + } +} + +// changeType offers a source or sink the other types of its role; a filter +// or the format changes type on its type row +func (a *app) changeType() { + n, ok := a.node() + if l := a.lines(); a.inspect && ok && len(l) > 0 { + n = l[min(a.insp.cursor, len(l)-1)].node + } + switch { + case !ok || n.Role == "filters" && !a.inspect: + a.note = "c changes the type of a source, sink, filter or format" + case n.Role == "source" || n.Role == "sink": + a.dialog = a.typeMenu(n) + case n.Role == "filters" && n.Index < len(a.pipeline().Flow.Filters), n.Role == "format": + a.choose(n) + a.inspectAt(func(l line) bool { return l.node == n && l.key.Name == "type" }) + default: + a.note = "c changes the type of a source, sink, filter or format" + } +} + +// run leaves to start the pipelines, once they are valid +func (a *app) run() { + if a.problem != nil { + a.status = "not valid yet: ! shows the problem" + return + } + a.result, a.done = Result{Exit: Start, Pipelines: a.comp.Pipelines}, true +} + // switchStage turns a flow stage on with its defaults, or off; a stage with // an enabled key switches that func (a *app) switchStage(n compose.Node) { @@ -308,7 +390,8 @@ func slicesHasKey(keys []config.Key, name string) bool { return false } -// paste starts over from a pasted command line +// paste starts over from a pasted command line, asking first when there are +// edited pipelines to lose func (a *app) paste(text string) { c, err := compose.FromCommandLine(text, config.HostIsDir) if err != nil { @@ -320,7 +403,7 @@ func (a *app) paste(text string) { a.note = fmt.Sprintf("pasted %d pipeline(s)", len(c.Pipelines)) a.check() } - if !a.changed { + if !a.changed || len(a.comp.Pipelines) == 0 { replace(a) return } @@ -341,18 +424,25 @@ func (a *app) showProblem() { a.note = "the pipelines are valid" return } - if pi, n, key, ok := locate(a.problem); ok && pi < len(a.comp.Pipelines) { + if pi, n, key, _, ok := locate(a.problem); ok && pi < len(a.comp.Pipelines) { a.pi = pi a.choose(n) - a.inspect = true - a.insp.cursor = 0 - for i, l := range a.lines() { - if l.node == n && l.key.Name == key { - a.insp.cursor = i - } - } + a.inspectAt(func(l line) bool { return l.node == n && l.key.Name == key }) } - a.dialog = ¬ice{title: "Problem", lines: ui.WrapText(a.problem.Error(), 60)} + a.dialog = textNotice("Problem", a.problem.Error()) +} + +// inspectFirst opens the inspector on a node's first required key, or on a +// filter's group +func (a *app) inspectFirst(n compose.Node) { + a.inspectAt(func(l line) bool { return l.node == n && (l.key.Required || l.key.Name == "") }) +} + +// inspectAt opens the inspector, which takes the keys from the bar, on the +// first line at holds for, or on its top +func (a *app) inspectAt(at func(line) bool) { + a.inspect, a.bar = true, false + a.insp.cursor = max(0, slices.IndexFunc(a.lines(), at)) } // choose puts the canvas cursor on a node, a filter on the filters stage @@ -374,13 +464,12 @@ func (a *app) choose(n compose.Node) { // it from 110 columns or below it, a status row, and any dialog over them func (a *app) draw(r ui.Region) { r.FillStyle(a.th.Text) + status, style := a.bottom(r.W) if len(a.comp.Pipelines) > 0 { a.pi = min(a.pi, len(a.comp.Pipelines)-1) a.node() // a delete can leave the row past its column's end - p := a.pipeline() - title := fmt.Sprintf("LogWisp pipeline %s %d of %d", p.Name, a.pi+1, len(a.comp.Pipelines)) - r.TextStyled(1, 0, ui.Truncate(title, r.W-2), a.th.Accent) - body := r.Sub(0, 1, r.W, r.H-2) + a.drawBar(r.Sub(0, 0, r.W, 1)) + body := r.Sub(0, 1, r.W, r.H-1-len(status)) if r.W >= 110 { iw := min(60, r.W*2/5) a.drawCanvas(body.Sub(0, 0, r.W-iw-1, body.H)) @@ -391,12 +480,56 @@ func (a *app) draw(r ui.Region) { a.drawInspector(body.Sub(0, ch+1, r.W, body.H-ch-1)) } } - a.drawStatus(r.Sub(0, r.H-1, r.W, 1)) + for i, l := range status { + r.TextStyled(1, r.H-len(status)+i, l, style) + } + if a.status == "" && a.note == "" && a.problem == nil { + keys := a.hints(r.W) + r.TextStyled(r.W-ui.RuneLen(keys)-1, r.H-1, keys, a.th.Muted) + } if a.dialog != nil { a.dialog.draw(a, r) } } +// drawBar draws the title row: the pipelines by name, the one shown marked, +// or it and its place when they do not fit; with the keys, the bar's verbs +func (a *app) drawBar(r ui.Region) { + th, f := a.th, a.font + if a.bar { + r.TextStyled(0, 0, string(f.Focus), th.Accent) + hint := "a add " + f.enter + " rename d delete" + if r.W >= 60 { + r.TextStyled(r.W-ui.RuneLen(hint)-1, 0, hint, th.Muted) + r = r.Sub(0, 0, r.W-ui.RuneLen(hint)-2, 1) + } + } + r.TextStyled(2, 0, "LogWisp", th.Accent) + names, w := a.comp.Pipelines, 11 + for _, p := range names { + w += ui.RuneLen(p.Name) + 4 + } + if w > r.W { + names = names[a.pi : a.pi+1] + } + x := 11 + for _, p := range names { + mark, style := f.Off, th.Muted + if p.Name == a.pipeline().Name { + mark, style = f.On, th.Text + if a.bar { + style = th.Text.On(th.Selected) + } + } + label := ui.Truncate(string(mark)+" "+p.Name, max(0, r.W-x)) + r.TextStyled(x, 0, label, style) + x += ui.RuneLen(label) + 2 + } + if len(names) < len(a.comp.Pipelines) { + r.TextStyled(x, 0, fmt.Sprintf("%d/%d", a.pi+1, len(a.comp.Pipelines)), th.Muted) + } +} + // paint draws the screen into cells, each rune one the font's terminal shows func (a *app) paint(cells []terminal.Cell, w, h int) { a.draw(ui.NewRegion(cells, w, 0, 0, w, h)) @@ -405,23 +538,45 @@ func (a *app) paint(cells []terminal.Cell, w, h int) { } } -// drawStatus shows an edit's refusal, a note, or whether the pipelines are -// valid, and the keys -func (a *app) drawStatus(r ui.Region) { +// bottom is the status rows, wrapped to three at most: an edit's refusal, a +// note, the first problem by its part, or the keys while all is valid +func (a *app) bottom(w int) ([]string, ui.Style) { th, f := a.th, a.font - text, style := f.valid+" valid", th.Accent + text, style := "", th.Error switch { case a.status != "": - text, style = a.status, th.Error + text = a.status case a.note != "": text, style = a.note, th.Muted case a.problem != nil: - text, style = f.invalid+" "+a.problem.Error(), th.Error + pi, n, key, msg, _ := locate(a.problem) + if where := a.where(pi, n, key); where != "" { + msg = where + ": " + msg + } + text = f.invalid + " " + msg + default: + return []string{f.valid + " valid"}, th.Accent } - keys := "a add d delete " + f.space + " on/off J/K move p preset o output ? help" - if ui.RuneLen(keys)+24 > r.W { + return clip(ui.WrapText(text, w-2), 3), style +} + +// hints are the status row's keys, shown while nothing else needs the row +func (a *app) hints(w int) string { + keys := "a add d delete " + a.font.space + " on/off J/K move p preset o output ? help" + if ui.RuneLen(keys)+12 > w { keys = "? help" } - r.TextStyled(1, 0, ui.Truncate(text, r.W-ui.RuneLen(keys)-4), style) - r.TextStyled(r.W-ui.RuneLen(keys)-1, 0, keys, th.Muted) + return keys +} + +// clip keeps n lines, the last cut short when more follow +func clip(lines []string, n int) []string { + if len(lines) > n { + if n < 1 { + return nil + } + lines = lines[:n] + lines[n-1] = ui.Truncate(lines[n-1]+"…", ui.RuneLen(lines[n-1])) + } + return lines } diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index 3328502..87e07b7 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -197,8 +197,9 @@ func TestSelectionChangesAGlyph(t *testing.T) { } // The inspector types a value and sets it, typed by the key's kind, and Tab -// moves on; empty or d returns it to its default; arrows step a choice, -// setting nothing past either end, and Space a switch +// or Up moves on; empty or d returns it to its default; arrows step a choice, +// setting nothing past either end, and set a switch; Enter opens a choice's +// options, at its value, and Space flips a switch func TestInspectorSetsKeys(t *testing.T) { a := mono(pipeline(t, []string{"file"}, []string{"null"})) press(a, terminal.KeyEnter) @@ -221,8 +222,24 @@ func TestInspectorSetsKeys(t *testing.T) { focusKey(t, a, "from") press(a, terminal.KeyLeft) focusKey(t, a, "raw") - press(a, " ") - if o := opts(); o["directory"] != "/srv/a b" || o["check_interval_ms"] != int64(15) || o["from"] != "start" || o["raw"] != true { + if press(a, " ", terminal.KeyLeft, terminal.KeyLeft); opts()["raw"] != false { + t.Fatalf("Left left raw %v", opts()["raw"]) + } + press(a, terminal.KeyRight, terminal.KeyRight) + focusKey(t, a, "pattern") + at = a.insp.cursor + if press(a, terminal.KeyEnter, "x", terminal.KeyUp); a.insp.cursor != at-1 || opts()["pattern"] != "x" { + t.Fatalf("Up left the cursor on row %d, pattern %v", a.insp.cursor, opts()["pattern"]) + } + focusKey(t, a, "from") + if press(a, terminal.KeyEnter); a.dialog.(*chooser).list.Cursor != 0 { + t.Fatalf("the options open off the value: %+v", a.dialog) + } + if press(a, terminal.KeyDown, terminal.KeyEscape); opts()["from"] != "start" { + t.Fatalf("Esc set %v", opts()["from"]) + } + press(a, terminal.KeyEnter, terminal.KeyDown, terminal.KeyEnter) + if o := opts(); o["directory"] != "/srv/a b" || o["check_interval_ms"] != int64(15) || o["from"] != "end" || o["raw"] != true { t.Fatalf("%v", o) } focusKey(t, a, "directory") @@ -294,8 +311,9 @@ func TestFiltersMoveWithTheirFold(t *testing.T) { } } -// o leaves with the chosen form, printed as the engine writes it, or with -// the pipelines to run; a composition that does not validate stays +// o leaves with the form its letter picks, printed as the engine writes it, +// or with the pipelines to run, as r does; a composition that does not +// validate stays func TestOutputLeavesWithTheForm(t *testing.T) { c := pipeline(t, []string{"null"}, []string{"null"}) line, err := c.CommandLine() @@ -303,7 +321,7 @@ func TestOutputLeavesWithTheForm(t *testing.T) { t.Fatal(err) } a := mono(c) - press(a, "o", "comm", terminal.KeyEnter) + press(a, "o", "c") if !a.done || a.result.Exit != Print || a.result.Output != line { t.Fatalf("%+v", a.result) } @@ -312,8 +330,13 @@ func TestOutputLeavesWithTheForm(t *testing.T) { if !a.done || a.result.Exit != Start || len(a.result.Pipelines) != 1 { t.Fatalf("%+v", a.result) } + a = mono(pipeline(t, []string{"null"}, []string{"null"})) + press(a, "r") + if !a.done || a.result.Exit != Start || len(a.result.Pipelines) != 1 { + t.Fatalf("r: %+v", a.result) + } a = mono(pipeline(t, []string{"file"}, []string{"null"})) - press(a, "o", terminal.KeyEnter) + press(a, "o", terminal.KeyEnter, "r") if a.done || a.status == "" { t.Fatalf("ran an invalid composition: %+v", a.result) } @@ -351,7 +374,8 @@ func TestPresetsStartThePipeline(t *testing.T) { } // A pasted command line replaces the pipelines, from the preset menu too, -// asking first once they were edited; one that names none is refused +// asking first once they were edited and while some remain; one that names +// none is refused func TestPasteReplacesThePipelines(t *testing.T) { a := mono(&compose.Composition{}) if press(a, []byte("lw\n")); a.status == "" { @@ -368,22 +392,42 @@ func TestPasteReplacesThePipelines(t *testing.T) { if a.pipeline().Name != "x" { t.Fatalf("replaced edited pipelines without asking: %+v", a.comp.Pipelines) } + a = mono(pipeline(t, []string{"null"}, []string{"null"})) + press(a, "k", "d", "y", []byte("lw --pipeline z --source null --sink null")) + if a.dialog != nil || len(a.comp.Pipelines) != 1 || a.pipeline().Name != "z" { + t.Fatalf("a paste after the last pipeline was deleted: %T %+v", a.dialog, a.comp.Pipelines) + } } -// The first problem marks the part it names, and c moves there, to the key +// The first problem marks the part it names, and ! moves there, to the key func TestProblemNamesItsPart(t *testing.T) { a := mono(pipeline(t, []string{"null"}, []string{"null", "tcp"})) sink := compose.Node{Role: "sink", ID: "tcp"} if !a.faulty(sink) || a.faulty(compose.Node{Role: "source", ID: "null"}) || a.faulty(compose.Node{Role: "sink", ID: "null"}) { t.Fatalf("problem %v", a.problem) } - press(a, "c") + press(a, "!") n, _ := a.node() if l := a.lines()[a.insp.cursor]; n != sink || !a.inspect || l.key.Name != "port" { t.Fatalf("on %+v, row %+v", n, l) } } +// ! and c on the pipeline bar open the inspector, which then takes the keys: +// Enter edits the key it shows, not the pipeline's name +func TestTheInspectorTakesTheKeysFromTheBar(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, []string{"tcp"})) + press(a, "k", "!", terminal.KeyEscape, terminal.KeyEnter) + if a.bar || a.insp.field == nil { + t.Fatalf("Enter after ! on the bar: bar %v, %T", a.bar, a.dialog) + } + a = mono(pipeline(t, []string{"null"}, []string{"null"})) + press(a, "k", "c", terminal.KeyEnter, terminal.KeyEnter) + if a.bar || !a.inspect || a.dialog != nil { + t.Fatalf("Enter after c on the bar: bar %v, %T", a.bar, a.dialog) + } +} + // 16 colors and none keep the terminal's own background; more paint the site's func TestOnlyFullColorPaintsTheBackground(t *testing.T) { for mode, l := range looks { @@ -451,7 +495,9 @@ func TestPresetFormShowsTheFocusedField(t *testing.T) { presets := config.Presets() p := slices.MaxFunc(presets, func(x, y config.Preset) int { return len(x.Params) - len(y.Params) }) a := mono(pipeline(t, []string{"null"}, []string{"null"})) - a.dialog = newPresetForm(p) + f := &form{title: p.Name, params: p.Params} + f.fill(nil) + a.dialog = f press(a, terminal.KeyBacktab) last := p.Params[len(p.Params)-1].Name if screen := strings.Join(text(render(a, 80, 12), 80), "\n"); !strings.Contains(screen, last) { @@ -459,6 +505,96 @@ func TestPresetFormShowsTheFocusedField(t *testing.T) { } } +// presetForm is a preset's form, open on a +func presetForm(a *app, name string) *form { + presets := config.Presets() + p := presets[slices.IndexFunc(presets, func(p config.Preset) bool { return p.Name == name })] + f := &form{title: "preset " + p.Name, summary: p.Summary, params: p.Params} + f.fill(nil) + a.dialog = f + return f +} + +// A preset form is usable at 25 columns: labels stack above their values, +// which take the row, so typed text shows whole +func TestPresetFormFitsANarrowScreen(t *testing.T) { + a := mono(pipeline(t, nil, nil)) + presetForm(a, "serve") + press(a, terminal.KeyTab, terminal.KeyTab, terminal.KeyTab, "127.0.0.1:9000") + if screen := strings.Join(text(render(a, 25, 20), 25), "\n"); !strings.Contains(screen, "listen") || !strings.Contains(screen, "127.0.0.1:9000") { + t.Fatalf("listen and its value not whole at 25 columns:\n%s", screen) + } +} + +// A dialog is sized to what it holds: its first and last rows inside the +// box have text, a form without a summary included +func TestDialogsHaveNoBlankEdgeRows(t *testing.T) { + var a *app + for name, open := range map[string]func(){ + "add": func() { press(a, "a") }, + "presets": func() { press(a, "p") }, + "output": func() { press(a, "o") }, + "confirm": func() { press(a, "q") }, + "help": func() { press(a, "?") }, + "problem": func() { press(a, "!") }, + "rename": func() { press(a, terminal.KeyUp, terminal.KeyEnter) }, + "form": func() { presetForm(a, "serve") }, + } { + a = mono(pipeline(t, nil, nil)) + a.changed, a.dialog = true, nil + open() + rows := text(render(a, 80, 40), 80) + bottom := slices.IndexFunc(rows, func(r string) bool { return strings.Contains(r, "└") }) + top := slices.IndexFunc(rows, func(r string) bool { return strings.Contains(r, "┌") }) + if a.dialog == nil || top < 0 || bottom < top+2 { + t.Errorf("%s: no dialog:\n%s", name, strings.Join(rows, "\n")) + continue + } + edge := []rune(rows[bottom]) + left, right := slices.Index(edge, '└'), slices.Index(edge, '┘') + for _, y := range []int{top + 1, bottom - 1} { + if inside := strings.Trim(string([]rune(rows[y])[left:right]), " │"); inside == "" { + t.Errorf("%s: row %d blank:\n%s", name, y, strings.Join(rows[top:bottom+1], "\n")) + } + } + } +} + +// A question is answered by y or n, or by Enter on the answer the arrows +// chose; no is chosen first +func TestConfirmAnswersOnEnter(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, []string{"null"})) + a.changed = true + if press(a, "q", terminal.KeyEnter); a.done || a.dialog != nil { + t.Fatalf("Enter on no: done %v, dialog %T", a.done, a.dialog) + } + if press(a, "q", terminal.KeyLeft, terminal.KeyEnter); !a.done { + t.Fatal("Enter on yes did not quit") + } +} + +// The canvas scrolls to keep the chosen part in view, its bar solid +func TestCanvasKeepsTheChosenInView(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, slices.Repeat([]string{"null"}, 8))) + a.col, a.row[sinksCol] = sinksCol, 7 + if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.ContainsRune(screen, a.font.barOn) { + t.Fatalf("the eighth sink is out of view:\n%s", screen) + } +} + +// A notice taller than the screen scrolls with j and k and says where it is +func TestNoticeScrolls(t *testing.T) { + a := mono(pipeline(t, nil, nil)) + press(a, "?") + screen := func() string { return strings.Join(text(render(a, 25, 20), 25), "\n") } + if s := screen(); strings.Contains(s, "quit") || !strings.Contains(s, "j k: 1-") { + t.Fatalf("help at 25 columns:\n%s", s) + } + if press(a, strings.Repeat("j", 60)); a.dialog == nil || !strings.Contains(screen(), "quit") { + t.Fatalf("scrolled to the end:\n%s", screen()) + } +} + // A dialog's text wraps to a narrow screen func TestDialogsWrapToTheScreen(t *testing.T) { a := mono(pipeline(t, []string{"null"}, []string{"null"})) @@ -472,3 +608,103 @@ func TestDialogsWrapToTheScreen(t *testing.T) { } } } + +// The inspector opens on a part's first required key, after a adds it or +// Enter chooses it +func TestInspectorOpensOnTheFirstRequiredKey(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, []string{"tcp_chain"})) + a.col = sinksCol + press(a, terminal.KeyEnter) + at := func() string { return a.lines()[a.insp.cursor].key.Name } + if got := at(); got != "host" { + t.Fatalf("Enter on tcp_chain: %s", got) + } + press(a, terminal.KeyEscape, "a", "http", terminal.KeyEnter) + if got := at(); got != "port" { + t.Fatalf("a adding http: %s", got) + } +} + +// The status shows the first problem whole, by its part and key, not by its +// path, on as many rows as it takes +func TestTheProblemShowsWhole(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, []string{"http"})) + sink := compose.Node{Role: "sink", ID: "http"} + for key, value := range map[string]string{"port": "8080", "viewer_page": "true"} { + if err := a.comp.Set(0, sink, key, value); err != nil { + t.Fatal(err) + } + } + a.check() + for _, w := range []int{60, 87, 146} { + screen := strings.Join(text(render(a, w, 30), w), "\n") + if !strings.Contains(screen, "sink http · viewer_page: only") || !strings.Contains(screen, "at /") || strings.Contains(screen, "pipelines[0]") { + t.Errorf("%d columns:\n%s", w, screen) + } + } +} + +// Enter on a row that adds acts as a does: an empty column's slot, the +// filters stage with none, the inspector's last filter row +func TestEnterOnAnAddRowAdds(t *testing.T) { + a := mono(pipeline(t, nil, []string{"null"})) + if press(a, terminal.KeyEnter); a.dialog == nil { + t.Fatal("Enter on the empty sources column") + } + press(a, "null", terminal.KeyEnter, terminal.KeyEscape, terminal.KeyRight, terminal.KeyDown, terminal.KeyEnter, "exc", terminal.KeyEnter) + if f := a.pipeline().Flow.Filters; len(f) != 1 || f[0].Type != "exclude" { + t.Fatalf("Enter on filters: %+v", f) + } + lines := a.lines() + a.insp.cursor = len(lines) - 1 + press(a, terminal.KeyEnter, terminal.KeyEnter) + if f := a.pipeline().Flow.Filters; len(f) != 2 { + t.Fatalf("Enter on the add row: %+v", f) + } +} + +// c changes a source's or sink's type in its place and opens it; on a filter +// or the format it goes to the type row +func TestChangeTypeKeepsThePlace(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, []string{"null", "file"})) + a.col = sinksCol + press(a, "c", "tcp", terminal.KeyEnter) + sinks := a.pipeline().PluginSinks + if sinks[0].Type != "tcp" || sinks[0].ID != "tcp" || sinks[1].Type != "file" || !a.inspect || a.lines()[a.insp.cursor].key.Name != "port" { + t.Fatalf("%+v, cursor on %+v", sinks, a.lines()[a.insp.cursor]) + } + press(a, terminal.KeyEscape) + a.choose(compose.Node{Role: "format"}) + if press(a, "c"); !a.inspect || a.lines()[a.insp.cursor].key.Name != "type" { + t.Fatalf("c on format: %+v", a.lines()[a.insp.cursor]) + } +} + +// The pipeline bar, above the parts, switches pipelines, adds one under a free +// name, renames one under a name no other has, and deletes one, asking first +// when it has parts +func TestThePipelineBarAddsRenamesDeletes(t *testing.T) { + a := mono(pipeline(t, []string{"null"}, []string{"null"})) + press(a, "k", "a", "empty", terminal.KeyEnter, "k", "a", "empty", terminal.KeyEnter) + names := func() (out []string) { + for _, p := range a.comp.Pipelines { + out = append(out, p.Name) + } + return out + } + if got := names(); !slices.Equal(got, []string{"p", "default", "default_2"}) || a.pi != 2 { + t.Fatalf("added %q, showing %d", got, a.pi) + } + press(a, "k", terminal.KeyEnter, terminal.KeyCtrlU, "p", terminal.KeyEnter) + if _, open := a.dialog.(*form); !open { + t.Fatalf("renamed to a name another has: %q", names()) + } + press(a, terminal.KeyCtrlU, "x", terminal.KeyEnter, "h", "h", "d") + if got := names(); !slices.Equal(got, []string{"p", "default", "x"}) || a.dialog == nil { + t.Fatalf("deleted without asking: %q", got) + } + press(a, "y", "l", "d") + if got := names(); !slices.Equal(got, []string{"default"}) || a.dialog != nil { + t.Fatalf("%q %T", got, a.dialog) + } +} diff --git a/test/acl-test.sh b/test/acl-test.sh index 355dd2d..1afb91a 100755 --- a/test/acl-test.sh +++ b/test/acl-test.sh @@ -96,22 +96,22 @@ section "Rules" rejects() { ! lw --check --source null --sink "$1" >/dev/null 2>"$LOG/check.err" && grep -q -- "$2" "$LOG/check.err" && echo 1 || echo 0; } # warns SINK PATTERN: lw --check builds SINK and WARNs PATTERN warns() { lw --check --source null --sink "$1" 2>&1 >/dev/null | grep ' WARN ' | grep -q -- "$2" && echo 1 || echo 0; } -check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp,host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")" -check "an acl on a dialer is refused" "$(rejects 'tcp_chain,host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')" -check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp,host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')" -check "a deny list alone on a wildcard listener warns" "$(warns 'tcp,port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')" -check "an allow entry of a whole family warns" "$(warns 'tcp,port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')" +check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp:host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")" +check "an acl on a dialer is refused" "$(rejects 'tcp_chain:host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')" +check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp:host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')" +check "a deny list alone on a wildcard listener warns" "$(warns 'tcp:port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')" +check "an allow entry of a whole family warns" "$(warns 'tcp:port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')" check "a proxy_from entry reaching public addresses warns" \ - "$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=203.0.113.0/24' 'proxy_from reaches public addresses')" + "$(warns 'tcp:host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=203.0.113.0/24' 'proxy_from reaches public addresses')" check "a request rate on a tcp listener is refused" \ - "$(rejects 'tcp,host=127.0.0.1,port=15881,acl.requests_per_second_per_client=5' 'applies only to HTTP listeners')" + "$(rejects 'tcp:host=127.0.0.1,port=15881,acl.requests_per_second_per_client=5' 'applies only to HTTP listeners')" check "a connection cap behind trusted_proxies, which would count the proxy, is refused" \ - "$(rejects "http,host=127.0.0.1,port=15881,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.max_connections_per_client=2" 'behind trusted_proxies')" + "$(rejects "http:host=127.0.0.1,port=15881,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.max_connections_per_client=2" 'behind trusted_proxies')" check "optional proxy_protocol warns that a headerless client passes as the proxy" \ - "$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=127.0.0.1' 'passes as the proxy')" + "$(warns 'tcp:host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=127.0.0.1' 'passes as the proxy')" section "HTTP sink" -spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2" +spawn serve "$BIN" --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2" wait_port 15881 || abort "the serve preset did not listen" serve check "a client the allow list names reads the status" "$(get http://127.0.0.1:15881/status | grep -q '"acl":"allow=1 deny=1"' && echo 1 || echo 0)" get --interface 127.0.0.2 -o /dev/null http://127.0.0.1:15881/status @@ -127,7 +127,7 @@ section "TCP sink" # the lines of a finite input, once the clients had time to connect seq 1 20000 | sed 's/^/flush /' >"$RUN/flush.in" { wait_port 15882; sleep 1; cat "$RUN/flush.in"; } | - lw --sink "tcp,host=127.0.0.1,port=15882,acl.allow=127.0.0.1,buffer_size=20000,client_buffer_size=20000" 2>"$LOG/tcp.err" & + lw --sink "tcp:host=127.0.0.1,port=15882,acl.allow=127.0.0.1,buffer_size=20000,client_buffer_size=20000" 2>"$LOG/tcp.err" & fpid=$! wait_port 15882 tcp_read 15882 15 >"$RUN/flush.out" & @@ -141,7 +141,7 @@ check "one it denies gets nothing ($denied bytes) and is logged" \ section "Behind an L7 proxy" # proxy mode over plaintext loopback: curl is the proxy, its headers name the client -spawn web-proxied "$BIN" --source null --sink "http,host=127.0.0.1,port=15882,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.deny=198.51.100.5,acl.deny=2001:db8::/32,acl.requests_per_second_per_client=1" +spawn web-proxied "$BIN" --source null --sink "http:host=127.0.0.1,port=15882,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.deny=198.51.100.5,acl.deny=2001:db8::/32,acl.requests_per_second_per_client=1" wait_port 15882 || abort "the proxy-mode http sink did not listen" web-proxied # as CLIENT [PATH [CURL ARGS]]: the status of a request forwarded for CLIENT, # GET /status by default; there 401 is past the acl, short of a login @@ -162,9 +162,9 @@ section "Chain sources" # pin DAEMON: the pin_sha256 a self-signed listener logged pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; } received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; } -edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; } -spawn agg-allow "$BIN" --preset "aggregator,listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow" -spawn agg-deny "$BIN" --preset "aggregator,listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny" +edge() { spawn "$1" "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; } +spawn agg-allow "$BIN" --preset "aggregator:listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow" +spawn agg-deny "$BIN" --preset "aggregator:listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny" wait_port 15884 && wait_port 15885 || abort "an aggregator did not listen" agg-allow edge edge-allow 127.0.0.1:15884 tcp "$(pin agg-allow)" edge edge-deny 127.0.0.1:15885 http "$(pin agg-deny)" @@ -181,16 +181,16 @@ section "Behind a PROXY header" # connect from; each is up before a proxy shares its port on another address behind=acl.proxy_protocol=required,acl.proxy_from=127.0.0.1 scram="tls.enabled=true,tls.self_signed=true,auth.type=scram,auth.credentials_file=$RUN/users.toml" -spawn sink-behind "$BIN" --logging.level=info --source null --sink "http,host=127.0.0.1,port=15886,$scram,$behind,acl.deny=198.51.100.2" -spawn tcp-capped "$BIN" --source null --sink "tcp,host=127.0.0.1,port=15889,$behind,acl.max_connections_per_client=2" +spawn sink-behind "$BIN" --logging.level=info --source null --sink "http:host=127.0.0.1,port=15886,$scram,$behind,acl.deny=198.51.100.2" +spawn tcp-capped "$BIN" --source null --sink "tcp:host=127.0.0.1,port=15889,$behind,acl.max_connections_per_client=2" for t in tcp http; do spawn "agg-$t-behind" "$BIN" --logging.level=info \ - --source "${t}_chain,host=127.0.0.1,port=$([[ $t == tcp ]] && echo 15888 || echo 15883),$scram,$behind" \ - --sink "file,directory=$RUN/out/$t-behind,name=aggregate" + --source "${t}_chain:host=127.0.0.1,port=$([[ $t == tcp ]] && echo 15888 || echo 15883),$scram,$behind" \ + --sink "file:directory=$RUN/out/$t-behind,name=aggregate" done # the tcp sink sends its input once the clients connected { wait_until 30 test -e "$RUN/feed"; seq 1 100 | sed 's/^/behind /'; } | - lw --sink "tcp,host=127.0.0.1,port=15887,$behind,acl.deny=198.51.100.2,acl.deny=127.0.0.6" 2>"$LOG/tcp-behind.out" & + lw --sink "tcp:host=127.0.0.1,port=15887,$behind,acl.deny=198.51.100.2,acl.deny=127.0.0.6" 2>"$LOG/tcp-behind.out" & fpid=$! for p in 15883 15886 15887 15888 15889; do wait_port $p || abort "a listener behind the proxy did not listen on $p" sink-behind; done spawn proxy "$RUN/proxy/proxy" 127.0.0.11:15886,127.0.0.1:15886,1,198.51.100.1 \ diff --git a/test/pipe-test.sh b/test/pipe-test.sh index 2d579a7..2b8540e 100755 --- a/test/pipe-test.sh +++ b/test/pipe-test.sh @@ -55,14 +55,14 @@ rc=${PIPESTATUS[1]} check "reader gone: lw exits on SIGPIPE ($rc)" "$((rc == 141))" # 7. filters and formats compose with stdin and stdout -check "--filter keeps matching lines" "$(is "$(printf 'x INFO\ny ERROR\n' | lw bare --filter include,patterns=ERROR)" "y ERROR")" +check "--filter keeps matching lines" "$(is "$(printf 'x INFO\ny ERROR\n' | lw bare --filter include:patterns=ERROR)" "y ERROR")" # 8. control characters pass to a pipe unchanged and are escaped on request check "control bytes reach a pipe unchanged" "$(is "$(printf 'a\033[31mb\n' | lw bare | od -An -tx1 | tr -d ' ')" 611b5b33316d620a)" -check "escape=always writes them as " "$(is "$(printf 'a\033[31mb\n' | lw bare --sink console,escape=always)" 'a<1b>[31mb')" +check "escape=always writes them as " "$(is "$(printf 'a\033[31mb\n' | lw bare --sink console:escape=always)" 'a<1b>[31mb')" # 9. at the end of input the file sink writes its queue before lw exits -seq 1 5000 | lw bare --sink "file,directory=$RUN/fsink,name=out,buffer_size=10000" +seq 1 5000 | lw bare --sink "file:directory=$RUN/fsink,name=out,buffer_size=10000" n=$(cat "$RUN"/fsink/out*.log 2>/dev/null | wc -l) check "file sink holds all 5000 lines at exit ($n)" "$((n == 5000))" diff --git a/test/preset-test.sh b/test/preset-test.sh index 7690eb2..4165d85 100755 --- a/test/preset-test.sh +++ b/test/preset-test.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash # logwisp preset test: colors, presets, --dump, lw tls, listeners that make # their certificate (self-signed and pinned, or issued), the serve preset's -# browser viewer, and the network sinks delivering a finite input whole. +# browser viewer, the network sinks delivering a finite input whole, and a +# temporary pipe: one SCRAM user whose password is on no disk. # One-shot: both modes run the checks. -# Ports: 15890-15896 on 127.0.0.1. -# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only. +# Ports: 15890-15898 on 127.0.0.1. +# Requires: bash 5+, coreutils, curl, openssl; script(1) for the terminal +# checks. Linux dev host only. set -u . "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" @@ -14,7 +16,7 @@ e2e_init "$@" section "Setup" need curl openssl -ports_free 15890 15891 15892 15893 15894 15895 15896 +ports_free 15890 15891 15892 15893 15894 15895 15896 15897 15898 rm -rf "$RUN" mkdir -p "$RUN/out" "$LOG" cd "$RUN" || abort "no run directory" @@ -31,15 +33,15 @@ check "--color paints the level name" "$(is "$(printf 'a ERROR b\n' | lw --color check "--color never, and auto on a pipe, leave it plain" \ "$(is "$(printf 'a ERROR b\n' | lw --color never)$(printf 'a ERROR b\n' | lw)" "a ERROR ba ERROR b")" check "LOGWISP_COLOR=always, a sink's own color=never wins" \ - "$(is "$(printf 'x WARN\n' | LOGWISP_COLOR=always lw --sink console,color=never)" "x WARN")" + "$(is "$(printf 'x WARN\n' | LOGWISP_COLOR=always lw --sink console:color=never)" "x WARN")" section "Presets" seq 1 5000 >"$RUN/seq.txt" lw --preset pipe <"$RUN/seq.txt" >"$RUN/pipe.out" check "--preset pipe copies stdin line for line" "$(cmp -s "$RUN/seq.txt" "$RUN/pipe.out" && echo 1 || echo 0)" -out=$(timeout 3 "$BIN" --preset "tail,path=$RUN/edge.log,from=start" /dev/null | head -2 | tr '\n' '|') +out=$(timeout 3 "$BIN" --preset "tail:path=$RUN/edge.log,from=start" /dev/null | head -2 | tr '\n' '|') check "--preset tail follows a file" "$(is "$out" "edge line 1|edge line 2|")" -lw --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15890,tls=self,hosts=logs.example" --color --dump >"$RUN/dump.toml" +lw --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15890,tls=self,hosts=logs.example" --color --dump >"$RUN/dump.toml" lw -c "$RUN/dump.toml" --dump >"$RUN/dump2.toml" check "--dump prints a file that reads back to the same dump" "$(cmp -s "$RUN/dump.toml" "$RUN/dump2.toml" && echo 1 || echo 0)" check "lw --check builds the dumped preset" "$(lw --check -c "$RUN/dump.toml" 2>"$LOG/check.err" | grep -q 'configuration ok' && echo 1 || echo 0)" @@ -65,24 +67,24 @@ add_users "$RUN/users.toml" edge-01 pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; } received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; } -spawn agg-self "$BIN" --preset "aggregator,listen=127.0.0.1:15893,users=$RUN/users.toml,out=$RUN/out/self" +spawn agg-self "$BIN" --preset "aggregator:listen=127.0.0.1:15893,users=$RUN/users.toml,out=$RUN/out/self" wait_port 15893 || abort "the self-signed aggregator did not listen" agg-self PIN=$(pin agg-self) check "a self-signed aggregator logs its pin" "$(is_set "$PIN")" -spawn edge-pin "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" -spawn edge-wrong "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=sha256//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,user=edge-01,password_file=$RUN/edge-01.pass" +spawn edge-pin "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" +spawn edge-wrong "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=sha256//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,user=edge-01,password_file=$RUN/edge-01.pass" wait_until 10 eval '(( $(received self) >= 300 ))' check "an edge pinning it delivers every line over SCRAM ($(received self))" "$(( $(received self) == 300 ))" wait_until 10 logged edge-wrong 'matches no tls.pin_sha256' check "an edge with another pin is refused before it logs in" "$(logged edge-wrong 'matches no tls.pin_sha256' && echo 1 || echo 0)" -spawn agg-issuer "$BIN" --preset "aggregator,listen=127.0.0.1:15894,transport=http,tls=issuer,issuer_cert=$RUN/pki/ca.crt,issuer_key=$RUN/pki/ca.key,users=$RUN/users.toml,out=$RUN/out/issuer" +spawn agg-issuer "$BIN" --preset "aggregator:listen=127.0.0.1:15894,transport=http,tls=issuer,issuer_cert=$RUN/pki/ca.crt,issuer_key=$RUN/pki/ca.key,users=$RUN/users.toml,out=$RUN/out/issuer" wait_port 15894 || abort "the issuing aggregator did not listen" agg-issuer -spawn edge-ca "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" +spawn edge-ca "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" wait_until 10 eval '(( $(received issuer) >= 300 ))' check "an aggregator signing its own certificate is verified by the CA ($(received issuer))" "$(( $(received issuer) == 300 ))" -spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15895,tls=self" +spawn serve "$BIN" --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15895,tls=self" wait_port 15895 || abort "the serve preset did not listen" serve SERVE_PIN=$(pin serve) check "curl --pinnedpubkey reads a self-signed stream's status" \ @@ -93,7 +95,7 @@ check "a browser at the self-signed root lands on the viewer" \ "$(curl -sLk --pinnedpubkey "$SERVE_PIN" https://127.0.0.1:15895/ | grep -q 'name="logwisp-login" content="none"' && echo 1 || echo 0)" section "Viewer" -spawn serve-plain "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15896" +spawn serve-plain "$BIN" --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15896" wait_port 15896 || abort "the plain serve preset did not listen" serve-plain root=$(curl -s -o /dev/null -w '%{http_code} %{redirect_url}' http://127.0.0.1:15896/) check "GET / sends a browser to the viewer, which needs no login over plain http ($root)" \ @@ -103,13 +105,13 @@ section "Flush at exit" # feed SECONDS PORT: the lines, once a client had time to connect feed() { wait_port "$2"; sleep "$1"; seq 1 20000 | sed 's/^/flush /'; } queues='buffer_size=20000,client_buffer_size=20000' -feed 1 15891 | lw --sink "tcp,host=127.0.0.1,port=15891,$queues" 2>"$LOG/flush-tcp.err" & +feed 1 15891 | lw --sink "tcp:host=127.0.0.1,port=15891,$queues" 2>"$LOG/flush-tcp.err" & fpid=$! wait_port 15891 n=$(tcp_read 15891 15 | grep -c '^flush ') wait "$fpid" check "a tcp client gets all 20000 lines of a finite input ($n)" "$((n == 20000))" -feed 1 15892 | lw --sink "http,host=127.0.0.1,port=15892,$queues" 2>"$LOG/flush-http.err" & +feed 1 15892 | lw --sink "http:host=127.0.0.1,port=15892,$queues" 2>"$LOG/flush-http.err" & fpid=$! wait_port 15892 curl -sN --max-time 15 http://127.0.0.1:15892/stream >"$RUN/flush-http.out" @@ -119,11 +121,65 @@ check "an http stream gets all 20000 lines, then the disconnect event ($n)" \ "$( ((n == 20000)) && tail -3 "$RUN/flush-http.out" | grep -q '^event: disconnect' && echo 1 || echo 0)" # 1000 lines: the chain sink's queue, which holds them until the link is up flushed() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c '"flush '; } -seq 1 1000 | sed 's/^/flush /' | lw --preset "edge,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-tcp-chain.err" +seq 1 1000 | sed 's/^/flush /' | lw --preset "edge:to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-tcp-chain.err" wait_until 5 eval '(( $(flushed self) >= 1000 ))' check "a tcp_chain edge delivers all 1000 lines before it exits ($(flushed self))" "$(( $(flushed self) == 1000 ))" -seq 1 1000 | sed 's/^/flush /' | lw --preset "edge,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-http-chain.err" +seq 1 1000 | sed 's/^/flush /' | lw --preset "edge:to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-http-chain.err" wait_until 5 eval '(( $(flushed issuer) >= 1000 ))' check "an http_chain edge delivers all 1000 lines before it exits ($(flushed issuer))" "$(( $(flushed issuer) == 1000 ))" +section "Temporary pipe" +# One user on both ends, its password through a descriptor; printf is a +# builtin, so the password is in no argv either +PW=$(head -c 18 /dev/urandom | base64 | tr '+/' '-_') +pw() { printf '%s\n' "$1"; } +piped() { cat "$RUN/out/pipe"/aggregate*.log 2>/dev/null | grep -c '^pipe line'; } +piped_line() { cat "$RUN/out/pipe"/aggregate*.log 2>/dev/null | grep -qx "pipe line $1"; } +seq 1 100 | sed 's/^/pipe line /' >"$RUN/pipe.log" +spawn agg-pipe "$BIN" --logging.level=info --preset "aggregator:listen=127.0.0.1:15897,user=pipe,password_file=/dev/fd/3,format=raw,out=$RUN/out/pipe" 3< <(pw "$PW") +AGG_PIPE=${PIDS[-1]} +wait_port 15897 || abort "the pipe aggregator did not listen" agg-pipe +PIPE_PIN=$(pin agg-pipe) +spawn edge-pipe "$BIN" --logging.level=info --preset "edge:path=$RUN/pipe.log,from=start,to=127.0.0.1:15897,pin=$PIPE_PIN,user=pipe,password_file=/dev/fd/3" 3< <(pw "$PW") +EDGE_PIPE=${PIDS[-1]} +spawn edge-pipe-wrong "$BIN" --preset "edge:path=$RUN/pipe.log,from=start,to=127.0.0.1:15897,pin=$PIPE_PIN,user=pipe,password_file=/dev/fd/3" 3< <(pw another-password) +wait_until 10 eval '(( $(piped) >= 100 ))' +check "an aggregator and an edge given one password on /dev/fd/3 deliver every line ($(piped))" "$(( $(piped) == 100 ))" +wait_until 10 logged edge-pipe-wrong 'authentication failed' +check "an edge with another password is refused" "$(logged edge-pipe-wrong 'authentication failed' && echo 1 || echo 0)" +check "the descriptor, read and closed, draws no permission warning" "$(logged agg-pipe 'readable by all users' && echo 0 || echo 1)" +reloaded() { logged agg-pipe 'hot reload completed' && logged edge-pipe 'hot reload completed'; } +kill -HUP "$AGG_PIPE" "$EDGE_PIPE" +wait_until 10 reloaded +check "SIGHUP reloads both, the password kept from the first read" "$(reloaded && echo 1 || echo 0)" +seq 101 150 | sed 's/^/pipe line /' >>"$RUN/pipe.log" +wait_until 10 piped_line 150 +check "a second batch is delivered after the reload" "$(piped_line 150 && echo 1 || echo 0)" +leaks=$(grep -rlF -- "$PW" "$RUN"; for p in "$AGG_PIPE" "$EDGE_PIPE"; do tr '\0' '\n' <"/proc/$p/cmdline"; tr '\0' '\n' <"/proc/$p/environ"; done | grep -F -- "$PW") +check "the password is in no file of the run directory, no argv and no environment" "$(is "$leaks" "")" + +if command -v script >/dev/null && command -v setsid >/dev/null; then + setsid -w "$BIN" --check --preset aggregator:listen=127.0.0.1:15898,user=gen "$LOG/no-tty.out" 2>&1 + check "with no terminal to ask on, a listener's user fails closed" "$(grep -q 'no terminal to ask on' "$LOG/no-tty.out" && echo 1 || echo 0)" + # script(1) is lw's terminal, what lw shows there lands in agg-gen-tty.out; + # not spawn, whose background command would read /dev/null, not the Enter + script -qefc "$(printf '%q ' "$BIN" --preset "aggregator:listen=127.0.0.1:15898,transport=http,user=gen,format=raw,out=$RUN/out/gen")2>$(printf %q "$LOG/agg-gen.out")" /dev/null \ + < <(printf '\n') >"$LOG/agg-gen-tty.out" 2>&1 & + PIDS+=($!) DAEMONS+=(agg-gen-tty) + wait_until 10 logged agg-gen-tty 'generated password' + GEN=$(grep -o 'shown once: [A-Z2-7]*' "$LOG/agg-gen-tty.out" | cut -d' ' -f3) + check "Enter alone at a listener's prompt shows a generated password on its terminal" "$(is_set "$GEN")" + wait_port 15898 || abort "the generating aggregator did not listen" agg-gen-tty + GEN_PIN=$(pin agg-gen) + check "and not in lw's log or stderr" "$([[ -n $GEN ]] && ! grep -qF -- "$GEN" "$LOG/agg-gen.out" && echo 1 || echo 0)" + generated() { cat "$RUN/out/gen"/aggregate*.log 2>/dev/null | grep -c '^edge line'; } + spawn edge-gen "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15898,transport=http,pin=$GEN_PIN,user=gen,password_file=/dev/fd/3" 3< <(pw "$GEN") + wait_until 10 eval '(( $(generated) >= 300 ))' + check "an edge given the generated password delivers every line ($(generated))" "$(( $(generated) == 300 ))" + token=$(pw "$GEN" | script -qefc "$(printf '%q ' "$BIN" auth token --url https://127.0.0.1:15898 -u gen --pin-sha256 "$GEN_PIN")" /dev/null | tr -d '\r' | tail -1) + check "lw auth token asks on the terminal without --password-file" "$([[ $token =~ ^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]] && echo 1 || echo 0)" +else + skip "the terminal checks: no script(1) or setsid(1)" +fi + summary