lixen/logwisp
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.31.0 refinements

Browse Source
This commit is contained in:
lixen committed 2026-10-10 12:01:54 -04:00
1 parent bfcc15613a
commit 6bd5e12a60
52 files changed
+1915 -650
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

README.md
+6 -6
View File
@@ -23,7 +23,7 @@ streams, or downstream LogWisp nodes.
to stdout line for line and exits at the end of input; flags add filters,
formats, sources and sinks
- **Presets**: `tail`, `serve`, `edge` and `aggregator` build the common
pipelines from a few keys (`lw --preset tail,path=/var/log/app`);
pipelines from a few keys (`lw --preset tail:path=/var/log/app`);
`lw --dump` turns any command line into a configuration file
- **Independent pipelines**, each `sources → flow → sinks`, running concurrently
in one process
@@ -132,18 +132,18 @@ one without `--sink` writes stdout:
```bash
./bin/lw < app.log > copy.log
tail -F app.log | ./bin/lw --filter include,patterns=ERROR,patterns=WARN
tail -F app.log | ./bin/lw --filter include:patterns=ERROR,patterns=WARN
# stdin as a live SSE stream, with a browser viewer at http://127.0.0.1:8080/;
# the http sink binds 0.0.0.0 unless told
journalctl -f | ./bin/lw --sink http,host=127.0.0.1,port=8080
journalctl -f | ./bin/lw --sink http:host=127.0.0.1,port=8080
# tail a directory instead of stdin
./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \
--format json,sanitizer_policy=json > all.json
./bin/lw --source 'file:directory=/var/log/myapp,pattern=*.log' \
--format json:sanitizer_policy=json > all.json
# the same with a preset; on a terminal the level names are in color
./bin/lw --preset tail,path=/var/log/myapp
./bin/lw --preset tail:path=/var/log/myapp
```
As a service, a configuration file holds the pipelines:
cmd/lw/auth.go
+7 -7
View File
@@ -35,12 +35,12 @@ var authCommands = []subcommand{
{"remove-user", "--credentials FILE -u NAME",
"Remove a user from a credentials file",
[]string{"credentials", "user"}, defineRemoveUser},
{"token", "--url https://HOST:PORT -u NAME --password-file FILE [--unbound] [TLS flags]",
{"token", "--url https://HOST:PORT -u NAME [--password-file FILE] [--unbound] [TLS flags]",
"Log in to an http sink or http_chain source and print a bearer token",
[]string{"url", "user", "password-file"}, defineToken},
{"stream", "--addr HOST:PORT -u NAME --password-file FILE [TLS flags]",
[]string{"url", "user"}, defineToken},
{"stream", "--addr HOST:PORT -u NAME [--password-file FILE] [TLS flags]",
"Log in to a tcp sink and copy its stream to stdout until interrupted",
[]string{"addr", "user", "password-file"}, defineStream},
[]string{"addr", "user"}, defineStream},
}
func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
@@ -62,7 +62,7 @@ func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
var password string
if *passwordFile != "" && !*generate {
password, err = authz.ReadPassword(*passwordFile)
password, err = authz.ReadPassword(*passwordFile, *user, false)
switch {
case err == nil && len(password) < 8:
return fmt.Errorf("the password in %s is shorter than 8 bytes", *passwordFile)
@@ -155,7 +155,7 @@ func defineRemoveUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT or https://[IPV6]:PORT (with --unbound, a proxy's mount URL may carry a path)")
user := fs.String("user", "", "user `name`")
passwordFile := fs.String("password-file", "", "`file` holding the password")
passwordFile := fs.String("password-file", "/dev/tty", "`file` holding the password; a terminal asks")
unbound := fs.Bool("unbound", false, "log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)")
tlsOpts := tlsFlags(fs)
return func(stdout, _ io.Writer) error {
@@ -205,7 +205,7 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
addr := fs.String("addr", "", "tcp sink address, `HOST:PORT` or [IPV6]:PORT")
user := fs.String("user", "", "user `name`")
passwordFile := fs.String("password-file", "", "`file` holding the password")
passwordFile := fs.String("password-file", "/dev/tty", "`file` holding the password; a terminal asks")
tlsOpts := tlsFlags(fs)
return func(stdout, _ io.Writer) error {
host, _, err := net.SplitHostPort(*addr)
cmd/lw/auth_test.go
+1 -1
View File
@@ -63,7 +63,7 @@ func TestAddUserCreatesPrivateFilesWithAMatchingVerifier(t *testing.T) {
if err != nil {
t.Fatal(err)
}
password, err := authz.ReadPassword(pass)
password, err := authz.ReadPassword(pass, "edge-01", false)
if err != nil {
t.Fatal(err)
}
cmd/lw/bootstrap.go
+2
View File
@@ -22,6 +22,7 @@ import (
_ "github.com/lixenwraith/logwisp/internal/sink/tcp"
_ "github.com/lixenwraith/logwisp/internal/sink/tcpchain"
"github.com/lixenwraith/logwisp/internal/authz"
"github.com/lixenwraith/logwisp/internal/config"
"github.com/lixenwraith/logwisp/internal/service"
"github.com/lixenwraith/logwisp/internal/version"
@@ -37,6 +38,7 @@ func bootstrapInitial(ctx context.Context, cfg *config.Config) (*service.Service
if err != nil {
return nil, nil, fmt.Errorf("failed to bootstrap service: %w", err)
}
authz.EndStartup() // a reload reads no descriptor, pipe or terminal
if err := svc.Start(); err != nil {
svc.Shutdown()
cmd/lw/cli.go
+1 -1
View File
@@ -40,7 +40,7 @@ var commands = []command{
{"tls", "A CA and certificates for TLS and mTLS",
"Files are never overwritten; keys are written mode 0600.", tlsCommands},
{"preset", "Show the pipeline a preset makes, as TOML",
"lw --preset NAME,key=value runs one; flags here are its keys.", presetCommands()},
"lw --preset NAME:key=value runs one; flags here are its keys.", presetCommands()},
{"config", "Place the annotated default configuration",
"Files are never overwritten; lw --dump prints the effective configuration.", configCommands},
}
cmd/lw/cli_test.go
+13 -13
View File
@@ -31,9 +31,9 @@ func TestCommandLineGrammar(t *testing.T) {
}{
{[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}},
{[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}},
{[]string{"--source=null", "--sink", "http,port=1"},
invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}}}}},
{[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}},
{[]string{"--source=null", "--sink", "http:port=1"},
invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http:port=1"}}}}},
{[]string{"--preset", "tail:path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail:path=x"}, {Flag: "pipeline", Value: "b"}}}}},
{[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}},
{[]string{"--quiet", "--"}, invocation{load: config.Args{Overrides: []string{"--quiet"}}}},
{[]string{"--quiet", "-h"}, invocation{help: true, load: config.Args{Overrides: []string{"--quiet"}}}},
@@ -41,9 +41,9 @@ func TestCommandLineGrammar(t *testing.T) {
{[]string{"help"}, invocation{help: true}},
{[]string{"-T", "-c", "a.toml"}, invocation{on: map[string]bool{"tui": true}, load: config.Args{File: "a.toml"}}},
{[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}},
{[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"},
{[]string{"-q", "-t", "-V", "-p", "tail:path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"},
invocation{on: map[string]bool{"check": true, "version": true, "dump": true},
load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}},
load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail:path=x"}},
Overrides: []string{"--quiet", "--logging.level=debug", "--logging.file.retention_hours=-1"}}}},
} {
got, err := parseCommandLine(c.argv)
@@ -173,8 +173,8 @@ func TestDumpReadsBack(t *testing.T) {
if err := os.WriteFile(empty, nil, 0o600); err != nil {
t.Fatal(err)
}
args := []string{"--color", "--preset", "serve,listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example",
"--filter", `exclude,patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"}
args := []string{"--color", "--preset", "serve:listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example",
"--filter", `exclude:patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"}
cfg := loadTestConfig(t, empty, args...)
var dump bytes.Buffer
if code := dumpConfig(loadTestConfig(t, empty, args...), &dump); code != 0 {
@@ -225,7 +225,7 @@ func TestEveryFormLoadsTheSamePipelines(t *testing.T) {
written := filepath.Join(dir, "written.toml")
if err := os.WriteFile(written, []byte(`[[pipelines]]
name = "written"
flow.rate_limit.rate = 5
flow.rate_limit.entries_per_second = 5
plugin_sources = [{id = "in", type = "null"}]
plugin_sinks = [{id = "out", type = "http", config = {port = 15844, acl = {requests_per_second_per_client = 2}}}]
`), 0o600); err != nil {
@@ -321,11 +321,11 @@ func TestEveryPresetBuilds(t *testing.T) {
}
pin := "sha256//" + strings.Repeat("A", 43) + "="
samples := map[string]string{
"pipe": "pipe,format=txt",
"tail": "tail,path=" + dir,
"serve": "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2",
"edge": "edge,to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass,
"aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1",
"pipe": "pipe:format=txt",
"tail": "tail:path=" + dir,
"serve": "serve:path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2",
"edge": "edge:to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass,
"aggregator": "aggregator:listen=127.0.0.1:15843,user=edge-01,password_file=" + pass + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1",
}
for _, p := range config.Presets() {
sample, ok := samples[p.Name]
cmd/lw/help.go
+19 -14
View File
@@ -81,7 +81,7 @@ key alone means true:
chooses the console
Pipelines (they replace the configuration file's pipelines):
-p, --preset NAME[,KEY=VALUE...]
-p, --preset NAME[:KEY=VALUE,...]
Start a pipeline with a preset: pipe, tail,
serve, edge, aggregator; lw preset NAME -h
lists its keys, lw preset NAME prints it
@@ -89,27 +89,32 @@ Pipelines (they replace the configuration file's pipelines):
pipeline options after it configure; those
before any --pipeline make one named after
its --preset, or "cli" without one
--source SPEC TYPE[,KEY=VALUE...], repeatable
--sink SPEC e.g. http,host=0.0.0.0,port=8080, repeatable
--filter SPEC include|exclude,patterns=RE, repeatable
--format SPEC json|txt|raw[,KEY=VALUE...]
--rate-limit SPEC rate=N[,burst=N,policy=drop|pass]
--source SPEC TYPE[:KEY=VALUE,...], repeatable
--sink SPEC e.g. http:host=0.0.0.0,port=8080, repeatable
--filter SPEC include|exclude:patterns=RE, repeatable
--format SPEC json|txt|raw[:KEY=VALUE,...]
--rate-limit SPEC entries_per_second=N[,burst_entries=N,
policy=drop|pass]
--heartbeat SPEC interval_ms=N[,include_stats=true,...]
A pipeline without --source reads stdin, one without --sink writes stdout.
Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and
'\' escapes ',' '=' '\' in values.
TYPE ends at the first ':'. Keys nest with '.' (tls.cert_file=...), a
repeated key makes a list, and '\' escapes ',' '=' '\' in values.
Examples:
lw < app.log > copy.log Copy, line for line
tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN
journalctl -f | lw --sink http,host=127.0.0.1,port=8080
tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN
journalctl -f | lw --sink http:host=127.0.0.1,port=8080
Serve stdin live; browse http://127.0.0.1:8080/
lw --source file,directory=/var/log/app,pattern='*.log' --format txt
lw -p tail,path=/var/log/app Follow a directory's files
lw --preset serve,path=/var/log/app,tls=self,users=users.toml
lw --source file:directory=/var/log/app,pattern='*.log' --format txt
lw -p tail:path=/var/log/app Follow a directory's files
lw --preset serve:path=/var/log/app,tls=self,users=users.toml
HTTPS stream, SCRAM logins
lw --preset edge,path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\
lw --preset edge:path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\
password_file=edge-01.pass Forward to an aggregator
lw -p aggregator:user=pipe,format=raw A temporary pipe's end: Enter
at the prompt draws a password
cmd | lw -p edge:to=HOST:9000,pin=PIN,user=pipe
Its source, asked the password
Environment:
LOGWISP_<KEY> A configuration key, '.' as '_', uppercase:
cmd/lw/reload_test.go
+23 -3
View File
@@ -11,6 +11,7 @@ import (
"testing"
"time"
"github.com/lixenwraith/logwisp/internal/authz"
"github.com/lixenwraith/logwisp/internal/plugin"
"github.com/lixenwraith/logwisp/internal/session"
"github.com/lixenwraith/logwisp/internal/testutil"
@@ -60,6 +61,25 @@ func TestShippedConfigurationBuildsWithCheckedPluginDecoder(t *testing.T) {
svc.Shutdown() // A later reload failure or process exit can close it again.
}
// Once the first service is built a reload reads no descriptor, pipe or
// terminal, which a console source may be reading
func TestStartupEndsWithTheFirstService(t *testing.T) {
testLogger(t)
path := filepath.Join(t.TempDir(), "empty.toml")
testutil.WriteFile(t, path, "")
svc, cancel, err := bootstrapInitial(context.Background(), loadTestConfig(t, path, "--source", "null", "--sink", "null"))
if err != nil {
t.Fatal(err)
}
defer svc.Shutdown()
if cancel != nil {
defer cancel()
}
if _, err := authz.ReadPassword(os.DevNull, "late", false); err == nil || !strings.Contains(err.Error(), "only at startup") {
t.Fatalf("a device after the first service: %v", err)
}
}
func TestInvalidReloadLeavesStatusReporterRunning(t *testing.T) {
testLogger(t)
path := filepath.Join(t.TempDir(), "config.toml")
@@ -124,8 +144,8 @@ func TestCheckBuildsWithoutStarting(t *testing.T) {
dir := t.TempDir()
path, out := filepath.Join(dir, "empty.toml"), filepath.Join(dir, "out")
testutil.WriteFile(t, path, "")
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15862",
"--sink", "file,name=check,directory="+out)); code != 0 {
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http:host=127.0.0.1,port=15862",
"--sink", "file:name=check,directory="+out)); code != 0 {
t.Fatalf("valid configuration: exit %d", code)
}
ln, err := net.Listen("tcp4", "127.0.0.1:15862")
@@ -138,7 +158,7 @@ func TestCheckBuildsWithoutStarting(t *testing.T) {
}
missing := filepath.Join(dir, "missing")
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink",
"http,port=15862,tls.enabled=true,tls.cert_file="+missing+",tls.key_file="+missing)); code != 1 {
"http:port=15862,tls.enabled=true,tls.cert_file="+missing+",tls.key_file="+missing)); code != 1 {
t.Fatalf("listener TLS without its certificate file: exit %d, want 1", code)
}
}
config/logwisp.toml
+9 -6
View File
@@ -64,10 +64,10 @@ name = "default"
###============================================================================
## policy="pass" short-circuits the size check too: enforcing a size cap needs
## rate > 0 AND policy = "drop".
## entries_per_second > 0 AND policy = "drop".
# [pipelines.flow.rate_limit]
# rate = 0.0 # Entries/second (0 = limiter disabled)
# burst = 0.0 # Burst capacity (defaults to rate)
# entries_per_second = 0.0 # 0 = limiter disabled
# burst_entries = 0.0 # Bucket capacity (0 = entries_per_second)
# policy = "pass" # pass|drop
# max_entry_size_bytes = 0 # 0 = unlimited
@@ -131,7 +131,8 @@ name = "default"
### On the http sink the block gates stream_path AND status_path.
### Listeners (tcp/http sinks, tcp_chain/http_chain sources):
### mtls authorizes the client certificate; REQUIRES tls.client_auth.
### scram admits the users in credentials_file (`lw auth add-user`).
### scram admits the users in credentials_file (`lw auth add-user`), or
### the one username whose password is in password_file.
### Dialers (tcp_chain/http_chain sinks):
### mtls pins the server identity.
### scram logs in as username with the password in password_file.
@@ -153,8 +154,10 @@ name = "default"
### Overrides trust_node. See doc/security.md.
### credentials_file = "" scram listeners: verifiers; read at (re)load, not watched
### token_lifetime_ms = 0 scram, http sink and http_chain source: 10 s to 24 h, 0 = 15 minutes
### username = "" scram dialers
### password_file = "" scram dialers: one trailing line break is trimmed
### username = "" scram dialers, or a listener's one user
### password_file = "" scram, with username: one trailing line break is
### trimmed; /dev/fd/N or a pipe is read once and kept
### for reloads, /dev/tty asks once
### trusted_proxies = [] scram http sink: TLS-terminating reverse proxies
### (addresses or CIDRs); browsers log in through them
###============================================================================
deploy/package/completion/_lw
+9 -5
View File
@@ -31,7 +31,7 @@ auth:*)
'--ca-file=[CA file that verifies the server (default: system roots)]:value:_default' \
'--cert-file=[client certificate file, for listeners with tls.client_auth]:value:_default' \
'--key-file=[client key file]:value:_default' \
'--password-file=[file holding the password]:value:_default' \
'--password-file=[file holding the password; a terminal asks]:value:_default' \
'--pin-sha256=[sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)]:value:_default' \
'--server-name=[name the server certificate must carry (default: the host)]:value:_default' \
'--unbound[log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)]' \
@@ -43,7 +43,7 @@ auth:*)
'--ca-file=[CA file that verifies the server (default: system roots)]:value:_default' \
'--cert-file=[client certificate file, for listeners with tls.client_auth]:value:_default' \
'--key-file=[client key file]:value:_default' \
'--password-file=[file holding the password]:value:_default' \
'--password-file=[file holding the password; a terminal asks]:value:_default' \
'--pin-sha256=[sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)]:value:_default' \
'--server-name=[name the server certificate must carry (default: the host)]:value:_default' \
'(-u --user)'{-u=,--user=}'[user name]:value:_default' \
@@ -112,11 +112,13 @@ preset:*)
'--issuer_key=[CA key file, for tls=issuer]:value:_default' \
'--key=[key file, for tls=files]:value:_default' \
'--listen=[HOST:PORT; [IPV6\]:PORT]:value:_default' \
'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
'--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \
'--proxy=[addresses or CIDRs of the TLS-terminating proxies browsers come through; '\'','\'' between them]:value:_default' \
'--tls=[off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \
'--users=[credentials file (lw auth add-user): readers log in with SCRAM]:value:_default' \
'--viewer=[true: the login page and viewer for users, behind proxy; without users the viewer is always on]:value:_default' \
'--viewer=[true: the login page and viewer, behind proxy; without user or users the viewer is always on]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;;
edge) _arguments -S \
'--ca=[CA file that verifies the aggregator; default: system roots]:value:_default' \
@@ -124,13 +126,13 @@ preset:*)
'--from=[start|end: where reading a file starts]:value:_default' \
'--key=[client key file]:value:_default' \
'--node=[origin label; default: this host'\''s name]:value:_default' \
'--password_file=[file holding the SCRAM password]:value:_default' \
'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
'--path=[file, directory (its files) or glob to follow; default: standard input]:value:_default' \
'--pin=[sha256//BASE64 of the aggregator'\''s key, which it logs with tls=self]:value:_default' \
'--server_name=[name the aggregator'\''s certificate carries; default: to'\''s host]:value:_default' \
'--to=[the aggregator'\''s HOST:PORT; [IPV6\]:PORT]:value:_default' \
'--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user (lw auth add-user on the aggregator)]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;;
aggregator) _arguments -S \
'--allow=[addresses or CIDRs that may connect, of listen'\''s family (either behind proxy=); default: all; '\'','\'' between them]:value:_default' \
@@ -144,8 +146,10 @@ preset:*)
'--key=[key file, for tls=files]:value:_default' \
'--listen=[HOST:PORT; [IPV6\]:PORT; [::\] for IPv6 only]:value:_default' \
'--out=[directory for the received entries; default: standard output]:value:_default' \
'--password_file=[file holding user'\''s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks]:value:_default' \
'--tls=[self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files]:value:_default' \
'--transport=[tcp|http: tcp_chain or http_chain]:value:_default' \
'(-u --user)'{-u=,--user=}'[SCRAM user: an edge'\''s login, or a listener'\''s only user in place of users]:value:_default' \
'--users=[credentials file (lw auth add-user): edges log in with SCRAM]:value:_default' \
'(-h --help)'{-h,--help}'[print the flags]' ;;
esac
deploy/package/completion/lw.bash
+4 -4
View File
@@ -66,9 +66,9 @@ _lw() {
words='--format --from --path --help -h' ;;
serve)
case $prev in
--allow|-allow|--cert|-cert|--deny|-deny|--format|-format|--from|-from|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--path|-path|--proxy|-proxy|--tls|-tls|--users|-users|--viewer|-viewer) COMPREPLY=(); return ;;
--allow|-allow|--cert|-cert|--deny|-deny|--format|-format|--from|-from|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--password_file|-password_file|--path|-path|--proxy|-proxy|--tls|-tls|--user|-user|-u|--users|-users|--viewer|-viewer) COMPREPLY=(); return ;;
esac
words='--allow --cert --deny --format --from --hosts --issuer_cert --issuer_key --key --listen --path --proxy --tls --users --viewer --help -h' ;;
words='--allow --cert --deny --format --from --hosts --issuer_cert --issuer_key --key --listen --password_file --path --proxy --tls --user -u --users --viewer --help -h' ;;
edge)
case $prev in
--ca|-ca|--cert|-cert|--from|-from|--key|-key|--node|-node|--password_file|-password_file|--path|-path|--pin|-pin|--server_name|-server_name|--to|-to|--transport|-transport|--user|-user|-u) COMPREPLY=(); return ;;
@@ -76,9 +76,9 @@ _lw() {
words='--ca --cert --from --key --node --password_file --path --pin --server_name --to --transport --user -u --help -h' ;;
aggregator)
case $prev in
--allow|-allow|--cert|-cert|--client_ca|-client_ca|--deny|-deny|--format|-format|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--out|-out|--tls|-tls|--transport|-transport|--users|-users) COMPREPLY=(); return ;;
--allow|-allow|--cert|-cert|--client_ca|-client_ca|--deny|-deny|--format|-format|--hosts|-hosts|--issuer_cert|-issuer_cert|--issuer_key|-issuer_key|--key|-key|--listen|-listen|--out|-out|--password_file|-password_file|--tls|-tls|--transport|-transport|--user|-user|-u|--users|-users) COMPREPLY=(); return ;;
esac
words='--allow --cert --client_ca --deny --format --hosts --issuer_cert --issuer_key --key --listen --out --tls --transport --users --help -h' ;;
words='--allow --cert --client_ca --deny --format --hosts --issuer_cert --issuer_key --key --listen --out --password_file --tls --transport --user -u --users --help -h' ;;
esac ;;
config:2) words='init' ;;
config:*)
deploy/package/completion/lw.fish
+9 -5
View File
@@ -59,7 +59,7 @@ complete -c lw -n '__lw_in auth remove-user' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in auth token' -l ca-file -r -F -d 'CA file that verifies the server (default: system roots)'
complete -c lw -n '__lw_in auth token' -l cert-file -r -F -d 'client certificate file, for listeners with tls.client_auth'
complete -c lw -n '__lw_in auth token' -l key-file -r -F -d 'client key file'
complete -c lw -n '__lw_in auth token' -l password-file -r -F -d 'file holding the password'
complete -c lw -n '__lw_in auth token' -l password-file -r -F -d 'file holding the password; a terminal asks'
complete -c lw -n '__lw_in auth token' -l pin-sha256 -r -F -d 'sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)'
complete -c lw -n '__lw_in auth token' -l server-name -r -F -d 'name the server certificate must carry (default: the host)'
complete -c lw -n '__lw_in auth token' -l unbound -d 'log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)'
@@ -70,7 +70,7 @@ complete -c lw -n '__lw_in auth stream' -l addr -r -F -d 'tcp sink address, HOST
complete -c lw -n '__lw_in auth stream' -l ca-file -r -F -d 'CA file that verifies the server (default: system roots)'
complete -c lw -n '__lw_in auth stream' -l cert-file -r -F -d 'client certificate file, for listeners with tls.client_auth'
complete -c lw -n '__lw_in auth stream' -l key-file -r -F -d 'client key file'
complete -c lw -n '__lw_in auth stream' -l password-file -r -F -d 'file holding the password'
complete -c lw -n '__lw_in auth stream' -l password-file -r -F -d 'file holding the password; a terminal asks'
complete -c lw -n '__lw_in auth stream' -l pin-sha256 -r -F -d 'sha256//BASE64 pin of the server key, in place of --ca-file (a self_signed listener logs it)'
complete -c lw -n '__lw_in auth stream' -l server-name -r -F -d 'name the server certificate must carry (default: the host)'
complete -c lw -n '__lw_in auth stream' -l user -o u -r -F -d 'user name'
@@ -112,24 +112,26 @@ complete -c lw -n '__lw_in preset serve' -l issuer_cert -r -F -d 'CA certificate
complete -c lw -n '__lw_in preset serve' -l issuer_key -r -F -d 'CA key file, for tls=issuer'
complete -c lw -n '__lw_in preset serve' -l key -r -F -d 'key file, for tls=files'
complete -c lw -n '__lw_in preset serve' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT'
complete -c lw -n '__lw_in preset serve' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
complete -c lw -n '__lw_in preset serve' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input'
complete -c lw -n '__lw_in preset serve' -l proxy -r -F -d 'addresses or CIDRs of the TLS-terminating proxies browsers come through; \',\' between them'
complete -c lw -n '__lw_in preset serve' -l tls -r -F -d 'off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files'
complete -c lw -n '__lw_in preset serve' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users'
complete -c lw -n '__lw_in preset serve' -l users -r -F -d 'credentials file (lw auth add-user): readers log in with SCRAM'
complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true: the login page and viewer for users, behind proxy; without users the viewer is always on'
complete -c lw -n '__lw_in preset serve' -l viewer -r -F -d 'true: the login page and viewer, behind proxy; without user or users the viewer is always on'
complete -c lw -n '__lw_in preset serve' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in preset edge' -l ca -r -F -d 'CA file that verifies the aggregator; default: system roots'
complete -c lw -n '__lw_in preset edge' -l cert -r -F -d 'client certificate file, for an aggregator with client_ca'
complete -c lw -n '__lw_in preset edge' -l from -r -F -d 'start|end: where reading a file starts'
complete -c lw -n '__lw_in preset edge' -l key -r -F -d 'client key file'
complete -c lw -n '__lw_in preset edge' -l node -r -F -d 'origin label; default: this host\'s name'
complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding the SCRAM password'
complete -c lw -n '__lw_in preset edge' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
complete -c lw -n '__lw_in preset edge' -l path -r -F -d 'file, directory (its files) or glob to follow; default: standard input'
complete -c lw -n '__lw_in preset edge' -l pin -r -F -d 'sha256//BASE64 of the aggregator\'s key, which it logs with tls=self'
complete -c lw -n '__lw_in preset edge' -l server_name -r -F -d 'name the aggregator\'s certificate carries; default: to\'s host'
complete -c lw -n '__lw_in preset edge' -l to -r -F -d 'the aggregator\'s HOST:PORT; [IPV6]:PORT'
complete -c lw -n '__lw_in preset edge' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain'
complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user (lw auth add-user on the aggregator)'
complete -c lw -n '__lw_in preset edge' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users'
complete -c lw -n '__lw_in preset edge' -l help -o h -d 'print the flags'
complete -c lw -n '__lw_in preset aggregator' -l allow -r -F -d 'addresses or CIDRs that may connect, of listen\'s family (either behind proxy=); default: all; \',\' between them'
complete -c lw -n '__lw_in preset aggregator' -l cert -r -F -d 'certificate file, for tls=files'
@@ -142,8 +144,10 @@ complete -c lw -n '__lw_in preset aggregator' -l issuer_key -r -F -d 'CA key fil
complete -c lw -n '__lw_in preset aggregator' -l key -r -F -d 'key file, for tls=files'
complete -c lw -n '__lw_in preset aggregator' -l listen -r -F -d 'HOST:PORT; [IPV6]:PORT; [::] for IPv6 only'
complete -c lw -n '__lw_in preset aggregator' -l out -r -F -d 'directory for the received entries; default: standard output'
complete -c lw -n '__lw_in preset aggregator' -l password_file -r -F -d 'file holding user\'s password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks'
complete -c lw -n '__lw_in preset aggregator' -l tls -r -F -d 'self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files'
complete -c lw -n '__lw_in preset aggregator' -l transport -r -F -d 'tcp|http: tcp_chain or http_chain'
complete -c lw -n '__lw_in preset aggregator' -l user -o u -r -F -d 'SCRAM user: an edge\'s login, or a listener\'s only user in place of users'
complete -c lw -n '__lw_in preset aggregator' -l users -r -F -d 'credentials file (lw auth add-user): edges log in with SCRAM'
complete -c lw -n '__lw_in preset aggregator' -l help -o h -d 'print the flags'
doc/README.md
+1 -1
View File
@@ -90,7 +90,7 @@ stdout:
```bash
lw < app.log > copy.log
tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN
tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN
```
As a service, a file holds the pipelines:
doc/chaining.md
+3 -2
View File
@@ -255,8 +255,9 @@ is what their certificates say, regardless of the `node` each edge configured.
`test/mtls-chain-test.sh` builds exactly this shape against a throwaway PKI.
To use passwords instead of client certificates, replace both `auth` blocks:
`type = "scram"` with `username` and `password_file` on the edge, and with a
`credentials_file` on the relay. `client_auth` and the edge's certificate become
`type = "scram"` with `username` and `password_file` on the edge, and on the
relay a `credentials_file`, or one `username` with its `password_file`.
`client_auth` and the edge's certificate become
optional, and entries are labelled with the username. See
[Password Authentication](security.md#password-authentication-scram).
doc/cli.md
+91 -66
View File
@@ -3,7 +3,7 @@
```
lw [options] [< input]
lw -c|--config FILE [options]
lw -p|--preset NAME[,KEY=VALUE...] [options]
lw -p|--preset NAME[:KEY=VALUE,...] [options]
lw -t|--check [options]
lw --dump [options]
lw -T|--tui [options]
@@ -45,7 +45,7 @@ first argument.
tail`, `lw -t FILE`) are usage errors, named on stderr.
- lw's multiword options join words with `-` (`--rate-limit`,
`--password-file`); configuration keys, spec keys and preset keys keep
TOML's `_` and `.` (`--status_reporter`, `--sink http,tls.cert_file=F`,
TOML's `_` and `.` (`--status_reporter`, `--sink http:tls.cert_file=F`,
`lw preset edge --password_file F`).
- The commands also take a long option after one dash (`-user NAME`).
@@ -121,14 +121,15 @@ pipelines and the built-in default; every other key keeps its precedence.
- stdin has one reader, so the whole configuration holds at most one console
source: `pipeline "b": a console source already reads stdin in pipeline "a"`
A SPEC is a comma-separated list:
A SPEC is `TYPE:key=value,...`, or `key=value,...` without a TYPE:
- source, sink, filter and format specs start with a TYPE
- source, sink, filter and format specs start with a TYPE, which ends at the
first `:`; a TYPE alone (`--sink null`, `--format txt`) takes the defaults
- the plugin type: [Sources](sources.md), [Sinks](sinks.md)
- `include` or `exclude`: [Filters](filters.md)
- `json`, `txt` or `raw`: [Formatters](formatters.md)
- rate-limit and heartbeat specs have no TYPE
- the rest are `key=value` pairs with the TOML keys of the plugin's `config`
- the pairs are comma-separated, with the TOML keys of the plugin's `config`
table or of the [flow stage](configuration.md#flow-stages)
- a dotted key reaches a nested table: `tls.cert_file=...`, `auth.type=scram`
- a repeated key makes a list: `patterns=ERROR,patterns=WARN`
@@ -136,7 +137,8 @@ A SPEC is a comma-separated list:
is one regex, `auth.allow=a,auth.allow=b` two entries
- values convert to the option's type: `port=8080`, `raw=true`
- `\` escapes `,`, `=` and `\` in a value; any other backslash stays, so regex
escapes such as `\d` pass unchanged
escapes such as `\d` pass unchanged; a `:` in a value needs no escape
(`host=::1`, `listen=127.0.0.1:8080`)
- `id=NAME` names a source or sink; the default is its TYPE, then `TYPE_2`,
`TYPE_3`, ...
- naming a stage turns it on: `--rate-limit` defaults `policy` to `drop`,
@@ -145,8 +147,9 @@ A SPEC is a comma-separated list:
Errors name the flag and the offending part, and nothing starts:
```
--sink http,port: missing "="
--rate-limit rate=100,polcy=drop: unknown key "polcy"
--sink http:port: missing "="
--rate-limit entries_per_second=100,polcy=drop: unknown key "polcy"
--source file,directory=/x: TYPE ends at ':', as in file:directory=/x
```
A misspelled plugin key fails when the plugin is built, before any listener
@@ -155,35 +158,39 @@ opens: `failed to create sink http: ... unknown key "tls.enabeld"`.
```bash
# tail a directory, serve it over SSE on every interface; a browser at
# http://HOST:8080/ shows it
lw --source 'file,directory=/var/log/app,pattern=*.log' \
--sink http,host=0.0.0.0,port=8080
lw --source 'file:directory=/var/log/app,pattern=*.log' \
--sink http:host=0.0.0.0,port=8080
# errors and warnings as JSON, over TLS with SCRAM logins
lw --source file,directory=/var/log/app \
--filter include,patterns=ERROR,patterns=WARN --format json \
--sink "http,port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\
lw --source file:directory=/var/log/app \
--filter include:patterns=ERROR,patterns=WARN --format json \
--sink "http:port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\
tls.enabled=true,tls.cert_file=/etc/logwisp/server.crt,tls.key_file=/etc/logwisp/server.key"
# two pipelines: app writes stdout, relay a file
lw --pipeline app --source file,directory=/var/log/app \
--pipeline relay --source tcp_chain,port=9000 \
--sink file,directory=/var/log/relay,name=relay
lw --pipeline app --source file:directory=/var/log/app \
--pipeline relay --source tcp_chain:port=9000 \
--sink file:directory=/var/log/relay,name=relay
```
Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns).
## Presets
`--preset NAME,key=value,...` takes the [SPEC](#pipelines) syntax. A list
key (`hosts`, `proxy`, `allow`, `deny`) takes several values, repeated
(`hosts=a,hosts=b`) or `,`-separated; any other key takes one.
`--preset NAME:key=value,...` takes the [SPEC](#pipelines) syntax. A list
key (`hosts`, `proxy`, `allow`, `deny`) takes several values: repeated
(`hosts=a,hosts=b`) or `\,`-separated in a SPEC, `,`-separated in `lw preset`
flags and the TUI; any other key takes one.
`lw preset NAME [--key value ...]` prints the pipeline a preset expands to,
ready for a configuration file; its flags are the keys, spelled as in the SPEC
(`--password_file`), and `-u` is `--user`;
`lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An
unknown key fails and lists the valid ones. A `path` is a file, a directory
(its files) or a glob; a preset that takes one reads stdin without it, and
`from` (`end`, or `start`) is where reading a file starts.
`from` (`end`, or `start`) is where reading a file starts. `user` names one
SCRAM user, an edge's login or a listener's only user in place of `users`;
`password_file` holds its password and defaults to `/dev/tty`, which asks
([password sources](security.md#one-user-without-a-credentials-file)).
- `pipe`: stdin to stdout, the built-in default
- `format` (`raw`)
@@ -196,41 +203,49 @@ unknown key fails and lists the valid ones. A `path` is a file, a directory
- `tls`: `off` (default), `self` (a self-signed certificate made at
startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or
`files` (`cert` and `key`); `hosts` adds names to a made certificate
- without `users` the viewer needs no login, over plain http too
- without `user` or `users` the viewer needs no login, over plain http too
- `allow` and `deny`: addresses or CIDRs, the sink's
[acl](security.md#the-acl-block)
- `users`: a credentials file; readers then log in with SCRAM, and browsers
need `proxy` and `viewer=true`: the login page and viewer behind the
TLS-terminating proxies `proxy` names, since a browser cannot bind its
login to the TLS channel
- `users` (a credentials file) or `user`: readers then log in with SCRAM,
which needs `tls` or `proxy`; browsers need `proxy` and `viewer=true`: the
login page and viewer behind the TLS-terminating proxies `proxy` names,
since a browser cannot bind its login to the TLS channel
- `edge`: forward to an aggregator over TLS (`tcp_chain`, or `http_chain`
with `transport=http`)
- `to` (required), `path`, `from`, `transport` (`tcp`), `node`
- verify the aggregator by `ca` (a CA file; default: system roots) or `pin`
(its `tls.pin_sha256`), and `server_name`
- authenticate by `user` and `password_file` (SCRAM), or a client `cert` and
`key` (mTLS): without either the preset refuses to run
- authenticate by `user` (SCRAM), or a client `cert` and `key` (mTLS):
without either the preset refuses to run
- `aggregator`: receive from edges over TLS, to stdout or files
- `listen` (`0.0.0.0:9000`), `transport` (`tcp`), `format` (`json`)
- `tls`: `self` (default), `issuer` or `files`, as for `serve`; never `off`
- authenticate by `users` (SCRAM) or `client_ca` (mTLS): one is required
- authenticate by `users` or `user` (SCRAM), or `client_ca` (mTLS): one is
required
- `allow` and `deny`, as for `serve`
- `out`: a directory for `aggregate*.log` files; default stdout
```bash
# follow a directory, level names in color on a terminal
lw --preset tail,path=/var/log/app
lw --preset tail:path=/var/log/app
# serve it to a browser at http://127.0.0.1:8080/
lw --preset serve,path=/var/log/app
lw --preset serve:path=/var/log/app
# serve it over HTTPS, self-signed, to SCRAM users; keep the result as a file
lw --preset serve,path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml
lw --preset serve:path=/var/log/app,tls=self,users=/etc/logwisp/users.toml --dump > serve.toml
# an aggregator with a certificate made at startup, and an edge pinning it
lw --preset aggregator,users=/etc/logwisp/users.toml,out=/var/log/edges
lw --preset edge,path=/var/log/app,to=agg.example.org:9000,pin=sha256//BASE64,\
lw --preset aggregator:users=/etc/logwisp/users.toml,out=/var/log/edges
lw --preset edge:path=/var/log/app,to=agg.example.org:9000,pin=sha256//BASE64,\
user=edge-01,password_file=/etc/logwisp/edge-01.pass
# a temporary pipe, nothing on disk: Enter at the receiver's prompt draws the
# password, typed at the sender's; or both read it from a descriptor
lw -p aggregator:listen=0.0.0.0:9000,user=pipe,format=raw,allow=SENDER/32
cmd | lw -p edge:to=RECEIVER:9000,pin=sha256//BASE64,user=pipe
cmd | lw -p edge:to=RECEIVER:9000,pin=sha256//BASE64,user=pipe,\
password_file=/dev/fd/3 3< <(pass show lw/pipe)
```
A self-signed aggregator logs its pin at startup (`pin_sha256`, a warning, so
@@ -278,9 +293,9 @@ A container needs no configuration file:
```bash
docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \
-e LOGWISP_PIPELINE=app \
-e LOGWISP_SOURCE='file,directory=/logs,pattern=*.log' \
-e LOGWISP_FILTER='exclude,patterns=DEBUG' \
-e LOGWISP_SINK='http,host=0.0.0.0,port=8080' \
-e LOGWISP_SOURCE='file:directory=/logs,pattern=*.log' \
-e LOGWISP_FILTER='exclude:patterns=DEBUG' \
-e LOGWISP_SINK='http:host=0.0.0.0,port=8080' \
-e LOGWISP_SINK_1=console \
-e LOGWISP_LOGGING_LEVEL=info \
logwisp
@@ -382,13 +397,13 @@ sink gives its clients or its downstream its `write_timeout_ms`
lw < app.log > copy.log
# errors and warnings only
tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN
tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN
# stdin as a live SSE stream; the http and tcp sinks bind 0.0.0.0 by default
journalctl -f | lw --sink http,host=127.0.0.1,port=8080
journalctl -f | lw --sink http:host=127.0.0.1,port=8080
# a directory's files to one file, through stdout
lw --source file,directory=/var/log/app,pattern='*.log' > all.log
lw --source file:directory=/var/log/app,pattern='*.log' > all.log
```
A console sink never drops: when its reader is slow, the pipeline waits, and
@@ -411,7 +426,7 @@ prints `configuration ok: N pipeline(s)` and exits 0, or the error and exits 1
```bash
lw --check -c /etc/logwisp/logwisp.toml
lw --check --source file,directory=/var/log/app --sink http,port=8080
lw --check --source file:directory=/var/log/app --sink http:port=8080
```
**Production**
@@ -443,21 +458,30 @@ WebAssembly.
pipeline flags, `LOGWISP_SOURCE`, a discovered file); with none but the
built-in pipe, the preset menu (or `empty`). A pasted `lw` command line
replaces the pipelines, in the preset menu too.
- Screen: SOURCES, FLOW and SINKS side by side from 64 columns, stacked
below; each source and sink a tinted bar with its type, whether it listens
or dials, and a summary; the four flow stages in their order in a dashed
box, the ones off dimmed. The inspector, the chosen part's options, sits
below the canvas, and beside it from 110 columns.
- Keys: arrows or `hjkl` move; Enter inspects, then edits a value (empty
returns it to its default); Tab sets it and goes to the next; Esc back; `a` adds a source,
sink or filter; `d` deletes, or returns a value to its default; Space turns
a flow stage or a switch on or off; `J`/`K` move a filter; `[` `]` switch
pipelines; `p` presets; `c` shows the first problem; `o` output; `?` keys;
`q` quits.
- Output, `o`: run the pipelines with the other options given (so `--check`
and `--dump` apply to them, and reloads keep them), or print one of the
command line, the environment variables (one pipeline) and the file's
pipelines on standard output once the screen closes.
- Screen: the pipeline bar on top; SOURCES, FLOW and SINKS side by side
from 64 columns, stacked below; each source and sink a tinted bar with its
type, whether it listens or dials, and a summary; the four flow stages in
their order in a dashed box, the ones off dimmed. The inspector, the
chosen part's options with radio buttons for few choices, sits below the
canvas, and beside it from 110 columns. Narrow panes stack labels, wrap
text and scroll dialogs; the status line wraps to three rows.
- Keys: arrows or `hjkl` move; Enter inspects a part, opening on its first
required key, then edits a value (empty returns it to its default), opens
a list's choices or adds on an add row; in a value, Enter sets it, and Tab
or Up/Down set it and move. Left/Right step a choice or set a switch;
Space flips it; Esc goes back. `a` adds a source, sink or filter; `c`
changes a source's or sink's type in its place; `d` deletes, or returns a
value to its default; Space turns a flow stage on or off; `J`/`K` move a
filter; `[` `]` switch pipelines; `p` presets; `!` shows the first
problem; `r` runs; `o` output; `?` keys; `q` quits.
- Pipelines: Up from a column's top row reaches the bar, where `[` `]` or
Left/Right switch, `a` adds one (empty or from a preset), Enter renames and
`d` deletes, asking first when it has parts.
- Output, `o`, each picked by its letter: `r` runs the pipelines with the
other options given (so `--check` and `--dump` apply to them, and reloads
keep them), or `c`, `e` and `f` print the command line, the environment
variables (one pipeline) or the file's pipelines on standard output once
the screen closes.
- The screen draws on the controlling terminal: `lw -T > pipelines.toml` and
`cmd | lw -T` keep standard input and output as data. lw never writes the
file it starts from, but the shell empties a redirect's target before lw
@@ -476,9 +500,9 @@ them as a viewer. See
```
lw auth add-user --credentials FILE -u NAME [--password-file FILE] [--generate]
lw auth remove-user --credentials FILE -u NAME
lw auth token --url https://HOST:PORT[/PATH] -u NAME --password-file FILE
lw auth token --url https://HOST:PORT[/PATH] -u NAME [--password-file FILE]
[--unbound] [TLS flags]
lw auth stream --addr HOST:PORT -u NAME --password-file FILE [TLS flags]
lw auth stream --addr HOST:PORT -u NAME [--password-file FILE] [TLS flags]
```
`-u` is short for `--user`.
@@ -496,7 +520,7 @@ success, `1` on failure and `2` on a usage error.
**`add-user`** creates the credentials file, with a fresh `decoy_key`, when it
does not exist or is empty (create it empty first to choose its owner). The
password comes from `--password-file` when that file exists (at least 8 bytes;
one trailing line break is trimmed). Otherwise a random
one trailing line break is trimmed; `/dev/tty` asks). Otherwise a random
26-character password (130 bits) is generated and written to `--password-file`,
or printed once to stdout when there is none. Replacing an existing user's
password needs an existing `--password-file` or `--generate`, so a mistyped path
@@ -512,11 +536,13 @@ Neither touches a running LogWisp: send `SIGHUP`, since `auto_reload` does not
watch the credentials file.
**`token`** and **`stream`** build the same TLS and SCRAM client as a chain
sink. TLS flags: `--ca-file` (default: system roots) or `--pin-sha256` (the
`tls.pin_sha256` a self-signed listener logs), `--server-name` (default: the
host), and `--cert-file` / `--key-file` for a listener with `tls.client_auth`.
There is no flag to skip verification: an unverified server could relay the
login. Redirects are not followed. `--unbound` logs in to an `http` sink behind
sink. `--password-file` defaults to `/dev/tty`: the password is asked without
echo; a file, `/dev/fd/N` or a pipe is read as by a
[plugin](security.md#one-user-without-a-credentials-file). TLS flags:
`--ca-file` (default: system roots) or `--pin-sha256` (the `tls.pin_sha256` a
self-signed listener logs), `--server-name` (default: the host), and
`--cert-file` / `--key-file` for a listener with `tls.client_auth`. There is no
flag to skip verification: an unverified server could relay the login. Redirects are not followed. `--unbound` logs in to an `http` sink behind
a TLS-terminating proxy (`auth.trusted_proxies`), still pinning the proxy's
certificate across the two requests; only then may `--url` carry the path the
proxy mounts LogWisp at. `stream` exits `0` on `SIGINT` or `SIGTERM`, and `1`
@@ -549,9 +575,8 @@ curl --cacert ca.crt -H @<(printf 'Authorization: Bearer %s\n' \
"$(lw auth token --url https://HOST:PORT --user viewer \
--password-file viewer.pass --ca-file ca.crt)") https://HOST:PORT/status
# follow a tcp sink
lw auth stream --addr HOST:PORT --user viewer --password-file viewer.pass \
--ca-file ca.crt
# follow a tcp sink, asked for the password
lw auth stream --addr HOST:PORT --user viewer --pin-sha256 sha256//BASE64
```
## `lw tls`
doc/configuration.md
+8 -8
View File
@@ -111,8 +111,8 @@ name = "app" # required, unique across pipelines
# --- flow: everything between sources and sinks ---
[pipelines.flow.rate_limit]
rate = 1000.0
burst = 2000.0
entries_per_second = 1000.0
burst_entries = 2000.0
policy = "drop"
max_entry_size_bytes = 65536
@@ -168,19 +168,19 @@ Every stage is optional:
Options, each as type and default:
- `rate` (float, `0`): entries per second; `<= 0` disables the limiter
entirely.
- `burst` (float, `rate`): token bucket capacity.
- `entries_per_second` (float, `0`): `<= 0` disables the limiter entirely.
- `burst_entries` (float, `entries_per_second`): token bucket capacity.
- `policy` (string, `pass`): `pass` allows everything through, `drop` discards
over-limit entries.
- `max_entry_size_bytes` (int, `0`): per-entry byte cap; `0` = unlimited.
Two behaviours are easy to trip over:
- The limiter is constructed only when `rate > 0`. With `rate = 0`,
- The limiter is constructed only when `entries_per_second > 0`. With `0`,
`max_entry_size_bytes` is never enforced.
- `policy = "pass"` short-circuits the whole check, including the size cap.
To enforce a size cap you need `rate > 0` **and** `policy = "drop"`.
To enforce a size cap you need `entries_per_second > 0` **and**
`policy = "drop"`.
#### Heartbeat
@@ -258,7 +258,7 @@ Startup validation is intentionally split.
"pipelines[0].plugin_sinks[0].confg"`. A misspelled table path would otherwise
drop the whole table it heads. A top-level `config_file` is ignored
- pipeline [specs](cli.md#pipelines): syntax, and unknown flow-stage keys such
as `--rate-limit rate=1,polcy=drop`
as `--rate-limit entries_per_second=1,polcy=drop`
- at least one pipeline
- unique, non-empty pipeline names
- at least one source and one sink per pipeline
doc/installation.md
+1 -1
View File
@@ -317,7 +317,7 @@ docker run -d --name logwisp \
```bash
-e LOGWISP_PIPELINE=relay \
-e LOGWISP_SOURCE=tcp_chain,port=9000,tls.enabled=true,tls.cert_file=/run/secrets/tls.crt,tls.key_file=/run/secrets/tls.key,auth.type=scram,auth.credentials_file=/run/secrets/users.toml \
-e LOGWISP_SOURCE=tcp_chain:port=9000,tls.enabled=true,tls.cert_file=/run/secrets/tls.crt,tls.key_file=/run/secrets/tls.key,auth.type=scram,auth.credentials_file=/run/secrets/users.toml \
-e LOGWISP_SINK=console
```
doc/lw.1
+64 -26
View File
@@ -342,17 +342,20 @@ Set the pipeline's formatter, rate limit and heartbeat.
.Pp
A source, sink, filter or format
.Ar spec
is a type followed by comma-separated
is a type, then a colon and comma-separated
.Ar key Ns = Ns Ar value
pairs; a rate limit or heartbeat
pairs; the first
.Ql \&:
ends the type, and a type alone takes the defaults.
A rate limit or heartbeat
.Ar spec
has the pairs only:
.Bd -literal -offset indent
\-\-source file,directory=/var/log/app,pattern=*.log
\-\-sink http,host=127.0.0.1,port=8080,id=web
\-\-filter include,patterns=ERROR,patterns=WARN
\-\-source file:directory=/var/log/app,pattern=*.log
\-\-sink http:host=127.0.0.1,port=8080,id=web
\-\-filter include:patterns=ERROR,patterns=WARN
\-\-format json
\-\-rate\-limit rate=100,burst=200,policy=drop
\-\-rate\-limit entries_per_second=100,burst_entries=200,policy=drop
.Ed
.Pp
Keys are dotted paths into the plugin's options, such as
@@ -366,6 +369,8 @@ and
inside a value; quote such a
.Ar spec
so the shell keeps the backslash.
A colon in a value needs no escape, as in
.Cm host=::1 .
.Cm id= Ns Ar name
names a source or sink; the default is its type, then
.Ar type Ns _2 ,
@@ -377,7 +382,7 @@ ignore the pipeline variables in
.Sx ENVIRONMENT .
.Ss Presets
A preset builds a common pipeline from a few keys:
.Fl \-preset Ar name Ns Op , Ns Ar key Ns = Ns Ar value ... ,
.Fl \-preset Ar name Ns Op : Ns Ar key Ns = Ns Ar value ... ,
in the syntax of the other specifications;
.Nm
.Cm preset Ar name Op Fl \- Ns Ar key Ar value ...
@@ -392,6 +397,16 @@ A
.Ar path
is a file, a directory, whose files are followed, or a glob; without one, a
preset that takes it reads standard input.
.Cm user
names one SCRAM user, an edge's login or a listener's only user in place of
.Cm users ;
.Cm password_file
holds its password, read once at startup when it is
.Pa /dev/fd/ Ns Ar N
or a pipe, and defaults to
.Pa /dev/tty ,
which asks at startup; on a listener, Enter alone generates a password and
shows it there.
.Bl -tag -width Ds
.It Cm pipe Op Cm format
Standard input to standard output, the pipeline
@@ -400,7 +415,7 @@ runs without a configuration.
.It Cm tail path Op Cm from , format
Follow files to standard output, like
.Ql tail \-F .
.It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer , allow , deny Oc
.It Cm serve Oo Cm path , listen , format , tls , users , user , password_file , proxy , viewer , allow , deny Oc
Serve a live HTTP stream on
.Cm listen
.Pq default Li 127.0.0.1:8080 ,
@@ -416,9 +431,13 @@ or
.Cm files
.Pq Cm cert No and Cm key ;
.Cm users
names a credentials file, so readers log in with SCRAM.
A browser at the root gets a live viewer; with
.Cm users ,
names a credentials file, or
.Cm user
one user, so readers log in with SCRAM, which needs
.Cm tls
or
.Cm proxy .
A browser at the root gets a live viewer; with a user,
only behind a TLS-terminating proxy named by
.Cm proxy ,
with
@@ -437,14 +456,12 @@ or, with
.Cm transport=http ,
.Cm http_chain .
It refuses to run without credentials:
.Cm user
and
.Cm password_file ,
.Cm user ,
or a client
.Cm cert
and
.Cm key .
.It Cm aggregator Oo Cm listen , transport , tls , users , client_ca , out , format , allow , deny Oc
.It Cm aggregator Oo Cm listen , transport , tls , users , user , password_file , client_ca , out , format , allow , deny Oc
Receive from edges on
.Cm listen
.Pq default Li 0.0.0.0:9000
@@ -452,7 +469,9 @@ over TLS, never off
.Pq default Cm tls=self ,
authenticated by the
.Cm users
credentials file or the client certificates
credentials file, the one
.Cm user ,
or the client certificates
.Cm client_ca
vouches for, and write to standard output or to files in
.Cm out ;
@@ -534,7 +553,7 @@ one keeps its owner and mode.
.It Cm remove-user Fl \-credentials Ar file Fl u Ar name
Remove a user.
The last user cannot be removed.
.It Cm token Fl \-url Ar https://host:port Fl u Ar name Fl \-password\-file Ar file Oo Ar tls-flags Oc Op Fl \-unbound
.It Cm token Fl \-url Ar https://host:port Fl u Ar name Oo Fl \-password\-file Ar file Oc Oo Ar tls-flags Oc Op Fl \-unbound
Log in to an
.Cm http
sink or
@@ -544,12 +563,21 @@ source and print a bearer token.
logs in without channel binding, to an
.Cm http
sink behind a TLS-terminating proxy; the URL may then carry the proxy's path.
.It Cm stream Fl \-addr Ar host:port Fl u Ar name Fl \-password\-file Ar file Op Ar tls-flags
.It Cm stream Fl \-addr Ar host:port Fl u Ar name Oo Fl \-password\-file Ar file Oc Op Ar tls-flags
Log in to a
.Cm tcp
sink and copy its stream to standard output until interrupted.
.El
.Pp
For
.Cm token
and
.Cm stream ,
.Fl \-password\-file
defaults to
.Pa /dev/tty ,
where the password is asked without echo.
.Pp
The
.Ar tls-flags
are
@@ -627,7 +655,8 @@ is looked for.
.Pp
Secrets never go in the environment: TLS keys, credentials files and
password files are named by path
.Pq Cm tls.key_file , auth.credentials_file , auth.password_file .
.Pq Cm tls.key_file , auth.credentials_file , auth.password_file ,
or asked on the terminal.
.Sh FILES
Without
.Fl c
@@ -728,7 +757,7 @@ $ lw < app.log > copy.log
.Pp
Follow a log and keep only errors and warnings:
.Bd -literal -offset indent
$ tail \-F app.log | lw \-\-filter include,patterns=ERROR,patterns=WARN
$ tail \-F app.log | lw \-\-filter include:patterns=ERROR,patterns=WARN
.Ed
.Pp
Prefix each line with the time, the level found in it and the source:
@@ -743,7 +772,7 @@ sink listens on every address by default, and read it from another terminal
as server-sent events; a client gets the lines that arrive while it is
connected:
.Bd -literal -offset indent
$ journalctl \-f | lw \-\-sink http,host=127.0.0.1,port=8080
$ journalctl \-f | lw \-\-sink http:host=127.0.0.1,port=8080
$ curl \-N http://127.0.0.1:8080/stream
.Ed
.Pp
@@ -753,19 +782,19 @@ file in a directory, as
.Ql tail \-F
does, and write the new lines unparsed to one file:
.Bd -literal -offset indent
$ lw \-\-source 'file,directory=/var/log/app,pattern=*.log,raw=true' \e
$ lw \-\-source 'file:directory=/var/log/app,pattern=*.log,raw=true' \e
> all.log
.Ed
.Pp
Follow every file in a directory, with the level names in color on a
terminal:
.Bd -literal -offset indent
$ lw \-\-preset tail,path=/var/log/app
$ lw \-\-preset tail:path=/var/log/app
.Ed
.Pp
Print the configuration a set of options makes, to keep as a file:
.Bd -literal -offset indent
$ lw \-\-preset serve,path=/var/log/app,tls=self \-\-dump > serve.toml
$ lw \-\-preset serve:path=/var/log/app,tls=self \-\-dump > serve.toml
.Ed
.Pp
Ship a directory from an edge to an aggregator: on the aggregator, create
@@ -777,13 +806,22 @@ and the password file to the edge; then run both:
# lw tls cert \-\-ca-dir /etc/logwisp/pki \-\-name agg.example.org \-\-server
# lw auth add-user \-\-credentials /etc/logwisp/users.toml \e
\-\-user edge-01 \-\-password-file /root/edge-01.pass
agg# lw \-\-preset aggregator,tls=files,users=/etc/logwisp/users.toml,\e
agg# lw \-\-preset aggregator:tls=files,users=/etc/logwisp/users.toml,\e
cert=/etc/logwisp/pki/agg.example.org.crt,\e
key=/etc/logwisp/pki/agg.example.org.key,out=/var/log/edges
edge# lw \-\-preset edge,path=/var/log/app,to=agg.example.org:9000,\e
edge# lw \-\-preset edge:path=/var/log/app,to=agg.example.org:9000,\e
ca=/etc/logwisp/pki/ca.crt,user=edge-01,password_file=/root/edge-01.pass
.Ed
.Pp
A temporary pipe with nothing on disk: the receiver makes a self-signed
certificate, logs its pin, and generates the password when Enter alone
answers its prompt; the sender pins the certificate and is asked for the
password:
.Bd -literal -offset indent
recv$ lw \-p aggregator:listen=0.0.0.0:9000,user=pipe,format=raw
send$ cmd | lw \-p edge:to=RECEIVER:9000,pin=sha256//BASE64,user=pipe
.Ed
.Pp
Write the annotated configuration to the file
.Nm
reads without
doc/operations.md
+6 -6
View File
@@ -7,7 +7,7 @@ Running, monitoring, and maintaining LogWisp.
```bash
# a filter: stdin to stdout, exits 0 at the end of input
lw < app.log > copy.log
tail -F app.log | lw --filter include,patterns=ERROR
tail -F app.log | lw --filter include:patterns=ERROR
# a service: the file's pipelines, until SIGINT or SIGTERM
lw -c /etc/logwisp/logwisp.toml
@@ -249,16 +249,16 @@ client_buffer_size = 1024
```toml
[pipelines.flow.rate_limit]
rate = 1000.0
burst = 2000.0
entries_per_second = 1000.0
burst_entries = 2000.0
policy = "drop"
max_entry_size_bytes = 65536
```
Two behaviours to keep in mind: the limiter does not exist at all when
`rate <= 0`, and `policy = "pass"` short-circuits the size cap as well as the
rate check. Enforcing `max_entry_size_bytes` therefore requires `rate > 0` and
`policy = "drop"`.
`entries_per_second <= 0`, and `policy = "pass"` short-circuits the size cap as
well as the rate check. Enforcing `max_entry_size_bytes` therefore requires
`entries_per_second > 0` and `policy = "drop"`.
### Formatting
doc/scram-auth-plan.md
+33 -14
View File
@@ -15,8 +15,10 @@ next to the existing certificate method, without weakening what mTLS gives.
- Password authentication for chain links (`tcp_chain`, `http_chain`: sources
verify, sinks present) and for viewers of the `tcp` and `http` sinks.
- No password or password-equivalent on the wire, no KDF work on listeners, and
no credential relay through a TLS-terminating MITM.
- No password or password-equivalent on the wire, no KDF work per login on
listeners, and no credential relay through a TLS-terminating MITM.
- A temporary pipe needs no file: one listener user from a descriptor, a pipe
or the terminal, with a certificate made at startup.
- Default stays `auth.type = "none"`; existing configurations are unaffected.
- Config in TOML, credential management and viewer clients in a `lw auth`
CLI.
@@ -66,7 +68,7 @@ curl by the CLI is unpinned.
# listener (tcp_chain/http_chain source, tcp/http sink)
[pipelines.plugin_sources.config.auth]
type = "scram"
credentials_file = "/etc/logwisp/users.toml"
credentials_file = "/etc/logwisp/users.toml" # or username + password_file
token_lifetime_ms = 900000 # http sink and http_chain source only
node_binding = "force" # chain sources; binds to the username
@@ -81,12 +83,20 @@ Validation at construction:
- `scram` requires TLS, except on an `http` sink in proxy mode; a dialer also
forbids `insecure_skip_verify`.
- Listeners require `credentials_file` and reject `allow`/`allow_patterns` (the
credentials file is the allow list), `username` and `password_file`. Use one
file per listener when listeners admit different users.
- Dialers require `username` + `password_file` (1–1024 bytes after trimming one
line break), and reject `credentials_file`, `token_lifetime_ms`, `allow*` and
`identity` (server pinning stays an mTLS feature).
- Listeners require `credentials_file`, or `username` and `password_file`
(one user, its verifier derived in memory at construction), not both, and
reject `allow`/`allow_patterns`: the users are the allow list. Use one file
per listener when listeners admit different users.
- Dialers require `username` + `password_file`, and reject `credentials_file`,
`token_lifetime_ms`, `allow*` and `identity` (server pinning stays an mTLS
feature).
- A password is 1–1024 bytes after trimming one line break, 8 or more on a
listener. A regular file is read at every construction; `/dev/fd/N` (the
inherited descriptor, then closed), a pipe, a FIFO or another device once, at
startup, and kept for reloads; a terminal is asked once, at startup, without
echo, and on a listener Enter alone generates a password shown only there.
The open file decides, not stat: FreeBSD stats `/dev/stdin` as a device
whatever it holds. No terminal fails closed.
- `identity` on a listener requires `tls.client_auth` and **binds the certificate
to the user**: the certificate's identity field must equal the SCRAM username,
so a peer needs its own certificate and its own password (PostgreSQL's
@@ -207,8 +217,8 @@ closed.
```
lw auth add-user --credentials F --user U [--password-file P] [--generate]
lw auth remove-user --credentials F --user U
lw auth token --url https://host:port[/path] --user U --password-file P [--unbound] [TLS flags]
lw auth stream --addr host:port --user U --password-file P [TLS flags]
lw auth token --url https://host:port[/path] --user U [--password-file P] [--unbound] [TLS flags]
lw auth stream --addr host:port --user U [--password-file P] [TLS flags]
```
- `add-user` takes the password from `--password-file` when it exists (at least
@@ -224,7 +234,7 @@ lw auth stream --addr host:port --user U --password-file P [TLS flags]
- `token` prints a bearer token for curl (`-H @<(...)` keeps it out of argv);
`stream` authenticates to a `tcp` sink and copies the stream to stdout until
interrupted. `--url` takes a path only with `--unbound`, and redirects are not
followed.
followed. `--password-file` defaults to `/dev/tty`, which asks.
- TLS flags: `--ca-file` or `--pin-sha256`, `--server-name`, `--cert-file`,
`--key-file`. There is no insecure flag.
- Exit status: 0 success, 1 failure, 2 usage error.
@@ -340,7 +350,11 @@ a challenge below its Argon2 floor; the HTTP exchange on a TLS test server
(token accepted, garbage and foreign tokens `401`, forged final yields no token,
a different certificate after the challenge refused before the proof is sent,
renewal ahead of expiry); `Authorize` failing closed under SCRAM; password
trimming; refused hellos freeing their slot; unbound logins pinned too. Proxy
trimming; a one-user listener admitting its password, refusing another and an
unknown user alike; a descriptor's password surviving a reload, closed and
refused to a second user; a descriptor lw opened itself never read; refused
hellos freeing their slot; unbound logins pinned too. In `tlsx`, terminals,
pipes and descriptors never warned as world-readable. Proxy
mode: trusted peers, https and the client hop; throttling on the forwarded
client and IPv6 /64; an unbound browser login with the cookie's attributes, the
cookie opening endpoints (beside Basic auth) until logout revokes it, cookie
@@ -371,7 +385,12 @@ viewer's own form, not a loop, its token kept out of storage and the URL; a
profile blocking every cookie signing in on the viewer, its proofs asking for
no cookie, streaming with a bearer, sign-out revoking the token, and a reload's
`401` asking again; `token --unbound` and curl through the proxy; direct peers
and plaintext-forwarded requests `403`. The existing scripts keep passing.
and plaintext-forwarded requests `403`. `test/preset-test.sh --auto` (ports
15897-15898): an aggregator and an edge given one password on `/dev/fd/3`
deliver, another password is refused, both survive `SIGHUP` and deliver again,
and the password is in no file, argv or environment; under `script(1)` a
listener's generated password shows on its terminal and not in its log, and
an edge and `lw auth token` log in with it. The existing scripts keep passing.
## Not Implemented
doc/security.md
+68 -9
View File
@@ -159,8 +159,8 @@ allow_patterns = [] # mtls
node_binding = "force" # chain sources only
credentials_file = "" # scram listeners
token_lifetime_ms = 0 # scram, HTTP listeners; 10 s to 24 h, 0 = 15 minutes
username = "" # scram dialers
password_file = "" # scram dialers
username = "" # scram dialers, or a listener's one user
password_file = "" # scram, with username
trusted_proxies = [] # scram http sink behind a TLS-terminating proxy
```
@@ -181,7 +181,9 @@ Options by where they apply, each with its default:
them yourself.
- `scram` listeners
- `credentials_file`: verifiers written by
[`lw auth add-user`](cli.md#lw-auth).
[`lw auth add-user`](cli.md#lw-auth); or, in its place, `username` and
`password_file`, see
[One user without a credentials file](#one-user-without-a-credentials-file).
- `token_lifetime_ms` (15 minutes): bearer token lifetime, 10 s to 24 h; the
`http` sink and `http_chain` source only.
- `trusted_proxies` (`[]`): addresses or CIDRs of the reverse proxies that
@@ -190,7 +192,7 @@ Options by where they apply, each with its default:
- `scram` dialers
- `username`: user to log in as.
- `password_file`: file holding the password; one trailing line break is
trimmed.
trimmed. `/dev/fd/N`, a pipe or a terminal are read once, as on listeners.
**Roles by plugin**, and what the block decides:
@@ -306,9 +308,10 @@ Misconfiguration fails at plugin construction, before the pipeline starts:
type forgotten
- a key of the other method: `allow` or `allow_patterns` under `scram`, a
`scram` key under `mtls`
- a `scram` listener without a loadable `credentials_file` (see
[Credentials file](#credentials-file)), with `username` or `password_file`,
with a `token_lifetime_ms` outside 10 s to 24 h or on a TCP listener, or with
- a `scram` listener with neither a loadable `credentials_file` (see
[Credentials file](#credentials-file)) nor `username` and `password_file`,
or with both, with only one of `username` and `password_file`, with a
`token_lifetime_ms` outside 10 s to 24 h or on a TCP listener, or with
`identity` but no `tls.client_auth`
- a `scram` dialer without `username` and `password_file`, with
`credentials_file`, `token_lifetime_ms`, `trusted_proxies` or `identity`
@@ -435,8 +438,8 @@ when it is constructed, so startup and every reload repeat them:
- an `allow_patterns` entry not anchored at both ends of every alternative
(`^a|b$` admits `a…` and `…b`)
- a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every
local user can read
(once per path per process)
local user can read (once per path per process); terminals, pipes and
descriptors hold nothing at rest and are not reported
- an `acl` entry with host bits set (`10.0.0.1/8`), naming the network it
matches; an `acl.allow` entry of a whole family (`0.0.0.0/0`, `::/0`); a
wildcard listener whose `acl` has `deny` but no `allow`; an `acl.proxy_from`
@@ -477,6 +480,18 @@ lw tls cert --ca-dir /etc/logwisp/pki --name agg.example.org --server
lw tls cert --ca-dir /etc/logwisp/pki --name edge-01 --client
```
A temporary pipe needs no file at all: a self-signed certificate and
[one user](#one-user-without-a-credentials-file) whose password is typed, or
generated, at startup. Both last until lw exits.
```bash
# receiver: Enter at the prompt generates the password; copy it, and the
# pin_sha256 of the WARN line (-q would hide it)
lw -p aggregator:listen=0.0.0.0:9000,tls=self,user=pipe,format=raw,allow=SENDER/32
# sender: asked for the password
cmd | lw -p edge:to=RECEIVER:9000,pin=sha256//PIN,user=pipe
```
## Enabling mTLS
### 1. Generate a CA and certificates
@@ -702,6 +717,50 @@ only when the plugin is built and `auto_reload` does not watch it: send
`SIGHUP` after every change. Use one file per listener when listeners admit
different users.
### One user without a credentials file
`username` and `password_file` on a listener replace `credentials_file`: one
user whose verifier is derived in memory when the plugin is built (one
Argon2id at startup and each reload, never per login), with a decoy key drawn
once per process. Nothing is written. It suits a temporary pipe or a single
edge. `password_file` is read by what it names, on either side:
- A regular file: at every build, so `SIGHUP` rotates it. A systemd
`LoadCredential=` file sits on tmpfs.
- `/dev/fd/N`, a pipe, a FIFO or another device: once, at startup, then kept
in memory for reloads; restart lw to change it. A reload naming a new one
fails and the running pipelines stay: a console source may be reading the
same input, and the read would hold the reload and its signals.
- `/dev/fd/N` is the descriptor lw was given (`3< <(pass show lw/pipe)`),
read and closed; it needs no fdescfs on FreeBSD. A descriptor lw opened
itself is refused.
- One such source holds one user's password. `/dev/stdin` would take the
console source's input.
- A terminal: `/dev/tty`, the presets' default with `user`, asked once, at
startup, without echo. lw asks whatever file is a terminal once opened, so
`/dev/stdin` holding a pipe is read on FreeBSD too, where stat calls it a
device.
- On a listener, Enter alone generates a 130-bit password and shows it once
on that terminal, never in the log or on stderr: logs persist, are
forwarded, and stderr is often a file.
- With no terminal (systemd, cron) the plugin fails. `lw ... &` stops on
`SIGTTIN` at the prompt.
Trade-offs:
- The password lives as long as the process, as dialers' passwords do:
`tcp_chain` logs in again on every reconnect and `http_chain` at every token
renewal, so it cannot be single-use.
- A generated password stays in terminal scrollback, tmux or screen logs, and
the clipboard that carries it. A typed one may be as short as 8 bytes; on an
exposed listener prefer the generated one and `acl.allow`.
- It never goes in argv or the environment: `/proc/PID/cmdline` is
world-readable, and `--dump` keeps only the path.
- Each build draws a new salt, which, like a rotation, shows a prober that the
name exists.
- Older bash and zsh back a here-string (`<<<`) with a temporary file; prefer
`3< <(...)`.
### Rollout, rotation and revocation
- **Rollout.** Turning `scram` on cuts off every dialer of that listener that
doc/sinks.md
+6 -5
View File
@@ -173,11 +173,12 @@ Options, each as type and default:
- `replay_lines` (int, `0`; `1000` from the `serve` preset): the last entries
a new stream gets after its `connected` event, before the live ones, so a
browser opened late sees a backlog; `0` = none.
- `login_page` (bool, `false`): `scram` in proxy mode only (an error
elsewhere): the browser login page at `/auth/login`.
- `viewer_page` (bool, `false`): `scram` in proxy mode only (an error
elsewhere): the live viewer at `/auth/view`; needs `login_page`. Without
auth, or under `mtls`, the viewer is always served.
- `login_page` (bool, `false`): only for `scram` behind
`auth.trusted_proxies`, where browsers log in (an error elsewhere): the
login page at `/auth/login`.
- `viewer_page` (bool, `false`): the same, the live viewer at `/auth/view`
after a login; needs `login_page`. Without auth, or under `mtls`, the
viewer is always at `/`, with neither key.
- `tls` (table): listener TLS; see [Security](security.md).
- `auth` (table): client authentication (`mtls` or `scram`); see
[Security](security.md#the-auth-block).
doc/sources.md
+2 -1
View File
@@ -227,7 +227,8 @@ Options, each as type and default:
[Security](security.md#the-acl-block).
For passwords instead of certificates, the `auth` block takes `type = "scram"`
and a `credentials_file`, and `client_auth` becomes optional; see
and a `credentials_file`, or one `username` with its `password_file`, and
`client_auth` becomes optional; see
[Password Authentication](security.md#password-authentication-scram).
**Behaviour**
doc/todo.md
+6 -2
View File
@@ -102,7 +102,9 @@ What exists:
auth and acl tables, sources, sinks and presets, as one JSON document.
- `github.com/lixenwraith/terminal` (pinned) and its `tui` package: regions,
layout, boxes, List, Tree, TabBar, StatusBar, TextField, Form, Modal,
ConfirmDialog, scrolling, mouse. `github.com/lixenwraith/color`: RGB,
ConfirmDialog, scrolling, mouse; themed Radio, Field, TextInput,
OptionList and menus, Frame, Rule, Wires and Window, which `lw --tui`
draws with. `github.com/lixenwraith/color`: RGB,
blending, RGBTo256, RGBTo16.
Steps:
@@ -112,7 +114,9 @@ Steps:
4. `lixenwraith/terminal` v0.2.0: done: 16 and no color, the terminal's
default colors, `/dev/tty`, bracketed paste, themed form controls. Still
optional, in `color`: export the 16 xterm reference shades (RGBTo16 maps
the site's hues badly, so the 16-color tier is chosen by hand).
the site's hues badly, so the 16-color tier is chosen by hand). v0.3.0:
radio buttons, narrow form fields and option lists, menus, frames, rules,
wires and windows, moved from `lw --tui`.
5. `lw --tui`: done ([doc/cli.md](cli.md#lw---tui)).
6. WebAssembly: done. `make wasm` builds `bin/lwconf.wasm`, version stamped,
beside the toolchain's `wasm_exec.js`, and prints both checksums: about
go.mod
+2 -2
View File
@@ -1,13 +1,13 @@
module github.com/lixenwraith/logwisp
go 1.27.1
go 1.27.2
require (
github.com/lixenwraith/auth v0.5.1
github.com/lixenwraith/color v0.1.3
github.com/lixenwraith/config v0.2.2
github.com/lixenwraith/log v0.1.11
github.com/lixenwraith/terminal v0.2.0
github.com/lixenwraith/terminal v0.2.1
github.com/lixenwraith/toml v0.1.3
golang.org/x/term v0.47.0
)
go.sum
+4 -2
View File
@@ -8,8 +8,10 @@ github.com/lixenwraith/config v0.2.2 h1:X+l3Tf8Rbk49cipYBJU7/pfdw5wDwD4hRxtFH7rZ
github.com/lixenwraith/config v0.2.2/go.mod h1:yfJQwKS+/6Dg70GIoHeYc0eYztOPETpqe6nUIbakcyg=
github.com/lixenwraith/log v0.1.11 h1:dKWLnaR8dhXxj4l/aU5XZiFIsr8geaJQbOf8m7SiKhU=
github.com/lixenwraith/log v0.1.11/go.mod h1:EE8i5nSAW2yZBTytKuhdgehmgzrby6kW1k5DyH1SvBg=
github.com/lixenwraith/terminal v0.2.0 h1:Xwsg8LoAeV7E//JLIFmEBTTV9e03jpze3EUAOdqS6P8=
github.com/lixenwraith/terminal v0.2.0/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44=
github.com/lixenwraith/terminal v0.2.1-0.20261010134811-bbb83d6c20df h1:HPfMYYSfNva8cMlgyTtZntu3oUwNKXW48rCwpmTEPoU=
github.com/lixenwraith/terminal v0.2.1-0.20261010134811-bbb83d6c20df/go.mod h1:f9Ntk+xKBS2LWQEe6gk/1TOpG22HH0scTe4wlPhEy44=
github.com/lixenwraith/terminal v0.2.1 h1:KsMfmxrRicBFWMdXJIEJ3cpLWVw6IoNF8GwgsxbD3kQ=
github.com/lixenwraith/terminal v0.2.1/go.mod h1:V0P/FDpY+UeSvx1VIVuTG7llo0RbK3XrSJLDA7GqHqY=
github.com/lixenwraith/toml v0.1.3 h1:mhnNVLNogdaqjGd3eC4rV9ZqY3j1Sv+DK1erZ+SPFeI=
github.com/lixenwraith/toml v0.1.3/go.mod h1:itcT/Vt5mwmX9KVGCMsWZr6vbU/JgZ24wV/I0J5cMt0=
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
internal/authz/authz.go
+1 -1
View File
@@ -448,7 +448,7 @@ func (p *Policy) Enabled() bool { return p != nil }
func (p *Policy) NeedsLogin() bool { return p != nil && p.listener != nil }
// Unrestricted reports whether an mtls policy admits any identity the CA
// vouches for. Under scram the credentials file is the allow list.
// vouches for. Under scram the users are the allow list.
func (p *Policy) Unrestricted() bool {
return p != nil && p.method == MethodMTLS && len(p.allow) == 0 && len(p.patterns) == 0
}
internal/authz/authz_test.go
+7 -4
View File
@@ -139,9 +139,10 @@ func TestNewValidation(t *testing.T) {
{"dialer skips verify", &config.AuthOptions{Type: MethodMTLS}, &tls.Config{InsecureSkipVerify: true}, config.Dialer, TCP, "insecure_skip_verify"},
{"mtls with a password", &config.AuthOptions{Type: MethodMTLS, PasswordFile: pw}, &tls.Config{}, config.Dialer, TCP, "password_file applies only"},
{"scram without tls", listener(config.AuthOptions{}), nil, config.Listener, TCP, "requires tls.enabled"},
{"scram without credentials", scram(config.AuthOptions{}), f.serverTLS, config.Listener, TCP, "requires credentials_file"},
{"scram with an allow list", listener(config.AuthOptions{Allow: []string{"edge-01"}}), f.serverTLS, config.Listener, TCP, "credentials file is the allow list"},
{"scram listener with a password", listener(config.AuthOptions{Username: "edge-01"}), f.serverTLS, config.Listener, TCP, "only to dialers"},
{"scram without users", scram(config.AuthOptions{}), f.serverTLS, config.Listener, TCP, "requires credentials_file, or username and password_file"},
{"scram with an allow list", listener(config.AuthOptions{Allow: []string{"edge-01"}}), f.serverTLS, config.Listener, TCP, "the users are the allow list"},
{"credentials and a user", listener(config.AuthOptions{Username: "edge-01"}), f.serverTLS, config.Listener, TCP, "not both"},
{"user without password_file", scram(config.AuthOptions{Username: "edge-01"}), f.serverTLS, config.Listener, TCP, "go together"},
{"scram certificate binding without client_auth", listener(config.AuthOptions{Identity: "cn"}), f.serverTLS, config.Listener, TCP, "requires tls.client_auth"},
{"token lifetime on tcp", listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, config.Listener, TCP, "only to HTTP listeners"},
{"token lifetime too short", listener(config.AuthOptions{TokenLifetimeMS: 900}), f.serverTLS, config.Listener, HTTP, "must be from 10000 to 86400000"},
@@ -176,9 +177,11 @@ func TestNewValidation(t *testing.T) {
}
}
}
if _, err := New(listener(config.AuthOptions{TokenLifetimeMS: 60000}), f.serverTLS, nil, config.Listener, HTTP); err != nil {
for _, o := range []*config.AuthOptions{listener(config.AuthOptions{TokenLifetimeMS: 60000}), scram(config.AuthOptions{Username: "edge-01", PasswordFile: pw})} {
if _, err := New(o, f.serverTLS, nil, config.Listener, HTTP); err != nil {
t.Fatalf("scram listener rejected: %v", err)
}
}
// Behind proxies TLS ends at the proxy, so the hop may be plaintext
if _, err := New(listener(config.AuthOptions{TrustedProxies: proxies}), nil, nil, config.Listener, HTTP); err != nil {
t.Fatalf("proxy-mode listener rejected: %v", err)
internal/authz/scram.go
+158 -6
View File
@@ -16,8 +16,11 @@ import (
"net"
"net/http"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/lixenwraith/logwisp/internal/chain"
@@ -27,6 +30,7 @@ import (
"github.com/lixenwraith/auth"
"github.com/lixenwraith/toml"
"golang.org/x/term"
)
const (
@@ -162,10 +166,87 @@ func (c *Credentials) Marshal() ([]byte, error) {
return append([]byte(header), body...), nil
}
// ReadPassword reads a password file, trimming one trailing line break so a
// file from an editor and one from `lw auth add-user` agree.
func ReadPassword(path string) (string, error) {
// processDecoy is the decoy key of listeners without a credentials file:
// one per process, so reloads keep unknown-user challenges stable
var processDecoy = sync.OnceValue(func() []byte { key := make([]byte, 32); rand.Read(key); return key })
// oneUser is a listener's only user, derived in memory at the dialers' floor
// cost, the default outside tests
func oneUser(user, path string) (*Credentials, error) {
pw, err := ReadPassword(path, user, true)
if err != nil {
return nil, err
}
cred, err := auth.NewCredential(user, pw, auth.WithTime(minArgonTime), auth.WithMemory(minArgonMemory))
if err != nil {
return nil, fmt.Errorf("auth: user %q: %w", user, err)
}
return &Credentials{DecoyKey: processDecoy(), Users: []*auth.Credential{cred}}, nil
}
// readOnce keeps what a reload cannot read again, by path and user, and
// whether a terminal was asked: one may be asked for several users
var readOnce = struct {
sync.Mutex
m map[[2]string]asked
}{m: map[[2]string]asked{}}
type asked struct {
pw string
tty bool
}
// startupOver is set once lw's first service is built
var startupOver atomic.Bool
// EndStartup stops reading descriptors, pipes, FIFOs and terminals: at a
// reload a console source may be reading the same one, and a read would hold
// the reload and its signals. What was read before stays for reloads.
func EndStartup() { startupOver.Store(true) }
// ReadPassword reads user's password from path, trimming one trailing line
// break so a file from an editor and one from `lw auth add-user` agree. A
// regular file is read each time, for rotation by reload; /dev/fd/N, a pipe,
// a FIFO or another device once, before EndStartup; a terminal is asked once,
// where generate lets Enter alone draw the password.
func ReadPassword(path, user string, generate bool) (string, error) {
fd, inherited := descriptor(path)
if fi, err := os.Stat(path); !inherited && (err != nil || fi.Mode().IsRegular()) {
data, err := os.ReadFile(path)
return checkPassword(path, data, err)
}
readOnce.Lock()
defer readOnce.Unlock()
key := [2]string{path, user}
if a, ok := readOnce.m[key]; ok {
return a.pw, nil
}
if startupOver.Load() {
return "", fmt.Errorf("auth: password_file %s is read only at startup; name a regular file or restart lw", path)
}
for k, a := range readOnce.m {
if k[0] == path && !a.tty {
return "", fmt.Errorf("auth: password_file %s was read for user %q and holds one password", path, k[1])
}
}
var (
data []byte
tty bool
err error
)
if inherited {
data, err = readDescriptor(fd, path)
} else {
data, tty, err = readDevice(path, user, generate)
}
pw, err := checkPassword(path, data, err)
if err == nil {
readOnce.m[key] = asked{pw, tty}
}
return pw, err
}
func checkPassword(path string, data []byte, err error) (string, error) {
if err != nil {
return "", fmt.Errorf("auth: password_file: %w", err)
}
@@ -176,6 +257,70 @@ func ReadPassword(path string) (string, error) {
return pw, nil
}
// descriptor is N of /dev/fd/N, which ReadPassword closes: the path then
// names another file. 0-2 stay open, or stdio would go to the next file.
func descriptor(path string) (int, bool) {
n, ok := strings.CutPrefix(path, "/dev/fd/")
fd, err := strconv.Atoi(n)
return fd, ok && err == nil && fd > 2
}
// readDescriptor reads an inherited descriptor to its end and closes it,
// which needs no fdescfs on FreeBSD. Go opens its own close-on-exec, so a
// descriptor with the flag is lw's: reading or closing it would break lw.
func readDescriptor(fd int, path string) ([]byte, error) {
flags, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFD, 0)
if errno != 0 || flags&syscall.FD_CLOEXEC != 0 {
return nil, fmt.Errorf("%s was not passed to lw: redirect it, as in %d<FILE", path, fd)
}
f := os.NewFile(uintptr(fd), path)
defer f.Close()
return io.ReadAll(io.LimitReader(f, auth.MaxPasswordLen+3))
}
// readDevice reads a pipe, FIFO or device to its end, or asks when it is a
// terminal. The open file decides: on FreeBSD stat calls /dev/stdin a device
// whatever it holds.
func readDevice(path, user string, generate bool) ([]byte, bool, error) {
f, err := os.Open(path)
if errors.Is(err, syscall.ENXIO) {
err = fmt.Errorf("%s: no terminal to ask on; name a password file", path)
}
if err != nil {
return nil, false, err
}
defer f.Close()
if term.IsTerminal(int(f.Fd())) {
data, err := ask(path, user, generate)
return data, true, err
}
data, err := io.ReadAll(io.LimitReader(f, auth.MaxPasswordLen+3))
return data, false, err
}
// ask prompts on the terminal at path without echo. A generated password is
// shown there only: logs persist, and stderr is often a file.
func ask(path, user string, generate bool) ([]byte, error) {
tty, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return nil, err
}
defer tty.Close()
hint := ""
if generate {
hint = " (Enter generates one)"
}
fmt.Fprintf(tty, "password for %s%s: ", user, hint)
data, err := term.ReadPassword(int(tty.Fd()))
fmt.Fprintln(tty)
if err == nil && len(data) == 0 && generate {
pw := rand.Text()
fmt.Fprintf(tty, "generated password for %s, shown once: %s\n", user, pw)
return []byte(pw), nil
}
return data, err
}
// --- Listener ---
type scramListener struct {
@@ -215,7 +360,14 @@ func (p *Policy) compileSCRAM(o *config.AuthOptions, tlsCfg *tls.Config) error {
cb := sha256.Sum256(tlsCfg.Certificates[0].Certificate[0])
l.cb = cb[:]
}
if l.creds, err = LoadCredentials(o.CredentialsFile); err != nil {
secret := secretFile{"auth.credentials_file", o.CredentialsFile}
if o.CredentialsFile != "" {
l.creds, err = LoadCredentials(o.CredentialsFile)
} else {
secret = secretFile{"auth.password_file", o.PasswordFile}
l.creds, err = oneUser(o.Username, o.PasswordFile)
}
if err != nil {
return err
}
if p.transport == HTTP {
@@ -231,7 +383,7 @@ func (p *Policy) compileSCRAM(o *config.AuthOptions, tlsCfg *tls.Config) error {
}
}
p.listener = l
p.secrets = append(p.secrets, secretFile{"auth.credentials_file", o.CredentialsFile})
p.secrets = append(p.secrets, secret)
return nil
}
@@ -543,7 +695,7 @@ type scramDialer struct {
}
func (p *Policy) compileSCRAMDialer(o *config.AuthOptions) error {
password, err := ReadPassword(o.PasswordFile)
password, err := ReadPassword(o.PasswordFile, o.Username, false)
if err != nil {
return err
}
internal/authz/scram_test.go
+119 -7
View File
@@ -19,6 +19,7 @@ import (
"regexp"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
@@ -101,7 +102,9 @@ func (f *fixture) pool(t *testing.T) *x509.CertPool {
func (f *fixture) listener(t *testing.T, o config.AuthOptions, tlsCfg *tls.Config, role config.Side, transport Transport) *Policy {
t.Helper()
o.Type, o.CredentialsFile = MethodSCRAM, f.creds
if o.Type = MethodSCRAM; o.Username == "" {
o.CredentialsFile = f.creds
}
p, err := New(&o, tlsCfg, nil, role, transport)
if err != nil {
t.Fatal(err)
@@ -348,13 +351,15 @@ func TestCertificateBindsToUser(t *testing.T) {
}
}
// An unknown user's challenge salt comes from the file's decoy key, so it is
// the same after a restart and a prober learns nothing from restarts.
// An unknown user's challenge salt comes from the decoy key, the file's or,
// for one user without a file, the process's, so it is the same after a
// rebuild and a prober learns nothing from restarts or reloads.
func TestUnknownUserSaltSurvivesRestart(t *testing.T) {
f := newFixture(t)
for _, o := range []config.AuthOptions{{}, {Username: "pipe", PasswordFile: f.write(t, "pipe.pass", "pipe-secret\n")}} {
var salts []string
for range 2 {
l := f.listener(t, config.AuthOptions{}, f.serverTLS, config.Listener, TCP)
l := f.listener(t, o, f.serverTLS, config.Listener, TCP)
c, err := l.listener.server.Load().ProcessClientFirstMessage("nobody", strings.Repeat("n", 32))
if err != nil {
t.Fatal(err)
@@ -362,7 +367,8 @@ func TestUnknownUserSaltSurvivesRestart(t *testing.T) {
salts = append(salts, c.Salt)
}
if salts[0] != salts[1] {
t.Fatalf("unknown user salts differ across restarts: %q, %q", salts[0], salts[1])
t.Fatalf("%+v: unknown user salts differ across rebuilds: %q, %q", o, salts[0], salts[1])
}
}
}
@@ -615,15 +621,121 @@ func TestRefusedHellosFreeTheirSlot(t *testing.T) {
func TestReadPasswordTrimsOneLineBreak(t *testing.T) {
f := newFixture(t)
for _, contents := range []string{"secret", "secret\n", "secret\r\n"} {
if pw, err := ReadPassword(f.write(t, "pw", contents)); err != nil || pw != "secret" {
if pw, err := ReadPassword(f.write(t, "pw", contents), "edge-01", false); err != nil || pw != "secret" {
t.Errorf("ReadPassword(%q) = %q, %v", contents, pw, err)
}
}
if _, err := ReadPassword(f.write(t, "pw", "\n")); err == nil {
if _, err := ReadPassword(f.write(t, "pw", "\n"), "edge-01", false); err == nil {
t.Error("an empty password was accepted")
}
}
// A descriptor's password is read once and the descriptor closed: a reload
// gets it from memory, and another user naming it is refused.
func TestPasswordFromADescriptorSurvivesReload(t *testing.T) {
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
fd, err := syscall.Dup(int(r.Fd())) // not close-on-exec, as a shell passes it
r.Close()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { readOnce.Lock(); clear(readOnce.m); readOnce.Unlock() })
var before, after syscall.Stat_t
syscall.Fstat(fd, &before)
w.WriteString("pipe-secret\n")
w.Close()
path := fmt.Sprintf("/dev/fd/%d", fd)
for range 2 {
if pw, err := ReadPassword(path, "pipe", true); err != nil || pw != "pipe-secret" {
t.Fatalf("ReadPassword(%s) = %q, %v", path, pw, err)
}
}
if syscall.Fstat(fd, &after) == nil && after.Ino == before.Ino {
t.Error("the descriptor was left open")
}
if _, err := ReadPassword(path, "other", false); err == nil || !strings.Contains(err.Error(), `read for user "pipe"`) {
t.Errorf("another user of the descriptor: %v", err)
}
}
// After startup a descriptor, pipe or terminal is not read: a console source
// may read the same one. One read at startup stays for reloads.
func TestOnceReadSourcesAreReadAtStartupOnly(t *testing.T) {
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
fd, err := syscall.Dup(int(r.Fd()))
r.Close()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { startupOver.Store(false); readOnce.Lock(); clear(readOnce.m); readOnce.Unlock() })
w.WriteString("pipe-secret\n")
w.Close()
path := fmt.Sprintf("/dev/fd/%d", fd)
if _, err := ReadPassword(path, "early", false); err != nil {
t.Fatalf("at startup: %v", err)
}
EndStartup()
if _, err := ReadPassword(os.DevNull, "late", false); err == nil || !strings.Contains(err.Error(), "only at startup") {
t.Fatalf("a device after startup: %v", err)
}
if pw, err := ReadPassword(path, "early", false); err != nil || pw != "pipe-secret" {
t.Fatalf("a reload of the early user: %q, %v", pw, err)
}
}
// A device is classed by the file it opens, not by stat: FreeBSD stats
// /dev/stdin as a device whatever it holds, so one that is no terminal is read.
func TestADeviceThatIsNoTerminalIsRead(t *testing.T) {
if _, err := ReadPassword(os.DevNull, "dev", false); err == nil || !strings.Contains(err.Error(), "password must be") {
t.Fatalf("%s: %v; want it read, and empty", os.DevNull, err)
}
}
// A /dev/fd/N lw opened itself, close-on-exec as Go opens every file, was
// never passed to it: it is refused and left open.
func TestOwnDescriptorIsNeverRead(t *testing.T) {
f, err := os.Open(os.DevNull)
if err != nil {
t.Fatal(err)
}
defer f.Close()
if _, err := ReadPassword(fmt.Sprintf("/dev/fd/%d", f.Fd()), "pipe", false); err == nil || !strings.Contains(err.Error(), "not passed to lw") {
t.Fatalf("lw's own descriptor: %v", err)
}
if _, err := f.Stat(); err != nil {
t.Fatalf("lw's own descriptor was closed: %v", err)
}
}
// A listener's one user, from username and password_file, logs in with its
// password; another password and an unknown user fail alike.
func TestOneUserListenerAdmitsItsPassword(t *testing.T) {
f := newFixture(t)
o := config.AuthOptions{Username: "pipe", PasswordFile: f.write(t, "pipe.pass", "pipe-secret\n")}
l := f.listener(t, o, f.serverTLS, config.ChainListener, TCP)
res, _, err := f.connect(t, l, f.serverTLS, f.dialer(t, "pipe", "pipe-secret"), nil)
if err != nil || res.err != nil || res.adm.Identity.Name != "pipe" {
t.Fatalf("exchange: dialer %v, listener %v", err, res.err)
}
var seen []string
for _, d := range []*Policy{f.dialer(t, "pipe", "edge-01-secret"), f.dialer(t, "edge-01", "pipe-secret")} {
res, _, err := f.connect(t, l, f.serverTLS, d, nil)
if !errors.Is(err, ErrRefused) || !errors.Is(res.err, ErrRefused) {
t.Fatalf("dialer %v, listener %v; want both refused", err, res.err)
}
seen = append(seen, err.Error())
}
if seen[0] != seen[1] {
t.Fatalf("wrong password and unknown user differ: %q vs %q", seen[0], seen[1])
}
}
// A credentials file is validated whole, and Marshal round-trips. Each case
// breaks one rule of an otherwise valid file.
func TestCredentialsFileValidation(t *testing.T) {
internal/compose/compose.go
+74 -7
View File
@@ -181,13 +181,34 @@ func (c *Composition) pipeline(i int) (*config.PipelineConfig, error) {
// AddPipeline appends an empty pipeline under a name no other has
func (c *Composition) AddPipeline(name string) error {
if name == "" || slices.ContainsFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }) {
return fmt.Errorf("pipeline %q: a name no other pipeline has", name)
if err := c.free(name, -1); err != nil {
return err
}
c.Pipelines = append(c.Pipelines, config.PipelineConfig{Name: name, Flow: &config.FlowConfig{}})
return nil
}
// RenamePipeline gives pipeline i a name no other has
func (c *Composition) RenamePipeline(i int, name string) error {
p, err := c.pipeline(i)
if err == nil {
err = c.free(name, i)
}
if err != nil {
return err
}
p.Name = name
return nil
}
// free refuses a name that is empty or held by a pipeline other than i
func (c *Composition) free(name string, i int) error {
if j := slices.IndexFunc(c.Pipelines, func(p config.PipelineConfig) bool { return p.Name == name }); name == "" || j >= 0 && j != i {
return fmt.Errorf("pipeline %q: a name no other pipeline has", name)
}
return nil
}
func (c *Composition) RemovePipeline(i int) error {
if _, err := c.pipeline(i); err != nil {
return err
@@ -233,6 +254,52 @@ func (c *Composition) Add(pi int, role, typ string) (Node, error) {
return n, nil
}
// Retype gives a source or sink another type of its role, in its place: a
// chosen id stays and an automatic one follows the type. Options of the same
// name and kind carry over to a type on the same network side, where they
// mean the same; across sides host, port, tls and auth do not.
func (c *Composition) Retype(pi int, n Node, typ string) (Node, error) {
p, err := c.pipeline(pi)
if err != nil {
return Node{}, err
}
to, ok := config.LookupPlugin(n.Role, typ)
if !ok {
return Node{}, fmt.Errorf("unknown %s type %q", n.Role, typ)
}
var id, old *string
var opts *map[string]any
var ids []string
switch i := index(p, n); {
case i >= 0 && n.Role == "source":
id, old, opts = &p.PluginSources[i].ID, &p.PluginSources[i].Type, &p.PluginSources[i].Config
for _, s := range p.PluginSources {
ids = append(ids, s.ID)
}
case i >= 0 && n.Role == "sink":
id, old, opts = &p.PluginSinks[i].ID, &p.PluginSinks[i].Type, &p.PluginSinks[i].Config
for _, s := range p.PluginSinks {
ids = append(ids, s.ID)
}
default:
return Node{}, n.missing()
}
from, _ := config.LookupPlugin(n.Role, *old)
kept := map[string]any{}
for _, k := range from.Keys() {
same := func(t config.Key) bool { return t.Name == k.Name && t.Kind == k.Kind }
if v, set := (*opts)[k.Name]; set && from.Side == to.Side && slices.ContainsFunc(to.Keys(), same) {
kept[k.Name] = v
}
}
suffix, auto := strings.CutPrefix(*id, *old)
if n, err := strconv.Atoi(strings.TrimPrefix(suffix, "_")); auto && (suffix == "" || suffix[0] == '_' && err == nil && n > 1) {
*id = config.FreeID(typ, slices.DeleteFunc(ids, func(s string) bool { return s == *id }))
}
*old, *opts = typ, kept
return Node{Role: n.Role, ID: *id}, nil
}
// Remove deletes a source, sink or filter, or turns a flow stage off
func (c *Composition) Remove(pi int, n Node) error {
p, err := c.pipeline(pi)
@@ -353,22 +420,22 @@ func (c *Composition) edit(pi int, n Node, change func(map[string]any) error) er
return plugin(p.PluginSinks[i].Type, &p.PluginSinks[i].Config)
case n.Role == "filters":
filter := &p.Flow.Filters[i]
if err := editStage(reflect.ValueOf(&filter).Elem(), change); err != nil {
if err := editStage(reflect.ValueOf(&filter).Elem(), "", change); err != nil {
return err
}
p.Flow.Filters[i] = *filter
return nil
}
if stage, ok := stages(p.Flow)[n.Role]; ok {
return editStage(stage, change)
return editStage(stage, n.Role, change)
}
return n.missing()
}
// editStage edits a stage's keys as a spec sets them; the table they scan into
// replaces the stage, which turns on
func editStage(stage reflect.Value, change func(map[string]any) error) error {
m := map[string]any{}
// replaces the stage, which turns on as a spec turns it on
func editStage(stage reflect.Value, key string, change func(map[string]any) error) error {
m := config.StageOn(key, nil)
if !stage.IsNil() {
m = config.Values(stage.Interface())
}
internal/compose/compose_test.go
+70 -1
View File
@@ -1,6 +1,7 @@
package compose
import (
"fmt"
"os"
"os/exec"
"reflect"
@@ -64,7 +65,7 @@ func TestWrittenFormsStartTheSameComposition(t *testing.T) {
_, err = c.Add(0, "filters", "include")
must(err)
must(c.MoveFilter(0, 1, 0))
must(c.Set(0, Node{Role: "rate_limit"}, "rate", "100"))
must(c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "100"))
must(c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "5000"))
must(c.Set(0, Node{Role: "heartbeat"}, "enabled", "true"))
must(c.AddPipeline("-b"))
@@ -190,3 +191,71 @@ func TestUnnamedPartsTakeFreeIDs(t *testing.T) {
t.Fatalf("%v\n%s", err, line)
}
}
// Retype keeps the part's place and a chosen id, renames an automatic one,
// and carries the options of the same name and kind only within a side
func TestRetypeKeepsThePlace(t *testing.T) {
c := &Composition{}
if err := c.AddPipeline("p"); err != nil {
t.Fatal(err)
}
for _, typ := range []string{"tcp", "http", "tcp"} {
if _, err := c.Add(0, "sink", typ); err != nil {
t.Fatal(err)
}
}
tcp2, chosen := Node{Role: "sink", ID: "tcp_2"}, Node{Role: "sink", ID: "web"}
c.Pipelines[0].PluginSinks[1].ID = chosen.ID
for _, err := range []error{c.Set(0, tcp2, "port", "9000"), c.Set(0, tcp2, "keep_alive", "false"), c.Set(0, chosen, "port", "80")} {
if err != nil {
t.Fatal(err)
}
}
got, err := c.Retype(0, tcp2, "http")
if err != nil || got.ID != "http" {
t.Fatalf("%+v %v", got, err)
}
if _, err := c.Retype(0, chosen, "tcp_chain"); err != nil {
t.Fatal(err)
}
var parts []string
for _, s := range c.Pipelines[0].PluginSinks {
parts = append(parts, fmt.Sprintf("%s=%s %v", s.ID, s.Type, s.Config))
}
if want := []string{"tcp=tcp map[]", "web=tcp_chain map[]", "http=http map[port:9000]"}; !slices.Equal(parts, want) {
t.Fatalf("%q, want %q", parts, want)
}
}
// A pipeline takes a name no other has, renamed as when added
func TestPipelineNamesAreUnique(t *testing.T) {
c := &Composition{}
for _, name := range []string{"a", "b"} {
if err := c.AddPipeline(name); err != nil {
t.Fatal(err)
}
}
if c.AddPipeline("a") == nil || c.RenamePipeline(1, "a") == nil || c.RenamePipeline(1, "") == nil {
t.Fatal("took a name another pipeline has")
}
if err := c.RenamePipeline(1, "b"); err != nil || c.RenamePipeline(1, "c") != nil || c.Pipelines[1].Name != "c" {
t.Fatalf("%v, %+v", err, c.Pipelines)
}
}
// A stage an edit turns on does what its spec would: a rate limit drops what
// passes its rate, a heartbeat beats
func TestAStageTurnsOnAsItsSpecDoes(t *testing.T) {
c := &Composition{}
if err := c.AddPipeline("p"); err != nil {
t.Fatal(err)
}
for _, err := range []error{c.Set(0, Node{Role: "rate_limit"}, "entries_per_second", "5"), c.Set(0, Node{Role: "heartbeat"}, "interval_ms", "500")} {
if err != nil {
t.Fatal(err)
}
}
if f := c.Pipelines[0].Flow; f.RateLimit.Policy != "drop" || !f.Heartbeat.Enabled {
t.Fatalf("%+v %+v", f.RateLimit, f.Heartbeat)
}
}
internal/config/config.go
+9 -9
View File
@@ -103,8 +103,8 @@ const (
// RateLimitConfig defines the configuration for pipeline-level rate limiting
type RateLimitConfig struct {
Rate float64 `toml:"rate" lw:"min=0,zero=off" help:"entries a second"`
Burst float64 `toml:"burst" lw:"min=0,zero=the rate" help:"entries at once"`
EntriesPerSecond float64 `toml:"entries_per_second" lw:"min=0,zero=off" help:"entries a second the limit lets through"`
BurstEntries float64 `toml:"burst_entries" lw:"min=0,zero=one second's worth" help:"entries let through at once before the rate applies"`
Policy string `toml:"policy" default:"pass" lw:"enum=pass|drop" help:"drop entries over the rate, or pass them"`
MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes" lw:"min=0,zero=unlimited" help:"drop larger entries"`
}
@@ -262,8 +262,8 @@ type HTTPSinkOptions struct {
WriteTimeoutMS int64 `toml:"write_timeout_ms" lw:"min=0,zero=none" help:"milliseconds an event write may take"`
MaxConnections int64 `toml:"max_connections" lw:"min=0,zero=unlimited" help:"clients at once"`
ReplayLines int64 `toml:"replay_lines" lw:"min=0,zero=none" help:"recent entries a new stream gets first"`
LoginPage bool `toml:"login_page" help:"serve the login page; scram behind auth.trusted_proxies"`
ViewerPage bool `toml:"viewer_page" help:"serve the viewer to users logged in; needs login_page"`
LoginPage bool `toml:"login_page" help:"scram behind auth.trusted_proxies only: a login page at /auth/login"`
ViewerPage bool `toml:"viewer_page" help:"scram behind auth.trusted_proxies only: the viewer after a login; without scram it is always at /"`
Auth *AuthOptions `toml:"auth"`
ACL *ACLOptions `toml:"acl"`
}
@@ -307,9 +307,9 @@ type HTTPChainSinkOptions struct {
// AuthOptions selects how a network plugin authenticates its peer. It sits
// beside `tls`: TLS answers "is this channel private", auth answers "may this
// peer do this". Listeners verify (mtls: certificate identity; scram: password
// via credentials_file); dialers prove themselves (scram) or pin the server
// (mtls). AuthOptions.Check holds the rules of each side.
// peer do this". Listeners verify (mtls: certificate identity; scram: the
// credentials_file's users, or one username and password_file); dialers prove
// themselves (scram) or pin the server (mtls). Check holds each side's rules.
type AuthOptions struct {
Type string `toml:"type" default:"none" lw:"enum=none|mtls|scram" help:"how peers authenticate"`
Identity string `toml:"identity" lw:"enum=cn|san_dns|san_uri|san_email,zero=cn under mtls; no binding under scram" help:"certificate field naming the peer; under scram it must equal the user"`
@@ -321,8 +321,8 @@ type AuthOptions struct {
// leaves room for renewal ahead of expiry
TokenLifetimeMS int64 `toml:"token_lifetime_ms" lw:"listener,min=10000,max=86400000,zero=15 minutes" help:"scram on HTTP: bearer token lifetime"`
TrustedProxies []string `toml:"trusted_proxies" lw:"listener,hint=cidr" help:"scram on the http sink: proxies ending the browsers' TLS; logins are then unbound"`
Username string `toml:"username" lw:"dialer" help:"scram: the user this dialer logs in as"`
PasswordFile string `toml:"password_file" lw:"dialer,hint=file" help:"scram: file holding the user's password"`
Username string `toml:"username" help:"scram: the user a dialer logs in as, or a listener's one user"`
PasswordFile string `toml:"password_file" lw:"hint=file" help:"scram: file holding the password; /dev/fd/N or a pipe is read once, /dev/tty asks"`
}
// ACLOptions is a listener's address rules, entries being addresses or CIDRs:
internal/config/loader_test.go
+1 -1
View File
@@ -340,7 +340,7 @@ func TestFileStringIsOneListEntry(t *testing.T) {
// its own; other sinks do not take the key.
func TestTopLevelColorIsEachConsoleSinksDefault(t *testing.T) {
isolateConfig(t)
m, err := Load(Args{Specs: specs("sink", "console", "sink", "console,color=never", "sink", "null"), Overrides: []string{"--color=always"}})
m, err := Load(Args{Specs: specs("sink", "console", "sink", "console:color=never", "sink", "null"), Overrides: []string{"--color=always"}})
if err != nil {
t.Fatal(err)
}
internal/config/preset.go
+48 -21
View File
@@ -13,7 +13,7 @@ import (
"strings"
)
// Preset is a pipeline built from a few keys: --preset NAME,key=value starts
// Preset is a pipeline built from a few keys: --preset NAME:key=value starts
// a pipeline with it. Later flags add sources, sinks and filters, or replace
// its format, rate limit and heartbeat.
type Preset struct {
@@ -64,6 +64,10 @@ var (
{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"},
}
transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
userParams = []PresetParam{
{Name: "user", Help: "SCRAM user: an edge's login, or a listener's only user in place of users"},
{Name: "password_file", Help: "file holding user's password; /dev/fd/N or a pipe is read once; default: /dev/tty, which asks"},
}
)
var presets = []Preset{
@@ -88,8 +92,9 @@ var presets = []Preset{
{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
}, userParams, []PresetParam{
{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
{Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"},
{Name: "viewer", Default: "false", Help: "true: the login page and viewer, behind proxy; without user or users the viewer is always on"},
}, aclParams, tlsParams),
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir)
@@ -101,14 +106,18 @@ var presets = []Preset{
if err != nil {
return err
}
if v["users"] != "" {
sink["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
auth, err := scramAuth(v)
switch {
case err != nil:
return err
case auth != nil && v["tls"] == "off" && v["proxy"] == "":
return fmt.Errorf("user and users need tls (self, issuer or files), or proxy")
case auth != nil:
sink["auth"] = auth
case v["proxy"] != "":
return fmt.Errorf("proxy needs user or users: browsers log in with SCRAM")
}
if v["proxy"] != "" {
auth, ok := sink["auth"].(map[string]any)
if !ok {
return fmt.Errorf("proxy needs users: browsers log in with SCRAM")
}
auth["trusted_proxies"] = list(v["proxy"])
}
switch v["viewer"] {
@@ -127,18 +136,17 @@ var presets = []Preset{
return nil
}},
{"edge", "Forward files or standard input to an aggregator over TLS, authenticated",
[]PresetParam{pathParam, fromParam,
slices.Concat([]PresetParam{pathParam, fromParam,
{Name: "to", Required: true, Help: "the aggregator's HOST:PORT; [IPV6]:PORT"},
transportParam,
{Name: "ca", Help: "CA file that verifies the aggregator; default: system roots"},
{Name: "pin", Help: "sha256//BASE64 of the aggregator's key, which it logs with tls=self"},
{Name: "server_name", Help: "name the aggregator's certificate carries; default: to's host"},
{Name: "user", Help: "SCRAM user (lw auth add-user on the aggregator)"},
{Name: "password_file", Help: "file holding the SCRAM password"},
}, userParams, []PresetParam{
{Name: "cert", Help: "client certificate file, for an aggregator with client_ca"},
{Name: "key", Help: "client key file"},
{Name: "node", Help: "origin label; default: this host's name"},
},
}),
func(p *PipelineConfig, v map[string]string, isDir IsDir) error {
sources, err := pathOrStdin(v, isDir)
if err != nil {
@@ -159,16 +167,15 @@ var presets = []Preset{
tls[option] = v[key]
}
}
var auth map[string]any
auth, err := scramAuth(v)
switch {
case v["user"] != "" && v["password_file"] != "":
auth = map[string]any{"type": "scram", "username": v["user"], "password_file": v["password_file"]}
case v["user"] != "" || v["password_file"] != "":
return fmt.Errorf("user and password_file go together")
case err != nil:
return err
case auth != nil:
case v["cert"] != "" && v["key"] != "":
auth = map[string]any{"type": "mtls"}
default:
return fmt.Errorf("edge needs user and password_file, or cert and key: it never sends unauthenticated")
return fmt.Errorf("edge needs user, or cert and key: it never sends unauthenticated")
}
sink := map[string]any{"host": host, "port": port, "tls": tls, "auth": auth}
if v["node"] != "" {
@@ -183,6 +190,7 @@ var presets = []Preset{
transportParam,
{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
{Name: "users", Help: "credentials file (lw auth add-user): edges log in with SCRAM"},
}, userParams, []PresetParam{
{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
{Name: "out", Help: "directory for the received entries; default: standard output"},
formatParam("json"),
@@ -203,13 +211,16 @@ var presets = []Preset{
tls := src["tls"].(map[string]any)
tls["client_auth"], tls["client_ca_file"] = true, v["client_ca"]
}
auth, err := scramAuth(v)
switch {
case v["users"] != "":
src["auth"] = map[string]any{"type": "scram", "credentials_file": v["users"]}
case err != nil:
return err
case auth != nil:
src["auth"] = auth
case v["client_ca"] != "":
src["auth"] = map[string]any{"type": "mtls"}
default:
return fmt.Errorf("aggregator needs users or client_ca: it never receives unauthenticated")
return fmt.Errorf("aggregator needs user, users or client_ca: it never receives unauthenticated")
}
p.PluginSources = append(p.PluginSources, PluginSourceConfig{ID: "edges", Type: typ, Config: src})
p.PluginSinks = pipeDefault("").PluginSinks
@@ -356,6 +367,22 @@ func listener(v map[string]string, def string) (map[string]any, error) {
return opts, nil
}
// scramAuth is the scram table of user, whose password the terminal asks for
// unless password_file names it, or of a listener's users file; nil without
func scramAuth(v map[string]string) (map[string]any, error) {
switch {
case v["user"] != "" && v["users"] != "":
return nil, fmt.Errorf("user or users, not both")
case v["user"] != "":
return map[string]any{"type": "scram", "username": v["user"], "password_file": cmp.Or(v["password_file"], "/dev/tty")}, nil
case v["password_file"] != "":
return nil, fmt.Errorf("password_file needs user")
case v["users"] != "":
return map[string]any{"type": "scram", "credentials_file": v["users"]}, nil
}
return nil, nil
}
func splitAddr(key, addr string) (string, int64, error) {
host, p, err := net.SplitHostPort(addr)
port, perr := strconv.ParseUint(p, 10, 16)
internal/config/preset_test.go
+28 -14
View File
@@ -11,7 +11,7 @@ import (
// sinks add to its own, a later format replaces its own.
func TestPresetStartsAndNamesItsPipeline(t *testing.T) {
isolateConfig(t)
got := loadPipelines(t, "preset", "tail,path=/var/log/app.log", "sink", "null", "format", "json",
got := loadPipelines(t, "preset", "tail:path=/var/log/app.log", "sink", "null", "format", "json",
"pipeline", "mine", "preset", "pipe")
want := []PipelineConfig{{
Name: "tail",
@@ -41,20 +41,23 @@ func TestPresetKeysAreChecked(t *testing.T) {
isolateConfig(t)
for spec, want := range map[string]string{
"nope": `no preset "nope" (valid: pipe, tail, serve, edge, aggregator)`,
"tail,path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`,
"tail:path=x,colour=1": `preset tail has no key "colour" (valid: path, from, format)`,
"tail": "preset tail needs path",
"tail,path=x,format.type=json": "preset keys do not nest",
"tail,path=x,path=y": `key "path" takes one value`,
"serve,listen=8080": `listen "8080": want HOST:PORT`,
"serve,tls=self,cert=c": "cert does not apply to tls=self",
"serve,tls=files,cert=c": "tls=files needs key",
"serve,viewer=yes": `viewer "yes"`,
"serve,proxy=127.0.0.1": "proxy needs users",
"serve,viewer=true": "viewer=true needs proxy",
"edge,to=agg:9000": "it never sends unauthenticated",
"edge,to=agg:9000,user=u": "user and password_file go together",
"edge,to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
"aggregator,users=u,tls=off": "the aggregator always serves TLS",
"tail:path=x,format.type=json": "preset keys do not nest",
"tail:path=x,path=y": `key "path" takes one value`,
"serve:listen=8080": `listen "8080": want HOST:PORT`,
"serve:tls=self,cert=c": "cert does not apply to tls=self",
"serve:tls=files,cert=c": "tls=files needs key",
"serve:viewer=yes": `viewer "yes"`,
"serve:proxy=127.0.0.1": "proxy needs user or users",
"serve:viewer=true": "viewer=true needs proxy",
"serve:user=u": "user and users need tls (self, issuer or files), or proxy",
"serve:tls=self,user=u,users=f": "user or users, not both",
"edge:to=agg:9000": "it never sends unauthenticated",
"edge:to=agg:9000,password_file=p": "password_file needs user",
"edge:to=agg:9000,user=u,password_file=p,transport=udp": `transport "udp"`,
"aggregator:users=u,tls=off": "the aggregator always serves TLS",
"aggregator:user=u,users=f": "user or users, not both",
"aggregator": "it never receives unauthenticated",
} {
_, err := Load(Args{Specs: specs("preset", spec)})
@@ -64,6 +67,17 @@ func TestPresetKeysAreChecked(t *testing.T) {
}
}
// A preset user without password_file is asked for on the terminal
func TestPresetUserWithoutPasswordFileAsks(t *testing.T) {
p, err := ExpandPreset("edge", map[string]string{"to": "agg:9000", "user": "u"}, nil)
if err != nil {
t.Fatal(err)
}
if auth := p.PluginSinks[0].Config["auth"].(map[string]any); auth["password_file"] != "/dev/tty" {
t.Fatalf("auth = %v, want password_file /dev/tty", auth)
}
}
// A preset path is what isDir answers, a directory named with brackets too;
// without isDir, off the target host, it is a directory when it ends in '/',
// a pattern when it holds a glob, and otherwise ErrPathKind asks
internal/config/schema_test.go
+2 -2
View File
@@ -117,8 +117,8 @@ func TestValidateConfigNamesThePath(t *testing.T) {
c.Pipelines[0].PluginSinks[0] = PluginSinkConfig{ID: "up", Type: "tcp_chain", Config: map[string]any{"host": "h", "port": int64(1),
"tls": map[string]any{"enabled": true, "insecure_skip_verify": true}, "auth": map[string]any{"type": "mtls"}}}
},
"pipelines[0].flow.rate_limit.rate: must be a finite number": func(c *Config) {
c.Pipelines[0].Flow.RateLimit = &RateLimitConfig{Rate: math.Inf(1)}
"pipelines[0].flow.rate_limit.entries_per_second: must be a finite number": func(c *Config) {
c.Pipelines[0].Flow.RateLimit = &RateLimitConfig{EntriesPerSecond: math.Inf(1)}
},
"color: must be one of auto, always, never, got \"\"": func(c *Config) {
c.Color = ""
internal/config/spec.go
+33 -17
View File
@@ -93,10 +93,9 @@ var specKinds = []specKind{
Fill(&format)
return []string{typedSpec(&format)}
}},
// Naming a stage turns it on: the file defaults (pass, disabled) would not.
{"rate-limit", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
p.Flow.RateLimit = &RateLimitConfig{Policy: "drop"}
return Scan(opts, p.Flow.RateLimit)
p.Flow.RateLimit = &RateLimitConfig{}
return Scan(StageOn("rate_limit", opts), p.Flow.RateLimit)
}, func(p *PipelineConfig) []string {
if p.Flow.RateLimit == nil {
return nil
@@ -106,8 +105,8 @@ var specKinds = []specKind{
return []string{formatSpec("", Values(&rate))}
}},
{"heartbeat", false, false, false, func(p *PipelineConfig, _ string, opts map[string]any, _ IsDir) error {
p.Flow.Heartbeat = &HeartbeatConfig{Enabled: true}
return Scan(opts, p.Flow.Heartbeat)
p.Flow.Heartbeat = &HeartbeatConfig{}
return Scan(StageOn("heartbeat", opts), p.Flow.Heartbeat)
}, func(p *PipelineConfig) []string {
if p.Flow.Heartbeat == nil {
return nil
@@ -127,6 +126,18 @@ type pipelineSpec struct {
value string
}
// stageOn are the values naming a flow stage sets, in a spec or an engine
// edit: the file's defaults (pass, disabled) would leave it doing nothing
var stageOn = map[string]map[string]any{"rate_limit": {"policy": "drop"}, "heartbeat": {"enabled": true}}
// StageOn returns opts over the values that turn a flow stage on
func StageOn(stage string, opts map[string]any) map[string]any {
on := map[string]any{}
maps.Copy(on, stageOn[stage])
maps.Copy(on, opts)
return on
}
// Spec is one pipeline flag: its name without dashes, and its value.
type Spec struct{ Flag, Value string }
@@ -282,13 +293,10 @@ func typedSpec(stage any) string {
return formatSpec(typ, m)
}
// formatSpec writes [TYPE,]key=value,...: tables as dotted keys, lists as a
// repeated key
// formatSpec writes TYPE[:key=value,...] or key=value,...: tables as dotted
// keys, lists as a repeated key
func formatSpec(typ string, tables ...map[string]any) string {
var parts []string
if typ != "" {
parts = append(parts, escape(typ))
}
var add func(prefix string, m map[string]any)
add = func(prefix string, m map[string]any) {
for _, k := range slices.Sorted(maps.Keys(m)) {
@@ -307,7 +315,11 @@ func formatSpec(typ string, tables ...map[string]any) string {
for _, m := range tables {
add("", m)
}
return strings.Join(parts, ",")
opts := strings.Join(parts, ",")
if typ == "" || opts == "" {
return typ + opts
}
return typ + ":" + opts
}
func text(v any) string {
@@ -365,16 +377,20 @@ func useStdio(p *PipelineConfig) {
}
}
// parseSpec reads [TYPE,]key=value,...: dotted keys nest, a repeated key makes
// a list, and a backslash escapes ',', '=' and '\'. Errors name the part.
// parseSpec reads TYPE[:key=value,...] or key=value,...: TYPE ends at the
// first ':', dotted keys nest, a repeated key makes a list, and a backslash
// escapes ',', '=' and '\'. Errors name the part.
func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err error) {
rest, more := spec, true
if typed {
typ, rest, more = cutUnescaped(spec, ',')
if _, _, eq := cutUnescaped(typ, '='); typ == "" || eq {
typ, rest, _ = strings.Cut(spec, ":")
more = rest != ""
if head, _, comma := cutUnescaped(typ, ','); comma && !strings.Contains(head, "=") {
return "", nil, fmt.Errorf("%s: TYPE ends at ':', as in %s:%s", spec, head, spec[len(head)+1:])
}
if typ == "" || strings.ContainsAny(typ, "=,\\") {
return "", nil, fmt.Errorf("%s: missing TYPE", spec)
}
typ = unescape(typ)
}
opts = map[string]any{}
for more {
@@ -390,7 +406,7 @@ func parseSpec(spec string, typed bool) (typ string, opts map[string]any, err er
err = SetOption(opts, key, unescape(value))
}
if err != nil {
return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+","+part, ","), err)
return "", nil, fmt.Errorf("%s: %w", strings.TrimPrefix(typ+":"+part, ":"), err)
}
}
return typ, opts, nil
internal/config/spec_test.go
+21 -20
View File
@@ -36,13 +36,13 @@ func loadPipelines(t *testing.T, pairs ...string) []PipelineConfig {
func TestPipelineSpecGrammar(t *testing.T) {
isolateConfig(t)
got := loadPipelines(t,
"source", `file,directory=/var/log/a\,b,pattern=*.log`,
"source", `file,id=app,directory=x\=y`,
"source", `file,directory=c:\\logs\\`,
"sink", "http,port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.enabled=true,tls.cert_file=c,tls.key_file=k",
"filter", "include,patterns=ERROR,patterns=WARN",
"filter", `exclude,patterns=\d{1\,3}`,
"format", `txt,timestamp_format=Jan 2\, 2006`,
"source", `file:directory=/var/log/a\,b,pattern=*.log`,
"source", `file:id=app,directory=x\=y`,
"source", `file:directory=c:\\logs\\`,
"sink", "http:port=8080,auth.type=scram,auth.credentials_file=users.toml,tls.enabled=true,tls.cert_file=c,tls.key_file=k",
"filter", "include:patterns=ERROR,patterns=WARN",
"filter", `exclude:patterns=\d{1\,3}`,
"format", `txt:timestamp_format=Jan 2\, 2006`,
"pipeline", "b", "source", "null", "sink", "null", "sink", "null", "heartbeat", "interval_ms=1000")
want := []PipelineConfig{{
Name: "cli",
@@ -76,13 +76,14 @@ func TestPipelineSpecGrammar(t *testing.T) {
t.Fatalf("pipelines:\n got %+v\nwant %+v", got, want)
}
for want, pairs := range map[string][]string{
`--sink http,port: missing "="`: {"sink", "http,port"},
`--sink http:port: missing "="`: {"sink", "http:port"},
`--source file,directory=/x: TYPE ends at ':', as in file:directory=/x`: {"source", "file,directory=/x"},
`--source directory=/x: missing TYPE`: {"source", "directory=/x"},
`--source file,type=x: TYPE already sets "type"`: {"source", "file,type=x"},
`--sink http,tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http,tls=1,tls.cert_file=c"},
`--source file:type=x: TYPE already sets "type"`: {"source", "file:type=x"},
`--sink http:tls.cert_file=c: "tls" is both a value and a table`: {"sink", "http:tls=1,tls.cert_file=c"},
`--sink requires a value`: {"sink", ""},
`--format txt: pipeline "cli" already has one`: {"format", "json", "format", "txt"},
`--rate-limit rate=1,polcy=drop: unknown key "polcy"`: {"rate-limit", "rate=1,polcy=drop"},
`--rate-limit entries_per_second=1,polcy=drop: unknown key "polcy"`: {"rate-limit", "entries_per_second=1,polcy=drop"},
} {
if _, err := Load(Args{Specs: specs(pairs...)}); err == nil || err.Error() != want {
t.Errorf("%q: err = %v, want %s", pairs, err, want)
@@ -95,7 +96,7 @@ func TestPipelineSpecGrammar(t *testing.T) {
func TestSpecValueIsOneListEntry(t *testing.T) {
isolateConfig(t)
sink := loadPipelines(t, "source", "null", "sink",
`tcp,port=9000,tls.enabled=true,tls.self_signed=true,tls.client_auth=true,tls.client_ca_file=ca,`+
`tcp:port=9000,tls.enabled=true,tls.self_signed=true,tls.client_auth=true,tls.client_ca_file=ca,`+
`auth.type=mtls,auth.allow_patterns=^edge-\d{1\,3}$,auth.allow=CN=a\,O=b,auth.allow=CN=c`)[0].PluginSinks[0]
var opts TCPSinkOptions
if err := Scan(sink.Config, &opts); err != nil {
@@ -114,12 +115,12 @@ func TestEnvironmentPipeline(t *testing.T) {
for name, value := range map[string]string{
"LOGWISP_PIPELINE": "edge",
"LOGWISP_SOURCE": "random",
"LOGWISP_SOURCE_10": "null,id=ten",
"LOGWISP_SOURCE_2": "null,id=two",
"LOGWISP_SOURCE_10": "null:id=ten",
"LOGWISP_SOURCE_2": "null:id=two",
"LOGWISP_SINK_1": "console",
"LOGWISP_FILTER": "",
"LOGWISP_FORMAT_1": "raw",
"LOGWISP_RATE_LIMIT": "rate=10",
"LOGWISP_RATE_LIMIT": "entries_per_second=10",
"LOGWISP_LOGGING_LEVEL": "debug",
} {
t.Setenv(name, value)
@@ -136,7 +137,7 @@ func TestEnvironmentPipeline(t *testing.T) {
p := cfg.Pipelines
if len(p) != 1 || p[0].Name != "edge" || len(p[0].PluginSources) != 3 || p[0].PluginSources[1].ID != "two" ||
p[0].PluginSources[2].ID != "ten" || len(p[0].PluginSinks) != 1 || p[0].Flow.Filters != nil || p[0].Flow.Format != nil ||
*p[0].Flow.RateLimit != (RateLimitConfig{Rate: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" {
*p[0].Flow.RateLimit != (RateLimitConfig{EntriesPerSecond: 10, Policy: "drop"}) || cfg.Logging.Level != "debug" {
t.Fatalf("environment pipeline: %+v %+v", p, cfg.Logging)
}
}
@@ -174,7 +175,7 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) {
// stdout, as a filter does; two pipelines cannot both read stdin.
func TestSpecPipelinesDefaultToStdio(t *testing.T) {
isolateConfig(t)
got := loadPipelines(t, "filter", "include,patterns=ERROR")
got := loadPipelines(t, "filter", "include:patterns=ERROR")
if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" ||
len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" {
t.Fatalf("pipeline: %+v", p)
@@ -190,7 +191,7 @@ func TestSpecPipelinesDefaultToStdio(t *testing.T) {
// removal fails the reload instead of keeping the removed file's values.
func TestReloadKeepsSpecPipelines(t *testing.T) {
isolateConfig(t)
m, err := Load(Args{Specs: specs("source", "random,special=true", "sink", "null")})
m, err := Load(Args{Specs: specs("source", "random:special=true", "sink", "null")})
if err != nil {
t.Fatal(err)
}
@@ -217,11 +218,11 @@ func TestReloadKeepsSpecPipelines(t *testing.T) {
// Written specs build the pipelines a file would: each flow stage with its
// defaults, so the rate limit keeps its file policy and the heartbeat its off
func TestWrittenSpecsBuildWhatAFileWould(t *testing.T) {
p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{Rate: 5}, Heartbeat: &HeartbeatConfig{}},
p := PipelineConfig{Name: "p", Flow: &FlowConfig{Format: &FormatConfig{}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5}, Heartbeat: &HeartbeatConfig{}},
PluginSources: []PluginSourceConfig{{ID: "in", Type: "null", Config: map[string]any{}}},
PluginSinks: []PluginSinkConfig{{ID: "out", Type: "null", Config: map[string]any{}}}}
got, err := SpecPipelines(PipelineSpecs([]PipelineConfig{p}), nil)
p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{Rate: 5, Policy: "pass"},
p.Flow = &FlowConfig{Format: &FormatConfig{Type: "raw"}, RateLimit: &RateLimitConfig{EntriesPerSecond: 5, Policy: "pass"},
Heartbeat: &HeartbeatConfig{IntervalMS: 1000, Format: "txt"}}
if err != nil || !reflect.DeepEqual(got, []PipelineConfig{p}) {
t.Fatalf("got %+v %v\nwant %+v", got, err, p)
internal/config/validate.go
+16 -8
View File
@@ -296,7 +296,7 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
}
return nil
case len(o.Allow) > 0 || len(o.AllowPatterns) > 0:
return fmt.Errorf("auth: allow and allow_patterns apply only to type %q; the credentials file is the allow list", "mtls")
return fmt.Errorf("auth: allow and allow_patterns apply only to type %q; the users are the allow list", "mtls")
case side == Dialer && (o.CredentialsFile != "" || o.TokenLifetimeMS != 0 || len(o.TrustedProxies) > 0):
return errors.New("auth: credentials_file, token_lifetime_ms and trusted_proxies apply only to listeners")
case side == Dialer && o.Identity != "":
@@ -305,10 +305,12 @@ func (o *AuthOptions) Check(side Side, http bool, t TLSState) error {
return fmt.Errorf("auth: type %q on a dialer requires username and password_file", "scram")
case side == Dialer:
return nil
case o.Username != "" || o.PasswordFile != "":
return errors.New("auth: username and password_file apply only to dialers")
case o.CredentialsFile == "":
return fmt.Errorf("auth: type %q requires credentials_file", "scram")
case o.CredentialsFile != "" && (o.Username != "" || o.PasswordFile != ""):
return errors.New("auth: credentials_file, or username and password_file: not both")
case (o.Username == "") != (o.PasswordFile == ""):
return errors.New("auth: username and password_file go together")
case o.CredentialsFile == "" && o.Username == "":
return fmt.Errorf("auth: type %q requires credentials_file, or username and password_file", "scram")
case o.TokenLifetimeMS > 0 && !http:
return errors.New("auth: token_lifetime_ms applies only to HTTP listeners")
case len(o.TrustedProxies) > 0 && (side != Listener || !http):
@@ -425,6 +427,10 @@ func and(keys []string) string {
return strings.Join(keys[:len(keys)-1], ", ") + " and " + keys[len(keys)-1]
}
// pagesBehind is why a login or viewer page needs a proxy: only there do
// browsers log in, and elsewhere the viewer needs no page
const pagesBehind = "only for scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls, the viewer is always at /"
// Check keeps a stream and a status path ServeMux and a URL take literally,
// apart, and off the authentication paths; a login page needs browsers that
// log in, behind a proxy
@@ -442,10 +448,12 @@ func (o *HTTPSinkOptions) Check() error {
switch {
case o.StreamPath == o.StatusPath:
return fmt.Errorf("stream_path and status_path must differ")
case (o.LoginPage || o.ViewerPage) && !behind:
return errors.New("login_page and viewer_page apply to scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls the viewer is always served")
case o.LoginPage && !behind:
return &KeyError{"login_page", errors.New(pagesBehind)}
case o.ViewerPage && !behind:
return &KeyError{"viewer_page", errors.New(pagesBehind)}
case o.ViewerPage && !o.LoginPage:
return errors.New("viewer_page needs login_page, where it sends a signed-out viewer")
return &KeyError{"viewer_page", errors.New("needs login_page, where it sends a signed-out viewer")}
}
return nil
}
internal/flow/ratelimiter.go
+6 -6
View File
@@ -27,12 +27,12 @@ func NewRateLimiter(cfg config.RateLimitConfig, logger *log.Logger) (*RateLimite
if err := config.Settle(&cfg); err != nil {
return nil, err
}
if cfg.Rate == 0 {
if cfg.EntriesPerSecond == 0 {
return nil, nil // off
}
burst := cfg.Burst
burst := cfg.BurstEntries
if burst == 0 {
burst = cfg.Rate
burst = cfg.EntriesPerSecond
}
policy := config.PolicyPass
if cfg.Policy == "drop" {
@@ -40,7 +40,7 @@ func NewRateLimiter(cfg config.RateLimitConfig, logger *log.Logger) (*RateLimite
}
l := &RateLimiter{
bucket: tokenbucket.New(burst, cfg.Rate),
bucket: tokenbucket.New(burst, cfg.EntriesPerSecond),
policy: policy,
logger: logger,
maxEntrySizeBytes: cfg.MaxEntrySizeBytes,
@@ -85,8 +85,8 @@ func (l *RateLimiter) GetStats() map[string]any {
stats := map[string]any{
"enabled": true,
"rate": l.bucket.Rate(),
"burst": l.bucket.Capacity(),
"entries_per_second": l.bucket.Rate(),
"burst_entries": l.bucket.Capacity(),
"dropped_total": l.droppedCount.Load(),
"dropped_by_size_total": l.droppedBySizeCount.Load(),
"policy": policyString(l.policy),
internal/tlsx/tlsx.go
+4 -3
View File
@@ -240,13 +240,14 @@ var warnedSecrets sync.Map
// SecretFileWarning describes a secret file every local user can read, once
// per path per process. Group access is left alone: container secret mounts
// commonly grant it on purpose.
// commonly grant it on purpose. Terminals, pipes and descriptors hold no
// secret at rest; lw closes /dev/fd/N after reading, and N then names another.
func SecretFileWarning(key, path string) string {
if path == "" {
if path == "" || strings.HasPrefix(path, "/dev/fd/") {
return ""
}
fi, err := os.Stat(path)
if err != nil || fi.Mode().Perm()&0o004 == 0 {
if err != nil || !fi.Mode().IsRegular() || fi.Mode().Perm()&0o004 == 0 {
return ""
}
if _, seen := warnedSecrets.LoadOrStore(path, struct{}{}); seen {
internal/tlsx/tlsx_test.go
+33
View File
@@ -4,10 +4,13 @@ import (
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"fmt"
"net"
"os"
"path/filepath"
"slices"
"strings"
"syscall"
"testing"
"time"
@@ -200,3 +203,33 @@ func TestTLSOptionConflicts(t *testing.T) {
}
}
}
// Terminals, pipes and descriptors hold no secret at rest: /dev/tty, mode
// 0666 like /dev/null, is not warned about, nor /dev/fd/N whatever it names,
// while a world-readable file is.
func TestSecretFileWarningSkipsTerminalsAndPipes(t *testing.T) {
dir := t.TempDir()
fifo, file := filepath.Join(dir, "fifo"), filepath.Join(dir, "pw")
if err := syscall.Mkfifo(fifo, 0o600); err != nil {
t.Fatal(err)
}
testutil.WriteFile(t, file, "secret\n")
for _, path := range []string{fifo, file} {
if err := os.Chmod(path, 0o644); err != nil { // past the umask
t.Fatal(err)
}
}
held, err := os.Open(file)
if err != nil {
t.Fatal(err)
}
defer held.Close()
for _, path := range []string{os.DevNull, fifo, fmt.Sprintf("/dev/fd/%d", held.Fd())} {
if w := SecretFileWarning("auth.password_file", path); w != "" {
t.Errorf("%s: %s", path, w)
}
}
if SecretFileWarning("auth.password_file", file) == "" {
t.Error("a world-readable file was not warned about")
}
}
internal/tui/canvas.go
+45 -66
View File
@@ -10,7 +10,6 @@ import (
"github.com/lixenwraith/logwisp/internal/compose"
"github.com/lixenwraith/logwisp/internal/config"
"github.com/lixenwraith/terminal"
ui "github.com/lixenwraith/terminal/tui"
)
@@ -92,22 +91,9 @@ func (a *app) layout(w int) layout {
func (a *app) drawCanvas(r ui.Region) {
l := a.layout(r.W)
chosen := l.nodes[a.col][min(a.row[a.col], len(l.nodes[a.col])-1)]
if chosen.y < a.scroll {
a.scroll = chosen.y
}
if tall := 2 - a.col%2; chosen.y+tall > a.scroll+r.H {
a.scroll = chosen.y + tall - r.H
}
a.scroll = max(0, min(a.scroll, l.h-r.H))
// Drawn whole off screen, then the rows in view copied
cells := make([]terminal.Cell, r.W*l.h)
full := ui.NewRegion(cells, r.W, 0, 0, r.W, l.h)
full.FillStyle(a.th.Text)
a.drawParts(full, l)
for y := range min(r.H, l.h-a.scroll) {
copy(r.Cells[(r.Y+y)*r.TotalW+r.X:][:r.W], cells[(a.scroll+y)*r.W:][:r.W])
}
a.scroll.SetDimensions(l.h, r.H)
a.scroll.EnsureRange(chosen.y, 2-a.col%2)
r.Window(l.h, &a.scroll, a.th.Text, func(full ui.Region) { a.drawParts(full, l) })
}
func (a *app) drawParts(r ui.Region, l layout) {
@@ -118,7 +104,7 @@ func (a *app) drawParts(r ui.Region, l layout) {
r.TextStyled(t.x, t.y, string(f.dot)+" "+c.title, tints[col].On(th.Text))
}
a.drawBox(r, l)
g := wires{w: r.W, arms: make([]uint8, r.W*l.h)}
g := ui.NewWires(r.W, l.h)
for _, col := range []int{sourcesCol, sinksCol} {
nodes := a.nodes(col)
for i, s := range l.nodes[col] {
@@ -130,7 +116,7 @@ func (a *app) drawParts(r ui.Region, l layout) {
a.drawNode(r, s, nodes[i], tints[col], chosen)
}
if l.wide && len(nodes) > 0 {
g.join(l, col)
join(g, l, col)
}
}
if !l.wide {
@@ -139,11 +125,7 @@ func (a *app) drawParts(r ui.Region, l layout) {
}
return
}
for i, arms := range g.arms {
if arms != 0 {
r.TextStyled(i%r.W, i/r.W, string(f.wire[arms]), th.Border)
}
}
r.DrawWires(g, f.Line, th.Border)
if len(a.nodes(sourcesCol)) > 0 {
r.TextStyled(l.box.x-1, l.entry, string(f.arrow), th.Border)
}
@@ -152,30 +134,9 @@ func (a *app) drawParts(r ui.Region, l layout) {
}
}
// wires are the canvas's cells by the arms they have: up 1, right 2, down 4,
// left 8, so every junction draws from what meets there
type wires struct {
w int
arms []uint8
}
func (g wires) h(y, x0, x1 int) {
for x := x0; x < x1; x++ {
g.arms[y*g.w+x] |= 2
g.arms[y*g.w+x+1] |= 8
}
}
func (g wires) v(x, y0, y1 int) {
for y := y0; y < y1; y++ {
g.arms[y*g.w+x] |= 4
g.arms[(y+1)*g.w+x] |= 1
}
}
// join wires a column's nodes to a trunk beside the flow box, and the trunk
// to the box's side
func (g wires) join(l layout, col int) {
func join(g ui.Wires, l layout, col int) {
ys := []int{l.entry}
if col == sinksCol {
ys[0] = l.exit
@@ -186,18 +147,18 @@ func (g wires) join(l layout, col int) {
if col == sourcesCol {
trunk := l.box.x - gutter + 1
for _, s := range l.nodes[col] {
g.h(s.y, trunk-1, trunk)
g.H(s.y, trunk-1, trunk)
}
g.v(trunk, slices.Min(ys), slices.Max(ys))
g.h(l.entry, trunk, l.box.x-1)
g.V(trunk, slices.Min(ys), slices.Max(ys))
g.H(l.entry, trunk, l.box.x-1)
return
}
right := l.box.x + l.box.w
trunk := right + 2
g.h(l.exit, right, trunk)
g.v(trunk, slices.Min(ys), slices.Max(ys))
g.H(l.exit, right, trunk)
g.V(trunk, slices.Min(ys), slices.Max(ys))
for _, s := range l.nodes[col] {
g.h(s.y, trunk, s.x-1)
g.H(s.y, trunk, s.x-1)
}
}
@@ -245,14 +206,7 @@ func (a *app) drawEmpty(r ui.Region, s spot, role string, chosen bool) {
func (a *app) drawBox(r ui.Region, l layout) {
th, f := a.th, a.font
b := l.box
edge := []rune(f.box)
across := strings.Repeat(string(edge[4]), max(0, b.w-2))
r.TextStyled(b.x, b.y, string(edge[0])+across+string(edge[1]), th.Border)
r.TextStyled(b.x, b.y+l.boxH-1, string(edge[2])+across+string(edge[3]), th.Border)
for y := b.y + 1; y < b.y+l.boxH-1; y++ {
r.TextStyled(b.x, y, string(edge[5]), th.Border)
r.TextStyled(b.x+b.w-1, y, string(edge[5]), th.Border)
}
r.Sub(b.x, b.y, b.w, l.boxH).BoxStyle(f.box, th.Border)
for i, n := range a.nodes(flowCol) {
s := l.nodes[flowCol][i]
chosen := a.col == flowCol && a.row[flowCol] == i
@@ -345,11 +299,14 @@ func show(v any) string {
// key, as config's key paths write them
var problemPath = regexp.MustCompile(`^pipelines?\[(\d+)\](?:\.(plugin_sources|plugin_sinks)\[([^\]]*)\](?:\.config\.?([\w.]*))?|\.flow\.(\w+)(?:\[(\d+)\])?\.?([\w.]*))?`)
func locate(err error) (pi int, n compose.Node, key string, ok bool) {
m := problemPath.FindStringSubmatch(err.Error())
if m == nil {
return 0, n, "", false
// locate reads a problem's part, and its text past the path
func locate(err error) (pi int, n compose.Node, key, text string, ok bool) {
at := problemPath.FindStringSubmatchIndex(err.Error())
if at == nil {
return 0, n, "", err.Error(), false
}
m := problemPath.FindStringSubmatch(err.Error())
text = strings.TrimPrefix(err.Error()[at[1]:], ": ")
pi, _ = strconv.Atoi(m[1])
switch {
case m[2] != "":
@@ -362,7 +319,7 @@ func locate(err error) (pi int, n compose.Node, key string, ok bool) {
}
key = m[7]
}
return pi, n, key, true
return pi, n, key, text, true
}
// faulty reports whether the first problem names this node, or a filter
@@ -371,6 +328,28 @@ func (a *app) faulty(n compose.Node) bool {
if a.problem == nil {
return false
}
pi, m, _, ok := locate(a.problem)
pi, m, _, _, ok := locate(a.problem)
return ok && pi == a.pi && m.Role == n.Role && m.ID == n.ID
}
// where names a problem's part as the screen shows it: its pipeline when
// another is shown, the part, and its key
func (a *app) where(pi int, n compose.Node, key string) string {
var out []string
if pi != a.pi && pi < len(a.comp.Pipelines) {
out = append(out, "pipeline "+a.comp.Pipelines[pi].Name)
}
switch n.Role {
case "":
case "source", "sink":
out = append(out, n.Role+" "+n.ID)
case "filters":
out = append(out, fmt.Sprintf("filter %d", n.Index+1))
default:
out = append(out, n.Role)
}
if key != "" {
out = append(out, key)
}
return strings.Join(out, " · ")
}
internal/tui/dialog.go
+245 -118
View File
@@ -1,6 +1,7 @@
package tui
import (
"fmt"
"strings"
"github.com/lixenwraith/logwisp/internal/compose"
@@ -16,23 +17,19 @@ type dialog interface {
handle(a *app, ev terminal.Event)
}
// frame draws a dialog's box, w by h at most, centered in r with its title,
// and returns the inside
// frame draws a dialog's box around an inside w by h, centered in r and
// clipped to it, with its title, and returns the inside; sized to what it
// holds, it ends on its last line
func (a *app) frame(r ui.Region, w, h int, title string) ui.Region {
b := ui.Center(r, min(w, r.W), min(h, r.H))
b.FillStyle(a.th.Text)
wire := a.font.wire
across := strings.Repeat(string(wire[10]), max(0, b.W-2))
b.TextStyled(0, 0, string(wire[6])+across+string(wire[12]), a.th.Border)
b.TextStyled(0, b.H-1, string(wire[3])+across+string(wire[9]), a.th.Border)
for y := 1; y < b.H-1; y++ {
b.TextStyled(0, y, string(wire[5]), a.th.Border)
b.TextStyled(b.W-1, y, string(wire[5]), a.th.Border)
}
b.TextStyled(2, 0, " "+ui.Truncate(title, b.W-6)+" ", a.th.Accent)
return b.Sub(2, 1, b.W-4, b.H-2)
return ui.Center(r, min(w+4, r.W), min(h+2, r.H)).Frame(title, a.th)
}
// inside is a dialog's widest inside on a screen w wide; one narrower than
// stacked puts labels and keys above what they name
func inside(w int) int { return max(1, min(72, w-4)) }
const stacked = 32
// chooser picks one option from a filtered list
type chooser struct {
title string
@@ -43,8 +40,8 @@ type chooser struct {
}
func (c *chooser) draw(a *app, r ui.Region) {
in := a.frame(r, 76, len(c.list.Options)+4, c.title)
in.OptionList(c.list, a.th)
w := inside(r.W)
a.frame(r, w, c.list.Rows(w), c.title).OptionList(c.list, a.th)
}
func (c *chooser) handle(a *app, ev terminal.Event) {
@@ -64,6 +61,11 @@ func (c *chooser) handle(a *app, ev terminal.Event) {
a.dialog = nil
c.pick(a, i)
}
case ev.Key == terminal.KeyRune && c.list.Menu:
if i, ok := c.list.Pick(ev.Rune); ok {
a.dialog = nil
c.pick(a, i)
}
default:
c.list.HandleKey(ev.Key, ev.Rune, ev.Modifiers)
}
@@ -95,56 +97,104 @@ func (a *app) addMenu() dialog {
return
}
a.choose(n)
a.inspect, a.insp.cursor = true, 0
for j, l := range a.lines() {
if l.group != "" && l.node == n {
a.insp.cursor = j
a.inspectFirst(n)
}}
}
// typeMenu changes a source's or sink's type, in its place
func (a *app) typeMenu(n compose.Node) dialog {
p, _ := a.plugin(n)
var options []ui.Option
for _, q := range config.Plugins() {
if q.Role == n.Role && q.Type != p.Type {
options = append(options, ui.Option{Name: q.Type, Hint: q.Summary})
}
}
return &chooser{title: "Change " + p.Type, list: ui.NewOptionListState(options), pick: func(a *app, i int) {
m, err := a.comp.Retype(a.pi, n, options[i].Name)
if a.edit(err) {
a.choose(m)
a.inspectFirst(m)
}
}}
}
// presetMenu starts the pipeline over from a preset, or the pipelines from a
// pasted command line; on a bare start, Esc starts empty
func (a *app) presetMenu() dialog {
presets := config.Presets()
// enumMenu sets an enum key to the option chosen, the cursor on its value
func (a *app) enumMenu(l line, cur any) dialog {
var options []ui.Option
for _, p := range presets {
options = append(options, ui.Option{Name: p.Name, Hint: p.Summary})
for _, e := range l.key.Enum {
hint := ""
if e == l.key.Default {
hint = "the default"
}
options = append(options, ui.Option{Name: "empty", Hint: "no sources or sinks: add your own"})
options = append(options, ui.Option{Name: e, Hint: hint})
}
list := ui.NewOptionListState(options)
list.Cursor = max(0, choice(l.key, cur))
return &chooser{title: label(l), list: list, pick: func(a *app, i int) {
a.edit(a.comp.Set(a.pi, l.node, l.key.Name, options[i].Name))
}}
}
// presetMenu replaces the pipeline shown from a preset, or the pipelines
// from a pasted command line; on a bare start, Esc starts empty
func (a *app) presetMenu() dialog {
title := "Start from a preset, or paste an lw command line"
if len(a.comp.Pipelines) > 0 {
title = "Replace pipeline " + a.pipeline().Name + " with a preset"
}
empty := func(a *app) {
a.use(config.PipelineConfig{Name: "default", Flow: &config.FlowConfig{}})
}
menu := &chooser{title: title, list: ui.NewOptionListState(options)}
menu := a.pipelineMenu(title, (*app).use)
menu.paste = func(a *app, text string) {
a.dialog = nil
if a.paste(text); len(a.comp.Pipelines) == 0 {
a.dialog = menu // refused: the status says why
}
}
menu.pick = func(a *app, i int) {
switch {
case i == len(presets):
empty(a)
case len(presets[i].Params) == 0:
(&presetForm{preset: presets[i]}).apply(a)
default:
a.dialog = newPresetForm(presets[i])
}
}
menu.back = func(a *app) {
if len(a.comp.Pipelines) == 0 {
empty(a)
a.use(empty())
}
}
return menu
}
// pipelineMenu offers the presets and an empty pipeline, which place puts
// on screen
func (a *app) pipelineMenu(title string, place func(a *app, p config.PipelineConfig)) *chooser {
presets := config.Presets()
var options []ui.Option
for _, p := range presets {
options = append(options, ui.Option{Name: p.Name, Hint: p.Summary})
}
options = append(options, ui.Option{Name: "empty", Hint: "no sources or sinks: add your own"})
menu := &chooser{title: title, list: ui.NewOptionListState(options)}
menu.pick = func(a *app, i int) {
if i == len(presets) {
place(a, empty())
return
}
p := presets[i]
f := &form{title: "preset " + p.Name, summary: p.Summary, params: p.Params, back: func(a *app) { a.dialog = menu },
apply: func(a *app, values map[string]string) error {
c, err := compose.FromPreset(p.Name, values, config.HostIsDir)
if err == nil {
place(a, c.Pipelines[0])
}
return err
}}
if f.fill(nil); len(p.Params) == 0 {
f.submit(a)
return
}
a.dialog = f
}
return menu
}
func empty() config.PipelineConfig {
return config.PipelineConfig{Name: "default", Flow: &config.FlowConfig{}}
}
// use puts a pipeline in place of the one shown, under its name, asking
// first when that one has sources or sinks; the first pipeline keeps its own,
// and its parts are work quit asks about
@@ -167,51 +217,102 @@ func (a *app) use(p config.PipelineConfig) {
a.dialog = &confirm{question: "Replace pipeline " + old.Name + "?", yes: replace}
}
// presetForm takes a preset's keys, its defaults as placeholders
type presetForm struct {
preset config.Preset
// append adds a pipeline after the others, under its name or the first free
// one after it, and shows it
func (a *app) append(p config.PipelineConfig) {
var names []string
for _, q := range a.comp.Pipelines {
names = append(names, q.Name)
}
p.Name = config.FreeID(p.Name, names)
a.comp.Pipelines = append(a.comp.Pipelines, p)
a.pi, a.row, a.bar, a.changed = len(a.comp.Pipelines)-1, [3]int{}, false, true
a.check()
}
// form takes a few keys, their defaults as placeholders, then applies them;
// a refusal stays in the form
type form struct {
title, summary string
params []config.PresetParam
fields []*ui.TextFieldState
focus ui.Focus
scroll int // the first field shown
scroll ui.ViewportScroll // the fields' rows
err string
apply func(a *app, values map[string]string) error
back func(a *app) // on Esc, after the form closes
}
func newPresetForm(p config.Preset) *presetForm {
f := &presetForm{preset: p, focus: ui.Focus{Len: len(p.Params)}}
for range p.Params {
f.fields = append(f.fields, ui.NewTextFieldState(""))
// fill gives the fields their first values
func (f *form) fill(values map[string]string) {
f.focus, f.fields = ui.Focus{Len: len(f.params)}, nil
for _, p := range f.params {
f.fields = append(f.fields, ui.NewTextFieldState(values[p.Name]))
}
return f
}
func (f *presetForm) draw(a *app, r ui.Region) {
th := a.th
in := a.frame(r, 76, len(f.fields)+6, "preset "+f.preset.Name)
in.TextStyled(0, 0, ui.Truncate(f.preset.Summary, in.W), th.Muted)
rows := max(1, in.H-3) // between the summary and the note
f.scroll = max(0, min(f.scroll, f.focus.Index, len(f.fields)-rows), f.focus.Index-rows+1)
for i := f.scroll; i < len(f.fields) && i-f.scroll < rows; i++ {
p := f.preset.Params[i]
v, _ := in.Field(2+i-f.scroll, 14, ui.Field{Label: p.Name, Required: p.Required}, i == f.focus.Index, th)
// draw lays the form out at the screen's width: the summary, the fields,
// their labels beside them or, when stacked, above, then the focused
// field's help or the refusal, in rows kept for the longest of them
func (f *form) draw(a *app, r ui.Region) {
th, w := a.th, inside(r.W)
labelW, tall := 0, 2
if w >= stacked {
for _, p := range f.params {
labelW = max(labelW, ui.RuneLen(p.Name)+2)
}
tall = 1
}
summary, noteRows := wrap(f.summary, w, 2), len(wrap(f.err, w, 3))
for _, p := range f.params {
noteRows = max(noteRows, len(wrap(p.Help, w, 3)))
}
top, bottom := len(summary)+min(1, len(summary)), noteRows+min(1, noteRows)
in := a.frame(r, w, top+len(f.fields)*tall+bottom, f.title)
for i, l := range summary {
in.TextStyled(0, i, l, th.Muted)
}
view := in.Sub(0, top, in.W, max(1, in.H-top-bottom))
f.scroll.SetDimensions(len(f.fields)*tall, view.H)
f.scroll.EnsureRange(f.focus.Index*tall, tall)
view.Window(len(f.fields)*tall, &f.scroll, th.Text, func(all ui.Region) {
for i, p := range f.params {
v, _ := all.Field(i*tall, labelW, ui.Field{Label: p.Name, Required: p.Required}, i == f.focus.Index, th)
v.TextInput(f.fields[i], p.Default, i == f.focus.Index, th)
}
note, style := f.preset.Params[f.focus.Index].Help, th.Muted
})
note, style := f.params[f.focus.Index].Help, th.Muted
if f.err != "" {
note, style = f.err, th.Error
}
in.TextStyled(0, in.H-1, ui.Truncate(note, in.W), style)
lines := wrap(note, in.W, noteRows) // on the last rows, the box ending on text
for i, l := range lines {
in.TextStyled(0, in.H-len(lines)+i, l, style)
}
}
func (f *presetForm) handle(a *app, ev terminal.Event) {
// wrap is s wrapped to w, at most n lines, the last cut with '…'; none when
// s is empty
func wrap(s string, w, n int) []string {
if s == "" {
return nil
}
return clip(ui.WrapText(s, w), n)
}
func (f *form) handle(a *app, ev terminal.Event) {
field := f.fields[f.focus.Index]
switch {
case ev.Type == terminal.EventPaste:
field.Paste(ev.Text)
case ev.Type != terminal.EventKey:
case is(ev, terminal.KeyEscape, ""):
a.dialog = a.presetMenu()
a.dialog = nil
if f.back != nil {
f.back(a)
}
case is(ev, terminal.KeyEnter, ""):
f.apply(a)
f.submit(a)
case is(ev, terminal.KeyUp, ""):
f.focus.Index = max(0, f.focus.Index-1)
case is(ev, terminal.KeyDown, ""):
@@ -222,39 +323,33 @@ func (f *presetForm) handle(a *app, ev terminal.Event) {
}
}
// apply builds the preset's pipeline, on this host: a path is a directory
// when one is there
func (f *presetForm) apply(a *app) {
// submit applies the values typed, closing the form unless they are refused
func (f *form) submit(a *app) {
values := map[string]string{}
for i, p := range f.preset.Params {
for i, p := range f.params {
if v := f.fields[i].Value(); v != "" {
values[p.Name] = v
}
}
c, err := compose.FromPreset(f.preset.Name, values, config.HostIsDir)
if err != nil {
f.err = err.Error()
return
}
a.dialog = nil
a.use(c.Pipelines[0])
if err := f.apply(a, values); err != nil {
f.err, a.dialog = err.Error(), f
}
}
// outputMenu is what o offers: run, or print one of the forms
func (a *app) outputMenu() dialog {
forms := compose.Forms()
options := []ui.Option{{Name: "run", Hint: "start the pipelines now, with lw's other settings"}}
options := []ui.Option{{Name: "run", Key: 'r', Hint: "start the pipelines now, with lw's other settings"}}
for _, f := range forms {
options = append(options, ui.Option{Name: f.Name, Hint: "print " + f.Hint})
options = append(options, ui.Option{Name: f.Name, Key: rune(f.Name[0]), Hint: "print " + f.Hint})
}
return &chooser{title: "Output, once the screen closes", list: ui.NewOptionListState(options),
list := ui.NewOptionListState(options)
list.Menu = true
return &chooser{title: "Output, once the screen closes", list: list,
pick: func(a *app, i int) {
if i == 0 {
if a.problem != nil {
a.status = "not valid yet: c shows the problem"
return
}
a.result, a.done = Result{Exit: Start, Pipelines: a.comp.Pipelines}, true
a.run()
return
}
text, err := forms[i-1].Write(a.comp)
@@ -266,88 +361,120 @@ func (a *app) outputMenu() dialog {
}}
}
// notice shows lines until any key
// notice shows text, wrapped to the screen, until a key other than one
// that scrolls it
type notice struct {
title string
lines []string
body func(w int) []string // the lines at a width
view ui.ViewportScroll
}
// textNotice is a notice of plain text, its paragraphs split at '\n'
func textNotice(title, body string) *notice {
return &notice{title: title, body: func(w int) []string { return ui.WrapText(body, w) }}
}
func (n *notice) draw(a *app, r ui.Region) {
w, lines := ui.RuneLen(n.title)+8, fit(n.lines, r.W)
lines, w := n.body(inside(r.W)), ui.RuneLen(n.title)+2
for _, l := range lines {
w = max(w, ui.RuneLen(l)+6)
w = max(w, ui.RuneLen(l))
}
in := a.frame(r, w, len(lines)+4, n.title)
for i, l := range lines {
in.TextStyled(0, 1+i, l, a.th.Text)
in := a.frame(r, min(w, inside(r.W)), len(lines), n.title)
rows := in.H
if len(lines) > in.H { // the last row says where the view is
rows = max(1, in.H-1)
}
n.view.SetDimensions(len(lines), rows)
for y := range min(rows, len(lines)-n.view.Offset) {
in.TextStyled(0, y, lines[n.view.Offset+y], a.th.Text)
}
// fit wraps lines to a dialog's inside on a screen w wide
func fit(lines []string, w int) []string {
var out []string
for _, l := range lines {
out = append(out, ui.WrapText(l, max(1, w-6))...)
if rows < in.H {
where := fmt.Sprintf("j k: %d-%d of %d", n.view.Offset+1, n.view.Offset+rows, len(lines))
in.TextStyled(0, rows, ui.Truncate(where, in.W), a.th.Muted)
}
return out
}
func (n *notice) handle(a *app, ev terminal.Event) {
if ev.Type == terminal.EventKey {
switch {
case ev.Type != terminal.EventKey:
case is(ev, terminal.KeyUp, "k"):
n.view.ScrollBy(-1)
case is(ev, terminal.KeyDown, "j"):
n.view.ScrollBy(1)
case is(ev, terminal.KeyPageUp, ""):
n.view.PageUp()
case is(ev, terminal.KeyPageDown, ""):
n.view.PageDown()
default:
a.dialog = nil
}
}
// confirm asks a yes or no question
// confirm asks a yes or no question: y or n answers, as does Enter on the
// answer the arrows chose; Esc is no
type confirm struct {
question string
yes func(a *app)
state ui.ConfirmState
}
func (c *confirm) draw(a *app, r ui.Region) {
lines := fit([]string{c.question}, r.W)
w := 0
lines := ui.WrapText(c.question, inside(r.W))
w := 12 // the answers
for _, l := range lines {
w = max(w, ui.RuneLen(l)+6)
w = max(w, ui.RuneLen(l))
}
in := a.frame(r, w, len(lines)+5, "Confirm")
in := a.frame(r, w, len(lines)+1, "Confirm")
for i, l := range lines {
in.TextStyled(0, 1+i, l, a.th.Text)
in.TextStyled(0, i, l, a.th.Text)
}
in.TextStyled(0, 1+len(lines), "y yes n no", a.th.Muted)
in.Radio(0, in.H-1, []string{"yes", "no"}, map[bool]int{true: 0, false: 1}[c.state.FocusYes], a.th)
}
func (c *confirm) handle(a *app, ev terminal.Event) {
switch {
case is(ev, terminal.KeyNone, "yY"):
a.dialog = nil
if ev.Type != terminal.EventKey || !c.state.HandleKey(ev.Key, ev.Rune) {
return
}
if a.dialog = nil; c.state.Result == ui.ConfirmYes {
c.yes(a)
case is(ev, terminal.KeyEscape, "nNq"):
a.dialog = nil
}
}
// help lists the keys
// help lists the keys, each beside its description or, narrow, above it
func help(f font) dialog {
keys := [][2]string{
{"arrows, hjkl", "move"},
{f.enter, "inspect the chosen part; edit a value"},
{"arrows, hjkl", "move; up from the top row to the pipeline bar"},
{f.enter, "inspect a part; edit a value; open a list; add"},
{"tab", "next value"},
{"esc", "back"},
{"a", "add a source, sink or filter"},
{"a", "add a source, sink or filter; on the bar, a pipeline"},
{"c", "change a source's or sink's type"},
{"d", "delete it; a value back to its default"},
{f.space, "a flow stage or a switch on or off"},
{"J K", "move a filter down or up"},
{"[ ]", "previous or next pipeline"},
{"p", "start over from a preset"},
{"r", "run the pipelines"},
{"o", "run, or print a command line, environment or file"},
{"c", "the first problem"},
{"!", "the first problem"},
{"paste", "an lw command line replaces the pipelines"},
{"q", "quit"},
}
n := &notice{title: "Keys"}
return &notice{title: "Keys", body: func(w int) []string {
var lines []string
for _, k := range keys {
n.lines = append(n.lines, ui.PadRight(k[0], 14)+k[1])
if w < stacked {
lines = append(lines, k[0])
for _, l := range ui.WrapText(k[1], w-2) {
lines = append(lines, " "+l)
}
return n
continue
}
key := k[0]
for _, l := range ui.WrapText(k[1], w-14) {
lines, key = append(lines, ui.PadRight(key, 14)+l), ""
}
}
return lines
}}
}
internal/tui/inspector.go
+58 -37
View File
@@ -21,15 +21,21 @@ type inspector struct {
field *ui.TextFieldState // the value being typed, nil while none
}
// line is one inspector row: a key, a group that folds, or a list's element
// line is one inspector row: a key, a group that folds, a list's element, or
// the row that adds an element or a filter
type line struct {
node compose.Node
key config.Key // its Name is the dotted path in the node's options
key config.Key // its Name is the dotted path in the node's options, "" on a filter's rows
depth int
group string // a group's title, "" for a value
item int // a list element's index, the list's length on the row that adds one; -1 elsewhere
}
// adds reports whether a row adds: a list's last row, or the filters'
func (a *app) adds(l line) bool {
return l.item >= 0 && (l.key.Name == "" || l.item == len(a.items(l.node, l.key.Name)))
}
func (a *app) foldKey(n compose.Node, path string) string {
return fmt.Sprintf("%d/%s/%s/%d/%s", a.pi, n.Role, n.ID, n.Index, path)
}
@@ -44,14 +50,15 @@ func (a *app) lines() []line {
return a.keyLines(nil, n, a.keys(n), "", 0)
}
var out []line
for i, f := range a.pipeline().Flow.Filters {
filters := a.pipeline().Flow.Filters
for i, f := range filters {
fn := compose.Node{Role: "filters", Index: i}
out = append(out, line{node: fn, group: fmt.Sprintf("%d %s", i+1, f.Type), item: -1})
if a.insp.open[a.foldKey(n, strconv.Itoa(i))] {
out = a.keyLines(out, fn, a.keys(n), "", 1)
}
}
return out
return append(out, line{node: compose.Node{Role: "filters", Index: len(filters)}, item: len(filters)})
}
func (a *app) keyLines(out []line, n compose.Node, keys []config.Key, prefix string, depth int) []line {
@@ -120,31 +127,31 @@ func (a *app) drawInspector(r ui.Region) {
if a.inspect {
hint = f.enter + " edit esc back"
}
r.TextStyled(0, 0, strings.Repeat(string(f.rule), r.W), th.Border)
r.TextStyled(2, 0, " "+ui.Truncate(title, r.W-ui.RuneLen(hint)-10)+" ", th.Text)
r.TextStyled(r.W-ui.RuneLen(hint)-3, 0, " "+hint+" ", th.Muted)
r.Rule(0, title, hint, th)
lines := a.lines()
switch {
case !ok:
r.TextStyled(2, 2, "a adds a "+columns[a.col].role, th.Muted)
return
case len(lines) == 0 && n.Role == "filters":
r.TextStyled(2, 2, "no filters: a adds one", th.Muted)
return
case len(lines) == 0:
r.TextStyled(2, 2, "no options", th.Muted)
return
}
a.insp.cursor = max(0, min(a.insp.cursor, len(lines)-1))
rows := r.H - 2
var help []string
style := th.Muted
if a.inspect {
var text string
text, style = a.help(lines[a.insp.cursor])
help = clip(ui.WrapText(text, r.W-4), 2)
}
rows := r.H - 1 - len(help)
cols, perCol := 1, rows
if half := (len(lines) + 1) / 2; r.W >= 80 && half <= rows {
cols, perCol = 2, half
}
if cols == 1 {
a.insp.scroll = max(0, min(a.insp.scroll, a.insp.cursor, len(lines)-rows))
a.insp.scroll = max(a.insp.scroll, a.insp.cursor-rows+1)
} else {
a.insp.scroll = ui.AdjustScroll(a.insp.cursor, ui.ClampScroll(a.insp.scroll, rows, len(lines)), rows, len(lines))
if cols == 2 {
a.insp.scroll = 0
}
colW, labelW := r.W/cols, 0
@@ -157,9 +164,8 @@ func (a *app) drawInspector(r ui.Region) {
row := r.Sub(at/perCol*colW, 1+at%perCol, colW-1, 1)
a.drawLine(row, lines[i], labelW, a.inspect && i == a.insp.cursor)
}
if a.inspect && rows > 0 {
help, style := a.help(lines[a.insp.cursor])
r.TextStyled(2, r.H-1, ui.Truncate(help, r.W-4), style)
for i, h := range help {
r.TextStyled(2, r.H-len(help)+i, h, style)
}
}
@@ -177,12 +183,15 @@ func (a *app) nodeTitle(n compose.Node) string {
// help is the focused row's problem, or its key's help and default
func (a *app) help(l line) (string, ui.Style) {
if a.problem != nil {
if pi, n, key, ok := locate(a.problem); ok && pi == a.pi && n == l.node && key == l.key.Name {
return a.problem.Error(), a.th.Error
if pi, n, key, text, ok := locate(a.problem); ok && pi == a.pi && n == l.node && key == l.key.Name {
return text, a.th.Error
}
}
if l.group != "" && l.key.Name == "" {
return "J and K move this filter; d deletes it", a.th.Muted
switch {
case l.group != "" && l.key.Name == "":
return "J and K move this filter; c changes its type; d deletes it", a.th.Muted
case a.adds(l) && l.key.Name == "":
return "adds a filter: include passes only the entries that match, exclude drops them", a.th.Muted
}
help := l.key.Help
if d := placeholder(l.key); d != "" && l.group == "" {
@@ -217,12 +226,12 @@ func (a *app) drawLine(r ui.Region, l line, labelW int, focused bool) {
switch {
case focused && a.insp.field != nil:
v.TextInput(a.insp.field, placeholder(k), true, th)
case l.item == len(items):
case a.adds(l):
v.TextStyled(0, 0, "+ add", th.Muted)
case l.item >= 0:
v.TextStyled(0, 0, ui.Truncate(items[l.item], v.W), th.Text)
case len(k.Enum) > 0:
v.Choice(0, 0, k.Enum, choice(k, cur), th)
v.Radio(0, 0, k.Enum, choice(k, cur), th)
case k.Kind == "bool":
v.Toggle(0, 0, cur == true || cur == nil && k.Default == "true", th)
case cur == nil:
@@ -289,7 +298,8 @@ func (a *app) inspectKey(ev terminal.Event) bool {
k := l.key
cur := a.value(l.node, k.Name)
focus := ui.Focus{Index: a.insp.cursor, Len: len(lines)}
toggle := is(ev, terminal.KeyEnter, " ") || is(ev, terminal.KeySpace, "")
enter, space := is(ev, terminal.KeyEnter, ""), is(ev, terminal.KeySpace, " ")
left, right := is(ev, terminal.KeyLeft, "h"), is(ev, terminal.KeyRight, "l")
switch {
case focus.HandleKey(ev.Key, ev.Modifiers):
a.insp.cursor = focus.Index
@@ -297,31 +307,35 @@ func (a *app) inspectKey(ev terminal.Event) bool {
a.insp.cursor = max(0, a.insp.cursor-1)
case is(ev, terminal.KeyDown, "j"):
a.insp.cursor = min(len(lines)-1, a.insp.cursor+1)
case l.group != "" && (toggle || is(ev, terminal.KeyRight, "l") || is(ev, terminal.KeyLeft, "h")):
case a.adds(l) && k.Name == "" && enter:
a.dialog = a.addMenu()
case l.group != "" && (enter || space || left || right):
fold := a.foldKey(l.node, k.Name)
if k.Name == "" {
fold = a.foldKey(compose.Node{Role: "filters"}, strconv.Itoa(l.node.Index))
}
a.insp.open[fold] = toggle && !a.insp.open[fold] || is(ev, terminal.KeyRight, "l")
case l.group != "" && l.key.Name == "" && is(ev, terminal.KeyNone, "JK"):
a.insp.open[fold] = (enter || space) && !a.insp.open[fold] || right
case l.group != "" && k.Name == "" && is(ev, terminal.KeyNone, "JK"):
a.moveFilter(l.node.Index, map[rune]int{'J': 1, 'K': -1}[ev.Rune])
case is(ev, terminal.KeyNone, "d"):
a.clear(l)
case len(k.Enum) > 0 && l.item < 0 && (toggle || is(ev, terminal.KeyLeft, "h") || is(ev, terminal.KeyRight, "l")):
case len(k.Enum) > 0 && l.item < 0 && enter:
a.dialog = a.enumMenu(l, cur)
case len(k.Enum) > 0 && l.item < 0 && (space || left || right):
i := choice(k, cur)
switch next, moved := ui.StepChoice(ev.Key, ev.Rune, i, len(k.Enum)); {
case moved:
i = next
case !toggle:
case !space:
return true // an arrow at either end
default:
i = (i + 1) % len(k.Enum)
}
a.edit(a.comp.Set(a.pi, l.node, k.Name, k.Enum[i]))
case k.Kind == "bool" && l.item < 0 && toggle:
case k.Kind == "bool" && l.item < 0 && (enter || space || left || right):
on := cur == true || cur == nil && k.Default == "true"
a.edit(a.comp.Set(a.pi, l.node, k.Name, strconv.FormatBool(!on)))
case is(ev, terminal.KeyEnter, ""):
a.edit(a.comp.Set(a.pi, l.node, k.Name, strconv.FormatBool(right || !left && !on)))
case enter:
text := show(cmp.Or(cur, any("")))
if items := a.items(l.node, k.Name); l.item >= 0 {
text = ""
@@ -337,13 +351,14 @@ func (a *app) inspectKey(ev terminal.Event) bool {
return true
}
// typeKey edits the value being typed: Enter sets it, Tab sets it and moves
// on, Esc drops it
// typeKey edits the value being typed: Enter sets it, Tab and the arrows up
// and down set it and move on, Esc drops it
func (a *app) typeKey(ev terminal.Event, l line) {
up, down := is(ev, terminal.KeyUp, ""), is(ev, terminal.KeyDown, "")
switch {
case is(ev, terminal.KeyEscape, ""):
a.insp.field = nil
case is(ev, terminal.KeyEnter, "") || is(ev, terminal.KeyTab, "") || is(ev, terminal.KeyBacktab, ""):
case is(ev, terminal.KeyEnter, "") || is(ev, terminal.KeyTab, "") || is(ev, terminal.KeyBacktab, "") || up || down:
value := a.insp.field.Value()
if a.commit(l, value) {
a.insp.field = nil
@@ -351,7 +366,12 @@ func (a *app) typeKey(ev terminal.Event, l line) {
a.insp.cursor++ // stays on the row that adds one
}
focus := ui.Focus{Index: a.insp.cursor, Len: len(a.lines())}
if focus.HandleKey(ev.Key, ev.Modifiers) {
switch {
case up:
a.insp.cursor = max(0, a.insp.cursor-1)
case down:
a.insp.cursor = min(focus.Len-1, a.insp.cursor+1)
case focus.HandleKey(ev.Key, ev.Modifiers):
a.insp.cursor = focus.Index
}
}
@@ -390,6 +410,7 @@ func (a *app) commit(l line, value string) bool {
// value, which returns to its default
func (a *app) clear(l line) {
switch {
case a.adds(l):
case l.group != "" && l.key.Name == "":
a.edit(a.comp.Remove(a.pi, l.node))
case l.item >= 0:
internal/tui/theme.go
+10 -10
View File
@@ -55,14 +55,14 @@ var looks = map[terminal.ColorMode]look{
Source: ui.MonoTheme.Text, Flow: ui.MonoTheme.Text, Sink: ui.MonoTheme.Text},
}
// font is the marks one character set draws: the widgets' and the canvas's
// font is the marks one character set draws: the widgets' and the canvas's;
// Glyphs.Line draws frames, rules and wires
type font struct {
ui.Glyphs
wire []rune // a wire cell by its arms: up 1, right 2, down 4, left 8
box []rune // the flow box: top left, top right, bottom left, bottom right, across, down
box ui.LineType // the flow box
arrow, down rune // into the flow box and a sink; between stacked parts
bar, barOn rune // a node, and the chosen node
dot, rule rune // before a column's title; the inspector's rule
dot rune // before a column's title
valid string
invalid string
enter string
@@ -85,13 +85,13 @@ func (f font) fold(r rune) rune {
// fonts: Unicode, a text console's CP437, and ASCII outside UTF-8
var (
unicodeFont = font{Glyphs: ui.GlyphsUnicode, wire: []rune(" │─└││┌├─┘─┴┐┤┬┼"), box: []rune("╭╮╰╯╌╎"),
arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '●', rule: '─', valid: "✓", invalid: "✗", enter: "⏎", space: "␣"}
cp437Font = font{Glyphs: ui.GlyphsCP437, wire: []rune(" │─└││┌├─┘─┴┐┤┬┼"), box: []rune("┌┐└┘─│"),
arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '•', rule: '─', valid: "√", invalid: "x", enter: "enter", space: "space",
unicodeFont = font{Glyphs: ui.GlyphsUnicode, box: ui.LineDashed,
arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '●', valid: "✓", invalid: "✗", enter: "⏎", space: "␣"}
cp437Font = font{Glyphs: ui.GlyphsCP437, box: ui.LineSingle,
arrow: '►', down: '▼', bar: '▌', barOn: '█', dot: '•', valid: "√", invalid: "x", enter: "enter", space: "space",
folds: map[rune]rune{'…': '.'}}
asciiFont = font{Glyphs: ui.GlyphsASCII, wire: []rune(" |-+||++-+-+++++"), box: []rune("++++-|"),
arrow: '>', down: 'v', bar: '|', barOn: '#', dot: '*', rule: '-', valid: "ok", invalid: "x", enter: "enter", space: "space",
asciiFont = font{Glyphs: ui.GlyphsASCII, box: ui.LineASCII,
arrow: '>', down: 'v', bar: '|', barOn: '#', dot: '*', valid: "ok", invalid: "x", enter: "enter", space: "space",
folds: map[rune]rune{'…': '.', '·': '-'}, ascii: true}
)
internal/tui/tui.go
+184 -29
View File
@@ -7,6 +7,7 @@ package tui
import (
"errors"
"fmt"
"slices"
"strings"
"github.com/lixenwraith/logwisp/internal/compose"
@@ -88,7 +89,8 @@ type app struct {
pi int // the pipeline shown
col int // the chosen column
row [3]int // the chosen node of each column
scroll int // the canvas's first row
scroll ui.ViewportScroll // the canvas's rows
bar bool // keys go to the pipeline bar
inspect bool // keys go to the inspector
insp inspector
dialog dialog // takes every key while open
@@ -217,6 +219,8 @@ func (a *app) handle(ev terminal.Event) {
case ev.Type == terminal.EventPaste:
a.paste(ev.Text)
case ev.Type != terminal.EventKey:
case a.bar:
a.barKey(ev)
case !a.inspect:
a.canvasKey(ev)
case !a.inspectKey(ev):
@@ -238,13 +242,17 @@ func (a *app) canvasKey(ev terminal.Event) {
a.col = min(2, a.col+1)
case is(ev, terminal.KeyTab, ""):
a.col = (a.col + 1) % 3
case is(ev, terminal.KeyUp, "k") && a.row[a.col] == 0:
a.bar = true
case is(ev, terminal.KeyUp, "k"):
a.row[a.col] = max(0, a.row[a.col]-1)
a.row[a.col]--
case is(ev, terminal.KeyDown, "j"):
a.row[a.col]++
a.node()
case is(ev, terminal.KeyEnter, "") && ok:
a.inspect, a.insp.cursor = true, 0
case is(ev, terminal.KeyEnter, "") && (!ok || n.Role == "filters" && len(a.pipeline().Flow.Filters) == 0):
a.dialog = a.addMenu() // an empty column, or no filters, as a
case is(ev, terminal.KeyEnter, ""):
a.inspectFirst(n)
case is(ev, terminal.KeyNone, "a"):
a.dialog = a.addMenu()
case is(ev, terminal.KeyNone, "d") && ok && n.Role == "filters":
@@ -269,8 +277,12 @@ func (a *app) globalKey(ev terminal.Event) {
a.dialog = a.outputMenu()
case is(ev, terminal.KeyNone, "?"):
a.dialog = help(a.font)
case is(ev, terminal.KeyNone, "c"):
case is(ev, terminal.KeyNone, "!"):
a.showProblem()
case is(ev, terminal.KeyNone, "c"):
a.changeType()
case is(ev, terminal.KeyNone, "r"):
a.run()
case is(ev, terminal.KeyNone, "]") && n > 0:
a.pi, a.row, a.inspect = (a.pi+1)%n, [3]int{}, false
case is(ev, terminal.KeyNone, "[") && n > 0:
@@ -280,6 +292,76 @@ func (a *app) globalKey(ev terminal.Event) {
}
}
// barKey takes the keys on the pipeline bar: switch, add, rename, delete
func (a *app) barKey(ev terminal.Event) {
n := len(a.comp.Pipelines)
switch p := a.pipeline(); {
case is(ev, terminal.KeyLeft, "h["):
a.pi, a.row = (a.pi+n-1)%n, [3]int{}
case is(ev, terminal.KeyRight, "l]"):
a.pi, a.row = (a.pi+1)%n, [3]int{}
case is(ev, terminal.KeyDown, "j") || is(ev, terminal.KeyEscape, ""):
a.bar = false
case is(ev, terminal.KeyNone, "a"):
a.dialog = a.pipelineMenu("Add a pipeline", (*app).append)
case is(ev, terminal.KeyEnter, ""):
f := &form{title: "Rename pipeline " + p.Name, params: []config.PresetParam{{Name: "name", Required: true, Help: "a name no other pipeline has"}},
apply: func(a *app, v map[string]string) error {
err := a.comp.RenamePipeline(a.pi, v["name"])
if err == nil {
a.edit(nil)
}
return err
}}
f.fill(map[string]string{"name": p.Name})
a.dialog = f
case is(ev, terminal.KeyNone, "d"):
remove := func(a *app) {
a.edit(a.comp.RemovePipeline(a.pi))
a.pi, a.row, a.insp.open = max(0, a.pi-1), [3]int{}, map[string]bool{} // folds are keyed by index
if len(a.comp.Pipelines) == 0 {
a.bar, a.dialog = false, a.presetMenu()
}
}
if len(p.PluginSources)+len(p.PluginSinks) == 0 {
remove(a)
return
}
a.dialog = &confirm{question: "Delete pipeline " + p.Name + "?", yes: remove}
default:
a.globalKey(ev)
}
}
// changeType offers a source or sink the other types of its role; a filter
// or the format changes type on its type row
func (a *app) changeType() {
n, ok := a.node()
if l := a.lines(); a.inspect && ok && len(l) > 0 {
n = l[min(a.insp.cursor, len(l)-1)].node
}
switch {
case !ok || n.Role == "filters" && !a.inspect:
a.note = "c changes the type of a source, sink, filter or format"
case n.Role == "source" || n.Role == "sink":
a.dialog = a.typeMenu(n)
case n.Role == "filters" && n.Index < len(a.pipeline().Flow.Filters), n.Role == "format":
a.choose(n)
a.inspectAt(func(l line) bool { return l.node == n && l.key.Name == "type" })
default:
a.note = "c changes the type of a source, sink, filter or format"
}
}
// run leaves to start the pipelines, once they are valid
func (a *app) run() {
if a.problem != nil {
a.status = "not valid yet: ! shows the problem"
return
}
a.result, a.done = Result{Exit: Start, Pipelines: a.comp.Pipelines}, true
}
// switchStage turns a flow stage on with its defaults, or off; a stage with
// an enabled key switches that
func (a *app) switchStage(n compose.Node) {
@@ -308,7 +390,8 @@ func slicesHasKey(keys []config.Key, name string) bool {
return false
}
// paste starts over from a pasted command line
// paste starts over from a pasted command line, asking first when there are
// edited pipelines to lose
func (a *app) paste(text string) {
c, err := compose.FromCommandLine(text, config.HostIsDir)
if err != nil {
@@ -320,7 +403,7 @@ func (a *app) paste(text string) {
a.note = fmt.Sprintf("pasted %d pipeline(s)", len(c.Pipelines))
a.check()
}
if !a.changed {
if !a.changed || len(a.comp.Pipelines) == 0 {
replace(a)
return
}
@@ -341,18 +424,25 @@ func (a *app) showProblem() {
a.note = "the pipelines are valid"
return
}
if pi, n, key, ok := locate(a.problem); ok && pi < len(a.comp.Pipelines) {
if pi, n, key, _, ok := locate(a.problem); ok && pi < len(a.comp.Pipelines) {
a.pi = pi
a.choose(n)
a.inspect = true
a.insp.cursor = 0
for i, l := range a.lines() {
if l.node == n && l.key.Name == key {
a.insp.cursor = i
a.inspectAt(func(l line) bool { return l.node == n && l.key.Name == key })
}
a.dialog = textNotice("Problem", a.problem.Error())
}
// inspectFirst opens the inspector on a node's first required key, or on a
// filter's group
func (a *app) inspectFirst(n compose.Node) {
a.inspectAt(func(l line) bool { return l.node == n && (l.key.Required || l.key.Name == "") })
}
a.dialog = &notice{title: "Problem", lines: ui.WrapText(a.problem.Error(), 60)}
// inspectAt opens the inspector, which takes the keys from the bar, on the
// first line at holds for, or on its top
func (a *app) inspectAt(at func(line) bool) {
a.inspect, a.bar = true, false
a.insp.cursor = max(0, slices.IndexFunc(a.lines(), at))
}
// choose puts the canvas cursor on a node, a filter on the filters stage
@@ -374,13 +464,12 @@ func (a *app) choose(n compose.Node) {
// it from 110 columns or below it, a status row, and any dialog over them
func (a *app) draw(r ui.Region) {
r.FillStyle(a.th.Text)
status, style := a.bottom(r.W)
if len(a.comp.Pipelines) > 0 {
a.pi = min(a.pi, len(a.comp.Pipelines)-1)
a.node() // a delete can leave the row past its column's end
p := a.pipeline()
title := fmt.Sprintf("LogWisp pipeline %s %d of %d", p.Name, a.pi+1, len(a.comp.Pipelines))
r.TextStyled(1, 0, ui.Truncate(title, r.W-2), a.th.Accent)
body := r.Sub(0, 1, r.W, r.H-2)
a.drawBar(r.Sub(0, 0, r.W, 1))
body := r.Sub(0, 1, r.W, r.H-1-len(status))
if r.W >= 110 {
iw := min(60, r.W*2/5)
a.drawCanvas(body.Sub(0, 0, r.W-iw-1, body.H))
@@ -391,12 +480,56 @@ func (a *app) draw(r ui.Region) {
a.drawInspector(body.Sub(0, ch+1, r.W, body.H-ch-1))
}
}
a.drawStatus(r.Sub(0, r.H-1, r.W, 1))
for i, l := range status {
r.TextStyled(1, r.H-len(status)+i, l, style)
}
if a.status == "" && a.note == "" && a.problem == nil {
keys := a.hints(r.W)
r.TextStyled(r.W-ui.RuneLen(keys)-1, r.H-1, keys, a.th.Muted)
}
if a.dialog != nil {
a.dialog.draw(a, r)
}
}
// drawBar draws the title row: the pipelines by name, the one shown marked,
// or it and its place when they do not fit; with the keys, the bar's verbs
func (a *app) drawBar(r ui.Region) {
th, f := a.th, a.font
if a.bar {
r.TextStyled(0, 0, string(f.Focus), th.Accent)
hint := "a add " + f.enter + " rename d delete"
if r.W >= 60 {
r.TextStyled(r.W-ui.RuneLen(hint)-1, 0, hint, th.Muted)
r = r.Sub(0, 0, r.W-ui.RuneLen(hint)-2, 1)
}
}
r.TextStyled(2, 0, "LogWisp", th.Accent)
names, w := a.comp.Pipelines, 11
for _, p := range names {
w += ui.RuneLen(p.Name) + 4
}
if w > r.W {
names = names[a.pi : a.pi+1]
}
x := 11
for _, p := range names {
mark, style := f.Off, th.Muted
if p.Name == a.pipeline().Name {
mark, style = f.On, th.Text
if a.bar {
style = th.Text.On(th.Selected)
}
}
label := ui.Truncate(string(mark)+" "+p.Name, max(0, r.W-x))
r.TextStyled(x, 0, label, style)
x += ui.RuneLen(label) + 2
}
if len(names) < len(a.comp.Pipelines) {
r.TextStyled(x, 0, fmt.Sprintf("%d/%d", a.pi+1, len(a.comp.Pipelines)), th.Muted)
}
}
// paint draws the screen into cells, each rune one the font's terminal shows
func (a *app) paint(cells []terminal.Cell, w, h int) {
a.draw(ui.NewRegion(cells, w, 0, 0, w, h))
@@ -405,23 +538,45 @@ func (a *app) paint(cells []terminal.Cell, w, h int) {
}
}
// drawStatus shows an edit's refusal, a note, or whether the pipelines are
// valid, and the keys
func (a *app) drawStatus(r ui.Region) {
// bottom is the status rows, wrapped to three at most: an edit's refusal, a
// note, the first problem by its part, or the keys while all is valid
func (a *app) bottom(w int) ([]string, ui.Style) {
th, f := a.th, a.font
text, style := f.valid+" valid", th.Accent
text, style := "", th.Error
switch {
case a.status != "":
text, style = a.status, th.Error
text = a.status
case a.note != "":
text, style = a.note, th.Muted
case a.problem != nil:
text, style = f.invalid+" "+a.problem.Error(), th.Error
pi, n, key, msg, _ := locate(a.problem)
if where := a.where(pi, n, key); where != "" {
msg = where + ": " + msg
}
keys := "a add d delete " + f.space + " on/off J/K move p preset o output ? help"
if ui.RuneLen(keys)+24 > r.W {
text = f.invalid + " " + msg
default:
return []string{f.valid + " valid"}, th.Accent
}
return clip(ui.WrapText(text, w-2), 3), style
}
// hints are the status row's keys, shown while nothing else needs the row
func (a *app) hints(w int) string {
keys := "a add d delete " + a.font.space + " on/off J/K move p preset o output ? help"
if ui.RuneLen(keys)+12 > w {
keys = "? help"
}
r.TextStyled(1, 0, ui.Truncate(text, r.W-ui.RuneLen(keys)-4), style)
r.TextStyled(r.W-ui.RuneLen(keys)-1, 0, keys, th.Muted)
return keys
}
// clip keeps n lines, the last cut short when more follow
func clip(lines []string, n int) []string {
if len(lines) > n {
if n < 1 {
return nil
}
lines = lines[:n]
lines[n-1] = ui.Truncate(lines[n-1]+"…", ui.RuneLen(lines[n-1]))
}
return lines
}
internal/tui/tui_test.go
+248 -12
View File
@@ -197,8 +197,9 @@ func TestSelectionChangesAGlyph(t *testing.T) {
}
// The inspector types a value and sets it, typed by the key's kind, and Tab
// moves on; empty or d returns it to its default; arrows step a choice,
// setting nothing past either end, and Space a switch
// or Up moves on; empty or d returns it to its default; arrows step a choice,
// setting nothing past either end, and set a switch; Enter opens a choice's
// options, at its value, and Space flips a switch
func TestInspectorSetsKeys(t *testing.T) {
a := mono(pipeline(t, []string{"file"}, []string{"null"}))
press(a, terminal.KeyEnter)
@@ -221,8 +222,24 @@ func TestInspectorSetsKeys(t *testing.T) {
focusKey(t, a, "from")
press(a, terminal.KeyLeft)
focusKey(t, a, "raw")
press(a, " ")
if o := opts(); o["directory"] != "/srv/a b" || o["check_interval_ms"] != int64(15) || o["from"] != "start" || o["raw"] != true {
if press(a, " ", terminal.KeyLeft, terminal.KeyLeft); opts()["raw"] != false {
t.Fatalf("Left left raw %v", opts()["raw"])
}
press(a, terminal.KeyRight, terminal.KeyRight)
focusKey(t, a, "pattern")
at = a.insp.cursor
if press(a, terminal.KeyEnter, "x", terminal.KeyUp); a.insp.cursor != at-1 || opts()["pattern"] != "x" {
t.Fatalf("Up left the cursor on row %d, pattern %v", a.insp.cursor, opts()["pattern"])
}
focusKey(t, a, "from")
if press(a, terminal.KeyEnter); a.dialog.(*chooser).list.Cursor != 0 {
t.Fatalf("the options open off the value: %+v", a.dialog)
}
if press(a, terminal.KeyDown, terminal.KeyEscape); opts()["from"] != "start" {
t.Fatalf("Esc set %v", opts()["from"])
}
press(a, terminal.KeyEnter, terminal.KeyDown, terminal.KeyEnter)
if o := opts(); o["directory"] != "/srv/a b" || o["check_interval_ms"] != int64(15) || o["from"] != "end" || o["raw"] != true {
t.Fatalf("%v", o)
}
focusKey(t, a, "directory")
@@ -294,8 +311,9 @@ func TestFiltersMoveWithTheirFold(t *testing.T) {
}
}
// o leaves with the chosen form, printed as the engine writes it, or with
// the pipelines to run; a composition that does not validate stays
// o leaves with the form its letter picks, printed as the engine writes it,
// or with the pipelines to run, as r does; a composition that does not
// validate stays
func TestOutputLeavesWithTheForm(t *testing.T) {
c := pipeline(t, []string{"null"}, []string{"null"})
line, err := c.CommandLine()
@@ -303,7 +321,7 @@ func TestOutputLeavesWithTheForm(t *testing.T) {
t.Fatal(err)
}
a := mono(c)
press(a, "o", "comm", terminal.KeyEnter)
press(a, "o", "c")
if !a.done || a.result.Exit != Print || a.result.Output != line {
t.Fatalf("%+v", a.result)
}
@@ -312,8 +330,13 @@ func TestOutputLeavesWithTheForm(t *testing.T) {
if !a.done || a.result.Exit != Start || len(a.result.Pipelines) != 1 {
t.Fatalf("%+v", a.result)
}
a = mono(pipeline(t, []string{"null"}, []string{"null"}))
press(a, "r")
if !a.done || a.result.Exit != Start || len(a.result.Pipelines) != 1 {
t.Fatalf("r: %+v", a.result)
}
a = mono(pipeline(t, []string{"file"}, []string{"null"}))
press(a, "o", terminal.KeyEnter)
press(a, "o", terminal.KeyEnter, "r")
if a.done || a.status == "" {
t.Fatalf("ran an invalid composition: %+v", a.result)
}
@@ -351,7 +374,8 @@ func TestPresetsStartThePipeline(t *testing.T) {
}
// A pasted command line replaces the pipelines, from the preset menu too,
// asking first once they were edited; one that names none is refused
// asking first once they were edited and while some remain; one that names
// none is refused
func TestPasteReplacesThePipelines(t *testing.T) {
a := mono(&compose.Composition{})
if press(a, []byte("lw\n")); a.status == "" {
@@ -368,22 +392,42 @@ func TestPasteReplacesThePipelines(t *testing.T) {
if a.pipeline().Name != "x" {
t.Fatalf("replaced edited pipelines without asking: %+v", a.comp.Pipelines)
}
a = mono(pipeline(t, []string{"null"}, []string{"null"}))
press(a, "k", "d", "y", []byte("lw --pipeline z --source null --sink null"))
if a.dialog != nil || len(a.comp.Pipelines) != 1 || a.pipeline().Name != "z" {
t.Fatalf("a paste after the last pipeline was deleted: %T %+v", a.dialog, a.comp.Pipelines)
}
}
// The first problem marks the part it names, and c moves there, to the key
// The first problem marks the part it names, and ! moves there, to the key
func TestProblemNamesItsPart(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"null", "tcp"}))
sink := compose.Node{Role: "sink", ID: "tcp"}
if !a.faulty(sink) || a.faulty(compose.Node{Role: "source", ID: "null"}) || a.faulty(compose.Node{Role: "sink", ID: "null"}) {
t.Fatalf("problem %v", a.problem)
}
press(a, "c")
press(a, "!")
n, _ := a.node()
if l := a.lines()[a.insp.cursor]; n != sink || !a.inspect || l.key.Name != "port" {
t.Fatalf("on %+v, row %+v", n, l)
}
}
// ! and c on the pipeline bar open the inspector, which then takes the keys:
// Enter edits the key it shows, not the pipeline's name
func TestTheInspectorTakesTheKeysFromTheBar(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"tcp"}))
press(a, "k", "!", terminal.KeyEscape, terminal.KeyEnter)
if a.bar || a.insp.field == nil {
t.Fatalf("Enter after ! on the bar: bar %v, %T", a.bar, a.dialog)
}
a = mono(pipeline(t, []string{"null"}, []string{"null"}))
press(a, "k", "c", terminal.KeyEnter, terminal.KeyEnter)
if a.bar || !a.inspect || a.dialog != nil {
t.Fatalf("Enter after c on the bar: bar %v, %T", a.bar, a.dialog)
}
}
// 16 colors and none keep the terminal's own background; more paint the site's
func TestOnlyFullColorPaintsTheBackground(t *testing.T) {
for mode, l := range looks {
@@ -451,7 +495,9 @@ func TestPresetFormShowsTheFocusedField(t *testing.T) {
presets := config.Presets()
p := slices.MaxFunc(presets, func(x, y config.Preset) int { return len(x.Params) - len(y.Params) })
a := mono(pipeline(t, []string{"null"}, []string{"null"}))
a.dialog = newPresetForm(p)
f := &form{title: p.Name, params: p.Params}
f.fill(nil)
a.dialog = f
press(a, terminal.KeyBacktab)
last := p.Params[len(p.Params)-1].Name
if screen := strings.Join(text(render(a, 80, 12), 80), "\n"); !strings.Contains(screen, last) {
@@ -459,6 +505,96 @@ func TestPresetFormShowsTheFocusedField(t *testing.T) {
}
}
// presetForm is a preset's form, open on a
func presetForm(a *app, name string) *form {
presets := config.Presets()
p := presets[slices.IndexFunc(presets, func(p config.Preset) bool { return p.Name == name })]
f := &form{title: "preset " + p.Name, summary: p.Summary, params: p.Params}
f.fill(nil)
a.dialog = f
return f
}
// A preset form is usable at 25 columns: labels stack above their values,
// which take the row, so typed text shows whole
func TestPresetFormFitsANarrowScreen(t *testing.T) {
a := mono(pipeline(t, nil, nil))
presetForm(a, "serve")
press(a, terminal.KeyTab, terminal.KeyTab, terminal.KeyTab, "127.0.0.1:9000")
if screen := strings.Join(text(render(a, 25, 20), 25), "\n"); !strings.Contains(screen, "listen") || !strings.Contains(screen, "127.0.0.1:9000") {
t.Fatalf("listen and its value not whole at 25 columns:\n%s", screen)
}
}
// A dialog is sized to what it holds: its first and last rows inside the
// box have text, a form without a summary included
func TestDialogsHaveNoBlankEdgeRows(t *testing.T) {
var a *app
for name, open := range map[string]func(){
"add": func() { press(a, "a") },
"presets": func() { press(a, "p") },
"output": func() { press(a, "o") },
"confirm": func() { press(a, "q") },
"help": func() { press(a, "?") },
"problem": func() { press(a, "!") },
"rename": func() { press(a, terminal.KeyUp, terminal.KeyEnter) },
"form": func() { presetForm(a, "serve") },
} {
a = mono(pipeline(t, nil, nil))
a.changed, a.dialog = true, nil
open()
rows := text(render(a, 80, 40), 80)
bottom := slices.IndexFunc(rows, func(r string) bool { return strings.Contains(r, "└") })
top := slices.IndexFunc(rows, func(r string) bool { return strings.Contains(r, "┌") })
if a.dialog == nil || top < 0 || bottom < top+2 {
t.Errorf("%s: no dialog:\n%s", name, strings.Join(rows, "\n"))
continue
}
edge := []rune(rows[bottom])
left, right := slices.Index(edge, '└'), slices.Index(edge, '┘')
for _, y := range []int{top + 1, bottom - 1} {
if inside := strings.Trim(string([]rune(rows[y])[left:right]), " │"); inside == "" {
t.Errorf("%s: row %d blank:\n%s", name, y, strings.Join(rows[top:bottom+1], "\n"))
}
}
}
}
// A question is answered by y or n, or by Enter on the answer the arrows
// chose; no is chosen first
func TestConfirmAnswersOnEnter(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"null"}))
a.changed = true
if press(a, "q", terminal.KeyEnter); a.done || a.dialog != nil {
t.Fatalf("Enter on no: done %v, dialog %T", a.done, a.dialog)
}
if press(a, "q", terminal.KeyLeft, terminal.KeyEnter); !a.done {
t.Fatal("Enter on yes did not quit")
}
}
// The canvas scrolls to keep the chosen part in view, its bar solid
func TestCanvasKeepsTheChosenInView(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, slices.Repeat([]string{"null"}, 8)))
a.col, a.row[sinksCol] = sinksCol, 7
if screen := strings.Join(text(render(a, 80, 24), 80), "\n"); !strings.ContainsRune(screen, a.font.barOn) {
t.Fatalf("the eighth sink is out of view:\n%s", screen)
}
}
// A notice taller than the screen scrolls with j and k and says where it is
func TestNoticeScrolls(t *testing.T) {
a := mono(pipeline(t, nil, nil))
press(a, "?")
screen := func() string { return strings.Join(text(render(a, 25, 20), 25), "\n") }
if s := screen(); strings.Contains(s, "quit") || !strings.Contains(s, "j k: 1-") {
t.Fatalf("help at 25 columns:\n%s", s)
}
if press(a, strings.Repeat("j", 60)); a.dialog == nil || !strings.Contains(screen(), "quit") {
t.Fatalf("scrolled to the end:\n%s", screen())
}
}
// A dialog's text wraps to a narrow screen
func TestDialogsWrapToTheScreen(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"null"}))
@@ -472,3 +608,103 @@ func TestDialogsWrapToTheScreen(t *testing.T) {
}
}
}
// The inspector opens on a part's first required key, after a adds it or
// Enter chooses it
func TestInspectorOpensOnTheFirstRequiredKey(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"tcp_chain"}))
a.col = sinksCol
press(a, terminal.KeyEnter)
at := func() string { return a.lines()[a.insp.cursor].key.Name }
if got := at(); got != "host" {
t.Fatalf("Enter on tcp_chain: %s", got)
}
press(a, terminal.KeyEscape, "a", "http", terminal.KeyEnter)
if got := at(); got != "port" {
t.Fatalf("a adding http: %s", got)
}
}
// The status shows the first problem whole, by its part and key, not by its
// path, on as many rows as it takes
func TestTheProblemShowsWhole(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"http"}))
sink := compose.Node{Role: "sink", ID: "http"}
for key, value := range map[string]string{"port": "8080", "viewer_page": "true"} {
if err := a.comp.Set(0, sink, key, value); err != nil {
t.Fatal(err)
}
}
a.check()
for _, w := range []int{60, 87, 146} {
screen := strings.Join(text(render(a, w, 30), w), "\n")
if !strings.Contains(screen, "sink http · viewer_page: only") || !strings.Contains(screen, "at /") || strings.Contains(screen, "pipelines[0]") {
t.Errorf("%d columns:\n%s", w, screen)
}
}
}
// Enter on a row that adds acts as a does: an empty column's slot, the
// filters stage with none, the inspector's last filter row
func TestEnterOnAnAddRowAdds(t *testing.T) {
a := mono(pipeline(t, nil, []string{"null"}))
if press(a, terminal.KeyEnter); a.dialog == nil {
t.Fatal("Enter on the empty sources column")
}
press(a, "null", terminal.KeyEnter, terminal.KeyEscape, terminal.KeyRight, terminal.KeyDown, terminal.KeyEnter, "exc", terminal.KeyEnter)
if f := a.pipeline().Flow.Filters; len(f) != 1 || f[0].Type != "exclude" {
t.Fatalf("Enter on filters: %+v", f)
}
lines := a.lines()
a.insp.cursor = len(lines) - 1
press(a, terminal.KeyEnter, terminal.KeyEnter)
if f := a.pipeline().Flow.Filters; len(f) != 2 {
t.Fatalf("Enter on the add row: %+v", f)
}
}
// c changes a source's or sink's type in its place and opens it; on a filter
// or the format it goes to the type row
func TestChangeTypeKeepsThePlace(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"null", "file"}))
a.col = sinksCol
press(a, "c", "tcp", terminal.KeyEnter)
sinks := a.pipeline().PluginSinks
if sinks[0].Type != "tcp" || sinks[0].ID != "tcp" || sinks[1].Type != "file" || !a.inspect || a.lines()[a.insp.cursor].key.Name != "port" {
t.Fatalf("%+v, cursor on %+v", sinks, a.lines()[a.insp.cursor])
}
press(a, terminal.KeyEscape)
a.choose(compose.Node{Role: "format"})
if press(a, "c"); !a.inspect || a.lines()[a.insp.cursor].key.Name != "type" {
t.Fatalf("c on format: %+v", a.lines()[a.insp.cursor])
}
}
// The pipeline bar, above the parts, switches pipelines, adds one under a free
// name, renames one under a name no other has, and deletes one, asking first
// when it has parts
func TestThePipelineBarAddsRenamesDeletes(t *testing.T) {
a := mono(pipeline(t, []string{"null"}, []string{"null"}))
press(a, "k", "a", "empty", terminal.KeyEnter, "k", "a", "empty", terminal.KeyEnter)
names := func() (out []string) {
for _, p := range a.comp.Pipelines {
out = append(out, p.Name)
}
return out
}
if got := names(); !slices.Equal(got, []string{"p", "default", "default_2"}) || a.pi != 2 {
t.Fatalf("added %q, showing %d", got, a.pi)
}
press(a, "k", terminal.KeyEnter, terminal.KeyCtrlU, "p", terminal.KeyEnter)
if _, open := a.dialog.(*form); !open {
t.Fatalf("renamed to a name another has: %q", names())
}
press(a, terminal.KeyCtrlU, "x", terminal.KeyEnter, "h", "h", "d")
if got := names(); !slices.Equal(got, []string{"p", "default", "x"}) || a.dialog == nil {
t.Fatalf("deleted without asking: %q", got)
}
press(a, "y", "l", "d")
if got := names(); !slices.Equal(got, []string{"default"}) || a.dialog != nil {
t.Fatalf("%q %T", got, a.dialog)
}
}
test/acl-test.sh
+20 -20
View File
@@ -96,22 +96,22 @@ section "Rules"
rejects() { ! lw --check --source null --sink "$1" >/dev/null 2>"$LOG/check.err" && grep -q -- "$2" "$LOG/check.err" && echo 1 || echo 0; }
# warns SINK PATTERN: lw --check builds SINK and WARNs PATTERN
warns() { lw --check --source null --sink "$1" 2>&1 >/dev/null | grep ' WARN ' | grep -q -- "$2" && echo 1 || echo 0; }
check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp,host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")"
check "an acl on a dialer is refused" "$(rejects 'tcp_chain,host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')"
check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp,host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')"
check "a deny list alone on a wildcard listener warns" "$(warns 'tcp,port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')"
check "an allow entry of a whole family warns" "$(warns 'tcp,port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')"
check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp:host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")"
check "an acl on a dialer is refused" "$(rejects 'tcp_chain:host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')"
check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp:host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')"
check "a deny list alone on a wildcard listener warns" "$(warns 'tcp:port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')"
check "an allow entry of a whole family warns" "$(warns 'tcp:port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')"
check "a proxy_from entry reaching public addresses warns" \
"$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=203.0.113.0/24' 'proxy_from reaches public addresses')"
"$(warns 'tcp:host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=203.0.113.0/24' 'proxy_from reaches public addresses')"
check "a request rate on a tcp listener is refused" \
"$(rejects 'tcp,host=127.0.0.1,port=15881,acl.requests_per_second_per_client=5' 'applies only to HTTP listeners')"
"$(rejects 'tcp:host=127.0.0.1,port=15881,acl.requests_per_second_per_client=5' 'applies only to HTTP listeners')"
check "a connection cap behind trusted_proxies, which would count the proxy, is refused" \
"$(rejects "http,host=127.0.0.1,port=15881,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.max_connections_per_client=2" 'behind trusted_proxies')"
"$(rejects "http:host=127.0.0.1,port=15881,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.max_connections_per_client=2" 'behind trusted_proxies')"
check "optional proxy_protocol warns that a headerless client passes as the proxy" \
"$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=127.0.0.1' 'passes as the proxy')"
"$(warns 'tcp:host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=127.0.0.1' 'passes as the proxy')"
section "HTTP sink"
spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2"
spawn serve "$BIN" --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2"
wait_port 15881 || abort "the serve preset did not listen" serve
check "a client the allow list names reads the status" "$(get http://127.0.0.1:15881/status | grep -q '"acl":"allow=1 deny=1"' && echo 1 || echo 0)"
get --interface 127.0.0.2 -o /dev/null http://127.0.0.1:15881/status
@@ -127,7 +127,7 @@ section "TCP sink"
# the lines of a finite input, once the clients had time to connect
seq 1 20000 | sed 's/^/flush /' >"$RUN/flush.in"
{ wait_port 15882; sleep 1; cat "$RUN/flush.in"; } |
lw --sink "tcp,host=127.0.0.1,port=15882,acl.allow=127.0.0.1,buffer_size=20000,client_buffer_size=20000" 2>"$LOG/tcp.err" &
lw --sink "tcp:host=127.0.0.1,port=15882,acl.allow=127.0.0.1,buffer_size=20000,client_buffer_size=20000" 2>"$LOG/tcp.err" &
fpid=$!
wait_port 15882
tcp_read 15882 15 >"$RUN/flush.out" &
@@ -141,7 +141,7 @@ check "one it denies gets nothing ($denied bytes) and is logged" \
section "Behind an L7 proxy"
# proxy mode over plaintext loopback: curl is the proxy, its headers name the client
spawn web-proxied "$BIN" --source null --sink "http,host=127.0.0.1,port=15882,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.deny=198.51.100.5,acl.deny=2001:db8::/32,acl.requests_per_second_per_client=1"
spawn web-proxied "$BIN" --source null --sink "http:host=127.0.0.1,port=15882,auth.type=scram,auth.credentials_file=$RUN/users.toml,auth.trusted_proxies=127.0.0.1,acl.deny=198.51.100.5,acl.deny=2001:db8::/32,acl.requests_per_second_per_client=1"
wait_port 15882 || abort "the proxy-mode http sink did not listen" web-proxied
# as CLIENT [PATH [CURL ARGS]]: the status of a request forwarded for CLIENT,
# GET /status by default; there 401 is past the acl, short of a login
@@ -162,9 +162,9 @@ section "Chain sources"
# pin DAEMON: the pin_sha256 a self-signed listener logged
pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; }
received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; }
edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; }
spawn agg-allow "$BIN" --preset "aggregator,listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow"
spawn agg-deny "$BIN" --preset "aggregator,listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny"
edge() { spawn "$1" "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; }
spawn agg-allow "$BIN" --preset "aggregator:listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow"
spawn agg-deny "$BIN" --preset "aggregator:listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny"
wait_port 15884 && wait_port 15885 || abort "an aggregator did not listen" agg-allow
edge edge-allow 127.0.0.1:15884 tcp "$(pin agg-allow)"
edge edge-deny 127.0.0.1:15885 http "$(pin agg-deny)"
@@ -181,16 +181,16 @@ section "Behind a PROXY header"
# connect from; each is up before a proxy shares its port on another address
behind=acl.proxy_protocol=required,acl.proxy_from=127.0.0.1
scram="tls.enabled=true,tls.self_signed=true,auth.type=scram,auth.credentials_file=$RUN/users.toml"
spawn sink-behind "$BIN" --logging.level=info --source null --sink "http,host=127.0.0.1,port=15886,$scram,$behind,acl.deny=198.51.100.2"
spawn tcp-capped "$BIN" --source null --sink "tcp,host=127.0.0.1,port=15889,$behind,acl.max_connections_per_client=2"
spawn sink-behind "$BIN" --logging.level=info --source null --sink "http:host=127.0.0.1,port=15886,$scram,$behind,acl.deny=198.51.100.2"
spawn tcp-capped "$BIN" --source null --sink "tcp:host=127.0.0.1,port=15889,$behind,acl.max_connections_per_client=2"
for t in tcp http; do
spawn "agg-$t-behind" "$BIN" --logging.level=info \
--source "${t}_chain,host=127.0.0.1,port=$([[ $t == tcp ]] && echo 15888 || echo 15883),$scram,$behind" \
--sink "file,directory=$RUN/out/$t-behind,name=aggregate"
--source "${t}_chain:host=127.0.0.1,port=$([[ $t == tcp ]] && echo 15888 || echo 15883),$scram,$behind" \
--sink "file:directory=$RUN/out/$t-behind,name=aggregate"
done
# the tcp sink sends its input once the clients connected
{ wait_until 30 test -e "$RUN/feed"; seq 1 100 | sed 's/^/behind /'; } |
lw --sink "tcp,host=127.0.0.1,port=15887,$behind,acl.deny=198.51.100.2,acl.deny=127.0.0.6" 2>"$LOG/tcp-behind.out" &
lw --sink "tcp:host=127.0.0.1,port=15887,$behind,acl.deny=198.51.100.2,acl.deny=127.0.0.6" 2>"$LOG/tcp-behind.out" &
fpid=$!
for p in 15883 15886 15887 15888 15889; do wait_port $p || abort "a listener behind the proxy did not listen on $p" sink-behind; done
spawn proxy "$RUN/proxy/proxy" 127.0.0.11:15886,127.0.0.1:15886,1,198.51.100.1 \
test/pipe-test.sh
+3 -3
View File
@@ -55,14 +55,14 @@ rc=${PIPESTATUS[1]}
check "reader gone: lw exits on SIGPIPE ($rc)" "$((rc == 141))"
# 7. filters and formats compose with stdin and stdout
check "--filter keeps matching lines" "$(is "$(printf 'x INFO\ny ERROR\n' | lw bare --filter include,patterns=ERROR)" "y ERROR")"
check "--filter keeps matching lines" "$(is "$(printf 'x INFO\ny ERROR\n' | lw bare --filter include:patterns=ERROR)" "y ERROR")"
# 8. control characters pass to a pipe unchanged and are escaped on request
check "control bytes reach a pipe unchanged" "$(is "$(printf 'a\033[31mb\n' | lw bare | od -An -tx1 | tr -d ' ')" 611b5b33316d620a)"
check "escape=always writes them as <hex>" "$(is "$(printf 'a\033[31mb\n' | lw bare --sink console,escape=always)" 'a<1b>[31mb')"
check "escape=always writes them as <hex>" "$(is "$(printf 'a\033[31mb\n' | lw bare --sink console:escape=always)" 'a<1b>[31mb')"
# 9. at the end of input the file sink writes its queue before lw exits
seq 1 5000 | lw bare --sink "file,directory=$RUN/fsink,name=out,buffer_size=10000"
seq 1 5000 | lw bare --sink "file:directory=$RUN/fsink,name=out,buffer_size=10000"
n=$(cat "$RUN"/fsink/out*.log 2>/dev/null | wc -l)
check "file sink holds all 5000 lines at exit ($n)" "$((n == 5000))"
test/preset-test.sh
+74 -18
View File
@@ -1,10 +1,12 @@
#!/usr/bin/env bash
# logwisp preset test: colors, presets, --dump, lw tls, listeners that make
# their certificate (self-signed and pinned, or issued), the serve preset's
# browser viewer, and the network sinks delivering a finite input whole.
# browser viewer, the network sinks delivering a finite input whole, and a
# temporary pipe: one SCRAM user whose password is on no disk.
# One-shot: both modes run the checks.
# Ports: 15890-15896 on 127.0.0.1.
# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only.
# Ports: 15890-15898 on 127.0.0.1.
# Requires: bash 5+, coreutils, curl, openssl; script(1) for the terminal
# checks. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
@@ -14,7 +16,7 @@ e2e_init "$@"
section "Setup"
need curl openssl
ports_free 15890 15891 15892 15893 15894 15895 15896
ports_free 15890 15891 15892 15893 15894 15895 15896 15897 15898
rm -rf "$RUN"
mkdir -p "$RUN/out" "$LOG"
cd "$RUN" || abort "no run directory"
@@ -31,15 +33,15 @@ check "--color paints the level name" "$(is "$(printf 'a ERROR b\n' | lw --color
check "--color never, and auto on a pipe, leave it plain" \
"$(is "$(printf 'a ERROR b\n' | lw --color never)$(printf 'a ERROR b\n' | lw)" "a ERROR ba ERROR b")"
check "LOGWISP_COLOR=always, a sink's own color=never wins" \
"$(is "$(printf 'x WARN\n' | LOGWISP_COLOR=always lw --sink console,color=never)" "x WARN")"
"$(is "$(printf 'x WARN\n' | LOGWISP_COLOR=always lw --sink console:color=never)" "x WARN")"
section "Presets"
seq 1 5000 >"$RUN/seq.txt"
lw --preset pipe <"$RUN/seq.txt" >"$RUN/pipe.out"
check "--preset pipe copies stdin line for line" "$(cmp -s "$RUN/seq.txt" "$RUN/pipe.out" && echo 1 || echo 0)"
out=$(timeout 3 "$BIN" --preset "tail,path=$RUN/edge.log,from=start" </dev/null 2>/dev/null | head -2 | tr '\n' '|')
out=$(timeout 3 "$BIN" --preset "tail:path=$RUN/edge.log,from=start" </dev/null 2>/dev/null | head -2 | tr '\n' '|')
check "--preset tail follows a file" "$(is "$out" "edge line 1|edge line 2|")"
lw --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15890,tls=self,hosts=logs.example" --color --dump >"$RUN/dump.toml"
lw --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15890,tls=self,hosts=logs.example" --color --dump >"$RUN/dump.toml"
lw -c "$RUN/dump.toml" --dump >"$RUN/dump2.toml"
check "--dump prints a file that reads back to the same dump" "$(cmp -s "$RUN/dump.toml" "$RUN/dump2.toml" && echo 1 || echo 0)"
check "lw --check builds the dumped preset" "$(lw --check -c "$RUN/dump.toml" 2>"$LOG/check.err" | grep -q 'configuration ok' && echo 1 || echo 0)"
@@ -65,24 +67,24 @@ add_users "$RUN/users.toml" edge-01
pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; }
received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; }
spawn agg-self "$BIN" --preset "aggregator,listen=127.0.0.1:15893,users=$RUN/users.toml,out=$RUN/out/self"
spawn agg-self "$BIN" --preset "aggregator:listen=127.0.0.1:15893,users=$RUN/users.toml,out=$RUN/out/self"
wait_port 15893 || abort "the self-signed aggregator did not listen" agg-self
PIN=$(pin agg-self)
check "a self-signed aggregator logs its pin" "$(is_set "$PIN")"
spawn edge-pin "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass"
spawn edge-wrong "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=sha256//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,user=edge-01,password_file=$RUN/edge-01.pass"
spawn edge-pin "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass"
spawn edge-wrong "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15893,pin=sha256//AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=,user=edge-01,password_file=$RUN/edge-01.pass"
wait_until 10 eval '(( $(received self) >= 300 ))'
check "an edge pinning it delivers every line over SCRAM ($(received self))" "$(( $(received self) == 300 ))"
wait_until 10 logged edge-wrong 'matches no tls.pin_sha256'
check "an edge with another pin is refused before it logs in" "$(logged edge-wrong 'matches no tls.pin_sha256' && echo 1 || echo 0)"
spawn agg-issuer "$BIN" --preset "aggregator,listen=127.0.0.1:15894,transport=http,tls=issuer,issuer_cert=$RUN/pki/ca.crt,issuer_key=$RUN/pki/ca.key,users=$RUN/users.toml,out=$RUN/out/issuer"
spawn agg-issuer "$BIN" --preset "aggregator:listen=127.0.0.1:15894,transport=http,tls=issuer,issuer_cert=$RUN/pki/ca.crt,issuer_key=$RUN/pki/ca.key,users=$RUN/users.toml,out=$RUN/out/issuer"
wait_port 15894 || abort "the issuing aggregator did not listen" agg-issuer
spawn edge-ca "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass"
spawn edge-ca "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass"
wait_until 10 eval '(( $(received issuer) >= 300 ))'
check "an aggregator signing its own certificate is verified by the CA ($(received issuer))" "$(( $(received issuer) == 300 ))"
spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15895,tls=self"
spawn serve "$BIN" --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15895,tls=self"
wait_port 15895 || abort "the serve preset did not listen" serve
SERVE_PIN=$(pin serve)
check "curl --pinnedpubkey reads a self-signed stream's status" \
@@ -93,7 +95,7 @@ check "a browser at the self-signed root lands on the viewer" \
"$(curl -sLk --pinnedpubkey "$SERVE_PIN" https://127.0.0.1:15895/ | grep -q 'name="logwisp-login" content="none"' && echo 1 || echo 0)"
section "Viewer"
spawn serve-plain "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15896"
spawn serve-plain "$BIN" --preset "serve:path=$RUN/edge.log,listen=127.0.0.1:15896"
wait_port 15896 || abort "the plain serve preset did not listen" serve-plain
root=$(curl -s -o /dev/null -w '%{http_code} %{redirect_url}' http://127.0.0.1:15896/)
check "GET / sends a browser to the viewer, which needs no login over plain http ($root)" \
@@ -103,13 +105,13 @@ section "Flush at exit"
# feed SECONDS PORT: the lines, once a client had time to connect
feed() { wait_port "$2"; sleep "$1"; seq 1 20000 | sed 's/^/flush /'; }
queues='buffer_size=20000,client_buffer_size=20000'
feed 1 15891 | lw --sink "tcp,host=127.0.0.1,port=15891,$queues" 2>"$LOG/flush-tcp.err" &
feed 1 15891 | lw --sink "tcp:host=127.0.0.1,port=15891,$queues" 2>"$LOG/flush-tcp.err" &
fpid=$!
wait_port 15891
n=$(tcp_read 15891 15 | grep -c '^flush ')
wait "$fpid"
check "a tcp client gets all 20000 lines of a finite input ($n)" "$((n == 20000))"
feed 1 15892 | lw --sink "http,host=127.0.0.1,port=15892,$queues" 2>"$LOG/flush-http.err" &
feed 1 15892 | lw --sink "http:host=127.0.0.1,port=15892,$queues" 2>"$LOG/flush-http.err" &
fpid=$!
wait_port 15892
curl -sN --max-time 15 http://127.0.0.1:15892/stream >"$RUN/flush-http.out"
@@ -119,11 +121,65 @@ check "an http stream gets all 20000 lines, then the disconnect event ($n)" \
"$( ((n == 20000)) && tail -3 "$RUN/flush-http.out" | grep -q '^event: disconnect' && echo 1 || echo 0)"
# 1000 lines: the chain sink's queue, which holds them until the link is up
flushed() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c '"flush '; }
seq 1 1000 | sed 's/^/flush /' | lw --preset "edge,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-tcp-chain.err"
seq 1 1000 | sed 's/^/flush /' | lw --preset "edge:to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-tcp-chain.err"
wait_until 5 eval '(( $(flushed self) >= 1000 ))'
check "a tcp_chain edge delivers all 1000 lines before it exits ($(flushed self))" "$(( $(flushed self) == 1000 ))"
seq 1 1000 | sed 's/^/flush /' | lw --preset "edge,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-http-chain.err"
seq 1 1000 | sed 's/^/flush /' | lw --preset "edge:to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-http-chain.err"
wait_until 5 eval '(( $(flushed issuer) >= 1000 ))'
check "an http_chain edge delivers all 1000 lines before it exits ($(flushed issuer))" "$(( $(flushed issuer) == 1000 ))"
section "Temporary pipe"
# One user on both ends, its password through a descriptor; printf is a
# builtin, so the password is in no argv either
PW=$(head -c 18 /dev/urandom | base64 | tr '+/' '-_')
pw() { printf '%s\n' "$1"; }
piped() { cat "$RUN/out/pipe"/aggregate*.log 2>/dev/null | grep -c '^pipe line'; }
piped_line() { cat "$RUN/out/pipe"/aggregate*.log 2>/dev/null | grep -qx "pipe line $1"; }
seq 1 100 | sed 's/^/pipe line /' >"$RUN/pipe.log"
spawn agg-pipe "$BIN" --logging.level=info --preset "aggregator:listen=127.0.0.1:15897,user=pipe,password_file=/dev/fd/3,format=raw,out=$RUN/out/pipe" 3< <(pw "$PW")
AGG_PIPE=${PIDS[-1]}
wait_port 15897 || abort "the pipe aggregator did not listen" agg-pipe
PIPE_PIN=$(pin agg-pipe)
spawn edge-pipe "$BIN" --logging.level=info --preset "edge:path=$RUN/pipe.log,from=start,to=127.0.0.1:15897,pin=$PIPE_PIN,user=pipe,password_file=/dev/fd/3" 3< <(pw "$PW")
EDGE_PIPE=${PIDS[-1]}
spawn edge-pipe-wrong "$BIN" --preset "edge:path=$RUN/pipe.log,from=start,to=127.0.0.1:15897,pin=$PIPE_PIN,user=pipe,password_file=/dev/fd/3" 3< <(pw another-password)
wait_until 10 eval '(( $(piped) >= 100 ))'
check "an aggregator and an edge given one password on /dev/fd/3 deliver every line ($(piped))" "$(( $(piped) == 100 ))"
wait_until 10 logged edge-pipe-wrong 'authentication failed'
check "an edge with another password is refused" "$(logged edge-pipe-wrong 'authentication failed' && echo 1 || echo 0)"
check "the descriptor, read and closed, draws no permission warning" "$(logged agg-pipe 'readable by all users' && echo 0 || echo 1)"
reloaded() { logged agg-pipe 'hot reload completed' && logged edge-pipe 'hot reload completed'; }
kill -HUP "$AGG_PIPE" "$EDGE_PIPE"
wait_until 10 reloaded
check "SIGHUP reloads both, the password kept from the first read" "$(reloaded && echo 1 || echo 0)"
seq 101 150 | sed 's/^/pipe line /' >>"$RUN/pipe.log"
wait_until 10 piped_line 150
check "a second batch is delivered after the reload" "$(piped_line 150 && echo 1 || echo 0)"
leaks=$(grep -rlF -- "$PW" "$RUN"; for p in "$AGG_PIPE" "$EDGE_PIPE"; do tr '\0' '\n' <"/proc/$p/cmdline"; tr '\0' '\n' <"/proc/$p/environ"; done | grep -F -- "$PW")
check "the password is in no file of the run directory, no argv and no environment" "$(is "$leaks" "")"
if command -v script >/dev/null && command -v setsid >/dev/null; then
setsid -w "$BIN" --check --preset aggregator:listen=127.0.0.1:15898,user=gen </dev/null >"$LOG/no-tty.out" 2>&1
check "with no terminal to ask on, a listener's user fails closed" "$(grep -q 'no terminal to ask on' "$LOG/no-tty.out" && echo 1 || echo 0)"
# script(1) is lw's terminal, what lw shows there lands in agg-gen-tty.out;
# not spawn, whose background command would read /dev/null, not the Enter
script -qefc "$(printf '%q ' "$BIN" --preset "aggregator:listen=127.0.0.1:15898,transport=http,user=gen,format=raw,out=$RUN/out/gen")2>$(printf %q "$LOG/agg-gen.out")" /dev/null \
< <(printf '\n') >"$LOG/agg-gen-tty.out" 2>&1 &
PIDS+=($!) DAEMONS+=(agg-gen-tty)
wait_until 10 logged agg-gen-tty 'generated password'
GEN=$(grep -o 'shown once: [A-Z2-7]*' "$LOG/agg-gen-tty.out" | cut -d' ' -f3)
check "Enter alone at a listener's prompt shows a generated password on its terminal" "$(is_set "$GEN")"
wait_port 15898 || abort "the generating aggregator did not listen" agg-gen-tty
GEN_PIN=$(pin agg-gen)
check "and not in lw's log or stderr" "$([[ -n $GEN ]] && ! grep -qF -- "$GEN" "$LOG/agg-gen.out" && echo 1 || echo 0)"
generated() { cat "$RUN/out/gen"/aggregate*.log 2>/dev/null | grep -c '^edge line'; }
spawn edge-gen "$BIN" --preset "edge:path=$RUN/edge.log,from=start,to=127.0.0.1:15898,transport=http,pin=$GEN_PIN,user=gen,password_file=/dev/fd/3" 3< <(pw "$GEN")
wait_until 10 eval '(( $(generated) >= 300 ))'
check "an edge given the generated password delivers every line ($(generated))" "$(( $(generated) == 300 ))"
token=$(pw "$GEN" | script -qefc "$(printf '%q ' "$BIN" auth token --url https://127.0.0.1:15898 -u gen --pin-sha256 "$GEN_PIN")" /dev/null | tr -d '\r' | tail -1)
check "lw auth token asks on the terminal without --password-file" "$([[ $token =~ ^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$ ]] && echo 1 || echo 0)"
else
skip "the terminal checks: no script(1) or setsid(1)"
fi
summary