lixen/logwisp
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.31.0 refinements

Browse Source
This commit is contained in:
lixen committed 2026-10-10 12:01:54 -04:00
1 parent bfcc15613a
commit 6bd5e12a60
52 files changed
+1948 -683
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/lw/auth.go
+7 -7
View File
@@ -35,12 +35,12 @@ var authCommands = []subcommand{
{"remove-user", "--credentials FILE -u NAME",
"Remove a user from a credentials file",
[]string{"credentials", "user"}, defineRemoveUser},
{"token", "--url https://HOST:PORT -u NAME --password-file FILE [--unbound] [TLS flags]",
{"token", "--url https://HOST:PORT -u NAME [--password-file FILE] [--unbound] [TLS flags]",
"Log in to an http sink or http_chain source and print a bearer token",
[]string{"url", "user", "password-file"}, defineToken},
{"stream", "--addr HOST:PORT -u NAME --password-file FILE [TLS flags]",
[]string{"url", "user"}, defineToken},
{"stream", "--addr HOST:PORT -u NAME [--password-file FILE] [TLS flags]",
"Log in to a tcp sink and copy its stream to stdout until interrupted",
[]string{"addr", "user", "password-file"}, defineStream},
[]string{"addr", "user"}, defineStream},
}
func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
@@ -62,7 +62,7 @@ func defineAddUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
var password string
if *passwordFile != "" && !*generate {
password, err = authz.ReadPassword(*passwordFile)
password, err = authz.ReadPassword(*passwordFile, *user, false)
switch {
case err == nil && len(password) < 8:
return fmt.Errorf("the password in %s is shorter than 8 bytes", *passwordFile)
@@ -155,7 +155,7 @@ func defineRemoveUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT or https://[IPV6]:PORT (with --unbound, a proxy's mount URL may carry a path)")
user := fs.String("user", "", "user `name`")
passwordFile := fs.String("password-file", "", "`file` holding the password")
passwordFile := fs.String("password-file", "/dev/tty", "`file` holding the password; a terminal asks")
unbound := fs.Bool("unbound", false, "log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)")
tlsOpts := tlsFlags(fs)
return func(stdout, _ io.Writer) error {
@@ -205,7 +205,7 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
addr := fs.String("addr", "", "tcp sink address, `HOST:PORT` or [IPV6]:PORT")
user := fs.String("user", "", "user `name`")
passwordFile := fs.String("password-file", "", "`file` holding the password")
passwordFile := fs.String("password-file", "/dev/tty", "`file` holding the password; a terminal asks")
tlsOpts := tlsFlags(fs)
return func(stdout, _ io.Writer) error {
host, _, err := net.SplitHostPort(*addr)
cmd/lw/auth_test.go
+1 -1
View File
@@ -63,7 +63,7 @@ func TestAddUserCreatesPrivateFilesWithAMatchingVerifier(t *testing.T) {
if err != nil {
t.Fatal(err)
}
password, err := authz.ReadPassword(pass)
password, err := authz.ReadPassword(pass, "edge-01", false)
if err != nil {
t.Fatal(err)
}
cmd/lw/bootstrap.go
+2
View File
@@ -22,6 +22,7 @@ import (
_ "github.com/lixenwraith/logwisp/internal/sink/tcp"
_ "github.com/lixenwraith/logwisp/internal/sink/tcpchain"
"github.com/lixenwraith/logwisp/internal/authz"
"github.com/lixenwraith/logwisp/internal/config"
"github.com/lixenwraith/logwisp/internal/service"
"github.com/lixenwraith/logwisp/internal/version"
@@ -37,6 +38,7 @@ func bootstrapInitial(ctx context.Context, cfg *config.Config) (*service.Service
if err != nil {
return nil, nil, fmt.Errorf("failed to bootstrap service: %w", err)
}
authz.EndStartup() // a reload reads no descriptor, pipe or terminal
if err := svc.Start(); err != nil {
svc.Shutdown()
cmd/lw/cli.go
+1 -1
View File
@@ -40,7 +40,7 @@ var commands = []command{
{"tls", "A CA and certificates for TLS and mTLS",
"Files are never overwritten; keys are written mode 0600.", tlsCommands},
{"preset", "Show the pipeline a preset makes, as TOML",
"lw --preset NAME,key=value runs one; flags here are its keys.", presetCommands()},
"lw --preset NAME:key=value runs one; flags here are its keys.", presetCommands()},
{"config", "Place the annotated default configuration",
"Files are never overwritten; lw --dump prints the effective configuration.", configCommands},
}
cmd/lw/cli_test.go
+13 -13
View File
@@ -31,9 +31,9 @@ func TestCommandLineGrammar(t *testing.T) {
}{
{[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}},
{[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}},
{[]string{"--source=null", "--sink", "http,port=1"},
invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}}}}},
{[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}},
{[]string{"--source=null", "--sink", "http:port=1"},
invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http:port=1"}}}}},
{[]string{"--preset", "tail:path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail:path=x"}, {Flag: "pipeline", Value: "b"}}}}},
{[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}},
{[]string{"--quiet", "--"}, invocation{load: config.Args{Overrides: []string{"--quiet"}}}},
{[]string{"--quiet", "-h"}, invocation{help: true, load: config.Args{Overrides: []string{"--quiet"}}}},
@@ -41,9 +41,9 @@ func TestCommandLineGrammar(t *testing.T) {
{[]string{"help"}, invocation{help: true}},
{[]string{"-T", "-c", "a.toml"}, invocation{on: map[string]bool{"tui": true}, load: config.Args{File: "a.toml"}}},
{[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}},
{[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"},
{[]string{"-q", "-t", "-V", "-p", "tail:path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"},
invocation{on: map[string]bool{"check": true, "version": true, "dump": true},
load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}},
load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail:path=x"}},
Overrides: []string{"--quiet", "--logging.level=debug", "--logging.file.retention_hours=-1"}}}},
} {
got, err := parseCommandLine(c.argv)
@@ -173,8 +173,8 @@ func TestDumpReadsBack(t *testing.T) {
if err := os.WriteFile(empty, nil, 0o600); err != nil {
t.Fatal(err)
}
args := []string{"--color", "--preset", "serve,listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example",
"--filter", `exclude,patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"}
args := []string{"--color", "--preset", "serve:listen=127.0.0.1:15841,tls=self,hosts=a.example,hosts=b.example",
"--filter", `exclude:patterns=password=\S{8\,64}`, "--sink", "console", "--logging.level=debug"}
cfg := loadTestConfig(t, empty, args...)
var dump bytes.Buffer
if code := dumpConfig(loadTestConfig(t, empty, args...), &dump); code != 0 {
@@ -225,7 +225,7 @@ func TestEveryFormLoadsTheSamePipelines(t *testing.T) {
written := filepath.Join(dir, "written.toml")
if err := os.WriteFile(written, []byte(`[[pipelines]]
name = "written"
flow.rate_limit.rate = 5
flow.rate_limit.entries_per_second = 5
plugin_sources = [{id = "in", type = "null"}]
plugin_sinks = [{id = "out", type = "http", config = {port = 15844, acl = {requests_per_second_per_client = 2}}}]
`), 0o600); err != nil {
@@ -321,11 +321,11 @@ func TestEveryPresetBuilds(t *testing.T) {
}
pin := "sha256//" + strings.Repeat("A", 43) + "="
samples := map[string]string{
"pipe": "pipe,format=txt",
"tail": "tail,path=" + dir,
"serve": "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2",
"edge": "edge,to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass,
"aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1",
"pipe": "pipe:format=txt",
"tail": "tail:path=" + dir,
"serve": "serve:path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2",
"edge": "edge:to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass,
"aggregator": "aggregator:listen=127.0.0.1:15843,user=edge-01,password_file=" + pass + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1",
}
for _, p := range config.Presets() {
sample, ok := samples[p.Name]
cmd/lw/help.go
+19 -14
View File
@@ -81,7 +81,7 @@ key alone means true:
chooses the console
Pipelines (they replace the configuration file's pipelines):
-p, --preset NAME[,KEY=VALUE...]
-p, --preset NAME[:KEY=VALUE,...]
Start a pipeline with a preset: pipe, tail,
serve, edge, aggregator; lw preset NAME -h
lists its keys, lw preset NAME prints it
@@ -89,27 +89,32 @@ Pipelines (they replace the configuration file's pipelines):
pipeline options after it configure; those
before any --pipeline make one named after
its --preset, or "cli" without one
--source SPEC TYPE[,KEY=VALUE...], repeatable
--sink SPEC e.g. http,host=0.0.0.0,port=8080, repeatable
--filter SPEC include|exclude,patterns=RE, repeatable
--format SPEC json|txt|raw[,KEY=VALUE...]
--rate-limit SPEC rate=N[,burst=N,policy=drop|pass]
--source SPEC TYPE[:KEY=VALUE,...], repeatable
--sink SPEC e.g. http:host=0.0.0.0,port=8080, repeatable
--filter SPEC include|exclude:patterns=RE, repeatable
--format SPEC json|txt|raw[:KEY=VALUE,...]
--rate-limit SPEC entries_per_second=N[,burst_entries=N,
policy=drop|pass]
--heartbeat SPEC interval_ms=N[,include_stats=true,...]
A pipeline without --source reads stdin, one without --sink writes stdout.
Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and
'\' escapes ',' '=' '\' in values.
TYPE ends at the first ':'. Keys nest with '.' (tls.cert_file=...), a
repeated key makes a list, and '\' escapes ',' '=' '\' in values.
Examples:
lw < app.log > copy.log Copy, line for line
tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN
journalctl -f | lw --sink http,host=127.0.0.1,port=8080
tail -F app.log | lw --filter include:patterns=ERROR,patterns=WARN
journalctl -f | lw --sink http:host=127.0.0.1,port=8080
Serve stdin live; browse http://127.0.0.1:8080/
lw --source file,directory=/var/log/app,pattern='*.log' --format txt
lw -p tail,path=/var/log/app Follow a directory's files
lw --preset serve,path=/var/log/app,tls=self,users=users.toml
lw --source file:directory=/var/log/app,pattern='*.log' --format txt
lw -p tail:path=/var/log/app Follow a directory's files
lw --preset serve:path=/var/log/app,tls=self,users=users.toml
HTTPS stream, SCRAM logins
lw --preset edge,path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\
lw --preset edge:path=/var/log/app,to=agg:9000,ca=ca.crt,user=edge-01,\
password_file=edge-01.pass Forward to an aggregator
lw -p aggregator:user=pipe,format=raw A temporary pipe's end: Enter
at the prompt draws a password
cmd | lw -p edge:to=HOST:9000,pin=PIN,user=pipe
Its source, asked the password
Environment:
LOGWISP_<KEY> A configuration key, '.' as '_', uppercase:
cmd/lw/reload_test.go
+23 -3
View File
@@ -11,6 +11,7 @@ import (
"testing"
"time"
"github.com/lixenwraith/logwisp/internal/authz"
"github.com/lixenwraith/logwisp/internal/plugin"
"github.com/lixenwraith/logwisp/internal/session"
"github.com/lixenwraith/logwisp/internal/testutil"
@@ -60,6 +61,25 @@ func TestShippedConfigurationBuildsWithCheckedPluginDecoder(t *testing.T) {
svc.Shutdown() // A later reload failure or process exit can close it again.
}
// Once the first service is built a reload reads no descriptor, pipe or
// terminal, which a console source may be reading
func TestStartupEndsWithTheFirstService(t *testing.T) {
testLogger(t)
path := filepath.Join(t.TempDir(), "empty.toml")
testutil.WriteFile(t, path, "")
svc, cancel, err := bootstrapInitial(context.Background(), loadTestConfig(t, path, "--source", "null", "--sink", "null"))
if err != nil {
t.Fatal(err)
}
defer svc.Shutdown()
if cancel != nil {
defer cancel()
}
if _, err := authz.ReadPassword(os.DevNull, "late", false); err == nil || !strings.Contains(err.Error(), "only at startup") {
t.Fatalf("a device after the first service: %v", err)
}
}
func TestInvalidReloadLeavesStatusReporterRunning(t *testing.T) {
testLogger(t)
path := filepath.Join(t.TempDir(), "config.toml")
@@ -124,8 +144,8 @@ func TestCheckBuildsWithoutStarting(t *testing.T) {
dir := t.TempDir()
path, out := filepath.Join(dir, "empty.toml"), filepath.Join(dir, "out")
testutil.WriteFile(t, path, "")
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15862",
"--sink", "file,name=check,directory="+out)); code != 0 {
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http:host=127.0.0.1,port=15862",
"--sink", "file:name=check,directory="+out)); code != 0 {
t.Fatalf("valid configuration: exit %d", code)
}
ln, err := net.Listen("tcp4", "127.0.0.1:15862")
@@ -138,7 +158,7 @@ func TestCheckBuildsWithoutStarting(t *testing.T) {
}
missing := filepath.Join(dir, "missing")
if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink",
"http,port=15862,tls.enabled=true,tls.cert_file="+missing+",tls.key_file="+missing)); code != 1 {
"http:port=15862,tls.enabled=true,tls.cert_file="+missing+",tls.key_file="+missing)); code != 1 {
t.Fatalf("listener TLS without its certificate file: exit %d, want 1", code)
}
}