From 4eba6fc834cf3134904b554639fe11a4f19ee4d7b0238077e76bf0fa243b7c22 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sat, 3 Oct 2026 21:57:41 -0400 Subject: [PATCH] v0.22.1 cli flag and behavior, and test output improvement --- .github/workflows/test.yml | 12 +- Dockerfile | 2 +- Makefile | 6 +- README.md | 54 ++++--- cmd/lw/auth.go | 8 +- cmd/lw/auth_test.go | 2 +- cmd/lw/bootstrap.go | 39 ++--- cmd/lw/help.go | 23 ++- cmd/lw/main.go | 36 ++++- cmd/lw/reload_test.go | 6 +- cmd/lw/status.go | 2 +- cmd/lw/test_helpers_test.go | 4 +- config/logwisp.toml | 11 +- deploy/package/arch/PKGBUILD | 4 +- deploy/package/freebsd/Makefile | 12 +- doc/README.md | 24 ++- doc/architecture.md | 29 ++-- doc/cli.md | 133 ++++++++++++---- doc/configuration.md | 21 ++- doc/formatters.md | 16 +- doc/installation.md | 21 ++- doc/lw.1 | 152 ++++++++++++++++--- doc/operations.md | 51 +++++-- doc/security.md | 6 + doc/sinks.md | 35 ++++- doc/sources.md | 23 ++- doc/todo.md | 98 +++++++----- go.mod | 3 +- go.sum | 2 + internal/authz/authz.go | 6 +- internal/authz/authz_test.go | 4 +- internal/authz/proxy_test.go | 4 +- internal/authz/scram.go | 8 +- internal/authz/scram_test.go | 6 +- internal/chain/proto.go | 3 +- internal/config/config.go | 1 + internal/config/loader.go | 72 +++------ internal/config/loader_test.go | 39 ++++- internal/config/spec.go | 12 +- internal/config/spec_test.go | 24 ++- internal/config/test_helpers_test.go | 2 +- internal/config/validate.go | 10 ++ internal/core/capability.go | 3 + internal/core/const_test.go | 2 +- internal/filter/chain.go | 4 +- internal/filter/filter.go | 4 +- internal/flow/flow.go | 8 +- internal/flow/heartbeat.go | 6 +- internal/flow/heartbeat_test.go | 4 +- internal/flow/ratelimiter.go | 6 +- internal/format/adapter.go | 18 ++- internal/format/adapter_test.go | 34 +++++ internal/format/format.go | 4 +- internal/pipeline/pipeline.go | 95 ++++++++---- internal/pipeline/pipeline_test.go | 132 ++++++++++++++++ internal/pipeline/registry.go | 8 +- internal/plugin/factory.go | 8 +- internal/sanitize/sanitize.go | 70 --------- internal/service/service.go | 23 ++- internal/session/session.go | 2 +- internal/sink/console/console.go | 158 ++++++++++++++----- internal/sink/console/console_test.go | 25 +++ internal/sink/file/file.go | 56 ++++--- internal/sink/http/http.go | 18 +-- internal/sink/http/http_test.go | 6 +- internal/sink/http/test_helpers_test.go | 6 +- internal/sink/http/web.go | 6 +- internal/sink/httpchain/httpchain.go | 16 +- internal/sink/httpchain/httpchain_test.go | 14 +- internal/sink/null/null.go | 10 +- internal/sink/sink.go | 2 +- internal/sink/tcp/tcp.go | 14 +- internal/sink/tcp/tcp_test.go | 4 +- internal/sink/tcpchain/tcpchain.go | 16 +- internal/sink/tcpchain/tcpchain_test.go | 4 +- internal/source/console/console.go | 177 +++++++++++++++------- internal/source/file/file.go | 10 +- internal/source/file/file_test.go | 4 +- internal/source/file/file_watcher.go | 7 +- internal/source/httpchain/httpchain.go | 16 +- internal/source/null/null.go | 10 +- internal/source/random/random.go | 10 +- internal/source/source.go | 2 +- internal/source/tcpchain/tcpchain.go | 16 +- internal/tlsx/tlsx.go | 2 +- internal/tlsx/tlsx_test.go | 2 +- internal/version/version.go | 28 +++- test/lib.sh | 22 +-- test/pipe-test.sh | 74 +++++++++ 89 files changed, 1531 insertions(+), 661 deletions(-) create mode 100644 internal/format/adapter_test.go create mode 100644 internal/pipeline/pipeline_test.go delete mode 100644 internal/sanitize/sanitize.go create mode 100644 internal/sink/console/console_test.go create mode 100755 test/pipe-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d3c188..90ecadd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,11 +7,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 with: go-version-file: go.mod - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: node-version: 22 - run: go test -timeout 60s -race ./... @@ -28,11 +28,11 @@ jobs: env: FORCE_COLOR: "1" steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 with: go-version-file: go.mod - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v5 with: node-version: 22 # "chromium" fetches the headless shell and full Chromium, which the diff --git a/Dockerfile b/Dockerfile index fc976e2..97647d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ ARG BUILD_TIME=unknown # REVISION identifies the commit: the context has no .git, so no VCS stamping. RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ go build -trimpath -buildvcs=false \ - -ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION} -X logwisp/internal/version.BuildTime=${BUILD_TIME}" \ + -ldflags="-s -w -X github.com/lixenwraith/logwisp/internal/version.Version=${VERSION} -X github.com/lixenwraith/logwisp/internal/version.GitCommit=${REVISION} -X github.com/lixenwraith/logwisp/internal/version.BuildTime=${BUILD_TIME}" \ -o /out/lw ./cmd/lw FROM scratch diff --git a/Makefile b/Makefile index c9201c5..9742d41 100644 --- a/Makefile +++ b/Makefile @@ -18,9 +18,9 @@ BUILD_TIME ?= $(BUILD_TIME_DEFAULT) # The image stamps the commit time, so a rebuild of one commit stays cached IMAGE_BUILD_TIME_DEFAULT != TZ=UTC git log -1 --format=%cd --date=format-local:%Y-%m-%d_%H:%M:%S 2>/dev/null || echo unknown IMAGE_BUILD_TIME ?= $(IMAGE_BUILD_TIME_DEFAULT) -VERSION_LDFLAGS = -X 'logwisp/internal/version.Version=$(VERSION)' \ - -X 'logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \ - -X 'logwisp/internal/version.BuildTime=$(BUILD_TIME)' +VERSION_LDFLAGS = -X 'github.com/lixenwraith/logwisp/internal/version.Version=$(VERSION)' \ + -X 'github.com/lixenwraith/logwisp/internal/version.GitCommit=$(GIT_COMMIT)' \ + -X 'github.com/lixenwraith/logwisp/internal/version.BuildTime=$(BUILD_TIME)' # Not LDFLAGS/GOFLAGS: packagers export those for the C linker and go itself. GO_BUILDFLAGS ?= GO_LDFLAGS ?= diff --git a/README.md b/README.md index 7232252..900d8c9 100644 --- a/README.md +++ b/README.md @@ -19,23 +19,28 @@ streams, or downstream LogWisp nodes. ### Pipeline +- **A Unix filter out of the box**: with no configuration, `lw` copies stdin + to stdout line for line and exits at the end of input; flags add filters, + formats, sources and sinks - **Independent pipelines**, each `sources → flow → sinks`, running concurrently in one process - **Fan-in and fan-out**: many sources and many sinks per pipeline -- **Never blocks**: a stalled sink drops its own events and is counted, rather - than stalling the pipeline or its sibling sinks +- **Isolated sinks**: a stalled sink drops and counts its own events rather + than stalling the pipeline or its sibling sinks; only the console sink + waits, so a filter loses no line to a slow reader - **Hot reload** via `SIGHUP`/`SIGUSR1` or a config file watch, with the new configuration validated before the old service is torn down ### Inputs `file` (directory tail with rotation detection and JSON line parsing), -`console` (stdin), `random` (synthetic generator), `null`, and the chain ingest -listeners `tcp_chain` and `http_chain`. +`console` (stdin, one reader per process), `random` (synthetic generator), +`null`, and the chain ingest listeners `tcp_chain` and `http_chain`. ### Outputs -`console`, `file` (rotating with retention), `http` (Server-Sent Events plus a +`console` (control characters escaped on a terminal), `file` (rotating with +retention), `http` (Server-Sent Events plus a JSON status endpoint), `tcp` (broadcast server), `null`, and the chain forwarders `tcp_chain` and `http_chain`. @@ -99,8 +104,8 @@ authentication designs for the rationale and what is deliberately left out. - [Security](doc/security.md): TLS, mTLS, peer authorization, threat model - Design notes: [mTLS](doc/mtls-auth-plan.md) and [password](doc/scram-auth-plan.md) authentication -- [CLI](doc/cli.md): flags, pipeline specifications, signals, exit codes, - `lw auth`; also the `lw(1)` manual, [`doc/lw.1`](doc/lw.1) +- [CLI](doc/cli.md): filter use, flags, pipeline specifications, signals, + exit codes, `lw auth`; also the `lw(1)` manual, [`doc/lw.1`](doc/lw.1) - [Operations](doc/operations.md): running, monitoring, tuning, troubleshooting - [To Do](doc/todo.md): planned work in priority order: network access @@ -114,16 +119,30 @@ A fully annotated configuration covering every option ships as ```bash make build # plain `make` lists the targets -./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \ - --format json,sanitizer_policy=json --sink console ``` -Or as a configuration file: +Without a configuration file, `lw` is a filter: stdin to stdout, line for +line, exiting 0 at the end of input. A pipeline without `--source` reads stdin, +one without `--sink` writes stdout: + +```bash +./bin/lw < app.log > copy.log +tail -F app.log | ./bin/lw --filter include,patterns=ERROR,patterns=WARN + +# stdin as a live SSE stream; the http sink binds 0.0.0.0 unless told +journalctl -f | ./bin/lw --sink http,host=127.0.0.1,port=8080 + +# tail a directory instead of stdin +./bin/lw --source 'file,directory=/var/log/myapp,pattern=*.log' \ + --format json,sanitizer_policy=json > all.json +``` + +As a service, a configuration file holds the pipelines: ```toml # logwisp.toml [[pipelines]] -name = "default" +name = "app" [pipelines.flow.format] type = "json" @@ -137,19 +156,18 @@ directory = "/var/log/myapp" pattern = "*.log" [[pipelines.plugin_sinks]] -id = "stdout" -type = "console" +id = "sse" +type = "http" [pipelines.plugin_sinks.config] -target = "stdout" +host = "127.0.0.1" +port = 8080 ``` ```bash -./bin/lw -c logwisp.toml +./bin/lw -c logwisp.toml # until SIGINT or SIGTERM; logs to stderr +curl -N http://127.0.0.1:8080/stream # from another shell ``` -Running with neither starts a self-demonstrating pipeline: a synthetic -generator writing JSON to stdout. - In a container, read-only and without capabilities: ```bash diff --git a/cmd/lw/auth.go b/cmd/lw/auth.go index 5125426..6e86e89 100644 --- a/cmd/lw/auth.go +++ b/cmd/lw/auth.go @@ -18,10 +18,10 @@ import ( "strings" "syscall" - "logwisp/internal/authz" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/tlsx" "github.com/lixenwraith/auth" ) diff --git a/cmd/lw/auth_test.go b/cmd/lw/auth_test.go index 57e281c..5151635 100644 --- a/cmd/lw/auth_test.go +++ b/cmd/lw/auth_test.go @@ -8,7 +8,7 @@ import ( "strings" "testing" - "logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/authz" "github.com/lixenwraith/auth" ) diff --git a/cmd/lw/bootstrap.go b/cmd/lw/bootstrap.go index 0a1841b..0ead3e9 100644 --- a/cmd/lw/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -2,27 +2,28 @@ package main import ( "context" + "errors" "fmt" "os" - _ "logwisp/internal/source/console" - _ "logwisp/internal/source/file" - _ "logwisp/internal/source/httpchain" - _ "logwisp/internal/source/null" - _ "logwisp/internal/source/random" - _ "logwisp/internal/source/tcpchain" + _ "github.com/lixenwraith/logwisp/internal/source/console" + _ "github.com/lixenwraith/logwisp/internal/source/file" + _ "github.com/lixenwraith/logwisp/internal/source/httpchain" + _ "github.com/lixenwraith/logwisp/internal/source/null" + _ "github.com/lixenwraith/logwisp/internal/source/random" + _ "github.com/lixenwraith/logwisp/internal/source/tcpchain" - _ "logwisp/internal/sink/console" - _ "logwisp/internal/sink/file" - _ "logwisp/internal/sink/http" - _ "logwisp/internal/sink/httpchain" - _ "logwisp/internal/sink/null" - _ "logwisp/internal/sink/tcp" - _ "logwisp/internal/sink/tcpchain" + _ "github.com/lixenwraith/logwisp/internal/sink/console" + _ "github.com/lixenwraith/logwisp/internal/sink/file" + _ "github.com/lixenwraith/logwisp/internal/sink/http" + _ "github.com/lixenwraith/logwisp/internal/sink/httpchain" + _ "github.com/lixenwraith/logwisp/internal/sink/null" + _ "github.com/lixenwraith/logwisp/internal/sink/tcp" + _ "github.com/lixenwraith/logwisp/internal/sink/tcpchain" - "logwisp/internal/config" - "logwisp/internal/service" - "logwisp/internal/version" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/service" + "github.com/lixenwraith/logwisp/internal/version" "github.com/lixenwraith/log" "github.com/lixenwraith/log/sanitizer" @@ -48,6 +49,10 @@ func bootstrapInitial(ctx context.Context, cfg *config.Config) (*service.Service return svc, statusCancel, nil } +// errServiceStopped is a reload that failed after the old service stopped: +// no pipeline runs until a later reload succeeds. +var errServiceStopped = errors.New("old service stopped, new one failed to start") + // handleReload orchestrates the entire hot-reload process including status reporter lifecycle func handleReload(ctx context.Context, newCfg *config.Config, oldSvc *service.Service, statusCancel context.CancelFunc) (*service.Service, *config.Config, context.CancelFunc, error) { logger.Info("msg", "Starting configuration hot reload") @@ -78,7 +83,7 @@ func handleReload(ctx context.Context, newCfg *config.Config, oldSvc *service.Se if err := newService.Start(); err != nil { newService.Shutdown() logger.Error("msg", "Failed to start new service pipelines after reload. The application may be in a non-functional state.", "error", err) - return nil, nil, nil, fmt.Errorf("failed to start new service: %w", err) + return nil, nil, nil, fmt.Errorf("%w: %w", errServiceStopped, err) } // Manage status reporter lifecycle diff --git a/cmd/lw/help.go b/cmd/lw/help.go index 5e6eb67..a9a4b61 100644 --- a/cmd/lw/help.go +++ b/cmd/lw/help.go @@ -2,15 +2,18 @@ package main import ( "fmt" - "logwisp/internal/version" "os" + + "github.com/lixenwraith/logwisp/internal/version" ) // helpText is the CLI usage reference. Scalar flags map 1:1 to TOML config paths. const helpText = `LogWisp %s - log collection, processing, and distribution Usage: - lw [options] + lw [options] [< input] No file, no pipeline options: stdin to stdout, + line for line, like cat; exits at end of input + lw -c [options] Run the file's pipelines lw --check [options] Build every pipeline and plugin, report, exit lw help | -h | --help lw --version @@ -25,10 +28,12 @@ Scalar configuration keys are settable as flags using their TOML paths: Common options: -c, --config Configuration file (default: ./logwisp.toml) --quiet Suppress console output - --status_reporter= Periodic status logging (default: true) + --status_reporter= Periodic status logging (default: true with + a configuration file) --auto_reload= Config hot reload on file change (default: false) -Logging: +Logging (lw's own log goes to stderr, at info with a configuration file and +at warn without one): --logging.output= file|stdout|stderr|split|all|none --logging.level= debug|info|warn|error --logging.file.directory= @@ -41,9 +46,17 @@ Pipelines (replace the file's pipelines; see doc/cli.md): --format json|txt|raw[,key=value...] --rate-limit rate=N[,burst=N,policy=drop|pass] --heartbeat interval_ms=N[,include_stats=true,...] + A pipeline without --source reads stdin, one without --sink writes stdout. Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and '\' escapes ',' '=' '\' in values. -- ends option parsing. +Examples: + lw < app.log > copy.log Copy, line for line + tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN + journalctl -f | lw --sink http,host=127.0.0.1,port=8080 + Serve stdin as a live stream + lw --source file,directory=/var/log/app,pattern='*.log' --format txt + Environment: LOGWISP_ Config path, '.' -> '_', uppercase e.g. LOGWISP_LOGGING_LEVEL=debug @@ -59,7 +72,7 @@ Signals: SIGHUP, SIGUSR1 Reload configuration Exit codes: - 0 success + 0 success, including the end of input 1 general error 2 configuration file not found ` diff --git a/cmd/lw/main.go b/cmd/lw/main.go index 694158a..0accc10 100644 --- a/cmd/lw/main.go +++ b/cmd/lw/main.go @@ -9,11 +9,12 @@ import ( "reflect" "syscall" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/version" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/version" "github.com/lixenwraith/log" + "golang.org/x/term" ) var logger *log.Logger @@ -88,6 +89,10 @@ func main() { os.Exit(1) } + if !cfg.Quiet && readsStdin(cfg) && term.IsTerminal(int(os.Stdin.Fd())) { + fmt.Fprintln(os.Stderr, "lw: reading standard input; Ctrl-D ends it (lw --help for usage)") + } + // --- 3. Setup signals and shutdown --- sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGUSR1) @@ -112,6 +117,9 @@ func main() { logger.Info("msg", "Shutdown complete") }() + // Closed when every pipeline's input ended (stdin); nil while none runs + done := svc.Done() + reload := func(fromDisk bool) { var next *config.Config var err error @@ -130,8 +138,12 @@ func main() { return } newSvc, newCfg, newStatusCancel, err := handleReload(ctx, next, svc, statusReporterCancel) - if err == nil { + switch { + case err == nil: svc, cfg, statusReporterCancel = newSvc, newCfg, newStatusCancel + done = svc.Done() + case errors.Is(err, errServiceStopped): + done = nil // the stopped service's pipelines did not reach the end of input } } @@ -158,6 +170,10 @@ func main() { reload(false) } + case <-done: + logger.Info("msg", "Every pipeline finished: its input ended") + return + case <-ctx.Done(): return // Exit the loop and trigger deferred shutdown } @@ -172,3 +188,15 @@ func shutdownLogger() { } } } + +// readsStdin reports a console source: typed into, it waits for the keyboard +func readsStdin(cfg *config.Config) bool { + for _, p := range cfg.Pipelines { + for _, src := range p.PluginSources { + if src.Type == "console" { + return true + } + } + } + return false +} diff --git a/cmd/lw/reload_test.go b/cmd/lw/reload_test.go index bf86dec..43b6439 100644 --- a/cmd/lw/reload_test.go +++ b/cmd/lw/reload_test.go @@ -11,9 +11,9 @@ import ( "testing" "time" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/testutil" lconfig "github.com/lixenwraith/config" ) diff --git a/cmd/lw/status.go b/cmd/lw/status.go index 54f2ccb..adeba54 100644 --- a/cmd/lw/status.go +++ b/cmd/lw/status.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "logwisp/internal/service" + "github.com/lixenwraith/logwisp/internal/service" ) // startStatusReporter starts a new status reporter for a service and returns its cancel function. diff --git a/cmd/lw/test_helpers_test.go b/cmd/lw/test_helpers_test.go index a271ec0..d4d7507 100644 --- a/cmd/lw/test_helpers_test.go +++ b/cmd/lw/test_helpers_test.go @@ -3,8 +3,8 @@ package main import ( "testing" - "logwisp/internal/config" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/log" ) diff --git a/config/logwisp.toml b/config/logwisp.toml index 414a6f2..2c0db26 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -23,8 +23,8 @@ ### Global Settings ############################################################################### -quiet = false # Suppress console output -status_reporter = true # Periodic status logging (30s, DEBUG level) +quiet = false # No log of lw's own; pipeline output still flows +status_reporter = true # Periodic status logging (30s, DEBUG level); false without a file auto_reload = false # Watch file; read at startup (signals always reread it) ############################################################################### @@ -32,8 +32,8 @@ auto_reload = false # Watch file; read at startup (signals a ############################################################################### [logging] -output = "stdout" # file|stdout|stderr|split|all|none -level = "info" # debug|info|warn|error +output = "stderr" # file|stdout|stderr|split|all|none; stdout is the console sink's +level = "info" # debug|info|warn|error; warn without a file # format = "txt" # raw|txt|json # sanitization = "" # raw|json|txt|shell (empty = logger default) @@ -168,7 +168,7 @@ check_interval_ms = 100 # Directory rescan interval (min 10) raw = false # Keep the whole line as the message from = "end" # "end" or "start" of a newly discovered file -## Console source (stdin, single instance per pipeline) +## Console source (stdin; one in the whole configuration, ends lw's pipeline at end of input) # [[pipelines.plugin_sources]] # id = "console_in" # type = "console" @@ -256,6 +256,7 @@ type = "console" [pipelines.plugin_sinks.config] target = "stdout" # stdout|stderr ("split" NOT supported) # buffer_size = 1000 +# escape = "auto" # auto|always|never: control characters as , auto on a terminal ## File sink (rotating) # [[pipelines.plugin_sinks]] diff --git a/deploy/package/arch/PKGBUILD b/deploy/package/arch/PKGBUILD index b939baa..17e7cfd 100644 --- a/deploy/package/arch/PKGBUILD +++ b/deploy/package/arch/PKGBUILD @@ -1,6 +1,6 @@ # Maintainer: -# Skeleton: source and checksum wait for a tagged release; run namcap and -# makepkg --printsrcinfo > .SRCINFO before publishing to the AUR. +# Skeleton: pkgver, sha256sums and the maintainer wait for a tagged release; +# doc/todo.md lists the steps to publish it to the AUR. pkgname=logwisp pkgver=0.21.0 diff --git a/deploy/package/freebsd/Makefile b/deploy/package/freebsd/Makefile index 04991ed..9bcd957 100644 --- a/deploy/package/freebsd/Makefile +++ b/deploy/package/freebsd/Makefile @@ -1,7 +1,7 @@ -# Skeleton for sysutils/logwisp. GO_MODULE fetches the module by its canonical -# path, so it waits for the module rename to github.com/lixenwraith/logwisp -# (the -X paths below follow it), a tagged release, a maintainer, and a -# UIDs/GIDs entry for logwisp. files/logwisp.in links to ../logwisp.rc: cp -L. +# Skeleton for sysutils/logwisp: GO_MODULE fetches the tagged module from the +# Go proxy, so it waits for a release tag, a maintainer and a UIDs/GIDs entry +# for logwisp (doc/todo.md). files/logwisp.in is a symlink to +# deploy/package/logwisp.rc: copy the port with cp -RL. PORTNAME= logwisp DISTVERSIONPREFIX= v @@ -22,8 +22,8 @@ SUB_LIST= BINDIR=${PREFIX}/bin GO_MODULE= github.com/lixenwraith/logwisp GO_TARGET= ./cmd/lw GO_BUILDFLAGS= -ldflags="-s -w \ - -X logwisp/internal/version.Version=${DISTVERSION} \ - -X logwisp/internal/version.GitCommit=${DISTVERSIONFULL}" + -X github.com/lixenwraith/logwisp/internal/version.Version=${DISTVERSION} \ + -X github.com/lixenwraith/logwisp/internal/version.GitCommit=${DISTVERSIONFULL}" USERS= logwisp GROUPS= logwisp diff --git a/doc/README.md b/doc/README.md index 2a8426d..0d08da0 100644 --- a/doc/README.md +++ b/doc/README.md @@ -41,21 +41,24 @@ A fully annotated configuration covering every option lives at ### Pipeline +- A Unix filter without configuration: stdin to stdout, line for line - Independent named pipelines, each `sources → flow → sinks` - Fan-in (many sources per pipeline) and fan-out (many sinks per pipeline) -- Non-blocking sink dispatch: a stalled sink drops its own events and never - stalls the pipeline or its sibling sinks +- Isolated sinks: a stalled sink drops its own events rather than stall the + pipeline; only the console sink waits, so a filter loses no line - Hot reload of pipeline configuration via `SIGHUP`/`SIGUSR1` or a file watch ### Inputs -`file` (directory tail with rotation detection), `console` (stdin), +`file` (directory tail with rotation detection), `console` (stdin, one +reader per process), `random` (synthetic generator), `null`, and the chain ingest listeners `tcp_chain` and `http_chain`. ### Outputs -`console`, `file` (rotating), `http` (Server-Sent Events plus a JSON status +`console` (control characters escaped on a terminal), `file` (rotating), +`http` (Server-Sent Events plus a JSON status endpoint), `tcp` (broadcast server), `null`, and the chain forwarders `tcp_chain` and `http_chain`. @@ -83,9 +86,20 @@ See [Security](security.md) for what each layer does and does not give you. ## Quick Start +Without a configuration file `lw` is a filter, exiting 0 at the end of input; +a pipeline without `--source` reads stdin and one without `--sink` writes +stdout: + +```bash +lw < app.log > copy.log +tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN +``` + +As a service, a file holds the pipelines: + ```toml [[pipelines]] -name = "default" +name = "app" [pipelines.flow.format] type = "json" diff --git a/doc/architecture.md b/doc/architecture.md index 9316595..c9255d9 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -64,14 +64,16 @@ registry rejects a second instance of any such type. ### Entry lifecycle 1. **Source** produces a `core.LogEntry` and publishes it to every subscriber - channel it has handed out. Publication is non-blocking: a full subscriber - channel increments the source's `dropped_entries` counter. + channel it has handed out; the pipeline subscribes before the source + starts. Publication is non-blocking: a full subscriber channel increments + the source's `dropped_entries` counter. The console source alone waits, + since stdin can simply be read later. 2. **Flow** applies, in order: rate limit → filter chain → formatter. A drop at any stage ends the entry's life and increments `flow.total_dropped`. 3. The formatter output becomes a `core.TransportEvent`, which carries both the formatted `Payload` and the original structured `Entry`. 4. **Dispatch** sends the event to every sink's input channel with a - non-blocking send. + non-blocking send, except to a sink declaring `CapBackpressure`. `LogEntry` fields: @@ -90,15 +92,17 @@ structured entry rather than shipping whatever text the local formatter chose. ### Back-pressure and drops -There is exactly one drop policy and it is not configurable: **never block**. -Each stage drops and counts instead of waiting: +The drop policy is not configurable: **never block**, except where waiting +loses nothing. Each stage drops and counts instead of waiting: - Source → subscriber: a full channel drops the entry; source - `dropped_entries`. + `dropped_entries`. The console source waits instead and reads stdin later. - Flow: the rate limiter, a filter rejection or a format error drops it; `flow.total_dropped`. - Pipeline → sink: a full sink input drops it for that sink only; pipeline - `total_dropped_by_sink`. + `total_dropped_by_sink`. A sink declaring `core.CapBackpressure`, the console + sink, makes dispatch wait instead, so a filter loses no line to a slow + reader and its pipeline runs at the reader's pace; `Stop` ends the wait. - TCP/HTTP sink → client queue: a full queue drops it for that client only; sink `dropped_writes`. @@ -126,9 +130,16 @@ on a non-retryable response or on shutdown (`dropped_batches`). `Pipeline.Stop` is deliberately ordered so in-flight data drains: -1. Stop all sources concurrently; each closes its subscriber channels. +1. Release dispatch waits on backpressure sinks; stop all sources + concurrently, each closing its subscriber channels. 2. Wait for the run loop, which ends when every subscription channel closes. -3. Stop all sinks concurrently. +3. Stop all sinks concurrently; console and file sinks write their queue + first. Then cancel the pipeline context. + +A finite source ends on its own: the console source at the end of stdin. The +run loop then returns and the pipeline's `Finished` channel closes; once every +pipeline has finished, the service's `Done` closes and `main` shuts down as on +`SIGTERM`, exiting 0. ## Network Architecture diff --git a/doc/cli.md b/doc/cli.md index 5e5baa8..32513a2 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -1,13 +1,19 @@ # Command Line Interface ``` -lw [options] +lw [options] [< input] +lw -c [options] lw --check [options] lw auth [flags] lw help | -h | --help lw --version ``` +Without a configuration file or pipeline options, lw is a filter: stdin to +stdout, line for line, until the end of input ([Built-in +Defaults](#built-in-defaults)). With a file it runs the file's pipelines as a +service. + `lw auth` manages SCRAM credentials and logs in to `scram` listeners; see [below](#lw-auth). There is no certificate-generation command: use `openssl` or your PKI tooling — see [Security](security.md#enabling-mtls). @@ -25,12 +31,15 @@ pipelines have [their own flags](#pipelines): ### Common -- `-c `, `--config `, `--config=`: the configuration file, - default `./logwisp.toml` +- `-c `, `--config `, `--config=`: the configuration file; + unnamed, `~/.config/logwisp/logwisp.toml` if it exists, else + `./logwisp.toml` ([Configuration](configuration.md#file-location)) - `--check`: build every pipeline and plugin, bind nothing, report and exit; see [Usage Patterns](#usage-patterns) -- `--quiet`: suppress all application output, default `false` -- `--status_reporter=`: periodic status logging, default `true` +- `--quiet`: silence lw's own log and notices, default `false`; pipeline + output still flows +- `--status_reporter=`: periodic status logging, default `true` with a + configuration file, `false` without - `--auto_reload=`: reload when the configuration file changes, default `false` - `--version`: print the version and exit @@ -41,8 +50,10 @@ path returns an error, and `--` ends option parsing. ### Logging -- `--logging.output`: `file`, `stdout`, `stderr`, `split`, `all` or `none` -- `--logging.level`: `debug`, `info`, `warn` or `error` +- `--logging.output`: `file`, `stdout`, `stderr`, `split`, `all` or `none`, + default `stderr`, so stdout carries only data +- `--logging.level`: `debug`, `info`, `warn` or `error`, default `info` with a + configuration file, `warn` without - `--logging.format`: `raw`, `txt` or `json` - `--logging.sanitization`: `raw`, `json`, `txt` or `shell` - `--logging.file.directory`: a path @@ -63,6 +74,10 @@ pipelines and the built-in default; every other key keeps its precedence. - `--source SPEC`, `--sink SPEC` and `--filter SPEC` add a stage, repeatable - `--format SPEC`, `--rate-limit SPEC` and `--heartbeat SPEC`, once per pipeline - both `--flag SPEC` and `--flag=SPEC` work; `--` ends options +- a pipeline without `--source` reads stdin (console source `stdin`), one + without `--sink` writes stdout (console sink `stdout`) +- stdin has one reader, so the whole configuration holds at most one console + source: `pipeline "b": a console source already reads stdin in pipeline "a"` A SPEC is a comma-separated list: @@ -96,7 +111,7 @@ A misspelled plugin key fails when the plugin is built, before any listener opens: `failed to create sink http: ... unknown key "tls.enabeld"`. ```bash -# tail a directory, serve it over SSE +# tail a directory, serve it over SSE on every interface lw --source 'file,directory=/var/log/app,pattern=*.log' \ --sink http,host=0.0.0.0,port=8080 @@ -106,12 +121,14 @@ lw --source file,directory=/var/log/app \ --sink "http,port=8443,auth.type=scram,auth.credentials_file=/etc/logwisp/users.toml,\ tls.enabled=true,tls.cert_file=/etc/logwisp/server.crt,tls.key_file=/etc/logwisp/server.key" -# two pipelines -lw --pipeline app --source file,directory=/var/log/app --sink console \ +# two pipelines: app writes stdout, relay a file +lw --pipeline app --source file,directory=/var/log/app \ --pipeline relay --source tcp_chain,port=9000 \ --sink file,directory=/var/log/relay,name=relay ``` +Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns). + ## Environment Variables Configuration paths map to environment variables by replacing `.` with `_`, @@ -154,21 +171,28 @@ docker run --rm -p 8080:8080 -v /var/log/app:/logs:ro \ -e LOGWISP_FILTER='exclude,patterns=DEBUG' \ -e LOGWISP_SINK='http,host=0.0.0.0,port=8080' \ -e LOGWISP_SINK_1=console \ + -e LOGWISP_LOGGING_LEVEL=info \ logwisp ``` +Without a file, lw logs at `warn` with no status reporter, so a container +configured through the environment passes `LOGWISP_LOGGING_LEVEL=info` (and +`LOGWISP_STATUS_REPORTER=true`) to keep the service defaults. The image does +not set them: environment variables would override a mounted file's values. + ## Precedence - Pipelines - pipeline flags, if any are given - else pipeline variables, if any are set - else the configuration file's `[[pipelines]]` - - else the built-in default + - else the built-in `pipe` ([Built-in Defaults](#built-in-defaults)) - Every other key 1. command-line flags 2. environment variables 3. configuration file - 4. built-in defaults + 4. built-in defaults; `logging.level` and `status_reporter` depend on + whether a file was found A reload rereads the file and keeps the command-line or environment pipelines. @@ -186,10 +210,13 @@ service untouched; see [Configuration](configuration.md#hot-reload). ## Exit Codes -- `0`: clean shutdown, `--version` / `--help`, or a valid `--check` +- `0`: the end of input, a clean shutdown (`SIGINT`, `SIGTERM`), `--version` / + `--help`, or a valid `--check` - `1`: general error: a configuration load or validation failure (`--check` included), a logger init failure, a service bootstrap failure - `2`: an explicitly requested configuration file is not found +- killed by `SIGPIPE` (shell status `141`): the reader of stdout went away, as + in `lw | head`; `cat` ends the same way Exit code 2 applies only when the file was named explicitly (`-c`, `--config=`, or the `LOGWISP_CONFIG_*` variables). A missing discovered default @@ -197,35 +224,77 @@ is not an error, and LogWisp starts on built-in defaults. ## Built-in Defaults -With no configuration file present, LogWisp runs one pipeline named -`default_pipeline`: a `random` source with `special = true`, JSON formatting, -a rate limit of 5 entries/second with a burst of 10 and `policy = "drop"`, and a -`console` sink on stdout. It is a self-demonstrating idle mode, not a useful -production configuration. +Without a configuration file and without pipeline flags or variables, lw runs +one built-in pipeline, `pipe`: console source `stdin`, `raw` format, console +sink `stdout`, no filter, no rate limit. lw is then a Unix filter, like `cat`: -As soon as the file defines `[[pipelines]]`, or pipeline flags or variables are -given, that entire default pipeline — rate limit included — is replaced rather -than merged. +```bash +lw < app.log > copy.log +``` + +- Each line of stdin is one entry and one line of stdout. + - The terminator (`\n` or `\r\n`) is written back as `\n`; an unterminated + last line is kept and terminated. + - Blank lines are skipped. + - A line over 1 MiB continues in the next entry; no byte is lost. +- Nothing is dropped: a slow reader slows the reading of stdin. +- On a terminal, control characters are written as `` (`ESC` becomes + `<1b>`); pipes and files get the bytes unchanged + ([console sink](sinks.md#console)). +- When a console source reads a terminal, lw says so once on stderr, unless + `--quiet`: `lw: reading standard input; Ctrl-D ends it (lw --help for usage)`. + +A file that defines no `[[pipelines]]` still runs `pipe`. The file's +`[[pipelines]]`, or pipeline flags or variables, replace it whole. + +Without a file lw also logs less: `logging.level` defaults to `warn` and +`status_reporter` to `false`, against `info` and `true` with one. Explicit +values from the file, the environment or flags win. + +**End of input.** When stdin ends, the console source ends. A pipeline +finishes when all its sources have ended, and once every pipeline has +finished lw shuts down gracefully and exits 0. Only a console source ends on +its own: file and network sources run until a signal, and so does a pipeline +holding one. Console and file sinks write what is queued before the +exit. The `http` and `tcp` sinks do not yet flush their clients' queues, so a +connected client can miss the last entries of a finite input +([To Do](todo.md)). ## Usage Patterns +**Filter** + +```bash +# copy, line for line; exits 0 at the end of input +lw < app.log > copy.log + +# errors and warnings only +tail -F app.log | lw --filter include,patterns=ERROR,patterns=WARN + +# stdin as a live SSE stream; the http and tcp sinks bind 0.0.0.0 by default +journalctl -f | lw --sink http,host=127.0.0.1,port=8080 + +# a directory's files to one file, through stdout +lw --source file,directory=/var/log/app,pattern='*.log' > all.log +``` + +A console sink never drops: when its reader is slow, the pipeline waits, and +so does the reading of stdin. + **Development** ```bash -# verbose, everything to stderr -lw -c dev.toml --logging.output=stderr --logging.level=debug - -# no config at all: synthetic generator to stdout -lw +# a file's pipelines, verbose +lw -c dev.toml --logging.level=debug ``` **Configuration check** `lw --check` validates without running: it builds every pipeline and plugin as a start would (options, TLS files, credentials files, startup warnings), but -binds no port and opens, reads or creates no log file; it prints -`configuration ok` and exits 0, or the error and exits 1 (2 when a named file -is missing, see [Exit Codes](#exit-codes)): +binds no port, reads no stdin, and opens, reads or creates no log file; it +prints `configuration ok: N pipeline(s)` and exits 0, or the error and exits 1 +(2 when a named file is missing, see [Exit Codes](#exit-codes)): ```bash lw --check -c /etc/logwisp/logwisp.toml @@ -238,8 +307,10 @@ lw --check --source file,directory=/var/log/app --sink http,port=8080 lw -c /etc/logwisp/logwisp.toml --logging.output=file ``` -Run under a supervisor (systemd, rc.d) rather than backgrounding it — there is -no `--background` flag; earlier releases had one and it was removed. See +Run it in the foreground under a supervisor (systemd, rc.d, a container +runtime); there is no `--background` flag. A supervisor's stdin is usually +`/dev/null`, where a console source ends at once, so a service's file defines +its pipelines. See [Operations](operations.md#starting) and [Installation](installation.md). **Reload** diff --git a/doc/configuration.md b/doc/configuration.md index e682e08..4018d6f 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -41,9 +41,9 @@ runtime metadata and cannot be redirected by a `config_file` key inside the file Top-level keys, each as type and default: - `quiet` (bool, `false`): disable all application logging and console - diagnostics. -- `status_reporter` (bool, `true`): emit a status report every 30 s at DEBUG - level. + diagnostics; pipeline output still flows. +- `status_reporter` (bool, `true` with a configuration file, `false` without): + emit a status report every 30 s at DEBUG level. - `auto_reload` (bool, `false`): watch the config file and reload pipelines on change. @@ -56,11 +56,12 @@ Note that `status_reporter` writes at DEBUG level, so it produces nothing unless ## Application Logging This configures LogWisp's own operational log, not the log data it transports. +It goes to stderr by default, so stdout carries only a console sink's data. ```toml [logging] -output = "stdout" # file | stdout | stderr | split | all | none -level = "info" # debug | info | warn | error +output = "stderr" # file | stdout | stderr | split | all | none +level = "info" # debug | info | warn | error; warn without a file format = "txt" # raw | txt | json # sanitization = "" # raw | json | txt | shell @@ -81,7 +82,15 @@ retention_hours = 168.0 - `all`: files plus split console. - `none`: no application logging. -`[logging.file]` applies only to the `file` and `all` modes. +`[logging.file]` applies only to the `file` and `all` modes. The `stdout`, +`split` and `all` modes mix the log into a console sink's stdout. + +Without a configuration file (none named, none found) lw is a command-line +tool: `level` defaults to `warn` and `status_reporter` to `false`. Explicit +values from the environment or flags still win. A container configured only +through variables or flags passes `LOGWISP_LOGGING_LEVEL=info` to log as a +service; the image does not set it, since a variable would override a mounted +file's value. > `[logging.console].target` is accepted and validated (`stdout`, `stderr`, > `split`) but **not applied**. The console destination is derived from diff --git a/doc/formatters.md b/doc/formatters.md index a609cd3..7053fa8 100644 --- a/doc/formatters.md +++ b/doc/formatters.md @@ -11,8 +11,8 @@ flags = 0 timestamp_format = "" ``` -One formatter serves the whole pipeline. Sinks receive an identical payload; -there is no per-sink formatting. When you need two shapes of the same data, run +One formatter serves the whole pipeline. Sinks receive an identical payload, +one record ending in exactly one newline; there is no per-sink formatting. When you need two shapes of the same data, run two pipelines, or chain to a node that formats differently. Omitting `[pipelines.flow.format]` entirely selects `raw`. @@ -33,9 +33,10 @@ Each as type and default: ### raw Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches -the sink exactly as the source produced it, with no timestamp, level or source -prefix added. An entry that also carries `fields` gets the fields JSON appended -verbatim after a single space — `raw` never drops data and never re-encodes it. +the sink as the source produced it, with no timestamp, level or source prefix +added, and a newline appended unless the message ends in one. An entry that also +carries `fields` gets the fields JSON appended verbatim after a single space — +`raw` never drops data and never re-encodes it. ```toml [pipelines.flow.format] @@ -47,8 +48,9 @@ its final form. Note that it also bypasses sanitization, so control characters in the source data reach your sinks intact. Byte-exact transport needs a source that does not split the line: the `console` -source, or the `file` source with `raw = true`. Both put the whole line — -newline included — in the message and leave `fields` empty. The `file` source's +source, or the `file` source with `raw = true`. Both put the whole line, its +terminator removed, in the message and leave `fields` empty; `raw` writes it +back as one `\n`. The `file` source's JSON branch splits a line into message and fields, so `raw` reassembles it as ` ` rather than reproducing the original object. diff --git a/doc/installation.md b/doc/installation.md index a86c588..0f5533d 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -52,9 +52,11 @@ Variables: - `SOURCE_DATE_EPOCH`: when set, the embedded build time comes from it, so package builds are reproducible. -A plain `go build -o bin/lw ./cmd/lw` works too, and reports `dev` for the -version metadata. `go install github.com/lixenwraith/logwisp/cmd/lw@latest` -does not work until the module is renamed to its canonical path. +A plain `go build -o bin/lw ./cmd/lw` works too, as does +`go install github.com/lixenwraith/logwisp/cmd/lw@latest` (or `@vX.Y.Z`). +Without the Makefile's `-ldflags`, `lw --version` reports what Go stamps into +the binary: the module version, or a pseudo-version with the commit in a +checkout. ## Installing @@ -201,6 +203,12 @@ docker run -d --name logwisp \ -e LOGWISP_SINK=console ``` + Without a file lw logs warnings only and runs no status reporter; add + `-e LOGWISP_LOGGING_LEVEL=info` (and `-e LOGWISP_STATUS_REPORTER=true`) for + the service defaults. The image cannot set them: a variable would override + a mounted file's value. Without `-i`, stdin is `/dev/null`, so a container + needs a source other than `console`. + - Secrets: only as mounted files, at `/run/secrets` (Docker or Compose secrets, a Kubernetes secret volume), named by `tls.key_file`, `auth.credentials_file` and `auth.password_file`. Never put a password in @@ -237,10 +245,9 @@ The foundation for distribution packages is in place: - skeletons: `deploy/package/arch/PKGBUILD` and a `sysutils/logwisp` port in `deploy/package/freebsd/` -Still missing: tagged, signed release tarballs; the module rename to -`github.com/lixenwraith/logwisp`, which `go install` and FreeBSD's -`go:modules` need; finishing and submitting the AUR package and the port; -Debian packaging. [To Do](todo.md) has the steps. +Still missing: release tags, which both skeletons download; finishing and +submitting the AUR package and the port; Debian packaging. +[To Do](todo.md) has the steps. ## Verification diff --git a/doc/lw.1 b/doc/lw.1 index 01af4ad..06d6250 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -1,4 +1,4 @@ -.Dd October 2, 2026 +.Dd October 3, 2026 .Dt LW 1 .Os .Sh NAME @@ -36,14 +36,65 @@ forward to other nodes. Every network source and sink can use TLS, mutual TLS and Argon2id-SCRAM password authentication. .Pp +With no configuration file and no pipeline options, .Nm -stays in the foreground and logs to standard output; run it under a -supervisor such as +is a filter like +.Xr cat 1 : +it copies standard input to standard output line by line and exits 0 at the +end of input. +This built-in pipeline, +.Dq pipe , +joins the console source +.Dq stdin +to the console sink +.Dq stdout +in the raw format, without a filter or rate limit; a configuration file that +defines no pipelines runs it too. +Each line is written with one newline, whether it ended in a newline, a +carriage return and newline, or nothing; blank lines are skipped, and a line +over 1 MiB continues in the next one. +When standard input is a terminal, +.Nm +prints a reminder on standard error that Control-D ends the input. +.Pp +A pipeline finishes when all its sources have ended. +Only a console source ends, at the end of standard input; every other +source runs until shutdown. +When every pipeline has finished, +.Nm +shuts down gracefully and exits 0. +Console and file sinks first write what they hold; the +.Cm http +and +.Cm tcp +sinks do not yet, so a client can miss the last lines of a finite input. +Under a supervisor, standard input is usually +.Pa /dev/null : +a console source there ends at once. +.Pp +The console source and sink never drop a line. +A busy pipeline slows the reading of standard input; a console sink whose +output is slow makes its pipeline wait, other sinks included. +On a terminal the console sink writes control characters, escape sequences +included, as the hex of their bytes in angle brackets, such as +.Ql <1b> , +so that a log line cannot drive the terminal; pipes and files get the bytes +unchanged. +Its +.Cm escape +option sets this: +.Cm auto +.Pq the default , +.Cm always +or +.Cm never . +.Pp +.Nm +stays in the foreground and logs to standard error; run a configuration as a +service under a supervisor such as .Xr systemd 1 or .Xr daemon 8 . -With no configuration file it runs a demonstration pipeline: a synthetic -generator writing JSON to standard output. .Pp The options are as follows: .Bl -tag -width Ds @@ -59,23 +110,39 @@ A bare .Fl \- Ns Ar key sets a boolean to true. .It Fl \-quiet -Suppress console output. +Turn off the log of +.Nm +and the terminal reminder; console sinks still write. .It Fl \-status_reporter= Ns Ar bool Log pipeline statistics every 30 seconds at the debug level -.Pq default true . +.Pq default true with a configuration file, false without . .It Fl \-auto_reload= Ns Ar bool Reload when the configuration file changes .Pq default false . .It Fl \-logging.output= Ns Ar mode One of -.Cm file , stdout , stderr , split , all +.Cm stderr +.Pq the default , +.Cm stdout , +.Cm split +.Pq warnings and errors to standard error, the rest to standard output , +.Cm file , +.Cm all +.Pq split and file or .Cm none . +The modes that use standard output mix the log into what a console sink +writes there. .It Fl \-logging.level= Ns Ar level One of .Cm debug , info , warn or -.Cm error . +.Cm error ; +the default is +.Cm info +with a configuration file and +.Cm warn +without. .It Fl \-check Build every pipeline and plugin as a start would, loading options, TLS files and credentials files and printing startup warnings, without binding a port @@ -97,6 +164,12 @@ the configuration file. They then replace the file's pipelines and the built-in default; every other key keeps its usual precedence. A reload rereads the file but keeps these pipelines. +A pipeline without a source reads standard input with the console source +.Dq stdin ; +one without a sink writes standard output with the console sink +.Dq stdout . +Standard input has one reader: a second console source anywhere in the +configuration fails validation. .Bl -tag -width Ds .It Fl \-pipeline Ar name Start a new pipeline. @@ -265,7 +338,7 @@ The service's working directory on Linux and on .El .Sh EXIT STATUS .Nm -exits 0 after a clean shutdown, +exits 0 at the end of input, after a clean shutdown, .Fl \-version or help; 1 on a configuration, logger or startup error; and 2 when a configuration file named by @@ -273,6 +346,15 @@ configuration file named by or .Ev LOGWISP_CONFIG_* does not exist. +A reader of standard output that goes away, as in +.Ql lw | head , +ends +.Nm +with +.Dv SIGPIPE , +as it ends +.Xr cat 1 ; +the shell reports status 141. .Pp .Nm .Cm auth @@ -287,23 +369,59 @@ every pipeline. An invalid configuration leaves the running service untouched. .El .Sh EXAMPLES +Copy a file line by line: +.Bd -literal -offset indent +$ lw < app.log > copy.log +.Ed +.Pp +Follow a log and keep only errors and warnings: +.Bd -literal -offset indent +$ tail \-F app.log | lw \-\-filter include,patterns=ERROR,patterns=WARN +.Ed +.Pp +Prefix each line with the time, the level found in it and the source: +.Bd -literal -offset indent +$ tail \-F app.log | lw \-\-format txt +.Ed +.Pp +Serve standard input as a live stream on the loopback address only, since +the +.Cm http +sink listens on every address by default, and read it from another terminal +as server-sent events; a client gets the lines that arrive while it is +connected: +.Bd -literal -offset indent +$ journalctl \-f | lw \-\-sink http,host=127.0.0.1,port=8080 +$ curl \-N http://127.0.0.1:8080/stream +.Ed +.Pp +Follow every +.Pa .log +file in a directory, as +.Ql tail \-F +does, and write the new lines unparsed to one file: +.Bd -literal -offset indent +$ lw \-\-source 'file,directory=/var/log/app,pattern=*.log,raw=true' \e + > all.log +.Ed +.Pp +Check a configuration without binding a port or touching a log file: +.Bd -literal -offset indent +$ lw \-\-check \-c /etc/logwisp/logwisp.toml +.Ed +.Pp Run a configuration in the foreground with debug logging: .Bd -literal -offset indent $ lw \-c /etc/logwisp/logwisp.toml \-\-logging.level=debug .Ed .Pp -Tail a directory and serve it as a live stream, without a file: -.Bd -literal -offset indent -$ lw \-\-source 'file,directory=/var/log/app,pattern=*.log' \e - \-\-sink http,host=127.0.0.1,port=8080 -.Ed -.Pp Create a credentials file the service account can read, add a user, letting .Nm generate the password into a file, then reload the server: .Bd -literal -offset indent -# install \-m 0600 \-o logwisp \-g logwisp /dev/null /etc/logwisp/users.toml +# install \-m 0600 \-o logwisp \-g logwisp \e + /dev/null /etc/logwisp/users.toml # lw auth add-user \-credentials /etc/logwisp/users.toml \e \-user alice \-password-file /root/alice.pass # systemctl reload logwisp diff --git a/doc/operations.md b/doc/operations.md index bc95ce7..a366ff5 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -5,17 +5,32 @@ Running, monitoring, and maintaining LogWisp. ## Starting ```bash -# foreground, explicit config -lw -c /etc/logwisp/logwisp.toml +# a filter: stdin to stdout, exits 0 at the end of input +lw < app.log > copy.log +tail -F app.log | lw --filter include,patterns=ERROR -# no config: built-in demo pipeline (random source -> stdout) -lw +# a service: the file's pipelines, until SIGINT or SIGTERM +lw -c /etc/logwisp/logwisp.toml ``` +lw exits on its own only at the end of input: once every pipeline has +finished, a pipeline finishing when all its sources have ended. Only a console +source ends, when stdin does; file and network sources run until a signal, and +lw shuts down gracefully, exit 0, either way. +Console and file sinks write what is queued before the exit; the `http` and +`tcp` sinks do not yet flush their clients' queues ([To Do](todo.md)). See +[CLI](cli.md#built-in-defaults). + There is no built-in daemon mode. Run LogWisp in the foreground under a supervisor — systemd, rc.d, or a container runtime — which is where restart, log capture, and resource limits belong. See [Installation](installation.md). +A supervisor's stdin is `/dev/null` (systemd's default, `docker run` without +`-i`), where a console source ends at once. So a service defines its +pipelines: a file without `[[pipelines]]` runs the built-in stdin-to-stdout +pipeline and exits 0 at once, which `Restart=on-failure` does not restart, and +a container configured through variables needs a `LOGWISP_SOURCE`. + **systemd** ```bash @@ -64,6 +79,9 @@ What reload does *not* do: construction and `auto_reload` watches only the configuration file, so rotation requires `SIGHUP`. +Standard input survives a reload: one reader serves the process, so the new +console source continues where the old one stopped, and no line is read twice. + Plan reloads on a busy relay the way you would plan a restart. Listener bind/start failures happen after the old service stops; these can leave the application without working pipelines until a corrected configuration reloads. @@ -94,8 +112,9 @@ proves itself when the pipeline is actually built. ### Status reporter -Enabled by default, every 30 seconds. It logs at **DEBUG**, so it produces -nothing unless `logging.level = "debug"` — a common surprise. +Enabled by default with a configuration file (off without one), every 30 +seconds. It logs at **DEBUG**, so it produces nothing unless +`logging.level = "debug"` — a common surprise. ```toml status_reporter = true @@ -153,10 +172,12 @@ query it with a client certificate or a token from `lw auth token`. Each counter, where it is reported, and what a rise means: -- `dropped_entries` (source): downstream cannot keep up with the source +- `dropped_entries` (source): downstream cannot keep up with the source; the + console source never drops, it reads stdin slower - `total_dropped` (flow): the rate limit or filters are discarding entries, often by intent -- `total_dropped_by_sink` (pipeline): a sink's input queue is full +- `total_dropped_by_sink` (pipeline): a sink's input queue is full; a console + sink waits instead, so its entries count only when shutdown cuts the wait - `dropped_writes` (`tcp` and `http` sinks): a client's queue overflowed, because it is too slow or one burst exceeded `client_buffer_size` - `rejected_conns` (`tcp` sink, `tcp_chain` source), `rejected_clients` @@ -172,7 +193,11 @@ Each counter, where it is reported, and what a rise means: ## Log Management -LogWisp's own operational log: +LogWisp's own operational log goes to stderr by default, at `info` with a +configuration file and at `warn` without one; explicit values win. A container +configured only through variables or flags therefore logs at `warn`: pass +`-e LOGWISP_LOGGING_LEVEL=info`. `lw-deploy.sh` always passes `-c`, so its +nodes log at `info`. To files: ```toml [logging] @@ -253,6 +278,14 @@ Walk the pipeline in order and read the counters: source `total_entries` (is anything being produced?), flow `total_dropped` (filters or rate limit?), pipeline `total_dropped_by_sink` (sink backed up?), sink `total_processed`. +**A pipeline stalls behind a console sink** + +A console sink never drops: when its output is slow (a paused terminal, a full +pipe, a stalled log collector) its pipeline waits, and the pipeline's other +sinks with it. Stdin is read slower; other sources queue, then drop and count +`dropped_entries`. Where a service's stdout may stall, give it a `file` sink +instead. A closed stdout pipe ends lw with `SIGPIPE`. + **File source reads nothing** - The watcher seeks to end-of-file on start; only content appended afterwards is diff --git a/doc/security.md b/doc/security.md index 81cb0cd..338603f 100644 --- a/doc/security.md +++ b/doc/security.md @@ -853,3 +853,9 @@ Choose a sanitizer policy that matches the sink — `json` for JSON output, `txt` for files and consoles — so control characters in log data cannot break framing or inject terminal escapes downstream. See [Formatters](formatters.md). + +The `raw` format passes control characters through. A console sink writing to +a terminal escapes them itself (`escape = "auto"`), so a log line cannot move +the cursor, retitle the window, write the clipboard (OSC 52) or reorder text +with bidi controls; pipes and files get the bytes unchanged. See +[Sinks](sinks.md#console). diff --git a/doc/sinks.md b/doc/sinks.md index c18bdbf..513d5f3 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -16,7 +16,8 @@ sibling sinks are unaffected. ## console -Writes formatted payloads to stdout or stderr. +Writes formatted payloads to stdout or stderr. It is the sink of the built-in +`pipe` pipeline, and of every spec pipeline given no `--sink`. ```toml [[pipelines.plugin_sinks]] @@ -25,19 +26,35 @@ type = "console" [pipelines.plugin_sinks.config] target = "stdout" buffer_size = 1000 +escape = "auto" ``` Options, each as type and default: - `target` (string, `stdout`): `stdout` or `stderr`. - `buffer_size` (int, `1000`): sink input queue depth. +- `escape` (string, `auto`): write control characters as `` of their + bytes, `ESC` as `<1b>`: `auto` when the output is a terminal, `always`, or + `never`. > `split` is **not** a valid target for this sink and is rejected at startup. > Level-based splitting exists only for LogWisp's own application log > (`logging.output = "split"`). -Payloads are written verbatim; the sink adds no framing. Whether entries are -newline-terminated is decided by the formatter. +Payloads are written as the formatter made them, one record per line. + +- Escaping: a log line must not drive the terminal that shows it (cursor and + screen control, title changes, OSC 52 clipboard writes, bidi spoofing). C0 + and C1 controls other than tab, DEL, the bidi controls, the line and + paragraph separators and invalid UTF-8 are escaped; other invisible + characters, such as the joiners emoji and Indic scripts need, are kept. + `never` keeps an application's own ANSI colours; under `auto`, pipes and + files get the bytes unchanged. +- Backpressure: the sink never drops. When its output is slow (a slow reader, + a paused terminal, a stalled log collector) the pipeline waits for it, with + its other sinks. A service whose stdout may stall writes to a `file` sink + instead. A closed stdout pipe ends lw with `SIGPIPE`, as it ends `cat`. +- On stop it writes what is queued. --- @@ -68,7 +85,8 @@ Options, each as type and default: - `max_total_size_mb` (int, `1000`): cap across all rotated files. - `min_disk_free_mb` (int, `0`): free-space floor before writing; `0` = none. - `retention_hours` (float, `168.0`): delete rotated files older than this. -- `buffer_size` (int, `1000`): sink input queue depth. +- `buffer_size` (int, `1000`): sink input queue depth, and the depth of the + internal writer's queue, which drops and counts when full. - `flush_interval_ms` (int, `100`): forced flush interval. > `min_disk_free_mb` has an unusual default. The constructor replaces only @@ -76,7 +94,8 @@ Options, each as type and default: > free-space floor. Set it explicitly if you want one. The sink drives an internal writer configured for raw output with timestamps and -levels disabled, so what lands on disk is exactly the formatted payload. +levels disabled, so what lands on disk is exactly the formatted payload. On stop +it hands the writer what is queued before closing the file. --- @@ -187,7 +206,9 @@ Options, each as type and default: evicted by the broker. With the idle comment above, that reaches only a peer that has stopped accepting bytes on a sink configured `write_timeout_ms = 0`. - On shutdown, connected clients receive - `event: disconnect / data: {"reason":"server_shutdown"}`. + `event: disconnect / data: {"reason":"server_shutdown"}`. Entries still + queued for them are not flushed first, so a client can miss the last ones + of a finite input ([To Do](todo.md)). - HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1. **Status endpoint** returns service and version identity, host, port, TLS flag, @@ -266,6 +287,8 @@ Options, each as type and default: whole timeout, so the client is disconnected immediately rather than retried. - A client whose send queue is full has that event dropped (`dropped_writes`) and stays connected. +- On shutdown, entries still queued for clients are not flushed first, as + with the `http` sink. - With TLS enabled the handshake runs under a 10 s bound *after* the `max_connections` check, so concurrent handshakes are bounded too. - With an `auth` block, authorization runs after that handshake and *before* diff --git a/doc/sources.md b/doc/sources.md index eaebe3f..fdea872 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -84,7 +84,8 @@ last read time, plus `active_watchers`. ## console -Reads newline-delimited entries from standard input. +Reads newline-delimited entries from standard input. It is the source of the +built-in `pipe` pipeline, and of every spec pipeline given no `--source`. ```toml [[pipelines.plugin_sources]] @@ -98,9 +99,23 @@ Option, as type and default: - `buffer_size` (int, `1000`): subscriber channel depth. -At most **one** instance per pipeline: the type is registered with -`MaxInstances: 1`, and a second instance is rejected at pipeline construction. -The level is inferred from the line text, and `Source` is set to `console`. +Lines: +- Each line is one entry, its terminator (`\n` or `\r\n`) removed; the + formatter writes one back. An unterminated last line is kept. +- Blank lines are skipped. +- A line over 1 MiB continues in the next entry; no byte is lost. +- The level is inferred from the line text, and `Source` is set to `console`. + +Delivery and lifetime: +- Nothing is dropped: the source waits for its pipeline, so a slow pipeline + slows the reading of stdin. +- One reader serves the process. A reload's new source continues where the old + one stopped; no line is read twice. Hence at most **one** console source in + the whole configuration, checked at validation. +- At the end of input the source ends, and with it a pipeline whose sources + have all ended; lw exits once every pipeline has + ([CLI](cli.md#built-in-defaults)). Under a supervisor whose stdin is + `/dev/null` that is at once. --- diff --git a/doc/todo.md b/doc/todo.md index 3344f66..c960705 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -122,7 +122,16 @@ connection level, below TLS, so a refused peer costs no handshake. - Denied and allowed clients; per-client SCRAM budgets behind one proxy address. -## 2. Hardening review of `lixenwraith/config` and `lixenwraith/toml` +## 2. Flush network sinks at shutdown + +At the end of a finite input lw shuts down, and the `http` and `tcp` sinks +disconnect their clients without writing the entries still queued for them, so +a client can miss the last ones. On `Stop`, each sink's broker should first move +its input queue into the client queues, then let every client writer drain its +queue within a bound (`write_timeout_ms`) before the disconnect frame. Verify +with a finite stdin into each sink and a connected client counting lines. + +## 3. Hardening review of `lixenwraith/config` and `lixenwraith/toml` Both parse untrusted-shaped input (config files, environment, command line) and were not reviewed with the auth work. Review them as `auth` was reviewed: @@ -151,52 +160,59 @@ the library repository, then a dependency bump here. - Fuzz targets committed in the library repositories. - A note in this repository's security.md once done. -## 3. Packaging: AUR, FreeBSD ports, Debian +## 4. Packaging: AUR, FreeBSD ports, Debian The foundation exists: -- The `lw` binary name. +- The `lw` binary name, free in Arch (official repositories and AUR), Ubuntu + 24.04 and 26.04, and the FreeBSD 15.1 ports tree. +- The canonical module path `github.com/lixenwraith/logwisp`, which + `go install`, FreeBSD's `USES=go:modules` and Debian's dh-golang expect. - `make install` with `DESTDIR`, `PREFIX` and `SYSCONFDIR`. - The `doc/lw.1` manual. - Service files in `deploy/package/`: systemd unit, sysusers, tmpfiles and the FreeBSD rc.d script. - Skeletons in `deploy/package/arch/` and `deploy/package/freebsd/`. -What remains: +What remains. Steps 1 to 3 are the maintainer's: they need push rights and an +identity, and both skeletons download the tagged source. -- Releases: - - The repository has no tags, so versions come from `git describe` and fall - back to a commit hash. Tag releases `vX.Y.Z`. - - Publish source tarballs with checksums, and sign them. -- Module path: - - The module is `logwisp`, so `go install github.com/lixenwraith/logwisp/cmd/lw@latest` - does not work. - - FreeBSD's `USES=go:modules` and Debian's dh-golang expect the canonical - path. - - Rename the module to `github.com/lixenwraith/logwisp`: a mechanical - import rewrite in one commit. -- Name check before submission: - - Arch: `pkgfile -s lw` against the official repositories, and the AUR - package list for `lw` and `logwisp`. - - Already checked: Ubuntu 26.04 and 24.04 contents and the FreeBSD 15.1 - ports tree ship no `bin/lw`. -- Arch (AUR): `deploy/package/arch/PKGBUILD` already has the source URL, the - Arch Go flags (`-trimpath -buildmode=pie -mod=readonly -modcacherw`), - `backup` for the config and `check()`. Remaining: - - the checksum of a tagged tarball in place of `SKIP`, and a maintainer; - - namcap, `.SRCINFO`, and publishing `logwisp` (optionally `logwisp-git`). -- FreeBSD: `deploy/package/freebsd/` already has `USES=go:1.27,modules`, - `USE_RC_SUBR=logwisp`, `USERS`/`GROUPS`, and a pkg-plist with the `.sample` - config and the manual. Remaining: - - `distinfo` (or `GH_TUPLE`), which needs the module rename and a tag; - - a real `MAINTAINER` and a `UIDs`/`GIDs` entry for `logwisp`; - - poudriere on 14.x and 15.x, then submission through Bugzilla - (`sysutils/logwisp`). -- Debian: - - Debian policy wants every Go dependency packaged; the four `lixenwraith` - libraries are not. - - Either package them too (dh-golang), or start with an `.deb` built by the - Makefile `install` target (nfpm or `dpkg-deb`) and a PPA, and move to the - archive later. -- Shell completion for bash, zsh and fish, installed by `make install`. -- A packaging CI job: build the AUR package in an Arch container and the port - in a FreeBSD VM, run `lw --version` and `make image-check`. +1. Tag the release on the merged main commit, `vX.Y.Z`: + ``` + git tag -a vX.Y.Z -m vX.Y.Z # git tag -s signs it with your key + git push origin vX.Y.Z + ``` + - `make` then stamps `vX.Y.Z` (from `git describe`), and + `go install github.com/lixenwraith/logwisp/cmd/lw@vX.Y.Z` works. + - A GitHub release for the tag is optional; the tag alone serves + `archive/vX.Y.Z.tar.gz`, which the PKGBUILD downloads. +2. Arch (AUR), `deploy/package/arch/PKGBUILD`: + - `pkgver=X.Y.Z` and the `# Maintainer: Name ` line. + - `updpkgsums` (pacman-contrib) replaces `sha256sums=('SKIP')` with the + tarball's checksum. + - `namcap PKGBUILD`, then `makepkg -si` (builds, runs `check()`, + installs), then `makepkg --printsrcinfo > .SRCINFO`. + - Push `PKGBUILD` and `.SRCINFO` to + `ssh://aur@aur.archlinux.org/logwisp.git`; optionally a `logwisp-git` + package built from the main branch. +3. FreeBSD, `deploy/package/freebsd/` (`sysutils/logwisp`): + - `DISTVERSION=X.Y.Z` and `MAINTAINER=` your address. + - `make makesum` writes `distinfo`: the module zip that the Go proxy + serves for `GO_MODULE` at the tag. + - The `logwisp` user needs an ID registered in the ports tree. Take a + number below 1000 that is free in both `/usr/ports/UIDs` and + `/usr/ports/GIDs`, and add to the same patch: + ``` + UIDs: logwisp:*:NNN:NNN::0:0:LogWisp daemon:/nonexistent:/usr/sbin/nologin + GIDs: logwisp:*:NNN: + ``` + - `portlint -AC`, `poudriere testport` on 14.x and 15.x jails, then a + Bugzilla report with the port directory and the UIDs/GIDs diff. +4. Debian: + - Debian policy wants every Go dependency packaged; the four `lixenwraith` + libraries are not. + - Either package them too (dh-golang), or start with an `.deb` built by the + Makefile `install` target (nfpm or `dpkg-deb`) and a PPA, and move to the + archive later. +5. Shell completion for bash, zsh and fish, installed by `make install`. +6. A packaging CI job: build the AUR package in an Arch container and the port + in a FreeBSD VM, run `lw --version` and `make image-check`. diff --git a/go.mod b/go.mod index f4fced9..0ee85ac 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module logwisp +module github.com/lixenwraith/logwisp go 1.27.1 @@ -7,6 +7,7 @@ require ( github.com/lixenwraith/config v0.2.1 github.com/lixenwraith/log v0.1.11 github.com/lixenwraith/toml v0.0.0-20261001212158-cede215365c0 + golang.org/x/term v0.46.0 ) require ( diff --git a/go.sum b/go.sum index 10e32c5..55c23a0 100644 --- a/go.sum +++ b/go.sum @@ -12,3 +12,5 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= diff --git a/internal/authz/authz.go b/internal/authz/authz.go index 20e812c..578b769 100644 --- a/internal/authz/authz.go +++ b/internal/authz/authz.go @@ -19,9 +19,9 @@ import ( "sync/atomic" "time" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/tlsx" "github.com/lixenwraith/log" ) diff --git a/internal/authz/authz_test.go b/internal/authz/authz_test.go index 01dbfd7..d44f7d2 100644 --- a/internal/authz/authz_test.go +++ b/internal/authz/authz_test.go @@ -8,8 +8,8 @@ import ( "strings" "testing" - "logwisp/internal/config" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/tlsx" ) // peerState fakes a completed handshake. Only the leaf's identity fields are diff --git a/internal/authz/proxy_test.go b/internal/authz/proxy_test.go index d047e70..b0776dd 100644 --- a/internal/authz/proxy_test.go +++ b/internal/authz/proxy_test.go @@ -9,8 +9,8 @@ import ( "strings" "testing" - "logwisp/internal/chain" - "logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/auth" ) diff --git a/internal/authz/scram.go b/internal/authz/scram.go index c0da9b4..481c936 100644 --- a/internal/authz/scram.go +++ b/internal/authz/scram.go @@ -22,10 +22,10 @@ import ( "sync/atomic" "time" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/tlsx" - "logwisp/internal/tokenbucket" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/tokenbucket" "github.com/lixenwraith/auth" "github.com/lixenwraith/toml" diff --git a/internal/authz/scram_test.go b/internal/authz/scram_test.go index 7b5874c..ebd17e3 100644 --- a/internal/authz/scram_test.go +++ b/internal/authz/scram_test.go @@ -21,9 +21,9 @@ import ( "testing" "time" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/auth" ) diff --git a/internal/chain/proto.go b/internal/chain/proto.go index 9663650..37549d6 100644 --- a/internal/chain/proto.go +++ b/internal/chain/proto.go @@ -3,9 +3,10 @@ package chain import ( "encoding/json" "fmt" - "logwisp/internal/core" "math/rand/v2" "time" + + "github.com/lixenwraith/logwisp/internal/core" ) // ProtocolVersion is declared in the hello preamble diff --git a/internal/config/config.go b/internal/config/config.go index b2fd783..387693d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -244,6 +244,7 @@ type NullSinkOptions struct{} type ConsoleSinkOptions struct { Target string `toml:"target"` // "stdout", "stderr" BufferSize int64 `toml:"buffer_size"` + Escape string `toml:"escape"` // control characters as : "auto" (on a terminal), "always", "never" } // FileSinkOptions defines settings for a file-based sink diff --git a/internal/config/loader.go b/internal/config/loader.go index ddfd386..d1c83f7 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -3,11 +3,12 @@ package config import ( "errors" "fmt" + "io/fs" "os" "path/filepath" "strings" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/core" lconfig "github.com/lixenwraith/config" ) @@ -44,6 +45,11 @@ func Load(args []string) (*Manager, error) { } m := &Manager{path: configPath, explicit: isExplicit, specs: specs} initial := defaults() + if _, err := os.Stat(configPath); errors.Is(err, fs.ErrNotExist) { + // Without a file lw is a command-line tool: warnings and errors only + initial.Logging.Level = "warn" + initial.StatusReporter = false + } cfg, err := lconfig.NewBuilder(). WithTarget(initial). WithEnvPrefix("LOGWISP_"). @@ -134,20 +140,13 @@ func (m *Manager) Watch() <-chan string { func (m *Manager) Close() { m.config.StopAutoUpdate() } -// defaults provides the default configuration values for the application +// defaults provides the default configuration values for the application. +// Data goes to stdout, so lw's own log goes to stderr. func defaults() *Config { return &Config{ - // Top-level flag defaults - ShowVersion: false, - Quiet: false, - - // Runtime behavior defaults - StatusReporter: true, - ConfigAutoReload: false, - - // Existing defaults + StatusReporter: true, Logging: &LogConfig{ - Output: "stdout", + Output: "stderr", Level: "info", Format: "txt", File: &LogFileConfig{ @@ -158,45 +157,20 @@ func defaults() *Config { RetentionHours: 168, // 7 days }, Console: &LogConsoleConfig{ - Target: "stdout", - }, - }, - Pipelines: []PipelineConfig{ - { - Name: "default_pipeline", - Flow: &FlowConfig{ - RateLimit: &RateLimitConfig{ - Rate: 5, - Burst: 10, - Policy: "drop", - MaxEntrySizeBytes: 65536, - }, - Format: &FormatConfig{ - Type: "json", - SanitizerPolicy: "json", - }, - }, - PluginSources: []PluginSourceConfig{ - { - ID: "default_source", - Type: "random", - Config: map[string]any{ - "special": true, - }, - }, - }, - PluginSinks: []PluginSinkConfig{ - { - ID: "default_sink", - Type: "console", - Config: map[string]any{ - "target": "stdout", - "buffer_size": 100, - }, - }, - }, + Target: "stderr", }, }, + Pipelines: []PipelineConfig{pipeDefault("pipe")}, + } +} + +// pipeDefault is a filter in the Unix sense: stdin to stdout, line for line +func pipeDefault(name string) PipelineConfig { + return PipelineConfig{ + Name: name, + Flow: &FlowConfig{Format: &FormatConfig{Type: "raw"}}, + PluginSources: []PluginSourceConfig{{ID: "stdin", Type: "console", Config: map[string]any{}}}, + PluginSinks: []PluginSinkConfig{{ID: "stdout", Type: "console", Config: map[string]any{}}}, } } diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go index 146f495..d7e84b7 100644 --- a/internal/config/loader_test.go +++ b/internal/config/loader_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/testutil" ) func TestResolveConfigArguments(t *testing.T) { @@ -56,9 +56,9 @@ func TestLoadPrecedenceAndDetachedSnapshots(t *testing.T) { t.Fatalf("snapshot: %+v %v", first, err) } first.Logging.Level = "mutated" - first.Pipelines[0].PluginSources[0].Config["special"] = false + first.Pipelines[0].PluginSources[0].Config["buffer_size"] = 1 second, err := m.Snapshot() - if err != nil || second.Logging.Level != "warn" || second.Pipelines[0].PluginSources[0].Config["special"] != true { + if _, aliased := second.Pipelines[0].PluginSources[0].Config["buffer_size"]; err != nil || second.Logging.Level != "warn" || aliased { t.Fatalf("snapshot aliases another result: %+v %v", second, err) } env, err := Load([]string{"-c=selected.toml"}) @@ -268,3 +268,36 @@ port = 9000 } } } + +// A file makes lw a service, logging at info with the status reporter; without +// one it is a command-line tool that reports warnings and errors only. +func TestNoConfigFileMeansQuietLogging(t *testing.T) { + isolateConfig(t) + for _, c := range []struct { + file, level string + reporter bool + }{{"", "warn", false}, {"[logging]\noutput = \"stderr\"\n", "info", true}} { + if c.file != "" { + testutil.WriteFile(t, "logwisp.toml", c.file) + } + m, err := Load(nil) + if err != nil { + t.Fatal(err) + } + cfg, err := m.Snapshot() + m.Close() + if err != nil || cfg.Logging.Level != c.level || cfg.StatusReporter != c.reporter || cfg.Logging.Output != "stderr" { + t.Fatalf("file %q: %+v %+v %v", c.file, cfg, cfg.Logging, err) + } + } + t.Setenv("LOGWISP_LOGGING_LEVEL", "debug") + os.Remove("logwisp.toml") + m, err := Load(nil) + if err != nil { + t.Fatal(err) + } + defer m.Close() + if cfg, err := m.Snapshot(); err != nil || cfg.Logging.Level != "debug" { + t.Fatalf("environment over the file-less default: %+v %v", cfg, err) + } +} diff --git a/internal/config/spec.go b/internal/config/spec.go index 3e44a8c..e38fb26 100644 --- a/internal/config/spec.go +++ b/internal/config/spec.go @@ -131,7 +131,8 @@ func envPipelineSpecs() []pipelineSpec { } // buildPipelines turns specs, in order, into pipelines; specs before the first -// --pipeline go to one named "cli". Every call returns fresh maps. +// --pipeline go to one named "cli". A pipeline without a source reads stdin, +// one without a sink writes stdout. Every call returns fresh maps. func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { var pipelines []PipelineConfig var seen map[*specKind]bool @@ -160,6 +161,15 @@ func buildPipelines(specs []pipelineSpec) ([]PipelineConfig, error) { return nil, fmt.Errorf("%s %s: %w", s.name, s.value, err) } } + for i := range pipelines { + pipe := pipeDefault(pipelines[i].Name) + if len(pipelines[i].PluginSources) == 0 { + pipelines[i].PluginSources = pipe.PluginSources + } + if len(pipelines[i].PluginSinks) == 0 { + pipelines[i].PluginSinks = pipe.PluginSinks + } + } return pipelines, nil } diff --git a/internal/config/spec_test.go b/internal/config/spec_test.go index 56c00bc..a0699f0 100644 --- a/internal/config/spec_test.go +++ b/internal/config/spec_test.go @@ -4,9 +4,10 @@ import ( "errors" "os" "reflect" + "strings" "testing" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/testutil" ) func loadPipelines(t *testing.T, args ...string) []PipelineConfig { @@ -160,6 +161,21 @@ func TestSpecPipelinesReplaceFilePipelines(t *testing.T) { } } +// A spec pipeline without a source reads stdin and one without a sink writes +// stdout, as a filter does; two pipelines cannot both read stdin. +func TestSpecPipelinesDefaultToStdio(t *testing.T) { + isolateConfig(t) + got := loadPipelines(t, "--filter", "include,patterns=ERROR") + if p := got[0]; len(p.PluginSources) != 1 || p.PluginSources[0].Type != "console" || p.PluginSources[0].ID != "stdin" || + len(p.PluginSinks) != 1 || p.PluginSinks[0].Type != "console" || p.PluginSinks[0].ID != "stdout" { + t.Fatalf("pipeline: %+v", p) + } + _, err := Load([]string{"--pipeline", "a", "--sink", "null", "--pipeline", "b", "--sink", "null"}) + if err == nil || !strings.Contains(err.Error(), `console source already reads stdin in pipeline "a"`) { + t.Fatalf("second stdin reader: %v", err) + } +} + // Without a file a reload still rebuilds; once the discovered default // appears it is read, and its pipelines still yield to the specs. Its later // removal fails the reload instead of keeping the removed file's values. @@ -171,13 +187,13 @@ func TestReloadKeepsSpecPipelines(t *testing.T) { } defer m.Close() first, err := m.Reload() - if err != nil || !first.StatusReporter { + if err != nil || first.StatusReporter { t.Fatalf("reload without a file: %+v %v", first, err) } first.Pipelines[0].PluginSources[0].Config["special"] = "mutated" - testutil.WriteFile(t, "logwisp.toml", "status_reporter = false\n[[pipelines]]\nname = \"file\"\n") + testutil.WriteFile(t, "logwisp.toml", "status_reporter = true\n[[pipelines]]\nname = \"file\"\n") next, err := m.Reload() - if err != nil || next.StatusReporter || len(next.Pipelines) != 1 || next.Pipelines[0].Name != "cli" || + if err != nil || !next.StatusReporter || len(next.Pipelines) != 1 || next.Pipelines[0].Name != "cli" || next.Pipelines[0].PluginSources[0].Config["special"] != "true" { t.Fatalf("reload: %+v %v", next, err) } diff --git a/internal/config/test_helpers_test.go b/internal/config/test_helpers_test.go index 78d974f..20613fe 100644 --- a/internal/config/test_helpers_test.go +++ b/internal/config/test_helpers_test.go @@ -3,7 +3,7 @@ package config import ( "testing" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/testutil" ) func isolateConfig(t *testing.T) { diff --git a/internal/config/validate.go b/internal/config/validate.go index f8800d0..ab89cd3 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -38,6 +38,7 @@ func ValidateConfig(cfg *Config) error { return fmt.Errorf("logging: %w", err) } + stdinReader := "" for i, p := range cfg.Pipelines { if err := lconfig.NonEmpty(p.Name); err != nil { return fmt.Errorf("pipeline[%d].name: %w", i, err) @@ -48,6 +49,15 @@ func ValidateConfig(cfg *Config) error { if len(p.PluginSinks) == 0 { return fmt.Errorf("pipeline[%d]: no sinks defined", i) } + for _, src := range p.PluginSources { + if src.Type != "console" { + continue + } + if stdinReader != "" { + return fmt.Errorf("pipeline %q: a console source already reads stdin in pipeline %q", p.Name, stdinReader) + } + stdinReader = p.Name + } } return nil diff --git a/internal/core/capability.go b/internal/core/capability.go index 8d3abcb..a25c4cf 100644 --- a/internal/core/capability.go +++ b/internal/core/capability.go @@ -15,6 +15,9 @@ const ( CapMultiSession Capability = "multi_session" CapSingleInstance Capability = "single_instance" + // Flow capabilities + CapBackpressure Capability = "backpressure" // a full input stalls the pipeline instead of dropping + // Stream capabilities CapBidirectional Capability = "bidirectional" CapCompression Capability = "compression" diff --git a/internal/core/const_test.go b/internal/core/const_test.go index 3b07545..14af31c 100644 --- a/internal/core/const_test.go +++ b/internal/core/const_test.go @@ -4,7 +4,7 @@ import ( "net" "testing" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/testutil" ) // The host literal picks the family, strictly: an IPv4 literal or the empty diff --git a/internal/filter/chain.go b/internal/filter/chain.go index 0738769..a714a05 100644 --- a/internal/filter/chain.go +++ b/internal/filter/chain.go @@ -4,8 +4,8 @@ import ( "fmt" "sync/atomic" - "logwisp/internal/config" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" "github.com/lixenwraith/log" ) diff --git a/internal/filter/filter.go b/internal/filter/filter.go index b860fd4..e25c677 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -6,8 +6,8 @@ import ( "sync" "sync/atomic" - "logwisp/internal/config" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/flow/flow.go b/internal/flow/flow.go index 2cfa76a..2ca9088 100644 --- a/internal/flow/flow.go +++ b/internal/flow/flow.go @@ -5,10 +5,10 @@ import ( "fmt" "sync/atomic" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/filter" - "logwisp/internal/format" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/filter" + "github.com/lixenwraith/logwisp/internal/format" "github.com/lixenwraith/log" ) diff --git a/internal/flow/heartbeat.go b/internal/flow/heartbeat.go index 09f1b20..b6371a5 100644 --- a/internal/flow/heartbeat.go +++ b/internal/flow/heartbeat.go @@ -8,9 +8,9 @@ import ( "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/format" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/format" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/flow/heartbeat_test.go b/internal/flow/heartbeat_test.go index 0649bf9..1afaeb9 100644 --- a/internal/flow/heartbeat_test.go +++ b/internal/flow/heartbeat_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" - "logwisp/internal/config" - "logwisp/internal/format" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/format" ) // include_timestamp = false drops the timestamp the formatter shows by default diff --git a/internal/flow/ratelimiter.go b/internal/flow/ratelimiter.go index bdf3c6d..286b2ff 100644 --- a/internal/flow/ratelimiter.go +++ b/internal/flow/ratelimiter.go @@ -5,9 +5,9 @@ import ( "strings" "sync/atomic" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/tokenbucket" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/tokenbucket" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/format/adapter.go b/internal/format/adapter.go index 16a81cb..0cac80f 100644 --- a/internal/format/adapter.go +++ b/internal/format/adapter.go @@ -6,8 +6,8 @@ import ( "fmt" "sync" - "logwisp/internal/config" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log/formatter" @@ -98,15 +98,21 @@ func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int6 return a.serialize(entry, customFlags), nil } -// serialize renders an entry under the given flags. The returned slice is a -// copy: the underlying formatter reuses one buffer and sinks retain payloads. +// serialize renders an entry under the given flags as one record ending in +// one newline, which raw output lacks unless the message carried it. The +// result is a copy: the formatter reuses one buffer and sinks keep payloads. func (a *FormatterAdapter) serialize(entry core.LogEntry, flags int64) []byte { args, flags := formatArgs(entry, flags) a.mu.Lock() - out := bytes.Clone(a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args)) + out := a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args) + record := make([]byte, len(out), len(out)+1) + copy(record, out) a.mu.Unlock() - return out + if !bytes.HasSuffix(record, []byte{'\n'}) { + record = append(record, '\n') + } + return record } // formatArgs pairs the entry with its flags. FlagRaw keeps the fields JSON diff --git a/internal/format/adapter_test.go b/internal/format/adapter_test.go new file mode 100644 index 0000000..06b2716 --- /dev/null +++ b/internal/format/adapter_test.go @@ -0,0 +1,34 @@ +package format + +import ( + "bytes" + "testing" + "time" + + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" +) + +// Every payload is one record ending in exactly one newline, whatever the +// type and whether or not the message carried its own. +func TestEveryRecordEndsInOneNewline(t *testing.T) { + entries := []core.LogEntry{ + {Message: "plain"}, + {Message: "carried\n"}, + {Fields: []byte(`{"k":1}`)}, + {Message: "with", Fields: []byte(`{"k":1}`)}, + } + for _, typ := range []string{"raw", "txt", "json"} { + f, err := NewFormatter(&config.FormatConfig{Type: typ}) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + e.Time = time.Now() + out, _ := f.Format(e) + if !bytes.HasSuffix(out, []byte("\n")) || bytes.HasSuffix(out, []byte("\n\n")) { + t.Errorf("%s %+v: %q", typ, e, out) + } + } + } +} diff --git a/internal/format/format.go b/internal/format/format.go index 59177c9..5cafc00 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -1,8 +1,8 @@ package format import ( - "logwisp/internal/config" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" ) // Formatter defines the interface for transforming a LogEntry into a byte slice diff --git a/internal/pipeline/pipeline.go b/internal/pipeline/pipeline.go index 2d07f2e..20e37de 100644 --- a/internal/pipeline/pipeline.go +++ b/internal/pipeline/pipeline.go @@ -7,12 +7,12 @@ import ( "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/flow" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/flow" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) @@ -32,11 +32,17 @@ type Pipeline struct { Stats *PipelineStats logger *log.Logger + // Sinks that stall dispatch rather than drop (core.CapBackpressure) + backpressure map[string]bool + // Runtime - ctx context.Context - cancel context.CancelFunc - wg sync.WaitGroup - running atomic.Bool + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + running atomic.Bool + halt chan struct{} // closed by Stop: backpressure sinks drop again + finished chan struct{} // closed when every source has ended + finishedOnce sync.Once } // PipelineStats contains runtime statistics for a pipeline @@ -70,6 +76,9 @@ func NewPipeline( logger: logger, ctx: pipelineCtx, cancel: pipelineCancel, + + backpressure: make(map[string]bool), + finished: make(chan struct{}), } // Create flow processor @@ -199,6 +208,9 @@ func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig var hasTLS, hasAuth bool for _, c := range s.Capabilities() { switch c { + case core.CapBackpressure: + p.backpressure[cfg.ID] = true + // Network capabilities case core.CapNetLimit: continue // No-op for now, placeholder @@ -224,31 +236,31 @@ func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig return nil } -// run is the central processing loop that connects sources, flow, and sinks -func (p *Pipeline) run() { +// run connects the subscribed sources through the flow to the sinks. It +// returns once every source channel has closed, at Stop or at end of input. +func (p *Pipeline) run(subscriptions []<-chan core.LogEntry) { defer p.wg.Done() + defer p.finish() defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name) var componentWg sync.WaitGroup - // Start a goroutine for each source to fan-in data - for _, src := range p.Sources { + for _, ch := range subscriptions { componentWg.Add(1) - go func(s source.Source) { + go func() { defer componentWg.Done() - ch := s.Subscribe() - // Range allows in-flight data to drain cleanly once Source.Stop() closes the channel + // Range drains in-flight entries once the source closes the channel for entry := range ch { if event, passed := p.Flow.Process(entry); passed { - // Use non-blocking dispatcher p.dispatch(event) } } - }(src) + }() } + // Sinks keep p.ctx until Stop, so the heartbeat ends on its own context + hbCtx, hbCancel := context.WithCancel(p.ctx) var hbWg sync.WaitGroup - // Start heartbeat generator if enabled - if heartbeatCh := p.Flow.StartHeartbeat(p.ctx); heartbeatCh != nil { + if heartbeatCh := p.Flow.StartHeartbeat(hbCtx); heartbeatCh != nil { hbWg.Add(1) go func() { defer hbWg.Done() @@ -258,9 +270,8 @@ func (p *Pipeline) run() { if !ok { return } - // Use non-blocking dispatcher p.dispatch(event) - case <-p.ctx.Done(): + case <-hbCtx.Done(): return } } @@ -268,25 +279,36 @@ func (p *Pipeline) run() { } componentWg.Wait() - - // Terminate internal contexts (heartbeat) once flow is complete - p.cancel() + hbCancel() hbWg.Wait() } -// dispatch performs a non-blocking send to all sinks. -// A full/stalled sink must never block the run loop or starve sibling sinks. +// dispatch sends to every sink. A full sink drops the event so it cannot +// stall its siblings, except a backpressure sink, which waits until Stop. func (p *Pipeline) dispatch(event core.TransportEvent) { - for _, snk := range p.Sinks { + for id, snk := range p.Sinks { + if p.backpressure[id] { + select { + case snk.Input() <- event: + case <-p.halt: + p.Stats.TotalEntriesDroppedBySink.Add(1) + } + continue + } select { case snk.Input() <- event: default: - // Buffer full - drop to avoid deadlocking the pipeline p.Stats.TotalEntriesDroppedBySink.Add(1) } } } +func (p *Pipeline) finish() { p.finishedOnce.Do(func() { close(p.finished) }) } + +// Finished is closed when every source has ended, or the pipeline shut down. +// Only a finite source (console at end of input) ends without Stop. +func (p *Pipeline) Finished() <-chan struct{} { return p.finished } + // Start starts the pipeline operation and all its components including flow, sources, and sinks func (p *Pipeline) Start() error { if !p.running.CompareAndSwap(false, true) { @@ -295,6 +317,7 @@ func (p *Pipeline) Start() error { p.logger.Info("msg", "Starting pipeline", "pipeline", p.Config.Name) p.ctx, p.cancel = context.WithCancel(context.Background()) + p.halt = make(chan struct{}) // Start all sinks for id, s := range p.Sinks { @@ -303,7 +326,11 @@ func (p *Pipeline) Start() error { } } - // Start all sources + // Subscribed before Start: a source publishes from its first entry on + subscriptions := make([]<-chan core.LogEntry, 0, len(p.Sources)) + for _, src := range p.Sources { + subscriptions = append(subscriptions, src.Subscribe()) + } for id, src := range p.Sources { if err := src.Start(); err != nil { return fmt.Errorf("failed to start source %s: %w", id, err) @@ -313,7 +340,7 @@ func (p *Pipeline) Start() error { // Start the central processing loop p.Stats.StartTime = time.Now() p.wg.Add(1) - go p.run() + go p.run(subscriptions) return nil } @@ -325,6 +352,7 @@ func (p *Pipeline) Stop() error { } p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name) + close(p.halt) // 1. Stop all sources concurrently to halt new data ingress and close their channels var sourceWg sync.WaitGroup @@ -338,7 +366,6 @@ func (p *Pipeline) Stop() error { sourceWg.Wait() // 2. Wait for the run loop to finish processing and sending all in-flight data - // run() inherently calls p.cancel() when the source channels are empty p.wg.Wait() // 3. Stop all sinks concurrently now that no new data will be sent @@ -351,6 +378,7 @@ func (p *Pipeline) Stop() error { }(s) } sinkWg.Wait() + p.cancel() p.logger.Info("msg", "Pipeline stopped", "pipeline", p.Config.Name) return nil @@ -373,6 +401,7 @@ func (p *Pipeline) Shutdown() { if p.Sessions != nil { p.Sessions.Stop() } + p.finish() // a pipeline that never started has no run loop to close it p.logger.Info("msg", "Pipeline shutdown complete", "component", "pipeline", diff --git a/internal/pipeline/pipeline_test.go b/internal/pipeline/pipeline_test.go new file mode 100644 index 0000000..301dd6f --- /dev/null +++ b/internal/pipeline/pipeline_test.go @@ -0,0 +1,132 @@ +package pipeline + +import ( + "context" + "fmt" + "strconv" + "testing" + "time" + + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/source" + + "github.com/lixenwraith/log" +) + +// finiteSource publishes n entries as soon as it starts, then ends +type finiteSource struct { + n int + subs []chan core.LogEntry +} + +func (s *finiteSource) Capabilities() []core.Capability { return nil } +func (s *finiteSource) Subscribe() <-chan core.LogEntry { + ch := make(chan core.LogEntry, 1) + s.subs = append(s.subs, ch) + return ch +} +func (s *finiteSource) Start() error { + go func() { + for i := range s.n { + for _, ch := range s.subs { + ch <- core.LogEntry{Time: time.Now(), Message: strconv.Itoa(i)} + } + } + for _, ch := range s.subs { + close(ch) + } + }() + return nil +} +func (s *finiteSource) Stop() {} +func (s *finiteSource) GetStats() source.SourceStats { return source.SourceStats{} } + +// slowSink applies backpressure; a nil out never reads at all +type slowSink struct { + in chan core.TransportEvent + out chan string +} + +func (s *slowSink) Capabilities() []core.Capability { return []core.Capability{core.CapBackpressure} } +func (s *slowSink) Input() chan<- core.TransportEvent { return s.in } +func (s *slowSink) Start(context.Context) error { + if s.out != nil { + go func() { + for e := range s.in { + time.Sleep(10 * time.Microsecond) + s.out <- e.Entry.Message + } + }() + } + return nil +} +func (s *slowSink) Stop() {} +func (s *slowSink) GetStats() sink.SinkStats { return sink.SinkStats{} } + +var testSinks = map[string]*slowSink{} + +func init() { + plugin.RegisterSource("test_finite", func(id string, cfg map[string]any, _ *log.Logger, _ *session.Proxy) (source.Source, error) { + n, _ := strconv.Atoi(fmt.Sprint(cfg["n"])) + return &finiteSource{n: n}, nil + }) + plugin.RegisterSink("test_slow", func(id string, _ map[string]any, _ *log.Logger, _ *session.Proxy) (sink.Sink, error) { + return testSinks[id], nil + }) +} + +func newTestPipeline(t *testing.T, n int, s *slowSink) *Pipeline { + t.Helper() + testSinks[t.Name()] = s + p, err := NewPipeline(&config.PipelineConfig{ + Name: t.Name(), + PluginSources: []config.PluginSourceConfig{{ID: "src", Type: "test_finite", Config: map[string]any{"n": n}}}, + PluginSinks: []config.PluginSinkConfig{{ID: t.Name(), Type: "test_slow"}}, + }, log.NewLogger()) + if err != nil { + t.Fatal(err) + } + return p +} + +// A source that publishes from its first moment loses nothing to a slow sink, +// and the pipeline reports Finished when its input ends. +func TestFiniteInputReachesBackpressureSinkWhole(t *testing.T) { + const n = 2000 + s := &slowSink{in: make(chan core.TransportEvent, 1), out: make(chan string, n)} + p := newTestPipeline(t, n, s) + if err := p.Start(); err != nil { + t.Fatal(err) + } + select { + case <-p.Finished(): + case <-time.After(10 * time.Second): + t.Fatal("pipeline did not finish at the end of its input") + } + for i := range n { + if got := <-s.out; got != strconv.Itoa(i) { + t.Fatalf("entry %d = %s", i, got) + } + } + p.Shutdown() +} + +// Stop releases a dispatch blocked on a sink that never reads +func TestStopReleasesStalledBackpressureSink(t *testing.T) { + p := newTestPipeline(t, 100, &slowSink{in: make(chan core.TransportEvent, 1)}) + if err := p.Start(); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + stopped := make(chan struct{}) + go func() { p.Shutdown(); close(stopped) }() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("Stop blocked behind a stalled backpressure sink") + } +} diff --git a/internal/pipeline/registry.go b/internal/pipeline/registry.go index ce74dec..d31f44a 100644 --- a/internal/pipeline/registry.go +++ b/internal/pipeline/registry.go @@ -4,10 +4,10 @@ import ( "fmt" "sync" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) diff --git a/internal/plugin/factory.go b/internal/plugin/factory.go index d92c6c4..c05c294 100644 --- a/internal/plugin/factory.go +++ b/internal/plugin/factory.go @@ -4,10 +4,10 @@ import ( "fmt" "sync" - "logwisp/internal/core" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) diff --git a/internal/sanitize/sanitize.go b/internal/sanitize/sanitize.go deleted file mode 100644 index 365b922..0000000 --- a/internal/sanitize/sanitize.go +++ /dev/null @@ -1,70 +0,0 @@ -package sanitize - -import ( - "encoding/hex" - "strconv" - "strings" - "unicode/utf8" -) - -// String sanitizes a string by replacing non-printable characters with hex encoding -// Non-printable characters are encoded as (e.g., newline becomes <0a>) -func String(data string) string { - // Fast path: check if sanitization is needed - needsSanitization := false - for _, r := range data { - if !strconv.IsPrint(r) { - needsSanitization = true - break - } - } - - if !needsSanitization { - return data - } - - // Pre-allocate builder for efficiency - var builder strings.Builder - builder.Grow(len(data)) - - for _, r := range data { - if strconv.IsPrint(r) { - builder.WriteRune(r) - } else { - // Encode non-printable rune as - var runeBytes [utf8.UTFMax]byte - n := utf8.EncodeRune(runeBytes[:], r) - builder.WriteByte('<') - builder.WriteString(hex.EncodeToString(runeBytes[:n])) - builder.WriteByte('>') - } - } - - return builder.String() -} - -// Bytes sanitizes a byte slice by converting to string and sanitizing -func Bytes(data []byte) []byte { - return []byte(String(string(data))) -} - -// Rune sanitizes a single rune, returning its string representation -func Rune(r rune) string { - if strconv.IsPrint(r) { - return string(r) - } - - var runeBytes [utf8.UTFMax]byte - n := utf8.EncodeRune(runeBytes[:], r) - return "<" + hex.EncodeToString(runeBytes[:n]) + ">" -} - -// IsSafe checks if a string contains only printable characters -func IsSafe(data string) bool { - for _, r := range data { - if !strconv.IsPrint(r) { - return false - } - } - return true -} diff --git a/internal/service/service.go b/internal/service/service.go index c72cd28..180b73b 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -6,8 +6,8 @@ import ( "fmt" "sync" - "logwisp/internal/config" - "logwisp/internal/pipeline" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/pipeline" "github.com/lixenwraith/log" ) @@ -21,6 +21,8 @@ type Service struct { wg sync.WaitGroup logger *log.Logger shutdownOnce sync.Once + watchOnce sync.Once + done chan struct{} // closed when every pipeline has finished } // NewService creates a new, empty service @@ -31,6 +33,7 @@ func NewService(ctx context.Context, cfg *config.Config, logger *log.Logger) (*S ctx: serviceCtx, cancel: cancel, logger: logger, + done: make(chan struct{}), } var errs error @@ -89,10 +92,26 @@ func (svc *Service) Start(names ...string) error { } svc.logger.Debug("msg", "Finished starting pipeline(s)", "pipelines", names) + svc.watchOnce.Do(func() { + pipelines := make([]*pipeline.Pipeline, 0, len(svc.pipelines)) + for _, p := range svc.pipelines { + pipelines = append(pipelines, p) + } + go func() { + for _, p := range pipelines { + <-p.Finished() + } + close(svc.done) + }() + }) return errs } +// Done is closed once every pipeline has finished: all their sources reached +// the end of their input (stdin), or the service shut down. +func (svc *Service) Done() <-chan struct{} { return svc.done } + // Stop stops all or specific pipeline func (svc *Service) Stop(names ...string) error { svc.mu.RLock() diff --git a/internal/session/session.go b/internal/session/session.go index ce54790..836af17 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -8,7 +8,7 @@ import ( "sync" "time" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/core" ) // Session represents a connection session diff --git a/internal/sink/console/console.go b/internal/sink/console/console.go index 113174d..96cf6b4 100644 --- a/internal/sink/console/console.go +++ b/internal/sink/console/console.go @@ -1,21 +1,26 @@ package console import ( + "bytes" "context" + "encoding/hex" "fmt" "io" "os" + "sync" "sync/atomic" "time" + "unicode/utf8" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" + "golang.org/x/term" ) // init registers the component in plugin factory @@ -25,7 +30,7 @@ func init() { } } -// ConsoleSink writes log entries to the console (stdout/stderr) using an dedicated logger instance +// ConsoleSink writes formatted entries to stdout or stderr type ConsoleSink struct { // Plugin identity and session management id string @@ -34,6 +39,7 @@ type ConsoleSink struct { // Configuration config *config.ConsoleSinkOptions + escape bool // Application input chan core.TransportEvent @@ -42,6 +48,9 @@ type ConsoleSink struct { // Runtime done chan struct{} + exited chan struct{} + started atomic.Bool + stopOnce sync.Once startTime time.Time // Statistics @@ -53,6 +62,7 @@ const ( // Defaults DefaultConsoleTarget = "stdout" DefaultConsoleBufferSize = 1000 + DefaultConsoleEscape = "auto" ) // NewConsoleSinkPlugin creates a console sink through plugin factory @@ -62,28 +72,22 @@ func NewConsoleSinkPlugin( logger *log.Logger, proxy *session.Proxy, ) (sink.Sink, error) { - opts := &config.ConsoleSinkOptions{} + opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape} // Scan config map into struct if err := config.Scan(configMap, opts); err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) } - // Validate and apply defaults - if opts.Target == "" { - opts.Target = DefaultConsoleTarget - } else { - validateTarget := lconfig.OneOf("stdout", "stderr") - if err := validateTarget(opts.Target); err != nil { - return nil, fmt.Errorf("target: %w", err) - } + if err := lconfig.OneOf("stdout", "stderr")(opts.Target); err != nil { + return nil, fmt.Errorf("target: %w", err) + } + if err := lconfig.OneOf("auto", "always", "never")(opts.Escape); err != nil { + return nil, fmt.Errorf("escape: %w", err) } - var output io.Writer - switch opts.Target { - case "stdout": - output = os.Stdout - case "stderr": + output := os.Stdout + if opts.Target == "stderr" { output = os.Stderr } @@ -96,9 +100,11 @@ func NewConsoleSinkPlugin( id: id, proxy: proxy, config: opts, + escape: opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())), input: make(chan core.TransportEvent, opts.BufferSize), output: output, done: make(chan struct{}), + exited: make(chan struct{}), logger: logger, } cs.lastProcessed.Store(time.Time{}) @@ -117,6 +123,7 @@ func NewConsoleSinkPlugin( "component", "console_sink", "instance_id", id, "target", opts.Target, + "escape", cs.escape, ) return cs, nil @@ -126,6 +133,7 @@ func NewConsoleSinkPlugin( func (cs *ConsoleSink) Capabilities() []core.Capability { return []core.Capability{ core.CapSessionAware, // Single output session + core.CapBackpressure, // a slow reader slows the pipeline, as with any filter } } @@ -137,6 +145,7 @@ func (cs *ConsoleSink) Input() chan<- core.TransportEvent { // Start begins the processing loop func (cs *ConsoleSink) Start(ctx context.Context) error { cs.startTime = time.Now() + cs.started.Store(true) go cs.processLoop(ctx) cs.logger.Info("msg", "Console sink started", "component", "console_sink", @@ -144,7 +153,7 @@ func (cs *ConsoleSink) Start(ctx context.Context) error { return nil } -// Stop gracefully shuts down the sink +// Stop writes what is queued, then returns func (cs *ConsoleSink) Stop() { cs.logger.Info("msg", "Stopping console sink", "target", cs.config.Target) @@ -153,12 +162,14 @@ func (cs *ConsoleSink) Stop() { cs.proxy.RemoveSession(cs.session.ID) } - close(cs.done) + cs.stopOnce.Do(func() { close(cs.done) }) + if cs.started.Load() { + <-cs.exited + } cs.logger.Info("msg", "Console sink stopped", "instance_id", cs.id, "target", cs.config.Target, - "instance_id", cs.id, ) } @@ -175,35 +186,100 @@ func (cs *ConsoleSink) GetStats() sink.SinkStats { Details: map[string]any{ "target": cs.config.Target, "buffer_size": cs.config.BufferSize, + "escape": cs.escape, }, } } -// processLoop reads transport events and writes to console +// processLoop writes transport events until stopped, then what is queued func (cs *ConsoleSink) processLoop(ctx context.Context) { + defer close(cs.exited) for { select { - case event, ok := <-cs.input: - if !ok { - return - } - - // Write pre-formatted payload directly to output - if _, err := cs.output.Write(event.Payload); err != nil { - cs.logger.Error("msg", "Failed to write to console", - "component", "console_sink", - "target", cs.config.Target, - "error", err) - continue - } - - cs.totalProcessed.Add(1) - cs.lastProcessed.Store(time.Now()) - + case event := <-cs.input: + cs.write(event) case <-ctx.Done(): + cs.drain() return case <-cs.done: + cs.drain() return } } } + +func (cs *ConsoleSink) drain() { + for { + select { + case event := <-cs.input: + cs.write(event) + default: + return + } + } +} + +func (cs *ConsoleSink) write(event core.TransportEvent) { + payload := event.Payload + if cs.escape { + payload = escapeControls(payload) + } + if _, err := cs.output.Write(payload); err != nil { + cs.logger.Error("msg", "Failed to write to console", + "component", "console_sink", + "target", cs.config.Target, + "error", err) + return + } + cs.totalProcessed.Add(1) + cs.lastProcessed.Store(time.Now()) +} + +// escapeControls writes what a terminal would act on or reorder (C0 and C1 +// controls but tab, DEL, bidi controls, line separators, invalid UTF-8) as +// , keeping the final newline: a log line cannot drive the terminal. +// Other invisible characters stay: emoji and Indic scripts need ZWJ. +func escapeControls(p []byte) []byte { + body, newline := bytes.CutSuffix(p, []byte{'\n'}) + if printableASCII(body) { + return p + } + out := make([]byte, 0, len(p)+16) + for len(body) > 0 { + r, n := utf8.DecodeRune(body) + if !terminalControl(r, n) { + out = append(out, body[:n]...) + } else { + out = append(out, '<') + out = hex.AppendEncode(out, body[:n]) + out = append(out, '>') + } + body = body[n:] + } + if newline { + out = append(out, '\n') + } + return out +} + +func terminalControl(r rune, size int) bool { + switch { + case r == utf8.RuneError && size == 1: + return true + case r < 0x20: + return r != '\t' + case r >= 0x7f && r <= 0x9f, r == 0x061c, r == 0x200e, r == 0x200f, + r >= 0x2028 && r <= 0x202e, r >= 0x2066 && r <= 0x2069: + return true + } + return false +} + +func printableASCII(b []byte) bool { + for _, c := range b { + if (c < 0x20 || c > 0x7e) && c != '\t' { + return false + } + } + return true +} diff --git a/internal/sink/console/console_test.go b/internal/sink/console/console_test.go new file mode 100644 index 0000000..695022e --- /dev/null +++ b/internal/sink/console/console_test.go @@ -0,0 +1,25 @@ +package console + +import "testing" + +// Everything a terminal would act on is escaped; tabs and the final newline, +// which it only lays out, are kept. +func TestEscapeControlsKeepsOnlyLayout(t *testing.T) { + for in, want := range map[string]string{ + "plain\ttext\n": "plain\ttext\n", + "a\x1b[2Jb\n": "a<1b>[2Jb\n", + "bell\x07 del\x7f\n": "bell<07> del<7f>\n", + "inner\nnewline\n": "inner<0a>newline\n", + "c1 \u009b31m\n": "c1 31m\n", + "bidi \u202eevil\n": "bidi evil\n", + "isolate \u2066x\u2069\n": "isolate x\n", + "line\u2028sep\n": "linesep\n", + "zwj \U0001f468\u200d\U0001f469\n": "zwj \U0001f468\u200d\U0001f469\n", + "bad \xff utf8": "bad utf8", + "ünïcødé and \ufffd kept\n": "ünïcødé and \ufffd kept\n", + } { + if got := string(escapeControls([]byte(in))); got != want { + t.Errorf("%q: got %q, want %q", in, got, want) + } + } +} diff --git a/internal/sink/file/file.go b/internal/sink/file/file.go index da8f0bb..f5abe56 100644 --- a/internal/sink/file/file.go +++ b/internal/sink/file/file.go @@ -6,11 +6,11 @@ import ( "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" @@ -41,6 +41,8 @@ type FileSink struct { // Runtime done chan struct{} + exited chan struct{} + started atomic.Bool startTime time.Time // Statistics @@ -132,6 +134,7 @@ func NewFileSinkPlugin( writer: log.NewLogger(), writerConfig: writerConfig, done: make(chan struct{}), + exited: make(chan struct{}), logger: logger, } fs.lastProcessed.Store(time.Time{}) @@ -178,6 +181,7 @@ func (fs *FileSink) Start(ctx context.Context) error { } fs.startTime = time.Now() + fs.started.Store(true) go fs.processLoop(ctx) fs.logger.Info("msg", "File sink started", @@ -205,7 +209,11 @@ func (fs *FileSink) Stop() { "directory", fs.config.Directory, "name", fs.config.Name) + // The loop writes what is queued before the writer shuts down close(fs.done) + if fs.started.Load() { + <-fs.exited + } // Remove session if fs.session != nil { @@ -240,27 +248,37 @@ func (fs *FileSink) GetStats() sink.SinkStats { } } -// processLoop reads transport events and writes to file +// processLoop writes transport events until stopped, then what is queued func (fs *FileSink) processLoop(ctx context.Context) { + defer close(fs.exited) for { select { - case event, ok := <-fs.input: - if !ok { - return - } - - // Write the pre-formatted payload directly - // The writer handles rotation automatically based on configuration - fs.writer.Write(string(event.Payload)) - - fs.totalProcessed.Add(1) - fs.lastProcessed.Store(time.Now()) - + case event := <-fs.input: + fs.write(event) case <-ctx.Done(): + fs.drain() return - case <-fs.done: + fs.drain() return } } } + +func (fs *FileSink) drain() { + for { + select { + case event := <-fs.input: + fs.write(event) + default: + return + } + } +} + +// write hands the formatted payload to the writer, which rotates files +func (fs *FileSink) write(event core.TransportEvent) { + fs.writer.Write(string(event.Payload)) + fs.totalProcessed.Add(1) + fs.lastProcessed.Store(time.Now()) +} diff --git a/internal/sink/http/http.go b/internal/sink/http/http.go index 961d2ea..187828a 100644 --- a/internal/sink/http/http.go +++ b/internal/sink/http/http.go @@ -16,15 +16,15 @@ import ( "sync/atomic" "time" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/tlsx" - "logwisp/internal/version" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/version" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/sink/http/http_test.go b/internal/sink/http/http_test.go index 7445906..ac1deff 100644 --- a/internal/sink/http/http_test.go +++ b/internal/sink/http/http_test.go @@ -16,9 +16,9 @@ import ( "testing" "time" - "logwisp/internal/core" - "logwisp/internal/sink" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/auth" "github.com/lixenwraith/log" diff --git a/internal/sink/http/test_helpers_test.go b/internal/sink/http/test_helpers_test.go index ec3894c..4e2cf78 100644 --- a/internal/sink/http/test_helpers_test.go +++ b/internal/sink/http/test_helpers_test.go @@ -9,9 +9,9 @@ import ( "testing" "time" - "logwisp/internal/authz" - "logwisp/internal/session" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/auth" "github.com/lixenwraith/log" diff --git a/internal/sink/http/web.go b/internal/sink/http/web.go index 91cf0a7..a023002 100644 --- a/internal/sink/http/web.go +++ b/internal/sink/http/web.go @@ -9,9 +9,9 @@ import ( "path" "strings" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" ) //go:embed web/*.js web/*.html web/*.css diff --git a/internal/sink/httpchain/httpchain.go b/internal/sink/httpchain/httpchain.go index 34e439a..3d15a10 100644 --- a/internal/sink/httpchain/httpchain.go +++ b/internal/sink/httpchain/httpchain.go @@ -17,14 +17,14 @@ import ( "sync/atomic" "time" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/tlsx" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/sink/httpchain/httpchain_test.go b/internal/sink/httpchain/httpchain_test.go index 9853551..5b3196f 100644 --- a/internal/sink/httpchain/httpchain_test.go +++ b/internal/sink/httpchain/httpchain_test.go @@ -12,13 +12,13 @@ import ( "testing" "time" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/session" - ingest "logwisp/internal/source/httpchain" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/session" + ingest "github.com/lixenwraith/logwisp/internal/source/httpchain" + "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/auth" "github.com/lixenwraith/log" diff --git a/internal/sink/null/null.go b/internal/sink/null/null.go index 897bad7..3703c72 100644 --- a/internal/sink/null/null.go +++ b/internal/sink/null/null.go @@ -6,11 +6,11 @@ import ( "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" "github.com/lixenwraith/log" ) diff --git a/internal/sink/sink.go b/internal/sink/sink.go index 9c7af68..14fd4fd 100644 --- a/internal/sink/sink.go +++ b/internal/sink/sink.go @@ -4,7 +4,7 @@ import ( "context" "time" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/core" ) // Sink represents an output data stream. diff --git a/internal/sink/tcp/tcp.go b/internal/sink/tcp/tcp.go index 39df5cc..87cec18 100644 --- a/internal/sink/tcp/tcp.go +++ b/internal/sink/tcp/tcp.go @@ -12,13 +12,13 @@ import ( "sync/atomic" "time" - "logwisp/internal/authz" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/tlsx" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/sink/tcp/tcp_test.go b/internal/sink/tcp/tcp_test.go index b880123..1aa8712 100644 --- a/internal/sink/tcp/tcp_test.go +++ b/internal/sink/tcp/tcp_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" - "logwisp/internal/session" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/testutil" "github.com/lixenwraith/log" ) diff --git a/internal/sink/tcpchain/tcpchain.go b/internal/sink/tcpchain/tcpchain.go index fbb2e6d..0c4c8bb 100644 --- a/internal/sink/tcpchain/tcpchain.go +++ b/internal/sink/tcpchain/tcpchain.go @@ -14,14 +14,14 @@ import ( "sync/atomic" "time" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/sink" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/sink" + "github.com/lixenwraith/logwisp/internal/tlsx" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/sink/tcpchain/tcpchain_test.go b/internal/sink/tcpchain/tcpchain_test.go index 2aa9cf0..fca7f8b 100644 --- a/internal/sink/tcpchain/tcpchain_test.go +++ b/internal/sink/tcpchain/tcpchain_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "logwisp/internal/core" - "logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/session" "github.com/lixenwraith/log" ) diff --git a/internal/source/console/console.go b/internal/source/console/console.go index 4140dd7..c7d74bd 100644 --- a/internal/source/console/console.go +++ b/internal/source/console/console.go @@ -2,16 +2,19 @@ package console import ( "bufio" + "errors" "fmt" + "io" "os" + "sync" "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) @@ -31,6 +34,61 @@ func init() { } } +// One reader serves the process: after a reload the new source continues +// where the old one stopped, and no line is read twice. The channel is +// unbuffered, so a busy pipeline stops the reading of stdin, not its lines. +var ( + stdinOnce sync.Once + stdinLines chan string + stdinErr error // set before stdinLines closes +) + +func lines() <-chan string { + stdinOnce.Do(func() { + stdinLines = make(chan string) + go func() { + stdinErr = readLines(os.Stdin, stdinLines) + close(stdinLines) + }() + }) + return stdinLines +} + +// readLines sends r's lines without their terminator (\n or \r\n), the last +// one too when unterminated, until the end of input. A line longer than +// core.MaxLogEntryBytes continues in the next one. +func readLines(r io.Reader, out chan<- string) error { + br := bufio.NewReaderSize(r, 64*1024) + var line []byte + for { + chunk, err := br.ReadSlice('\n') + line = append(line, chunk...) + for len(line) > core.MaxLogEntryBytes { + out <- string(line[:core.MaxLogEntryBytes]) + line = append(line[:0], line[core.MaxLogEntryBytes:]...) + } + switch { + case errors.Is(err, bufio.ErrBufferFull): + continue + case err == nil: + line = line[:len(line)-1] + if n := len(line); n > 0 && line[n-1] == '\r' { + line = line[:n-1] + } + } + if len(line) > 0 || err == nil { + out <- string(line) + } + line = line[:0] + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + } +} + // ConsoleSource reads log entries from the standard input stream type ConsoleSource struct { // Plugin identity and session management @@ -46,13 +104,14 @@ type ConsoleSource struct { logger *log.Logger // Runtime - done chan struct{} + done chan struct{} + stopOnce sync.Once // Statistics - totalEntries atomic.Uint64 - droppedEntries atomic.Uint64 - startTime time.Time - lastEntryTime atomic.Value // time.Time + totalEntries atomic.Uint64 + startTime time.Time + lastEntryTime atomic.Value // time.Time + ended atomic.Bool // standard input reached its end } const ( @@ -122,7 +181,7 @@ func (s *ConsoleSource) Subscribe() <-chan core.LogEntry { // Start begins reading from the standard input. func (s *ConsoleSource) Start() error { s.startTime = time.Now() - go s.readLoop() + go s.readLoop(lines()) // Update session activity s.proxy.UpdateActivity(s.session.ID) @@ -133,20 +192,15 @@ func (s *ConsoleSource) Start() error { return nil } -// Stop signals the source to stop reading. +// Stop signals the source to stop reading; readLoop closes the subscribers. func (s *ConsoleSource) Stop() { - close(s.done) + s.stopOnce.Do(func() { close(s.done) }) // Remove session if s.session != nil { s.proxy.RemoveSession(s.session.ID) } - // Close subscriber channels - for _, ch := range s.subscribers { - close(ch) - } - s.logger.Info("msg", "Console source stopped", "component", "console_source", "instance_id", s.id) @@ -157,67 +211,76 @@ func (s *ConsoleSource) GetStats() source.SourceStats { lastEntry, _ := s.lastEntryTime.Load().(time.Time) return source.SourceStats{ - Type: "console", - TotalEntries: s.totalEntries.Load(), - DroppedEntries: s.droppedEntries.Load(), - StartTime: s.startTime, - LastEntryTime: lastEntry, - Details: map[string]any{}, + ID: s.id, + Type: "console", + TotalEntries: s.totalEntries.Load(), + StartTime: s.startTime, + LastEntryTime: lastEntry, + Details: map[string]any{"ended": s.ended.Load()}, } } -// readLoop continuously reads lines from stdin and publishes them -func (s *ConsoleSource) readLoop() { - scanner := bufio.NewScanner(os.Stdin) - for scanner.Scan() { +// readLoop publishes stdin's lines until Stop or the end of input, then +// closes the subscriber channels: it is their only sender. +func (s *ConsoleSource) readLoop(lines <-chan string) { + defer func() { + for _, ch := range s.subscribers { + close(ch) + } + }() + for { select { case <-s.done: return - default: - // Update session activity on each read - s.proxy.UpdateActivity(s.session.ID) - - // Get raw line - lineBytes := scanner.Bytes() - if len(lineBytes) == 0 { + case line, ok := <-lines: + if !ok { + s.ended.Store(true) + if stdinErr != nil { + s.logger.Error("msg", "Failed to read standard input", + "component", "console_source", + "instance_id", s.id, + "error", stdinErr) + } + s.logger.Info("msg", "Standard input ended", + "component", "console_source", + "instance_id", s.id) + return + } + if line == "" { continue } - - // Add newline back (scanner strips it) - lineWithNewline := append(lineBytes, '\n') - + s.proxy.UpdateActivity(s.session.ID) entry := core.LogEntry{ Time: time.Now(), Source: "console", - Message: string(lineWithNewline), // Keep newline - Level: source.ExtractLogLevel(string(lineBytes)), - RawSize: int64(len(lineWithNewline)), + Message: line, + Level: source.ExtractLogLevel(line), + RawSize: int64(len(line)), + } + if !s.publish(entry) { + return } - - s.publish(entry) } } - - if err := scanner.Err(); err != nil { - s.logger.Error("msg", "Scanner error reading stdin", - "component", "console_source", - "instance_id", s.id, - "error", err) - } } -// publish sends a log entry to all subscribers -func (s *ConsoleSource) publish(entry core.LogEntry) { +// publish waits for every subscriber: stdin is pulled, so a full pipeline +// slows the reading instead of losing lines. False when stopped meanwhile. +func (s *ConsoleSource) publish(entry core.LogEntry) bool { s.totalEntries.Add(1) s.lastEntryTime.Store(entry.Time) for _, ch := range s.subscribers { select { - case ch <- entry: + case ch <- entry: // first: a select with both ready picks at random + continue default: - s.droppedEntries.Add(1) - s.logger.Debug("msg", "Dropped log entry - subscriber buffer full", - "component", "console_source") + } + select { + case ch <- entry: + case <-s.done: + return false } } + return true } diff --git a/internal/source/file/file.go b/internal/source/file/file.go index b27e6db..06b3e0a 100644 --- a/internal/source/file/file.go +++ b/internal/source/file/file.go @@ -13,11 +13,11 @@ import ( "syscall" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/source" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/source/file/file_test.go b/internal/source/file/file_test.go index 895b890..45bd66b 100644 --- a/internal/source/file/file_test.go +++ b/internal/source/file/file_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "logwisp/internal/core" - "logwisp/internal/testutil" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/testutil" ) func TestStoppedWatcherReturnsNormally(t *testing.T) { diff --git a/internal/source/file/file_watcher.go b/internal/source/file/file_watcher.go index 0cf26d8..1d28b2a 100644 --- a/internal/source/file/file_watcher.go +++ b/internal/source/file/file_watcher.go @@ -13,8 +13,8 @@ import ( "syscall" "time" - "logwisp/internal/core" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) @@ -379,12 +379,11 @@ func (w *fileWatcher) isStopped() bool { // parseLine converts a line into an entry, as JSON when nothing would be lost func (w *fileWatcher) parseLine(line string) core.LogEntry { if w.raw { - // Newline restored: sinks write the payload as it stands return core.LogEntry{ Time: time.Now(), Source: filepath.Base(w.directory), Level: source.ExtractLogLevel(line), - Message: line + "\n", + Message: line, } } diff --git a/internal/source/httpchain/httpchain.go b/internal/source/httpchain/httpchain.go index d9922ed..2850021 100644 --- a/internal/source/httpchain/httpchain.go +++ b/internal/source/httpchain/httpchain.go @@ -15,14 +15,14 @@ import ( "sync/atomic" "time" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/source" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/tlsx" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/source/null/null.go b/internal/source/null/null.go index 2d7a598..b2f3ade 100644 --- a/internal/source/null/null.go +++ b/internal/source/null/null.go @@ -5,11 +5,11 @@ import ( "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/source" "github.com/lixenwraith/log" ) diff --git a/internal/source/random/random.go b/internal/source/random/random.go index bf640c9..6ee1037 100644 --- a/internal/source/random/random.go +++ b/internal/source/random/random.go @@ -8,11 +8,11 @@ import ( "sync/atomic" "time" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/source" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/source/source.go b/internal/source/source.go index f342b23..516e93a 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -4,7 +4,7 @@ import ( "strings" "time" - "logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/core" ) // Source represents an input data stream for log entries diff --git a/internal/source/tcpchain/tcpchain.go b/internal/source/tcpchain/tcpchain.go index afb6930..1dcd7e9 100644 --- a/internal/source/tcpchain/tcpchain.go +++ b/internal/source/tcpchain/tcpchain.go @@ -13,14 +13,14 @@ import ( "sync/atomic" "time" - "logwisp/internal/authz" - "logwisp/internal/chain" - "logwisp/internal/config" - "logwisp/internal/core" - "logwisp/internal/plugin" - "logwisp/internal/session" - "logwisp/internal/source" - "logwisp/internal/tlsx" + "github.com/lixenwraith/logwisp/internal/authz" + "github.com/lixenwraith/logwisp/internal/chain" + "github.com/lixenwraith/logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/plugin" + "github.com/lixenwraith/logwisp/internal/session" + "github.com/lixenwraith/logwisp/internal/source" + "github.com/lixenwraith/logwisp/internal/tlsx" lconfig "github.com/lixenwraith/config" "github.com/lixenwraith/log" diff --git a/internal/tlsx/tlsx.go b/internal/tlsx/tlsx.go index b29884c..fbcc0c8 100644 --- a/internal/tlsx/tlsx.go +++ b/internal/tlsx/tlsx.go @@ -14,7 +14,7 @@ import ( "sync" "time" - "logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/log" ) diff --git a/internal/tlsx/tlsx_test.go b/internal/tlsx/tlsx_test.go index d2a6155..0bb4b2c 100644 --- a/internal/tlsx/tlsx_test.go +++ b/internal/tlsx/tlsx_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "logwisp/internal/config" + "github.com/lixenwraith/logwisp/internal/config" ) // A dialer verifies an IPv6 target by its address: the zone names a local diff --git a/internal/version/version.go b/internal/version/version.go index 630b683..545a5eb 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -1,6 +1,9 @@ package version -import "fmt" +import ( + "fmt" + "runtime/debug" +) var ( // Version is the application version, set at compile time via -ldflags @@ -11,11 +14,28 @@ var ( BuildTime = "unknown" ) +// init fills what -ldflags left unset from the build info: `go install +// module@version` stamps the module version, a build in a checkout the commit. +func init() { + info, ok := debug.ReadBuildInfo() + if !ok { + return + } + if v := info.Main.Version; Version == "dev" && v != "" && v != "(devel)" { + Version = v + } + for _, s := range info.Settings { + switch { + case s.Key == "vcs.revision" && GitCommit == "unknown": + GitCommit = s.Value[:min(len(s.Value), 12)] + case s.Key == "vcs.time" && BuildTime == "unknown": + BuildTime = s.Value + } + } +} + // String returns a detailed, formatted version string including commit and build time func String() string { - if Version == "dev" { - return fmt.Sprintf("dev (commit: %s, built: %s)", GitCommit, BuildTime) - } return fmt.Sprintf("%s (commit: %s, built: %s)", Version, GitCommit, BuildTime) } diff --git a/test/lib.sh b/test/lib.sh index dc40759..3a0e591 100755 --- a/test/lib.sh +++ b/test/lib.sh @@ -18,6 +18,9 @@ if [[ ${FORCE_COLOR:-0} != 0 ]] || [[ -t 1 && -z ${NO_COLOR:-} ]]; then else C_BOLD='' C_DIM='' C_RED='' C_GREEN='' C_YELLOW='' C_CYAN='' C_OFF='' fi +# Rules fit a narrower terminal; wider ones keep 64 columns +COLS=${COLUMNS:-$(stty size <&2 2>/dev/null | cut -d' ' -f2)} +[[ $COLS =~ ^[0-9]+$ ]] && ((COLS >= 20 && COLS < 64)) || COLS=64 usage() { sed -n '2,/^[^#]/s/^# \{0,1\}//p' "$0" @@ -55,9 +58,9 @@ e2e_init() { # "$@" of the script # --- output --- -rule() { local s='' n=$1; while ((n-- > 0)); do s+='─'; done; printf '%s' "$s"; } +rule() { local s='' n=$1; while ((n-- > 0)); do s+=${2:-─}; done; printf '%s' "$s"; } # N [CHAR] short() { if [[ $1 == "$PWD"/* ]]; then printf '%s' "${1#"$PWD"/}"; else printf '%s' "$1"; fi; } -section() { printf '\n%s── %s %s%s\n' "$C_BOLD$C_CYAN" "$1" "$(rule $((60 - ${#1})))" "$C_OFF"; } +section() { printf '\n%s── %s %s%s\n' "$C_BOLD$C_CYAN" "$1" "$(rule $((COLS - 4 - ${#1})))" "$C_OFF"; } info() { printf ' %s·%s %s\n' "$C_DIM" "$C_OFF" "$*"; } pass() { E2E_PASSED=$((E2E_PASSED + 1)); printf ' %sPASS%s %s\n' "$C_BOLD$C_GREEN" "$C_OFF" "$1"; } @@ -67,20 +70,21 @@ check() { if (($2)); then pass "$1"; else fail "$1"; fi; } # LABEL 1|0 is() { [[ $1 == "$2" ]] && echo 1 || echo 0; } # ACTUAL EXPECTED is_set() { [[ -n $1 ]] && echo 1 || echo 0; } -# guide TITLE, the body on stdin: "> " lines are commands, " NNNNN " lines -# ports, lines ending in ":" headings. +# guide TITLE, the body on stdin, between two rules and without a side border, +# so lines wrapped by a narrow terminal stay readable: "> " lines are commands, +# " NNNNN " lines ports, lines ending in ":" headings. guide() { local line - printf '\n%s┌── %s %s%s\n' "$C_BOLD" "$1" "$(rule $((59 - ${#1})))" "$C_OFF" + printf '\n%s══ %s %s%s\n' "$C_BOLD" "$1" "$(rule $((COLS - 4 - ${#1})) ═)" "$C_OFF" while IFS= read -r line; do case $line in '> '*) line=" $C_CYAN${line#> }$C_OFF" ;; ' '[0-9][0-9][0-9][0-9][0-9]' '*) line=" $C_YELLOW${line:2:5}$C_OFF${line:7}" ;; *:) line="$C_BOLD$line$C_OFF" ;; esac - printf '%s│%s %s\n' "$C_BOLD" "$C_OFF" "$line" + printf '%s\n' "$line" done - printf '%s└%s%s\n' "$C_BOLD" "$(rule 63)" "$C_OFF" + printf '%s%s%s\n' "$C_BOLD" "$(rule "$COLS" ═)" "$C_OFF" } # write_env NAME=VALUE...: $RUN/env, which the guide's commands source @@ -119,7 +123,7 @@ summary() { [[ $name == "${ABORTED:-}" ]] || excerpt "$name" '^([^ ]+ ERROR |Error: )' done fi - printf '%s%s%s\n' "$C_DIM" "$(rule 64)" "$C_OFF" + printf '%s%s%s\n' "$C_DIM" "$(rule "$COLS")" "$C_OFF" if ((E2E_FAILED)); then printf '%sRESULT: FAILURES%s (%s); logs in %s/\n' "$C_BOLD$C_RED" "$C_OFF" "$counts" "$(short "$LOG")" exit 1 @@ -134,7 +138,7 @@ summary() { skip_all() { # REASON: nothing in the script applies to this host skip "$1" - printf '%s%s%s\n%sRESULT: SKIPPED%s\n' "$C_DIM" "$(rule 64)" "$C_OFF" "$C_BOLD$C_YELLOW" "$C_OFF" + printf '%s%s%s\n%sRESULT: SKIPPED%s\n' "$C_DIM" "$(rule "$COLS")" "$C_OFF" "$C_BOLD$C_YELLOW" "$C_OFF" exit 77 } diff --git a/test/pipe-test.sh b/test/pipe-test.sh new file mode 100755 index 0000000..2d579a7 --- /dev/null +++ b/test/pipe-test.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# logwisp pipe test: without a configuration file lw is a filter, stdin to +# stdout line for line, without loss, quiet on stderr, exiting 0 at the end of +# input. One-shot: both modes run the checks; no ports. +# Requires: bash 5+, coreutils (timeout, seq, head, cmp, od). Linux dev host only. + +set -u +. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" + +RUN=$E2E_DIR/run/pipe +e2e_init "$@" + +section "Setup" +need +rm -rf "$RUN" +mkdir -p "$RUN/bare" "$RUN/withfile" "$LOG" +printf '[logging]\noutput = "stderr"\n' >"$RUN/withfile/logwisp.toml" +# lw DIR ARGS...: in DIR, with no user configuration or LOGWISP_ variable +lw() { + ( + cd "$RUN/$1" || exit 1 + for v in $(compgen -e LOGWISP_); do unset "$v"; done + HOME=$RUN timeout 30 "$BIN" "${@:2}" + ) +} +info "runs lw in $(short "$RUN")/bare (no file) and $(short "$RUN")/withfile" + +section "Checks" +# 1. line for line, without loss, and nothing on stderr +seq 1 200000 >"$RUN/seq.txt" +lw bare <"$RUN/seq.txt" >"$RUN/seq.out" 2>"$LOG/seq.err" +rc=$? +check "200000 lines pass unchanged, exit $rc" $((rc == 0)) +check "output identical to input" "$(cmp -s "$RUN/seq.txt" "$RUN/seq.out" && echo 1 || echo 0)" +check "nothing on stderr without a configuration file" "$( [[ -s $LOG/seq.err ]] && echo 0 || echo 1)" + +# 2. sources are subscribed before they start: the first line is not lost +check "a single line arrives" "$(is "$(printf 'first\n' | lw bare)" first)" + +# 3. terminators: CRLF stripped, blank lines skipped, last line unterminated +check "\\r\\n, blank and unterminated lines" "$(is "$(printf 'a\r\nb\n\nc' | lw bare | od -An -c | tr -s ' ')" " a \n b \n c \n")" + +# 4. a line over the 1 MiB entry cap continues in the next entry, no byte lost +head -c 2500000 /dev/zero | tr '\0' x | lw bare >"$RUN/long.out" +check "2.5 MB line: entries of $(awk '{printf "%s ", length($0)}' "$RUN/long.out")" \ + "$(is "$(awk '{n += length($0)} END {print NR, n}' "$RUN/long.out")" "3 2500000")" + +# 5. a slow reader slows lw down instead of losing lines +n=$(seq 1 50000 | lw bare | { sleep 1; wc -l; }) +check "slow reader receives all 50000 lines ($n)" "$((n == 50000))" + +# 6. a reader that leaves ends lw as it ends cat: SIGPIPE +seq 1 5000000 | lw bare | head -1 >/dev/null +rc=${PIPESTATUS[1]} +check "reader gone: lw exits on SIGPIPE ($rc)" "$((rc == 141))" + +# 7. filters and formats compose with stdin and stdout +check "--filter keeps matching lines" "$(is "$(printf 'x INFO\ny ERROR\n' | lw bare --filter include,patterns=ERROR)" "y ERROR")" + +# 8. control characters pass to a pipe unchanged and are escaped on request +check "control bytes reach a pipe unchanged" "$(is "$(printf 'a\033[31mb\n' | lw bare | od -An -tx1 | tr -d ' ')" 611b5b33316d620a)" +check "escape=always writes them as " "$(is "$(printf 'a\033[31mb\n' | lw bare --sink console,escape=always)" 'a<1b>[31mb')" + +# 9. at the end of input the file sink writes its queue before lw exits +seq 1 5000 | lw bare --sink "file,directory=$RUN/fsink,name=out,buffer_size=10000" +n=$(cat "$RUN"/fsink/out*.log 2>/dev/null | wc -l) +check "file sink holds all 5000 lines at exit ($n)" "$((n == 5000))" + +# 10. a configuration file makes lw a service: info on stderr, data on stdout +out=$(printf 'x\n' | lw withfile 2>"$LOG/withfile.err") +check "with a file: data still on stdout" "$(is "$out" x)" +check "with a file: info logging on stderr" "$(grep -q 'LogWisp starting' "$LOG/withfile.err" && echo 1 || echo 0)" + +summary