v0.26.6 cli, doc, and web viewer updates

cmd/lw/cli_test.go

@@ -127,6 +127,47 @@ func TestUsageListsEachFlagOnce(t *testing.T) {
 	}
 }
 
+// lw --help and its manual list every option lw takes, and only those, a short
+// beside its long: lw's own, each configuration key, and the pipeline flags
+func TestHelpAndManualListEveryOption(t *testing.T) {
+	manual, err := os.ReadFile("../../doc/lw.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var help bytes.Buffer
+	printHelp(&help)
+	options := map[string]string{} // long -> its short, if lw takes one
+	for _, long := range config.SpecFlags() {
+		options[long] = ""
+	}
+	for long := range config.Settings() {
+		options[long] = ""
+	}
+	for letter, short := range shorts {
+		if short.top {
+			options[short.long] = letter
+		}
+	}
+	// mdoc's Fl writes the first dash, and the others as \-
+	fl := func(flag string) string { return regexp.QuoteMeta("Fl " + strings.ReplaceAll(flag[1:], "-", `\-`)) }
+	for long, letter := range options {
+		short, item := "    ", `^\.It (Fl \\-.*)?` // only long options before it
+		if letter != "" {
+			short, item = "-"+letter+", ", `^\.It `+fl("-"+letter)+` .*`
+		}
+		option := short + "--" + long
+		if !regexp.MustCompile(`(?m)^  ` + regexp.QuoteMeta(option) + `([^\w.-]|$)`).Match(help.Bytes()) {
+			t.Errorf("lw --help does not list %s", strings.TrimSpace(option))
+		}
+		if !regexp.MustCompile(`(?m)` + item + fl("--"+long) + `([^\w.\\]|$)`).Match(manual) {
+			t.Errorf("doc/lw.1 does not list %s", strings.TrimSpace(option))
+		}
+	}
+	if heads := regexp.MustCompile(`(?m)^  (-., |    )--\w`).FindAll(help.Bytes(), -1); len(heads) != len(options) {
+		t.Errorf("lw --help lists %d options, want %d", len(heads), len(options))
+	}
+}
+
 // --dump prints a file that lw -c reads back to the same configuration, its
 // console sinks still taking the top-level color
 func TestDumpReadsBack(t *testing.T) {

cmd/lw/help.go

@@ -2,12 +2,13 @@ package main
 
 import (
 	"fmt"
+	"io"
 	"strings"
 
 	"github.com/lixenwraith/logwisp/internal/version"
 )
 
-// helpText is the CLI usage reference. Scalar flags map 1:1 to TOML config paths.
+// helpText lists every option lw takes; man lw is the full reference.
 const helpText = `LogWisp %s - log collection, processing, and distribution
 
 Usage:
@@ -32,30 +33,55 @@ after '='; short options do not combine. lw takes options only.
       --dump                    Print the effective configuration, exit
   -q, --quiet                   Silence lw's own log and notices; pipeline
                                 output still flows
-      --color [WHEN]            Level names in color on console sinks:
+      --color [WHEN]            Level names in color on console sinks: auto
-                                auto (a terminal, NO_COLOR unset; default),
+                                (on a terminal; default), always (a bare
-                                always (bare --color) or never
+                                --color) or never
   -V, --version                 Print the version and exit
   -h, --help                    Print this help and exit
       --                        End the options
 
-Configuration keys are flags by their TOML path, '_' and '.' kept:
+Configuration keys are options by their TOML path, '_' and '.' kept; a BOOL
-      --KEY=VALUE               e.g. --logging.level=debug
+key alone means true:
-      --status_reporter=BOOL    Periodic status logging (default: true with
+      --status_reporter=BOOL    Log pipeline statistics every 30 s at debug
-                                a configuration file)
+                                (default: true with a configuration file)
       --auto_reload=BOOL        Reload when the file changes (default: false)
-  Logging (lw's own log goes to stderr, at info with a configuration file and
+  lw's own log:
-  at warn without one):
+      --logging.output=MODE     stderr (default), stdout, split (warn and
-      --logging.output=MODE     file|stdout|stderr|split|all|none
+                                error to stderr, the rest to stdout), file,
-      --logging.level=LEVEL     debug|info|warn|error
+                                all (split and file) or none
+      --logging.level=LEVEL     debug|info|warn|error (default: info with a
+                                configuration file, warn without)
+      --logging.format=FORMAT   txt|json|raw (default: txt)
+      --logging.sanitization=POLICY
+                                raw (default: unchanged), txt (hex for '<'
+                                and non-printables), json (escapes control
+                                characters) or shell (strips; lossy)
       --logging.file.directory=DIR
+                                Where file and all write (default: ./log);
+                                lw's alone: the limits below delete any old
+                                .log file in it
+      --logging.file.name=NAME  File name base (default: logwisp)
+      --logging.file.max_size_mb=N
+                                Start a new file past N MB (default: 100)
+      --logging.file.max_total_size_mb=N
+                                Delete the oldest .log files there past N MB
+                                in all (default: 1000)
+      --logging.file.retention_hours=H
+                                Delete .log files there older than H hours
+                                (default: 168; 0 keeps them)
+      --logging.console.target=TARGET
+                                Accepted without effect: --logging.output
+                                chooses the console
 
-Pipelines (replace the file's pipelines; see doc/cli.md):
+Pipelines (they replace the configuration file's pipelines):
   -p, --preset NAME[,KEY=VALUE...]
                                 Start a pipeline with a preset: pipe, tail,
                                 serve, edge, aggregator; lw preset NAME -h
                                 lists its keys, lw preset NAME prints it
-      --pipeline NAME           Start a pipeline; earlier specs go to "cli"
+      --pipeline NAME           Start a pipeline named NAME, which the
+                                pipeline options after it configure; those
+                                before any --pipeline make one named after
+                                its --preset, or "cli" without one
       --source SPEC             TYPE[,KEY=VALUE...], repeatable
       --sink SPEC               e.g. http,host=0.0.0.0,port=8080, repeatable
       --filter SPEC             include|exclude,patterns=RE, repeatable
@@ -79,29 +105,32 @@ Examples:
 password_file=edge-01.pass                        Forward to an aggregator
 
 Environment:
-  LOGWISP_<PATH>                Config path, '.' -> '_', uppercase
+  LOGWISP_<KEY>                 A configuration key, '.' as '_', uppercase:
-                                e.g. LOGWISP_LOGGING_LEVEL=debug
+                                LOGWISP_LOGGING_LEVEL=debug
   LOGWISP_CONFIG_FILE           Configuration file path
   LOGWISP_CONFIG_DIR            Configuration directory
   LOGWISP_PIPELINE, LOGWISP_PRESET, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N],
   LOGWISP_FILTER[_N], LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT
                                 One pipeline, specs as the flags (_N adds
                                 more); ignored when a pipeline flag is given
+  NO_COLOR, TERM=dumb           Turn --color auto off
 
 Signals:
   SIGINT, SIGTERM               Graceful shutdown
   SIGHUP, SIGUSR1               Reload configuration
 
 Exit codes:
-  0  success, including the end of input
+  0  Success, including the end of input
-  1  general error, including a configuration that does not load
+  1  Failure, including a configuration that does not load
-  2  usage error, or a named configuration file not found
+  2  Usage error, or a named configuration file not found
+
+The full reference, with every key, preset and command: man lw
 `
 
-func printHelp() {
+func printHelp(w io.Writer) {
 	var list strings.Builder
 	for _, c := range commands {
 		fmt.Fprintf(&list, "  lw %-26s %s\n", c.name+" COMMAND", c.summary)
 	}
-	fmt.Printf(helpText, version.Short(), list.String())
+	fmt.Fprintf(w, helpText, version.Short(), list.String())
 }

cmd/lw/main.go

@@ -28,7 +28,7 @@ func main() {
 	case inv.command != nil:
 		os.Exit(inv.command.run(inv.args, os.Stdout, os.Stderr))
 	case inv.help:
-		printHelp()
+		printHelp(os.Stdout)
 		os.Exit(0)
 	}
 

doc/cli.md

@@ -75,7 +75,8 @@ pipelines have [their own flags](#pipelines):
 - `--auto_reload=BOOL`: reload when the configuration file changes, default
   `false`
 - `-V`, `--version`: print the version and exit
-- `-h`, `--help`, `help`: print usage and exit
+- `-h`, `--help`, `help`: print every option, pointing to `man lw` for the
+  full reference, and exit
 
 The last `-c` wins; a missing or empty path is an error.
 
@@ -255,7 +256,8 @@ The path resolver reads these variables directly:
 One pipeline can come from the environment, with the [SPEC](#pipelines) syntax.
 Any pipeline flag on the command line makes LogWisp ignore all of them.
 
-- `LOGWISP_PIPELINE`: the pipeline's name, default `cli`
+- `LOGWISP_PIPELINE`: the pipeline's name, default the `LOGWISP_PRESET` name,
+  else `cli`
 - `LOGWISP_PRESET`: the [preset](#presets) that starts it
 - `LOGWISP_SOURCE`, `LOGWISP_SINK`, `LOGWISP_FILTER`: one stage each
   - `LOGWISP_SOURCE_1` .. `LOGWISP_SOURCE_N` add more, after the unnumbered

doc/lw.1

@@ -1,4 +1,4 @@
-.Dd October 4, 2026
+.Dd October 5, 2026
 .Dt LW 1
 .Os
 .Sh NAME
@@ -138,12 +138,23 @@ Read the configuration from
 .Ar file ,
 which must exist.
 The last one given wins.
-.It Fl \- Ns Ar key Ns = Ns Ar value , Fl \- Ns Ar key Ar value
+.It Fl t , Fl \-check
-Set any scalar configuration key by its TOML path, such as
+Build every pipeline and plugin as a start would, loading options, TLS files
-.Fl \-logging.level=debug .
+and credentials files and printing startup warnings, without binding a port
-A bare
+or opening, reading or creating a log file; print
-.Fl \- Ns Ar key
+.Dq configuration ok
-sets a boolean to true.
+and exit 0, or the error and exit 1, or 2 when a named configuration file is
+missing
+.Pq see Sx EXIT STATUS .
+.It Fl \-dump
+Print the effective configuration, with the options, environment and
+presets resolved into it, as a file
+.Fl c
+reads back, and exit.
+.It Fl q , Fl \-quiet
+Turn off the log of
+.Nm
+and the terminal reminder; console sinks still write.
 .It Fl \-color Op Ar when
 Color the level names console sinks write:
 .Cm auto
@@ -160,10 +171,25 @@ program parses keeps
 .Cm auto
 or
 .Cm never .
-.It Fl q , Fl \-quiet
+.It Fl V , Fl \-version
-Turn off the log of
+Print the version, commit and build time, and exit.
-.Nm
+.It Fl h , Fl \-help , Cm help
-and the terminal reminder; console sinks still write.
+Print a summary of every option and exit.
+.It Fl \-
+End option processing.
+.El
+.Ss Configuration keys
+Every scalar configuration key is an option by its TOML path,
+.Fl \- Ns Ar key Ns = Ns Ar value
+or
+.Fl \- Ns Ar key Ar value ,
+such as
+.Fl \-logging.level=debug ;
+a bare
+.Fl \- Ns Ar key
+sets a boolean to true.
+Besides the options above, the keys are:
+.Bl -tag -width Ds
 .It Fl \-status_reporter= Ns Ar bool
 Log pipeline statistics every 30 seconds at the debug level
 .Pq default true with a configuration file, false without .
@@ -171,7 +197,9 @@ Log pipeline statistics every 30 seconds at the debug level
 Reload when the configuration file changes
 .Pq default false .
 .It Fl \-logging.output= Ns Ar mode
-One of
+Where the log of
+.Nm
+itself goes:
 .Cm stderr
 .Pq the default ,
 .Cm stdout ,
@@ -194,25 +222,70 @@ the default is
 with a configuration file and
 .Cm warn
 without.
-.It Fl t , Fl \-check
+.It Fl \-logging.format= Ns Ar format
-Build every pipeline and plugin as a start would, loading options, TLS files
+One of
-and credentials files and printing startup warnings, without binding a port
+.Cm txt
-or opening, reading or creating a log file; print
+.Pq the default ,
-.Dq configuration ok
+.Cm json
-and exit 0, or the error and exit 1, or 2 when a named configuration file is
+or
-missing
+.Cm raw .
-.Pq see Sx EXIT STATUS .
+.It Fl \-logging.sanitization= Ns Ar policy
-.It Fl \-dump
+How the log's text is cleaned:
-Print the effective configuration, with the options, environment and
+.Cm raw
-presets resolved into it, as a file
+.Pq the default: unchanged ,
-.Fl c
+.Cm txt
-reads back, and exit.
+.Pq Ql < No and non-printable characters as hex ,
-.It Fl V , Fl \-version
+.Cm json
-Print the version, commit and build time, and exit.
+.Pq control characters escaped
-.It Fl h , Fl \-help , Cm help
+or
-Print usage and exit.
+.Cm shell
-.It Fl \-
+.Pq shell metacharacters, whitespace and control characters removed; lossy .
-End option processing.
+.It Fl \-logging.file.directory= Ns Ar dir
+The directory the
+.Cm file
+and
+.Cm all
+modes write in
+.Pq default Pa ./log .
+It must be
+.Nm Ns 's
+alone: the limits below count and delete every
+.Pa .log
+file in it but the one being written, and old ones are also deleted when the
+disk has less than 10 MB free.
+.It Fl \-logging.file.name= Ns Ar name
+The log file's name before
+.Pa .log
+.Pq default Cm logwisp ;
+see
+.Sx FILES .
+.It Fl \-logging.file.max_size_mb= Ns Ar n
+Start a new file when the current one would pass
+.Ar n
+megabytes
+.Pq default 100, 0 for no limit .
+.It Fl \-logging.file.max_total_size_mb= Ns Ar n
+Delete the oldest
+.Pa .log
+files in the directory when all of them pass
+.Ar n
+megabytes
+.Pq default 1000, 0 for no limit .
+.It Fl \-logging.file.retention_hours= Ns Ar hours
+Delete the
+.Pa .log
+files in the directory older than
+.Ar hours
+.Pq default 168, 0 to keep them .
+.It Fl \-logging.console.target= Ns Ar target
+Accepted, as
+.Cm stdout ,
+.Cm stderr
+or
+.Cm split ,
+and ignored:
+.Fl \-logging.output
+chooses the stream.
 .El
 .Ss Pipeline specifications
 Pipelines can be given on the command line or in the environment instead of
@@ -228,12 +301,16 @@ Standard input has one reader: a second console source anywhere in the
 configuration fails validation.
 .Bl -tag -width Ds
 .It Fl \-pipeline Ar name
-Start a new pipeline.
+Start a pipeline named
-Specifications before the first
+.Ar name ,
-.Fl \-pipeline
+which the pipeline options after it configure.
-belong to a pipeline named
+Those before any
-.Dq cli ,
+.Fl \-pipeline ,
-or after the preset that starts them.
+and the variables without
+.Ev LOGWISP_PIPELINE ,
+make a pipeline named after its preset, or
+.Dq cli
+without one.
 .It Fl p Ar spec , Fl \-preset Ar spec
 Start the pipeline with a preset, described in
 .Sx Presets ;
@@ -510,6 +587,10 @@ turns
 .Cm auto
 color off, as does
 .Ev TERM=dumb .
+.It Ev HOME
+Where
+.Pa ~/.config/logwisp/logwisp.toml
+is looked for.
 .El
 .Pp
 Secrets never go in the environment: TLS keys, credentials files and
@@ -546,13 +627,31 @@ and used by
 .It Pa /var/lib/logwisp , /var/db/logwisp
 The service's working directory on Linux and on
 .Fx .
+.It Pa ./log/logwisp.log
+The log of
+.Nm
+itself with
+.Fl \-logging.output=file
+or
+.Cm all ,
+in
+.Fl \-logging.file.directory
+and named by
+.Fl \-logging.file.name ;
+a full one is renamed with its time, as
+.Pa logwisp_YYMMDD_HHMMSS.log .
 .El
 .Sh EXIT STATUS
 .Nm
-exits 0 at the end of input, after a clean shutdown,
+exits 0 at the end of input, after a clean shutdown, and for
-.Fl \-version
+.Fl \-version ,
-or help; 1 on a configuration, logger or startup error; and 2 on a usage
+help,
-error or when a configuration file named by
+.Fl \-dump
+and a configuration
+.Fl \-check
+accepts; 1 on a failure, such as a configuration that does not load, or a
+logger or startup error; and 2 on a usage error or when a configuration file
+named by
 .Fl c
 or
 .Ev LOGWISP_CONFIG_*
@@ -581,6 +680,10 @@ Shut down gracefully.
 Reread the configuration file, certificates and credentials, and rebuild
 every pipeline.
 An invalid configuration leaves the running service untouched.
+Before the service runs,
+.Dv SIGHUP
+is ignored, as under
+.Xr nohup 1 .
 .El
 .Sh EXAMPLES
 Copy a file line by line:
@@ -674,10 +777,10 @@ generate the password into a file, then reload the server:
 .Xr tmpfiles.d 5 ,
 .Xr daemon 8
 .Pp
-The documentation installed in
+The guides to configuration, plugins, security and deployment, installed in
 .Pa share/doc/logwisp
-under the installation prefix, and
+under the installation prefix and kept at
-.Lk https://github.com/lixenwraith/logwisp
+.Lk https://github.com/lixenwraith/logwisp/tree/main/doc
 .Sh SECURITY CONSIDERATIONS
 A listener without an
 .Cm auth

doc/sinks.md

@@ -217,8 +217,9 @@ Options, each as type and default:
   - an endpoint at `/` keeps the root, and the viewer stays at `/auth/view`.
   The viewer starts with the `replay_lines` backlog, holds one stream against
   `max_connections`, and says so when all are taken (`server full`). Its
-  search keeps the lines containing a text, in any case, and its level list
+  search keeps the lines containing a text, in any case unless its `Aa` toggle
-  the entries at or above a level, read as the sources read it.
+  matches case, and its level list the entries at or above a level, read as
+  the sources read it.
 - Refusals are logged at WARN and counted in `auth_rejected`. The authorized
   identity is recorded in the client's session as `auth_method` /
   `auth_identity`.

internal/config/loader.go

@@ -41,7 +41,9 @@ func Settings() map[string]bool {
 	}
 	settings := map[string]bool{}
 	for path, value := range c.GetRegisteredPathsWithDefaults() {
-		_, settings[path] = value.(bool)
+		if _, tables := value.([]PipelineConfig); !tables { // the pipeline flags set those
+			_, settings[path] = value.(bool)
+		}
 	}
 	return settings
 }

internal/sink/http/web/style.css

@@ -104,6 +104,59 @@ progress {
   border-bottom: 1px solid var(--line);
 }
 
+/* The search field and its match-case toggle read as one input */
+.search {
+  display: flex;
+  border: 1px solid var(--line);
+  border-radius: 6px;
+  background: var(--bg);
+  overflow: hidden;
+}
+
+/* The field's ring is the input's own focus; the toggle draws its own */
+.search:has(#search:focus) {
+  outline: 2px solid var(--accent);
+  outline-offset: -1px;
+}
+
+.search input {
+  width: 11rem;
+  border: 0;
+  outline: none;
+}
+
+.search button {
+  padding: 0 0.6rem;
+  border-left: 1px solid var(--line);
+  border-radius: 0;
+  background: none;
+  color: var(--muted);
+  font-size: 0.85rem;
+  font-weight: 600;
+}
+
+.search button:hover {
+  color: var(--fg);
+}
+
+.search button[aria-pressed="true"] {
+  background: var(--accent);
+  color: var(--panel);
+}
+
+@media (forced-colors: active) {
+  .search button[aria-pressed="true"] {
+    forced-color-adjust: none;
+    background: Highlight;
+    color: HighlightText;
+  }
+}
+
+.search button:focus-visible {
+  outline: 2px solid var(--accent);
+  outline-offset: -4px;
+}
+
 #state {
   flex: 1;
   color: var(--muted);

internal/sink/http/web/view.html

@@ -14,7 +14,10 @@
 <body class="viewer">
 <header>
   <h1>LogWisp</h1>
-  <input id="search" type="search" placeholder="Search" aria-label="Show lines containing">
+  <span class="search">
+    <input id="search" type="search" placeholder="Search" aria-label="Show lines containing">
+    <button id="case" type="button" aria-pressed="false" aria-label="Match case" title="Match case">Aa</button>
+  </span>
   <select id="level" aria-label="Lowest level shown"><option value="">All lines</option></select>
   <span id="state" role="status">connecting…</span>
   <button id="logout" type="button">Sign out</button>

internal/sink/http/web/view.js

@@ -14,6 +14,7 @@ const form = document.getElementById("login");
 const progress = document.getElementById("progress");
 const status = document.getElementById("status");
 const search = document.getElementById("search");
+const matchCase = document.getElementById("case");
 const lowest = document.getElementById("level");
 // The server's levels by rising severity, each with the words naming it
 const levels = JSON.parse(meta("levels"));
@@ -43,13 +44,15 @@ function levelOf(text) {
   return "";
 }
 
-// A line shows when it contains the search, in any case, and its entry's level
+// A line shows when it contains the search, in any case unless matchCase is
-// reaches the lowest chosen; with none chosen, lines without a level show too
+// pressed, and its entry's level reaches the lowest chosen; with none chosen,
+// lines without a level show too
 function shown(row) {
   const floor = rank.get(lowest.value);
   if (floor !== undefined && !(rank.get(row.dataset.level) >= floor)) return false;
-  const text = search.value.trim().toLowerCase();
+  const fold = matchCase.getAttribute("aria-pressed") === "true" ? (s) => s : (s) => s.toLowerCase();
-  return !text || row.textContent.toLowerCase().includes(text);
+  const text = fold(search.value.trim());
+  return !text || fold(row.textContent).includes(text);
 }
 
 function refilter() {
@@ -165,6 +168,10 @@ function retry(reason) {
 }
 
 search.addEventListener("input", refilter);
+matchCase.addEventListener("click", () => {
+  matchCase.setAttribute("aria-pressed", String(matchCase.getAttribute("aria-pressed") !== "true"));
+  refilter();
+});
 lowest.addEventListener("change", refilter);
 
 form.addEventListener("submit", async (event) => {