v0.22.0 several enhancements and fixes for security, network, doc, test, packaging

cmd/lw/auth.go

@@ -20,6 +20,7 @@ import (
 
 	"logwisp/internal/authz"
 	"logwisp/internal/config"
+	"logwisp/internal/core"
 	"logwisp/internal/tlsx"
 
 	"github.com/lixenwraith/auth"
@@ -27,7 +28,7 @@ import (
 
 const reloadHint = "send SIGHUP to logwisp to apply (auto_reload does not watch the credentials file)"
 
-// authCommand is one `logwisp auth` subcommand: define registers its flags
+// authCommand is one `lw auth` subcommand: define registers its flags
 // and returns the command, which runs once they are parsed and checked.
 type authCommand struct {
 	name, synopsis, summary string
@@ -55,7 +56,7 @@ type usageError string
 
 func (e usageError) Error() string { return string(e) }
 
-// runAuth runs `logwisp auth` and returns the exit status: 0 success or
+// runAuth runs `lw auth` and returns the exit status: 0 success or
 // help, 1 failure, 2 usage error.
 func runAuth(args []string, stdout, stderr io.Writer) int {
 	if len(args) == 0 || slices.Contains([]string{"-h", "-help", "--help", "help"}, args[0]) {
@@ -67,12 +68,12 @@ func runAuth(args []string, stdout, stderr io.Writer) int {
 	}
 	i := slices.IndexFunc(authCommands, func(c authCommand) bool { return c.name == args[0] })
 	if i < 0 {
-		fmt.Fprintf(stderr, "logwisp auth: unknown command %q\n\n", args[0])
+		fmt.Fprintf(stderr, "lw auth: unknown command %q\n\n", args[0])
 		printAuthUsage(stderr)
 		return 2
 	}
 	c := authCommands[i]
-	fs := flag.NewFlagSet("logwisp auth "+c.name, flag.ContinueOnError)
+	fs := flag.NewFlagSet("lw auth "+c.name, flag.ContinueOnError)
 	fs.SetOutput(stderr)
 	fs.Usage = func() {
 		fmt.Fprintf(stderr, "Usage: %s %s\n\n%s.\n\n", fs.Name(), c.synopsis, c.summary)
@@ -101,11 +102,11 @@ func runAuth(args []string, stdout, stderr io.Writer) int {
 }
 
 func printAuthUsage(w io.Writer) {
-	fmt.Fprint(w, "Usage: logwisp auth <command> [flags]\n\n")
+	fmt.Fprint(w, "Usage: lw auth <command> [flags]\n\n")
 	for _, c := range authCommands {
 		fmt.Fprintf(w, "  %-12s %s\n", c.name, c.summary)
 	}
-	fmt.Fprint(w, "\nRun logwisp auth <command> -h for its flags. Credential changes apply on SIGHUP.\n"+
+	fmt.Fprint(w, "\nRun lw auth <command> -h for its flags. Credential changes apply on SIGHUP.\n"+
 		"Exit status: 0 success, 1 failure, 2 usage error.\n")
 }
 
@@ -231,7 +232,7 @@ func defineRemoveUser(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 }
 
 func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
-	rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT (with -unbound, a proxy's mount URL may carry a path)")
+	rawURL := fs.String("url", "", "listener `URL`, https://HOST:PORT or https://[IPV6]:PORT (with -unbound, a proxy's mount URL may carry a path)")
 	user := fs.String("user", "", "user `name`")
 	passwordFile := fs.String("password-file", "", "`file` holding the password")
 	unbound := fs.Bool("unbound", false, "log in without channel binding, to an http sink behind a TLS-terminating proxy (auth.trusted_proxies)")
@@ -245,6 +246,10 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 			// TLS passthrough cannot route by path: only a terminating proxy can
 			return usageError("-url takes a path only with -unbound")
 		}
+		network, err := core.Network(u.Hostname())
+		if err != nil {
+			return usageError("-url: " + err.Error())
+		}
 		tlsCfg, policy, err := dialPolicy(tlsOpts, u.Hostname(), *user, *passwordFile, authz.HTTP)
 		if err != nil {
 			return err
@@ -256,7 +261,7 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 			Transport: &http.Transport{
 				DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) {
 					var d net.Dialer
-					return d.DialContext(ctx, "tcp4", addr)
+					return d.DialContext(ctx, network, addr)
 				},
 				TLSClientConfig:     tlsCfg,
 				TLSHandshakeTimeout: tlsx.HandshakeTimeout,
@@ -264,7 +269,10 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 			// A redirect would carry the login to another endpoint
 			CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
 		}
-		token, err := policy.Token(context.Background(), client, "https://"+u.Host+u.Path)
+		// String escapes an IPv6 zone, which u.Host holds raw; the parts drop
+		// an empty "?" that would swallow the /auth path
+		base := url.URL{Scheme: u.Scheme, Host: u.Host, Path: u.Path, RawPath: u.RawPath}
+		token, err := policy.Token(context.Background(), client, base.String())
 		if err != nil {
 			return err
 		}
@@ -274,14 +282,15 @@ func defineToken(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 }
 
 func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
-	addr := fs.String("addr", "", "tcp sink `address`, HOST:PORT")
+	addr := fs.String("addr", "", "tcp sink `address`, HOST:PORT or [IPV6]:PORT")
 	user := fs.String("user", "", "user `name`")
 	passwordFile := fs.String("password-file", "", "`file` holding the password")
 	tlsOpts := tlsFlags(fs)
 	return func(stdout, _ io.Writer) error {
 		host, _, err := net.SplitHostPort(*addr)
-		if err != nil || host == "" {
-			return usageError("-addr must be HOST:PORT")
+		network, nerr := core.Network(host)
+		if err != nil || nerr != nil || host == "" {
+			return usageError("-addr must be HOST:PORT, with an IPv6 address in brackets")
 		}
 		tlsCfg, policy, err := dialPolicy(tlsOpts, host, *user, *passwordFile, authz.TCP)
 		if err != nil {
@@ -290,7 +299,7 @@ func defineStream(fs *flag.FlagSet) func(stdout, stderr io.Writer) error {
 		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 		defer stop()
 		dctx, cancel := context.WithTimeout(ctx, tlsx.HandshakeTimeout)
-		conn, err := (&tls.Dialer{Config: tlsCfg}).DialContext(dctx, "tcp4", *addr)
+		conn, err := (&tls.Dialer{Config: tlsCfg}).DialContext(dctx, network, *addr)
 		cancel()
 		if err == nil {
 			defer conn.Close()

cmd/lw/auth_test.go

@@ -190,7 +190,23 @@ func TestRemoveUserRefusesTheLastUser(t *testing.T) {
 
 func TestAuthCommandHelpPrintsAuthUsage(t *testing.T) {
 	code, _, stderr := runAuthTest(t, "add-user", "-h")
-	if code != 0 || !strings.Contains(stderr, "Usage: logwisp auth add-user -credentials FILE") {
+	if code != 0 || !strings.Contains(stderr, "Usage: lw auth add-user -credentials FILE") {
 		t.Fatalf("exit %d, stderr: %s", code, stderr)
 	}
 }
+
+// The login is posted to the -url given plus /auth: a link-local URL keeps
+// its zone escaped, not turned into one that no longer parses
+func TestTokenLogsInAtTheURLGiven(t *testing.T) {
+	pass := filepath.Join(t.TempDir(), "edge-01.pass")
+	if err := os.WriteFile(pass, []byte("correct horse battery\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	// No such interface or listener: the dial fails at once, after the URL is formed
+	for _, rawURL := range []string{"https://[fe80::1%25nosuchif0]:1", "https://127.0.0.1:1?"} {
+		code, _, stderr := runAuthTest(t, "token", "-url", rawURL, "-user", "edge-01", "-password-file", pass)
+		if want := `"` + strings.TrimSuffix(rawURL, "?") + `/auth"`; code != 1 || !strings.Contains(stderr, want) {
+			t.Errorf("%s: exit %d, stderr: %s", rawURL, code, stderr)
+		}
+	}
+}

cmd/lw/bootstrap.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 
 	_ "logwisp/internal/source/console"
 	_ "logwisp/internal/source/file"
@@ -108,6 +109,22 @@ func bootstrapService(ctx context.Context, cfg *config.Config) (*service.Service
 	return svc, nil
 }
 
+// checkConfig builds every pipeline and plugin as a start would, so TLS files,
+// credentials and options are loaded and their warnings print, then exits
+// before anything binds or reads: plugins start goroutines only in Start.
+func checkConfig(cfg *config.Config) int {
+	cfg.Logging.Output, cfg.Logging.Level = "stderr", "warn"
+	if err := initializeLogger(cfg); err == nil && logger.Start() == nil {
+		defer shutdownLogger()
+	}
+	if _, err := service.NewService(context.Background(), cfg, logger); err != nil {
+		fmt.Fprintf(os.Stderr, "configuration invalid: %v\n", err)
+		return 1
+	}
+	fmt.Printf("configuration ok: %d pipeline(s)\n", len(cfg.Pipelines))
+	return 0
+}
+
 // initializeLogger sets up the global logger based on the application's configuration
 func initializeLogger(cfg *config.Config) error {
 	logger = log.NewLogger()

cmd/lw/help.go

@@ -6,17 +6,18 @@ import (
 	"os"
 )
 
-// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths.
+// helpText is the CLI usage reference. Scalar flags map 1:1 to TOML config paths.
 const helpText = `LogWisp %s - log collection, processing, and distribution
 
 Usage:
-  logwisp [options]
-  logwisp help | -h | --help
-  logwisp --version
+  lw [options]
+  lw --check [options]          Build every pipeline and plugin, report, exit
+  lw help | -h | --help
+  lw --version
 
 Subcommands:
-  logwisp auth <command>        SCRAM credentials files, bearer tokens and tcp
-                                sink viewing; logwisp auth -h lists commands
+  lw auth <command>             SCRAM credentials files, bearer tokens and tcp
+                                sink viewing; lw auth -h lists commands
 
 Scalar configuration keys are settable as flags using their TOML paths:
   --<path>=<value>              e.g. --logging.level=debug
@@ -32,14 +33,26 @@ Logging:
       --logging.level=<level>   debug|info|warn|error
       --logging.file.directory=<path>
 
-Pipelines, plugins and filters are configured in TOML; array-indexed CLI and
-environment overrides are unsupported. -- ends option parsing.
+Pipelines (replace the file's pipelines; see doc/cli.md):
+      --pipeline <name>         Start a pipeline; earlier specs go to "cli"
+      --source <spec>           TYPE[,key=value...], repeatable
+      --sink <spec>             e.g. http,host=0.0.0.0,port=8080, repeatable
+      --filter <spec>           include|exclude,patterns=RE, repeatable
+      --format <spec>           json|txt|raw[,key=value...]
+      --rate-limit <spec>       rate=N[,burst=N,policy=drop|pass]
+      --heartbeat <spec>        interval_ms=N[,include_stats=true,...]
+  Keys nest with '.' (tls.cert_file=...), a repeated key makes a list, and
+  '\' escapes ',' '=' '\' in values. -- ends option parsing.
 
 Environment:
   LOGWISP_<PATH>                Config path, '.' -> '_', uppercase
                                 e.g. LOGWISP_LOGGING_LEVEL=debug
   LOGWISP_CONFIG_FILE           Configuration file path
   LOGWISP_CONFIG_DIR            Configuration directory
+  LOGWISP_PIPELINE, LOGWISP_SOURCE[_N], LOGWISP_SINK[_N], LOGWISP_FILTER[_N],
+  LOGWISP_FORMAT, LOGWISP_RATE_LIMIT, LOGWISP_HEARTBEAT
+                                One pipeline, specs as the flags (_N adds
+                                more); ignored when a pipeline flag is given
 
 Signals:
   SIGINT, SIGTERM               Graceful shutdown

cmd/lw/main.go

@@ -19,7 +19,7 @@ import (
 var logger *log.Logger
 
 func main() {
-	// Before handleHelp, so `logwisp auth <command> -h` prints the auth usage
+	// Before handleHelp, so `lw auth <command> -h` prints the auth usage
 	if len(os.Args) > 1 && os.Args[1] == "auth" {
 		os.Exit(runAuth(os.Args[2:], os.Stdout, os.Stderr))
 	}
@@ -53,6 +53,9 @@ func main() {
 		fmt.Println(version.String())
 		os.Exit(0)
 	}
+	if cfg.Check {
+		os.Exit(checkConfig(cfg))
+	}
 
 	if err := initializeLogger(cfg); err != nil {
 		FatalError(1, "Failed to initialize logger: %v\n", err)
@@ -63,9 +66,13 @@ func main() {
 		FatalError(1, "Failed to start logger: %v\n", err)
 	}
 
+	configFile := cfg.ConfigFile
+	if _, err := os.Stat(configFile); err != nil {
+		configFile = "none" // flags, environment and built-in defaults only
+	}
 	logger.Info("msg", "LogWisp starting",
 		"version", version.String(),
-		"config_file", cfg.ConfigFile,
+		"config_file", configFile,
 		"log_output", cfg.Logging.Output,
 		"status_reporter", cfg.StatusReporter,
 		"auto_reload", cfg.ConfigAutoReload)

cmd/lw/output.go

@@ -90,4 +90,4 @@ func (o *OutputHandler) SetQuiet(quiet bool) {
 	o.mu.Lock()
 	defer o.mu.Unlock()
 	o.quiet = quiet
-}
+}

cmd/lw/reload_test.go

@@ -2,6 +2,10 @@ package main
 
 import (
 	"context"
+	"errors"
+	"io/fs"
+	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -106,3 +110,43 @@ func TestEveryPluginRejectsUnknownKeys(t *testing.T) {
 		check("sink", name, err)
 	}
 }
+
+// A misspelled spec key reaches the plugin as the nested key it names, so the
+// sink refuses to build instead of starting without the TLS it was asked for.
+func TestMisspelledSpecKeyFailsPluginConstruction(t *testing.T) {
+	testLogger(t)
+	path := filepath.Join(t.TempDir(), "empty.toml")
+	testutil.WriteFile(t, path, "")
+	cfg := loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15861,tls.enabeld=true")
+	svc, err := bootstrapService(context.Background(), cfg)
+	if err == nil {
+		svc.Shutdown()
+	}
+	if err == nil || !strings.Contains(err.Error(), `unknown key "tls.enabeld"`) {
+		t.Fatalf("misspelled spec key accepted: %v", err)
+	}
+}
+
+// lw --check builds every plugin, so a bad option fails it, but starts none:
+// the port a valid sink names stays free and its log directory uncreated.
+func TestCheckBuildsWithoutStarting(t *testing.T) {
+	testLogger(t)
+	dir := t.TempDir()
+	path, out := filepath.Join(dir, "empty.toml"), filepath.Join(dir, "out")
+	testutil.WriteFile(t, path, "")
+	if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,host=127.0.0.1,port=15862",
+		"--sink", "file,name=check,directory="+out)); code != 0 {
+		t.Fatalf("valid configuration: exit %d", code)
+	}
+	ln, err := net.Listen("tcp4", "127.0.0.1:15862")
+	if err != nil {
+		t.Fatalf("check bound the sink's port: %v", err)
+	}
+	ln.Close()
+	if _, err := os.Stat(out); !errors.Is(err, fs.ErrNotExist) {
+		t.Fatalf("check created the file sink's directory: %v", err)
+	}
+	if code := checkConfig(loadTestConfig(t, path, "--source", "null", "--sink", "http,port=15862,tls.enabled=true")); code != 1 {
+		t.Fatalf("listener TLS without a certificate: exit %d, want 1", code)
+	}
+}

cmd/lw/test_helpers_test.go

@@ -16,10 +16,10 @@ func testLogger(t *testing.T) {
 	t.Cleanup(func() { logger = previous })
 }
 
-func loadTestConfig(t *testing.T, path string) *config.Config {
+func loadTestConfig(t *testing.T, path string, args ...string) *config.Config {
 	t.Helper()
 	testutil.ClearEnvPrefix(t, "LOGWISP_")
-	m, err := config.Load([]string{"-c", path, "--quiet", "--status_reporter=false"})
+	m, err := config.Load(append([]string{"-c", path, "--quiet", "--status_reporter=false"}, args...))
 	if err != nil {
 		t.Fatal(err)
 	}