v0.27.2 minor update: long lines, switches, tls

cmd/lw/bootstrap.go

@@ -135,7 +135,6 @@ func checkConfig(cfg *config.Config) int {
 // dumpConfig prints the effective configuration, flags and environment
 // resolved into it, as a file lw -c reads back.
 func dumpConfig(cfg *config.Config, w io.Writer) int {
-	cfg.Dump = false
 	// Snapshot gave each console sink the top-level color: keep it inherited
 	for _, p := range cfg.Pipelines {
 		for _, s := range p.PluginSinks {

cmd/lw/cli.go

@@ -56,6 +56,10 @@ var shorts = map[string]struct {
 	"u": {"user", false},
 }
 
+// switches are lw's own options that take no value and are no setting, so no
+// file or environment variable turns one on
+var switches = []string{"check", "dump", "version"}
+
 // usageError is a command-line mistake: exit status 2 rather than 1
 type usageError string
 
@@ -168,11 +172,12 @@ func checkArgs(fs *flag.FlagSet, required []string) error {
 }
 
 // invocation is a parsed command line: a command with its arguments, a help
-// request, or what config.Load reads.
+// request, or the switches given and what config.Load reads.
 type invocation struct {
 	command *command
 	args    []string
 	help    bool
+	on      map[string]bool
 	load    config.Args
 }
 
@@ -225,6 +230,14 @@ func parseCommandLine(argv []string) (inv invocation, err error) {
 				err = cmp.Or(err, fmt.Errorf("%s requires a configuration file path", typed))
 			}
 			inv.load.File = value
+		case strings.HasPrefix(name, "--") && slices.Contains(switches, name[2:]):
+			if inline {
+				err = cmp.Or(err, fmt.Errorf("%s takes no value", typed))
+			}
+			if inv.on == nil {
+				inv.on = map[string]bool{}
+			}
+			inv.on[name[2:]] = true
 		case arg == "--color":
 			// Bare, it means always: config takes no bare flag for a string key
 			inv.load.Overrides = append(inv.load.Overrides, "--color="+cmp.Or(next(&i), "always"))
@@ -232,6 +245,9 @@ func parseCommandLine(argv []string) (inv invocation, err error) {
 			if !inline {
 				value = next(&i)
 			}
+			if value == "" {
+				err = cmp.Or(err, fmt.Errorf("%s requires a value; one that starts with '-' follows '='", typed))
+			}
 			inv.load.Specs = append(inv.load.Specs, config.Spec{Flag: name[2:], Value: value})
 		case len(name) > 1 && name[0] == '-' && unicode.IsLetter(rune(name[1])):
 			// lw's only single-dash options are the shorts

cmd/lw/cli_test.go

@@ -28,8 +28,8 @@ func TestCommandLineGrammar(t *testing.T) {
 	}{
 		{[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}},
 		{[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}},
-		{[]string{"--source=null", "--sink", "http,port=1", "--filter", "--sink"},
+		{[]string{"--source=null", "--sink", "http,port=1"},
-			invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}, {Flag: "filter"}, {Flag: "sink"}}}}},
+			invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}}}}},
 		{[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}},
 		{[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}},
 		{[]string{"--quiet", "--"}, invocation{load: config.Args{Overrides: []string{"--quiet"}}}},
@@ -37,9 +37,10 @@ func TestCommandLineGrammar(t *testing.T) {
 		{[]string{"-c", "-h"}, invocation{help: true}},
 		{[]string{"help"}, invocation{help: true}},
 		{[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}},
-		{[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump=false"},
+		{[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"},
-			invocation{load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}},
+			invocation{on: map[string]bool{"check": true, "version": true, "dump": true},
-				Overrides: []string{"--quiet", "--check", "--version", "--logging.level=debug", "--logging.file.retention_hours=-1", "--dump=false"}}}},
+				load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}},
+					Overrides: []string{"--quiet", "--logging.level=debug", "--logging.file.retention_hours=-1"}}}},
 	} {
 		got, err := parseCommandLine(c.argv)
 		if err != nil || !reflect.DeepEqual(got, c.want) {
@@ -47,7 +48,8 @@ func TestCommandLineGrammar(t *testing.T) {
 		}
 	}
 	for _, argv := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"},
-		{"-v"}, {"-qt"}, {"-u", "x"}, {"-config", "x"}, {"--logging.level"}, {"--logging.file.retention_hours", "-1"}} {
+		{"-v"}, {"-qt"}, {"-u", "x"}, {"-config", "x"}, {"--logging.level"}, {"--logging.file.retention_hours", "-1"},
+		{"--check=false"}, {"--version=true"}, {"--source"}, {"--sink=", "x"}, {"-p"}, {"--filter", "--sink", "x"}} {
 		if _, err := parseCommandLine(argv); err == nil {
 			t.Errorf("%q accepted", argv)
 		}

cmd/lw/completion.go

@@ -83,6 +83,9 @@ func options() map[string]string {
 	for long := range config.Settings() {
 		options[long] = ""
 	}
+	for _, long := range switches {
+		options[long] = ""
+	}
 	for letter, short := range shorts {
 		if short.top {
 			options[short.long] = letter
@@ -99,7 +102,7 @@ func completionModel() (top []compFlag, cmds []compCommand, presets string) {
 		f := compFlag{long: long, short: shortOf[long], value: "text", repeat: slices.Contains(specFlags, long)}
 		if v, ok := valueOf[long]; ok {
 			f.value = v
-		} else if settings[long] {
+		} else if settings[long] || slices.Contains(switches, long) {
 			f.value = ""
 		}
 		top = append(top, f)

cmd/lw/main.go

@@ -30,6 +30,9 @@ func main() {
 	case inv.help:
 		printHelp(os.Stdout)
 		os.Exit(0)
+	case inv.on["version"]:
+		fmt.Println(version.String())
+		os.Exit(0)
 	}
 
 	// --- 1. Initial setup ---
@@ -54,14 +57,10 @@ func main() {
 
 	InitOutputHandler(cfg.Quiet)
 
-	if cfg.ShowVersion {
+	if inv.on["check"] {
-		fmt.Println(version.String())
-		os.Exit(0)
-	}
-	if cfg.Check {
 		os.Exit(checkConfig(cfg))
 	}
-	if cfg.Dump {
+	if inv.on["dump"] {
 		os.Exit(dumpConfig(cfg, os.Stdout))
 	}
 

doc/security.md

@@ -131,7 +131,10 @@ Misconfiguration fails at plugin construction, before the pipeline starts:
   `self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one; an
   issuer that is no CA (`lw tls ca` makes one)
 - `pin_sha256` with `ca_file` or `insecure_skip_verify`, or not of the
-  `sha256//BASE64` form; a listener key on a dialer, or `pin_sha256` on a
+  `sha256//BASE64` form
+- a key of the other role: `self_signed`, `issuer_cert_file`,
+  `issuer_key_file`, `hosts`, `client_auth` or `client_ca_file` on a dialer;
+  `ca_file`, `server_name`, `insecure_skip_verify` or `pin_sha256` on a
   listener
 - `client_auth = true` with no `client_ca_file`
 - a dialer with only one of `cert_file` / `key_file`

doc/sources.md

@@ -52,6 +52,10 @@ Options, each as type and default:
   interval that this option does not change.
 - Each matched file gets its own watcher. Watchers for files that disappear are
   stopped and removed on the next scan.
+- The file lw's stdout or stderr is redirected to is never followed, with a
+  warning: `lw > out.log` in the directory would read its own output back.
+- A line longer than 1 MiB continues in the next entry, as on the console
+  source.
 - A new watcher seeks to end-of-file. Positions live in memory only, so a
   restart resumes from the current end of each file and content written while
   LogWisp was down is not read. `from = "start"` reads each file whole when its

doc/todo.md

@@ -168,10 +168,10 @@ Steps:
      the site's hues badly (green to cyan, red to bright black), so the
      8-color tier is chosen by hand.
 5. `lw --tui`.
-   - A parser switch, no short form. With `-c FILE`, pipeline flags or
+   - A parser switch, `-T` for short (`-t` is `--check`). With `-c FILE`,
-     `LOGWISP_SOURCE` it starts from that composition, and never rewrites FILE
+     pipeline flags or `LOGWISP_SOURCE` it starts from that composition, and
-     (TOML encoding drops its comments); bare, it opens the preset menu (pipe,
+     never rewrites FILE (TOML encoding drops its comments); bare, it opens the
-     tail, serve, edge, aggregator, empty).
+     preset menu (pipe, tail, serve, edge, aggregator, empty).
    - The pipeline as the site draws it: SOURCES, FLOW, SINKS; a node is a
      tinted bar, its type, listens or dials, and a summary line; the four flow
      stages in order inside a dashed box, the ones off dimmed; wires drawn from

internal/config/config.go

@@ -4,10 +4,6 @@ package config
 
 // Config is the top-level configuration structure for the LogWisp application
 type Config struct {
-	// Top-level flags for application control
-	ShowVersion bool `toml:"version"`
-	Check       bool `toml:"check"` // build every plugin, report, exit: lw --check
-	Dump        bool `toml:"dump"`  // print the effective configuration, exit: lw --dump
 	Quiet bool `toml:"quiet"`
 
 	// Console sinks without their own color: "auto" (a terminal, NO_COLOR

internal/config/loader_test.go

@@ -229,7 +229,8 @@ port = 9000
 }
 
 // A misspelled table path above a plugin's config drops that whole table, so
-// the file itself is checked too; only config_file is tolerated, as documented.
+// the file itself is checked too; only config_file, as documented, and the
+// switches earlier lw --dump output wrote are tolerated.
 func TestConfigFileRejectsUnknownKeys(t *testing.T) {
 	isolateConfig(t)
 	for key, body := range map[string]string{
@@ -237,6 +238,9 @@ func TestConfigFileRejectsUnknownKeys(t *testing.T) {
 		"logging.levle":                      "[logging]\nlevle = \"debug\"",
 	} {
 		testutil.WriteFile(t, "typo.toml", `config_file = "ignored.toml"
+check = false
+dump = false
+version = false
 [[pipelines]]
 name = "typo"
 [[pipelines.plugin_sources]]

internal/config/validate.go

@@ -174,5 +174,8 @@ func checkFileKeys(path string) error {
 		return err
 	}
 	delete(root, "config_file") // runtime metadata, documented as ignored in the file
+	for _, key := range []string{"version", "check", "dump"} {
+		delete(root, key) // command-line switches, which earlier lw --dump output wrote
+	}
 	return checkKeys(root, reflect.TypeOf(Config{}), "")
 }

internal/source/file/file.go

@@ -40,6 +40,12 @@ type FileSource struct {
 	// Configuration
 	config *config.FileSourceOptions
 
+	// lw's stdout and stderr when they are files: a console sink writes there,
+	// so following them would feed lw its own output. ownSkipped is per path
+	// already reported, touched only by monitorLoop.
+	own        []os.FileInfo
+	ownSkipped map[string]bool
+
 	// Application
 	subscribers []chan core.LogEntry
 	watchers    map[string]*fileWatcher
@@ -105,8 +111,14 @@ func NewFileSourcePlugin(
 		config:      opts,
 		subscribers: make([]chan core.LogEntry, 0),
 		watchers:    make(map[string]*fileWatcher),
+		ownSkipped:  make(map[string]bool),
 		logger:      logger,
 	}
+	for _, f := range []*os.File{os.Stdout, os.Stderr} {
+		if info, err := f.Stat(); err == nil && info.Mode().IsRegular() {
+			fs.own = append(fs.own, info)
+		}
+	}
 	fs.lastEntryTime.Store(time.Time{})
 
 	fs.session = proxy.CreateSession(
@@ -388,15 +400,39 @@ func (fs *FileSource) scanFile() ([]string, error) {
 			continue
 		}
 
-		name := entry.Name()
+		path := filepath.Join(fs.config.Directory, entry.Name())
-		if re.MatchString(name) {
+		if re.MatchString(entry.Name()) && !fs.isOwn(path) {
-			files = append(files, filepath.Join(fs.config.Directory, name))
+			files = append(files, path)
 		}
 	}
 
 	return files, nil
 }
 
+// isOwn reports whether path is lw's own stdout or stderr, warning once per path
+func (fs *FileSource) isOwn(path string) bool {
+	if len(fs.own) == 0 {
+		return false
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	for _, own := range fs.own {
+		if os.SameFile(info, own) {
+			if !fs.ownSkipped[path] {
+				fs.ownSkipped[path] = true
+				fs.logger.Warn("msg", "Not following lw's own output",
+					"component", "file_source",
+					"instance_id", fs.id,
+					"path", path)
+			}
+			return true
+		}
+	}
+	return false
+}
+
 // globToRegex converts a simple glob pattern to a regular expression.
 func globToRegex(glob string) string {
 	regex := regexp.QuoteMeta(glob)

internal/source/file/file_test.go

@@ -4,14 +4,41 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"slices"
+	"strings"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/lixenwraith/logwisp/internal/config"
 	"github.com/lixenwraith/logwisp/internal/core"
 	"github.com/lixenwraith/logwisp/internal/testutil"
+
+	"github.com/lixenwraith/log"
 )
 
+// A file source never follows the file lw's stdout or stderr is: `lw > out.log`
+// in the directory it follows would read its own output back, endlessly
+func TestFileSourceSkipsItsOwnOutput(t *testing.T) {
+	dir := t.TempDir()
+	testutil.WriteFile(t, filepath.Join(dir, "app.log"), "")
+	testutil.WriteFile(t, filepath.Join(dir, "out.log"), "")
+	own, err := os.Stat(filepath.Join(dir, "out.log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	source := &FileSource{
+		config:     &config.FileSourceOptions{Directory: dir, Pattern: "*.log"},
+		own:        []os.FileInfo{own},
+		ownSkipped: map[string]bool{},
+		logger:     log.NewLogger(),
+	}
+	files, err := source.scanFile()
+	if err != nil || len(files) != 1 || filepath.Base(files[0]) != "app.log" {
+		t.Fatalf("scanFile = %v, %v; want app.log alone", files, err)
+	}
+}
+
 func TestStoppedWatcherReturnsNormally(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "session.jsonl")
 	testutil.WriteFile(t, path, "")
@@ -45,6 +72,25 @@ func TestRemoveWatcherPreservesReplacement(t *testing.T) {
 	}
 }
 
+// A line longer than core.MaxLogEntryBytes continues in the next entry, and the
+// lines around it are read once: the watcher used to stop at it for good
+func TestLongLineContinuesInTheNextEntry(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "app.log")
+	long := strings.Repeat("x", core.MaxLogEntryBytes+10)
+	testutil.WriteFile(t, path, "first\n"+long+"\nafter\n")
+	var got []string
+	w := newFileWatcher(path, true, true, func(e core.LogEntry) { got = append(got, e.Message) }, log.NewLogger())
+	for range 2 {
+		if err := w.checkFile(); err != nil {
+			t.Fatal(err)
+		}
+	}
+	want := []string{"first", long[:core.MaxLogEntryBytes], long[core.MaxLogEntryBytes:], "after"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("read %d entries, want %d: first, the long line in two, after", len(got), len(want))
+	}
+}
+
 // A rotated file reappears under its archive name with the same inode. Its
 // replacement watcher resumes where the original stopped, so a `from = "start"`
 // source does not re-emit every record the file already delivered.

internal/source/file/file_watcher.go

@@ -251,6 +251,7 @@ func (w *fileWatcher) checkFile() error {
 
 		scanner := bufio.NewScanner(file)
 		scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
+		scanner.Split(scanLinesCapped)
 
 		for scanner.Scan() {
 			line := scanner.Text()
@@ -311,6 +312,17 @@ func (w *fileWatcher) checkFile() error {
 
 // initPosition records the file's metadata and, unless the watcher was created
 // to read from the start, sets the initial read position to the end
+// scanLinesCapped is bufio.ScanLines, except that a line longer than
+// core.MaxLogEntryBytes continues in the next one, as on the console source:
+// the scanner would otherwise stop at it and every poll re-read the lines before.
+func scanLinesCapped(data []byte, atEOF bool) (int, []byte, error) {
+	advance, token, err := bufio.ScanLines(data, atEOF)
+	if advance == 0 && token == nil && err == nil && len(data) >= core.MaxLogEntryBytes {
+		return core.MaxLogEntryBytes, data[:core.MaxLogEntryBytes], nil
+	}
+	return advance, token, err
+}
+
 func (w *fileWatcher) initPosition() error {
 	file, err := os.Open(w.directory)
 	if err != nil {

internal/tlsx/tlsx.go

@@ -36,8 +36,8 @@ func Server(o *config.TLSOptions, host string) (*tls.Config, error) {
 	}
 	files, issuer := o.CertFile != "" || o.KeyFile != "", o.IssuerCertFile != "" || o.IssuerKeyFile != ""
 	switch {
-	case o.PinSHA256 != "":
+	case o.CAFile != "" || o.ServerName != "" || o.InsecureSkipVerify || o.PinSHA256 != "":
-		return nil, fmt.Errorf("tls: pin_sha256 applies to dialers")
+		return nil, fmt.Errorf("tls: ca_file, server_name, insecure_skip_verify and pin_sha256 apply to dialers")
 	case files && (issuer || o.SelfSigned) || issuer && o.SelfSigned:
 		return nil, fmt.Errorf("tls: set one of cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file")
 	case len(o.Hosts) > 0 && !issuer && !o.SelfSigned:
@@ -91,8 +91,8 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) {
 		return nil, nil
 	}
 	switch {
-	case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0:
+	case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0 || o.ClientAuth || o.ClientCAFile != "":
-		return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file and hosts apply to listeners")
+		return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file, hosts, client_auth and client_ca_file apply to listeners")
 	case o.PinSHA256 != "" && (o.CAFile != "" || o.InsecureSkipVerify):
 		return nil, fmt.Errorf("tls: pin_sha256 replaces ca_file and insecure_skip_verify: set one")
 	}

internal/tlsx/tlsx_test.go

@@ -179,9 +179,11 @@ func TestTLSOptionConflicts(t *testing.T) {
 		{config.TLSOptions{IssuerCertFile: "c"}, true, "must be set together"},
 		{config.TLSOptions{CertFile: "c"}, true, "must be set together"},
 		{config.TLSOptions{Hosts: []string{"a"}, CertFile: "c", KeyFile: "k"}, true, "hosts applies to"},
-		{config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "applies to dialers"},
+		{config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "apply to dialers"},
+		{config.TLSOptions{SelfSigned: true, CAFile: "ca"}, true, "apply to dialers"},
 		{config.TLSOptions{}, true, "listeners need"},
 		{config.TLSOptions{SelfSigned: true}, false, "apply to listeners"},
+		{config.TLSOptions{ClientAuth: true, ClientCAFile: "ca"}, false, "apply to listeners"},
 		{config.TLSOptions{PinSHA256: pin, CAFile: "ca"}, false, "set one"},
 		{config.TLSOptions{PinSHA256: pin, InsecureSkipVerify: true}, false, "set one"},
 		{config.TLSOptions{PinSHA256: "sha256//short"}, false, "want sha256//BASE64"},