From a1648ce17bc266cbe7aa65395b7bb205a216e0a9d03c14c24edc254aab072e3a Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Fri, 9 Oct 2026 14:03:28 -0400 Subject: [PATCH] v0.27.2 minor update: long lines, switches, tls --- cmd/lw/bootstrap.go | 1 - cmd/lw/cli.go | 18 ++++++++++- cmd/lw/cli_test.go | 14 +++++---- cmd/lw/completion.go | 5 ++- cmd/lw/main.go | 11 +++---- doc/security.md | 5 ++- doc/sources.md | 4 +++ doc/todo.md | 8 ++--- internal/config/config.go | 6 +--- internal/config/loader_test.go | 6 +++- internal/config/validate.go | 3 ++ internal/source/file/file.go | 42 +++++++++++++++++++++++-- internal/source/file/file_test.go | 46 ++++++++++++++++++++++++++++ internal/source/file/file_watcher.go | 12 ++++++++ internal/tlsx/tlsx.go | 8 ++--- internal/tlsx/tlsx_test.go | 4 ++- 16 files changed, 159 insertions(+), 34 deletions(-) diff --git a/cmd/lw/bootstrap.go b/cmd/lw/bootstrap.go index cbf5e98..3750daa 100644 --- a/cmd/lw/bootstrap.go +++ b/cmd/lw/bootstrap.go @@ -135,7 +135,6 @@ func checkConfig(cfg *config.Config) int { // dumpConfig prints the effective configuration, flags and environment // resolved into it, as a file lw -c reads back. func dumpConfig(cfg *config.Config, w io.Writer) int { - cfg.Dump = false // Snapshot gave each console sink the top-level color: keep it inherited for _, p := range cfg.Pipelines { for _, s := range p.PluginSinks { diff --git a/cmd/lw/cli.go b/cmd/lw/cli.go index cd6a85f..b6766fe 100644 --- a/cmd/lw/cli.go +++ b/cmd/lw/cli.go @@ -56,6 +56,10 @@ var shorts = map[string]struct { "u": {"user", false}, } +// switches are lw's own options that take no value and are no setting, so no +// file or environment variable turns one on +var switches = []string{"check", "dump", "version"} + // usageError is a command-line mistake: exit status 2 rather than 1 type usageError string @@ -168,11 +172,12 @@ func checkArgs(fs *flag.FlagSet, required []string) error { } // invocation is a parsed command line: a command with its arguments, a help -// request, or what config.Load reads. +// request, or the switches given and what config.Load reads. type invocation struct { command *command args []string help bool + on map[string]bool load config.Args } @@ -225,6 +230,14 @@ func parseCommandLine(argv []string) (inv invocation, err error) { err = cmp.Or(err, fmt.Errorf("%s requires a configuration file path", typed)) } inv.load.File = value + case strings.HasPrefix(name, "--") && slices.Contains(switches, name[2:]): + if inline { + err = cmp.Or(err, fmt.Errorf("%s takes no value", typed)) + } + if inv.on == nil { + inv.on = map[string]bool{} + } + inv.on[name[2:]] = true case arg == "--color": // Bare, it means always: config takes no bare flag for a string key inv.load.Overrides = append(inv.load.Overrides, "--color="+cmp.Or(next(&i), "always")) @@ -232,6 +245,9 @@ func parseCommandLine(argv []string) (inv invocation, err error) { if !inline { value = next(&i) } + if value == "" { + err = cmp.Or(err, fmt.Errorf("%s requires a value; one that starts with '-' follows '='", typed)) + } inv.load.Specs = append(inv.load.Specs, config.Spec{Flag: name[2:], Value: value}) case len(name) > 1 && name[0] == '-' && unicode.IsLetter(rune(name[1])): // lw's only single-dash options are the shorts diff --git a/cmd/lw/cli_test.go b/cmd/lw/cli_test.go index bab073b..b552889 100644 --- a/cmd/lw/cli_test.go +++ b/cmd/lw/cli_test.go @@ -28,8 +28,8 @@ func TestCommandLineGrammar(t *testing.T) { }{ {[]string{"-c", "a.toml", "--config=b.toml", "--quiet"}, invocation{load: config.Args{File: "b.toml", Overrides: []string{"--quiet"}}}}, {[]string{"-c=a.toml", "--config", "b.toml"}, invocation{load: config.Args{File: "b.toml"}}}, - {[]string{"--source=null", "--sink", "http,port=1", "--filter", "--sink"}, - invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}, {Flag: "filter"}, {Flag: "sink"}}}}}, + {[]string{"--source=null", "--sink", "http,port=1"}, + invocation{load: config.Args{Specs: []config.Spec{{Flag: "source", Value: "null"}, {Flag: "sink", Value: "http,port=1"}}}}}, {[]string{"--preset", "tail,path=x", "--pipeline", "b"}, invocation{load: config.Args{Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}, {Flag: "pipeline", Value: "b"}}}}}, {[]string{"--color", "--color", "never", "--color=auto"}, invocation{load: config.Args{Overrides: []string{"--color=always", "--color=never", "--color=auto"}}}}, {[]string{"--quiet", "--"}, invocation{load: config.Args{Overrides: []string{"--quiet"}}}}, @@ -37,9 +37,10 @@ func TestCommandLineGrammar(t *testing.T) { {[]string{"-c", "-h"}, invocation{help: true}}, {[]string{"help"}, invocation{help: true}}, {[]string{"tls", "ca", "-h"}, invocation{command: &commands[1], args: []string{"ca", "-h"}}}, - {[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump=false"}, - invocation{load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}}, - Overrides: []string{"--quiet", "--check", "--version", "--logging.level=debug", "--logging.file.retention_hours=-1", "--dump=false"}}}}, + {[]string{"-q", "-t", "-V", "-p", "tail,path=x", "-c=a.toml", "--logging.level", "debug", "--logging.file.retention_hours=-1", "--dump"}, + invocation{on: map[string]bool{"check": true, "version": true, "dump": true}, + load: config.Args{File: "a.toml", Specs: []config.Spec{{Flag: "preset", Value: "tail,path=x"}}, + Overrides: []string{"--quiet", "--logging.level=debug", "--logging.file.retention_hours=-1"}}}}, } { got, err := parseCommandLine(c.argv) if err != nil || !reflect.DeepEqual(got, c.want) { @@ -47,7 +48,8 @@ func TestCommandLineGrammar(t *testing.T) { } } for _, argv := range [][]string{{"-c"}, {"--config"}, {"--config="}, {"-c="}, {"-c", "--quiet"}, - {"-v"}, {"-qt"}, {"-u", "x"}, {"-config", "x"}, {"--logging.level"}, {"--logging.file.retention_hours", "-1"}} { + {"-v"}, {"-qt"}, {"-u", "x"}, {"-config", "x"}, {"--logging.level"}, {"--logging.file.retention_hours", "-1"}, + {"--check=false"}, {"--version=true"}, {"--source"}, {"--sink=", "x"}, {"-p"}, {"--filter", "--sink", "x"}} { if _, err := parseCommandLine(argv); err == nil { t.Errorf("%q accepted", argv) } diff --git a/cmd/lw/completion.go b/cmd/lw/completion.go index f499862..ca536eb 100644 --- a/cmd/lw/completion.go +++ b/cmd/lw/completion.go @@ -83,6 +83,9 @@ func options() map[string]string { for long := range config.Settings() { options[long] = "" } + for _, long := range switches { + options[long] = "" + } for letter, short := range shorts { if short.top { options[short.long] = letter @@ -99,7 +102,7 @@ func completionModel() (top []compFlag, cmds []compCommand, presets string) { f := compFlag{long: long, short: shortOf[long], value: "text", repeat: slices.Contains(specFlags, long)} if v, ok := valueOf[long]; ok { f.value = v - } else if settings[long] { + } else if settings[long] || slices.Contains(switches, long) { f.value = "" } top = append(top, f) diff --git a/cmd/lw/main.go b/cmd/lw/main.go index 3bd18f9..84bff39 100644 --- a/cmd/lw/main.go +++ b/cmd/lw/main.go @@ -30,6 +30,9 @@ func main() { case inv.help: printHelp(os.Stdout) os.Exit(0) + case inv.on["version"]: + fmt.Println(version.String()) + os.Exit(0) } // --- 1. Initial setup --- @@ -54,14 +57,10 @@ func main() { InitOutputHandler(cfg.Quiet) - if cfg.ShowVersion { - fmt.Println(version.String()) - os.Exit(0) - } - if cfg.Check { + if inv.on["check"] { os.Exit(checkConfig(cfg)) } - if cfg.Dump { + if inv.on["dump"] { os.Exit(dumpConfig(cfg, os.Stdout)) } diff --git a/doc/security.md b/doc/security.md index 151ae43..9e0347c 100644 --- a/doc/security.md +++ b/doc/security.md @@ -131,7 +131,10 @@ Misconfiguration fails at plugin construction, before the pipeline starts: `self_signed`, `issuer_cert_file`/`issuer_key_file`, or more than one; an issuer that is no CA (`lw tls ca` makes one) - `pin_sha256` with `ca_file` or `insecure_skip_verify`, or not of the - `sha256//BASE64` form; a listener key on a dialer, or `pin_sha256` on a + `sha256//BASE64` form +- a key of the other role: `self_signed`, `issuer_cert_file`, + `issuer_key_file`, `hosts`, `client_auth` or `client_ca_file` on a dialer; + `ca_file`, `server_name`, `insecure_skip_verify` or `pin_sha256` on a listener - `client_auth = true` with no `client_ca_file` - a dialer with only one of `cert_file` / `key_file` diff --git a/doc/sources.md b/doc/sources.md index 136c377..74cc98f 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -52,6 +52,10 @@ Options, each as type and default: interval that this option does not change. - Each matched file gets its own watcher. Watchers for files that disappear are stopped and removed on the next scan. +- The file lw's stdout or stderr is redirected to is never followed, with a + warning: `lw > out.log` in the directory would read its own output back. +- A line longer than 1 MiB continues in the next entry, as on the console + source. - A new watcher seeks to end-of-file. Positions live in memory only, so a restart resumes from the current end of each file and content written while LogWisp was down is not read. `from = "start"` reads each file whole when its diff --git a/doc/todo.md b/doc/todo.md index 76892b3..2df95ad 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -168,10 +168,10 @@ Steps: the site's hues badly (green to cyan, red to bright black), so the 8-color tier is chosen by hand. 5. `lw --tui`. - - A parser switch, no short form. With `-c FILE`, pipeline flags or - `LOGWISP_SOURCE` it starts from that composition, and never rewrites FILE - (TOML encoding drops its comments); bare, it opens the preset menu (pipe, - tail, serve, edge, aggregator, empty). + - A parser switch, `-T` for short (`-t` is `--check`). With `-c FILE`, + pipeline flags or `LOGWISP_SOURCE` it starts from that composition, and + never rewrites FILE (TOML encoding drops its comments); bare, it opens the + preset menu (pipe, tail, serve, edge, aggregator, empty). - The pipeline as the site draws it: SOURCES, FLOW, SINKS; a node is a tinted bar, its type, listens or dials, and a summary line; the four flow stages in order inside a dashed box, the ones off dimmed; wires drawn from diff --git a/internal/config/config.go b/internal/config/config.go index c685794..1035811 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,11 +4,7 @@ package config // Config is the top-level configuration structure for the LogWisp application type Config struct { - // Top-level flags for application control - ShowVersion bool `toml:"version"` - Check bool `toml:"check"` // build every plugin, report, exit: lw --check - Dump bool `toml:"dump"` // print the effective configuration, exit: lw --dump - Quiet bool `toml:"quiet"` + Quiet bool `toml:"quiet"` // Console sinks without their own color: "auto" (a terminal, NO_COLOR // unset, TERM not dumb), "always", "never" diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go index fc0c29e..7907803 100644 --- a/internal/config/loader_test.go +++ b/internal/config/loader_test.go @@ -229,7 +229,8 @@ port = 9000 } // A misspelled table path above a plugin's config drops that whole table, so -// the file itself is checked too; only config_file is tolerated, as documented. +// the file itself is checked too; only config_file, as documented, and the +// switches earlier lw --dump output wrote are tolerated. func TestConfigFileRejectsUnknownKeys(t *testing.T) { isolateConfig(t) for key, body := range map[string]string{ @@ -237,6 +238,9 @@ func TestConfigFileRejectsUnknownKeys(t *testing.T) { "logging.levle": "[logging]\nlevle = \"debug\"", } { testutil.WriteFile(t, "typo.toml", `config_file = "ignored.toml" +check = false +dump = false +version = false [[pipelines]] name = "typo" [[pipelines.plugin_sources]] diff --git a/internal/config/validate.go b/internal/config/validate.go index e4af7f8..d8b4a0b 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -174,5 +174,8 @@ func checkFileKeys(path string) error { return err } delete(root, "config_file") // runtime metadata, documented as ignored in the file + for _, key := range []string{"version", "check", "dump"} { + delete(root, key) // command-line switches, which earlier lw --dump output wrote + } return checkKeys(root, reflect.TypeOf(Config{}), "") } diff --git a/internal/source/file/file.go b/internal/source/file/file.go index 06b3e0a..2c4ae50 100644 --- a/internal/source/file/file.go +++ b/internal/source/file/file.go @@ -40,6 +40,12 @@ type FileSource struct { // Configuration config *config.FileSourceOptions + // lw's stdout and stderr when they are files: a console sink writes there, + // so following them would feed lw its own output. ownSkipped is per path + // already reported, touched only by monitorLoop. + own []os.FileInfo + ownSkipped map[string]bool + // Application subscribers []chan core.LogEntry watchers map[string]*fileWatcher @@ -105,8 +111,14 @@ func NewFileSourcePlugin( config: opts, subscribers: make([]chan core.LogEntry, 0), watchers: make(map[string]*fileWatcher), + ownSkipped: make(map[string]bool), logger: logger, } + for _, f := range []*os.File{os.Stdout, os.Stderr} { + if info, err := f.Stat(); err == nil && info.Mode().IsRegular() { + fs.own = append(fs.own, info) + } + } fs.lastEntryTime.Store(time.Time{}) fs.session = proxy.CreateSession( @@ -388,15 +400,39 @@ func (fs *FileSource) scanFile() ([]string, error) { continue } - name := entry.Name() - if re.MatchString(name) { - files = append(files, filepath.Join(fs.config.Directory, name)) + path := filepath.Join(fs.config.Directory, entry.Name()) + if re.MatchString(entry.Name()) && !fs.isOwn(path) { + files = append(files, path) } } return files, nil } +// isOwn reports whether path is lw's own stdout or stderr, warning once per path +func (fs *FileSource) isOwn(path string) bool { + if len(fs.own) == 0 { + return false + } + info, err := os.Stat(path) + if err != nil { + return false + } + for _, own := range fs.own { + if os.SameFile(info, own) { + if !fs.ownSkipped[path] { + fs.ownSkipped[path] = true + fs.logger.Warn("msg", "Not following lw's own output", + "component", "file_source", + "instance_id", fs.id, + "path", path) + } + return true + } + } + return false +} + // globToRegex converts a simple glob pattern to a regular expression. func globToRegex(glob string) string { regex := regexp.QuoteMeta(glob) diff --git a/internal/source/file/file_test.go b/internal/source/file/file_test.go index 45bd66b..cadfc9a 100644 --- a/internal/source/file/file_test.go +++ b/internal/source/file/file_test.go @@ -4,14 +4,41 @@ import ( "context" "os" "path/filepath" + "slices" + "strings" "syscall" "testing" "time" + "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/logwisp/internal/core" "github.com/lixenwraith/logwisp/internal/testutil" + + "github.com/lixenwraith/log" ) +// A file source never follows the file lw's stdout or stderr is: `lw > out.log` +// in the directory it follows would read its own output back, endlessly +func TestFileSourceSkipsItsOwnOutput(t *testing.T) { + dir := t.TempDir() + testutil.WriteFile(t, filepath.Join(dir, "app.log"), "") + testutil.WriteFile(t, filepath.Join(dir, "out.log"), "") + own, err := os.Stat(filepath.Join(dir, "out.log")) + if err != nil { + t.Fatal(err) + } + source := &FileSource{ + config: &config.FileSourceOptions{Directory: dir, Pattern: "*.log"}, + own: []os.FileInfo{own}, + ownSkipped: map[string]bool{}, + logger: log.NewLogger(), + } + files, err := source.scanFile() + if err != nil || len(files) != 1 || filepath.Base(files[0]) != "app.log" { + t.Fatalf("scanFile = %v, %v; want app.log alone", files, err) + } +} + func TestStoppedWatcherReturnsNormally(t *testing.T) { path := filepath.Join(t.TempDir(), "session.jsonl") testutil.WriteFile(t, path, "") @@ -45,6 +72,25 @@ func TestRemoveWatcherPreservesReplacement(t *testing.T) { } } +// A line longer than core.MaxLogEntryBytes continues in the next entry, and the +// lines around it are read once: the watcher used to stop at it for good +func TestLongLineContinuesInTheNextEntry(t *testing.T) { + path := filepath.Join(t.TempDir(), "app.log") + long := strings.Repeat("x", core.MaxLogEntryBytes+10) + testutil.WriteFile(t, path, "first\n"+long+"\nafter\n") + var got []string + w := newFileWatcher(path, true, true, func(e core.LogEntry) { got = append(got, e.Message) }, log.NewLogger()) + for range 2 { + if err := w.checkFile(); err != nil { + t.Fatal(err) + } + } + want := []string{"first", long[:core.MaxLogEntryBytes], long[core.MaxLogEntryBytes:], "after"} + if !slices.Equal(got, want) { + t.Fatalf("read %d entries, want %d: first, the long line in two, after", len(got), len(want)) + } +} + // A rotated file reappears under its archive name with the same inode. Its // replacement watcher resumes where the original stopped, so a `from = "start"` // source does not re-emit every record the file already delivered. diff --git a/internal/source/file/file_watcher.go b/internal/source/file/file_watcher.go index 1d28b2a..32c99c0 100644 --- a/internal/source/file/file_watcher.go +++ b/internal/source/file/file_watcher.go @@ -251,6 +251,7 @@ func (w *fileWatcher) checkFile() error { scanner := bufio.NewScanner(file) scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes) + scanner.Split(scanLinesCapped) for scanner.Scan() { line := scanner.Text() @@ -311,6 +312,17 @@ func (w *fileWatcher) checkFile() error { // initPosition records the file's metadata and, unless the watcher was created // to read from the start, sets the initial read position to the end +// scanLinesCapped is bufio.ScanLines, except that a line longer than +// core.MaxLogEntryBytes continues in the next one, as on the console source: +// the scanner would otherwise stop at it and every poll re-read the lines before. +func scanLinesCapped(data []byte, atEOF bool) (int, []byte, error) { + advance, token, err := bufio.ScanLines(data, atEOF) + if advance == 0 && token == nil && err == nil && len(data) >= core.MaxLogEntryBytes { + return core.MaxLogEntryBytes, data[:core.MaxLogEntryBytes], nil + } + return advance, token, err +} + func (w *fileWatcher) initPosition() error { file, err := os.Open(w.directory) if err != nil { diff --git a/internal/tlsx/tlsx.go b/internal/tlsx/tlsx.go index bc0ce00..d103933 100644 --- a/internal/tlsx/tlsx.go +++ b/internal/tlsx/tlsx.go @@ -36,8 +36,8 @@ func Server(o *config.TLSOptions, host string) (*tls.Config, error) { } files, issuer := o.CertFile != "" || o.KeyFile != "", o.IssuerCertFile != "" || o.IssuerKeyFile != "" switch { - case o.PinSHA256 != "": - return nil, fmt.Errorf("tls: pin_sha256 applies to dialers") + case o.CAFile != "" || o.ServerName != "" || o.InsecureSkipVerify || o.PinSHA256 != "": + return nil, fmt.Errorf("tls: ca_file, server_name, insecure_skip_verify and pin_sha256 apply to dialers") case files && (issuer || o.SelfSigned) || issuer && o.SelfSigned: return nil, fmt.Errorf("tls: set one of cert_file and key_file, self_signed, or issuer_cert_file and issuer_key_file") case len(o.Hosts) > 0 && !issuer && !o.SelfSigned: @@ -91,8 +91,8 @@ func Client(o *config.TLSOptions, host string) (*tls.Config, error) { return nil, nil } switch { - case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0: - return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file and hosts apply to listeners") + case o.SelfSigned || o.IssuerCertFile != "" || o.IssuerKeyFile != "" || len(o.Hosts) > 0 || o.ClientAuth || o.ClientCAFile != "": + return nil, fmt.Errorf("tls: self_signed, issuer_cert_file, issuer_key_file, hosts, client_auth and client_ca_file apply to listeners") case o.PinSHA256 != "" && (o.CAFile != "" || o.InsecureSkipVerify): return nil, fmt.Errorf("tls: pin_sha256 replaces ca_file and insecure_skip_verify: set one") } diff --git a/internal/tlsx/tlsx_test.go b/internal/tlsx/tlsx_test.go index f679c22..9279dfa 100644 --- a/internal/tlsx/tlsx_test.go +++ b/internal/tlsx/tlsx_test.go @@ -179,9 +179,11 @@ func TestTLSOptionConflicts(t *testing.T) { {config.TLSOptions{IssuerCertFile: "c"}, true, "must be set together"}, {config.TLSOptions{CertFile: "c"}, true, "must be set together"}, {config.TLSOptions{Hosts: []string{"a"}, CertFile: "c", KeyFile: "k"}, true, "hosts applies to"}, - {config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "applies to dialers"}, + {config.TLSOptions{SelfSigned: true, PinSHA256: pin}, true, "apply to dialers"}, + {config.TLSOptions{SelfSigned: true, CAFile: "ca"}, true, "apply to dialers"}, {config.TLSOptions{}, true, "listeners need"}, {config.TLSOptions{SelfSigned: true}, false, "apply to listeners"}, + {config.TLSOptions{ClientAuth: true, ClientCAFile: "ca"}, false, "apply to listeners"}, {config.TLSOptions{PinSHA256: pin, CAFile: "ca"}, false, "set one"}, {config.TLSOptions{PinSHA256: pin, InsecureSkipVerify: true}, false, "set one"}, {config.TLSOptions{PinSHA256: "sha256//short"}, false, "want sha256//BASE64"},