diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..58af5e6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# Builder pin and go.mod directive are the same patch release deliberately; +# an older builder reports the mismatch only after downloading the module graph. +ARG GO_VERSION=1.27.1 + +FROM docker.io/library/golang:${GO_VERSION}-alpine AS build + +# Git supplies Go's VCS build information; only /out/logwisp crosses stages. +RUN apk add --no-cache git +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . + +ARG TARGETOS=linux +ARG TARGETARCH=amd64 +ARG VERSION=dev +ARG REVISION=unknown +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath \ + -ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION}" \ + -o /out/logwisp ./cmd/logwisp + +FROM scratch + +ARG VERSION=dev +ARG REVISION=unknown + +LABEL org.opencontainers.image.title="logwisp" \ + org.opencontainers.image.description="Log transport: sources, flow, sinks" \ + org.opencontainers.image.source="https://github.com/lixenwraith/logwisp" \ + org.opencontainers.image.revision="${REVISION}" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.licenses="BSD-3-Clause" + +COPY --from=build /out/logwisp /logwisp + +# Numeric identity is required in scratch and satisfies a restricted pod spec. +USER 65532:65532 + +ENTRYPOINT ["/logwisp"] diff --git a/config/logwisp.toml b/config/logwisp.toml index df7ca85..5313e32 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -152,6 +152,9 @@ directory = "./" # Directory to monitor (required, not re pattern = "*.log" # Glob pattern (* and ? only) ## Tailing an already-open file polls at a fixed 100ms, regardless of this value. check_interval_ms = 100 # Directory rescan interval (min 10) +## raw = true never parses a line; with format type "raw" the file is relayed byte for byte. +raw = false # Keep the whole line as the message +from = "end" # "end" or "start" of a newly discovered file ## Console source (stdin, single instance per pipeline) # [[pipelines.plugin_sources]] diff --git a/doc/formatters.md b/doc/formatters.md index a68967e..335d6fe 100644 --- a/doc/formatters.md +++ b/doc/formatters.md @@ -30,8 +30,10 @@ Omitting `[pipelines.flow.format]` entirely selects `raw`. ### raw -Passthrough. `FlagRaw` bypasses both formatting and sanitization, so the -message reaches the sink exactly as the source produced it. +Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches +the sink exactly as the source produced it, with no timestamp, level or source +prefix added. An entry that also carries `fields` gets the fields JSON appended +verbatim after a single space — `raw` never drops data and never re-encodes it. ```toml [pipelines.flow.format] @@ -42,6 +44,12 @@ Fastest option, and the right one when you are relaying text that is already in its final form. Note that it also bypasses sanitization, so control characters in the source data reach your sinks intact. +Byte-exact transport needs a source that does not split the line: the `console` +source, or the `file` source with `raw = true`. Both put the whole line — +newline included — in the message and leave `fields` empty. The `file` source's +JSON branch splits a line into message and fields, so `raw` reassembles it as +` ` rather than reproducing the original object. + ### txt Human-readable line output with a timestamp and level. @@ -89,7 +97,7 @@ you need to override the defaults. With `flags = 0` the formatter selects `1` for `type = "raw"` and `6` (timestamp + level) for every other type. `8` is added automatically whenever an -entry carries parseable `fields`. +entry carries parseable `fields` and `1` is not set; `1` always wins. Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp only, no level. @@ -133,11 +141,11 @@ when writing downstream parsers or grep patterns. ## Structured Fields -When an entry carries `Fields` (raw JSON), the formatter parses it and switches -to structured rendering by adding the `StructuredJSON` flag automatically. -Fields reach a pipeline in two ways: from the `file` source when a tailed line -parses as JSON with a `fields` key, and from the heartbeat generator when -`include_stats = true`. +When an entry carries `Fields` (raw JSON) and `FlagRaw` is not set, the +formatter parses it and switches to structured rendering by adding the +`StructuredJSON` flag automatically. Fields reach a pipeline in two ways: from +the `file` source when a tailed line parses as JSON with a `fields` key, and +from the heartbeat generator when `include_stats = true`. ## Choosing a Configuration diff --git a/doc/installation.md b/doc/installation.md index f370604..0f7b296 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -4,7 +4,7 @@ - **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+) - **Architecture**: amd64 -- **Go**: 1.26 or newer, to build from source +- **Go**: 1.27.1 or newer, to build from source ## Building from Source @@ -37,6 +37,24 @@ go build -o bin/logwisp ./cmd/logwisp `go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with the same loss of version metadata. +## Container Image + +The root `Dockerfile` builds the same package into `scratch` under UID 65532, +static and stripped. There is no shell and no config in the image: mount one and +name it, as the binary has no daemon mode and no built-in defaults worth running. + +```bash +REV=$(git rev-parse HEAD) +docker build -t "logwisp:$(git rev-parse --short HEAD)" \ + --build-arg VERSION="$(git describe --tags --always)" \ + --build-arg REVISION="$REV" . +docker run --rm -v /etc/logwisp:/etc/logwisp:ro logwisp:... -c /etc/logwisp/logwisp.toml +``` + +Sinks that listen (`http`, `tcp`) need their ports published; the read-only +root filesystem and dropped capabilities a restricted runtime imposes are all +compatible with it, provided a `file` sink's directory is writable by 65532. + ## Configuration Copy the annotated reference configuration and edit it: @@ -173,27 +191,20 @@ mode; see [Operations](operations.md#checking-a-configuration). ## Test Scripts -Two end-to-end scripts under `test/` build multi-node chain topologies against a -local build: +End-to-end scripts under `test/` run against a local build: ```bash make ./test/chain-test.sh --auto # two independent relay pipelines ./test/chain-aggregate-test.sh --auto # fan-in: both edges into one pipeline +./test/mtls-chain-test.sh --auto # the same fan-in under mTLS +./test/passthrough-test.sh # file source relays a wide envelope intact ``` -Without `--auto` they run the relay in the foreground for interactive -inspection. They need bash 5+, coreutils, and curl, and they bind ports -15801–15804. Generated configuration and logs land in `test/run/`. - -> Two of the three `--auto` assertions currently report `FAIL` against a -> working build. They grep the sink output for `"source":"edge-tcp/` and -> `"node":"edge-http"`, but the JSON formatter emits the `node/source` label -> under the key `trace`. The transport itself is healthy — the -> `total_processed` assertion passes and the streamed entries carry -> `"trace":"edge-tcp/random_rand"` as expected. Until the assertions are -> updated, verify the streams by eye with `nc 127.0.0.1 15803` and -> `curl -sN http://127.0.0.1:15804/stream`. +Without `--auto` the chain scripts run the relay in the foreground for +interactive inspection. They need bash 5+, coreutils, and curl, and they bind +ports 15801–15804. The pass-through test binds nothing. Generated configuration +and logs land in `test/run/`. ## Uninstall diff --git a/doc/operations.md b/doc/operations.md index 9e497b1..892fd58 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -231,7 +231,8 @@ pipeline `total_dropped_by_sink` (sink backed up?), sink `total_processed`. - The watcher seeks to end-of-file on start; only content appended afterwards is read. Positions are in memory, so a restart re-seeks to end and anything - written during the downtime is lost. + written during the downtime is lost. `from = "start"` reads each file whole + instead, and replays it on every restart. - `pattern` is a filename glob with `*` and `?` only, and matching is not recursive. - `check_interval_ms` governs how quickly a *new file* is noticed; tailing an diff --git a/doc/sources.md b/doc/sources.md index 3d64a1f..9c0a184 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -32,6 +32,8 @@ type = "file" directory = "/var/log/myapp" pattern = "*.log" check_interval_ms = 100 +raw = false +from = "end" ``` | Option | Type | Default | Description | @@ -39,6 +41,8 @@ check_interval_ms = 100 | `directory` | string | **required** | Directory to scan; not recursive | | `pattern` | string | `*` | Glob over filenames; `*` and `?` only | | `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` | +| `raw` | bool | `false` | Never parse a line: the whole line is the message | +| `from` | string | `end` | Where a new watcher starts: `end` or `start` of the file | **Behaviour** @@ -49,15 +53,23 @@ check_interval_ms = 100 stopped and removed on the next scan. - A new watcher seeks to end-of-file. Positions live in memory only, so a restart resumes from the current end of each file and content written while - LogWisp was down is not read. + LogWisp was down is not read. `from = "start"` reads each file whole when its + watcher is created instead — what a process writing beside LogWisp needs, at + the cost of replaying a file already on disk at every restart. - Rotation is detected from size decrease, modification-time reset, a position beyond end-of-file, or an inode change. An inode change where the new file is already larger than the recorded position is treated as an atomic save, not a rotation, and the position is preserved. -- Lines are parsed as JSON when they contain `time`, `level`, `msg`, and - `fields` keys; `time` is read as RFC3339Nano. Anything else is kept as plain - text with the level inferred from common markers (`[ERROR]`, `WARN:`, and so - on). +- A line is parsed as JSON only when it is an object whose top-level keys are + all drawn from `time`, `level`, `msg` and `fields` — the four an entry can + carry. `time` is read as RFC3339Nano. Any other key, and any non-object line, + is kept whole as text with the level inferred from common markers + (`[ERROR]`, `WARN:`, and so on), because parsing it would drop the rest. +- `raw = true` skips the JSON branch entirely. The line, plus its newline, + becomes the message; `fields` stays empty, the time is the read time, and the + level is inferred from the text as for any unparsed line. Paired with + `format.type = "raw"` this is byte-exact transport for records LogWisp's + envelope cannot hold — see [Formatters](formatters.md#raw). - `Source` is set to the file's base name. **Statistics**: per-watcher size, position, entries read, rotation count, and diff --git a/go.mod b/go.mod index 22d66d1..c1f83f1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module logwisp -go 1.26.5 +go 1.27.1 require ( github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98 diff --git a/internal/config/config.go b/internal/config/config.go index 79b05dc..2cce0ff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -198,6 +198,8 @@ type FileSourceOptions struct { Directory string `toml:"directory"` Pattern string `toml:"pattern"` // glob pattern CheckIntervalMS int64 `toml:"check_interval_ms"` + Raw bool `toml:"raw"` // keep the whole line as the message, never parse it + From string `toml:"from"` // "end" (default) or "start" of a newly discovered file } // ConsoleSourceOptions defines settings for a stdin-based source diff --git a/internal/format/adapter.go b/internal/format/adapter.go index d6e1595..16a81cb 100644 --- a/internal/format/adapter.go +++ b/internal/format/adapter.go @@ -90,57 +90,44 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) { // Format implements Formatter interface func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) { - // Map logwisp LogEntry to formatter args - level := mapLevel(entry.Level) - // syslog-style origin prefix for chained entries - src := sourceLabel(entry) - - // Build args based on whether we have structured fields - var args []any - effectiveFlags := a.flags - - if len(entry.Fields) > 0 { - // Parse fields JSON - var fields map[string]any - if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 { - // Use structured JSON format for fields - args = []any{entry.Message, fields} - // Add structured flag to properly format fields as JSON object - effectiveFlags |= formatter.FlagStructuredJSON - return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil - } - } - if args == nil { - args = []any{entry.Message} - } - - a.mu.Lock() - out := bytes.Clone(a.formatter.Format(effectiveFlags, entry.Time, level, src, args)) - a.mu.Unlock() - return out, nil + return a.serialize(entry, a.flags), nil } // FormatWithFlags allows custom flags for specific formatting needs func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) { - level := mapLevel(entry.Level) - src := sourceLabel(entry) + return a.serialize(entry, customFlags), nil +} - var args []any - if len(entry.Fields) > 0 { - var fields map[string]any - if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 { - args = []any{entry.Message, fields} - customFlags |= formatter.FlagStructuredJSON - } - } - if args == nil { - args = []any{entry.Message} - } +// serialize renders an entry under the given flags. The returned slice is a +// copy: the underlying formatter reuses one buffer and sinks retain payloads. +func (a *FormatterAdapter) serialize(entry core.LogEntry, flags int64) []byte { + args, flags := formatArgs(entry, flags) a.mu.Lock() - out := bytes.Clone(a.formatter.Format(customFlags, entry.Time, level, src, args)) + out := bytes.Clone(a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args)) a.mu.Unlock() - return out, nil + return out +} + +// formatArgs pairs the entry with its flags. FlagRaw keeps the fields JSON +// verbatim beside the message rather than silently overriding the caller's +// choice of passthrough; every other mode renders it as a JSON object. +func formatArgs(entry core.LogEntry, flags int64) ([]any, int64) { + if len(entry.Fields) == 0 { + return []any{entry.Message}, flags + } + if flags&formatter.FlagRaw != 0 { + if entry.Message == "" { + return []any{[]byte(entry.Fields)}, flags + } + return []any{entry.Message, []byte(entry.Fields)}, flags + } + + var fields map[string]any + if err := json.Unmarshal(entry.Fields, &fields); err != nil || len(fields) == 0 { + return []any{entry.Message}, flags + } + return []any{entry.Message, fields}, flags | formatter.FlagStructuredJSON } // Name returns formatter type diff --git a/internal/source/file/file.go b/internal/source/file/file.go index 58c4ac5..a63aeb9 100644 --- a/internal/source/file/file.go +++ b/internal/source/file/file.go @@ -61,6 +61,7 @@ const ( DefaultFileSourcePattern = "*" DefaultFileSourceCheckIntervalMS = 100 MinFileSourceCheckIntervalMS = 10 + DefaultFileSourceFrom = "end" ) // NewFileSourcePlugin creates a file source through plugin factory @@ -90,6 +91,11 @@ func NewFileSourcePlugin( } else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS { return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS) } + if opts.From == "" { + opts.From = DefaultFileSourceFrom + } else if err := lconfig.OneOf("start", "end")(opts.From); err != nil { + return nil, fmt.Errorf("from: %w", err) + } // Create and return plugin instance fs := &FileSource{ @@ -116,7 +122,9 @@ func NewFileSourcePlugin( "component", "file_source", "instance_id", id, "directory", opts.Directory, - "pattern", opts.Pattern) + "pattern", opts.Pattern, + "raw", opts.Raw, + "from", opts.From) return fs, nil } @@ -261,7 +269,7 @@ func (fs *FileSource) ensureWatcher(path string) { return } - w := newFileWatcher(path, fs.publish, fs.logger) + w := newFileWatcher(path, fs.config.Raw, fs.config.From == "start", fs.publish, fs.logger) fs.watchers[path] = w fs.logger.Debug("msg", "Created file watcher", diff --git a/internal/source/file/file_watcher.go b/internal/source/file/file_watcher.go index 3119501..8af71eb 100644 --- a/internal/source/file/file_watcher.go +++ b/internal/source/file/file_watcher.go @@ -34,6 +34,7 @@ type WatcherInfo struct { type fileWatcher struct { directory string callback func(core.LogEntry) + raw bool position int64 size int64 inode uint64 @@ -46,12 +47,18 @@ type fileWatcher struct { logger *log.Logger } -// newFileWatcher creates a new watcher for a specific file path -func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher { +// newFileWatcher creates a new watcher for a specific file path. +// A start position of 0 reads an existing file whole; -1 seeks to its end. +func newFileWatcher(directory string, raw, fromStart bool, callback func(core.LogEntry), logger *log.Logger) *fileWatcher { + position := int64(-1) + if fromStart { + position = 0 + } w := &fileWatcher{ directory: directory, callback: callback, - position: -1, + raw: raw, + position: position, logger: logger, } w.lastReadTime.Store(time.Time{}) @@ -60,8 +67,8 @@ func newFileWatcher(directory string, callback func(core.LogEntry), logger *log. // watch starts the main monitoring loop for the file func (w *fileWatcher) watch(ctx context.Context) error { - if err := w.seekToEnd(); err != nil { - return fmt.Errorf("seekToEnd failed: %w", err) + if err := w.initPosition(); err != nil { + return fmt.Errorf("initPosition failed: %w", err) } ticker := time.NewTicker(core.FileWatcherPollInterval) @@ -297,8 +304,9 @@ func (w *fileWatcher) checkFile() error { return nil } -// seekToEnd sets the initial read position to the end of the file -func (w *fileWatcher) seekToEnd() error { +// initPosition records the file's metadata and, unless the watcher was created +// to read from the start, sets the initial read position to the end +func (w *fileWatcher) initPosition() error { file, err := os.Open(w.directory) if err != nil { if os.IsNotExist(err) { @@ -322,8 +330,6 @@ func (w *fileWatcher) seekToEnd() error { w.mu.Lock() defer w.mu.Unlock() - // Keep existing position (including 0) - // First time initialization seeks to the end of the file if w.position == -1 { pos, err := file.Seek(0, io.SeekEnd) if err != nil { @@ -348,36 +354,67 @@ func (w *fileWatcher) isStopped() bool { return w.stopped } -// parseLine attempts to parse a line as JSON, falling back to plain text +// parseLine converts a line into an entry, as JSON when nothing would be lost func (w *fileWatcher) parseLine(line string) core.LogEntry { - var jsonLog struct { - Time string `json:"time"` - Level string `json:"level"` - Message string `json:"msg"` - Fields json.RawMessage `json:"fields"` - } - - if err := json.Unmarshal([]byte(line), &jsonLog); err == nil { - timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time) - if err != nil { - timestamp = time.Now() - } - + if w.raw { + // Newline restored: sinks write the payload as it stands return core.LogEntry{ - Time: timestamp, + Time: time.Now(), Source: filepath.Base(w.directory), - Level: jsonLog.Level, - Message: jsonLog.Message, - Fields: jsonLog.Fields, + Level: source.ExtractLogLevel(line), + Message: line + "\n", } } - level := source.ExtractLogLevel(line) + if entry, ok := w.parseJSON(line); ok { + return entry + } return core.LogEntry{ Time: time.Now(), Source: filepath.Base(w.directory), - Level: level, + Level: source.ExtractLogLevel(line), Message: line, } -} \ No newline at end of file +} + +// parseJSON decodes a line into the entry envelope. A top-level key LogEntry +// cannot carry refuses the whole line, so a richer record reaches the pipeline +// as text rather than silently reduced to the four keys kept here. +func (w *fileWatcher) parseJSON(line string) (core.LogEntry, bool) { + if len(line) == 0 || line[0] != '{' { + return core.LogEntry{}, false + } + + var obj map[string]json.RawMessage + if err := json.Unmarshal([]byte(line), &obj); err != nil || len(obj) == 0 { + return core.LogEntry{}, false + } + + entry := core.LogEntry{Time: time.Now(), Source: filepath.Base(w.directory)} + for key, val := range obj { + var err error + switch key { + case "time": + var ts string + if json.Unmarshal(val, &ts) == nil { + if t, terr := time.Parse(time.RFC3339Nano, ts); terr == nil { + entry.Time = t + } + } + case "level": + err = json.Unmarshal(val, &entry.Level) + case "msg": + err = json.Unmarshal(val, &entry.Message) + case "fields": + entry.Fields = val + default: + return core.LogEntry{}, false + } + if err != nil { + return core.LogEntry{}, false + } + } + + return entry, true +} diff --git a/test/passthrough-test.sh b/test/passthrough-test.sh new file mode 100755 index 0000000..cceb421 --- /dev/null +++ b/test/passthrough-test.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# logwisp file source pass-through test +# +# file src (raw, from=start) --> raw format --> file sink byte-exact relay +# file src (defaults) --> raw format --> file sink no key dropped +# +# The fixture is a record whose envelope is wider than time/level/msg/fields, +# which is what the narrow JSON branch used to reduce to an empty message. +# +# Usage: ./passthrough-test.sh +# Requires: bash 5+, coreutils (timeout). Linux dev host only. + +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}" +RUN="$SCRIPT_DIR/run/passthrough" + +[[ -x $BIN ]] || { echo "no binary at $BIN; run make build" >&2; exit 1; } + +rm -rf "$RUN" +mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed" + +cat > "$RUN/in/wide.jsonl" <<'EOF' +{"time":"2026-09-08T21:35:44.178372768-04:00","level":"INFO","sub":"app","run":0,"tick":0,"frame":0,"fields":{"msg":"init begin","mode":"play"}} +{"time":"2026-09-08T21:35:44.178581366-04:00","level":"PROC","run":0,"tick":0,"frame":0,"fields":{"seq":1,"seed":1788917744178374836}} +plain text line, not JSON at all +EOF + +conf() { # sink_dir raw + cat < "$RUN/raw.toml" +conf "$RUN/out-parsed" false > "$RUN/parsed.toml" + +for c in raw parsed; do + timeout 5 "$BIN" -c "$RUN/$c.toml" > "$RUN/$c.out" 2>&1 +done + +fail=0 +check() { # label condition_result + if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi +} + +# 1. raw = true relays the file byte for byte +if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" > /dev/null; then + check "raw = true: output identical to input" 1 +else + check "raw = true: output identical to input" 0 + diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6 +fi + +# 2. the default parse keeps every key, JSON branch refused on the wide envelope +n=$(grep -c '"sub":"app"' "$RUN/out-parsed/relay.log") +check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $(( n == 1 )) +n=$(grep -c '1788917744178374836' "$RUN/out-parsed/relay.log") +check "defaults: large integers are not re-encoded through float64 ($n)" $(( n == 1 )) + +echo "================================================================" +if (( fail == 0 )); then + echo "RESULT: ALL PASS" +else + echo "RESULT: FAILURES — inspect $RUN/" +fi +exit "$fail"