v0.18.0 raw and json formatter improvements, dockerfile, go bump to 1.27.1
This commit is contained in:
+40
@@ -0,0 +1,40 @@
|
|||||||
|
# Builder pin and go.mod directive are the same patch release deliberately;
|
||||||
|
# an older builder reports the mismatch only after downloading the module graph.
|
||||||
|
ARG GO_VERSION=1.27.1
|
||||||
|
|
||||||
|
FROM docker.io/library/golang:${GO_VERSION}-alpine AS build
|
||||||
|
|
||||||
|
# Git supplies Go's VCS build information; only /out/logwisp crosses stages.
|
||||||
|
RUN apk add --no-cache git
|
||||||
|
WORKDIR /src
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
ARG TARGETOS=linux
|
||||||
|
ARG TARGETARCH=amd64
|
||||||
|
ARG VERSION=dev
|
||||||
|
ARG REVISION=unknown
|
||||||
|
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||||
|
go build -trimpath \
|
||||||
|
-ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION}" \
|
||||||
|
-o /out/logwisp ./cmd/logwisp
|
||||||
|
|
||||||
|
FROM scratch
|
||||||
|
|
||||||
|
ARG VERSION=dev
|
||||||
|
ARG REVISION=unknown
|
||||||
|
|
||||||
|
LABEL org.opencontainers.image.title="logwisp" \
|
||||||
|
org.opencontainers.image.description="Log transport: sources, flow, sinks" \
|
||||||
|
org.opencontainers.image.source="https://github.com/lixenwraith/logwisp" \
|
||||||
|
org.opencontainers.image.revision="${REVISION}" \
|
||||||
|
org.opencontainers.image.version="${VERSION}" \
|
||||||
|
org.opencontainers.image.licenses="BSD-3-Clause"
|
||||||
|
|
||||||
|
COPY --from=build /out/logwisp /logwisp
|
||||||
|
|
||||||
|
# Numeric identity is required in scratch and satisfies a restricted pod spec.
|
||||||
|
USER 65532:65532
|
||||||
|
|
||||||
|
ENTRYPOINT ["/logwisp"]
|
||||||
@@ -152,6 +152,9 @@ directory = "./" # Directory to monitor (required, not re
|
|||||||
pattern = "*.log" # Glob pattern (* and ? only)
|
pattern = "*.log" # Glob pattern (* and ? only)
|
||||||
## Tailing an already-open file polls at a fixed 100ms, regardless of this value.
|
## Tailing an already-open file polls at a fixed 100ms, regardless of this value.
|
||||||
check_interval_ms = 100 # Directory rescan interval (min 10)
|
check_interval_ms = 100 # Directory rescan interval (min 10)
|
||||||
|
## raw = true never parses a line; with format type "raw" the file is relayed byte for byte.
|
||||||
|
raw = false # Keep the whole line as the message
|
||||||
|
from = "end" # "end" or "start" of a newly discovered file
|
||||||
|
|
||||||
## Console source (stdin, single instance per pipeline)
|
## Console source (stdin, single instance per pipeline)
|
||||||
# [[pipelines.plugin_sources]]
|
# [[pipelines.plugin_sources]]
|
||||||
|
|||||||
+16
-8
@@ -30,8 +30,10 @@ Omitting `[pipelines.flow.format]` entirely selects `raw`.
|
|||||||
|
|
||||||
### raw
|
### raw
|
||||||
|
|
||||||
Passthrough. `FlagRaw` bypasses both formatting and sanitization, so the
|
Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches
|
||||||
message reaches the sink exactly as the source produced it.
|
the sink exactly as the source produced it, with no timestamp, level or source
|
||||||
|
prefix added. An entry that also carries `fields` gets the fields JSON appended
|
||||||
|
verbatim after a single space — `raw` never drops data and never re-encodes it.
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
[pipelines.flow.format]
|
[pipelines.flow.format]
|
||||||
@@ -42,6 +44,12 @@ Fastest option, and the right one when you are relaying text that is already in
|
|||||||
its final form. Note that it also bypasses sanitization, so control characters
|
its final form. Note that it also bypasses sanitization, so control characters
|
||||||
in the source data reach your sinks intact.
|
in the source data reach your sinks intact.
|
||||||
|
|
||||||
|
Byte-exact transport needs a source that does not split the line: the `console`
|
||||||
|
source, or the `file` source with `raw = true`. Both put the whole line —
|
||||||
|
newline included — in the message and leave `fields` empty. The `file` source's
|
||||||
|
JSON branch splits a line into message and fields, so `raw` reassembles it as
|
||||||
|
`<msg> <fields>` rather than reproducing the original object.
|
||||||
|
|
||||||
### txt
|
### txt
|
||||||
|
|
||||||
Human-readable line output with a timestamp and level.
|
Human-readable line output with a timestamp and level.
|
||||||
@@ -89,7 +97,7 @@ you need to override the defaults.
|
|||||||
|
|
||||||
With `flags = 0` the formatter selects `1` for `type = "raw"` and `6`
|
With `flags = 0` the formatter selects `1` for `type = "raw"` and `6`
|
||||||
(timestamp + level) for every other type. `8` is added automatically whenever an
|
(timestamp + level) for every other type. `8` is added automatically whenever an
|
||||||
entry carries parseable `fields`.
|
entry carries parseable `fields` and `1` is not set; `1` always wins.
|
||||||
|
|
||||||
Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp
|
Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp
|
||||||
only, no level.
|
only, no level.
|
||||||
@@ -133,11 +141,11 @@ when writing downstream parsers or grep patterns.
|
|||||||
|
|
||||||
## Structured Fields
|
## Structured Fields
|
||||||
|
|
||||||
When an entry carries `Fields` (raw JSON), the formatter parses it and switches
|
When an entry carries `Fields` (raw JSON) and `FlagRaw` is not set, the
|
||||||
to structured rendering by adding the `StructuredJSON` flag automatically.
|
formatter parses it and switches to structured rendering by adding the
|
||||||
Fields reach a pipeline in two ways: from the `file` source when a tailed line
|
`StructuredJSON` flag automatically. Fields reach a pipeline in two ways: from
|
||||||
parses as JSON with a `fields` key, and from the heartbeat generator when
|
the `file` source when a tailed line parses as JSON with a `fields` key, and
|
||||||
`include_stats = true`.
|
from the heartbeat generator when `include_stats = true`.
|
||||||
|
|
||||||
## Choosing a Configuration
|
## Choosing a Configuration
|
||||||
|
|
||||||
|
|||||||
+26
-15
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
|
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
|
||||||
- **Architecture**: amd64
|
- **Architecture**: amd64
|
||||||
- **Go**: 1.26 or newer, to build from source
|
- **Go**: 1.27.1 or newer, to build from source
|
||||||
|
|
||||||
## Building from Source
|
## Building from Source
|
||||||
|
|
||||||
@@ -37,6 +37,24 @@ go build -o bin/logwisp ./cmd/logwisp
|
|||||||
`go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with
|
`go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with
|
||||||
the same loss of version metadata.
|
the same loss of version metadata.
|
||||||
|
|
||||||
|
## Container Image
|
||||||
|
|
||||||
|
The root `Dockerfile` builds the same package into `scratch` under UID 65532,
|
||||||
|
static and stripped. There is no shell and no config in the image: mount one and
|
||||||
|
name it, as the binary has no daemon mode and no built-in defaults worth running.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
REV=$(git rev-parse HEAD)
|
||||||
|
docker build -t "logwisp:$(git rev-parse --short HEAD)" \
|
||||||
|
--build-arg VERSION="$(git describe --tags --always)" \
|
||||||
|
--build-arg REVISION="$REV" .
|
||||||
|
docker run --rm -v /etc/logwisp:/etc/logwisp:ro logwisp:... -c /etc/logwisp/logwisp.toml
|
||||||
|
```
|
||||||
|
|
||||||
|
Sinks that listen (`http`, `tcp`) need their ports published; the read-only
|
||||||
|
root filesystem and dropped capabilities a restricted runtime imposes are all
|
||||||
|
compatible with it, provided a `file` sink's directory is writable by 65532.
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
Copy the annotated reference configuration and edit it:
|
Copy the annotated reference configuration and edit it:
|
||||||
@@ -173,27 +191,20 @@ mode; see [Operations](operations.md#checking-a-configuration).
|
|||||||
|
|
||||||
## Test Scripts
|
## Test Scripts
|
||||||
|
|
||||||
Two end-to-end scripts under `test/` build multi-node chain topologies against a
|
End-to-end scripts under `test/` run against a local build:
|
||||||
local build:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make
|
make
|
||||||
./test/chain-test.sh --auto # two independent relay pipelines
|
./test/chain-test.sh --auto # two independent relay pipelines
|
||||||
./test/chain-aggregate-test.sh --auto # fan-in: both edges into one pipeline
|
./test/chain-aggregate-test.sh --auto # fan-in: both edges into one pipeline
|
||||||
|
./test/mtls-chain-test.sh --auto # the same fan-in under mTLS
|
||||||
|
./test/passthrough-test.sh # file source relays a wide envelope intact
|
||||||
```
|
```
|
||||||
|
|
||||||
Without `--auto` they run the relay in the foreground for interactive
|
Without `--auto` the chain scripts run the relay in the foreground for
|
||||||
inspection. They need bash 5+, coreutils, and curl, and they bind ports
|
interactive inspection. They need bash 5+, coreutils, and curl, and they bind
|
||||||
15801–15804. Generated configuration and logs land in `test/run/`.
|
ports 15801–15804. The pass-through test binds nothing. Generated configuration
|
||||||
|
and logs land in `test/run/`.
|
||||||
> Two of the three `--auto` assertions currently report `FAIL` against a
|
|
||||||
> working build. They grep the sink output for `"source":"edge-tcp/` and
|
|
||||||
> `"node":"edge-http"`, but the JSON formatter emits the `node/source` label
|
|
||||||
> under the key `trace`. The transport itself is healthy — the
|
|
||||||
> `total_processed` assertion passes and the streamed entries carry
|
|
||||||
> `"trace":"edge-tcp/random_rand"` as expected. Until the assertions are
|
|
||||||
> updated, verify the streams by eye with `nc 127.0.0.1 15803` and
|
|
||||||
> `curl -sN http://127.0.0.1:15804/stream`.
|
|
||||||
|
|
||||||
## Uninstall
|
## Uninstall
|
||||||
|
|
||||||
|
|||||||
+2
-1
@@ -231,7 +231,8 @@ pipeline `total_dropped_by_sink` (sink backed up?), sink `total_processed`.
|
|||||||
|
|
||||||
- The watcher seeks to end-of-file on start; only content appended afterwards is
|
- The watcher seeks to end-of-file on start; only content appended afterwards is
|
||||||
read. Positions are in memory, so a restart re-seeks to end and anything
|
read. Positions are in memory, so a restart re-seeks to end and anything
|
||||||
written during the downtime is lost.
|
written during the downtime is lost. `from = "start"` reads each file whole
|
||||||
|
instead, and replays it on every restart.
|
||||||
- `pattern` is a filename glob with `*` and `?` only, and matching is not
|
- `pattern` is a filename glob with `*` and `?` only, and matching is not
|
||||||
recursive.
|
recursive.
|
||||||
- `check_interval_ms` governs how quickly a *new file* is noticed; tailing an
|
- `check_interval_ms` governs how quickly a *new file* is noticed; tailing an
|
||||||
|
|||||||
+17
-5
@@ -32,6 +32,8 @@ type = "file"
|
|||||||
directory = "/var/log/myapp"
|
directory = "/var/log/myapp"
|
||||||
pattern = "*.log"
|
pattern = "*.log"
|
||||||
check_interval_ms = 100
|
check_interval_ms = 100
|
||||||
|
raw = false
|
||||||
|
from = "end"
|
||||||
```
|
```
|
||||||
|
|
||||||
| Option | Type | Default | Description |
|
| Option | Type | Default | Description |
|
||||||
@@ -39,6 +41,8 @@ check_interval_ms = 100
|
|||||||
| `directory` | string | **required** | Directory to scan; not recursive |
|
| `directory` | string | **required** | Directory to scan; not recursive |
|
||||||
| `pattern` | string | `*` | Glob over filenames; `*` and `?` only |
|
| `pattern` | string | `*` | Glob over filenames; `*` and `?` only |
|
||||||
| `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` |
|
| `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` |
|
||||||
|
| `raw` | bool | `false` | Never parse a line: the whole line is the message |
|
||||||
|
| `from` | string | `end` | Where a new watcher starts: `end` or `start` of the file |
|
||||||
|
|
||||||
**Behaviour**
|
**Behaviour**
|
||||||
|
|
||||||
@@ -49,15 +53,23 @@ check_interval_ms = 100
|
|||||||
stopped and removed on the next scan.
|
stopped and removed on the next scan.
|
||||||
- A new watcher seeks to end-of-file. Positions live in memory only, so a
|
- A new watcher seeks to end-of-file. Positions live in memory only, so a
|
||||||
restart resumes from the current end of each file and content written while
|
restart resumes from the current end of each file and content written while
|
||||||
LogWisp was down is not read.
|
LogWisp was down is not read. `from = "start"` reads each file whole when its
|
||||||
|
watcher is created instead — what a process writing beside LogWisp needs, at
|
||||||
|
the cost of replaying a file already on disk at every restart.
|
||||||
- Rotation is detected from size decrease, modification-time reset, a position
|
- Rotation is detected from size decrease, modification-time reset, a position
|
||||||
beyond end-of-file, or an inode change. An inode change where the new file is
|
beyond end-of-file, or an inode change. An inode change where the new file is
|
||||||
already larger than the recorded position is treated as an atomic save, not a
|
already larger than the recorded position is treated as an atomic save, not a
|
||||||
rotation, and the position is preserved.
|
rotation, and the position is preserved.
|
||||||
- Lines are parsed as JSON when they contain `time`, `level`, `msg`, and
|
- A line is parsed as JSON only when it is an object whose top-level keys are
|
||||||
`fields` keys; `time` is read as RFC3339Nano. Anything else is kept as plain
|
all drawn from `time`, `level`, `msg` and `fields` — the four an entry can
|
||||||
text with the level inferred from common markers (`[ERROR]`, `WARN:`, and so
|
carry. `time` is read as RFC3339Nano. Any other key, and any non-object line,
|
||||||
on).
|
is kept whole as text with the level inferred from common markers
|
||||||
|
(`[ERROR]`, `WARN:`, and so on), because parsing it would drop the rest.
|
||||||
|
- `raw = true` skips the JSON branch entirely. The line, plus its newline,
|
||||||
|
becomes the message; `fields` stays empty, the time is the read time, and the
|
||||||
|
level is inferred from the text as for any unparsed line. Paired with
|
||||||
|
`format.type = "raw"` this is byte-exact transport for records LogWisp's
|
||||||
|
envelope cannot hold — see [Formatters](formatters.md#raw).
|
||||||
- `Source` is set to the file's base name.
|
- `Source` is set to the file's base name.
|
||||||
|
|
||||||
**Statistics**: per-watcher size, position, entries read, rotation count, and
|
**Statistics**: per-watcher size, position, entries read, rotation count, and
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
module logwisp
|
module logwisp
|
||||||
|
|
||||||
go 1.26.5
|
go 1.27.1
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98
|
github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98
|
||||||
|
|||||||
@@ -198,6 +198,8 @@ type FileSourceOptions struct {
|
|||||||
Directory string `toml:"directory"`
|
Directory string `toml:"directory"`
|
||||||
Pattern string `toml:"pattern"` // glob pattern
|
Pattern string `toml:"pattern"` // glob pattern
|
||||||
CheckIntervalMS int64 `toml:"check_interval_ms"`
|
CheckIntervalMS int64 `toml:"check_interval_ms"`
|
||||||
|
Raw bool `toml:"raw"` // keep the whole line as the message, never parse it
|
||||||
|
From string `toml:"from"` // "end" (default) or "start" of a newly discovered file
|
||||||
}
|
}
|
||||||
|
|
||||||
// ConsoleSourceOptions defines settings for a stdin-based source
|
// ConsoleSourceOptions defines settings for a stdin-based source
|
||||||
|
|||||||
+30
-43
@@ -90,57 +90,44 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
|
|||||||
|
|
||||||
// Format implements Formatter interface
|
// Format implements Formatter interface
|
||||||
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
|
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
|
||||||
// Map logwisp LogEntry to formatter args
|
return a.serialize(entry, a.flags), nil
|
||||||
level := mapLevel(entry.Level)
|
|
||||||
// syslog-style origin prefix for chained entries
|
|
||||||
src := sourceLabel(entry)
|
|
||||||
|
|
||||||
// Build args based on whether we have structured fields
|
|
||||||
var args []any
|
|
||||||
effectiveFlags := a.flags
|
|
||||||
|
|
||||||
if len(entry.Fields) > 0 {
|
|
||||||
// Parse fields JSON
|
|
||||||
var fields map[string]any
|
|
||||||
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
|
|
||||||
// Use structured JSON format for fields
|
|
||||||
args = []any{entry.Message, fields}
|
|
||||||
// Add structured flag to properly format fields as JSON object
|
|
||||||
effectiveFlags |= formatter.FlagStructuredJSON
|
|
||||||
return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if args == nil {
|
|
||||||
args = []any{entry.Message}
|
|
||||||
}
|
|
||||||
|
|
||||||
a.mu.Lock()
|
|
||||||
out := bytes.Clone(a.formatter.Format(effectiveFlags, entry.Time, level, src, args))
|
|
||||||
a.mu.Unlock()
|
|
||||||
return out, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// FormatWithFlags allows custom flags for specific formatting needs
|
// FormatWithFlags allows custom flags for specific formatting needs
|
||||||
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
|
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
|
||||||
level := mapLevel(entry.Level)
|
return a.serialize(entry, customFlags), nil
|
||||||
src := sourceLabel(entry)
|
}
|
||||||
|
|
||||||
var args []any
|
// serialize renders an entry under the given flags. The returned slice is a
|
||||||
if len(entry.Fields) > 0 {
|
// copy: the underlying formatter reuses one buffer and sinks retain payloads.
|
||||||
var fields map[string]any
|
func (a *FormatterAdapter) serialize(entry core.LogEntry, flags int64) []byte {
|
||||||
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
|
args, flags := formatArgs(entry, flags)
|
||||||
args = []any{entry.Message, fields}
|
|
||||||
customFlags |= formatter.FlagStructuredJSON
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if args == nil {
|
|
||||||
args = []any{entry.Message}
|
|
||||||
}
|
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
out := bytes.Clone(a.formatter.Format(customFlags, entry.Time, level, src, args))
|
out := bytes.Clone(a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args))
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
return out, nil
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatArgs pairs the entry with its flags. FlagRaw keeps the fields JSON
|
||||||
|
// verbatim beside the message rather than silently overriding the caller's
|
||||||
|
// choice of passthrough; every other mode renders it as a JSON object.
|
||||||
|
func formatArgs(entry core.LogEntry, flags int64) ([]any, int64) {
|
||||||
|
if len(entry.Fields) == 0 {
|
||||||
|
return []any{entry.Message}, flags
|
||||||
|
}
|
||||||
|
if flags&formatter.FlagRaw != 0 {
|
||||||
|
if entry.Message == "" {
|
||||||
|
return []any{[]byte(entry.Fields)}, flags
|
||||||
|
}
|
||||||
|
return []any{entry.Message, []byte(entry.Fields)}, flags
|
||||||
|
}
|
||||||
|
|
||||||
|
var fields map[string]any
|
||||||
|
if err := json.Unmarshal(entry.Fields, &fields); err != nil || len(fields) == 0 {
|
||||||
|
return []any{entry.Message}, flags
|
||||||
|
}
|
||||||
|
return []any{entry.Message, fields}, flags | formatter.FlagStructuredJSON
|
||||||
}
|
}
|
||||||
|
|
||||||
// Name returns formatter type
|
// Name returns formatter type
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ const (
|
|||||||
DefaultFileSourcePattern = "*"
|
DefaultFileSourcePattern = "*"
|
||||||
DefaultFileSourceCheckIntervalMS = 100
|
DefaultFileSourceCheckIntervalMS = 100
|
||||||
MinFileSourceCheckIntervalMS = 10
|
MinFileSourceCheckIntervalMS = 10
|
||||||
|
DefaultFileSourceFrom = "end"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewFileSourcePlugin creates a file source through plugin factory
|
// NewFileSourcePlugin creates a file source through plugin factory
|
||||||
@@ -90,6 +91,11 @@ func NewFileSourcePlugin(
|
|||||||
} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
|
} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
|
||||||
return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
|
return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
|
||||||
}
|
}
|
||||||
|
if opts.From == "" {
|
||||||
|
opts.From = DefaultFileSourceFrom
|
||||||
|
} else if err := lconfig.OneOf("start", "end")(opts.From); err != nil {
|
||||||
|
return nil, fmt.Errorf("from: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Create and return plugin instance
|
// Create and return plugin instance
|
||||||
fs := &FileSource{
|
fs := &FileSource{
|
||||||
@@ -116,7 +122,9 @@ func NewFileSourcePlugin(
|
|||||||
"component", "file_source",
|
"component", "file_source",
|
||||||
"instance_id", id,
|
"instance_id", id,
|
||||||
"directory", opts.Directory,
|
"directory", opts.Directory,
|
||||||
"pattern", opts.Pattern)
|
"pattern", opts.Pattern,
|
||||||
|
"raw", opts.Raw,
|
||||||
|
"from", opts.From)
|
||||||
|
|
||||||
return fs, nil
|
return fs, nil
|
||||||
}
|
}
|
||||||
@@ -261,7 +269,7 @@ func (fs *FileSource) ensureWatcher(path string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w := newFileWatcher(path, fs.publish, fs.logger)
|
w := newFileWatcher(path, fs.config.Raw, fs.config.From == "start", fs.publish, fs.logger)
|
||||||
fs.watchers[path] = w
|
fs.watchers[path] = w
|
||||||
|
|
||||||
fs.logger.Debug("msg", "Created file watcher",
|
fs.logger.Debug("msg", "Created file watcher",
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ type WatcherInfo struct {
|
|||||||
type fileWatcher struct {
|
type fileWatcher struct {
|
||||||
directory string
|
directory string
|
||||||
callback func(core.LogEntry)
|
callback func(core.LogEntry)
|
||||||
|
raw bool
|
||||||
position int64
|
position int64
|
||||||
size int64
|
size int64
|
||||||
inode uint64
|
inode uint64
|
||||||
@@ -46,12 +47,18 @@ type fileWatcher struct {
|
|||||||
logger *log.Logger
|
logger *log.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// newFileWatcher creates a new watcher for a specific file path
|
// newFileWatcher creates a new watcher for a specific file path.
|
||||||
func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
|
// A start position of 0 reads an existing file whole; -1 seeks to its end.
|
||||||
|
func newFileWatcher(directory string, raw, fromStart bool, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
|
||||||
|
position := int64(-1)
|
||||||
|
if fromStart {
|
||||||
|
position = 0
|
||||||
|
}
|
||||||
w := &fileWatcher{
|
w := &fileWatcher{
|
||||||
directory: directory,
|
directory: directory,
|
||||||
callback: callback,
|
callback: callback,
|
||||||
position: -1,
|
raw: raw,
|
||||||
|
position: position,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
}
|
}
|
||||||
w.lastReadTime.Store(time.Time{})
|
w.lastReadTime.Store(time.Time{})
|
||||||
@@ -60,8 +67,8 @@ func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.
|
|||||||
|
|
||||||
// watch starts the main monitoring loop for the file
|
// watch starts the main monitoring loop for the file
|
||||||
func (w *fileWatcher) watch(ctx context.Context) error {
|
func (w *fileWatcher) watch(ctx context.Context) error {
|
||||||
if err := w.seekToEnd(); err != nil {
|
if err := w.initPosition(); err != nil {
|
||||||
return fmt.Errorf("seekToEnd failed: %w", err)
|
return fmt.Errorf("initPosition failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
ticker := time.NewTicker(core.FileWatcherPollInterval)
|
ticker := time.NewTicker(core.FileWatcherPollInterval)
|
||||||
@@ -297,8 +304,9 @@ func (w *fileWatcher) checkFile() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// seekToEnd sets the initial read position to the end of the file
|
// initPosition records the file's metadata and, unless the watcher was created
|
||||||
func (w *fileWatcher) seekToEnd() error {
|
// to read from the start, sets the initial read position to the end
|
||||||
|
func (w *fileWatcher) initPosition() error {
|
||||||
file, err := os.Open(w.directory)
|
file, err := os.Open(w.directory)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
@@ -322,8 +330,6 @@ func (w *fileWatcher) seekToEnd() error {
|
|||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
// Keep existing position (including 0)
|
|
||||||
// First time initialization seeks to the end of the file
|
|
||||||
if w.position == -1 {
|
if w.position == -1 {
|
||||||
pos, err := file.Seek(0, io.SeekEnd)
|
pos, err := file.Seek(0, io.SeekEnd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -348,36 +354,67 @@ func (w *fileWatcher) isStopped() bool {
|
|||||||
return w.stopped
|
return w.stopped
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseLine attempts to parse a line as JSON, falling back to plain text
|
// parseLine converts a line into an entry, as JSON when nothing would be lost
|
||||||
func (w *fileWatcher) parseLine(line string) core.LogEntry {
|
func (w *fileWatcher) parseLine(line string) core.LogEntry {
|
||||||
var jsonLog struct {
|
if w.raw {
|
||||||
Time string `json:"time"`
|
// Newline restored: sinks write the payload as it stands
|
||||||
Level string `json:"level"`
|
|
||||||
Message string `json:"msg"`
|
|
||||||
Fields json.RawMessage `json:"fields"`
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.Unmarshal([]byte(line), &jsonLog); err == nil {
|
|
||||||
timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time)
|
|
||||||
if err != nil {
|
|
||||||
timestamp = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
return core.LogEntry{
|
return core.LogEntry{
|
||||||
Time: timestamp,
|
Time: time.Now(),
|
||||||
Source: filepath.Base(w.directory),
|
Source: filepath.Base(w.directory),
|
||||||
Level: jsonLog.Level,
|
Level: source.ExtractLogLevel(line),
|
||||||
Message: jsonLog.Message,
|
Message: line + "\n",
|
||||||
Fields: jsonLog.Fields,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
level := source.ExtractLogLevel(line)
|
if entry, ok := w.parseJSON(line); ok {
|
||||||
|
return entry
|
||||||
|
}
|
||||||
|
|
||||||
return core.LogEntry{
|
return core.LogEntry{
|
||||||
Time: time.Now(),
|
Time: time.Now(),
|
||||||
Source: filepath.Base(w.directory),
|
Source: filepath.Base(w.directory),
|
||||||
Level: level,
|
Level: source.ExtractLogLevel(line),
|
||||||
Message: line,
|
Message: line,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseJSON decodes a line into the entry envelope. A top-level key LogEntry
|
||||||
|
// cannot carry refuses the whole line, so a richer record reaches the pipeline
|
||||||
|
// as text rather than silently reduced to the four keys kept here.
|
||||||
|
func (w *fileWatcher) parseJSON(line string) (core.LogEntry, bool) {
|
||||||
|
if len(line) == 0 || line[0] != '{' {
|
||||||
|
return core.LogEntry{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var obj map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal([]byte(line), &obj); err != nil || len(obj) == 0 {
|
||||||
|
return core.LogEntry{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
entry := core.LogEntry{Time: time.Now(), Source: filepath.Base(w.directory)}
|
||||||
|
for key, val := range obj {
|
||||||
|
var err error
|
||||||
|
switch key {
|
||||||
|
case "time":
|
||||||
|
var ts string
|
||||||
|
if json.Unmarshal(val, &ts) == nil {
|
||||||
|
if t, terr := time.Parse(time.RFC3339Nano, ts); terr == nil {
|
||||||
|
entry.Time = t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "level":
|
||||||
|
err = json.Unmarshal(val, &entry.Level)
|
||||||
|
case "msg":
|
||||||
|
err = json.Unmarshal(val, &entry.Message)
|
||||||
|
case "fields":
|
||||||
|
entry.Fields = val
|
||||||
|
default:
|
||||||
|
return core.LogEntry{}, false
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return core.LogEntry{}, false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry, true
|
||||||
|
}
|
||||||
|
|||||||
Executable
+92
@@ -0,0 +1,92 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# logwisp file source pass-through test
|
||||||
|
#
|
||||||
|
# file src (raw, from=start) --> raw format --> file sink byte-exact relay
|
||||||
|
# file src (defaults) --> raw format --> file sink no key dropped
|
||||||
|
#
|
||||||
|
# The fixture is a record whose envelope is wider than time/level/msg/fields,
|
||||||
|
# which is what the narrow JSON branch used to reduce to an empty message.
|
||||||
|
#
|
||||||
|
# Usage: ./passthrough-test.sh
|
||||||
|
# Requires: bash 5+, coreutils (timeout). Linux dev host only.
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
|
||||||
|
RUN="$SCRIPT_DIR/run/passthrough"
|
||||||
|
|
||||||
|
[[ -x $BIN ]] || { echo "no binary at $BIN; run make build" >&2; exit 1; }
|
||||||
|
|
||||||
|
rm -rf "$RUN"
|
||||||
|
mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed"
|
||||||
|
|
||||||
|
cat > "$RUN/in/wide.jsonl" <<'EOF'
|
||||||
|
{"time":"2026-09-08T21:35:44.178372768-04:00","level":"INFO","sub":"app","run":0,"tick":0,"frame":0,"fields":{"msg":"init begin","mode":"play"}}
|
||||||
|
{"time":"2026-09-08T21:35:44.178581366-04:00","level":"PROC","run":0,"tick":0,"frame":0,"fields":{"seq":1,"seed":1788917744178374836}}
|
||||||
|
plain text line, not JSON at all
|
||||||
|
EOF
|
||||||
|
|
||||||
|
conf() { # sink_dir raw
|
||||||
|
cat <<EOF
|
||||||
|
quiet = true
|
||||||
|
status_reporter = false
|
||||||
|
[logging]
|
||||||
|
output = "stderr"
|
||||||
|
level = "error"
|
||||||
|
[[pipelines]]
|
||||||
|
name = "passthrough"
|
||||||
|
[pipelines.flow.format]
|
||||||
|
type = "raw"
|
||||||
|
sanitizer_policy = "raw"
|
||||||
|
[[pipelines.plugin_sources]]
|
||||||
|
id = "wide"
|
||||||
|
type = "file"
|
||||||
|
[pipelines.plugin_sources.config]
|
||||||
|
directory = "$RUN/in"
|
||||||
|
pattern = "*.jsonl"
|
||||||
|
raw = $2
|
||||||
|
from = "start"
|
||||||
|
[[pipelines.plugin_sinks]]
|
||||||
|
id = "out"
|
||||||
|
type = "file"
|
||||||
|
[pipelines.plugin_sinks.config]
|
||||||
|
directory = "$1"
|
||||||
|
name = "relay"
|
||||||
|
flush_interval_ms = 100
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
conf "$RUN/out-raw" true > "$RUN/raw.toml"
|
||||||
|
conf "$RUN/out-parsed" false > "$RUN/parsed.toml"
|
||||||
|
|
||||||
|
for c in raw parsed; do
|
||||||
|
timeout 5 "$BIN" -c "$RUN/$c.toml" > "$RUN/$c.out" 2>&1
|
||||||
|
done
|
||||||
|
|
||||||
|
fail=0
|
||||||
|
check() { # label condition_result
|
||||||
|
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# 1. raw = true relays the file byte for byte
|
||||||
|
if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" > /dev/null; then
|
||||||
|
check "raw = true: output identical to input" 1
|
||||||
|
else
|
||||||
|
check "raw = true: output identical to input" 0
|
||||||
|
diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6
|
||||||
|
fi
|
||||||
|
|
||||||
|
# 2. the default parse keeps every key, JSON branch refused on the wide envelope
|
||||||
|
n=$(grep -c '"sub":"app"' "$RUN/out-parsed/relay.log")
|
||||||
|
check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $(( n == 1 ))
|
||||||
|
n=$(grep -c '1788917744178374836' "$RUN/out-parsed/relay.log")
|
||||||
|
check "defaults: large integers are not re-encoded through float64 ($n)" $(( n == 1 ))
|
||||||
|
|
||||||
|
echo "================================================================"
|
||||||
|
if (( fail == 0 )); then
|
||||||
|
echo "RESULT: ALL PASS"
|
||||||
|
else
|
||||||
|
echo "RESULT: FAILURES — inspect $RUN/"
|
||||||
|
fi
|
||||||
|
exit "$fail"
|
||||||
Reference in New Issue
Block a user