v0.18.0 raw and json formatter improvements, dockerfile, go bump to 1.27.1
This commit is contained in:
+40
@@ -0,0 +1,40 @@
|
||||
# Builder pin and go.mod directive are the same patch release deliberately;
|
||||
# an older builder reports the mismatch only after downloading the module graph.
|
||||
ARG GO_VERSION=1.27.1
|
||||
|
||||
FROM docker.io/library/golang:${GO_VERSION}-alpine AS build
|
||||
|
||||
# Git supplies Go's VCS build information; only /out/logwisp crosses stages.
|
||||
RUN apk add --no-cache git
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
|
||||
ARG TARGETOS=linux
|
||||
ARG TARGETARCH=amd64
|
||||
ARG VERSION=dev
|
||||
ARG REVISION=unknown
|
||||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \
|
||||
go build -trimpath \
|
||||
-ldflags="-s -w -X logwisp/internal/version.Version=${VERSION} -X logwisp/internal/version.GitCommit=${REVISION}" \
|
||||
-o /out/logwisp ./cmd/logwisp
|
||||
|
||||
FROM scratch
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG REVISION=unknown
|
||||
|
||||
LABEL org.opencontainers.image.title="logwisp" \
|
||||
org.opencontainers.image.description="Log transport: sources, flow, sinks" \
|
||||
org.opencontainers.image.source="https://github.com/lixenwraith/logwisp" \
|
||||
org.opencontainers.image.revision="${REVISION}" \
|
||||
org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.licenses="BSD-3-Clause"
|
||||
|
||||
COPY --from=build /out/logwisp /logwisp
|
||||
|
||||
# Numeric identity is required in scratch and satisfies a restricted pod spec.
|
||||
USER 65532:65532
|
||||
|
||||
ENTRYPOINT ["/logwisp"]
|
||||
@@ -152,6 +152,9 @@ directory = "./" # Directory to monitor (required, not re
|
||||
pattern = "*.log" # Glob pattern (* and ? only)
|
||||
## Tailing an already-open file polls at a fixed 100ms, regardless of this value.
|
||||
check_interval_ms = 100 # Directory rescan interval (min 10)
|
||||
## raw = true never parses a line; with format type "raw" the file is relayed byte for byte.
|
||||
raw = false # Keep the whole line as the message
|
||||
from = "end" # "end" or "start" of a newly discovered file
|
||||
|
||||
## Console source (stdin, single instance per pipeline)
|
||||
# [[pipelines.plugin_sources]]
|
||||
|
||||
+16
-8
@@ -30,8 +30,10 @@ Omitting `[pipelines.flow.format]` entirely selects `raw`.
|
||||
|
||||
### raw
|
||||
|
||||
Passthrough. `FlagRaw` bypasses both formatting and sanitization, so the
|
||||
message reaches the sink exactly as the source produced it.
|
||||
Passthrough. `FlagRaw` bypasses formatting and sanitization: the message reaches
|
||||
the sink exactly as the source produced it, with no timestamp, level or source
|
||||
prefix added. An entry that also carries `fields` gets the fields JSON appended
|
||||
verbatim after a single space — `raw` never drops data and never re-encodes it.
|
||||
|
||||
```toml
|
||||
[pipelines.flow.format]
|
||||
@@ -42,6 +44,12 @@ Fastest option, and the right one when you are relaying text that is already in
|
||||
its final form. Note that it also bypasses sanitization, so control characters
|
||||
in the source data reach your sinks intact.
|
||||
|
||||
Byte-exact transport needs a source that does not split the line: the `console`
|
||||
source, or the `file` source with `raw = true`. Both put the whole line —
|
||||
newline included — in the message and leave `fields` empty. The `file` source's
|
||||
JSON branch splits a line into message and fields, so `raw` reassembles it as
|
||||
`<msg> <fields>` rather than reproducing the original object.
|
||||
|
||||
### txt
|
||||
|
||||
Human-readable line output with a timestamp and level.
|
||||
@@ -89,7 +97,7 @@ you need to override the defaults.
|
||||
|
||||
With `flags = 0` the formatter selects `1` for `type = "raw"` and `6`
|
||||
(timestamp + level) for every other type. `8` is added automatically whenever an
|
||||
entry carries parseable `fields`.
|
||||
entry carries parseable `fields` and `1` is not set; `1` always wins.
|
||||
|
||||
Examples: `flags = 4` for level only, no timestamp; `flags = 2` for timestamp
|
||||
only, no level.
|
||||
@@ -133,11 +141,11 @@ when writing downstream parsers or grep patterns.
|
||||
|
||||
## Structured Fields
|
||||
|
||||
When an entry carries `Fields` (raw JSON), the formatter parses it and switches
|
||||
to structured rendering by adding the `StructuredJSON` flag automatically.
|
||||
Fields reach a pipeline in two ways: from the `file` source when a tailed line
|
||||
parses as JSON with a `fields` key, and from the heartbeat generator when
|
||||
`include_stats = true`.
|
||||
When an entry carries `Fields` (raw JSON) and `FlagRaw` is not set, the
|
||||
formatter parses it and switches to structured rendering by adding the
|
||||
`StructuredJSON` flag automatically. Fields reach a pipeline in two ways: from
|
||||
the `file` source when a tailed line parses as JSON with a `fields` key, and
|
||||
from the heartbeat generator when `include_stats = true`.
|
||||
|
||||
## Choosing a Configuration
|
||||
|
||||
|
||||
+26
-15
@@ -4,7 +4,7 @@
|
||||
|
||||
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
|
||||
- **Architecture**: amd64
|
||||
- **Go**: 1.26 or newer, to build from source
|
||||
- **Go**: 1.27.1 or newer, to build from source
|
||||
|
||||
## Building from Source
|
||||
|
||||
@@ -37,6 +37,24 @@ go build -o bin/logwisp ./cmd/logwisp
|
||||
`go install github.com/lixenwraith/logwisp/cmd/logwisp@latest` also works, with
|
||||
the same loss of version metadata.
|
||||
|
||||
## Container Image
|
||||
|
||||
The root `Dockerfile` builds the same package into `scratch` under UID 65532,
|
||||
static and stripped. There is no shell and no config in the image: mount one and
|
||||
name it, as the binary has no daemon mode and no built-in defaults worth running.
|
||||
|
||||
```bash
|
||||
REV=$(git rev-parse HEAD)
|
||||
docker build -t "logwisp:$(git rev-parse --short HEAD)" \
|
||||
--build-arg VERSION="$(git describe --tags --always)" \
|
||||
--build-arg REVISION="$REV" .
|
||||
docker run --rm -v /etc/logwisp:/etc/logwisp:ro logwisp:... -c /etc/logwisp/logwisp.toml
|
||||
```
|
||||
|
||||
Sinks that listen (`http`, `tcp`) need their ports published; the read-only
|
||||
root filesystem and dropped capabilities a restricted runtime imposes are all
|
||||
compatible with it, provided a `file` sink's directory is writable by 65532.
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy the annotated reference configuration and edit it:
|
||||
@@ -173,27 +191,20 @@ mode; see [Operations](operations.md#checking-a-configuration).
|
||||
|
||||
## Test Scripts
|
||||
|
||||
Two end-to-end scripts under `test/` build multi-node chain topologies against a
|
||||
local build:
|
||||
End-to-end scripts under `test/` run against a local build:
|
||||
|
||||
```bash
|
||||
make
|
||||
./test/chain-test.sh --auto # two independent relay pipelines
|
||||
./test/chain-aggregate-test.sh --auto # fan-in: both edges into one pipeline
|
||||
./test/mtls-chain-test.sh --auto # the same fan-in under mTLS
|
||||
./test/passthrough-test.sh # file source relays a wide envelope intact
|
||||
```
|
||||
|
||||
Without `--auto` they run the relay in the foreground for interactive
|
||||
inspection. They need bash 5+, coreutils, and curl, and they bind ports
|
||||
15801–15804. Generated configuration and logs land in `test/run/`.
|
||||
|
||||
> Two of the three `--auto` assertions currently report `FAIL` against a
|
||||
> working build. They grep the sink output for `"source":"edge-tcp/` and
|
||||
> `"node":"edge-http"`, but the JSON formatter emits the `node/source` label
|
||||
> under the key `trace`. The transport itself is healthy — the
|
||||
> `total_processed` assertion passes and the streamed entries carry
|
||||
> `"trace":"edge-tcp/random_rand"` as expected. Until the assertions are
|
||||
> updated, verify the streams by eye with `nc 127.0.0.1 15803` and
|
||||
> `curl -sN http://127.0.0.1:15804/stream`.
|
||||
Without `--auto` the chain scripts run the relay in the foreground for
|
||||
interactive inspection. They need bash 5+, coreutils, and curl, and they bind
|
||||
ports 15801–15804. The pass-through test binds nothing. Generated configuration
|
||||
and logs land in `test/run/`.
|
||||
|
||||
## Uninstall
|
||||
|
||||
|
||||
+2
-1
@@ -231,7 +231,8 @@ pipeline `total_dropped_by_sink` (sink backed up?), sink `total_processed`.
|
||||
|
||||
- The watcher seeks to end-of-file on start; only content appended afterwards is
|
||||
read. Positions are in memory, so a restart re-seeks to end and anything
|
||||
written during the downtime is lost.
|
||||
written during the downtime is lost. `from = "start"` reads each file whole
|
||||
instead, and replays it on every restart.
|
||||
- `pattern` is a filename glob with `*` and `?` only, and matching is not
|
||||
recursive.
|
||||
- `check_interval_ms` governs how quickly a *new file* is noticed; tailing an
|
||||
|
||||
+17
-5
@@ -32,6 +32,8 @@ type = "file"
|
||||
directory = "/var/log/myapp"
|
||||
pattern = "*.log"
|
||||
check_interval_ms = 100
|
||||
raw = false
|
||||
from = "end"
|
||||
```
|
||||
|
||||
| Option | Type | Default | Description |
|
||||
@@ -39,6 +41,8 @@ check_interval_ms = 100
|
||||
| `directory` | string | **required** | Directory to scan; not recursive |
|
||||
| `pattern` | string | `*` | Glob over filenames; `*` and `?` only |
|
||||
| `check_interval_ms` | int | `100` | Directory rescan interval; minimum `10` |
|
||||
| `raw` | bool | `false` | Never parse a line: the whole line is the message |
|
||||
| `from` | string | `end` | Where a new watcher starts: `end` or `start` of the file |
|
||||
|
||||
**Behaviour**
|
||||
|
||||
@@ -49,15 +53,23 @@ check_interval_ms = 100
|
||||
stopped and removed on the next scan.
|
||||
- A new watcher seeks to end-of-file. Positions live in memory only, so a
|
||||
restart resumes from the current end of each file and content written while
|
||||
LogWisp was down is not read.
|
||||
LogWisp was down is not read. `from = "start"` reads each file whole when its
|
||||
watcher is created instead — what a process writing beside LogWisp needs, at
|
||||
the cost of replaying a file already on disk at every restart.
|
||||
- Rotation is detected from size decrease, modification-time reset, a position
|
||||
beyond end-of-file, or an inode change. An inode change where the new file is
|
||||
already larger than the recorded position is treated as an atomic save, not a
|
||||
rotation, and the position is preserved.
|
||||
- Lines are parsed as JSON when they contain `time`, `level`, `msg`, and
|
||||
`fields` keys; `time` is read as RFC3339Nano. Anything else is kept as plain
|
||||
text with the level inferred from common markers (`[ERROR]`, `WARN:`, and so
|
||||
on).
|
||||
- A line is parsed as JSON only when it is an object whose top-level keys are
|
||||
all drawn from `time`, `level`, `msg` and `fields` — the four an entry can
|
||||
carry. `time` is read as RFC3339Nano. Any other key, and any non-object line,
|
||||
is kept whole as text with the level inferred from common markers
|
||||
(`[ERROR]`, `WARN:`, and so on), because parsing it would drop the rest.
|
||||
- `raw = true` skips the JSON branch entirely. The line, plus its newline,
|
||||
becomes the message; `fields` stays empty, the time is the read time, and the
|
||||
level is inferred from the text as for any unparsed line. Paired with
|
||||
`format.type = "raw"` this is byte-exact transport for records LogWisp's
|
||||
envelope cannot hold — see [Formatters](formatters.md#raw).
|
||||
- `Source` is set to the file's base name.
|
||||
|
||||
**Statistics**: per-watcher size, position, entries read, rotation count, and
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module logwisp
|
||||
|
||||
go 1.26.5
|
||||
go 1.27.1
|
||||
|
||||
require (
|
||||
github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98
|
||||
|
||||
@@ -198,6 +198,8 @@ type FileSourceOptions struct {
|
||||
Directory string `toml:"directory"`
|
||||
Pattern string `toml:"pattern"` // glob pattern
|
||||
CheckIntervalMS int64 `toml:"check_interval_ms"`
|
||||
Raw bool `toml:"raw"` // keep the whole line as the message, never parse it
|
||||
From string `toml:"from"` // "end" (default) or "start" of a newly discovered file
|
||||
}
|
||||
|
||||
// ConsoleSourceOptions defines settings for a stdin-based source
|
||||
|
||||
+30
-43
@@ -90,57 +90,44 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
|
||||
|
||||
// Format implements Formatter interface
|
||||
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
|
||||
// Map logwisp LogEntry to formatter args
|
||||
level := mapLevel(entry.Level)
|
||||
// syslog-style origin prefix for chained entries
|
||||
src := sourceLabel(entry)
|
||||
|
||||
// Build args based on whether we have structured fields
|
||||
var args []any
|
||||
effectiveFlags := a.flags
|
||||
|
||||
if len(entry.Fields) > 0 {
|
||||
// Parse fields JSON
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
|
||||
// Use structured JSON format for fields
|
||||
args = []any{entry.Message, fields}
|
||||
// Add structured flag to properly format fields as JSON object
|
||||
effectiveFlags |= formatter.FlagStructuredJSON
|
||||
return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil
|
||||
}
|
||||
}
|
||||
if args == nil {
|
||||
args = []any{entry.Message}
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
out := bytes.Clone(a.formatter.Format(effectiveFlags, entry.Time, level, src, args))
|
||||
a.mu.Unlock()
|
||||
return out, nil
|
||||
return a.serialize(entry, a.flags), nil
|
||||
}
|
||||
|
||||
// FormatWithFlags allows custom flags for specific formatting needs
|
||||
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
|
||||
level := mapLevel(entry.Level)
|
||||
src := sourceLabel(entry)
|
||||
return a.serialize(entry, customFlags), nil
|
||||
}
|
||||
|
||||
var args []any
|
||||
if len(entry.Fields) > 0 {
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
|
||||
args = []any{entry.Message, fields}
|
||||
customFlags |= formatter.FlagStructuredJSON
|
||||
}
|
||||
}
|
||||
if args == nil {
|
||||
args = []any{entry.Message}
|
||||
}
|
||||
// serialize renders an entry under the given flags. The returned slice is a
|
||||
// copy: the underlying formatter reuses one buffer and sinks retain payloads.
|
||||
func (a *FormatterAdapter) serialize(entry core.LogEntry, flags int64) []byte {
|
||||
args, flags := formatArgs(entry, flags)
|
||||
|
||||
a.mu.Lock()
|
||||
out := bytes.Clone(a.formatter.Format(customFlags, entry.Time, level, src, args))
|
||||
out := bytes.Clone(a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args))
|
||||
a.mu.Unlock()
|
||||
return out, nil
|
||||
return out
|
||||
}
|
||||
|
||||
// formatArgs pairs the entry with its flags. FlagRaw keeps the fields JSON
|
||||
// verbatim beside the message rather than silently overriding the caller's
|
||||
// choice of passthrough; every other mode renders it as a JSON object.
|
||||
func formatArgs(entry core.LogEntry, flags int64) ([]any, int64) {
|
||||
if len(entry.Fields) == 0 {
|
||||
return []any{entry.Message}, flags
|
||||
}
|
||||
if flags&formatter.FlagRaw != 0 {
|
||||
if entry.Message == "" {
|
||||
return []any{[]byte(entry.Fields)}, flags
|
||||
}
|
||||
return []any{entry.Message, []byte(entry.Fields)}, flags
|
||||
}
|
||||
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(entry.Fields, &fields); err != nil || len(fields) == 0 {
|
||||
return []any{entry.Message}, flags
|
||||
}
|
||||
return []any{entry.Message, fields}, flags | formatter.FlagStructuredJSON
|
||||
}
|
||||
|
||||
// Name returns formatter type
|
||||
|
||||
@@ -61,6 +61,7 @@ const (
|
||||
DefaultFileSourcePattern = "*"
|
||||
DefaultFileSourceCheckIntervalMS = 100
|
||||
MinFileSourceCheckIntervalMS = 10
|
||||
DefaultFileSourceFrom = "end"
|
||||
)
|
||||
|
||||
// NewFileSourcePlugin creates a file source through plugin factory
|
||||
@@ -90,6 +91,11 @@ func NewFileSourcePlugin(
|
||||
} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
|
||||
return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
|
||||
}
|
||||
if opts.From == "" {
|
||||
opts.From = DefaultFileSourceFrom
|
||||
} else if err := lconfig.OneOf("start", "end")(opts.From); err != nil {
|
||||
return nil, fmt.Errorf("from: %w", err)
|
||||
}
|
||||
|
||||
// Create and return plugin instance
|
||||
fs := &FileSource{
|
||||
@@ -116,7 +122,9 @@ func NewFileSourcePlugin(
|
||||
"component", "file_source",
|
||||
"instance_id", id,
|
||||
"directory", opts.Directory,
|
||||
"pattern", opts.Pattern)
|
||||
"pattern", opts.Pattern,
|
||||
"raw", opts.Raw,
|
||||
"from", opts.From)
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
@@ -261,7 +269,7 @@ func (fs *FileSource) ensureWatcher(path string) {
|
||||
return
|
||||
}
|
||||
|
||||
w := newFileWatcher(path, fs.publish, fs.logger)
|
||||
w := newFileWatcher(path, fs.config.Raw, fs.config.From == "start", fs.publish, fs.logger)
|
||||
fs.watchers[path] = w
|
||||
|
||||
fs.logger.Debug("msg", "Created file watcher",
|
||||
|
||||
@@ -34,6 +34,7 @@ type WatcherInfo struct {
|
||||
type fileWatcher struct {
|
||||
directory string
|
||||
callback func(core.LogEntry)
|
||||
raw bool
|
||||
position int64
|
||||
size int64
|
||||
inode uint64
|
||||
@@ -46,12 +47,18 @@ type fileWatcher struct {
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// newFileWatcher creates a new watcher for a specific file path
|
||||
func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
|
||||
// newFileWatcher creates a new watcher for a specific file path.
|
||||
// A start position of 0 reads an existing file whole; -1 seeks to its end.
|
||||
func newFileWatcher(directory string, raw, fromStart bool, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
|
||||
position := int64(-1)
|
||||
if fromStart {
|
||||
position = 0
|
||||
}
|
||||
w := &fileWatcher{
|
||||
directory: directory,
|
||||
callback: callback,
|
||||
position: -1,
|
||||
raw: raw,
|
||||
position: position,
|
||||
logger: logger,
|
||||
}
|
||||
w.lastReadTime.Store(time.Time{})
|
||||
@@ -60,8 +67,8 @@ func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.
|
||||
|
||||
// watch starts the main monitoring loop for the file
|
||||
func (w *fileWatcher) watch(ctx context.Context) error {
|
||||
if err := w.seekToEnd(); err != nil {
|
||||
return fmt.Errorf("seekToEnd failed: %w", err)
|
||||
if err := w.initPosition(); err != nil {
|
||||
return fmt.Errorf("initPosition failed: %w", err)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(core.FileWatcherPollInterval)
|
||||
@@ -297,8 +304,9 @@ func (w *fileWatcher) checkFile() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// seekToEnd sets the initial read position to the end of the file
|
||||
func (w *fileWatcher) seekToEnd() error {
|
||||
// initPosition records the file's metadata and, unless the watcher was created
|
||||
// to read from the start, sets the initial read position to the end
|
||||
func (w *fileWatcher) initPosition() error {
|
||||
file, err := os.Open(w.directory)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
@@ -322,8 +330,6 @@ func (w *fileWatcher) seekToEnd() error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
// Keep existing position (including 0)
|
||||
// First time initialization seeks to the end of the file
|
||||
if w.position == -1 {
|
||||
pos, err := file.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
@@ -348,36 +354,67 @@ func (w *fileWatcher) isStopped() bool {
|
||||
return w.stopped
|
||||
}
|
||||
|
||||
// parseLine attempts to parse a line as JSON, falling back to plain text
|
||||
// parseLine converts a line into an entry, as JSON when nothing would be lost
|
||||
func (w *fileWatcher) parseLine(line string) core.LogEntry {
|
||||
var jsonLog struct {
|
||||
Time string `json:"time"`
|
||||
Level string `json:"level"`
|
||||
Message string `json:"msg"`
|
||||
Fields json.RawMessage `json:"fields"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(line), &jsonLog); err == nil {
|
||||
timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time)
|
||||
if err != nil {
|
||||
timestamp = time.Now()
|
||||
}
|
||||
|
||||
if w.raw {
|
||||
// Newline restored: sinks write the payload as it stands
|
||||
return core.LogEntry{
|
||||
Time: timestamp,
|
||||
Time: time.Now(),
|
||||
Source: filepath.Base(w.directory),
|
||||
Level: jsonLog.Level,
|
||||
Message: jsonLog.Message,
|
||||
Fields: jsonLog.Fields,
|
||||
Level: source.ExtractLogLevel(line),
|
||||
Message: line + "\n",
|
||||
}
|
||||
}
|
||||
|
||||
level := source.ExtractLogLevel(line)
|
||||
if entry, ok := w.parseJSON(line); ok {
|
||||
return entry
|
||||
}
|
||||
|
||||
return core.LogEntry{
|
||||
Time: time.Now(),
|
||||
Source: filepath.Base(w.directory),
|
||||
Level: level,
|
||||
Level: source.ExtractLogLevel(line),
|
||||
Message: line,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseJSON decodes a line into the entry envelope. A top-level key LogEntry
|
||||
// cannot carry refuses the whole line, so a richer record reaches the pipeline
|
||||
// as text rather than silently reduced to the four keys kept here.
|
||||
func (w *fileWatcher) parseJSON(line string) (core.LogEntry, bool) {
|
||||
if len(line) == 0 || line[0] != '{' {
|
||||
return core.LogEntry{}, false
|
||||
}
|
||||
|
||||
var obj map[string]json.RawMessage
|
||||
if err := json.Unmarshal([]byte(line), &obj); err != nil || len(obj) == 0 {
|
||||
return core.LogEntry{}, false
|
||||
}
|
||||
|
||||
entry := core.LogEntry{Time: time.Now(), Source: filepath.Base(w.directory)}
|
||||
for key, val := range obj {
|
||||
var err error
|
||||
switch key {
|
||||
case "time":
|
||||
var ts string
|
||||
if json.Unmarshal(val, &ts) == nil {
|
||||
if t, terr := time.Parse(time.RFC3339Nano, ts); terr == nil {
|
||||
entry.Time = t
|
||||
}
|
||||
}
|
||||
case "level":
|
||||
err = json.Unmarshal(val, &entry.Level)
|
||||
case "msg":
|
||||
err = json.Unmarshal(val, &entry.Message)
|
||||
case "fields":
|
||||
entry.Fields = val
|
||||
default:
|
||||
return core.LogEntry{}, false
|
||||
}
|
||||
if err != nil {
|
||||
return core.LogEntry{}, false
|
||||
}
|
||||
}
|
||||
|
||||
return entry, true
|
||||
}
|
||||
|
||||
Executable
+92
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env bash
|
||||
# logwisp file source pass-through test
|
||||
#
|
||||
# file src (raw, from=start) --> raw format --> file sink byte-exact relay
|
||||
# file src (defaults) --> raw format --> file sink no key dropped
|
||||
#
|
||||
# The fixture is a record whose envelope is wider than time/level/msg/fields,
|
||||
# which is what the narrow JSON branch used to reduce to an empty message.
|
||||
#
|
||||
# Usage: ./passthrough-test.sh
|
||||
# Requires: bash 5+, coreutils (timeout). Linux dev host only.
|
||||
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
|
||||
RUN="$SCRIPT_DIR/run/passthrough"
|
||||
|
||||
[[ -x $BIN ]] || { echo "no binary at $BIN; run make build" >&2; exit 1; }
|
||||
|
||||
rm -rf "$RUN"
|
||||
mkdir -p "$RUN/in" "$RUN/out-raw" "$RUN/out-parsed"
|
||||
|
||||
cat > "$RUN/in/wide.jsonl" <<'EOF'
|
||||
{"time":"2026-09-08T21:35:44.178372768-04:00","level":"INFO","sub":"app","run":0,"tick":0,"frame":0,"fields":{"msg":"init begin","mode":"play"}}
|
||||
{"time":"2026-09-08T21:35:44.178581366-04:00","level":"PROC","run":0,"tick":0,"frame":0,"fields":{"seq":1,"seed":1788917744178374836}}
|
||||
plain text line, not JSON at all
|
||||
EOF
|
||||
|
||||
conf() { # sink_dir raw
|
||||
cat <<EOF
|
||||
quiet = true
|
||||
status_reporter = false
|
||||
[logging]
|
||||
output = "stderr"
|
||||
level = "error"
|
||||
[[pipelines]]
|
||||
name = "passthrough"
|
||||
[pipelines.flow.format]
|
||||
type = "raw"
|
||||
sanitizer_policy = "raw"
|
||||
[[pipelines.plugin_sources]]
|
||||
id = "wide"
|
||||
type = "file"
|
||||
[pipelines.plugin_sources.config]
|
||||
directory = "$RUN/in"
|
||||
pattern = "*.jsonl"
|
||||
raw = $2
|
||||
from = "start"
|
||||
[[pipelines.plugin_sinks]]
|
||||
id = "out"
|
||||
type = "file"
|
||||
[pipelines.plugin_sinks.config]
|
||||
directory = "$1"
|
||||
name = "relay"
|
||||
flush_interval_ms = 100
|
||||
EOF
|
||||
}
|
||||
|
||||
conf "$RUN/out-raw" true > "$RUN/raw.toml"
|
||||
conf "$RUN/out-parsed" false > "$RUN/parsed.toml"
|
||||
|
||||
for c in raw parsed; do
|
||||
timeout 5 "$BIN" -c "$RUN/$c.toml" > "$RUN/$c.out" 2>&1
|
||||
done
|
||||
|
||||
fail=0
|
||||
check() { # label condition_result
|
||||
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
|
||||
}
|
||||
|
||||
# 1. raw = true relays the file byte for byte
|
||||
if diff -q "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" > /dev/null; then
|
||||
check "raw = true: output identical to input" 1
|
||||
else
|
||||
check "raw = true: output identical to input" 0
|
||||
diff "$RUN/in/wide.jsonl" "$RUN/out-raw/relay.log" | head -6
|
||||
fi
|
||||
|
||||
# 2. the default parse keeps every key, JSON branch refused on the wide envelope
|
||||
n=$(grep -c '"sub":"app"' "$RUN/out-parsed/relay.log")
|
||||
check "defaults: wide envelope reaches the sink whole ($n line(s) carry sub)" $(( n == 1 ))
|
||||
n=$(grep -c '1788917744178374836' "$RUN/out-parsed/relay.log")
|
||||
check "defaults: large integers are not re-encoded through float64 ($n)" $(( n == 1 ))
|
||||
|
||||
echo "================================================================"
|
||||
if (( fail == 0 )); then
|
||||
echo "RESULT: ALL PASS"
|
||||
else
|
||||
echo "RESULT: FAILURES — inspect $RUN/"
|
||||
fi
|
||||
exit "$fail"
|
||||
Reference in New Issue
Block a user