v0.33.0 tui tap for live view and console enhancements
This commit is contained in:
lixen committed
1 parent
a1e9d21d06
commit
87f5576ccf
41 files changed
+4606
-449
No files matched your search
@@ -0,0 +1,408 @@
|
||||
// Package logline reads a log line of any shape (vif records, LogWisp's json,
|
||||
// lw's own log, pino, zap, slog, docker, journald, ECS, GCP, plain text) into
|
||||
// one set of roles, and names the runes a terminal would act on. It links no
|
||||
// terminal code, so a viewer, a sink and the configuration page can share it.
|
||||
package logline
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/hex"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
)
|
||||
|
||||
// Line is what a log line says, wherever its writer put it
|
||||
type Line struct {
|
||||
Time time.Time // zero when the line names none readable: the entry's time stands in
|
||||
Stamp string // the time as the line wrote it
|
||||
Level string // a core.Levels level, or a value outside their order as written (PROC)
|
||||
Tag string // who wrote it: sub, source, logger and the like, else the entry's source
|
||||
Message string
|
||||
Fields []Field // what no role took, in the writer's order
|
||||
Layout string // where the roles were found, so a reader can check the guess
|
||||
}
|
||||
|
||||
// Field is a value no role took. Key is its dotted path, "" for a positional
|
||||
// value; Value is a string's text or any other value as its JSON.
|
||||
type Field struct {
|
||||
Key, Value string
|
||||
}
|
||||
|
||||
const (
|
||||
roleTime = iota
|
||||
roleLevel
|
||||
roleTag
|
||||
roleMsg
|
||||
)
|
||||
|
||||
// roles are the keys each role is read from, the first present wins: a dotted
|
||||
// key is a path into nested objects or a flat key spelled with dots. trace is
|
||||
// the last tag, so it names LogWisp json's source but not vif's call chain.
|
||||
var roles = [...][]string{
|
||||
roleTime: {"time", "ts", "timestamp", "@timestamp", "t", "datetime", "__REALTIME_TIMESTAMP", "@t"},
|
||||
roleLevel: {"level", "lvl", "severity", "log.level", "levelname", "loglevel", "PRIORITY", "@l"},
|
||||
roleTag: {"sub", "source", "logger", "component", "name", "_SYSTEMD_UNIT", "trace"},
|
||||
roleMsg: {"msg", "message", "MESSAGE", "@m", "@mt", "log", "text", "event"},
|
||||
}
|
||||
|
||||
// envelopes hold a record's payload one level down; the first of them present
|
||||
// is opened, and without a message of its own the record takes the envelope's
|
||||
// first discriminator, or an array's first string (vif: fields.msg|type|ev)
|
||||
var (
|
||||
envelopes = []string{"fields", "data", "attrs", "payload"}
|
||||
discriminators = []string{"msg", "type", "ev", "message"}
|
||||
)
|
||||
|
||||
// nestMax bounds the depth flattened into dotted paths, and so their length
|
||||
const nestMax = 8
|
||||
|
||||
// member is one leaf of a record; no role takes the envelope's
|
||||
type member struct {
|
||||
key, val string
|
||||
kind byte
|
||||
env bool
|
||||
used bool
|
||||
}
|
||||
|
||||
// Read finds e's roles in its message, a JSON record (also one inside such a
|
||||
// record's message) or text with a leading RFC3339 stamp, and in its fields;
|
||||
// a role the line does not name is the entry's.
|
||||
func Read(e core.LogEntry) Line {
|
||||
l, at, ok := record(strings.TrimSpace(e.Message), "")
|
||||
if ok {
|
||||
if in, _, ok := record(strings.TrimSpace(l.Message), at); ok {
|
||||
l = merge(l, in)
|
||||
}
|
||||
} else {
|
||||
l = Line{Message: e.Message, Layout: "text"}
|
||||
if stamp, rest, ok := strings.Cut(e.Message, " "); ok {
|
||||
if at, ok := readStamp(stamp); ok {
|
||||
l.Time, l.Stamp, l.Message, l.Layout = at, stamp, strings.TrimLeft(rest, " "), "text, stamped"
|
||||
}
|
||||
}
|
||||
}
|
||||
if f := strings.TrimSpace(string(e.Fields)); f != "" {
|
||||
_, kind, _ := scanValue(f, 0)
|
||||
ms, msg, at := envelope(nil, "fields", f, kind)
|
||||
if l.Message == "" && msg >= 0 {
|
||||
ms[msg].used = true
|
||||
l.Message, l.Layout = unquote(ms[msg].val, kStr), "entry, message in "+at
|
||||
}
|
||||
l.Fields = appendFields(l.Fields, ms)
|
||||
}
|
||||
if l.Level == "" {
|
||||
l.Level = level("", e.Level)
|
||||
}
|
||||
if l.Tag == "" {
|
||||
l.Tag = e.Source
|
||||
}
|
||||
l.Message = strings.TrimRight(l.Message, "\r\n")
|
||||
return l
|
||||
}
|
||||
|
||||
// record reads s as a JSON object; in names where s sat as another record's
|
||||
// message, at where s's own message is. The members before a malformed part
|
||||
// count, so a clipped line still shows its roles; ok is false without any.
|
||||
func record(s, in string) (l Line, at string, ok bool) {
|
||||
var top []member
|
||||
whole := eachField(s, 0, func(k, v string, kind byte) bool {
|
||||
top = append(top, member{key: k, val: v, kind: kind})
|
||||
return true
|
||||
})
|
||||
if len(top) == 0 {
|
||||
return Line{}, "", false
|
||||
}
|
||||
env := -1
|
||||
for _, name := range envelopes {
|
||||
for i := range top {
|
||||
if env < 0 && top[i].key == name && (top[i].kind == kObj || top[i].kind == kArr) {
|
||||
env = i
|
||||
}
|
||||
}
|
||||
}
|
||||
var ms []member
|
||||
msg, envAt := -1, ""
|
||||
for i, m := range top {
|
||||
if i == env {
|
||||
ms, msg, envAt = envelope(ms, m.key, m.val, m.kind)
|
||||
} else {
|
||||
ms = leaves(ms, m.key, m.val, m.kind, false, 0)
|
||||
}
|
||||
}
|
||||
|
||||
if m := find(ms, roleTime); m != nil {
|
||||
l.Stamp = unquote(m.val, m.kind)
|
||||
l.Time, _ = readTime(l.Stamp)
|
||||
}
|
||||
if m := find(ms, roleLevel); m != nil {
|
||||
l.Level = level(m.key, unquote(m.val, m.kind))
|
||||
}
|
||||
if m := find(ms, roleTag); m != nil {
|
||||
l.Tag = unquote(m.val, m.kind)
|
||||
}
|
||||
if m := find(ms, roleMsg); m != nil {
|
||||
l.Message, at = unquote(m.val, m.kind), m.key
|
||||
} else if msg >= 0 {
|
||||
ms[msg].used = true
|
||||
l.Message, at = unquote(ms[msg].val, kStr), envAt
|
||||
}
|
||||
l.Fields = appendFields(nil, ms)
|
||||
|
||||
l.Layout = "json"
|
||||
if in != "" {
|
||||
l.Layout += " in " + in
|
||||
}
|
||||
if !whole {
|
||||
l.Layout += ", cut short"
|
||||
}
|
||||
if at != "" {
|
||||
l.Layout += ", message in " + at
|
||||
} else {
|
||||
l.Layout += ", no message"
|
||||
}
|
||||
return l, at, true
|
||||
}
|
||||
|
||||
// envelope appends the leaves of envelope name's value v; msg indexes its
|
||||
// message in ms and at says where it was, -1 and "" when it holds none
|
||||
func envelope(ms []member, name, v string, kind byte) (_ []member, msg int, at string) {
|
||||
msg, rank, slot := -1, len(discriminators), -1
|
||||
note := func(key string, kind byte, i int) {
|
||||
for r := 0; r < rank && kind == kStr; r++ {
|
||||
if discriminators[r] == key {
|
||||
msg, rank, slot = len(ms), r, i
|
||||
}
|
||||
}
|
||||
}
|
||||
switch kind {
|
||||
case kObj:
|
||||
eachField(v, 0, func(k, fv string, fk byte) bool {
|
||||
note(k, fk, -1)
|
||||
ms = leaves(ms, k, fv, fk, true, 1)
|
||||
return true
|
||||
})
|
||||
case kArr:
|
||||
// lw's own log writes key/value pairs; LogWisp json, the message alone
|
||||
n, pairs := 0, true
|
||||
eachItem(v, 0, func(_ string, k byte) {
|
||||
pairs = pairs && (n%2 == 1 || k == kStr)
|
||||
n++
|
||||
})
|
||||
pairs = pairs && n > 0 && n%2 == 0
|
||||
i, key := 0, ""
|
||||
eachItem(v, 0, func(iv string, ik byte) {
|
||||
switch {
|
||||
case pairs && i%2 == 0:
|
||||
key = unquote(iv, ik)
|
||||
case pairs:
|
||||
note(key, ik, i)
|
||||
ms = leaves(ms, key, iv, ik, true, 1)
|
||||
default:
|
||||
if i == 0 && ik == kStr {
|
||||
msg, slot = len(ms), 0
|
||||
}
|
||||
ms = leaves(ms, "", iv, ik, true, 1)
|
||||
}
|
||||
i++
|
||||
})
|
||||
default:
|
||||
ms = leaves(ms, name, v, kind, true, 0)
|
||||
}
|
||||
switch {
|
||||
case msg < 0:
|
||||
case slot < 0:
|
||||
at = name + "." + ms[msg].key
|
||||
default:
|
||||
at = name + "[" + strconv.Itoa(slot) + "]"
|
||||
}
|
||||
return ms, msg, at
|
||||
}
|
||||
|
||||
// leaves appends v under key, an object as one leaf per member at its dotted
|
||||
// path, so roles and readers see nested keys as if written flat
|
||||
func leaves(ms []member, key, v string, kind byte, env bool, depth int) []member {
|
||||
if kind == kObj && depth < nestMax {
|
||||
n := len(ms)
|
||||
if eachField(v, 0, func(k, fv string, fk byte) bool {
|
||||
if key != "" {
|
||||
k = key + "." + k
|
||||
}
|
||||
ms = leaves(ms, k, fv, fk, env, depth+1)
|
||||
return true
|
||||
}) && len(ms) > n {
|
||||
return ms
|
||||
}
|
||||
ms = ms[:n]
|
||||
}
|
||||
return append(ms, member{key: key, val: v, kind: kind, env: env})
|
||||
}
|
||||
|
||||
// find takes the leaf holding role r, outside the envelope and holding a
|
||||
// string or a number: a null, a flag or a list names no role
|
||||
func find(ms []member, r int) *member {
|
||||
for _, key := range roles[r] {
|
||||
for i := range ms {
|
||||
m := &ms[i]
|
||||
if m.key == key && !m.env && !m.used && (m.kind == kStr || m.kind == kNum) {
|
||||
m.used = true
|
||||
return m
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendFields(fs []Field, ms []member) []Field {
|
||||
for _, m := range ms {
|
||||
if !m.used {
|
||||
fs = append(fs, Field{m.key, unquote(m.val, m.kind)})
|
||||
}
|
||||
}
|
||||
return fs
|
||||
}
|
||||
|
||||
// merge lays a record found in out's message over out: its roles win, out's
|
||||
// fill the ones it does not name, and out's fields come first
|
||||
func merge(out, in Line) Line {
|
||||
if in.Time.IsZero() {
|
||||
in.Time, in.Stamp = out.Time, out.Stamp
|
||||
}
|
||||
if in.Level == "" {
|
||||
in.Level = out.Level
|
||||
}
|
||||
if in.Tag == "" {
|
||||
in.Tag = out.Tag
|
||||
}
|
||||
in.Fields = append(out.Fields, in.Fields...)
|
||||
return in
|
||||
}
|
||||
|
||||
// level reads a level value: a name in any case, or a number on its key's
|
||||
// scale (journald's syslog PRIORITY, pino's tens, else slog's and
|
||||
// lixenwraith/log's); any other value is kept as written, outside the order
|
||||
func level(key, v string) string {
|
||||
if n, ok := parseInt64(v); ok {
|
||||
switch {
|
||||
case key == "PRIORITY":
|
||||
return [...]string{"ERROR", "ERROR", "ERROR", "ERROR", "WARN", "INFO", "INFO", "DEBUG"}[min(max(n, 0), 7)]
|
||||
case n >= 10:
|
||||
return [...]string{"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "ERROR"}[min(n/10-1, 5)]
|
||||
case n <= -8:
|
||||
return "TRACE"
|
||||
case n < 0:
|
||||
return "DEBUG"
|
||||
case n < 4:
|
||||
return "INFO"
|
||||
case n < 8:
|
||||
return "WARN"
|
||||
}
|
||||
return "ERROR"
|
||||
}
|
||||
for _, l := range core.Levels {
|
||||
for _, names := range [2][]string{l.Names, l.Keyed} {
|
||||
for _, name := range names {
|
||||
if strings.EqualFold(v, name) {
|
||||
return l.Level
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// readStamp reads an RFC3339 stamp, one without a zone as UTC
|
||||
func readStamp(s string) (time.Time, bool) {
|
||||
at, err := time.Parse(time.RFC3339Nano, s)
|
||||
if err != nil {
|
||||
at, err = time.Parse("2006-01-02T15:04:05.999999999", s)
|
||||
}
|
||||
return at, err == nil
|
||||
}
|
||||
|
||||
// readTime reads an RFC3339 stamp, or a number since 1970 in seconds,
|
||||
// milliseconds or microseconds, told apart by its magnitude
|
||||
func readTime(s string) (time.Time, bool) {
|
||||
if at, ok := readStamp(s); ok {
|
||||
return at, true
|
||||
}
|
||||
whole, frac, _ := strings.Cut(s, ".")
|
||||
n, ok := parseInt64(whole)
|
||||
if !ok || n < 0 {
|
||||
return time.Time{}, false
|
||||
}
|
||||
unit := int64(1e3)
|
||||
switch {
|
||||
case n < 1e11:
|
||||
unit = 1e9
|
||||
case n < 1e14:
|
||||
unit = 1e6
|
||||
}
|
||||
if n > math.MaxInt64/unit {
|
||||
return time.Time{}, false
|
||||
}
|
||||
ns := n * unit
|
||||
for i := 0; i < len(frac); i++ {
|
||||
if frac[i] < '0' || frac[i] > '9' {
|
||||
return time.Time{}, false
|
||||
}
|
||||
unit /= 10
|
||||
ns += int64(frac[i]-'0') * unit
|
||||
}
|
||||
return time.Unix(0, ns), true
|
||||
}
|
||||
|
||||
// EscapeControls writes what a terminal would act on or reorder (Control) as
|
||||
// <hex>, keeping the final newline: a log line cannot drive the terminal.
|
||||
// Other invisible characters stay: emoji and Indic scripts need ZWJ.
|
||||
func EscapeControls(p []byte) []byte {
|
||||
body, newline := bytes.CutSuffix(p, []byte{'\n'})
|
||||
if printableASCII(body) {
|
||||
return p
|
||||
}
|
||||
out := make([]byte, 0, len(p)+16)
|
||||
for len(body) > 0 {
|
||||
r, n := utf8.DecodeRune(body)
|
||||
if !Control(r, n) {
|
||||
out = append(out, body[:n]...)
|
||||
} else {
|
||||
out = append(out, '<')
|
||||
out = hex.AppendEncode(out, body[:n])
|
||||
out = append(out, '>')
|
||||
}
|
||||
body = body[n:]
|
||||
}
|
||||
if newline {
|
||||
out = append(out, '\n')
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Control reports whether a terminal would act on the rune r, decoded from
|
||||
// size bytes, or reorder text by it: C0 and C1 controls but tab, DEL, bidi
|
||||
// controls, line separators and invalid UTF-8
|
||||
func Control(r rune, size int) bool {
|
||||
switch {
|
||||
case r == utf8.RuneError && size == 1:
|
||||
return true
|
||||
case r < 0x20:
|
||||
return r != '\t'
|
||||
case r >= 0x7f && r <= 0x9f, r == 0x061c, r == 0x200e, r == 0x200f,
|
||||
r >= 0x2028 && r <= 0x202e, r >= 0x2066 && r <= 0x2069:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printableASCII(b []byte) bool {
|
||||
for _, c := range b {
|
||||
if (c < 0x20 || c > 0x7e) && c != '\t' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package logline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
)
|
||||
|
||||
// arrived is a line as a source passes one it does not parse: whole, at its
|
||||
// arrival, with the level its first level word names
|
||||
func arrived(line string) core.LogEntry {
|
||||
level, _, _ := core.LevelWord(line, "")
|
||||
return core.LogEntry{Time: time.Unix(1, 0), Source: "app.log", Level: level, Message: line}
|
||||
}
|
||||
|
||||
// Each writer's line yields its time, level, tag and message from where that
|
||||
// writer puts them, the rest as fields in its order; the want is "layout |
|
||||
// stamp=time | level | tag | message | fields". vif's, lw's and slog's lines
|
||||
// are their writers' output; the others follow their formats' documentation.
|
||||
func TestReadFindsTheRolesWhereTheirWriterPutsThem(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
in core.LogEntry
|
||||
want string
|
||||
}{
|
||||
// vif: the tag is sub, the message the first of fields.msg, type and ev
|
||||
{arrived(`{"time":"2026-10-10T19:09:54.251623265Z","level":"INFO","sub":"fsm","run":3,"tick":1200,"fields":{"msg":"transition","from":"idle","to":"run","elapsed":1500000}}`),
|
||||
`json, message in fields.msg | 2026-10-10T19:09:54.251623265Z=2026-10-10T19:09:54.251623265Z | INFO | fsm | "transition" | run="3" tick="1200" from="idle" to="run" elapsed="1500000"`},
|
||||
{arrived(`{"time":"2026-10-10T19:09:54.251635197Z","level":"WARN","sub":"fsm","run":3,"tick":1200,"trace":"main -> main","fields":{"msg":"slow tick","ms":73.25}}`),
|
||||
`json, message in fields.msg | 2026-10-10T19:09:54.251635197Z=2026-10-10T19:09:54.251635197Z | WARN | fsm | "slow tick" | run="3" tick="1200" trace="main -> main" ms="73.25"`},
|
||||
{arrived(`{"time":"2026-10-10T19:09:54.251635488Z","level":"INFO","sub":"journal","run":0,"tick":0,"fields":{"jseq":7,"domain":"player","ev":"EventX","payload":"x = 1\ny = 2\n"}}`),
|
||||
`json, message in fields.ev | 2026-10-10T19:09:54.251635488Z=2026-10-10T19:09:54.251635488Z | INFO | journal | "EventX" | run="0" tick="0" jseq="7" domain="player" payload="x = 1\ny = 2\n"`},
|
||||
{arrived(`{"time":"2026-08-29T21:15:23.33805528-04:00","level":"PROC","run":0,"tick":0,"frame":0,"fields":{"type":"proc","sequence":1,"processed_logs":0}}`),
|
||||
`json, message in fields.type | 2026-08-29T21:15:23.33805528-04:00=2026-08-30T01:15:23.33805528Z | PROC | app.log | "proc" | run="0" tick="0" frame="0" sequence="1" processed_logs="0"`},
|
||||
{arrived(`{"time":"2026-08-30T01:27:39.654615818Z","level":"INFO","sub":"anchor","run":0,"tick":0,"frame":0,"fields":{"schema":10,"jseq":0,"tick_ns":50000000}}`),
|
||||
`json, no message | 2026-08-30T01:27:39.654615818Z=2026-08-30T01:27:39.654615818Z | INFO | anchor | "" | run="0" tick="0" frame="0" schema="10" jseq="0" tick_ns="50000000"`},
|
||||
// LogWisp json: trace is the source, the message alone is fields[0], and a
|
||||
// record the console source read whole sits there, over the outer roles
|
||||
{arrived(`{"time":"2026-10-10T12:00:00Z","level":"WARN","trace":"app.log","message":"disk almost full","fields":{"disk":"/dev/sda1","nested":{"a":[1,2]},"pct":91}}`),
|
||||
`json, message in message | 2026-10-10T12:00:00Z=2026-10-10T12:00:00Z | WARN | app.log | "disk almost full" | disk="/dev/sda1" nested.a="[1,2]" pct="91"`},
|
||||
{arrived(`{"time":"2026-10-10T20:30:12.385839584Z","level":"INFO","trace":"app.log","fields":["TRACE deep detail"]}`),
|
||||
`json, message in fields[0] | 2026-10-10T20:30:12.385839584Z=2026-10-10T20:30:12.385839584Z | INFO | app.log | "TRACE deep detail" | `},
|
||||
{arrived(`{"time":"2026-10-10T20:43:03.350727184Z","level":"INFO","trace":"console","fields":["{\"time\":\"2026-10-10T12:00:00Z\",\"msg\":\"disk almost full\",\"fields\":{\"disk\":\"/dev/sda1\"}}"]}`),
|
||||
`json in fields[0], message in msg | 2026-10-10T12:00:00Z=2026-10-10T12:00:00Z | INFO | console | "disk almost full" | disk="/dev/sda1"`},
|
||||
{arrived(`{"time":"2026-10-10T20:43:38.342629146Z","level":"INFO","trace":"console","fields":["{\"msg\":\"no time here\",\"fields\":{\"k\":1}}"]}`),
|
||||
`json in fields[0], message in msg | 2026-10-10T20:43:38.342629146Z=2026-10-10T20:43:38.342629146Z | INFO | console | "no time here" | k="1"`},
|
||||
{arrived(`{"time":"2026-10-10T19:06:34.86389785Z","node":"edge-01","source":"app.log","level":"WARN","message":"disk almost full","fields":{"pct":91}}`),
|
||||
`json, message in message | 2026-10-10T19:06:34.86389785Z=2026-10-10T19:06:34.86389785Z | WARN | app.log | "disk almost full" | node="edge-01" pct="91"`},
|
||||
// lw's own log: key/value pairs, in the line or in what the file source kept
|
||||
{arrived(`{"time":"2026-10-10T20:29:33.279610845Z","level":"INFO","fields":["msg","Initializing pipeline","pipeline","cli"]}`),
|
||||
`json, message in fields[1] | 2026-10-10T20:29:33.279610845Z=2026-10-10T20:29:33.279610845Z | INFO | app.log | "Initializing pipeline" | pipeline="cli"`},
|
||||
{core.LogEntry{Source: "lw.log", Level: "INFO", Fields: json.RawMessage(`["msg","LogWisp starting","status_reporter",false]`)},
|
||||
`entry, message in fields[1] | =- | INFO | lw.log | "LogWisp starting" | status_reporter="false"`},
|
||||
{core.LogEntry{Source: "app.log", Level: "warning", Message: "disk almost full", Fields: json.RawMessage(`{"pct":91}`)},
|
||||
`text | =- | WARN | app.log | "disk almost full" | pct="91"`},
|
||||
// pino's tens, zap's seconds, slog's scale and groups
|
||||
{arrived(`{"level":30,"time":1728561600000,"pid":1,"hostname":"h","msg":"pino style"}`),
|
||||
`json, message in msg | 1728561600000=2024-10-10T12:00:00Z | INFO | app.log | "pino style" | pid="1" hostname="h"`},
|
||||
{arrived(`{"level":"info","ts":1728561600.123456,"caller":"main.go:12","msg":"zap style","n":1}`),
|
||||
`json, message in msg | 1728561600.123456=2024-10-10T12:00:00.123456Z | INFO | app.log | "zap style" | caller="main.go:12" n="1"`},
|
||||
{arrived(`{"time":"2026-10-10T20:30:23.451515887Z","level":"INFO","msg":"slog style","user":{"id":7,"name":"a"}}`),
|
||||
`json, message in msg | 2026-10-10T20:30:23.451515887Z=2026-10-10T20:30:23.451515887Z | INFO | app.log | "slog style" | user.id="7" user.name="a"`},
|
||||
{arrived(`{"time":"2026-10-10T20:39:59.809016667Z","level":-4,"msg":"slog level as a number","attempt":2}`),
|
||||
`json, message in msg | 2026-10-10T20:39:59.809016667Z=2026-10-10T20:39:59.809016667Z | DEBUG | app.log | "slog level as a number" | attempt="2"`},
|
||||
// docker's log ends in its newline; the level is the entry's
|
||||
{arrived(`{"log":"docker line ERROR x\n","stream":"stderr","time":"2026-10-10T12:00:00.1Z"}`),
|
||||
`json, message in log | 2026-10-10T12:00:00.1Z=2026-10-10T12:00:00.1Z | ERROR | app.log | "docker line ERROR x" | stream="stderr"`},
|
||||
// journald writes numbers as strings: microseconds, syslog priority
|
||||
{arrived(`{"__REALTIME_TIMESTAMP":"1728561600123456","PRIORITY":"3","_SYSTEMD_UNIT":"nginx.service","MESSAGE":"journald"}`),
|
||||
`json, message in MESSAGE | 1728561600123456=2024-10-10T12:00:00.123456Z | ERROR | nginx.service | "journald" | `},
|
||||
{arrived(`{"@timestamp":"2026-10-10T12:00:00Z","log.level":"warn","message":"ecs flat"}`),
|
||||
`json, message in message | 2026-10-10T12:00:00Z=2026-10-10T12:00:00Z | WARN | app.log | "ecs flat" | `},
|
||||
{arrived(`{"@timestamp":"2026-10-10T12:00:00Z","log":{"level":"error"},"message":"ecs nested"}`),
|
||||
`json, message in message | 2026-10-10T12:00:00Z=2026-10-10T12:00:00Z | ERROR | app.log | "ecs nested" | `},
|
||||
{arrived(`{"severity":"CRITICAL","timestamp":"2026-10-10T12:00:00Z","message":"gcp"}`),
|
||||
`json, message in message | 2026-10-10T12:00:00Z=2026-10-10T12:00:00Z | ERROR | app.log | "gcp" | `},
|
||||
{arrived(`{"a":1,"b":"no known keys","source":null}`),
|
||||
`json, no message | =- | | app.log | "" | a="1" b="no known keys" source="null"`},
|
||||
// a line the tap clipped keeps the roles before the cut
|
||||
{arrived(`{"time":"2026-10-10T12:00:00Z","level":"INFO","msg":"clipped","big":"aaaa`),
|
||||
`json, cut short, message in msg | 2026-10-10T12:00:00Z=2026-10-10T12:00:00Z | INFO | app.log | "clipped" | `},
|
||||
// nesting past nestMax stays one value, bounding the paths
|
||||
{arrived(`{"msg":"deep","a":{"b":{"c":{"d":{"e":{"f":{"g":{"h":{"i":{"j":1}}}}}}}}}}`),
|
||||
`json, message in msg | =- | | app.log | "deep" | a.b.c.d.e.f.g.h.i="{\"j\":1}"`},
|
||||
{arrived(`2026-10-10T12:00:00+02:00 ERROR db: connection refused`),
|
||||
`text, stamped | 2026-10-10T12:00:00+02:00=2026-10-10T10:00:00Z | ERROR | app.log | "ERROR db: connection refused" | `},
|
||||
{arrived(`2026-10-10T12:00:00.5 INFO a stamp without a zone is UTC`),
|
||||
`text, stamped | 2026-10-10T12:00:00.5=2026-10-10T12:00:00.5Z | INFO | app.log | "INFO a stamp without a zone is UTC" | `},
|
||||
// what only looks like a stamp is text: no day is invented
|
||||
{arrived(`2026-13-45T99:99:99Z ERROR x`),
|
||||
`text | =- | ERROR | app.log | "2026-13-45T99:99:99Z ERROR x" | `},
|
||||
{arrived(`2026-10-10T12:00:00Zjunk ERROR x`),
|
||||
`text | =- | ERROR | app.log | "2026-10-10T12:00:00Zjunk ERROR x" | `},
|
||||
{arrived(`{not json} WARNING disk`),
|
||||
`text | =- | WARN | app.log | "{not json} WARNING disk" | `},
|
||||
} {
|
||||
if got := show(Read(c.in)); got != c.want {
|
||||
t.Errorf("%s\n got: %s\nwant: %s", c.in.Message, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func show(l Line) string {
|
||||
at := "-"
|
||||
if !l.Time.IsZero() {
|
||||
at = l.Time.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
var fs []string
|
||||
for _, f := range l.Fields {
|
||||
fs = append(fs, fmt.Sprintf("%s=%q", f.Key, f.Value))
|
||||
}
|
||||
return fmt.Sprintf("%s | %s=%s | %s | %s | %q | %s", l.Layout, l.Stamp, at, l.Level, l.Tag, l.Message, strings.Join(fs, " "))
|
||||
}
|
||||
|
||||
// Everything a terminal would act on is escaped; tabs and the final newline,
|
||||
// which it only lays out, are kept.
|
||||
func TestEscapeControlsKeepsOnlyLayout(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"plain\ttext\n": "plain\ttext\n",
|
||||
"a\x1b[2Jb\n": "a<1b>[2Jb\n",
|
||||
"bell\x07 del\x7f\n": "bell<07> del<7f>\n",
|
||||
"inner\nnewline\n": "inner<0a>newline\n",
|
||||
"c1 \u009b31m\n": "c1 <c29b>31m\n",
|
||||
"bidi \u202eevil\n": "bidi <e280ae>evil\n",
|
||||
"isolate \u2066x\u2069\n": "isolate <e281a6>x<e281a9>\n",
|
||||
"line\u2028sep\n": "line<e280a8>sep\n",
|
||||
"zwj \U0001f468\u200d\U0001f469\n": "zwj \U0001f468\u200d\U0001f469\n",
|
||||
"bad \xff utf8": "bad <ff> utf8",
|
||||
"ünïcødé and \ufffd kept\n": "ünïcødé and \ufffd kept\n",
|
||||
} {
|
||||
if got := string(EscapeControls([]byte(in))); got != want {
|
||||
t.Errorf("%q: got %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package logline
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
// JSON token kinds
|
||||
const (
|
||||
kNone byte = 0
|
||||
kStr byte = 's'
|
||||
kNum byte = 'n'
|
||||
kBool byte = 'b'
|
||||
kNull byte = 'z'
|
||||
kObj byte = 'o'
|
||||
kArr byte = 'a'
|
||||
)
|
||||
|
||||
func skipSpace(s string, i int) int {
|
||||
for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\r' || s[i] == '\n') {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func isNumByte(c byte) bool {
|
||||
return c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E' || (c >= '0' && c <= '9')
|
||||
}
|
||||
|
||||
// scanString returns the index past the closing quote; i indexes the opener.
|
||||
func scanString(s string, i int) (int, bool) {
|
||||
for i++; i < len(s); i++ {
|
||||
switch s[i] {
|
||||
case '\\':
|
||||
i++
|
||||
case '"':
|
||||
return i + 1, true
|
||||
}
|
||||
}
|
||||
return i, false
|
||||
}
|
||||
|
||||
// scanValue returns the end index and kind of the value starting at i.
|
||||
func scanValue(s string, i int) (int, byte, bool) {
|
||||
if i >= len(s) {
|
||||
return i, kNone, false
|
||||
}
|
||||
switch c := s[i]; c {
|
||||
case '"':
|
||||
e, ok := scanString(s, i)
|
||||
return e, kStr, ok
|
||||
|
||||
case '{', '[':
|
||||
opener, closer, kind := byte('{'), byte('}'), kObj
|
||||
if c == '[' {
|
||||
opener, closer, kind = '[', ']', kArr
|
||||
}
|
||||
depth := 0
|
||||
for i < len(s) {
|
||||
ch := s[i]
|
||||
if ch == '"' {
|
||||
e, ok := scanString(s, i)
|
||||
if !ok {
|
||||
return e, kind, false
|
||||
}
|
||||
i = e
|
||||
continue
|
||||
}
|
||||
if ch == opener {
|
||||
depth++
|
||||
} else if ch == closer {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i + 1, kind, true
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
return i, kind, false
|
||||
|
||||
case 't', 'f', 'n':
|
||||
kind := kBool
|
||||
if c == 'n' {
|
||||
kind = kNull
|
||||
}
|
||||
for i < len(s) && s[i] >= 'a' && s[i] <= 'z' {
|
||||
i++
|
||||
}
|
||||
return i, kind, true
|
||||
|
||||
default:
|
||||
e := i
|
||||
for e < len(s) && isNumByte(s[e]) {
|
||||
e++
|
||||
}
|
||||
return e, kNum, e > i
|
||||
}
|
||||
}
|
||||
|
||||
// eachField calls fn for each member of the object at i, which must index '{',
|
||||
// until fn returns false; it reports whether the object is well formed.
|
||||
func eachField(s string, i int, fn func(key, val string, kind byte) bool) bool {
|
||||
if i >= len(s) || s[i] != '{' {
|
||||
return false
|
||||
}
|
||||
i = skipSpace(s, i+1)
|
||||
if i < len(s) && s[i] == '}' {
|
||||
return true
|
||||
}
|
||||
for i < len(s) {
|
||||
if s[i] != '"' {
|
||||
return false
|
||||
}
|
||||
ke, ok := scanString(s, i)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
key := s[i+1 : ke-1]
|
||||
|
||||
i = skipSpace(s, ke)
|
||||
if i >= len(s) || s[i] != ':' {
|
||||
return false
|
||||
}
|
||||
i = skipSpace(s, i+1)
|
||||
|
||||
vs := i
|
||||
ve, kind, ok := scanValue(s, i)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if !fn(key, s[vs:ve], kind) {
|
||||
return true
|
||||
}
|
||||
|
||||
i = skipSpace(s, ve)
|
||||
if i >= len(s) {
|
||||
return false
|
||||
}
|
||||
switch s[i] {
|
||||
case ',':
|
||||
i = skipSpace(s, i+1)
|
||||
case '}':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// eachItem calls fn for each element of the array at i, which must index '[';
|
||||
// it reports whether the array is well formed.
|
||||
func eachItem(s string, i int, fn func(val string, kind byte)) bool {
|
||||
if i >= len(s) || s[i] != '[' {
|
||||
return false
|
||||
}
|
||||
i = skipSpace(s, i+1)
|
||||
if i < len(s) && s[i] == ']' {
|
||||
return true
|
||||
}
|
||||
for i < len(s) {
|
||||
ve, kind, ok := scanValue(s, i)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
fn(s[i:ve], kind)
|
||||
i = skipSpace(s, ve)
|
||||
if i >= len(s) {
|
||||
return false
|
||||
}
|
||||
switch s[i] {
|
||||
case ',':
|
||||
i = skipSpace(s, i+1)
|
||||
case ']':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// unquote returns the content of a string token, decoding only when it holds
|
||||
// an escape; any other token is returned as its JSON.
|
||||
func unquote(tok string, kind byte) string {
|
||||
if kind != kStr || len(tok) < 2 {
|
||||
return tok
|
||||
}
|
||||
in := tok[1 : len(tok)-1]
|
||||
for i := 0; i < len(in); i++ {
|
||||
if in[i] == '\\' {
|
||||
var s string
|
||||
if json.Unmarshal([]byte(tok), &s) == nil {
|
||||
return s
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
// parseInt64 parses a complete decimal integer token without allocating.
|
||||
func parseInt64(s string) (int64, bool) {
|
||||
if len(s) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
i, neg := 0, false
|
||||
if s[0] == '-' || s[0] == '+' {
|
||||
neg = s[0] == '-'
|
||||
i++
|
||||
}
|
||||
if i >= len(s) {
|
||||
return 0, false
|
||||
}
|
||||
var v int64
|
||||
for ; i < len(s); i++ {
|
||||
if s[i] < '0' || s[i] > '9' {
|
||||
return 0, false
|
||||
}
|
||||
v = v*10 + int64(s[i]-'0')
|
||||
if v < 0 {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
if neg {
|
||||
v = -v
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
Reference in new issue
Block a user