logwisp/internal/logline/logline.go

// Package logline reads a log line of any shape (vif records, LogWisp's json,
// lw's own log, pino, zap, slog, docker, journald, ECS, GCP, plain text) into
// one set of roles, and names the runes a terminal would act on. It links no
// terminal code, so a viewer, a sink and the configuration page can share it.
package logline

import (
	"bytes"
	"encoding/hex"
	"math"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/lixenwraith/logwisp/internal/core"
)

// Line is what a log line says, wherever its writer put it
type Line struct {
	Time    time.Time // zero when the line names none readable: the entry's time stands in
	Stamp   string    // the time as the line wrote it
	Level   string    // a core.Levels level, or a value outside their order as written (PROC)
	Tag     string    // who wrote it: sub, source, logger and the like, else the entry's source
	Message string
	Fields  []Field // what no role took, in the writer's order
	Layout  string  // where the roles were found, so a reader can check the guess
}

// Field is a value no role took. Key is its dotted path, "" for a positional
// value; Value is a string's text or any other value as its JSON.
type Field struct {
	Key, Value string
}

const (
	roleTime = iota
	roleLevel
	roleTag
	roleMsg
)

// roles are the keys each role is read from, the first present wins: a dotted
// key is a path into nested objects or a flat key spelled with dots. trace is
// the last tag, so it names LogWisp json's source but not vif's call chain.
var roles = [...][]string{
	roleTime:  {"time", "ts", "timestamp", "@timestamp", "t", "datetime", "__REALTIME_TIMESTAMP", "@t"},
	roleLevel: {"level", "lvl", "severity", "log.level", "levelname", "loglevel", "PRIORITY", "@l"},
	roleTag:   {"sub", "source", "logger", "component", "name", "_SYSTEMD_UNIT", "trace"},
	roleMsg:   {"msg", "message", "MESSAGE", "@m", "@mt", "log", "text", "event"},
}

// envelopes hold a record's payload one level down; the first of them present
// is opened, and without a message of its own the record takes the envelope's
// first discriminator, or an array's first string (vif: fields.msg|type|ev)
var (
	envelopes      = []string{"fields", "data", "attrs", "payload"}
	discriminators = []string{"msg", "type", "ev", "message"}
)

// nestMax bounds the depth flattened into dotted paths, and so their length
const nestMax = 8

// member is one leaf of a record; no role takes the envelope's
type member struct {
	key, val string
	kind     byte
	env      bool
	used     bool
}

// Read finds e's roles in its message, a JSON record (also one inside such a
// record's message) or text with a leading RFC3339 stamp, and in its fields;
// a role the line does not name is the entry's.
func Read(e core.LogEntry) Line {
	l, at, ok := record(strings.TrimSpace(e.Message), "")
	if ok {
		if in, _, ok := record(strings.TrimSpace(l.Message), at); ok {
			l = merge(l, in)
		}
	} else {
		l = Line{Message: e.Message, Layout: "text"}
		if stamp, rest, ok := strings.Cut(e.Message, " "); ok {
			if at, ok := readStamp(stamp); ok {
				l.Time, l.Stamp, l.Message, l.Layout = at, stamp, strings.TrimLeft(rest, " "), "text, stamped"
			}
		}
	}
	if f := strings.TrimSpace(string(e.Fields)); f != "" {
		_, kind, _ := scanValue(f, 0)
		ms, msg, at := envelope(nil, "fields", f, kind)
		if l.Message == "" && msg >= 0 {
			ms[msg].used = true
			l.Message, l.Layout = unquote(ms[msg].val, kStr), "entry, message in "+at
		}
		l.Fields = appendFields(l.Fields, ms)
	}
	if l.Level == "" {
		l.Level = level("", e.Level)
	}
	if l.Tag == "" {
		l.Tag = e.Source
	}
	l.Message = strings.TrimRight(l.Message, "\r\n")
	return l
}

// record reads s as a JSON object; in names where s sat as another record's
// message, at where s's own message is. The members before a malformed part
// count, so a clipped line still shows its roles; ok is false without any.
func record(s, in string) (l Line, at string, ok bool) {
	var top []member
	whole := eachField(s, 0, func(k, v string, kind byte) bool {
		top = append(top, member{key: k, val: v, kind: kind})
		return true
	})
	if len(top) == 0 {
		return Line{}, "", false
	}
	env := -1
	for _, name := range envelopes {
		for i := range top {
			if env < 0 && top[i].key == name && (top[i].kind == kObj || top[i].kind == kArr) {
				env = i
			}
		}
	}
	var ms []member
	msg, envAt := -1, ""
	for i, m := range top {
		if i == env {
			ms, msg, envAt = envelope(ms, m.key, m.val, m.kind)
		} else {
			ms = leaves(ms, m.key, m.val, m.kind, false, 0)
		}
	}

	if m := find(ms, roleTime); m != nil {
		l.Stamp = unquote(m.val, m.kind)
		l.Time, _ = readTime(l.Stamp)
	}
	if m := find(ms, roleLevel); m != nil {
		l.Level = level(m.key, unquote(m.val, m.kind))
	}
	if m := find(ms, roleTag); m != nil {
		l.Tag = unquote(m.val, m.kind)
	}
	if m := find(ms, roleMsg); m != nil {
		l.Message, at = unquote(m.val, m.kind), m.key
	} else if msg >= 0 {
		ms[msg].used = true
		l.Message, at = unquote(ms[msg].val, kStr), envAt
	}
	l.Fields = appendFields(nil, ms)

	l.Layout = "json"
	if in != "" {
		l.Layout += " in " + in
	}
	if !whole {
		l.Layout += ", cut short"
	}
	if at != "" {
		l.Layout += ", message in " + at
	} else {
		l.Layout += ", no message"
	}
	return l, at, true
}

// envelope appends the leaves of envelope name's value v; msg indexes its
// message in ms and at says where it was, -1 and "" when it holds none
func envelope(ms []member, name, v string, kind byte) (_ []member, msg int, at string) {
	msg, rank, slot := -1, len(discriminators), -1
	note := func(key string, kind byte, i int) {
		for r := 0; r < rank && kind == kStr; r++ {
			if discriminators[r] == key {
				msg, rank, slot = len(ms), r, i
			}
		}
	}
	switch kind {
	case kObj:
		eachField(v, 0, func(k, fv string, fk byte) bool {
			note(k, fk, -1)
			ms = leaves(ms, k, fv, fk, true, 1)
			return true
		})
	case kArr:
		// lw's own log writes key/value pairs; LogWisp json, the message alone
		n, pairs := 0, true
		eachItem(v, 0, func(_ string, k byte) {
			pairs = pairs && (n%2 == 1 || k == kStr)
			n++
		})
		pairs = pairs && n > 0 && n%2 == 0
		i, key := 0, ""
		eachItem(v, 0, func(iv string, ik byte) {
			switch {
			case pairs && i%2 == 0:
				key = unquote(iv, ik)
			case pairs:
				note(key, ik, i)
				ms = leaves(ms, key, iv, ik, true, 1)
			default:
				if i == 0 && ik == kStr {
					msg, slot = len(ms), 0
				}
				ms = leaves(ms, "", iv, ik, true, 1)
			}
			i++
		})
	default:
		ms = leaves(ms, name, v, kind, true, 0)
	}
	switch {
	case msg < 0:
	case slot < 0:
		at = name + "." + ms[msg].key
	default:
		at = name + "[" + strconv.Itoa(slot) + "]"
	}
	return ms, msg, at
}

// leaves appends v under key, an object as one leaf per member at its dotted
// path, so roles and readers see nested keys as if written flat
func leaves(ms []member, key, v string, kind byte, env bool, depth int) []member {
	if kind == kObj && depth < nestMax {
		n := len(ms)
		if eachField(v, 0, func(k, fv string, fk byte) bool {
			if key != "" {
				k = key + "." + k
			}
			ms = leaves(ms, k, fv, fk, env, depth+1)
			return true
		}) && len(ms) > n {
			return ms
		}
		ms = ms[:n]
	}
	return append(ms, member{key: key, val: v, kind: kind, env: env})
}

// find takes the leaf holding role r, outside the envelope and holding a
// string or a number: a null, a flag or a list names no role
func find(ms []member, r int) *member {
	for _, key := range roles[r] {
		for i := range ms {
			m := &ms[i]
			if m.key == key && !m.env && !m.used && (m.kind == kStr || m.kind == kNum) {
				m.used = true
				return m
			}
		}
	}
	return nil
}

func appendFields(fs []Field, ms []member) []Field {
	for _, m := range ms {
		if !m.used {
			fs = append(fs, Field{m.key, unquote(m.val, m.kind)})
		}
	}
	return fs
}

// merge lays a record found in out's message over out: its roles win, out's
// fill the ones it does not name, and out's fields come first
func merge(out, in Line) Line {
	if in.Time.IsZero() {
		in.Time, in.Stamp = out.Time, out.Stamp
	}
	if in.Level == "" {
		in.Level = out.Level
	}
	if in.Tag == "" {
		in.Tag = out.Tag
	}
	in.Fields = append(out.Fields, in.Fields...)
	return in
}

// level reads a level value: a name in any case, or a number on its key's
// scale (journald's syslog PRIORITY, pino's tens, else slog's and
// lixenwraith/log's); any other value is kept as written, outside the order
func level(key, v string) string {
	if n, ok := parseInt64(v); ok {
		switch {
		case key == "PRIORITY":
			return [...]string{"ERROR", "ERROR", "ERROR", "ERROR", "WARN", "INFO", "INFO", "DEBUG"}[min(max(n, 0), 7)]
		case n >= 10:
			return [...]string{"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "ERROR"}[min(n/10-1, 5)]
		case n <= -8:
			return "TRACE"
		case n < 0:
			return "DEBUG"
		case n < 4:
			return "INFO"
		case n < 8:
			return "WARN"
		}
		return "ERROR"
	}
	for _, l := range core.Levels {
		for _, names := range [2][]string{l.Names, l.Keyed} {
			for _, name := range names {
				if strings.EqualFold(v, name) {
					return l.Level
				}
			}
		}
	}
	return v
}

// readStamp reads an RFC3339 stamp, one without a zone as UTC
func readStamp(s string) (time.Time, bool) {
	at, err := time.Parse(time.RFC3339Nano, s)
	if err != nil {
		at, err = time.Parse("2006-01-02T15:04:05.999999999", s)
	}
	return at, err == nil
}

// readTime reads an RFC3339 stamp, or a number since 1970 in seconds,
// milliseconds or microseconds, told apart by its magnitude
func readTime(s string) (time.Time, bool) {
	if at, ok := readStamp(s); ok {
		return at, true
	}
	whole, frac, _ := strings.Cut(s, ".")
	n, ok := parseInt64(whole)
	if !ok || n < 0 {
		return time.Time{}, false
	}
	unit := int64(1e3)
	switch {
	case n < 1e11:
		unit = 1e9
	case n < 1e14:
		unit = 1e6
	}
	if n > math.MaxInt64/unit {
		return time.Time{}, false
	}
	ns := n * unit
	for i := 0; i < len(frac); i++ {
		if frac[i] < '0' || frac[i] > '9' {
			return time.Time{}, false
		}
		unit /= 10
		ns += int64(frac[i]-'0') * unit
	}
	return time.Unix(0, ns), true
}

// EscapeControls writes what a terminal would act on or reorder (Control) as
// <hex>, keeping the final newline: a log line cannot drive the terminal.
// Other invisible characters stay: emoji and Indic scripts need ZWJ.
func EscapeControls(p []byte) []byte {
	body, newline := bytes.CutSuffix(p, []byte{'\n'})
	if printableASCII(body) {
		return p
	}
	out := make([]byte, 0, len(p)+16)
	for len(body) > 0 {
		r, n := utf8.DecodeRune(body)
		if !Control(r, n) {
			out = append(out, body[:n]...)
		} else {
			out = append(out, '<')
			out = hex.AppendEncode(out, body[:n])
			out = append(out, '>')
		}
		body = body[n:]
	}
	if newline {
		out = append(out, '\n')
	}
	return out
}

// Control reports whether a terminal would act on the rune r, decoded from
// size bytes, or reorder text by it: C0 and C1 controls but tab, DEL, bidi
// controls, line separators and invalid UTF-8
func Control(r rune, size int) bool {
	switch {
	case r == utf8.RuneError && size == 1:
		return true
	case r < 0x20:
		return r != '\t'
	case r >= 0x7f && r <= 0x9f, r == 0x061c, r == 0x200e, r == 0x200f,
		r >= 0x2028 && r <= 0x202e, r >= 0x2066 && r <= 0x2069:
		return true
	}
	return false
}

func printableASCII(b []byte) bool {
	for _, c := range b {
		if (c < 0x20 || c > 0x7e) && c != '\t' {
			return false
		}
	}
	return true
}