v0.1.0 initial commit
This commit is contained in:
@@ -0,0 +1,346 @@
|
||||
package logfile
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Meta is one index row: 48 bytes, pointer-free, holds no line bytes.
|
||||
type Meta struct {
|
||||
Off int64
|
||||
TS int64
|
||||
Len uint32
|
||||
Run uint32
|
||||
Tick uint32
|
||||
Frame uint32
|
||||
Msg uint32 // interned fields.msg
|
||||
Snap uint32 // stat-snapshot group id, 0 = none
|
||||
Sub uint16 // interned sub
|
||||
Src uint16 // source file index
|
||||
Lvl Level
|
||||
Flags uint8
|
||||
}
|
||||
|
||||
// Meta flag bits.
|
||||
const (
|
||||
FlagMalformed uint8 = 1 << iota
|
||||
FlagSnapHead
|
||||
FlagTrace
|
||||
FlagNoTime // TS was inherited from the previous line, for ordering only
|
||||
)
|
||||
|
||||
// Snapshot groups the stat records sharing one (src, run, tick). Members are
|
||||
// not contiguous in the merged view; Count is authoritative, Head locates the
|
||||
// group's first row in the published order.
|
||||
type Snapshot struct {
|
||||
Head uint32
|
||||
Count uint32
|
||||
Run uint32
|
||||
Tick uint32
|
||||
Frame uint32
|
||||
Src uint16
|
||||
}
|
||||
|
||||
// Source is one indexed file.
|
||||
type Source struct {
|
||||
Path string
|
||||
Name string
|
||||
Size int64
|
||||
|
||||
scanned atomic.Int64
|
||||
bad atomic.Int64
|
||||
done atomic.Bool
|
||||
failure atomic.Pointer[scanErr]
|
||||
}
|
||||
|
||||
// Index is the append-only line index over one or more files. The scanner
|
||||
// publishes immutable slice headers; readers load them atomically, so the
|
||||
// render path never locks. A single source publishes incrementally; several
|
||||
// sources publish once, merged by timestamp, so row indices never shift.
|
||||
type Index struct {
|
||||
srcs []*Source
|
||||
|
||||
subN, msgN *interner
|
||||
|
||||
metas atomic.Pointer[[]Meta]
|
||||
snaps atomic.Pointer[[]Snapshot]
|
||||
subs atomic.Pointer[[]string]
|
||||
msgs atomic.Pointer[[]string]
|
||||
}
|
||||
|
||||
type scanErr struct{ err error }
|
||||
|
||||
// The accessors below tolerate a nil receiver: the viewer runs with no file
|
||||
// open until one is chosen, and the render path reads the index every frame.
|
||||
|
||||
// Sources returns the indexed files in source-id order.
|
||||
func (x *Index) Sources() []*Source {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
return x.srcs
|
||||
}
|
||||
|
||||
// SrcCount returns the number of indexed files.
|
||||
func (x *Index) SrcCount() int {
|
||||
if x == nil {
|
||||
return 0
|
||||
}
|
||||
return len(x.srcs)
|
||||
}
|
||||
|
||||
// SrcName resolves a source id to its base filename.
|
||||
func (x *Index) SrcName(id uint16) string {
|
||||
if x == nil || int(id) >= len(x.srcs) {
|
||||
return ""
|
||||
}
|
||||
return x.srcs[id].Name
|
||||
}
|
||||
|
||||
// SrcMark returns the one-character gutter label for a source id.
|
||||
func (x *Index) SrcMark(id uint16) rune {
|
||||
const marks = "123456789abcdefghijklmnopqrstuvwxyz"
|
||||
if int(id) >= len(marks) {
|
||||
return '?'
|
||||
}
|
||||
return rune(marks[id])
|
||||
}
|
||||
|
||||
// Metas returns the currently published index rows.
|
||||
func (x *Index) Metas() []Meta {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
if p := x.metas.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Len returns the number of indexed records.
|
||||
func (x *Index) Len() int { return len(x.Metas()) }
|
||||
|
||||
// Snaps returns the published snapshot groups.
|
||||
func (x *Index) Snaps() []Snapshot {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
if p := x.snaps.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Subs returns the interned subsystem table.
|
||||
func (x *Index) Subs() []string {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
return strTable(x.subs.Load())
|
||||
}
|
||||
|
||||
// Msgs returns the interned fields.msg table.
|
||||
func (x *Index) Msgs() []string {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
return strTable(x.msgs.Load())
|
||||
}
|
||||
|
||||
// SubName resolves an interned sub id.
|
||||
func (x *Index) SubName(id uint16) string {
|
||||
if x == nil {
|
||||
return ""
|
||||
}
|
||||
return lookup(x.subs.Load(), int(id))
|
||||
}
|
||||
|
||||
// MsgName resolves an interned fields.msg id.
|
||||
func (x *Index) MsgName(id uint32) string {
|
||||
if x == nil {
|
||||
return ""
|
||||
}
|
||||
return lookup(x.msgs.Load(), int(id))
|
||||
}
|
||||
|
||||
func strTable(p *[]string) []string {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
func lookup(p *[]string, i int) string {
|
||||
t := strTable(p)
|
||||
if i < 0 || i >= len(t) {
|
||||
return ""
|
||||
}
|
||||
return t[i]
|
||||
}
|
||||
|
||||
// Progress reports bytes scanned, total bytes and completion across all sources.
|
||||
func (x *Index) Progress() (scanned, total int64, complete bool) {
|
||||
if x == nil {
|
||||
return 0, 0, true
|
||||
}
|
||||
complete = true
|
||||
for _, s := range x.srcs {
|
||||
scanned += s.scanned.Load()
|
||||
total += s.Size
|
||||
if !s.done.Load() {
|
||||
complete = false
|
||||
}
|
||||
}
|
||||
return scanned, total, complete
|
||||
}
|
||||
|
||||
// Malformed returns the count of unparseable lines seen so far.
|
||||
func (x *Index) Malformed() int64 {
|
||||
if x == nil {
|
||||
return 0
|
||||
}
|
||||
var n int64
|
||||
for _, s := range x.srcs {
|
||||
n += s.bad.Load()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Err returns the first scan failure, if any.
|
||||
func (x *Index) Err() error {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
for _, s := range x.srcs {
|
||||
if p := s.failure.Load(); p != nil {
|
||||
return p.err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotOf returns the stat group a record belongs to. The group currently
|
||||
// being scanned is not published, so ok=false until it closes.
|
||||
func (x *Index) SnapshotOf(m Meta) (Snapshot, bool) {
|
||||
if x == nil || m.Snap == 0 {
|
||||
return Snapshot{}, false
|
||||
}
|
||||
s := x.Snaps()
|
||||
if int(m.Snap) > len(s) {
|
||||
return Snapshot{}, false
|
||||
}
|
||||
return s[m.Snap-1], true
|
||||
}
|
||||
|
||||
const readWindow = 256 << 10
|
||||
|
||||
// Reader fetches raw line bytes through a per-source sliding window.
|
||||
// Not safe for concurrent use; create one per goroutine.
|
||||
type Reader struct {
|
||||
paths []string
|
||||
win []window
|
||||
}
|
||||
|
||||
type window struct {
|
||||
f *os.File
|
||||
buf []byte
|
||||
base int64
|
||||
n int
|
||||
}
|
||||
|
||||
// NewReader opens an independent handle set on the indexed files. Handles are
|
||||
// opened lazily, so a reader that only touches one source costs one fd.
|
||||
func (x *Index) NewReader() (*Reader, error) {
|
||||
r := &Reader{
|
||||
paths: make([]string, len(x.srcs)),
|
||||
win: make([]window, len(x.srcs)),
|
||||
}
|
||||
for i, s := range x.srcs {
|
||||
r.paths[i] = s.Path
|
||||
r.win[i].base = -1
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Line returns the raw bytes of m, valid until the next call.
|
||||
func (r *Reader) Line(m Meta) ([]byte, error) {
|
||||
n := int(m.Len)
|
||||
if n == 0 || int(m.Src) >= len(r.win) {
|
||||
return nil, nil
|
||||
}
|
||||
w := &r.win[m.Src]
|
||||
if w.f == nil {
|
||||
f, err := os.Open(r.paths[m.Src])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.f, w.buf, w.base = f, make([]byte, readWindow), -1
|
||||
}
|
||||
if n+4096 > len(w.buf) {
|
||||
w.buf = make([]byte, n+4096)
|
||||
w.base = -1
|
||||
}
|
||||
if w.base < 0 || m.Off < w.base || m.Off+int64(n) > w.base+int64(w.n) {
|
||||
base := m.Off &^ 4095
|
||||
got, err := w.f.ReadAt(w.buf, base)
|
||||
if got == 0 && err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.base, w.n = base, got
|
||||
}
|
||||
s := int(m.Off - w.base)
|
||||
if s < 0 || s+n > w.n {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
return w.buf[s : s+n], nil
|
||||
}
|
||||
|
||||
// Close releases every open file handle.
|
||||
func (r *Reader) Close() error {
|
||||
var err error
|
||||
for i := range r.win {
|
||||
if r.win[i].f != nil {
|
||||
if e := r.win[i].f.Close(); e != nil && err == nil {
|
||||
err = e
|
||||
}
|
||||
r.win[i].f = nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// interner maps byte tokens to dense ids. Writer-side only; the id→name table
|
||||
// is published as an immutable snapshot. Locked: sources scan concurrently.
|
||||
type interner struct {
|
||||
mu sync.Mutex
|
||||
ids map[string]uint32
|
||||
tab []string
|
||||
}
|
||||
|
||||
func newInterner() *interner {
|
||||
n := &interner{ids: make(map[string]uint32, 64)}
|
||||
n.intern(nil) // id 0 == ""
|
||||
return n
|
||||
}
|
||||
|
||||
func (n *interner) intern(b []byte) uint32 {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
if id, ok := n.ids[string(b)]; ok {
|
||||
return id
|
||||
}
|
||||
s := string(b)
|
||||
id := uint32(len(n.tab))
|
||||
n.tab = append(n.tab, s)
|
||||
n.ids[s] = id
|
||||
return id
|
||||
}
|
||||
|
||||
// table returns a consistent header over the interned names.
|
||||
func (n *interner) table() []string {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
return n.tab[:len(n.tab):len(n.tab)]
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package logfile
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// JSON token kinds.
|
||||
const (
|
||||
KNone byte = 0
|
||||
KStr byte = 's'
|
||||
KNum byte = 'n'
|
||||
KBool byte = 'b'
|
||||
KNull byte = 'z'
|
||||
KObj byte = 'o'
|
||||
KArr byte = 'a'
|
||||
)
|
||||
|
||||
func skipSpace(b []byte, i int) int {
|
||||
for i < len(b) && (b[i] == ' ' || b[i] == '\t' || b[i] == '\r' || b[i] == '\n') {
|
||||
i++
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func isNumByte(c byte) bool {
|
||||
return c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E' || (c >= '0' && c <= '9')
|
||||
}
|
||||
|
||||
// scanString returns the index past the closing quote; i indexes the opener.
|
||||
func scanString(b []byte, i int) (int, bool) {
|
||||
for i++; i < len(b); i++ {
|
||||
switch b[i] {
|
||||
case '\\':
|
||||
i++
|
||||
case '"':
|
||||
return i + 1, true
|
||||
}
|
||||
}
|
||||
return i, false
|
||||
}
|
||||
|
||||
// scanValue returns the end index and kind of the value starting at i.
|
||||
func scanValue(b []byte, i int) (int, byte, bool) {
|
||||
if i >= len(b) {
|
||||
return i, KNone, false
|
||||
}
|
||||
switch c := b[i]; c {
|
||||
case '"':
|
||||
e, ok := scanString(b, i)
|
||||
return e, KStr, ok
|
||||
|
||||
case '{', '[':
|
||||
opener, closer, kind := byte('{'), byte('}'), KObj
|
||||
if c == '[' {
|
||||
opener, closer, kind = '[', ']', KArr
|
||||
}
|
||||
depth := 0
|
||||
for i < len(b) {
|
||||
ch := b[i]
|
||||
if ch == '"' {
|
||||
e, ok := scanString(b, i)
|
||||
if !ok {
|
||||
return e, kind, false
|
||||
}
|
||||
i = e
|
||||
continue
|
||||
}
|
||||
if ch == opener {
|
||||
depth++
|
||||
} else if ch == closer {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return i + 1, kind, true
|
||||
}
|
||||
}
|
||||
i++
|
||||
}
|
||||
return i, kind, false
|
||||
|
||||
case 't', 'f', 'n':
|
||||
kind := KBool
|
||||
if c == 'n' {
|
||||
kind = KNull
|
||||
}
|
||||
for i < len(b) && b[i] >= 'a' && b[i] <= 'z' {
|
||||
i++
|
||||
}
|
||||
return i, kind, true
|
||||
|
||||
default:
|
||||
for i < len(b) && isNumByte(b[i]) {
|
||||
i++
|
||||
}
|
||||
return i, KNum, true
|
||||
}
|
||||
}
|
||||
|
||||
// eachField calls fn for each member of the object at i, which must index '{'.
|
||||
// fn returning false stops iteration. Reports whether the object is well formed.
|
||||
func eachField(b []byte, i int, fn func(key, val []byte, kind byte) bool) bool {
|
||||
if i >= len(b) || b[i] != '{' {
|
||||
return false
|
||||
}
|
||||
i = skipSpace(b, i+1)
|
||||
if i < len(b) && b[i] == '}' {
|
||||
return true
|
||||
}
|
||||
for i < len(b) {
|
||||
if b[i] != '"' {
|
||||
return false
|
||||
}
|
||||
ke, ok := scanString(b, i)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
key := b[i+1 : ke-1]
|
||||
|
||||
i = skipSpace(b, ke)
|
||||
if i >= len(b) || b[i] != ':' {
|
||||
return false
|
||||
}
|
||||
i = skipSpace(b, i+1)
|
||||
|
||||
vs := i
|
||||
ve, kind, ok := scanValue(b, i)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if !fn(key, b[vs:ve], kind) {
|
||||
return true
|
||||
}
|
||||
|
||||
i = skipSpace(b, ve)
|
||||
if i >= len(b) {
|
||||
return false
|
||||
}
|
||||
switch b[i] {
|
||||
case ',':
|
||||
i = skipSpace(b, i+1)
|
||||
case '}':
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// strTok returns the undecoded content of a string token.
|
||||
func strTok(tok []byte) []byte {
|
||||
if len(tok) < 2 {
|
||||
return nil
|
||||
}
|
||||
return tok[1 : len(tok)-1]
|
||||
}
|
||||
|
||||
// unquote returns the content of a string token, decoding escapes only when present.
|
||||
func unquote(tok []byte) string {
|
||||
in := strTok(tok)
|
||||
if bytes.IndexByte(in, '\\') < 0 {
|
||||
return string(in)
|
||||
}
|
||||
if s, err := strconv.Unquote(string(tok)); err == nil {
|
||||
return s
|
||||
}
|
||||
return string(in)
|
||||
}
|
||||
|
||||
// parseUint32 parses a leading decimal run, saturating at the type maximum.
|
||||
func parseUint32(b []byte) uint32 {
|
||||
var v uint64
|
||||
for _, c := range b {
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
v = v*10 + uint64(c-'0')
|
||||
if v > 0xffffffff {
|
||||
return 0xffffffff
|
||||
}
|
||||
}
|
||||
return uint32(v)
|
||||
}
|
||||
|
||||
// parseInt64 parses a complete decimal integer token without allocating.
|
||||
func parseInt64(b []byte) (int64, bool) {
|
||||
if len(b) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
i, neg := 0, false
|
||||
if b[0] == '-' || b[0] == '+' {
|
||||
neg = b[0] == '-'
|
||||
i++
|
||||
}
|
||||
if i >= len(b) {
|
||||
return 0, false
|
||||
}
|
||||
var v int64
|
||||
for ; i < len(b); i++ {
|
||||
if b[i] < '0' || b[i] > '9' {
|
||||
return 0, false
|
||||
}
|
||||
v = v*10 + int64(b[i]-'0')
|
||||
if v < 0 {
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
if neg {
|
||||
v = -v
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
package logfile
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// Column identifies a list column for focus, search scope and sorting.
|
||||
type Column uint8
|
||||
|
||||
const (
|
||||
ColAll Column = iota
|
||||
ColTime
|
||||
ColTick
|
||||
ColSub
|
||||
ColMsg
|
||||
ColFields
|
||||
ColCount
|
||||
)
|
||||
|
||||
var columnName = [ColCount]string{"all", "time", "tick", "sub", "msg", "fields"}
|
||||
|
||||
func (c Column) String() string {
|
||||
if c < ColCount {
|
||||
return columnName[c]
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
// Next returns the column d steps away, wrapping.
|
||||
func (c Column) Next(d int) Column {
|
||||
return Column(((int(c)+d)%int(ColCount) + int(ColCount)) % int(ColCount))
|
||||
}
|
||||
|
||||
// Field is one member of the fields object; Val is the raw JSON token.
|
||||
type Field struct {
|
||||
Key string
|
||||
Val []byte
|
||||
Kind byte
|
||||
}
|
||||
|
||||
// Record is the parsed form of a line. It is the only type that knows the log
|
||||
// schema: a new sub, a renamed key or a second format touches this file alone.
|
||||
type Record struct {
|
||||
Meta Meta
|
||||
Raw []byte
|
||||
Time string
|
||||
Level string
|
||||
Sub string
|
||||
Trace string
|
||||
Fields []Field
|
||||
Bad bool
|
||||
|
||||
msgIdx int // index into Fields of the discriminator, -1 if none
|
||||
buf []byte // column rendering scratch, reused across calls
|
||||
}
|
||||
|
||||
// msgKeys are the discriminator keys in precedence order. Most records use
|
||||
// fields.msg; the logger self-report uses fields.type.
|
||||
var msgKeys = [...]string{"msg", "type"}
|
||||
|
||||
// Parse fills r from line. Slices alias line and are reused across calls, so
|
||||
// copy anything retained past the next Parse.
|
||||
func (r *Record) Parse(m Meta, line []byte) {
|
||||
r.Meta, r.Raw = m, line
|
||||
r.Time, r.Level, r.Sub, r.Trace = "", "", "", ""
|
||||
r.Fields = r.Fields[:0]
|
||||
r.msgIdx = -1
|
||||
r.Bad = m.Flags&FlagMalformed != 0 || len(line) == 0
|
||||
if r.Bad {
|
||||
return
|
||||
}
|
||||
|
||||
eachField(line, skipSpace(line, 0), func(k, v []byte, kind byte) bool {
|
||||
switch string(k) {
|
||||
case "time":
|
||||
r.Time = string(strTok(v))
|
||||
case "level":
|
||||
r.Level = string(strTok(v))
|
||||
case "sub":
|
||||
r.Sub = string(strTok(v))
|
||||
case "trace":
|
||||
r.Trace = unquote(v)
|
||||
case "fields":
|
||||
if kind == KObj {
|
||||
eachField(v, 0, func(fk, fv []byte, fkind byte) bool {
|
||||
r.Fields = append(r.Fields, Field{Key: string(fk), Val: fv, Kind: fkind})
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
// Resolve the discriminator once: input records carry both msg and type,
|
||||
// and only the one actually used may be dropped from the fields text.
|
||||
for _, k := range msgKeys {
|
||||
for i := range r.Fields {
|
||||
if r.Fields[i].Key == k && r.Fields[i].Kind == KStr {
|
||||
r.msgIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if r.msgIdx >= 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get returns the named field.
|
||||
func (r *Record) Get(key string) (Field, bool) {
|
||||
for _, f := range r.Fields {
|
||||
if f.Key == key {
|
||||
return f, true
|
||||
}
|
||||
}
|
||||
return Field{}, false
|
||||
}
|
||||
|
||||
// Msg returns the record's discriminator.
|
||||
func (r *Record) Msg() string {
|
||||
if r.msgIdx < 0 {
|
||||
return ""
|
||||
}
|
||||
return unquote(r.Fields[r.msgIdx].Val)
|
||||
}
|
||||
|
||||
// FollowValue returns the first string field other than the discriminator: the
|
||||
// value distinguishing records that share a (sub, msg) pair — ev for event
|
||||
// dispatch, service for service records. Empty when every field is numeric.
|
||||
func (r *Record) FollowValue() string {
|
||||
for i, f := range r.Fields {
|
||||
if i == r.msgIdx || f.Kind != KStr {
|
||||
continue
|
||||
}
|
||||
return unquote(f.Val)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
const fieldsTextCap = 512
|
||||
|
||||
// FieldsText is the fields column exactly as the record list renders it.
|
||||
func (r *Record) FieldsText() string {
|
||||
r.buf = r.appendFields(r.buf[:0])
|
||||
return string(r.buf)
|
||||
}
|
||||
|
||||
// ColumnBytes renders one column's searchable text into a reused buffer, so
|
||||
// search matches what is on screen rather than the raw JSON.
|
||||
func (r *Record) ColumnBytes(c Column) []byte {
|
||||
r.buf = r.buf[:0]
|
||||
switch c {
|
||||
case ColTime:
|
||||
r.buf = append(r.buf, StampText(r.Meta)...)
|
||||
case ColTick:
|
||||
r.buf = strconv.AppendUint(r.buf, uint64(r.Meta.Tick), 10)
|
||||
case ColSub:
|
||||
r.buf = append(r.buf, r.Sub...)
|
||||
case ColMsg:
|
||||
r.buf = r.appendMsg(r.buf)
|
||||
case ColFields:
|
||||
r.buf = r.appendFields(r.buf)
|
||||
default:
|
||||
r.buf = append(r.buf, StampText(r.Meta)...)
|
||||
r.buf = append(r.buf, ' ')
|
||||
r.buf = append(r.buf, r.Sub...)
|
||||
r.buf = append(r.buf, ' ')
|
||||
r.buf = r.appendMsg(r.buf)
|
||||
r.buf = append(r.buf, ' ')
|
||||
r.buf = r.appendFields(r.buf)
|
||||
}
|
||||
return r.buf
|
||||
}
|
||||
|
||||
func (r *Record) appendMsg(dst []byte) []byte {
|
||||
if r.msgIdx < 0 {
|
||||
return dst
|
||||
}
|
||||
return appendUnquoted(dst, r.Fields[r.msgIdx].Val)
|
||||
}
|
||||
|
||||
func (r *Record) appendFields(dst []byte) []byte {
|
||||
start := len(dst)
|
||||
for i, f := range r.Fields {
|
||||
if i == r.msgIdx {
|
||||
continue
|
||||
}
|
||||
if len(dst) > start {
|
||||
dst = append(dst, ' ')
|
||||
}
|
||||
dst = append(dst, f.Key...)
|
||||
dst = append(dst, '=')
|
||||
dst = r.appendDisplay(dst, f)
|
||||
if len(dst)-start > fieldsTextCap {
|
||||
break
|
||||
}
|
||||
}
|
||||
return dst
|
||||
}
|
||||
|
||||
// Display renders a field value: durations via the unit table, long float
|
||||
// tails trimmed, everything else verbatim.
|
||||
func (r *Record) Display(f Field) string {
|
||||
return string(r.appendDisplay(nil, f))
|
||||
}
|
||||
|
||||
func (r *Record) appendDisplay(dst []byte, f Field) []byte {
|
||||
switch f.Kind {
|
||||
case KStr:
|
||||
return appendUnquoted(dst, f.Val)
|
||||
case KNum:
|
||||
if u := DurationUnit(f.Key); u != DurNone {
|
||||
if v, ok := parseInt64(f.Val); ok {
|
||||
return append(dst, FormatDuration(v, u)...)
|
||||
}
|
||||
}
|
||||
return append(dst, trimFloat(f.Val)...)
|
||||
}
|
||||
return append(dst, f.Val...)
|
||||
}
|
||||
|
||||
func appendUnquoted(dst, tok []byte) []byte {
|
||||
in := strTok(tok)
|
||||
if bytes.IndexByte(in, '\\') < 0 {
|
||||
return append(dst, in...)
|
||||
}
|
||||
return append(dst, unquote(tok)...)
|
||||
}
|
||||
|
||||
// trimFloat shortens 17-digit float tails to 6 significant digits.
|
||||
func trimFloat(tok []byte) []byte {
|
||||
dot := bytes.IndexByte(tok, '.')
|
||||
if dot < 0 || len(tok)-dot <= 7 {
|
||||
return tok
|
||||
}
|
||||
if v, err := strconv.ParseFloat(string(tok), 64); err == nil {
|
||||
return strconv.AppendFloat(nil, v, 'g', 6, 64)
|
||||
}
|
||||
return tok
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package logfile
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
scanBufSize = 1 << 20
|
||||
publishEach = 4096
|
||||
)
|
||||
|
||||
// scanPart is one source's private scan output, merged after completion.
|
||||
type scanPart struct {
|
||||
metas []Meta
|
||||
snaps []Snapshot
|
||||
}
|
||||
|
||||
// Open indexes one or more files and starts background scanning. A single
|
||||
// source grows the published view incrementally; several sources publish once,
|
||||
// merged by timestamp, so a row index is stable for the life of the index.
|
||||
func Open(paths ...string) (*Index, error) {
|
||||
if len(paths) == 0 {
|
||||
return nil, os.ErrInvalid
|
||||
}
|
||||
x := &Index{subN: newInterner(), msgN: newInterner()}
|
||||
|
||||
files := make([]*os.File, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
f, err := os.Open(p)
|
||||
if err != nil {
|
||||
closeAll(files)
|
||||
return nil, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
closeAll(files)
|
||||
return nil, err
|
||||
}
|
||||
abs := p
|
||||
if a, err := filepath.Abs(p); err == nil {
|
||||
abs = a
|
||||
}
|
||||
x.srcs = append(x.srcs, &Source{Path: abs, Name: filepath.Base(p), Size: fi.Size()})
|
||||
files = append(files, f)
|
||||
}
|
||||
|
||||
x.publish(nil, nil, nil, nil)
|
||||
go x.scanAll(files)
|
||||
return x, nil
|
||||
}
|
||||
|
||||
func closeAll(fs []*os.File) {
|
||||
for _, f := range fs {
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// publish stores immutable slice headers; a later append reallocates rather
|
||||
// than mutating what readers already hold.
|
||||
func (x *Index) publish(metas []Meta, snaps []Snapshot, subs, msgs []string) {
|
||||
x.metas.Store(&metas)
|
||||
x.snaps.Store(&snaps)
|
||||
x.subs.Store(&subs)
|
||||
x.msgs.Store(&msgs)
|
||||
}
|
||||
|
||||
func (x *Index) scanAll(files []*os.File) {
|
||||
live := len(x.srcs) == 1
|
||||
parts := make([]scanPart, len(x.srcs))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := range x.srcs {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
x.scanSource(uint16(i), files[i], &parts[i], live)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if live {
|
||||
x.publish(parts[0].metas, parts[0].snaps, x.subN.table(), x.msgN.table())
|
||||
return
|
||||
}
|
||||
metas, snaps := mergeParts(parts)
|
||||
x.publish(metas, snaps, x.subN.table(), x.msgN.table())
|
||||
}
|
||||
|
||||
// scanSource indexes one file. In live mode it republishes as it goes.
|
||||
func (x *Index) scanSource(src uint16, f *os.File, part *scanPart, live bool) {
|
||||
s := x.srcs[src]
|
||||
defer f.Close()
|
||||
defer s.done.Store(true)
|
||||
|
||||
br := bufio.NewReaderSize(f, scanBufSize)
|
||||
|
||||
var (
|
||||
line []byte
|
||||
off int64
|
||||
lastPub int
|
||||
lastTS int64
|
||||
curID uint32
|
||||
curR uint32
|
||||
curT uint32
|
||||
haveSnap bool
|
||||
)
|
||||
|
||||
// Estimated row count avoids repeated regrowth on multi-MB files.
|
||||
if s.Size > 0 {
|
||||
part.metas = make([]Meta, 0, int(s.Size/160)+64)
|
||||
}
|
||||
|
||||
// closedSnaps excludes the still-growing last group: its Count is mutated
|
||||
// in place, and readers must never observe a header containing it.
|
||||
closedSnaps := func() []Snapshot {
|
||||
if len(part.snaps) == 0 {
|
||||
return nil
|
||||
}
|
||||
return part.snaps[:len(part.snaps)-1]
|
||||
}
|
||||
|
||||
for {
|
||||
var err error
|
||||
line, err = readLine(br, line)
|
||||
raw := trimEOL(line)
|
||||
|
||||
if len(raw) > 0 {
|
||||
m := parseMeta(raw, off, src, x.subN, x.msgN)
|
||||
if m.Flags&FlagMalformed != 0 {
|
||||
s.bad.Add(1)
|
||||
}
|
||||
// Ordering key: a line without a usable stamp inherits the previous
|
||||
// one and is rendered as unstamped.
|
||||
if m.TS == 0 {
|
||||
m.TS, m.Flags = lastTS, m.Flags|FlagNoTime
|
||||
} else {
|
||||
lastTS = m.TS
|
||||
}
|
||||
idx := uint32(len(part.metas))
|
||||
|
||||
// A stat record whose (run,tick) differs from the previous stat
|
||||
// record opens a new group. Frame is excluded: it is stamped by the
|
||||
// render goroutine and can change mid-snapshot.
|
||||
if x.subN.table()[m.Sub] == SubStat {
|
||||
if !haveSnap || curR != m.Run || curT != m.Tick {
|
||||
part.snaps = append(part.snaps, Snapshot{
|
||||
Head: idx, Run: m.Run, Tick: m.Tick, Frame: m.Frame, Src: src,
|
||||
})
|
||||
curID = uint32(len(part.snaps))
|
||||
curR, curT, haveSnap = m.Run, m.Tick, true
|
||||
m.Flags |= FlagSnapHead
|
||||
}
|
||||
part.snaps[curID-1].Count++
|
||||
m.Snap = curID
|
||||
}
|
||||
part.metas = append(part.metas, m)
|
||||
}
|
||||
|
||||
off += int64(len(line))
|
||||
s.scanned.Store(off)
|
||||
|
||||
if live && len(part.metas)-lastPub >= publishEach {
|
||||
x.publish(part.metas, closedSnaps(), x.subN.table(), x.msgN.table())
|
||||
lastPub = len(part.metas)
|
||||
}
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
s.failure.Store(&scanErr{err})
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
s.scanned.Store(off)
|
||||
}
|
||||
|
||||
// mergeParts interleaves per-source rows by timestamp and renumbers snapshot
|
||||
// ids into one global space. Sources are individually monotonic, so one linear
|
||||
// k-way pass suffices.
|
||||
func mergeParts(parts []scanPart) ([]Meta, []Snapshot) {
|
||||
total := 0
|
||||
base := make([]uint32, len(parts))
|
||||
var snaps []Snapshot
|
||||
for i := range parts {
|
||||
total += len(parts[i].metas)
|
||||
base[i] = uint32(len(snaps))
|
||||
snaps = append(snaps, parts[i].snaps...)
|
||||
}
|
||||
|
||||
out := make([]Meta, 0, total)
|
||||
cur := make([]int, len(parts))
|
||||
for {
|
||||
best := -1
|
||||
for i := range parts {
|
||||
if cur[i] >= len(parts[i].metas) {
|
||||
continue
|
||||
}
|
||||
if best < 0 || parts[i].metas[cur[i]].TS < parts[best].metas[cur[best]].TS {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
break
|
||||
}
|
||||
m := parts[best].metas[cur[best]]
|
||||
cur[best]++
|
||||
if m.Snap != 0 {
|
||||
m.Snap += base[best]
|
||||
if m.Flags&FlagSnapHead != 0 {
|
||||
snaps[m.Snap-1].Head = uint32(len(out))
|
||||
}
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, snaps
|
||||
}
|
||||
|
||||
// readLine appends the next line, terminator included, into buf.
|
||||
func readLine(br *bufio.Reader, buf []byte) ([]byte, error) {
|
||||
buf = buf[:0]
|
||||
for {
|
||||
chunk, err := br.ReadSlice('\n')
|
||||
buf = append(buf, chunk...)
|
||||
if err == bufio.ErrBufferFull {
|
||||
continue
|
||||
}
|
||||
return buf, err
|
||||
}
|
||||
}
|
||||
|
||||
func trimEOL(b []byte) []byte {
|
||||
for len(b) > 0 && (b[len(b)-1] == '\n' || b[len(b)-1] == '\r') {
|
||||
b = b[:len(b)-1]
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// parseMeta extracts the indexed fields from one line. Unparseable lines are
|
||||
// flagged and kept, never dropped.
|
||||
func parseMeta(line []byte, off int64, src uint16, subN, msgN *interner) Meta {
|
||||
m := Meta{Off: off, Len: uint32(len(line)), Src: src, Lvl: LevelBad, Flags: FlagMalformed}
|
||||
i := skipSpace(line, 0)
|
||||
if i >= len(line) || line[i] != '{' {
|
||||
return m
|
||||
}
|
||||
|
||||
ok := eachField(line, i, func(k, v []byte, kind byte) bool {
|
||||
switch string(k) {
|
||||
case "time":
|
||||
if kind == KStr {
|
||||
if ns, good := parseRFC3339Nano(strTok(v)); good {
|
||||
m.TS = ns
|
||||
}
|
||||
}
|
||||
case "level":
|
||||
if kind == KStr {
|
||||
m.Lvl = ParseLevel(strTok(v))
|
||||
}
|
||||
case "sub":
|
||||
if kind == KStr {
|
||||
if id := subN.intern(strTok(v)); id <= 0xffff {
|
||||
m.Sub = uint16(id)
|
||||
}
|
||||
}
|
||||
case "run":
|
||||
m.Run = parseUint32(v)
|
||||
case "tick":
|
||||
m.Tick = parseUint32(v)
|
||||
case "frame":
|
||||
m.Frame = parseUint32(v)
|
||||
case "trace":
|
||||
if kind == KStr && len(v) > 2 {
|
||||
m.Flags |= FlagTrace
|
||||
}
|
||||
case "fields":
|
||||
if kind == KObj {
|
||||
m.Msg = msgN.intern(discriminator(v))
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if ok {
|
||||
m.Flags &^= FlagMalformed
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// discriminator returns fields.msg, falling back to fields.type for records
|
||||
// that omit msg. Returns nil when neither is present; that interns to id 0.
|
||||
func discriminator(fields []byte) []byte {
|
||||
var msg, typ []byte
|
||||
eachField(fields, 0, func(k, v []byte, kind byte) bool {
|
||||
if kind != KStr {
|
||||
return true
|
||||
}
|
||||
switch string(k) {
|
||||
case "msg":
|
||||
msg = strTok(v)
|
||||
return false // msg wins and is always first; stop scanning
|
||||
case "type":
|
||||
typ = strTok(v)
|
||||
}
|
||||
return true
|
||||
})
|
||||
if msg != nil {
|
||||
return msg
|
||||
}
|
||||
return typ
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package logfile
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Level is record severity. LevelTrace..LevelError are ordered for threshold
|
||||
// filtering; LevelProc and LevelBad sit outside that order.
|
||||
type Level uint8
|
||||
|
||||
const (
|
||||
LevelTrace Level = iota
|
||||
LevelDebug
|
||||
LevelInfo
|
||||
LevelWarn
|
||||
LevelError
|
||||
LevelProc
|
||||
LevelBad
|
||||
LevelCount
|
||||
)
|
||||
|
||||
// OrderedCount is the number of threshold-comparable levels.
|
||||
const OrderedCount = int(LevelError) + 1
|
||||
|
||||
var levelName = [LevelCount]string{"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "PROC", "BAD"}
|
||||
var levelInitial = [LevelCount]byte{'T', 'D', 'I', 'W', 'E', 'P', 'B'}
|
||||
|
||||
func (l Level) String() string {
|
||||
if l < LevelCount {
|
||||
return levelName[l]
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
// Initial returns the single character used to toggle the level.
|
||||
func (l Level) Initial() byte {
|
||||
if l < LevelCount {
|
||||
return levelInitial[l]
|
||||
}
|
||||
return '?'
|
||||
}
|
||||
|
||||
// ParseLevel maps a level token to a Level; unknown tokens are LevelBad.
|
||||
func ParseLevel(b []byte) Level {
|
||||
switch string(b) {
|
||||
case "TRACE":
|
||||
return LevelTrace
|
||||
case "DEBUG":
|
||||
return LevelDebug
|
||||
case "INFO":
|
||||
return LevelInfo
|
||||
case "WARN":
|
||||
return LevelWarn
|
||||
case "ERROR":
|
||||
return LevelError
|
||||
case "PROC":
|
||||
return LevelProc
|
||||
}
|
||||
return LevelBad
|
||||
}
|
||||
|
||||
// LevelByInitial resolves a toggle character to a Level.
|
||||
func LevelByInitial(c byte) (Level, bool) {
|
||||
for i := Level(0); i < LevelCount; i++ {
|
||||
if levelInitial[i] == c {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
return LevelBad, false
|
||||
}
|
||||
|
||||
// SubStat marks stat snapshot records.
|
||||
const SubStat = "stat"
|
||||
|
||||
// StampText renders a record's wall-clock stamp, or a placeholder when the
|
||||
// line carried no usable time and inherited one for ordering.
|
||||
func StampText(m Meta) string {
|
||||
if m.Flags&FlagNoTime != 0 {
|
||||
return "--:--:--.---"
|
||||
}
|
||||
return FormatTS(m.TS)
|
||||
}
|
||||
|
||||
// KnownSubs is the closed subsystem vocabulary; ad-hoc taps are also accepted.
|
||||
var KnownSubs = []string{
|
||||
"app", "service", "fsm", "event", "dispatch", "push",
|
||||
"input", "stat", "rec", "lock", "race", "crash",
|
||||
}
|
||||
|
||||
// DurUnit is the time unit implied by a metric key's suffix.
|
||||
type DurUnit uint8
|
||||
|
||||
const (
|
||||
DurNone DurUnit = iota
|
||||
DurNs
|
||||
DurUs
|
||||
DurMs
|
||||
)
|
||||
|
||||
// durKeys maps a key name to the unit it implies. Order matters: the first
|
||||
// match wins, so unit suffixes are tested after the bare duration names.
|
||||
var durKeys = []struct {
|
||||
name string
|
||||
unit DurUnit
|
||||
}{
|
||||
{"timer", DurNs}, {"duration", DurNs}, {"elapsed", DurNs}, {"remaining", DurNs},
|
||||
{"ns", DurNs}, {"us", DurUs}, {"ms", DurMs},
|
||||
}
|
||||
|
||||
// DurationUnit reports the duration unit implied by a field key.
|
||||
func DurationUnit(key string) DurUnit {
|
||||
for _, d := range durKeys {
|
||||
if hasKeySegment(key, d.name) {
|
||||
return d.unit
|
||||
}
|
||||
}
|
||||
return DurNone
|
||||
}
|
||||
|
||||
// hasKeySegment matches name as the whole key or as its trailing '.'/'_'
|
||||
// segment: "elapsed", "max_duration" and "fsm.elapsed" match, "populations"
|
||||
// does not match "ns".
|
||||
func hasKeySegment(key, name string) bool {
|
||||
if len(key) < len(name) || key[len(key)-len(name):] != name {
|
||||
return false
|
||||
}
|
||||
if len(key) == len(name) {
|
||||
return true
|
||||
}
|
||||
c := key[len(key)-len(name)-1]
|
||||
return c == '.' || c == '_'
|
||||
}
|
||||
|
||||
// FormatDuration renders v, expressed in unit u, as a human duration.
|
||||
func FormatDuration(v int64, u DurUnit) string {
|
||||
switch u {
|
||||
case DurNs:
|
||||
return time.Duration(v).String()
|
||||
case DurUs:
|
||||
return (time.Duration(v) * time.Microsecond).String()
|
||||
case DurMs:
|
||||
return (time.Duration(v) * time.Millisecond).String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// FormatTS renders unix nanoseconds as a local wall-clock stamp.
|
||||
func FormatTS(ns int64) string {
|
||||
if ns == 0 {
|
||||
return "--:--:--.---"
|
||||
}
|
||||
return time.Unix(0, ns).Format("15:04:05.000")
|
||||
}
|
||||
|
||||
// Dash renders empty vocabulary values: records without sub, or without a
|
||||
// fields.msg/type discriminator.
|
||||
func Dash(s string) string {
|
||||
if s == "" {
|
||||
return "-"
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package logfile
|
||||
|
||||
// parseRFC3339Nano parses an RFC3339 stamp with optional fractional seconds
|
||||
// into unix nanoseconds. Hand-rolled to keep the index pass allocation-free.
|
||||
func parseRFC3339Nano(b []byte) (int64, bool) {
|
||||
if len(b) < 19 {
|
||||
return 0, false
|
||||
}
|
||||
if b[4] != '-' || b[7] != '-' || (b[10] != 'T' && b[10] != 't') || b[13] != ':' || b[16] != ':' {
|
||||
return 0, false
|
||||
}
|
||||
y, ok1 := numField(b[0:4])
|
||||
mo, ok2 := numField(b[5:7])
|
||||
d, ok3 := numField(b[8:10])
|
||||
h, ok4 := numField(b[11:13])
|
||||
mi, ok5 := numField(b[14:16])
|
||||
s, ok6 := numField(b[17:19])
|
||||
if !ok1 || !ok2 || !ok3 || !ok4 || !ok5 || !ok6 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
i := 19
|
||||
var frac int64
|
||||
if i < len(b) && b[i] == '.' {
|
||||
scale := int64(100000000)
|
||||
for i++; i < len(b) && b[i] >= '0' && b[i] <= '9'; i++ {
|
||||
if scale > 0 {
|
||||
frac += int64(b[i]-'0') * scale
|
||||
scale /= 10
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var offSec int64
|
||||
if i < len(b) {
|
||||
switch b[i] {
|
||||
case 'Z', 'z':
|
||||
case '+', '-':
|
||||
if i+6 > len(b) || b[i+3] != ':' {
|
||||
return 0, false
|
||||
}
|
||||
oh, okh := numField(b[i+1 : i+3])
|
||||
om, okm := numField(b[i+4 : i+6])
|
||||
if !okh || !okm {
|
||||
return 0, false
|
||||
}
|
||||
offSec = int64(oh)*3600 + int64(om)*60
|
||||
if b[i] == '-' {
|
||||
offSec = -offSec
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sec := daysFromCivil(y, mo, d)*86400 + int64(h)*3600 + int64(mi)*60 + int64(s) - offSec
|
||||
return sec*1e9 + frac, true
|
||||
}
|
||||
|
||||
func numField(b []byte) (int, bool) {
|
||||
v := 0
|
||||
for _, c := range b {
|
||||
if c < '0' || c > '9' {
|
||||
return 0, false
|
||||
}
|
||||
v = v*10 + int(c-'0')
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
|
||||
// daysFromCivil returns days since 1970-01-01 (Hinnant's civil-date algorithm).
|
||||
func daysFromCivil(y, m, d int) int64 {
|
||||
if m <= 2 {
|
||||
y--
|
||||
}
|
||||
era := y
|
||||
if era < 0 {
|
||||
era -= 399
|
||||
}
|
||||
era /= 400
|
||||
yoe := y - era*400
|
||||
mp := (m + 9) % 12
|
||||
doy := (153*mp+2)/5 + d - 1
|
||||
doe := yoe*365 + yoe/4 - yoe/100 + doy
|
||||
return int64(era)*146097 + int64(doe) - 719468
|
||||
}
|
||||
Reference in New Issue
Block a user