v0.1.0 initial commit
This commit is contained in:
@@ -0,0 +1,346 @@
|
||||
package logfile
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// Meta is one index row: 48 bytes, pointer-free, holds no line bytes.
|
||||
type Meta struct {
|
||||
Off int64
|
||||
TS int64
|
||||
Len uint32
|
||||
Run uint32
|
||||
Tick uint32
|
||||
Frame uint32
|
||||
Msg uint32 // interned fields.msg
|
||||
Snap uint32 // stat-snapshot group id, 0 = none
|
||||
Sub uint16 // interned sub
|
||||
Src uint16 // source file index
|
||||
Lvl Level
|
||||
Flags uint8
|
||||
}
|
||||
|
||||
// Meta flag bits.
|
||||
const (
|
||||
FlagMalformed uint8 = 1 << iota
|
||||
FlagSnapHead
|
||||
FlagTrace
|
||||
FlagNoTime // TS was inherited from the previous line, for ordering only
|
||||
)
|
||||
|
||||
// Snapshot groups the stat records sharing one (src, run, tick). Members are
|
||||
// not contiguous in the merged view; Count is authoritative, Head locates the
|
||||
// group's first row in the published order.
|
||||
type Snapshot struct {
|
||||
Head uint32
|
||||
Count uint32
|
||||
Run uint32
|
||||
Tick uint32
|
||||
Frame uint32
|
||||
Src uint16
|
||||
}
|
||||
|
||||
// Source is one indexed file.
|
||||
type Source struct {
|
||||
Path string
|
||||
Name string
|
||||
Size int64
|
||||
|
||||
scanned atomic.Int64
|
||||
bad atomic.Int64
|
||||
done atomic.Bool
|
||||
failure atomic.Pointer[scanErr]
|
||||
}
|
||||
|
||||
// Index is the append-only line index over one or more files. The scanner
|
||||
// publishes immutable slice headers; readers load them atomically, so the
|
||||
// render path never locks. A single source publishes incrementally; several
|
||||
// sources publish once, merged by timestamp, so row indices never shift.
|
||||
type Index struct {
|
||||
srcs []*Source
|
||||
|
||||
subN, msgN *interner
|
||||
|
||||
metas atomic.Pointer[[]Meta]
|
||||
snaps atomic.Pointer[[]Snapshot]
|
||||
subs atomic.Pointer[[]string]
|
||||
msgs atomic.Pointer[[]string]
|
||||
}
|
||||
|
||||
type scanErr struct{ err error }
|
||||
|
||||
// The accessors below tolerate a nil receiver: the viewer runs with no file
|
||||
// open until one is chosen, and the render path reads the index every frame.
|
||||
|
||||
// Sources returns the indexed files in source-id order.
|
||||
func (x *Index) Sources() []*Source {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
return x.srcs
|
||||
}
|
||||
|
||||
// SrcCount returns the number of indexed files.
|
||||
func (x *Index) SrcCount() int {
|
||||
if x == nil {
|
||||
return 0
|
||||
}
|
||||
return len(x.srcs)
|
||||
}
|
||||
|
||||
// SrcName resolves a source id to its base filename.
|
||||
func (x *Index) SrcName(id uint16) string {
|
||||
if x == nil || int(id) >= len(x.srcs) {
|
||||
return ""
|
||||
}
|
||||
return x.srcs[id].Name
|
||||
}
|
||||
|
||||
// SrcMark returns the one-character gutter label for a source id.
|
||||
func (x *Index) SrcMark(id uint16) rune {
|
||||
const marks = "123456789abcdefghijklmnopqrstuvwxyz"
|
||||
if int(id) >= len(marks) {
|
||||
return '?'
|
||||
}
|
||||
return rune(marks[id])
|
||||
}
|
||||
|
||||
// Metas returns the currently published index rows.
|
||||
func (x *Index) Metas() []Meta {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
if p := x.metas.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Len returns the number of indexed records.
|
||||
func (x *Index) Len() int { return len(x.Metas()) }
|
||||
|
||||
// Snaps returns the published snapshot groups.
|
||||
func (x *Index) Snaps() []Snapshot {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
if p := x.snaps.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Subs returns the interned subsystem table.
|
||||
func (x *Index) Subs() []string {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
return strTable(x.subs.Load())
|
||||
}
|
||||
|
||||
// Msgs returns the interned fields.msg table.
|
||||
func (x *Index) Msgs() []string {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
return strTable(x.msgs.Load())
|
||||
}
|
||||
|
||||
// SubName resolves an interned sub id.
|
||||
func (x *Index) SubName(id uint16) string {
|
||||
if x == nil {
|
||||
return ""
|
||||
}
|
||||
return lookup(x.subs.Load(), int(id))
|
||||
}
|
||||
|
||||
// MsgName resolves an interned fields.msg id.
|
||||
func (x *Index) MsgName(id uint32) string {
|
||||
if x == nil {
|
||||
return ""
|
||||
}
|
||||
return lookup(x.msgs.Load(), int(id))
|
||||
}
|
||||
|
||||
func strTable(p *[]string) []string {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return *p
|
||||
}
|
||||
|
||||
func lookup(p *[]string, i int) string {
|
||||
t := strTable(p)
|
||||
if i < 0 || i >= len(t) {
|
||||
return ""
|
||||
}
|
||||
return t[i]
|
||||
}
|
||||
|
||||
// Progress reports bytes scanned, total bytes and completion across all sources.
|
||||
func (x *Index) Progress() (scanned, total int64, complete bool) {
|
||||
if x == nil {
|
||||
return 0, 0, true
|
||||
}
|
||||
complete = true
|
||||
for _, s := range x.srcs {
|
||||
scanned += s.scanned.Load()
|
||||
total += s.Size
|
||||
if !s.done.Load() {
|
||||
complete = false
|
||||
}
|
||||
}
|
||||
return scanned, total, complete
|
||||
}
|
||||
|
||||
// Malformed returns the count of unparseable lines seen so far.
|
||||
func (x *Index) Malformed() int64 {
|
||||
if x == nil {
|
||||
return 0
|
||||
}
|
||||
var n int64
|
||||
for _, s := range x.srcs {
|
||||
n += s.bad.Load()
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Err returns the first scan failure, if any.
|
||||
func (x *Index) Err() error {
|
||||
if x == nil {
|
||||
return nil
|
||||
}
|
||||
for _, s := range x.srcs {
|
||||
if p := s.failure.Load(); p != nil {
|
||||
return p.err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SnapshotOf returns the stat group a record belongs to. The group currently
|
||||
// being scanned is not published, so ok=false until it closes.
|
||||
func (x *Index) SnapshotOf(m Meta) (Snapshot, bool) {
|
||||
if x == nil || m.Snap == 0 {
|
||||
return Snapshot{}, false
|
||||
}
|
||||
s := x.Snaps()
|
||||
if int(m.Snap) > len(s) {
|
||||
return Snapshot{}, false
|
||||
}
|
||||
return s[m.Snap-1], true
|
||||
}
|
||||
|
||||
const readWindow = 256 << 10
|
||||
|
||||
// Reader fetches raw line bytes through a per-source sliding window.
|
||||
// Not safe for concurrent use; create one per goroutine.
|
||||
type Reader struct {
|
||||
paths []string
|
||||
win []window
|
||||
}
|
||||
|
||||
type window struct {
|
||||
f *os.File
|
||||
buf []byte
|
||||
base int64
|
||||
n int
|
||||
}
|
||||
|
||||
// NewReader opens an independent handle set on the indexed files. Handles are
|
||||
// opened lazily, so a reader that only touches one source costs one fd.
|
||||
func (x *Index) NewReader() (*Reader, error) {
|
||||
r := &Reader{
|
||||
paths: make([]string, len(x.srcs)),
|
||||
win: make([]window, len(x.srcs)),
|
||||
}
|
||||
for i, s := range x.srcs {
|
||||
r.paths[i] = s.Path
|
||||
r.win[i].base = -1
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Line returns the raw bytes of m, valid until the next call.
|
||||
func (r *Reader) Line(m Meta) ([]byte, error) {
|
||||
n := int(m.Len)
|
||||
if n == 0 || int(m.Src) >= len(r.win) {
|
||||
return nil, nil
|
||||
}
|
||||
w := &r.win[m.Src]
|
||||
if w.f == nil {
|
||||
f, err := os.Open(r.paths[m.Src])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.f, w.buf, w.base = f, make([]byte, readWindow), -1
|
||||
}
|
||||
if n+4096 > len(w.buf) {
|
||||
w.buf = make([]byte, n+4096)
|
||||
w.base = -1
|
||||
}
|
||||
if w.base < 0 || m.Off < w.base || m.Off+int64(n) > w.base+int64(w.n) {
|
||||
base := m.Off &^ 4095
|
||||
got, err := w.f.ReadAt(w.buf, base)
|
||||
if got == 0 && err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.base, w.n = base, got
|
||||
}
|
||||
s := int(m.Off - w.base)
|
||||
if s < 0 || s+n > w.n {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
return w.buf[s : s+n], nil
|
||||
}
|
||||
|
||||
// Close releases every open file handle.
|
||||
func (r *Reader) Close() error {
|
||||
var err error
|
||||
for i := range r.win {
|
||||
if r.win[i].f != nil {
|
||||
if e := r.win[i].f.Close(); e != nil && err == nil {
|
||||
err = e
|
||||
}
|
||||
r.win[i].f = nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// interner maps byte tokens to dense ids. Writer-side only; the id→name table
|
||||
// is published as an immutable snapshot. Locked: sources scan concurrently.
|
||||
type interner struct {
|
||||
mu sync.Mutex
|
||||
ids map[string]uint32
|
||||
tab []string
|
||||
}
|
||||
|
||||
func newInterner() *interner {
|
||||
n := &interner{ids: make(map[string]uint32, 64)}
|
||||
n.intern(nil) // id 0 == ""
|
||||
return n
|
||||
}
|
||||
|
||||
func (n *interner) intern(b []byte) uint32 {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
if id, ok := n.ids[string(b)]; ok {
|
||||
return id
|
||||
}
|
||||
s := string(b)
|
||||
id := uint32(len(n.tab))
|
||||
n.tab = append(n.tab, s)
|
||||
n.ids[s] = id
|
||||
return id
|
||||
}
|
||||
|
||||
// table returns a consistent header over the interned names.
|
||||
func (n *interner) table() []string {
|
||||
n.mu.Lock()
|
||||
defer n.mu.Unlock()
|
||||
return n.tab[:len(n.tab):len(n.tab)]
|
||||
}
|
||||
Reference in New Issue
Block a user