lixen/log
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.1.11 harden logging lifecycle, sanitization, and formatter edge cases
Test / test (push) Canceled after 0s
Details
Test / fuzz (./compat, FuzzParseFormat) (push) Canceled after 0s
Details
Test / fuzz (./formatter, FuzzFormatJSON) (push) Canceled after 0s
Details
Test / fuzz (./formatter, FuzzFormatTextPolicy) (push) Canceled after 0s
Details
Test / fuzz (./sanitizer, FuzzSanitize) (push) Canceled after 0s
Details
Test / fuzz (./sanitizer, FuzzSerializerJSON) (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
lixen committed 2026-09-26 14:25:14 -04:00
1 parent 79a81db3ca
commit 2417e51da1
46 files changed
+1838 -1281
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/test.yml
+66
View File
@@ -0,0 +1,66 @@
name: Test
on:
push:
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: test-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
persist-credentials: false
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
with:
go-version-file: go.mod
cache: false # Standard-library-only module.
- name: Check formatting
run: test -z "$(gofmt -l $(git ls-files '*.go'))"
- run: go vet ./...
- name: Unit, regression, and stress tests with race detection
run: go test -race -count=1 -timeout=3m ./...
- name: Repeated lifecycle and stress tests
run: go test -race -run 'Test(Stress|StartDoesNotReplaceProcessor|ConcurrentOverridesMerge|TimedOutShutdown)' -count=5 -timeout=3m .
- name: Compile supported deployment targets
run: |
GOOS=freebsd GOARCH=amd64 go build ./...
GOOS=js GOARCH=wasm go build ./...
GOOS=windows GOARCH=amd64 go build ./...
fuzz:
runs-on: ubuntu-latest
timeout-minutes: 5
strategy:
fail-fast: false
matrix:
include:
- package: ./sanitizer
target: FuzzSanitize
- package: ./sanitizer
target: FuzzSerializerJSON
- package: ./formatter
target: FuzzFormatJSON
- package: ./formatter
target: FuzzFormatTextPolicy
- package: ./compat
target: FuzzParseFormat
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with:
persist-credentials: false
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
with:
go-version-file: go.mod
cache: false
- name: Bounded fuzzing
run: go test '${{ matrix.package }}' -run '^$' -fuzz '^${{ matrix.target }}$' -fuzztime=15s -parallel=2 -timeout=2m
LICENSE
+1 -1
View File
@@ -1,6 +1,6 @@
BSD 3-Clause License
Copyright (c) 2025, Lixen Wraith
Copyright (c) 2026, Lixen Wraith
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
README.md
+8 -12
View File
@@ -1,6 +1,6 @@
# Log
[![Go](https://img.shields.io/badge/Go-1.25+-00ADD8?style=flat&logo=go)](https://golang.org)
[![Go](https://img.shields.io/badge/Go-1.27.1+-00ADD8?style=flat&logo=go)](https://go.dev)
[![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause)
[![Documentation](https://img.shields.io/badge/Docs-Available-green.svg)](doc/)
@@ -8,10 +8,10 @@ A high-performance, buffered, rotating file logger for Go applications with buil
## Key Features
- **Lock-free async logging** with minimal application impact
- **Buffered async logging** with minimal application impact
- **Automatic file rotation** and disk space management
- **Operational heartbeats** for production monitoring
- **Hot reconfiguration** without data loss
- **Hot reconfiguration** with draining on restart
- **Framework adapters** for gnet v2, fasthttp, Fiber v2
- **Production-grade reliability** with graceful shutdown
@@ -29,7 +29,7 @@ import (
func main() {
// Create and initialize logger
logger := log.NewLogger()
err := logger.ApplyConfigString("directory=/var/log/myapp")
err := logger.ApplyConfigString("directory=/var/log/myapp", "enable_file=true", "format=txt")
if err != nil {
panic(fmt.Errorf("failed to apply logger config: %w", err))
}
@@ -59,21 +59,17 @@ go get github.com/lixenwraith/log
- **[Configuration Builder](doc/builder.md)** - Builder pattern guide
- **[API Reference](doc/api.md)** - Complete API documentation
- **[Logging Guide](doc/logging.md)** - Logging methods and best practices
+ **[Formatting & Sanitization](doc/formatting.md)** - Standalone formatter and sanitizer packages
- **[Formatting & Sanitization](doc/formatting.md)** - Standalone formatter and sanitizer packages
- **[Disk Management](doc/storage.md)** - File rotation and cleanup
- **[Heartbeat Monitoring](doc/heartbeat.md)** - Operational statistics
- **[Compatibility Adapters](doc/adapters.md)** - Framework integrations
- **[Quick Guide](doc/quick-guide_lixenwraith_log.md)** - Quick reference guide
- **[Testing and Performance](doc/testing.md)** - Race, stress, fuzz, and benchmark commands
## Architecture Overview
The logger uses a lock-free, channel-based architecture for high performance:
Applications enqueue records into a bounded channel. A single background processor formats records, writes output, rotates files, and performs cleanup. A short read lock coordinates enqueues with lifecycle changes; producers never wait for queue capacity or output I/O. Full queues drop records and increment counters.
```
Application → Log Methods → Buffered Channel → Background Processor → File/Console
↓ ↓
(non-blocking) (rotation, cleanup, monitoring)
```
Call `ApplyConfig` (or `Build`) and then `Start` before logging. `Stop` drains accepted records and can be followed by `Start`. `Shutdown` also closes the file; after a timeout, retry it to finish cleanup. See the [API reference](doc/api.md) for ownership and concurrency rules.
## Contributing
audit_test.go
+336
View File
@@ -0,0 +1,336 @@
package log
import (
"bytes"
"errors"
"math"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestConfigOwnershipAndRollback(t *testing.T) {
l, dir := newTestLogger(t)
cfg := l.GetConfig()
cfg.Format = "json"
mustNoErr(t, l.ApplyConfig(cfg), "apply snapshot")
cfg.Format = "raw"
equal(t, l.GetConfig().Format, "json", "caller config is not retained")
old := *l.GetConfig()
blocker := filepath.Join(dir, "blocker")
mustNoErr(t, os.WriteFile(blocker, nil, 0600), "create blocker")
mustNoErr(t, os.Mkdir(filepath.Join(dir, "directory"), 0700), "create directory blocker")
for _, path := range []string{filepath.Join(blocker, "child"), dir} {
bad := l.GetConfig()
bad.Directory = path
bad.Name = "directory"
bad.Extension = ""
bad.Format = "txt"
bad.Level = LevelError
mustErr(t, l.ApplyConfig(bad), "failed reconfiguration")
equal(t, *l.GetConfig(), old, "configuration rolled back")
isTrue(t, l.Enabled(LevelInfo), "failed config did not close emit gate")
}
l.Info("survives")
mustNoErr(t, l.Flush(time.Second), "flush old configuration")
contains(t, readLog(t, dir), `"fields":["survives"]`, "formatter rolled back")
}
func TestConfigBounds(t *testing.T) {
for name, mutate := range map[string]func(*Config){
"name traversal": func(c *Config) { c.Name = "../outside" },
"name dot": func(c *Config) { c.Name = ".." },
"extension traversal": func(c *Config) { c.Extension = "../log" },
"NUL": func(c *Config) { c.Name = "bad\x00name" },
"buffer": func(c *Config) { c.BufferSize = math.MaxInt64 },
"size": func(c *Config) { c.MaxSizeKB = math.MaxInt64 },
"flush": func(c *Config) { c.FlushIntervalMs = math.MaxInt64 },
"heartbeat": func(c *Config) { c.HeartbeatIntervalS = math.MaxInt64 },
"retention nan": func(c *Config) { c.RetentionPeriodHrs = math.NaN() },
"retention infinity": func(c *Config) { c.RetentionCheckMins = math.Inf(1) },
"retention overflow": func(c *Config) { c.RetentionPeriodHrs = 1e30 },
"retention underflow": func(c *Config) { c.RetentionCheckMins = 1e-30 },
} {
t.Run(name, func(t *testing.T) { c := DefaultConfig(); mutate(c); mustErr(t, c.Validate(), "validate") })
}
var nilConfig *Config
mustErr(t, nilConfig.Validate(), "nil config")
for _, b := range []*Builder{NewBuilder().MaxSizeMB(math.MaxInt64), NewBuilder().MaxTotalSizeMB(math.MaxInt64), NewBuilder().MinDiskFreeMB(math.MaxInt64)} {
_, err := b.Build()
mustErr(t, err, "MB overflow")
}
l := NewLogger()
isFalse(t, l.Enabled(math.MaxInt64), "closed level gate")
}
func TestConcurrentOverridesMerge(t *testing.T) {
l, _ := newTestLogger(t)
overrides := []string{"trace_depth=2", "level=debug", "format=json", "show_level=false", "show_timestamp=false", "sanitization=txt"}
var wg sync.WaitGroup
start := make(chan struct{})
for _, override := range overrides {
wg.Go(func() { <-start; noErr(t, l.ApplyConfigString(override), override) })
}
close(start)
wg.Wait()
c := l.GetConfig()
equal(t, c.TraceDepth, int64(2), "depth")
equal(t, c.Level, LevelDebug, "level")
equal(t, c.Format, "json", "format")
isFalse(t, c.ShowLevel, "show level")
isFalse(t, c.ShowTimestamp, "show timestamp")
equal(t, c.Sanitization, PolicyTxt, "sanitization")
}
func TestStartDoesNotReplaceProcessor(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
c.EnableConsole = false
mustNoErr(t, l.ApplyConfig(c), "apply")
var launches atomic.Int64
l.SetSpawn(func(fn func()) { launches.Add(1); go fn() })
var wg sync.WaitGroup
for range 64 {
wg.Go(func() { noErr(t, l.Start(), "concurrent start") })
}
wg.Wait()
equal(t, launches.Load(), int64(1), "only one processor")
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
mustNoErr(t, l.ApplyConfig(c), "reset after shutdown")
mustNoErr(t, l.Start(), "start reset logger")
isTrue(t, l.Enabled(LevelInfo), "reset reopens gate")
mustNoErr(t, l.Shutdown(time.Second), "shutdown reset logger")
}
type blockedWriter struct {
entered, release chan struct{}
once sync.Once
}
func (w *blockedWriter) Write(p []byte) (int, error) {
w.once.Do(func() { close(w.entered) })
<-w.release
return len(p), nil
}
func TestTimedOutShutdownKeepsProcessorResources(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
c.Directory = t.TempDir()
c.EnableFile = true
c.EnableConsole = true
mustNoErr(t, l.ApplyConfig(c), "apply")
w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})}
l.state.StdoutWriter.Store(&sink{w: w})
mustNoErr(t, l.Start(), "start")
l.Info("accepted before timeout")
select {
case <-w.entered:
case <-time.After(time.Second):
t.Fatal("processor never reached writer")
}
// Always release the writer even when an assertion fails.
defer func() { close(w.release); noErr(t, l.Shutdown(time.Second), "retry shutdown") }()
errContains(t, l.Shutdown(time.Millisecond), "timeout", "shutdown timeout")
f := l.state.CurrentFile.Load().(*os.File)
_, err := f.Stat()
mustNoErr(t, err, "file still owned by draining processor")
mustErr(t, l.Start(), "restart blocked")
mustErr(t, l.ApplyConfig(c), "resource replacement blocked")
}
func TestAsyncArgumentSnapshot(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
c.EnableConsole = false
c.EnableFile = true
c.Directory = t.TempDir()
c.Format = "json"
mustNoErr(t, l.ApplyConfig(c), "apply")
var run func()
l.SetSpawn(func(fn func()) { run = fn })
mustNoErr(t, l.Start(), "start deferred processor")
args := []any{"original"}
l.Info(args...)
args[0] = "mutated"
fields := map[string]any{"value": "original"}
l.LogStructured(LevelInfo, "message", fields)
fields["value"] = "mutated"
go run()
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
data := readLog(t, c.Directory)
notContains(t, data, "mutated", "snapshotted arguments and map")
equal(t, strings.Count(data, "original"), 2, "original values")
}
func TestFlushReportsSyncError(t *testing.T) {
l, _ := newTestLogger(t)
mustNoErr(t, l.Flush(time.Second), "initial flush")
file := l.state.CurrentFile.Load().(*os.File)
mustNoErr(t, file.Close(), "inject closed file")
err := l.Flush(time.Second)
if !errors.Is(err, os.ErrClosed) {
t.Fatalf("expected sync error, got %v", err)
}
}
func TestConsoleOnlyRetentionIsInert(t *testing.T) {
c := DefaultConfig()
c.EnableConsole = false
c.Directory = t.TempDir()
c.RetentionPeriodHrs = 1
name := filepath.Join(c.Directory, "old.log")
mustNoErr(t, os.WriteFile(name, []byte("keep"), 0600), "write")
old := time.Now().Add(-2 * time.Hour)
mustNoErr(t, os.Chtimes(name, old, old), "set age")
l := NewLogger()
mustNoErr(t, l.ApplyConfig(c), "apply")
l.state.EarliestFileTime.Store(old)
l.handleRetentionCheck()
mustNoErr(t, l.cleanExpiredLogs(old), "explicit cleanup")
data, err := os.ReadFile(name)
mustNoErr(t, err, "file survived")
equal(t, string(data), "keep", "contents")
timers := l.setupProcessingTimers()
defer l.stopProcessingTimers(timers)
if timers.retentionChan != nil {
t.Fatal("console-only retention timer enabled")
}
}
func TestArchiveCollisionBeyondOneThousand(t *testing.T) {
l, dir := newTestLogger(t)
stamp := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
base := "log_" + stamp.Format("060102_150405")
mustNoErr(t, os.Mkdir(filepath.Join(dir, base+".log"), 0700), "directory collision")
for i := 1; i <= 1000; i++ {
mustNoErr(t, os.WriteFile(filepath.Join(dir, base+"_"+itoa(i)+".log"), nil, 0600), "archive")
}
equal(t, l.generateArchiveLogFileName(stamp), base+"_1001.log", "no capped collision reuse")
}
func TestExtensionlessAccountingAndCleanup(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
c.EnableConsole = false
c.EnableFile = true
c.Directory = t.TempDir()
c.Extension = ""
mustNoErr(t, l.ApplyConfig(c), "apply")
defer l.Shutdown()
for _, name := range []string{"old", "other.txt"} {
mustNoErr(t, os.WriteFile(filepath.Join(c.Directory, name), []byte("123"), 0600), "write")
}
size, err := l.getLogDirSize(c.Directory, "")
mustNoErr(t, err, "size")
equal(t, size, int64(3), "extensionless size")
count, err := l.getLogFileCount(c.Directory, "")
mustNoErr(t, err, "count")
equal(t, count, 2, "extensionless count")
mustNoErr(t, l.cleanOldLogs(0), "zero demand does not remove archives")
mustNoErr(t, l.cleanOldLogs(3), "cleanup")
_, err = os.Stat(filepath.Join(c.Directory, "other.txt"))
mustNoErr(t, err, "other extension preserved")
}
func TestTextSanitizesMetadata(t *testing.T) {
var out bytes.Buffer
l := NewLogger()
c := DefaultConfig()
c.Format = "txt"
c.Sanitization = PolicyTxt
c.TimestampFormat = "bad\n\x1b"
c.EnableConsole = true
mustNoErr(t, l.ApplyConfig(c), "apply")
l.SetContextKeys("key\n\x1b")
l.state.StdoutWriter.Store(&sink{w: &out})
mustNoErr(t, l.Start(), "start")
l.LogContext(Context{Tag: "value"}, FlagKV, LevelInfo, 0, "key\n\x1b", "value")
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
equal(t, bytes.Count(out.Bytes(), []byte{'\n'}), 1, "one record line")
notContains(t, out.String(), "\x1b", "no terminal escape")
}
type shortWriter struct{}
func (shortWriter) Write(p []byte) (int, error) { return len(p) - 1, nil }
func TestConsoleWriteFailureAccounting(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
mustNoErr(t, l.ApplyConfig(c), "apply")
l.state.StdoutWriter.Store(&sink{w: shortWriter{}})
mustNoErr(t, l.Start(), "start")
l.Info("short write")
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
equal(t, l.state.TotalDroppedLogs.Load(), uint64(1), "console drop")
equal(t, l.state.DroppedLogs.Load(), uint64(1), "interval drop")
equal(t, l.state.TotalLogsProcessed.Load(), uint64(0), "not processed successfully")
}
func TestDrainFlushIsBounded(t *testing.T) {
// Refill after every write, reproducing a queue which never becomes empty.
// A snapshot drain must still complete after the original pending count.
l := NewLogger()
c := DefaultConfig()
c.Format = "raw"
mustNoErr(t, l.ApplyConfig(c), "apply")
ch := make(chan logRecord, 4)
for range cap(ch) {
ch <- logRecord{Args: []any{"x"}}
}
l.state.StdoutWriter.Store(&sink{w: writerFunc(func(p []byte) (int, error) { ch <- logRecord{Args: []any{"later"}}; return len(p), nil })})
done := make(chan struct{})
go func() { l.drain(ch); close(done) }()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("drain starved on refilled queue")
}
equal(t, l.state.TotalLogsProcessed.Load(), uint64(4), "bounded work")
equal(t, len(ch), 4, "later records left for next iteration")
}
type writerFunc func([]byte) (int, error)
func (fn writerFunc) Write(p []byte) (int, error) { return fn(p) }
func TestReconfigureDrainsOldFileAndFormat(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
c.Directory = t.TempDir()
c.EnableFile = true
c.EnableConsole = true
c.Format = "raw"
c.MaxSizeKB = 1
c.FlushIntervalMs = 1000
mustNoErr(t, l.ApplyConfig(c), "apply")
w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})}
l.state.StdoutWriter.Store(&sink{w: w})
var diagnostics []string
l.SetErrorHandler(func(msg string) { diagnostics = append(diagnostics, msg) })
mustNoErr(t, l.Start(), "start")
payload := strings.Repeat("x", 1500)
l.Info(payload)
<-w.entered
l.Info("old-format-tail")
updated := l.GetConfig()
updated.EnableFile = false
updated.EnableConsole = false
updated.Format = "json"
done := make(chan error, 1)
go func() { done <- l.ApplyConfig(updated) }()
mustEventually(t, time.Second, "stop began", func() bool { return !l.state.Started.Load() })
close(w.release)
mustNoErr(t, <-done, "reconfigure")
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
data := readAllLogs(t, c.Directory)
contains(t, data, payload, "original payload")
contains(t, data, "old-format-tail", "queued record used old format")
equal(t, len(data), len(payload)+len("old-format-tail"), "exact drained bytes")
equal(t, len(diagnostics), 0, "closed current handle after draining rotation")
}
benchmark_test.go
+4 -4
View File
@@ -18,7 +18,7 @@ func BenchmarkLoggerInfo(b *testing.B) {
}
}
// BenchmarkLoggerTxt measures the txt path, which includes quote analysis.
// BenchmarkLoggerTxt measures enqueueing with txt output selected.
func BenchmarkLoggerTxt(b *testing.B) {
logger, _ := newTestLogger(b)
@@ -46,7 +46,7 @@ func BenchmarkLoggerJSON(b *testing.B) {
}
}
// BenchmarkLoggerStructured measures the json.Marshal path for field maps.
// BenchmarkLoggerStructured measures enqueueing and snapshotting structured maps.
func BenchmarkLoggerStructured(b *testing.B) {
logger, _ := newTestLogger(b)
@@ -66,8 +66,8 @@ func BenchmarkLoggerStructured(b *testing.B) {
}
}
// BenchmarkLoggerSanitized measures PolicyTxt overhead on control-free input,
// where the sanitizer takes its no-allocation fast path.
// BenchmarkLoggerSanitized measures enqueueing with PolicyTxt selected;
// sanitizer CPU work runs asynchronously and is benchmarked in its own package.
func BenchmarkLoggerSanitized(b *testing.B) {
logger, _ := newTestLogger(b)
builder.go
+20 -4
View File
@@ -1,6 +1,9 @@
package log
import (
"math"
"slices"
"github.com/lixenwraith/log/sanitizer"
)
@@ -23,7 +26,7 @@ func NewBuilder() *Builder {
// ContextKeys names the record keys for Context values
func (b *Builder) ContextKeys(tag string, vals ...string) *Builder {
b.ctxTag = tag
b.ctxVals = vals
b.ctxVals = slices.Clone(vals)
return b
}
@@ -67,7 +70,7 @@ func (b *Builder) LevelString(level string) *Builder {
return b
}
// Name sets the log level
// Name sets the base filename
func (b *Builder) Name(name string) *Builder {
b.cfg.Name = name
return b
@@ -91,7 +94,7 @@ func (b *Builder) Sanitization(policy sanitizer.PolicyPreset) *Builder {
return b
}
// Extension sets the log level
// Extension sets the filename extension without a leading dot
func (b *Builder) Extension(ext string) *Builder {
b.cfg.Extension = ext
return b
@@ -111,6 +114,10 @@ func (b *Builder) MaxSizeKB(size int64) *Builder {
// MaxSizeMB sets the maximum log file size in MB
func (b *Builder) MaxSizeMB(size int64) *Builder {
if size < 0 || size > math.MaxInt64/(sizeMultiplier*sizeMultiplier) {
b.err = fmtErrorf("size in MB is out of range: %d", size)
return b
}
b.cfg.MaxSizeKB = size * sizeMultiplier
return b
}
@@ -127,7 +134,7 @@ func (b *Builder) HeartbeatLevel(level int64) *Builder {
return b
}
// HeartbeatIntervalS sets the heartbeat monitoring level
// HeartbeatIntervalS sets the heartbeat interval in seconds
func (b *Builder) HeartbeatIntervalS(interval int64) *Builder {
b.cfg.HeartbeatIntervalS = interval
return b
@@ -159,6 +166,10 @@ func (b *Builder) MaxTotalSizeKB(size int64) *Builder {
// MaxTotalSizeMB sets the maximum total size of all log files in MB
func (b *Builder) MaxTotalSizeMB(size int64) *Builder {
if size < 0 || size > math.MaxInt64/(sizeMultiplier*sizeMultiplier) {
b.err = fmtErrorf("size in MB is out of range: %d", size)
return b
}
b.cfg.MaxTotalSizeKB = size * sizeMultiplier
return b
}
@@ -171,6 +182,10 @@ func (b *Builder) MinDiskFreeKB(size int64) *Builder {
// MinDiskFreeMB sets the minimum required free disk space in MB
func (b *Builder) MinDiskFreeMB(size int64) *Builder {
if size < 0 || size > math.MaxInt64/(sizeMultiplier*sizeMultiplier) {
b.err = fmtErrorf("size in MB is out of range: %d", size)
return b
}
b.cfg.MinDiskFreeKB = size * sizeMultiplier
return b
}
@@ -259,6 +274,7 @@ func (b *Builder) EnableConsole(enable bool) *Builder {
// if err == nil {
//
// defer logger.Shutdown()
// if err := logger.Start(); err != nil { return }
// logger.Info("Logger initialized successfully")
//
// }
builder_test.go
-1
View File
@@ -137,4 +137,3 @@ func TestBuilderDefaults(t *testing.T) {
equal(t, cfg.BufferSize, def.BufferSize, "BufferSize")
equal(t, cfg.Sanitization, def.Sanitization, "Sanitization")
}
compat/builder.go
+4 -55
View File
@@ -97,7 +97,7 @@ func (b *Builder) BuildFastHTTP(opts ...FastHTTPOption) (*FastHTTPAdapter, error
return NewFastHTTPAdapter(l, opts...), nil
}
// BuildFiber creates a Fiber v2.54.x adapter
// BuildFiber creates a Fiber v2 adapter
func (b *Builder) BuildFiber(opts ...FiberOption) (*FiberAdapter, error) {
l, err := b.getLogger()
if err != nil {
@@ -112,57 +112,6 @@ func (b *Builder) GetLogger() (*log.Logger, error) {
return b.getLogger()
}
// --- Example Usage ---
//
// The following demonstrates how to integrate lixenwraith/log with gnet, fasthttp, and Fiber
// using a single, shared logger instance
//
// // 1. Create and configure application's main logger
// appLogger := log.NewLogger()
// logCfg := log.DefaultConfig()
// logCfg.Level = log.LevelDebug
// if err := appLogger.ApplyConfig(logCfg); err != nil {
// panic(fmt.Sprintf("failed to configure logger: %v", err))
// }
//
// // 2. Create a builder and provide the existing logger
// builder := compat.NewBuilder().WithLogger(appLogger)
//
// // 3. Build the required adapters
// gnetLogger, err := builder.BuildGnet()
// if err != nil { /* handle error */ }
//
// fasthttpLogger, err := builder.BuildFastHTTP()
// if err != nil { /* handle error */ }
//
// fiberLogger, err := builder.BuildFiber()
// if err != nil { /* handle error */ }
//
// // 4. Configure your servers with the adapters
//
// // For gnet:
// var events gnet.EventHandler // your-event-handler
// // The adapter is passed directly into the gnet options
// go gnet.Run(events, "tcp://:9000", gnet.WithLogger(gnetLogger))
//
// // For fasthttp:
// // The adapter is assigned directly to the server's Logger field
// server := &fasthttp.Server{
// Handler: func(ctx *fasthttp.RequestCtx) {
// ctx.WriteString("Hello, world!")
// },
// Logger: fasthttpLogger,
// }
// go server.ListenAndServe(":8080")
//
// // For Fiber v2.54.x:
// // The adapter is passed to fiber.New() via the config
// app := fiber.New(fiber.Config{
// AppName: "My Application",
// })
// app.UpdateConfig(fiber.Config{
// AppName: "My Application",
// })
// // Note: Set the logger after app creation if needed
// // fiber uses internal logging, adapter can be used in custom middleware
// go app.Listen(":3000")
// Configure and Start the shared Logger before building adapters. WithConfig
// initializes a new Logger but does not Start it; use GetLogger for lifecycle
// control. See doc/adapters.md for framework integration examples.
compat/compat_test.go
-1
View File
@@ -501,4 +501,3 @@ func TestFiberAdapterWriter(t *testing.T) {
eq(t, level, "INFO", "level")
checkFields(t, fields, []any{"msg", "writer output", "source", "fiber"}, "Write")
}
compat/fasthttp.go
+6 -2
View File
@@ -59,16 +59,20 @@ func (a *FastHTTPAdapter) Printf(format string, args ...any) {
}
}
// Log with appropriate level
switch level {
case log.LevelDebug:
a.logger.Debug("msg", msg, "source", "fasthttp")
case log.LevelInfo:
a.logger.Info("msg", msg, "source", "fasthttp")
case log.LevelWarn:
a.logger.Warn("msg", msg, "source", "fasthttp")
case log.LevelError:
a.logger.Error("msg", msg, "source", "fasthttp")
default:
a.logger.Info("msg", msg, "source", "fasthttp")
if a.logger.Enabled(level) {
a.logger.LogContext(log.Context{}, a.logger.Flags(), level, a.logger.GetConfig().TraceDepth,
"msg", msg, "source", "fasthttp")
}
}
}
compat/fiber.go
+37 -1
View File
@@ -9,7 +9,7 @@ import (
)
// FiberAdapter wraps lixenwraith/log.Logger to implement Fiber's CommonLogger interface
// This provides compatibility with Fiber v2.54.x logging requirements
// This provides compatibility with Fiber v2 logging methods
type FiberAdapter struct {
logger *log.Logger
fatalHandler func(msg string) // Customizable fatal behavior
@@ -62,24 +62,36 @@ func (a *FiberAdapter) Trace(v ...any) {
// Debug logs at debug level
func (a *FiberAdapter) Debug(v ...any) {
if !a.logger.Enabled(log.LevelDebug) {
return
}
msg := fmt.Sprint(v...)
a.logger.Debug("msg", msg, "source", "fiber")
}
// Info logs at info level
func (a *FiberAdapter) Info(v ...any) {
if !a.logger.Enabled(log.LevelInfo) {
return
}
msg := fmt.Sprint(v...)
a.logger.Info("msg", msg, "source", "fiber")
}
// Warn logs at warn level
func (a *FiberAdapter) Warn(v ...any) {
if !a.logger.Enabled(log.LevelWarn) {
return
}
msg := fmt.Sprint(v...)
a.logger.Warn("msg", msg, "source", "fiber")
}
// Error logs at error level
func (a *FiberAdapter) Error(v ...any) {
if !a.logger.Enabled(log.LevelError) {
return
}
msg := fmt.Sprint(v...)
a.logger.Error("msg", msg, "source", "fiber")
}
@@ -132,24 +144,36 @@ func (a *FiberAdapter) Tracef(format string, v ...any) {
// Debugf logs at debug level with printf-style formatting
func (a *FiberAdapter) Debugf(format string, v ...any) {
if !a.logger.Enabled(log.LevelDebug) {
return
}
msg := fmt.Sprintf(format, v...)
a.logger.Debug("msg", msg, "source", "fiber")
}
// Infof logs at info level with printf-style formatting
func (a *FiberAdapter) Infof(format string, v ...any) {
if !a.logger.Enabled(log.LevelInfo) {
return
}
msg := fmt.Sprintf(format, v...)
a.logger.Info("msg", msg, "source", "fiber")
}
// Warnf logs at warn level with printf-style formatting
func (a *FiberAdapter) Warnf(format string, v ...any) {
if !a.logger.Enabled(log.LevelWarn) {
return
}
msg := fmt.Sprintf(format, v...)
a.logger.Warn("msg", msg, "source", "fiber")
}
// Errorf logs at error level with printf-style formatting
func (a *FiberAdapter) Errorf(format string, v ...any) {
if !a.logger.Enabled(log.LevelError) {
return
}
msg := fmt.Sprintf(format, v...)
a.logger.Error("msg", msg, "source", "fiber")
}
@@ -192,6 +216,9 @@ func (a *FiberAdapter) Tracew(msg string, keysAndValues ...any) {
// Debugw logs at debug level with structured key-value pairs
func (a *FiberAdapter) Debugw(msg string, keysAndValues ...any) {
if !a.logger.Enabled(log.LevelDebug) {
return
}
fields := make([]any, 0, len(keysAndValues)+4)
fields = append(fields, "msg", msg, "source", "fiber")
fields = append(fields, keysAndValues...)
@@ -200,6 +227,9 @@ func (a *FiberAdapter) Debugw(msg string, keysAndValues ...any) {
// Infow logs at info level with structured key-value pairs
func (a *FiberAdapter) Infow(msg string, keysAndValues ...any) {
if !a.logger.Enabled(log.LevelInfo) {
return
}
fields := make([]any, 0, len(keysAndValues)+4)
fields = append(fields, "msg", msg, "source", "fiber")
fields = append(fields, keysAndValues...)
@@ -208,6 +238,9 @@ func (a *FiberAdapter) Infow(msg string, keysAndValues ...any) {
// Warnw logs at warn level with structured key-value pairs
func (a *FiberAdapter) Warnw(msg string, keysAndValues ...any) {
if !a.logger.Enabled(log.LevelWarn) {
return
}
fields := make([]any, 0, len(keysAndValues)+4)
fields = append(fields, "msg", msg, "source", "fiber")
fields = append(fields, keysAndValues...)
@@ -216,6 +249,9 @@ func (a *FiberAdapter) Warnw(msg string, keysAndValues ...any) {
// Errorw logs at error level with structured key-value pairs
func (a *FiberAdapter) Errorw(msg string, keysAndValues ...any) {
if !a.logger.Enabled(log.LevelError) {
return
}
fields := make([]any, 0, len(keysAndValues)+4)
fields = append(fields, "msg", msg, "source", "fiber")
fields = append(fields, keysAndValues...)
compat/gnet.go
+12
View File
@@ -42,24 +42,36 @@ func WithFatalHandler(handler func(string)) GnetOption {
// Debugf logs at debug level with printf-style formatting
func (a *GnetAdapter) Debugf(format string, args ...any) {
if !a.logger.Enabled(log.LevelDebug) {
return
}
msg := fmt.Sprintf(format, args...)
a.logger.Debug("msg", msg, "source", "gnet")
}
// Infof logs at info level with printf-style formatting
func (a *GnetAdapter) Infof(format string, args ...any) {
if !a.logger.Enabled(log.LevelInfo) {
return
}
msg := fmt.Sprintf(format, args...)
a.logger.Info("msg", msg, "source", "gnet")
}
// Warnf logs at warn level with printf-style formatting
func (a *GnetAdapter) Warnf(format string, args ...any) {
if !a.logger.Enabled(log.LevelWarn) {
return
}
msg := fmt.Sprintf(format, args...)
a.logger.Warn("msg", msg, "source", "gnet")
}
// Errorf logs at error level with printf-style formatting
func (a *GnetAdapter) Errorf(format string, args ...any) {
if !a.logger.Enabled(log.LevelError) {
return
}
msg := fmt.Sprintf(format, args...)
a.logger.Error("msg", msg, "source", "gnet")
}
compat/structured_format_test.go
+54
View File
@@ -0,0 +1,54 @@
package compat
import (
"fmt"
"reflect"
"testing"
)
func TestParseFormatKeepsArgumentAssociation(t *testing.T) {
for _, tc := range []struct {
format string
args []any
want []any
}{
{"prefix %s id=%d", []any{"alice", 42}, []any{"msg", "prefix alice id=42"}},
{"id=%[2]d name=%[1]s", []any{"alice", 42}, []any{"msg", "id=42 name=alice"}},
{"id=%d suffix", []any{42}, []any{"msg", "suffix", "id", 42}},
{"user=%s id=%d", []any{"alice", 42}, []any{"user", "alice", "id", 42}},
{"id=%d %s", []any{42, "tail"}, []any{"msg", "id=42 tail"}},
{"id=%*d", []any{4, 42}, []any{"msg", "id= 42"}},
{"literal %% id=%d", []any{42}, []any{"msg", "literal % id=42"}},
} {
t.Run(tc.format, func(t *testing.T) {
if got := parseFormat(tc.format, tc.args); !reflect.DeepEqual(got, tc.want) {
t.Fatalf("got %#v, want %#v", got, tc.want)
}
})
}
}
func FuzzParseFormat(f *testing.F) {
for _, format := range []string{"id=%d", "%s id=%d", "%%id=%d", "id=%[1]s"} {
f.Add(format, "value")
}
f.Fuzz(func(t *testing.T, format, value string) {
if len(format) > 1024 {
t.Skip()
}
fields := parseFormat(format, []any{value})
if len(fields)%2 != 0 {
t.Fatalf("odd fields %#v", fields)
}
for i := 0; i < len(fields); i += 2 {
if _, ok := fields[i].(string); !ok {
t.Fatalf("non-string key %#v", fields)
}
}
if len(fields) == 2 && fields[0] == "msg" {
if fields[1] != fmt.Sprintf(format, value) {
t.Fatal("fallback changed message")
}
}
})
}
compat/structured_gnet.go
+39 -56
View File
@@ -8,70 +8,41 @@ import (
"github.com/lixenwraith/log"
)
// parseFormat attempts to extract structured fields from printf-style format strings
// Useful for preserving structured logging semantics
// Only unindexed, single-argument verbs are eligible for extraction. Mixed
// printf expressions fall back intact, rather than attributing an argument to
// the wrong key (width, precision and explicit indexes may consume arguments).
var keyValuePattern = regexp.MustCompile(`(\w+)\s*[:=]\s*%[vsdqxXeEfFgGpbcU]`)
func parseFormat(format string, args []any) []any {
// Pattern to detect common structured patterns like "key=%v" or "key: %v"
keyValuePattern := regexp.MustCompile(`(\w+)\s*[:=]\s*%[vsdqxXeEfFgGpbcU]`)
fallback := func() []any { return []any{"msg", fmt.Sprintf(format, args...)} }
matches := keyValuePattern.FindAllStringSubmatchIndex(format, -1)
if len(matches) == 0 || len(matches) > len(args) {
// Fallback to simple message if pattern doesn't match
return []any{"msg", fmt.Sprintf(format, args...)}
if len(matches) == 0 || len(matches) != len(args) {
return fallback()
}
// Build structured fields
fields := make([]any, 0, len(matches)*2+2)
lastEnd := 0
argIndex := 0
for _, match := range matches {
// Add any text before this match as part of the message
if match[0] > lastEnd {
prefix := format[lastEnd:match[0]]
if strings.TrimSpace(prefix) != "" {
if len(fields) == 0 {
fields = append(fields, "msg", strings.TrimSpace(prefix))
}
}
var text []string
end := 0
for i, m := range matches {
gap := format[end:m[0]]
if strings.Contains(gap, "%") {
return fallback()
}
// Extract key name
keyStart := match[2]
keyEnd := match[3]
key := format[keyStart:keyEnd]
// Get corresponding value
if argIndex < len(args) {
fields = append(fields, key, args[argIndex])
argIndex++
if part := strings.TrimSpace(gap); part != "" {
text = append(text, part)
}
lastEnd = match[1]
fields = append(fields, format[m[2]:m[3]], args[i])
end = m[1]
}
// Handle remaining format string and args
if lastEnd < len(format) {
remainingFormat := format[lastEnd:]
remainingArgs := args[argIndex:]
if len(remainingArgs) > 0 {
remaining := fmt.Sprintf(remainingFormat, remainingArgs...)
if strings.TrimSpace(remaining) != "" {
if len(fields) == 0 {
fields = append(fields, "msg", strings.TrimSpace(remaining))
} else {
// Append to existing message
for i := 0; i < len(fields); i += 2 {
if fields[i] == "msg" {
fields[i+1] = fmt.Sprintf("%v %s", fields[i+1], strings.TrimSpace(remaining))
break
}
}
}
}
}
tail := format[end:]
if strings.Contains(tail, "%") {
return fallback()
}
if part := strings.TrimSpace(tail); part != "" {
text = append(text, part)
}
if len(text) > 0 {
fields = append([]any{"msg", strings.Join(text, " ")}, fields...)
}
return fields
}
@@ -91,6 +62,9 @@ func NewStructuredGnetAdapter(logger *log.Logger, opts ...GnetOption) *Structure
// Debugf logs with structured field extraction
func (a *StructuredGnetAdapter) Debugf(format string, args ...any) {
if !a.logger.Enabled(log.LevelDebug) {
return
}
if a.extractFields {
fields := parseFormat(format, args)
a.logger.Debug(append(fields, "source", "gnet")...)
@@ -101,6 +75,9 @@ func (a *StructuredGnetAdapter) Debugf(format string, args ...any) {
// Infof logs with structured field extraction
func (a *StructuredGnetAdapter) Infof(format string, args ...any) {
if !a.logger.Enabled(log.LevelInfo) {
return
}
if a.extractFields {
fields := parseFormat(format, args)
a.logger.Info(append(fields, "source", "gnet")...)
@@ -111,6 +88,9 @@ func (a *StructuredGnetAdapter) Infof(format string, args ...any) {
// Warnf logs with structured field extraction
func (a *StructuredGnetAdapter) Warnf(format string, args ...any) {
if !a.logger.Enabled(log.LevelWarn) {
return
}
if a.extractFields {
fields := parseFormat(format, args)
a.logger.Warn(append(fields, "source", "gnet")...)
@@ -121,6 +101,9 @@ func (a *StructuredGnetAdapter) Warnf(format string, args ...any) {
// Errorf logs with structured field extraction
func (a *StructuredGnetAdapter) Errorf(format string, args ...any) {
if !a.logger.Enabled(log.LevelError) {
return
}
if a.extractFields {
fields := parseFormat(format, args)
a.logger.Error(append(fields, "source", "gnet")...)
config.go
+45
View File
@@ -2,6 +2,7 @@ package log
import (
"fmt"
"math"
"strconv"
"strings"
"time"
@@ -117,11 +118,24 @@ func (c *Config) Clone() *Config {
// Validate performs validation on the configuration
func (c *Config) Validate() error {
if c == nil {
return fmtErrorf("configuration cannot be nil")
}
// String validations
if strings.TrimSpace(c.Name) == "" {
return fmtErrorf("log name cannot be empty")
}
if c.Name == "." || c.Name == ".." || strings.ContainsAny(c.Name, "/\\\x00") || strings.ContainsAny(c.Extension, "/\\\x00") {
return fmtErrorf("name and extension must be filename components, without path separators or NUL")
}
if c.EnableFile && strings.TrimSpace(c.Directory) == "" {
return fmtErrorf("directory cannot be empty when file output is enabled")
}
if c.Level == levelOff {
return fmtErrorf("level is reserved for disabling emission")
}
if c.Format != "txt" && c.Format != "json" && c.Format != "raw" {
return fmtErrorf("invalid format: '%s' (use txt, json, or raw)", c.Format)
}
@@ -150,6 +164,14 @@ func (c *Config) Validate() error {
return fmtErrorf("buffer_size must be positive: %d", c.BufferSize)
}
// Bound allocation before make(chan): one million records already reserves
// over 100 MiB and is well beyond the default queue capacity.
if c.BufferSize > 1<<20 {
return fmtErrorf("buffer_size cannot exceed 1048576 records")
}
if c.MaxSizeKB > math.MaxInt64/sizeMultiplier || c.MaxTotalSizeKB > math.MaxInt64/sizeMultiplier || c.MinDiskFreeKB > math.MaxInt64/sizeMultiplier {
return fmtErrorf("size limits overflow byte counts")
}
if c.MaxSizeKB < 0 || c.MaxTotalSizeKB < 0 || c.MinDiskFreeKB < 0 {
return fmtErrorf("size limits cannot be negative")
}
@@ -159,6 +181,27 @@ func (c *Config) Validate() error {
return fmtErrorf("interval settings must be positive")
}
for _, ms := range []int64{c.FlushIntervalMs, c.DiskCheckIntervalMs, c.MinCheckIntervalMs, c.MaxCheckIntervalMs} {
if ms > math.MaxInt64/int64(time.Millisecond) {
return fmtErrorf("interval overflows time.Duration")
}
}
if c.FlushIntervalMs > math.MaxInt64/(2*int64(time.Millisecond)) {
return fmtErrorf("flush interval overflows default stop timeout")
}
if c.HeartbeatIntervalS > math.MaxInt64/int64(time.Second) {
return fmtErrorf("heartbeat interval overflows time.Duration")
}
for _, setting := range []struct{ value, unit float64 }{
{c.RetentionPeriodHrs, float64(time.Hour)},
{c.RetentionCheckMins, float64(time.Minute)},
} {
ns := setting.value * setting.unit
if math.IsNaN(ns) || math.IsInf(ns, 0) || ns >= float64(math.MaxInt64) || setting.value > 0 && ns < 1 {
return fmtErrorf("retention settings must be finite and representable as time.Duration")
}
}
if c.TraceDepth < 0 || c.TraceDepth > 10 {
return fmtErrorf("trace_depth must be between 0 and 10: %d", c.TraceDepth)
}
@@ -381,6 +424,8 @@ func configRequiresRestart(oldCfg, newCfg *Config) bool {
if oldCfg.FlushIntervalMs != newCfg.FlushIntervalMs ||
oldCfg.DiskCheckIntervalMs != newCfg.DiskCheckIntervalMs ||
oldCfg.EnableAdaptiveInterval != newCfg.EnableAdaptiveInterval ||
oldCfg.MinCheckIntervalMs != newCfg.MinCheckIntervalMs ||
oldCfg.MaxCheckIntervalMs != newCfg.MaxCheckIntervalMs ||
oldCfg.HeartbeatIntervalS != newCfg.HeartbeatIntervalS ||
oldCfg.HeartbeatLevel != newCfg.HeartbeatLevel ||
oldCfg.RetentionCheckMins != newCfg.RetentionCheckMins ||
config_test.go
-1
View File
@@ -190,4 +190,3 @@ func TestConcurrentApplyConfig(t *testing.T) {
return strings.Contains(readLog(t, tmpDir), "after concurrent config")
})
}
doc/adapters.md
+75 -319
View File
@@ -1,359 +1,112 @@
# Compatibility Adapters
# Compatibility adapters
Guide to using lixenwraith/log with popular Go networking frameworks through compatibility adapters.
The `compat` package provides logging adapters for gnet v2, fasthttp, and Fiber v2. Framework dependencies are not imported by this module; applications supply them.
## Overview
## Shared logger setup
The `compat` package provides adapters that allow the lixenwraith/log logger to work seamlessly with:
- **gnet v2**: High-performance event-driven networking framework
- **fasthttp**: Fast HTTP implementation
### Features
- Full interface compatibility
- Preserves structured logging
- Configurable behavior
- Shared logger instances
- Optional field extraction
## gnet Adapter
### Basic Usage
Configure and start the logger before passing an adapter to a framework:
```go
import (
"github.com/lixenwraith/log"
"github.com/lixenwraith/log/compat"
"github.com/panjf2000/gnet/v2"
)
// Create logger
logger := log.NewLogger()
cfg := log.DefaultConfig()
cfg.Directory = "/var/log/gnet"
logger.ApplyConfig(cfg)
defer logger.Shutdown()
// Create adapter
adapter := compat.NewGnetAdapter(logger)
// Use with gnet
gnet.Run(eventHandler, "tcp://127.0.0.1:9000",
gnet.WithLogger(adapter),
)
```
### gnet Interface Implementation
The adapter implements all gnet logger methods:
```go
type GnetAdapter struct {
logger *log.Logger
logger, err := log.NewBuilder().
Directory("/var/log/service").
EnableFile(true).
Format("json").
LevelString("info").
Build()
if err != nil { return err }
if err := logger.Start(); err != nil {
_ = logger.Shutdown()
return err
}
defer logger.Shutdown(5 * time.Second)
// Methods implemented:
// - Debugf(format string, args ...any)
// - Infof(format string, args ...any)
// - Warnf(format string, args ...any)
// - Errorf(format string, args ...any)
// - Fatalf(format string, args ...any)
```
### Custom Fatal Behavior
Override default fatal handling:
```go
adapter := compat.NewGnetAdapter(logger,
compat.WithFatalHandler(func(msg string) {
// Custom cleanup
saveApplicationState()
notifyOperations(msg)
gracefulShutdown()
os.Exit(1)
}),
)
```
### Complete gnet Example
```go
type echoServer struct {
gnet.BuiltinEventEngine
logger gnet.Logger
}
func (es *echoServer) OnBoot(eng gnet.Engine) gnet.Action {
es.logger.Infof("Server started on %s", eng.Addrs)
return gnet.None
}
func (es *echoServer) OnTraffic(c gnet.Conn) gnet.Action {
buf, _ := c.Next(-1)
es.logger.Debugf("Received %d bytes from %s", len(buf), c.RemoteAddr())
c.Write(buf)
return gnet.None
}
func main() {
logger := log.NewLogger()
cfg := log.DefaultConfig()
cfg.Directory = "/var/log/gnet"
cfg.Format = "json"
cfg.BufferSize = 2048
logger.ApplyConfig(cfg)
defer logger.Shutdown()
adapter := compat.NewGnetAdapter(logger)
gnet.Run(
&echoServer{logger: adapter},
"tcp://127.0.0.1:9000",
gnet.WithMulticore(true),
gnet.WithLogger(adapter),
)
}
```
## fasthttp Adapter
### Basic Usage
```go
import (
"github.com/lixenwraith/log"
"github.com/lixenwraith/log/compat"
"github.com/valyala/fasthttp"
)
// Create logger
logger := log.NewLogger()
cfg := log.DefaultConfig()
cfg.Directory = "/var/log/fasthttp"
logger.ApplyConfig(cfg)
defer logger.Shutdown()
// Create adapter
adapter := compat.NewFastHTTPAdapter(logger)
// Configure server
server := &fasthttp.Server{
Handler: requestHandler,
Logger: adapter,
}
```
### Level Detection
The adapter automatically detects log levels from message content:
```go
// Default detection rules:
// - Contains "error", "failed", "fatal", "panic" → ERROR
// - Contains "warn", "warning", "deprecated" → WARN
// - Contains "debug", "trace" → DEBUG
// - Otherwise → INFO
```
### Custom Level Detection
```go
adapter := compat.NewFastHTTPAdapter(logger,
compat.WithDefaultLevel(log.LevelInfo),
compat.WithLevelDetector(func(msg string) int64 {
// Custom detection logic
if strings.Contains(msg, "CRITICAL") {
return log.LevelError
}
if strings.Contains(msg, "performance") {
return log.LevelWarn
}
// Return 0 to use the adapter's default log level (log.LevelInfo by default)
return 0
}),
)
```
## Builder Pattern
### Using Existing Logger (Recommended)
Share a configured logger across adapters:
```go
// Create and configure your main logger
logger := log.NewLogger()
cfg := log.DefaultConfig()
cfg.Level = log.LevelDebug
logger.ApplyConfig(cfg)
logger.Start()
defer logger.Shutdown()
// Create builder with existing logger
builder := compat.NewBuilder().WithLogger(logger)
// Build adapters
gnetAdapter, _ := builder.BuildGnet()
gnetAdapter, err := builder.BuildGnet()
if err != nil { return err }
fasthttpAdapter, _ := builder.BuildFastHTTP()
fastHTTPAdapter, err := builder.BuildFastHTTP()
if err != nil { return err }
fiberAdapter, err := builder.BuildFiber()
if err != nil { return err }
```
### Creating New Logger
Alternatively, `compat.NewBuilder().WithConfig(cfg)` creates an initialized logger on the first build. Retrieve it with `GetLogger()` and call `Start()` explicitly. `WithLogger` takes precedence over `WithConfig`. Builders and option application are single-goroutine operations; completed adapters may be shared with a shared Logger.
Let the builder create a logger with config:
## gnet
`GnetAdapter` implements `Debugf`, `Infof`, `Warnf`, `Errorf`, and `Fatalf`. Pass it through `gnet.WithLogger(adapter)`:
```go
// Option 1: With custom config
cfg := log.DefaultConfig()
cfg.Directory = "/var/log/app"
builder := compat.NewBuilder().WithConfig(cfg)
// Option 2: Default config (created on first build)
builder := compat.NewBuilder()
if err != nil { return err }
// Build adapters
gnetAdapter, _ := builder.BuildGnet()
logger, _ := builder.GetLogger() // Retrieve for direct use
adapter := compat.NewGnetAdapter(logger)
err := gnet.Run(eventHandler, "tcp://127.0.0.1:9000", gnet.WithLogger(adapter))
```
### Structured gnet Adapter
Ordinary messages include `msg` and `source` (`gnet`) as positional arguments. With JSON output, these appear in the `fields` array. Use the logger's `LogStructured` or `LogContext` with `FlagKV` directly if a keyed JSON object is required.
Extract fields from printf-style formats:
### Structured field extraction
```go
structuredAdapter, _ := builder.BuildStructuredGnet()
// "client=%s port=%d" → {"client": "...", "port": ...}
```
## Structured Logging
### Field Extraction
Structured adapters can extract fields from printf-style formats:
```go
// Regular adapter output:
// "client=192.168.1.1 port=8080"
// Structured adapter output:
// {"client": "192.168.1.1", "port": 8080, "source": "gnet"}
```
### Pattern Detection
The structured adapter recognizes patterns like:
- `key=%v`
- `key: %v`
- `key = %v`
`NewStructuredGnetAdapter` extracts simple `key=%v` or `key: %v` expressions with unindexed, single-argument verbs:
```go
adapter := compat.NewStructuredGnetAdapter(logger)
// These will extract structured fields:
adapter.Infof("client=%s port=%d", "192.168.1.1", 8080)
// → {"client": "192.168.1.1", "port": 8080}
adapter.Errorf("user: %s, error: %s", "john", "auth failed")
// → {"user": "john", "error": "auth failed"}
// These remain as messages:
adapter.Infof("Connected to server")
// → {"msg": "Connected to server"}
// JSON fields: ["client", "192.168.1.1", "port", 8080, "source", "gnet"]
```
### Integration Examples
Text outside the expressions is retained in `msg`. Formats mixing unrelated conversions, argument indexes, width/precision, escaped percent signs, or a different argument count fall back to a complete `fmt.Sprintf` message. Extraction never guesses which argument belongs to a key.
#### Microservice with Both Frameworks
## fasthttp
`FastHTTPAdapter` implements the `Printf` method accepted by `fasthttp.Server.Logger`:
```go
type Service struct {
gnetAdapter *compat.GnetAdapter
fasthttpAdapter *compat.FastHTTPAdapter
logger *log.Logger
}
func NewService() (*Service, error) {
// Create and configure logger
logger := log.NewLogger()
cfg := log.DefaultConfig()
cfg.Directory = "/var/log/service"
cfg.Format = "json"
cfg.HeartbeatLevel = 2
if err := logger.ApplyConfig(cfg); err != nil {
return nil, err
}
if err := logger.Start(); err != nil {
return nil, err
}
// Create builder with the logger
builder := compat.NewBuilder().WithLogger(logger)
// Build adapters
gnetAdapter, err := builder.BuildGnet()
if err != nil {
logger.Shutdown()
return nil, err
}
fasthttpAdapter, err := builder.BuildFastHTTP()
if err != nil {
logger.Shutdown()
return nil, err
}
return &Service{
gnetAdapter: gnetAdapter,
fasthttpAdapter: fasthttpAdapter,
logger: logger,
}, nil
adapter := compat.NewFastHTTPAdapter(logger)
server := &fasthttp.Server{
Handler: requestHandler,
Logger: adapter,
}
```
#### Middleware Integration
The default detector performs a case-insensitive substring check:
| Message contains | Level |
|---|---|
| `error`, `failed`, `fatal`, or `panic` | Error |
| `warn` or `deprecated` | Warn |
| `debug` or `trace` | Debug |
| Otherwise | Configured default (Info initially) |
This heuristic does not interpret HTTP status numbers. Disable it with `WithLevelDetector(nil)`, or provide a custom function. A detector result of zero selects the configured default; nonzero custom levels are preserved.
```go
// gnet middleware
func loggingMiddleware(adapter *compat.GnetAdapter) gnet.EventHandler {
return func(c gnet.Conn) gnet.Action {
start := time.Now()
addr := c.RemoteAddr()
// Process connection
action := next(c)
adapter.Infof("conn_duration=%v remote=%s action=%v",
time.Since(start), addr, action)
return action
}
}
// fasthttp middleware
func requestLogger(adapter *compat.FastHTTPAdapter) fasthttp.RequestHandler {
return func(ctx *fasthttp.RequestCtx) {
start := time.Now()
// Process request
next(ctx)
// Adapter will detect level from status
adapter.Printf("method=%s path=%s status=%d duration=%v",
ctx.Method(), ctx.Path(),
ctx.Response.StatusCode(),
time.Since(start))
}
}
adapter := compat.NewFastHTTPAdapter(logger,
compat.WithDefaultLevel(log.LevelWarn),
compat.WithLevelDetector(nil),
)
```
## Fiber
`FiberAdapter` provides plain, printf-style (`Infof`), and key/value (`Infow`) logging methods, and implements `io.Writer` through `Write([]byte)`. Trace methods currently map to Debug with a `level`, `trace` marker. Use it directly in Fiber middleware:
```go
adapter := compat.NewFiberAdapter(logger)
app.Use(func(c *fiber.Ctx) error {
err := c.Next()
adapter.Infow("request", "method", c.Method(), "path", c.Path(),
"status", c.Response().StatusCode())
return err
})
```
## Fatal and panic behavior
Fatal methods enqueue an Error record, attempt a flush for 100 ms, and invoke the configured handler. Defaults call `os.Exit(1)` for fatal and `panic(msg)` for Fiber panic methods. These effects still occur when the log level suppresses the record. A timed-out or failed flush cannot guarantee that the final message was written.
Override behavior with `WithFatalHandler`, `WithFiberFatalHandler`, or `WithFiberPanicHandler`. A nil handler disables that action. Shut down the application/framework before the shared logger so shutdown messages can still be recorded.
### Simple integration example suite
Below simple client and server examples can be used to test the basic functionality of the adapters. They are not included in the package to avoid dependency creep.
These client and server examples can be used to test the basic functionality of the adapters. They are not included in the package to avoid dependency creep.
#### gnet server
@@ -391,6 +144,7 @@ func main() {
// Minimal logger config
logger, err := log.NewBuilder().
Directory("./logs_gnet").
EnableFile(true).
Format("json").
LevelString("info").
HeartbeatLevel(0).
@@ -453,6 +207,7 @@ func main() {
// Minimal logger config
logger, err := log.NewBuilder().
Directory("./logs_fasthttp").
EnableFile(true).
Format("json").
LevelString("info").
HeartbeatLevel(0).
@@ -521,6 +276,7 @@ func main() {
// Minimal logger config
logger, err := log.NewBuilder().
Directory("./logs_fiber").
EnableFile(true).
Format("json").
LevelString("info").
HeartbeatLevel(0).
doc/api.md
+27 -9
View File
@@ -37,7 +37,7 @@ Applies a validated configuration to the logger. This is the recommended method
```go
logger := log.NewLogger()
cfg := log.GetConfig()
cfg := log.DefaultConfig()
cfg.Level = log.LevelDebug
cfg.Directory = "/var/log/app"
err := logger.ApplyConfig(cfg)
@@ -66,7 +66,7 @@ err := logger.ApplyConfigString("directory=/var/log/app", "name=app")
## Logging Methods
All logging methods accept variadic arguments, typically used as key-value pairs for structured logging.
Logging methods accept variadic arguments. Ordinary methods preserve them as positional values (a JSON array in JSON mode). Use `LogContext` with `FlagKV` for a keyed object, or `LogStructured` for a message and nested fields.
### Debug
@@ -217,7 +217,7 @@ logger.Log("Checkpoint reached", "step", 5)
func (l *Logger) Message(args ...any)
```
Logs raw message without timestamp or level.
Logs using the configured format and sanitization, without timestamp or level.
**Example:**
```go
@@ -239,13 +239,21 @@ logger.LogTrace(2, "Function boundary", "entering", true)
## Control Methods
### Start and Stop
`Start() error` begins processing after configuration; repeated starts are no-ops. `Stop(timeout ...time.Duration) error` closes emission, drains accepted records, and joins the processor. A timeout does not permit a second processor to start while the first still owns resources. Retry `Stop` or `Shutdown` to wait again. Lifecycle timeouts bound the processor join after acquiring the lifecycle lock.
### Context and fast filtering
`Enabled(level int64) bool` checks the current emit gate before constructing expensive arguments. `SetLevel(level int64)` updates the threshold without restarting. `SetContextKeys(tag string, vals ...string)` names a tag and up to three counters; `LogContext(ctx Context, flags, level, depth int64, args ...any)` stamps them onto a record. `Flags()` returns display defaults suitable for combining with `FlagKV`.
### Shutdown
```go
func (l *Logger) Shutdown(timeout ...time.Duration) error
```
Gracefully shuts down the logger, attempting to flush pending logs.
Gracefully drains accepted records and closes the log file. If the processor times out, its resources remain open; retry `Shutdown` to finish cleanup. A fresh `ApplyConfig` permits reuse after shutdown.
**Parameters:**
- `timeout`: Optional timeout duration (defaults to 2x flush interval)
@@ -267,7 +275,7 @@ if err != nil {
func (l *Logger) Flush(timeout time.Duration) error
```
Explicitly triggers a sync of the current log file buffer to disk. Uses **barrier semantics**: it guarantees that all log records enqueued *before* the `Flush` call are fully processed and formatted before the disk sync occurs and confirmation is returned.
Drains records enqueued before the request and syncs the file. Later arrivals do not extend the drain indefinitely. The timeout covers both queueing and confirmation. A successful return reports successful sync; previously dropped records cannot be recovered. A concurrent stop may return a stopped-processor error.
**Parameters:**
- `timeout`: Maximum time to wait for flush completion
@@ -286,6 +294,7 @@ err := logger.Flush(1 * time.Second)
```go
const (
LevelTrace int64 = -8
LevelDebug int64 = -4
LevelInfo int64 = 0
LevelWarn int64 = 4
@@ -316,7 +325,7 @@ func Level(levelStr string) (int64, error)
Converts level string to numeric constant.
**Parameters:**
- `levelStr`: Level name ("debug", "info", "warn", "error", "proc", "disk", "sys")
- `levelStr`: Level name ("trace", "debug", "info", "warn", "error", "proc", "disk", "sys")
**Returns:**
- `int64`: Numeric level value
@@ -337,6 +346,7 @@ const (
FlagStructuredJSON = formatter.FlagStructuredJSON // Structured JSON output
FlagNoTimestamp = formatter.FlagNoTimestamp // Suppress timestamp
FlagNoLevel = formatter.FlagNoLevel // Suppress level
FlagKV = formatter.FlagKV // Alternating string keys and values
FlagDefault = formatter.FlagDefault // Default flags
)
```
@@ -348,9 +358,9 @@ Control output formatting behavior. These flags are re-exported from the formatt
```go
const (
PolicyRaw = sanitizer.PolicyRaw // No sanitization
PolicyJSON = sanitizer.PolicyJSON // JSON-safe output
PolicyJSON = sanitizer.PolicyJSON // Content control-character escapes
PolicyTxt = sanitizer.PolicyTxt // Text file safe
PolicyShell = sanitizer.PolicyShell // Shell-safe output
PolicyShell = sanitizer.PolicyShell // Lossy filter, not shell quoting
)
```
@@ -376,7 +386,11 @@ The logger returns errors prefixed with "log: " for easy identification:
## Thread Safety
All public methods are thread-safe and can be called concurrently from multiple goroutines. The logger uses atomic operations and channels to ensure safe concurrent access without locks in the critical path.
Logger methods support concurrent calls. Lifecycle and configuration changes are serialized. Enqueues hold a short read lock to prevent records being stranded during a stop. Logger, Formatter, Sanitizer, and Builder instances must not be copied after use.
`ApplyConfig` takes a snapshot. Emission copies the argument slice, and `LogStructured` also copies the top-level fields map. Referenced values (byte slices, nested maps/slices, pointers, Stringers, and custom marshalers) remain caller-owned: keep them immutable until a successful `Flush` or `Stop`, or pass an application-created snapshot. Do not mutate configuration while an `ApplyConfig` call is reading it.
Builders and standalone formatter/sanitizer configuration are not synchronized; finish configuration before sharing. `SetSpawn` callbacks must launch the function asynchronously and return promptly. Error handlers must return promptly and must not invoke lifecycle/configuration methods or `Flush`.
### Usage Pattern Example
@@ -396,6 +410,10 @@ func NewService() (*Service, error) {
return nil, fmt.Errorf("logger init: %w", err)
}
if err := logger.Start(); err != nil {
_ = logger.Shutdown()
return nil, err
}
return &Service{logger: logger}, nil
}
doc/builder.md
+3 -3
View File
@@ -14,7 +14,7 @@ builder := log.NewBuilder()
## Builder Methods
All builder methods return `*Builder` for chaining. Errors are accumulated and returned by `Build()`.
All builder methods return `*Builder` for chaining. Validation errors are returned by `Build()`. Builders are not safe for concurrent mutation; each built logger owns a configuration snapshot.
### Common Methods
@@ -25,7 +25,7 @@ All builder methods return `*Builder` for chaining. Errors are accumulated and r
| `Name(name string)` | `name`: Base filename | Sets log file base name |
| `Directory(dir string)` | `dir`: Path | Sets log directory |
| `Format(format string)` | `format`: Output format | Sets format ("txt", "json", "raw") |
| `Sanitization(policy string)` | `policy`: Sanitization policy | Sets policy ("txt", "json", "raw", "shell") |
| `Sanitization(policy sanitizer.PolicyPreset)` | `policy`: Sanitization policy | Sets policy ("txt", "json", "raw", "shell") |
| `Extension(ext string)` | `ext`: File extension | Sets log file extension |
| `BufferSize(size int64)` | `size`: Buffer size | Sets channel buffer size |
| `MaxSizeKB(size int64)` | `size`: Size in KB | Sets max file size in KB |
@@ -36,7 +36,7 @@ All builder methods return `*Builder` for chaining. Errors are accumulated and r
| `MinDiskFreeMB(size int64)` | `size`: Size in MB | Sets minimum required free disk space in MB |
| `EnableConsole(enable bool)` | `enable`: Boolean | Enables console output |
| `EnableFile(enable bool)` | `enable`: Boolean | Enables file output |
| `ConsoleTarget(target string)` | `target`: "stdout"/"stderr" | Sets console output target |
| `ConsoleTarget(target string)` | `target`: "stdout"/"stderr"/"split" | Sets console output target |
| `ShowTimestamp(show bool)` | `show`: Boolean | Controls timestamp display |
| `ShowLevel(show bool)` | `show`: Boolean | Controls log level display |
| `TimestampFormat(format string)` | `format`: Time format | Sets timestamp format (Go time format) |
doc/configuration.md
+17 -20
View File
@@ -17,39 +17,36 @@ logger := log.NewLogger()
Direct struct configuration using the Config struct, or key-value overrides:
```go
logger := log.NewLogger() // logger instance created with DefaultConfig (using default values)
// Note: with default config, logs only go to stderr (file output disabled by default)
logger.Start() // Required before logging
logger.Info("info raw log record written to stderr")
// Directly change config struct
cfg := log.GetConfig()
logger := log.NewLogger()
cfg := log.DefaultConfig()
cfg.Level = log.LevelDebug
cfg.Name = "myapp"
cfg.Directory = "/var/log/myapp"
cfg.EnableFile = true
cfg.Format = "json"
cfg.MaxSizeKB = 100
err := logger.ApplyConfig(cfg)
if err := logger.ApplyConfig(cfg); err != nil { return err }
if err := logger.Start(); err != nil { return err }
defer logger.Shutdown()
logger.Info("application started")
logger.Info("info json log record written to /var/log/myapp/myapp.log")
// Override values with key-value string
err = logger.ApplyConfigString(
"directory=/var/log/",
"extension=txt"
"format=txt")
logger.Info("info txt log record written to /var/log/myapp.txt")
// Apply overrides to the current configuration atomically.
if err := logger.ApplyConfigString("extension=txt", "format=txt"); err != nil {
return err
}
```
`ApplyConfig` copies the supplied configuration. The caller may reuse or modify it after the call returns. Failed validation, directory creation, or file opening leaves the active configuration unchanged. Changes requiring a restart drain accepted records using the old file and configuration before publishing the replacement. Calls made while emission is stopped are ignored; a full queue drops records.
Names and extensions must be filename components, without path separators or NUL. The queue size must be between 1 and 1,048,576 records. Numeric settings are checked for byte-count and duration overflow; positive fractional retention settings must resolve to at least one nanosecond. Size settings use decimal units: 1 KB = 1,000 bytes and 1 MB = 1,000 KB.
## Configuration Parameters
### Basic Settings
| Parameter | Type | Description | Default |
|-----------|------|-------------|------------|
| `level` | `int64` | Minimum log level (-4=Debug, 0=Info, 4=Warn, 8=Error) | `0` |
| `level` | `int64` | Minimum log level (-8=Trace, -4=Debug, 0=Info, 4=Warn, 8=Error) | `0` |
| `name` | `string` | Base name for log files | `"log"` |
| `extension` | `string` | Log file extension (without dot) | `"log"` |
| `directory` | `string` | Directory to store log files | `"./log"` |
@@ -66,7 +63,7 @@ logger.Info("info txt log record written to /var/log/myapp.txt")
| `show_level` | `bool` | Include log level in entries | `true` |
| `enable_console` | `bool` | Enable console output (stdout/stderr) | `true` |
| `console_target` | `string` | Console target: `"stdout"`, `"stderr"`, or `"split"` | `"stderr"` |
| `enable_file` | `bool` | Enable file output (console-only) | `false` |
| `enable_file` | `bool` | Enable file output | `false` |
**Note:** When `console_target="split"`, INFO/DEBUG logs go to stdout while WARN/ERROR logs go to stderr.
doc/formatting.md
+34 -13
View File
@@ -65,6 +65,7 @@ const (
FlagStructuredJSON int64 = 0b1000 // Use structured JSON with message/fields
FlagNoTimestamp int64 = 0b010000 // Suppress timestamp
FlagNoLevel int64 = 0b100000 // Suppress level
FlagKV int64 = 0b1000000 // Alternating string keys and values
FlagDefault = FlagShowTimestamp | FlagShowLevel
)
```
@@ -105,9 +106,9 @@ clean = s.Sanitize("cmd; echo test") // "cmd echo test"
```go
const (
PolicyRaw PolicyPreset = "raw" // No-op passthrough
PolicyJSON PolicyPreset = "json" // JSON-safe strings
PolicyJSON PolicyPreset = "json" // Visible control-character escapes
PolicyTxt PolicyPreset = "txt" // Text file safe
PolicyShell PolicyPreset = "shell" // Shell command safe
PolicyShell PolicyPreset = "shell" // Lossy filter, not shell quoting
)
```
@@ -171,7 +172,26 @@ serializer.WriteNil(&buf) // "null"
The sanitizer is a content transform; JSON string escaping is transport
encoding applied afterward, unconditionally. Output is valid JSON for any
sanitization policy. Multi-byte UTF-8 passes through unescaped.
sanitization policy. Multi-byte UTF-8 passes through unescaped. Malformed UTF-8
is replaced before rule-based sanitization, preventing stripped text from joining
invalid byte fragments into control characters. A sanitizer with no rules remains
byte-preserving; the JSON transport layer always replaces malformed UTF-8.
JSON keys and custom timestamp layouts are escaped. Non-finite scalar floats
are emitted as strings (`"NaN"`, `"+Inf"`, `"-Inf"`); non-finite values in structured
maps follow `encoding/json` and produce a marshal error. As in the existing API,
`int32` values are treated as runes; cast to `int64` when numeric output is needed.
Choose unique context and KV keys; avoid context keys named `time`, `level`,
`trace`, `message`, or `fields`. Content policies can make distinct keys identical.
`PolicyTxt` covers text timestamps, trace, context, KV keys, and values. `PolicyRaw`
is intentionally unfiltered, and `FlagRaw` bypasses formatting and sanitization.
Generic complex-value text formatting is bounded to 64 levels and 10,000 visited
values; cycles and larger graphs produce `<cyclic or oversized value>`. This also
applies to complex values under `FlagRaw`, which use `%+v` text formatting.
Custom formatting and marshal callbacks must terminate and be safe for shared use.
A custom rule with no recognized transform preserves its matched rune. Predicate
callbacks must be safe for concurrent use when the sanitizer is shared.
- `format=json` + `sanitization=raw`: recommended; transport escaping only.
- `format=json` + `sanitization=txt`: non-printables appear as `<XX>` inside
@@ -179,8 +199,9 @@ sanitization policy. Multi-byte UTF-8 passes through unescaped.
- `format=json` + `sanitization=json`: redundant; produces visible `\\n`
double escapes. Use `raw` instead.
- Structured JSON (`FlagStructuredJSON`) marshals the fields map via
`encoding/json` and bypasses the sanitizer; validity is guaranteed,
content-level sanitization is not applied to field values.
`encoding/json`, then applies the content policy to every string, including
nested values and object keys. Value types and deterministic map order are preserved.
Marshal failures produce a valid `fields._marshal_error` string.
## PolicyShell Scope
@@ -215,13 +236,13 @@ logger := log.NewLogger()
// Configure sanitization policy
logger.ApplyConfigString(
"format=json",
"sanitization=json", // Uses PolicyJSON
"sanitization=raw", // JSON transport escaping remains enabled
)
// Or with custom formatter (advanced)
s := sanitizer.New().Policy(sanitizer.PolicyShell)
customFormatter := formatter.New(s).Type("txt")
// Note: Direct formatter injection requires using lower-level APIs
// Use customFormatter directly; Logger has no formatter injection API.
```
## Common Patterns
@@ -247,7 +268,7 @@ f := formatter.New()
f.Type("json").ShowTimestamp(false).ShowLevel(false)
// Create custom log entry
entry := f.FormatArgs("action", "purchase", "amount", 99.99)
entry := f.Format(formatter.FlagKV, time.Now(), 0, "", []any{"action", "purchase", "amount", 99.99})
sendToExternalSystem(entry)
```
@@ -262,17 +283,17 @@ shellSanitizer := sanitizer.New().Policy(sanitizer.PolicyShell)
jsonFormatter := formatter.New(jsonSanitizer).Type("json")
apiLog := jsonFormatter.Format(...)
// For shell script generation
// For a filtered text log
txtFormatter := formatter.New(shellSanitizer).Type("txt")
scriptLog := txtFormatter.Format(...)
textLog := txtFormatter.Format(...)
```
## Performance Considerations
- Both packages use pre-allocated buffers for efficiency
- Sanitizer rules are applied in a single pass
- Formatter append methods reuse caller-provided buffers
- Sanitizer avoids allocations when no rule matches valid UTF-8
- Formatter reuses internal buffers via `Reset()`
- No regex or reflection in hot paths
- Scalar formatting avoids reflection; complex values and structured marshaling may use it
## Ownership and Thread Safety
doc/getting-started.md
+8 -5
View File
@@ -4,7 +4,7 @@ This guide will help you get started with the lixenwraith/log package, from inst
## Installation
Install the logger package:
Requires Go 1.27.1 or later. Install the logger package:
```bash
go get github.com/lixenwraith/log
@@ -74,7 +74,7 @@ func main() {
Directory("/var/log/myapp"). // Log directory path
LevelString("info"). // Minimum log level
Format("json"). // Output format
Sanitization("json"). // Sanitization policy
Sanitization(log.PolicyRaw). // Sanitization policy
EnableFile(true). // Enable file output (disabled by default)
BufferSize(2048). // Channel buffer size
MaxSizeMB(10). // Max file size before rotation
@@ -102,8 +102,8 @@ func main() {
## Next Steps
1. **[Learn about configuration options](configuration.md)** - Customize behavior for your needs
2. **[Explore the API](api-reference.md)** - See all available methods
3. **[Logging patterns and examples](logging-guide.md)** - Write better logs
2. **[Explore the API](api.md)** - See all available methods
3. **[Logging patterns and examples](logging.md)** - Write better logs
## Common Patterns
@@ -125,7 +125,10 @@ func NewService() (*Service, error) {
return nil, fmt.Errorf("logger init failed: %w", err)
}
logger.Start()
if err := logger.Start(); err != nil {
_ = logger.Shutdown()
return nil, err
}
return &Service{
logger: logger,
doc/heartbeat.md
+17 -14
View File
@@ -8,7 +8,7 @@ Heartbeats are periodic log messages that provide operational statistics about t
### Key Features
- **Always Visible**: Heartbeats use special log levels that bypass filtering
- **Independent of level filtering**: Heartbeats can still be dropped on queue overflow or output failure
- **Multi-Level Detail**: Choose from process, disk, or system statistics
- **Production Monitoring**: Track logger health without debug logs
- **Metrics Source**: Parse heartbeats for monitoring dashboards
@@ -38,14 +38,15 @@ logger.ApplyConfigString(
**Output:**
```
2024-01-15T10:30:00Z PROC type="proc" sequence=1 uptime_hours="24.50" processed_logs=1847293 dropped_logs=0
2024-01-15T10:30:00Z PROC type=proc sequence=1 uptime_hours="24.50" processed_logs=1847293 total_dropped_logs=0
```
**Fields:**
- `sequence`: Incrementing counter
- `uptime_hours`: Logger uptime
- `processed_logs`: Successfully written logs
- `dropped_logs`: Logs lost due to buffer overflow
- `processed_logs`: Application records processed successfully, excluding heartbeats
- `total_dropped_logs`: Cumulative queue and output-processing drops, including heartbeats
- `dropped_since_last`: Interval drops, emitted only when non-zero; reset at each PROC heartbeat attempt
### Level 2: Process + Disk Statistics (DISK)
@@ -60,7 +61,7 @@ logger.ApplyConfigString(
**Additional Output:**
```
2024-01-15T10:30:00Z DISK type="disk" sequence=1 rotated_files=12 deleted_files=5 total_log_size_mb="487.32" log_file_count=8 current_file_size_mb="23.45" disk_status_ok=true disk_free_mb="5234.67"
2024-01-15T10:30:00Z DISK type=disk sequence=1 rotated_files=12 deleted_files=5 total_log_size_mb="487.32" log_file_count=8 current_file_size_mb="23.45" disk_status_ok=true disk_free_mb="5234.67"
```
**Additional Fields:**
@@ -85,7 +86,7 @@ logger.ApplyConfigString(
**Additional Output:**
```
2024-01-15T10:30:00Z SYS type="sys" sequence=1 alloc_mb="45.23" sys_mb="128.45" num_gc=1523 num_goroutine=42
2024-01-15T10:30:00Z SYS type=sys sequence=1 alloc_mb="45.23" sys_mb="128.45" num_gc=1523 num_goroutine=42
```
**Additional Fields:**
@@ -146,13 +147,13 @@ With `format=json`, heartbeats are structured for easy parsing:
{
"time": "2024-01-15T10:30:00.123456789Z",
"level": "PROC",
"fields": [
"type", "proc",
"sequence", 42,
"uptime_hours", "24.50",
"processed_logs", 1847293,
"dropped_logs", 0
]
"fields": {
"type": "proc",
"sequence": 42,
"uptime_hours": "24.50",
"processed_logs": 1847293,
"total_dropped_logs": 0
}
}
```
@@ -161,5 +162,7 @@ With `format=json`, heartbeats are structured for easy parsing:
With `format=txt`, heartbeats are human-readable:
```
2024-01-15T10:30:00.123456789Z PROC type="proc" sequence=42 uptime_hours="24.50" processed_logs=1847293 dropped_logs=0
2024-01-15T10:30:00.123456789Z PROC type=proc sequence=42 uptime_hours="24.50" processed_logs=1847293 total_dropped_logs=0
```
PROC uptime and totals span the lifetime of the Logger instance, including stop/start cycles. DISK fields ending in `_mb` retain historical binary MiB units (1,048,576 bytes); SYS memory fields use decimal MB. Configuration size limits use decimal KB.
doc/logging.md
+8 -8
View File
@@ -44,7 +44,7 @@ logger.ApplyConfigString("level=8") // Error only
### Key-Value Pairs
Use structured key-value pairs for machine-parseable logs:
Ordinary methods emit positional arguments (JSON arrays). Use `LogStructured`, or `LogContext` with `FlagKV`, when named JSON fields are required:
```go
logger.Info("User login",
@@ -117,9 +117,9 @@ func logWithContext(ctx context.Context, logger *log.Logger, level string, msg s
switch level {
case "info":
logger.Info(msg, fields...)
logger.Info(append([]any{msg}, fields...)...)
case "error":
logger.Error(msg, fields...)
logger.Error(append([]any{msg}, fields...)...)
}
}
```
@@ -130,11 +130,11 @@ The logger supports three output formats, each with configurable sanitization. T
### Txt Format (Human-Readable)
Default format for development and debugging:
Text format for development and debugging (select `format=txt`):
```
2024-01-15T10:30:45.123456789Z INFO User login user_id=42 email=user@example.com ip=192.168.1.100
2024-01-15T10:30:45.234567890Z WARN Rate limit approaching user_id=42 requests=95 limit=100
2024-01-15T10:30:45.123456789Z INFO "User login" user_id 42 email user@example.com ip 192.168.1.100
2024-01-15T10:30:45.234567890Z WARN "Rate limit approaching" user_id 42 requests 95 limit 100
```
Note: The txt format applies the configured sanitization policy (default: raw). Non-printable characters can be hex-encoded using `sanitization=txt` configuration.
@@ -204,8 +204,8 @@ func processPayment(amount float64) error {
Output includes function names:
```
2024-01-15T10:30:45.123456789Z INFO processPayment Processing payment amount=99.99
2024-01-15T10:30:45.234567890Z ERROR validateAmount -> processPayment -> main Payment validation failed amount=-10 error="negative amount"
2024-01-15T10:30:45.123456789Z INFO processPayment "Processing payment" amount 99.99
2024-01-15T10:30:45.234567890Z ERROR main -> processPayment "Payment validation failed" amount -10 error "negative amount"
```
### Trace Depth Guidelines
doc/storage.md
+17 -28
View File
@@ -10,31 +10,19 @@ Log files are automatically rotated when they reach the configured size limit in
```go
logger.ApplyConfigString(
"max_size_kb=102400", // Rotate at 100MB (102400 KB)
"max_size_kb=100000", // Rotate at 100 MB (100,000 KB)
)
```
### Rotation Behavior
1. **Size Check**: Before each write, the logger checks if the file would exceed `max_size_kb`
2. **New File Creation**: Creates a new file with timestamp: `appname_240115_103045_123456789.log`
3. **Seamless Transition**: No logs are lost during rotation
4. **Old File Closure**: Previous file is properly closed and synced
1. Before each write, the logger checks whether the active file would exceed `max_size_kb`.
2. It closes the active `{name}.{extension}` file and renames it to `{name}_{YYMMDD}_{HHMMSS}[_N].{extension}`.
3. It opens a new active file. A counter suffix avoids existing names when rotations share a second.
### File Naming Convention
For example: `myapp.log` becomes `myapp_240115_143022.log`, then `myapp_240115_143022_1.log` on the next collision. Rotation does not split a record, so a single oversized record can exceed the size limit. File errors can drop records and are counted. Periodic sync, explicit `Flush`, and shutdown provide sync points; closing a file alone does not guarantee crash durability.
```
{name}_{YYMMDD}_{HHMMSS}_{nanoseconds}.{extension}
Example: myapp_240115_143022_987654321.log
```
Components:
- `name`: Configured log name
- `YYMMDD`: Date (year, month, day)
- `HHMMSS`: Time (hour, minute, second)
- `nanoseconds`: For uniqueness
- `extension`: Configured extension
Use a directory owned exclusively by this logger. Cleanup and retention select regular files by extension, including files from earlier runs with different base names. They exclude the active filename, directories, and symlinks. With an empty extension, only extensionless files match. Console-only configurations never perform retention deletion. Total-size and free-space limits are checked periodically and can be exceeded between checks.
## Disk Space Management
@@ -45,7 +33,7 @@ The logger enforces two types of space limits:
```go
logger.ApplyConfigString(
"max_total_size_kb=1000", // Total log directory size
"min_disk_free_kb=5000", // Minimum free disk space
"min_disk_free_mb=5000", // Minimum free disk space
)
```
@@ -55,7 +43,7 @@ When limits are exceeded, the logger:
1. Identifies oldest log files
2. Deletes them until space requirements are met
3. Preserves the current active log file
4. Logs cleanup actions for audit
4. Increments deletion counters reported in disk heartbeats
### Example Configuration
@@ -64,21 +52,21 @@ When limits are exceeded, the logger:
logger.ApplyConfigString(
"max_size_kb=500", // 500 KB files
"max_total_size_kb=5000", // 5 MB total log directory limit
"min_disk_free_kb=1048576", // 1 GB free space required on disk
"min_disk_free_mb=1000000", // 1 GB free space required on disk
)
// Generous: Large files, external archival
logger.ApplyConfigString(
"max_size_kb=102400", // 100 MB files
"max_size_kb=100000", // 100 MB files
"max_total_size_kb=0", // No total limit
"min_disk_free_kb=10240", // 10 MB free required
"min_disk_free_mb=10000", // 10 MB free required
)
// Balanced: Production defaults
logger.ApplyConfigString(
"max_size_kb=102400", // 100 MB files
"max_total_size_kb=5242880", // 5 GB total limit
"min_disk_free_kb=512000", // 500 MB free required
"max_size_kb=100000", // 100 MB files
"max_total_size_kb=5000000", // 5 GB total limit
"min_disk_free_mb=500000", // 500 MB free required
)
```
@@ -161,7 +149,7 @@ logger.ApplyConfigString(
Output:
```
2024-01-15T10:30:00Z DISK type="disk" sequence=1 rotated_files=5 deleted_files=2 total_log_size_kb="487.32" log_file_count=8 current_file_size_kb="23.45" disk_status_ok=true disk_free_kb="5234.67"
2024-01-15T10:30:00Z DISK type="disk" sequence=1 rotated_files=5 deleted_files=2 total_log_size_mb="487.32" log_file_count=8 current_file_size_mb="23.45" disk_status_ok=true disk_free_mb="5234.67"
```
## Manual Recovery
@@ -176,7 +164,8 @@ df -h /var/log
find /var/log/myapp -name "*.log" -size +100M
# Manual cleanup (oldest first)
ls -t /var/log/myapp/*.log | tail -n 20 | xargs rm
# Review archive paths before deleting; preserve the active log file.
find /var/log/myapp -maxdepth 1 -type f -name 'myapp_*.log' -print
# Verify space
df -h /var/log
doc/testing.md
+29
View File
@@ -0,0 +1,29 @@
# Testing and performance
Use Go 1.27.1 or later. The module has no external dependencies.
```sh
go vet ./...
go test -race -count=1 -timeout=3m ./...
go test -race -run 'TestStress' -count=10 -timeout=3m .
```
`stress_test.go` uses fixed record counts and synchronized starts. It verifies exact queue overflow, unique IDs, processed-plus-dropped accounting across lifecycle changes, and byte preservation across rotations. It has no throughput pass threshold and no timed soak loop. Heartbeat and adaptive-timer tests use `testing/synctest` or explicit state transitions instead of wall-clock sleeps. Run benchmarks separately when comparing performance:
```sh
go test ./... -run '^$' -bench . -benchmem -count=5
```
Fuzz targets test malformed UTF-8, control characters, keys, custom timestamp layouts, numeric edge cases, structured marshal failures, and printf argument association. Each target runs independently:
```sh
go test ./sanitizer -run '^$' -fuzz '^FuzzSanitize$' -fuzztime=30s -parallel=2
go test ./sanitizer -run '^$' -fuzz '^FuzzSerializerJSON$' -fuzztime=30s -parallel=2
go test ./formatter -run '^$' -fuzz '^FuzzFormatJSON$' -fuzztime=30s -parallel=2
go test ./formatter -run '^$' -fuzz '^FuzzFormatTextPolicy$' -fuzztime=30s -parallel=2
go test ./compat -run '^$' -fuzz '^FuzzParseFormat$' -fuzztime=30s -parallel=2
```
Fuzzing is bounded by time in CI; execution counts depend on the runner. All committed seeds run during ordinary `go test`, including minimized failures retained under `testdata/fuzz`.
The GitHub workflow runs on pushes and pull requests. It checks formatting, vet, race detection, repeated lifecycle/stress workloads, bounded fuzzing, and compilation for FreeBSD, Windows, and browser WASM. Runtime tests execute on Linux; cross-compilation does not validate other operating systems' runtime behavior.
formatter/formatter.go
+63 -46
View File
@@ -15,8 +15,9 @@
package formatter
import (
"encoding/json"
"encoding/json/jsontext"
"fmt"
"math"
"strconv"
"time"
"unicode/utf8"
@@ -50,9 +51,10 @@ type Formatter struct {
// Serializers are stateless and format-fixed; built once to keep the
// per-record path allocation-free
serTxt *sanitizer.Serializer
serJSON *sanitizer.Serializer
serRaw *sanitizer.Serializer
serTxt *sanitizer.Serializer
serJSON *sanitizer.Serializer
serRaw *sanitizer.Serializer
serPlain *sanitizer.Serializer
}
// ContextSlots is the number of correlation values a Context carries
@@ -66,7 +68,8 @@ type Context struct {
}
// ContextKeys names the record keys for Context fields; empty names are omitted.
// Names are emitted verbatim and must be plain identifiers.
// Names are escaped for the output format. Use distinct names that do not
// collide with built-in time, level, trace, message, or fields keys.
type ContextKeys struct {
Tag string
Vals [ContextSlots]string
@@ -90,6 +93,7 @@ func New(s ...*sanitizer.Sanitizer) *Formatter {
serTxt: sanitizer.NewSerializer("txt", san),
serJSON: sanitizer.NewSerializer("json", san),
serRaw: sanitizer.NewSerializer("raw", san),
serPlain: sanitizer.NewSerializer("raw", sanitizer.New()),
}
}
@@ -198,16 +202,22 @@ func (f *Formatter) AppendFormatWithOptionsCtx(dst []byte, format string, ctx Co
case []byte:
dst = append(dst, v...)
case fmt.Stringer:
dst = append(dst, v.String()...)
dst = fmt.Append(dst, v)
case error:
dst = append(dst, v.Error()...)
dst = fmt.Append(dst, v)
default:
dst = append(dst, fmt.Sprint(v)...)
f.serPlain.WriteComplex(&dst, v)
}
}
return dst
}
if flags&FlagNoTimestamp != 0 {
flags &^= FlagShowTimestamp
}
if flags&FlagNoLevel != 0 {
flags &^= FlagShowLevel
}
format = normalizeFormat(format)
serializer := f.serializerFor(format)
@@ -271,10 +281,7 @@ func (f *Formatter) AppendArgs(dst []byte, args ...any) []byte {
return dst
}
// appendValue provides unified type conversion (was convertValue; now
// value-return style over caller buffer). Type switch body unchanged except
// buffer plumbing — replace every `serializer.WriteX(buf, ...)` with
// `serializer.WriteX(&dst, ...)` and `return dst`.
// appendValue converts one value using the selected serializer.
func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Serializer, needsSpace bool) []byte {
if needsSpace && len(dst) > 0 {
dst = append(dst, ' ')
@@ -289,17 +296,29 @@ func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Seriali
n := utf8.EncodeRune(runeStr[:], val)
serializer.WriteString(&dst, string(runeStr[:n]))
case int:
serializer.WriteNumber(&dst, string(strconv.AppendInt(nil, int64(val), 10)))
dst = strconv.AppendInt(dst, int64(val), 10)
case int8:
dst = strconv.AppendInt(dst, int64(val), 10)
case int16:
dst = strconv.AppendInt(dst, int64(val), 10)
case int64:
serializer.WriteNumber(&dst, string(strconv.AppendInt(nil, val, 10)))
dst = strconv.AppendInt(dst, val, 10)
case uint:
serializer.WriteNumber(&dst, string(strconv.AppendUint(nil, uint64(val), 10)))
dst = strconv.AppendUint(dst, uint64(val), 10)
case uint8:
dst = strconv.AppendUint(dst, uint64(val), 10)
case uint16:
dst = strconv.AppendUint(dst, uint64(val), 10)
case uint32:
dst = strconv.AppendUint(dst, uint64(val), 10)
case uintptr:
dst = strconv.AppendUint(dst, uint64(val), 10)
case uint64:
serializer.WriteNumber(&dst, string(strconv.AppendUint(nil, val, 10)))
dst = strconv.AppendUint(dst, val, 10)
case float32:
serializer.WriteNumber(&dst, string(strconv.AppendFloat(nil, float64(val), 'f', -1, 32)))
dst = appendFloat(dst, float64(val), 32, serializer)
case float64:
serializer.WriteNumber(&dst, string(strconv.AppendFloat(nil, val, 'f', -1, 64)))
dst = appendFloat(dst, val, 64, serializer)
case bool:
serializer.WriteBool(&dst, val)
case nil:
@@ -307,15 +326,25 @@ func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Seriali
case time.Time:
serializer.WriteString(&dst, val.Format(f.timestampFormat))
case error:
serializer.WriteString(&dst, val.Error())
serializer.WriteString(&dst, fmt.Sprint(val))
case fmt.Stringer:
serializer.WriteString(&dst, val.String())
serializer.WriteString(&dst, fmt.Sprint(val))
default:
serializer.WriteComplex(&dst, val)
}
return dst
}
// Non-finite floats have no JSON number representation. Keep their spelling
// as a string so a single bad measurement cannot corrupt the record.
func appendFloat(dst []byte, value float64, bits int, serializer *sanitizer.Serializer) []byte {
if math.IsNaN(value) || math.IsInf(value, 0) {
serializer.WriteString(&dst, strconv.FormatFloat(value, 'f', -1, bits))
return dst
}
return strconv.AppendFloat(dst, value, 'f', -1, bits)
}
// LevelToString converts integer level values to string
func LevelToString(level int64) string {
switch level {
@@ -342,10 +371,8 @@ func LevelToString(level int64) string {
// appendJSONKey writes a quoted literal key followed by ':'
func appendJSONKey(dst []byte, key string) []byte {
dst = append(dst, '"')
dst = append(dst, key...)
dst = append(dst, '"', ':')
return dst
dst, _ = jsontext.AppendQuote(dst, key)
return append(dst, ':')
}
// isKV reports whether args form an even-length list with string keys
@@ -367,9 +394,9 @@ func (f *Formatter) appendJSON(dst []byte, ctx Context, flags int64, timestamp t
needsComma := false
if flags&FlagShowTimestamp != 0 {
dst = append(dst, `"time":"`...)
dst = timestamp.AppendFormat(dst, f.timestampFormat)
dst = append(dst, '"')
dst = append(dst, `"time":`...)
var stamp [128]byte
serializer.WriteString(&dst, string(timestamp.AppendFormat(stamp[:0], f.timestampFormat)))
needsComma = true
}
@@ -425,13 +452,10 @@ func (f *Formatter) appendJSON(dst []byte, ctx Context, flags int64, timestamp t
dst = append(dst, `,"fields":`...)
marshaledFields, err := json.Marshal(fields)
if err != nil {
dst = append(dst, `{"_marshal_error":"`...)
if err := serializer.WriteJSON(&dst, fields); err != nil {
dst = append(dst, `{"_marshal_error":`...)
serializer.WriteString(&dst, err.Error())
dst = append(dst, `"}`...)
} else {
dst = append(dst, marshaledFields...)
dst = append(dst, '}')
}
dst = append(dst, '}', '\n')
@@ -477,7 +501,8 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti
needsSpace := false
if flags&FlagShowTimestamp != 0 {
dst = timestamp.AppendFormat(dst, f.timestampFormat)
var stamp [128]byte
dst = f.sanitizer.AppendSanitize(dst, string(timestamp.AppendFormat(stamp[:0], f.timestampFormat)))
needsSpace = true
}
@@ -493,7 +518,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti
if needsSpace {
dst = append(dst, ' ')
}
dst = append(dst, f.ctxKeys.Tag...)
serializer.WriteString(&dst, f.ctxKeys.Tag)
dst = append(dst, '=')
dst = f.appendValue(dst, ctx.Tag, serializer, false)
needsSpace = true
@@ -505,7 +530,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti
if needsSpace {
dst = append(dst, ' ')
}
dst = append(dst, key...)
serializer.WriteString(&dst, key)
dst = append(dst, '=')
dst = strconv.AppendUint(dst, ctx.Vals[i], 10)
needsSpace = true
@@ -515,15 +540,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti
if needsSpace {
dst = append(dst, ' ')
}
// Sanitize trace to prevent terminal control sequence injection
tempBuf := make([]byte, 0, len(trace)*2)
f.serTxt.WriteString(&tempBuf, trace)
// Extract content without quotes if added by txt serializer
if len(tempBuf) > 2 && tempBuf[0] == '"' && tempBuf[len(tempBuf)-1] == '"' {
dst = append(dst, tempBuf[1:len(tempBuf)-1]...)
} else {
dst = append(dst, tempBuf...)
}
dst = f.sanitizer.AppendSanitize(dst, trace)
needsSpace = true
}
@@ -532,7 +549,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti
if needsSpace {
dst = append(dst, ' ')
}
dst = append(dst, args[i].(string)...)
serializer.WriteString(&dst, args[i].(string))
dst = append(dst, '=')
dst = f.appendValue(dst, args[i+1], serializer, false)
needsSpace = true
formatter/fuzz_test.go
+118
View File
@@ -0,0 +1,118 @@
package formatter
import (
"bytes"
"encoding/json"
"math"
"strings"
"testing"
"unicode"
"unicode/utf8"
"github.com/lixenwraith/log/sanitizer"
)
func FuzzFormatJSON(f *testing.F) {
for _, s := range []string{"", "safe", "\xff\n\"\\", "\u0085\x1b[0m", "世界"} {
f.Add(s, s, s, uint64(0), uint8(0))
}
f.Fuzz(func(t *testing.T, value, key, layout string, bits uint64, mode uint8) {
policies := []sanitizer.PolicyPreset{sanitizer.PolicyRaw, sanitizer.PolicyTxt, sanitizer.PolicyJSON, sanitizer.PolicyShell}
fm := New(sanitizer.New().Policy(policies[int(mode)%4])).Type("json").TimestampFormat(layout).ContextKeys(key)
flags := FlagDefault
args := []any{value, []byte(value), math.Float64frombits(bits), uint32(bits)}
switch mode % 3 {
case 1:
flags |= FlagKV
args = []any{key, value, "number", math.Float64frombits(bits)}
case 2:
flags |= FlagStructuredJSON
args = []any{value, map[string]any{key: []any{value, math.Float64frombits(bits)}}}
}
prefix := []byte("prefix:")
out := fm.AppendFormatCtx(bytes.Clone(prefix), Context{Tag: value}, flags, testStamp, int64(bits), value, args)
if !bytes.HasPrefix(out, prefix) {
t.Fatal("prefix changed")
}
data := out[len(prefix):]
if !utf8.Valid(data) || !json.Valid(data) || bytes.Count(data, []byte{'\n'}) != 1 || data[len(data)-1] != '\n' {
t.Fatalf("invalid JSON record %q", data)
}
if second := fm.AppendFormatCtx(nil, Context{Tag: value}, flags, testStamp, int64(bits), value, args); !bytes.Equal(data, second) {
t.Fatal("non-deterministic formatter")
}
})
}
func FuzzFormatTextPolicy(f *testing.F) {
for _, s := range []string{"", "\n\r\x1b[31m", "\xff<00>\u0085", "a=b \"c\""} {
f.Add(s, s, s)
}
f.Fuzz(func(t *testing.T, value, key, layout string) {
fm := New(sanitizer.New().Policy(sanitizer.PolicyTxt)).Type("txt").TimestampFormat(layout).ContextKeys(key, key)
out := fm.AppendFormatCtx(nil, Context{Tag: value}, FlagDefault|FlagKV, testStamp, 0, value, []any{key, value})
if !bytes.HasSuffix(out, []byte{'\n'}) {
t.Fatal("missing terminator")
}
for _, r := range string(out[:len(out)-1]) {
if unicode.IsControl(r) || !unicode.IsPrint(r) {
t.Fatalf("unsafe rune %U in %q", r, out)
}
}
})
}
type panickingStringer struct{}
func (panickingStringer) String() string { panic("broken Stringer") }
func TestFormatHardening(t *testing.T) {
t.Run("marshal error", func(t *testing.T) {
cycle := map[string]any{}
cycle["self"] = cycle
for _, v := range []any{math.NaN(), make(chan int), cycle, json.RawMessage(`invalid`)} {
f := New().Type("json")
record := unmarshalRecord(t, f.Format(FlagStructuredJSON, testStamp, 0, "", []any{"msg", map[string]any{"bad": v}}))
if _, ok := record["fields"].(map[string]any)["_marshal_error"].(string); !ok {
t.Fatalf("missing marshal error: %v", record)
}
}
})
t.Run("nested sanitization", func(t *testing.T) {
f := New(sanitizer.New().Policy(sanitizer.PolicyTxt)).Type("json")
record := unmarshalRecord(t, f.Format(FlagStructuredJSON, testStamp, 0, "", []any{"msg", map[string]any{"k\n": []any{map[string]any{"v": "\x1b[31m"}, 42, true}}}))
nested := record["fields"].(map[string]any)["k<0a>"].([]any)
eq(t, nested[0].(map[string]any)["v"], any("<1b>[31m"), "nested content policy")
eq(t, nested[1], any(float64(42)), "number preserved")
})
t.Run("explicit suppression wins", func(t *testing.T) {
f := New().Type("json")
record := unmarshalRecord(t, f.FormatWithOptions("json", FlagDefault|FlagNoLevel|FlagNoTimestamp, testStamp, 0, "", nil))
eq(t, len(record), 0, "suppressed metadata")
})
t.Run("bad stringer is printable", func(t *testing.T) {
for _, flags := range []int64{0, FlagRaw} {
out := New().Format(flags, testStamp, 0, "", []any{panickingStringer{}})
if !strings.Contains(string(out), "PANIC") {
t.Fatalf("missing fmt panic marker: %q", out)
}
}
})
}
func TestCyclicComplexValues(t *testing.T) {
cycle := map[string]any{}
cycle["self"] = cycle
slice := make([]any, 1)
slice[0] = slice
for _, value := range []any{cycle, slice} {
for _, format := range []string{"txt", "json", "raw"} {
for _, flags := range []int64{0, FlagRaw} {
output := New().Type(format).Format(flags, testStamp, 0, "", []any{value})
if !strings.Contains(string(output), "cyclic or oversized value") {
t.Fatalf("%s: missing cycle marker: %q", format, output)
}
}
}
}
}
go.mod
+1 -1
View File
@@ -1,3 +1,3 @@
module github.com/lixenwraith/log
go 1.26.0
go 1.27.1
heartbeat.go
+1
View File
@@ -135,6 +135,7 @@ func (l *Logger) writeHeartbeatRecord(level int64, args []any) {
}
record := logRecord{
heartbeat: true,
Flags: FlagDefault | FlagKV,
TimeStamp: time.Now(),
Level: level,
integration_test.go
+1 -3
View File
@@ -96,7 +96,6 @@ func TestConcurrentOperations(t *testing.T) {
for i := range 3 {
// Non-fatal only: Fatal outside the test goroutine is undefined behavior
noErr(t, logger.ApplyConfigString(fmt.Sprintf("trace_depth=%d", i)), "ApplyConfigString")
time.Sleep(50 * time.Millisecond)
}
}()
@@ -104,9 +103,8 @@ func TestConcurrentOperations(t *testing.T) {
go func() {
defer wg.Done()
for range 5 {
// Timeout must exceed worst-case contention on flushMutex under load
// Allow the fixed workload to complete under race instrumentation
noErr(t, logger.Flush(2*time.Second), "concurrent Flush")
time.Sleep(30 * time.Millisecond)
}
}()
lifecycle_test.go
+1 -12
View File
@@ -123,16 +123,6 @@ func TestShutdownEdgeCases(t *testing.T) {
noErr(t, logger.Shutdown(), "second Shutdown")
})
t.Run("timeout", func(t *testing.T) {
logger, _ := newTestLogger(t)
for i := range 200 {
logger.Info("flood", i)
}
// Stop may time out; terminal state transitions are unconditional
_ = logger.Shutdown(time.Millisecond)
isTrue(t, logger.state.ShutdownCalled.Load(), "ShutdownCalled")
isFalse(t, logger.state.IsInitialized.Load(), "IsInitialized")
})
}
// TestFlush covers the success path and both failure modes.
@@ -150,7 +140,7 @@ func TestFlush(t *testing.T) {
t.Run("timeout", func(t *testing.T) {
logger, _ := newTestLogger(t)
errContains(t, logger.Flush(time.Nanosecond), "timeout", "Flush")
errContains(t, logger.Flush(0), "timeout", "Flush")
})
t.Run("on stopped logger", func(t *testing.T) {
@@ -165,4 +155,3 @@ func TestFlush(t *testing.T) {
errContains(t, logger.Flush(time.Second), "not initialized", "Flush")
})
}
logger.go
+173 -216
View File
@@ -4,6 +4,7 @@ import (
"errors"
"fmt"
"io"
"maps"
"math"
"os"
"sync"
@@ -22,7 +23,9 @@ type Logger struct {
spawner atomic.Pointer[func(func())]
errHandler atomic.Pointer[func(string)]
state State
initMu sync.Mutex
initMu sync.Mutex // serializes lifecycle and configuration changes
sendMu sync.RWMutex // joins enqueues before detaching a processor
processorErr error // written by processor, read only after ProcDone closes
}
// levelOff closes the emit gate without touching configuration
@@ -42,14 +45,6 @@ func NewLogger() *Logger {
l.state.Flags.Store(flagsFromConfig(defaultCfg))
l.state.TraceDepth.Store(defaultCfg.TraceDepth)
// // Initialize default formatter to prevent nil access
// defaultFormatter := formatter.New(sanitizer.New()).
// Type(defaultCfg.Format).
// TimestampFormat(defaultCfg.TimestampFormat).
// ShowLevel(defaultCfg.ShowLevel).
// ShowTimestamp(defaultCfg.ShowTimestamp)
// l.formatter.Store(defaultFormatter)
// Initialize the state
l.state.IsInitialized.Store(false)
l.state.LoggerDisabled.Store(false)
@@ -69,14 +64,6 @@ func NewLogger() *Logger {
// Typed nil: a non-blocking send on a nil channel always takes default
l.state.ActiveLogChannel.Store((chan logRecord)(nil))
l.state.flushRequestChan = make(chan chan struct{}, 1)
// // Create a closed channel initially to prevent nil pointer issues
// initialChan := make(chan logRecord)
// close(initialChan)
// l.state.ActiveLogChannel.Store(initialChan)
//
// l.state.flushRequestChan = make(chan chan struct{}, 1)
return l
}
@@ -88,6 +75,7 @@ func (l *Logger) ApplyConfig(cfg *Config) error {
return fmt.Errorf("log: configuration cannot be nil")
}
cfg = cfg.Clone()
if err := cfg.Validate(); err != nil {
return fmt.Errorf("log: invalid configuration: %w", err)
}
@@ -101,6 +89,8 @@ func (l *Logger) ApplyConfig(cfg *Config) error {
// ApplyConfigString applies string key-value overrides to the logger's current configuration
// Each override should be in the format "key=value"
func (l *Logger) ApplyConfigString(overrides ...string) error {
l.initMu.Lock()
defer l.initMu.Unlock()
cfg := l.getConfig().Clone()
var errors []error
@@ -121,7 +111,10 @@ func (l *Logger) ApplyConfigString(overrides ...string) error {
return combineConfigErrors(errors)
}
return l.ApplyConfig(cfg)
if err := cfg.Validate(); err != nil {
return fmtErrorf("invalid configuration: %w", err)
}
return l.applyConfig(cfg)
}
// GetConfig returns a copy of current configuration
@@ -137,280 +130,239 @@ func (l *Logger) getConfig() *Config {
// applyConfig is the internal implementation for applying configuration, assuming initMu is held
func (l *Logger) applyConfig(cfg *Config) error {
oldCfg := l.getConfig()
l.currentConfig.Store(cfg)
wasStarted := l.state.Started.Load()
needsRestart := wasStarted && configRequiresRestart(oldCfg, cfg)
// Shared formatter and sanitizer constructor with SetContextKeys
l.rebuildFormatter(cfg)
// A timed-out processor still owns its file and formatter. Do not replace
// either resource until it has actually exited.
if !wasStarted && !l.state.ProcessorExited.Load() {
return fmtErrorf("previous processor is still stopping")
}
// Emit fast-path mirrors
l.state.Flags.Store(flagsFromConfig(cfg))
l.state.TraceDepth.Store(cfg.TraceDepth)
// Ensure log directory exists if file output is enabled
currentFile, _ := l.state.CurrentFile.Load().(*os.File)
needsNewFile := cfg.EnableFile && (!l.state.IsInitialized.Load() || !oldCfg.EnableFile ||
!wasStarted && currentFile == nil || l.state.LoggerDisabled.Load() ||
oldCfg.Directory != cfg.Directory || oldCfg.Name != cfg.Name || oldCfg.Extension != cfg.Extension)
var newFile *os.File
if cfg.EnableFile {
if err := os.MkdirAll(cfg.Directory, 0755); err != nil {
l.state.LoggerDisabled.Store(true)
l.currentConfig.Store(oldCfg) // Rollback
l.refreshLevelGate()
return fmtErrorf("failed to create log directory '%s': %w", cfg.Directory, err)
}
if needsNewFile {
var err error
newFile, err = openLogFile(cfg)
if err != nil {
return fmtErrorf("failed to create log file: %w", err)
}
}
}
// Get current state
wasInitialized := l.state.IsInitialized.Load()
wasStarted := l.state.Started.Load()
// Determine if restart is needed
needsRestart := wasStarted && wasInitialized && configRequiresRestart(oldCfg, cfg)
// Stop processor if restart needed
if needsRestart {
if err := l.Stop(); err != nil {
l.currentConfig.Store(oldCfg) // Rollback
// Drain with the old configuration and file before publishing the new one.
if needsRestart || wasStarted && needsNewFile {
needsRestart = true
if err := l.stopLocked(); err != nil {
if newFile != nil {
_ = newFile.Close()
}
return fmtErrorf("failed to stop processor for restart: %w", err)
}
}
// Get current file handle
currentFilePtr := l.state.CurrentFile.Load()
var currentFile *os.File
if currentFilePtr != nil {
currentFile, _ = currentFilePtr.(*os.File)
}
// Determine if we need a new file
needsNewFile := !wasInitialized || currentFile == nil ||
oldCfg.Directory != cfg.Directory ||
oldCfg.Name != cfg.Name ||
oldCfg.Extension != cfg.Extension
// Handle file state transitions
if !cfg.EnableFile {
// When disabling file output, close the current file
// Draining may rotate the old file; close the handle the processor left behind.
currentFile, _ = l.state.CurrentFile.Load().(*os.File)
if !cfg.EnableFile || newFile != nil {
if currentFile != nil {
// Sync and close the file
_ = currentFile.Sync()
if err := currentFile.Close(); err != nil {
l.internalLog("warning - failed to close log file during disable: %v\n", err)
l.internalLog("failed to close old log file: %v\n", err)
}
}
l.state.CurrentFile.Store((*os.File)(nil))
l.state.CurrentFile.Store(newFile)
l.state.CurrentSize.Store(0)
} else if needsNewFile {
// When enabling file output or initializing, create new file
logFile, err := l.createNewLogFile()
if err != nil {
l.state.LoggerDisabled.Store(true)
l.currentConfig.Store(oldCfg) // Rollback
return fmtErrorf("failed to create log file: %w", err)
}
// Close old file if transitioning from one file to another
if currentFile != nil && currentFile != logFile {
_ = currentFile.Sync()
if err := currentFile.Close(); err != nil {
l.internalLog("warning - failed to close old log file: %v\n", err)
if newFile != nil {
if info, err := newFile.Stat(); err == nil {
l.state.CurrentSize.Store(info.Size())
}
}
l.state.CurrentFile.Store(logFile)
l.state.CurrentSize.Store(0)
if fi, errStat := logFile.Stat(); errStat == nil {
l.state.CurrentSize.Store(fi.Size())
}
}
// Setup console writer based on config
var writer io.Writer = io.Discard
if cfg.EnableConsole {
var writer io.Writer
writer = os.Stdout
if cfg.ConsoleTarget == "stderr" {
writer = os.Stderr
} else {
writer = os.Stdout
}
l.state.StdoutWriter.Store(&sink{w: writer})
} else {
l.state.StdoutWriter.Store(&sink{w: io.Discard})
}
// Mark as initialized
l.state.StdoutWriter.Store(&sink{w: writer})
l.currentConfig.Store(cfg)
l.rebuildFormatter(cfg)
l.state.Flags.Store(flagsFromConfig(cfg))
l.state.TraceDepth.Store(cfg.TraceDepth)
l.state.IsInitialized.Store(true)
l.state.ShutdownCalled.Store(false)
l.state.DiskFullLogged.Store(false)
// A running processor may have just reported an output failure. A hot
// configuration update must not revive its invalid file handle.
if !wasStarted || needsRestart {
l.state.LoggerDisabled.Store(false)
l.state.DiskFullLogged.Store(false)
}
l.state.DiskStatusOK.Store(true)
l.refreshLevelGate()
// Restart processor if it was running and needs restart
if needsRestart {
return l.Start()
return l.startLocked()
}
return nil
}
// Start begins log processing. Safe to call multiple times
// Returns error if logger is not initialized
// Start begins log processing. Repeated calls while running are no-ops.
// ApplyConfig must succeed first. A processor still stopping cannot be restarted.
func (l *Logger) Start() error {
if !l.state.IsInitialized.Load() {
l.initMu.Lock()
defer l.initMu.Unlock()
return l.startLocked()
}
func (l *Logger) startLocked() error {
if !l.state.IsInitialized.Load() || l.state.ShutdownCalled.Load() {
return fmtErrorf("logger not initialized, call ApplyConfig first")
}
// Check if processor didn't exit cleanly last time
if l.state.Started.Load() && !l.state.ProcessorExited.Load() {
// Force stop to clean up
l.internalLog("warning - processor still running from previous start, forcing stop\n")
if err := l.Stop(); err != nil {
return fmtErrorf("failed to stop hung processor: %w", err)
if l.state.Started.Load() {
if l.state.ProcessorExited.Load() {
return fmtErrorf("processor exited unexpectedly; call Stop before restarting")
}
return nil
}
if !l.state.ProcessorExited.Load() {
return fmtErrorf("previous processor is still stopping")
}
// Only start if not already started
if l.state.Started.CompareAndSwap(false, true) {
cfg := l.getConfig()
// Create log channels
ch := make(chan logRecord, cfg.BufferSize)
stop := make(chan struct{})
done := make(chan struct{})
l.state.ActiveLogChannel.Store(ch)
l.state.ProcStop.Store(stop)
l.state.ProcDone.Store(done)
l.state.ProcessorExited.Store(false)
// Start processor
l.spawn(func() { l.processLogs(ch, stop, done) })
}
ch := make(chan logRecord, l.getConfig().BufferSize)
stop, done := make(chan struct{}), make(chan struct{})
flush := make(chan flushRequest, 1)
l.processorErr = nil
l.sendMu.Lock()
l.state.ActiveLogChannel.Store(ch)
l.state.ProcStop.Store(stop)
l.state.ProcDone.Store(done)
l.state.flushRequestChan = flush
l.state.ProcessorExited.Store(false)
l.state.Started.Store(true)
l.refreshLevelGate()
l.sendMu.Unlock()
l.spawn(func() { l.processLogs(ch, stop, done, flush) })
return nil
}
// Stop halts log processing. Can be restarted with Start()
// The record channel is never closed: producers are detached to a nil channel
// first, so a concurrent send falls through to the drop counter instead of
// racing a close.
// Stop detaches producers, drains accepted records, and joins the processor.
// After a timeout, Stop can be retried; Start and resource replacement remain
// blocked until that processor exits. The default timeout is 2x flush interval.
func (l *Logger) Stop(timeout ...time.Duration) error {
if !l.state.Started.CompareAndSwap(true, false) {
return nil // Already stopped
}
l.refreshLevelGate()
l.initMu.Lock()
defer l.initMu.Unlock()
return l.stopLocked(timeout...)
}
// Calculate effective timeout
var effectiveTimeout time.Duration
func (l *Logger) stopLocked(timeout ...time.Duration) error {
l.sendMu.Lock()
if l.state.Started.Swap(false) {
l.refreshLevelGate()
l.state.ActiveLogChannel.Store((chan logRecord)(nil))
close(l.state.ProcStop.Load().(chan struct{}))
}
l.sendMu.Unlock()
done, _ := l.state.ProcDone.Load().(chan struct{})
if done == nil {
return nil
}
effectiveTimeout := 2 * time.Duration(l.getConfig().FlushIntervalMs) * time.Millisecond
if len(timeout) > 0 {
effectiveTimeout = timeout[0]
} else {
effectiveTimeout = 2 * time.Duration(l.getConfig().FlushIntervalMs) * time.Millisecond
}
if effectiveTimeout < minWaitTime {
effectiveTimeout = minWaitTime
}
// 1. Detach producers
l.state.ActiveLogChannel.Store((chan logRecord)(nil))
// 2. Signal the processor to drain and exit
if s, ok := l.state.ProcStop.Load().(chan struct{}); ok && s != nil {
close(s)
}
// 3. Join
d, _ := l.state.ProcDone.Load().(chan struct{})
if d == nil {
return nil
}
effectiveTimeout = max(effectiveTimeout, minWaitTime)
// Prefer an already completed processor even with a tiny timeout.
select {
case <-d:
return nil
case <-time.After(effectiveTimeout):
case <-done:
return l.processorErr
default:
}
timer := time.NewTimer(effectiveTimeout)
defer timer.Stop()
select {
case <-done:
return l.processorErr
case <-timer.C:
return fmtErrorf("processor did not exit within timeout (%v)", effectiveTimeout)
}
}
// Shutdown gracefully closes the logger, attempting to flush pending records
// If no timeout is provided, uses a default of 2x flush interval
// Shutdown stops the logger and closes its file after the processor exits.
// If joining times out, resources remain owned by the processor; retry Shutdown
// to finish cleanup. A completed shutdown can be reset by ApplyConfig.
func (l *Logger) Shutdown(timeout ...time.Duration) error {
if !l.state.ShutdownCalled.CompareAndSwap(false, true) {
l.initMu.Lock()
defer l.initMu.Unlock()
if !l.state.IsInitialized.Load() {
return nil
}
l.state.ShutdownCalled.Store(true)
l.state.LoggerDisabled.Store(true)
l.refreshLevelGate()
if !l.state.IsInitialized.Load() {
l.state.ShutdownCalled.Store(false)
l.state.LoggerDisabled.Store(false)
l.state.ProcessorExited.Store(true)
l.refreshLevelGate()
return nil
err := l.stopLocked(timeout...)
if !l.state.ProcessorExited.Load() {
return err
}
var stopErr error
if l.state.Started.Load() {
stopErr = l.Stop(timeout...)
}
l.state.IsInitialized.Store(false)
var finalErr error
cfPtr := l.state.CurrentFile.Load()
if cfPtr != nil {
if currentLogFile, ok := cfPtr.(*os.File); ok && currentLogFile != nil {
if err := currentLogFile.Sync(); err != nil {
syncErr := fmtErrorf("failed to sync log file '%s' during shutdown: %w", currentLogFile.Name(), err)
finalErr = errors.Join(finalErr, syncErr)
}
if err := currentLogFile.Close(); err != nil {
closeErr := fmtErrorf("failed to close log file '%s' during shutdown: %w", currentLogFile.Name(), err)
finalErr = errors.Join(finalErr, closeErr)
}
l.state.CurrentFile.Store((*os.File)(nil))
}
if file, _ := l.state.CurrentFile.Load().(*os.File); file != nil {
err = errors.Join(err, file.Sync(), file.Close())
l.state.CurrentFile.Store((*os.File)(nil))
}
if stopErr != nil {
finalErr = errors.Join(finalErr, stopErr)
}
return finalErr
return err
}
// Flush explicitly triggers a sync of the current log file buffer to disk and waits for completion or timeout
// Flush processes records queued before the request and syncs the current file.
// The timeout covers queueing and confirmation. Concurrent later records need
// not be drained, so sustained producers cannot starve the barrier.
func (l *Logger) Flush(timeout time.Duration) error {
l.state.flushMutex.Lock()
defer l.state.flushMutex.Unlock()
// State checks
l.sendMu.RLock()
if !l.state.IsInitialized.Load() || l.state.ShutdownCalled.Load() {
l.sendMu.RUnlock()
return fmtErrorf("logger not initialized or already shut down")
}
if !l.state.Started.Load() {
l.sendMu.RUnlock()
return fmtErrorf("logger not started")
}
// Create a channel to wait for confirmation from the processor
confirmChan := make(chan struct{})
// Send the request with the confirmation channel
select {
case l.state.flushRequestChan <- confirmChan:
// Request sent
case <-time.After(minWaitTime): // Short timeout to prevent blocking if processor is stuck
return fmtErrorf("failed to send flush request to processor (possible deadlock or high load)")
requests := l.state.flushRequestChan
done := l.state.ProcDone.Load().(chan struct{})
l.sendMu.RUnlock()
if timeout <= 0 {
return fmtErrorf("timeout waiting for flush confirmation (%v)", timeout)
}
timer := time.NewTimer(timeout)
defer timer.Stop()
request := flushRequest{done: make(chan error, 1)}
select {
case <-confirmChan:
return nil
case <-time.After(timeout):
case requests <- request:
case <-done:
return fmtErrorf("processor stopped during flush")
case <-timer.C:
return fmtErrorf("timeout sending flush request (%v)", timeout)
}
select {
case err := <-request.done:
return err
case <-done:
return fmtErrorf("processor stopped during flush")
case <-timer.C:
return fmtErrorf("timeout waiting for flush confirmation (%v)", timeout)
}
}
// SetSpawn installs the goroutine launcher used by Start. Hosts that own
// panic recovery and terminal teardown pass their own launcher here.
// Call before Start; a nil fn restores the default.
// Call before Start; fn must launch asynchronously and return promptly.
// A nil fn restores the default.
func (l *Logger) SetSpawn(fn func(func())) {
if fn == nil {
l.spawner.Store(nil)
@@ -420,7 +372,8 @@ func (l *Logger) SetSpawn(fn func(func())) {
}
// SetErrorHandler routes internal diagnostics to fn instead of stderr.
// Required for TUI hosts, where stderr writes corrupt the display.
// Required for TUI hosts, where stderr writes corrupt the display. The callback
// must return promptly and must not call lifecycle/configuration methods or Flush.
func (l *Logger) SetErrorHandler(fn func(string)) {
if fn == nil {
l.errHandler.Store(nil)
@@ -450,15 +403,16 @@ func (l *Logger) SetLevel(level int64) {
cfg := l.getConfig().Clone()
cfg.Level = level
l.currentConfig.Store(cfg)
l.initMu.Unlock()
l.refreshLevelGate()
l.initMu.Unlock()
}
// Enabled reports whether a record at level would be emitted. Single atomic
// load: the intended guard for hot call sites, where argument slices are
// built before the call and would otherwise escape to the heap.
func (l *Logger) Enabled(level int64) bool {
return level >= l.state.Level.Load()
threshold := l.state.Level.Load()
return threshold != levelOff && level >= threshold
}
// Flags returns the default record flags derived from display config
@@ -468,7 +422,7 @@ func (l *Logger) Flags() int64 {
// LogContext emits a record with caller-supplied context and explicit flags
func (l *Logger) LogContext(ctx Context, flags, level, depth int64, args ...any) {
if level < l.state.Level.Load() {
if !l.Enabled(level) {
return
}
l.emit(ctx, flags, level, depth, args)
@@ -591,7 +545,10 @@ func (l *Logger) LogTrace(depth int, args ...any) {
// LogStructured logs a message with structured fields as proper JSON
func (l *Logger) LogStructured(level int64, message string, fields map[string]any) {
l.LogContext(Context{}, l.getFlags()|FlagStructuredJSON, level, 0, message, fields)
if !l.Enabled(level) {
return
}
l.LogContext(Context{}, l.getFlags()|FlagStructuredJSON, level, 0, message, maps.Clone(fields))
}
// Write outputs raw, unformatted data ignoring configured format and sanitization
processor.go
+57 -46
View File
@@ -1,6 +1,7 @@
package log
import (
"io"
"os"
"time"
@@ -10,7 +11,7 @@ import (
// processLogs is the main log processing loop running in a separate goroutine.
// Exits on stop, draining buffered records first. No panic recovery: a fault
// here is fatal by design and is surfaced by the host's spawner.
func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done chan<- struct{}) {
func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done chan<- struct{}, flush <-chan flushRequest) {
defer func() {
l.state.ProcessorExited.Store(true)
close(done)
@@ -51,7 +52,7 @@ func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done cha
select {
case <-stop:
l.drain(ch)
l.performSync()
l.processorErr = l.performSync()
return
case record := <-ch:
@@ -84,9 +85,9 @@ func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done cha
lastCheckTime = time.Now()
}
case confirmChan := <-l.state.flushRequestChan:
case request := <-flush:
// Barrier: drain queued records before sync
l.handleFlushRequest(ch, confirmChan)
l.handleFlushRequest(ch, request)
case <-timers.retentionChan:
l.handleRetentionCheck()
@@ -99,22 +100,16 @@ func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done cha
// drain processes every buffered record without blocking
func (l *Logger) drain(ch <-chan logRecord) {
for {
select {
case record := <-ch:
l.processLogRecord(record)
default:
return
}
// A snapshot bounds the flush work even while producers keep the queue full.
// Stop has already joined producers, so this also drains its entire queue.
for range len(ch) {
l.processLogRecord(<-ch)
}
}
// handleFlushRequest drains pending records, then syncs. Gives Flush barrier
// semantics: records enqueued before the Flush call are processed first.
func (l *Logger) handleFlushRequest(ch <-chan logRecord, confirmChan chan struct{}) {
func (l *Logger) handleFlushRequest(ch <-chan logRecord, request flushRequest) {
l.drain(ch)
l.performSync()
close(confirmChan)
request.done <- l.performSync()
}
// processLogRecord handles individual log records and returns bytes written
@@ -123,8 +118,7 @@ func (l *Logger) processLogRecord(record logRecord) int64 {
enableFile := c.EnableFile
if enableFile && !l.state.DiskStatusOK.Load() {
// Simple increment of both counters
l.state.DroppedLogs.Add(1)
l.state.TotalDroppedLogs.Add(1)
l.handleFailedSend()
return 0
}
@@ -147,31 +141,33 @@ func (l *Logger) processLogRecord(record logRecord) int64 {
)
formattedDataLen := int64(len(formattedData))
// Write to console if enabled
enableConsole := c.EnableConsole
if enableConsole {
if s := l.state.StdoutWriter.Load(); s != nil {
if sinkWrapper, ok := s.(*sink); ok && sinkWrapper != nil {
// Handle split mode
if c.ConsoleTarget == "split" {
if record.Level >= LevelWarn {
// Write WARN and ERROR to stderr
_, _ = os.Stderr.Write(formattedData)
} else {
// Write INFO and DEBUG to stdout
_, _ = sinkWrapper.w.Write(formattedData)
}
} else {
// Write to the configured target (stdout or stderr)
_, _ = sinkWrapper.w.Write(formattedData)
}
var consoleErr error
if c.EnableConsole {
if target, _ := l.state.StdoutWriter.Load().(*sink); target != nil {
writer := target.w
if c.ConsoleTarget == "split" && record.Level >= LevelWarn {
writer = os.Stderr
}
n, err := writer.Write(formattedData)
if err == nil && n != len(formattedData) {
err = io.ErrShortWrite
}
consoleErr = err
if err != nil {
l.internalLog("failed to write to console: %v\n", err)
}
}
}
// Skip file operations if file output is disabled
if !enableFile {
l.state.TotalLogsProcessed.Add(1)
if consoleErr != nil {
l.handleFailedSend()
return 0
}
if !record.heartbeat {
l.state.TotalLogsProcessed.Add(1)
}
return formattedDataLen // Return data length for adaptive interval calculations
}
@@ -184,7 +180,7 @@ func (l *Logger) processLogRecord(record logRecord) int64 {
if err := l.rotateLogFile(); err != nil {
l.internalLog("failed to rotate log file: %v\n", err)
// Account for the dropped log that triggered the failed rotation
l.state.DroppedLogs.Add(1)
l.handleFailedSend()
return 0
}
}
@@ -195,16 +191,21 @@ func (l *Logger) processLogRecord(record logRecord) int64 {
n, err := currentLogFile.Write(formattedData)
if err != nil {
l.internalLog("failed to write to log file: %v\n", err)
l.state.DroppedLogs.Add(1)
l.handleFailedSend()
l.state.CurrentSize.Add(int64(n))
l.performDiskCheck(true)
return 0
} else {
l.state.CurrentSize.Add(int64(n))
l.state.TotalLogsProcessed.Add(1)
if consoleErr != nil {
l.handleFailedSend()
} else if !record.heartbeat {
l.state.TotalLogsProcessed.Add(1)
}
return int64(n)
}
} else {
l.state.DroppedLogs.Add(1)
l.handleFailedSend()
return 0
}
}
@@ -214,7 +215,9 @@ func (l *Logger) handleFlushTick() {
c := l.getConfig()
enableSync := c.EnablePeriodicSync
if enableSync {
l.performSync()
if err := l.performSync(); err != nil {
l.internalLog("failed to sync log file: %v\n", err)
}
}
}
@@ -224,7 +227,7 @@ func (l *Logger) handleRetentionCheck() {
retentionPeriodHrs := c.RetentionPeriodHrs
retentionDur := time.Duration(retentionPeriodHrs * float64(time.Hour))
if retentionDur > 0 {
if c.EnableFile && retentionDur > 0 {
etPtr := l.state.EarliestFileTime.Load()
if earliest, ok := etPtr.(time.Time); ok && !earliest.IsZero() {
if time.Since(earliest) > retentionDur {
@@ -253,13 +256,20 @@ func (l *Logger) adjustDiskCheckInterval(timers *TimerSet, lastCheckTime time.Ti
logsPerSecond := float64(logsSinceLastCheck) / elapsed.Seconds()
targetLogsPerSecond := float64(100) // Baseline
diskCheckIntervalMs := c.DiskCheckIntervalMs
currentDiskCheckInterval := time.Duration(diskCheckIntervalMs) * time.Millisecond
currentDiskCheckInterval := timers.diskInterval
if currentDiskCheckInterval == 0 {
currentDiskCheckInterval = time.Duration(c.DiskCheckIntervalMs) * time.Millisecond
}
// Calculate the new interval
var newInterval time.Duration
if logsPerSecond < targetLogsPerSecond/2 { // Load low -> increase interval
newInterval = time.Duration(float64(currentDiskCheckInterval) * adaptiveIntervalFactor)
scaled := float64(currentDiskCheckInterval) * adaptiveIntervalFactor
if scaled >= float64(time.Duration(c.MaxCheckIntervalMs)*time.Millisecond) {
newInterval = time.Duration(c.MaxCheckIntervalMs) * time.Millisecond
} else {
newInterval = time.Duration(scaled)
}
} else if logsPerSecond > targetLogsPerSecond*2 { // Load high -> decrease interval
newInterval = time.Duration(float64(currentDiskCheckInterval) * adaptiveSpeedUpFactor)
} else {
@@ -280,5 +290,6 @@ func (l *Logger) adjustDiskCheckInterval(timers *TimerSet, lastCheckTime time.Ti
newInterval = maxCheckInterval
}
timers.diskInterval = newInterval
timers.diskCheckTicker.Reset(newInterval)
}
processor_test.go
+86 -189
View File
@@ -1,40 +1,14 @@
package log
import (
"bytes"
"encoding/json"
"strings"
"testing"
"testing/synctest"
"time"
)
// // procRecords parses PROC heartbeat records out of json-formatted content.
// // Heartbeat arguments are emitted as a flat key/value array.
// func procRecords(tb testing.TB, content string) []map[string]any {
// tb.Helper()
// var out []map[string]any
// for _, line := range strings.Split(content, "\n") {
// if !strings.Contains(line, `"level":"PROC"`) {
// continue
// }
// var entry map[string]any
// if json.Unmarshal([]byte(line), &entry) != nil {
// continue
// }
// fields, ok := entry["fields"].([]any)
// if !ok {
// continue
// }
// rec := make(map[string]any, len(fields)/2)
// for i := 0; i+1 < len(fields); i += 2 {
// if key, ok := fields[i].(string); ok {
// rec[key] = fields[i+1]
// }
// }
// out = append(out, rec)
// }
// return out
// }
// procRecords parses PROC heartbeat records out of json-formatted content.
// Heartbeat arguments are emitted as a keyed object (FlagKV); the flat
// key/value array is still accepted for records written without the flag.
@@ -100,178 +74,101 @@ func TestLoggerHeartbeat(t *testing.T) {
// TestHeartbeatDisabled verifies level 0 emits nothing.
func TestHeartbeatDisabled(t *testing.T) {
logger, tmpDir := newTestLogger(t)
cfg := logger.GetConfig()
cfg.Format = "json"
cfg.HeartbeatLevel = 0
cfg.HeartbeatIntervalS = 1
mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig")
logger.Info("marker")
mustNoErr(t, logger.Flush(time.Second), "Flush")
time.Sleep(1200 * time.Millisecond) // span at least one interval
content := readLog(t, tmpDir)
contains(t, content, "marker", "regular record")
notContains(t, content, `"level":"PROC"`, "proc heartbeat")
equal(t, logger.state.HeartbeatSequence.Load(), uint64(0), "HeartbeatSequence")
synctest.Test(t, func(t *testing.T) {
l, out := memoryHeartbeatLogger(t, 0)
mustNoErr(t, l.Start(), "start")
synctest.Sleep(2 * time.Second)
l.Info("marker")
mustNoErr(t, l.Flush(time.Second), "flush")
contains(t, out.String(), "marker", "ordinary record")
notContains(t, out.String(), `"level":"PROC"`, "disabled heartbeat")
equal(t, l.state.HeartbeatSequence.Load(), uint64(0), "no heartbeat sequence")
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
})
}
// TestDroppedLogs verifies buffer overflow is counted and reported by the heartbeat.
func memoryHeartbeatLogger(t *testing.T, level int64) (*Logger, *bytes.Buffer) {
t.Helper()
l := NewLogger()
c := DefaultConfig()
c.Format = "json"
c.HeartbeatLevel = level
c.HeartbeatIntervalS = 1
c.BufferSize = 1
mustNoErr(t, l.ApplyConfig(c), "apply")
out := new(bytes.Buffer)
l.state.StdoutWriter.Store(&sink{w: out})
return l, out
}
// The processor cannot start until the flood is complete, making overflow exact.
func TestDroppedLogs(t *testing.T) {
logger := NewLogger()
cfg := DefaultConfig()
cfg.Directory = t.TempDir()
cfg.EnableConsole = false
cfg.EnableFile = true
cfg.Format = "json"
cfg.BufferSize = 1 // guarantees drops under flood
cfg.FlushIntervalMs = 10
cfg.HeartbeatLevel = 1
cfg.HeartbeatIntervalS = 1
mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig")
mustNoErr(t, logger.Start(), "Start")
t.Cleanup(func() { _ = logger.Shutdown() })
for i := range 100 {
logger.Info("flood", i)
}
dropped := logger.state.TotalDroppedLogs.Load()
if dropped == 0 {
t.Fatal("flood produced no drops")
}
// The interval counter is reported only when non-zero, so wait for the
// tick-driven heartbeat that follows the flood
mustEventually(t, 5*time.Second, "heartbeat reporting interval drops", func() bool {
for _, rec := range procRecords(t, readLog(t, cfg.Directory)) {
if _, ok := rec["dropped_since_last"]; ok {
return true
}
synctest.Test(t, func(t *testing.T) {
l, out := memoryHeartbeatLogger(t, 1)
release := make(chan struct{})
l.SetSpawn(func(fn func()) { go func() { <-release; fn() }() })
mustNoErr(t, l.Start(), "start")
for i := range 100 {
l.Info("flood", i)
}
return false
equal(t, l.state.TotalDroppedLogs.Load(), uint64(99), "exact overflow")
close(release)
synctest.Wait()
// The initial heartbeat encounters the full queue and adds one more drop.
synctest.Sleep(time.Second)
records := procRecords(t, out.String())
mustEqual(t, len(records), 1, "tick heartbeat")
equal(t, numField(records[0], "total_dropped_logs"), float64(100), "includes dropped initial heartbeat")
equal(t, numField(records[0], "processed_logs"), float64(1), "excludes heartbeats")
mustNoErr(t, l.Shutdown(time.Second), "shutdown")
})
records := procRecords(t, readLog(t, cfg.Directory))
last := records[len(records)-1]
if got := numField(last, "total_dropped_logs"); got < float64(dropped) {
t.Errorf("total_dropped_logs %v below observed drops %d", got, dropped)
}
}
// TestDroppedHeartbeatAccounting verifies a heartbeat discarded by the processor
// during a disk failure is still reflected in the total drop count reported by
// the next successful heartbeat.
func TestDroppedHeartbeatAccounting(t *testing.T) {
logger := NewLogger()
cfg := DefaultConfig()
cfg.Directory = t.TempDir()
cfg.EnableConsole = false
cfg.EnableFile = true
cfg.Format = "json"
cfg.BufferSize = 10
cfg.HeartbeatLevel = 1
cfg.HeartbeatIntervalS = 1
cfg.InternalErrorsToStderr = false // internal logs would add drops
mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig")
mustNoErr(t, logger.Start(), "Start")
t.Cleanup(func() { _ = logger.Shutdown() })
// Drops during the flood are nondeterministic; capture the actual count
for i := range int(cfg.BufferSize) + 50 {
logger.Info("flood", i)
}
floodDrops := logger.state.TotalDroppedLogs.Load()
if floodDrops == 0 {
t.Fatal("flood produced no drops")
}
// Let the first tick-driven heartbeat consume the interval counter
mustEventually(t, 3*time.Second, "first tick heartbeat", func() bool {
return logger.state.HeartbeatSequence.Load() >= 2
})
// Force the disk-unavailable state; the processor discards every record
diskFull := logger.GetConfig()
diskFull.MinDiskFreeKB = 1 << 40
mustNoErr(t, logger.ApplyConfig(diskFull), "ApplyConfig disk full")
isFalse(t, logger.performDiskCheck(true), "performDiskCheck under disk full")
isFalse(t, logger.state.DiskStatusOK.Load(), "DiskStatusOK")
// Hold the failure until a heartbeat has been produced and discarded
seq := logger.state.HeartbeatSequence.Load()
mustEventually(t, 3*time.Second, "heartbeat produced while disk full", func() bool {
return logger.state.HeartbeatSequence.Load() > seq
})
droppedWithDiskFull := logger.state.TotalDroppedLogs.Load()
if droppedWithDiskFull <= floodDrops {
t.Fatalf("processor did not drop during disk failure: %d", droppedWithDiskFull)
}
// Restore and wait for a heartbeat that reaches the file
diskOK := logger.GetConfig()
diskOK.MinDiskFreeKB = 0
mustNoErr(t, logger.ApplyConfig(diskOK), "ApplyConfig disk ok")
isTrue(t, logger.performDiskCheck(true), "performDiskCheck after recovery")
isTrue(t, logger.state.DiskStatusOK.Load(), "DiskStatusOK after recovery")
seq = logger.state.HeartbeatSequence.Load()
mustEventually(t, 4*time.Second, "heartbeat written after recovery", func() bool {
if logger.state.HeartbeatSequence.Load() <= seq {
return false
}
records := procRecords(t, readLog(t, cfg.Directory))
if len(records) == 0 {
return false
}
return numField(records[len(records)-1], "sequence") > float64(seq)
})
records := procRecords(t, readLog(t, cfg.Directory))
last := records[len(records)-1]
// The dropped heartbeat is unrecoverable in the interval counter but must
// remain visible in the monotonic total
if got := numField(last, "total_dropped_logs"); got < float64(droppedWithDiskFull) {
t.Errorf("total_dropped_logs %v does not cover drops observed during failure %d",
got, droppedWithDiskFull)
}
if got := numField(last, "processed_logs"); got == 0 {
t.Error("processed_logs must be non-zero after recovery")
}
l := NewLogger()
c := DefaultConfig()
c.EnableConsole = false
c.EnableFile = true
c.Directory = t.TempDir()
c.Format = "json"
mustNoErr(t, l.ApplyConfig(c), "apply")
defer l.Shutdown()
ch := make(chan logRecord, 1)
l.state.ActiveLogChannel.Store(ch)
l.processLogRecord(logRecord{Args: []any{"application record"}})
l.state.DiskStatusOK.Store(false)
l.logProcHeartbeat()
l.processLogRecord(<-ch)
equal(t, l.state.TotalDroppedLogs.Load(), uint64(1), "dropped heartbeat")
l.state.DiskStatusOK.Store(true)
l.logProcHeartbeat()
l.processLogRecord(<-ch)
records := procRecords(t, readLog(t, c.Directory))
mustEqual(t, len(records), 1, "recovery heartbeat")
equal(t, numField(records[0], "total_dropped_logs"), float64(1), "persistent drops")
equal(t, numField(records[0], "dropped_since_last"), float64(1), "interval drops")
equal(t, l.state.TotalLogsProcessed.Load(), uint64(1), "heartbeat excluded from processed count")
}
// TestAdaptiveDiskCheck exercises interval adjustment under varying log rates.
func TestAdaptiveDiskCheck(t *testing.T) {
logger, _ := newTestLogger(t)
cfg := logger.GetConfig()
cfg.EnableAdaptiveInterval = true
cfg.DiskCheckIntervalMs = 100
cfg.MinCheckIntervalMs = 50
cfg.MaxCheckIntervalMs = 500
mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig")
// Low rate, then burst: both adjustment branches
for i := range 10 {
logger.Info("adaptive test", i)
time.Sleep(10 * time.Millisecond)
}
for i := range 100 {
logger.Info("burst", i)
}
mustNoErr(t, logger.Flush(2*time.Second), "Flush")
isTrue(t, logger.state.DiskStatusOK.Load(), "DiskStatusOK")
if logger.state.TotalLogsProcessed.Load() == 0 {
t.Error("no records processed")
}
synctest.Test(t, func(t *testing.T) {
l := NewLogger()
c := DefaultConfig()
c.DiskCheckIntervalMs = 100
c.MinCheckIntervalMs = 50
c.MaxCheckIntervalMs = 500
mustNoErr(t, l.ApplyConfig(c), "apply")
timers := l.setupProcessingTimers()
defer l.stopProcessingTimers(timers)
for range 8 {
l.adjustDiskCheckInterval(timers, time.Now().Add(-time.Second), 0)
}
equal(t, timers.diskInterval, 500*time.Millisecond, "repeated low load reaches maximum")
for range 16 {
l.adjustDiskCheckInterval(timers, time.Now().Add(-time.Second), 1000)
}
equal(t, timers.diskInterval, 50*time.Millisecond, "repeated high load reaches minimum")
})
}
// TestFlushBarrier verifies records enqueued before Flush are written before it returns.
record.go
+6 -43
View File
@@ -3,6 +3,7 @@ package log
import (
"fmt"
"os"
"slices"
"strings"
"time"
)
@@ -18,9 +19,11 @@ func (l *Logger) getFlags() int64 {
return l.state.Flags.Load()
}
// sendLogRecord queues a record without blocking. The channel is never closed,
// so no recovery is needed: a detached (nil) channel takes the default branch.
// sendLogRecord never waits for queue capacity or I/O. The short read lock
// ensures Stop cannot finish draining while a producer still holds the old queue.
func (l *Logger) sendLogRecord(record logRecord) {
l.sendMu.RLock()
defer l.sendMu.RUnlock()
if l.state.LoggerDisabled.Load() {
l.handleFailedSend()
return
@@ -54,7 +57,7 @@ func (l *Logger) emit(ctx Context, flags, level, depth int64, args []any) {
TimeStamp: time.Now(),
Level: level,
Trace: trace,
Args: args,
Args: slices.Clone(args),
})
}
@@ -75,43 +78,3 @@ func (l *Logger) internalLog(format string, args ...any) {
}
fmt.Fprintf(os.Stderr, format, args...)
}
// // log handles the core logging logic
// func (l *Logger) log(flags int64, level int64, depth int64, args ...any) {
// // State checks
// if !l.state.IsInitialized.Load() {
// return
// }
//
// if !l.state.Started.Load() {
// // Log to internal error channel if configured
// cfg := l.getConfig()
// if cfg.InternalErrorsToStderr {
// l.internalLog("warning - logger not started, dropping log entry\n")
// }
// return
// }
//
// // Discard or proceed based on level
// cfg := l.getConfig()
// if level < cfg.Level {
// return
// }
//
// // Get trace info from runtime
// // Depth filter hard-coded based on call stack of current package design
// var trace string
// if depth > 0 {
// const skipTrace = 3 // log.Info -> log -> getTrace (Adjust if call stack changes)
// trace = getTrace(depth, skipTrace)
// }
//
// record := logRecord{
// Flags: flags,
// TimeStamp: time.Now(),
// Level: level,
// Trace: trace,
// Args: args,
// }
// l.sendLogRecord(record)
// }
sanitizer/fuzz_test.go
+92
View File
@@ -0,0 +1,92 @@
package sanitizer
import (
"encoding/json"
"strings"
"testing"
"unicode"
"unicode/utf8"
)
func FuzzSanitize(f *testing.F) {
for _, s := range []string{"", "plain", "\xff\xfe\n\x80", "<00>\x00", "\u0085\u009b[31m", "世界\u202e", "a'$(id)\\*"} {
f.Add(s, uint8(0), uint64(FilterControl), uint64(TransformHexEncode))
}
f.Fuzz(func(t *testing.T, input string, policy uint8, filter, transform uint64) {
policies := []PolicyPreset{PolicyRaw, PolicyTxt, PolicyJSON, PolicyShell}
s := New().Policy(policies[int(policy)%len(policies)]).Rule(filter, transform)
want := s.Sanitize(input)
got := s.AppendSanitize([]byte("prefix:"), input)
if string(got) != "prefix:"+want {
t.Fatalf("append/string disagreement: %q != %q", got, want)
}
if want != s.Sanitize(input) {
t.Fatal("non-deterministic sanitizer")
}
if policy%4 == 3 {
for _, r := range New().Policy(PolicyShell).Sanitize(input) {
if unicode.IsControl(r) || unicode.IsSpace(r) || matchesFilter(r, FilterShellSpecial) {
t.Fatalf("shell filter leaked %U", r)
}
}
}
})
}
func FuzzSerializerJSON(f *testing.F) {
for _, s := range []string{"", "\xff\xfe\x00", "a\"\\\nb", "\u0085", "世界"} {
f.Add(s, uint8(0))
}
f.Fuzz(func(t *testing.T, input string, policy uint8) {
policies := []PolicyPreset{PolicyRaw, PolicyTxt, PolicyJSON, PolicyShell}
s := New().Policy(policies[int(policy)%len(policies)])
out := []byte("prefix:")
NewSerializer("json", s).WriteString(&out, input)
if !strings.HasPrefix(string(out), "prefix:") {
t.Fatal("prefix changed")
}
data := out[len("prefix:"):]
if !utf8.Valid(data) || !json.Valid(data) {
t.Fatalf("invalid JSON string %q", data)
}
var got, want string
if err := json.Unmarshal(data, &got); err != nil {
t.Fatal(err)
}
// encoding/json is an independent oracle for malformed UTF-8 replacement.
oracle, err := json.Marshal(s.Sanitize(input))
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(oracle, &want); err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("round trip: %q != %q", got, want)
}
})
}
func TestSanitizerMalformedUTF8AndTransforms(t *testing.T) {
for _, input := range []string{"\xff", "\xff\n\xfe", "\n\xff"} {
s := New().Policy(PolicyJSON)
eq(t, string(s.AppendSanitize(nil, input)), s.Sanitize(input), "malformed UTF-8 consistency")
}
eq(t, New().Rule(FilterControl, 0).Sanitize("a\x00b"), "a\x00b", "zero transform preserves")
eq(t, New().Policy(PolicyJSON).Sanitize("\u0085\u009b"), `\u0085\u009b`, "C1 controls escaped")
var out []byte
NewSerializer("unknown", nil).WriteString(&out, "a b")
eq(t, string(out), `"a b"`, "nil sanitizer and unknown format")
}
func TestSerializerInvalidNumbers(t *testing.T) {
for _, input := range []string{"", "NaN", "+Inf", "-Inf", "01", "1,2", "null", "1\n", "1 "} {
var out []byte
NewSerializer("json", nil).WriteNumber(&out, input)
var got string
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("%q must be represented as string: %q (%v)", input, out, err)
}
eq(t, got, input, "invalid number round trip")
}
}
sanitizer/sanitizer.go
+132 -70
View File
@@ -9,8 +9,12 @@ package sanitizer
import (
"encoding/hex"
"encoding/json"
"encoding/json/jsontext"
"fmt"
"reflect"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
@@ -20,7 +24,7 @@ const (
FilterNonPrintable uint64 = 1 << iota // Matches runes not classified as printable by strconv.IsPrint
FilterControl // Matches control characters (unicode.IsControl)
FilterWhitespace // Matches whitespace characters (unicode.IsSpace)
FilterShellSpecial // Matches common shell metacharacters: '`', '$', ';', '|', '&', '>', '<', '(', ')', '#'
FilterShellSpecial // Matches shell metacharacters including quotes, backslash, globs, and: '`', '$', ';', '|', '&', '>', '<', '(', ')', '#'
)
// Transform flags for character transformation
@@ -38,7 +42,7 @@ const (
PolicyJSON PolicyPreset = "json" // Policy for sanitizing strings to be embedded in JSON
PolicyTxt PolicyPreset = "txt" // Policy for sanitizing text written to log files
// PolicyShell strips shell metacharacters, whitespace, and control characters. NOT sufficient for safe shell construction. Pass arguments via exec argv instead.
PolicyShell PolicyPreset = "shell" // Policy for sanitizing arguments passed to shell commands
PolicyShell PolicyPreset = "shell" // Lossy text filter; not shell quoting
)
// rule represents a single sanitization rule
@@ -62,27 +66,10 @@ var policyRules = map[PolicyPreset][]rule{
{fn: func(r rune) bool { return r == '<' }, transform: TransformHexEncode},
{filter: FilterNonPrintable, transform: TransformHexEncode},
},
// PolicyTxt: {{filter: FilterNonPrintable, transform: TransformHexEncode}},
PolicyJSON: {{filter: FilterControl, transform: TransformJSONEscape}},
PolicyShell: {{filter: FilterShellSpecial | FilterWhitespace | FilterControl, transform: TransformStrip}},
}
// filterCheckers maps individual filter flags to their check functions
var filterCheckers = map[uint64]func(rune) bool{
FilterNonPrintable: func(r rune) bool { return !strconv.IsPrint(r) },
FilterControl: unicode.IsControl,
FilterWhitespace: unicode.IsSpace,
FilterShellSpecial: func(r rune) bool {
switch r {
// CHANGED: D2 — added quotes, backslash, glob, braces, '~', '!'
case '`', '$', ';', '|', '&', '>', '<', '(', ')', '#',
'\'', '"', '\\', '*', '?', '[', ']', '{', '}', '~', '!':
return true
}
return false
},
}
// Sanitizer provides chainable text sanitization
type Sanitizer struct {
rules []rule
@@ -122,6 +109,7 @@ func (s *Sanitizer) Sanitize(data string) string {
if len(s.rules) == 0 {
return data
}
data = normalizeUTF8(data)
i := s.firstMatch(data)
if i < 0 {
return data
@@ -137,7 +125,16 @@ func (s *Sanitizer) AppendSanitize(dst []byte, data string) []byte {
if len(s.rules) == 0 {
return append(dst, data...)
}
return s.appendSanitized(dst, data)
return s.appendSanitized(dst, normalizeUTF8(data))
}
// Repair malformed sequences before removing characters: otherwise deleting
// separators can join invalid bytes into a new control rune or metacharacter.
func normalizeUTF8(data string) string {
if utf8.ValidString(data) {
return data
}
return strings.ToValidUTF8(data, "\ufffd")
}
// firstMatch returns the byte index of the first rune matching any rule, -1 if none
@@ -153,26 +150,36 @@ func (s *Sanitizer) firstMatch(data string) int {
}
func (s *Sanitizer) appendSanitized(dst []byte, data string) []byte {
for _, r := range data {
matched := false
start := 0
for i, r := range data {
for _, rl := range s.rules { // first match wins
if rl.matches(r) {
applyTransform(&dst, r, rl.transform)
matched = true
// A rule without a known transform is a passthrough rule.
if rl.transform&(TransformStrip|TransformHexEncode|TransformJSONEscape) != 0 {
dst = append(dst, data[start:i]...)
applyTransform(&dst, r, rl.transform)
_, size := utf8.DecodeRuneInString(data[i:])
start = i + size
}
break
}
}
if !matched {
dst = utf8.AppendRune(dst, r)
}
}
dst = append(dst, data[start:]...)
return dst
}
// matchesFilter checks if a rune matches any filter in the mask
func matchesFilter(r rune, filterMask uint64) bool {
for flag, checker := range filterCheckers {
if (filterMask&flag) != 0 && checker(r) {
if filterMask&FilterNonPrintable != 0 && !strconv.IsPrint(r) ||
filterMask&FilterControl != 0 && unicode.IsControl(r) ||
filterMask&FilterWhitespace != 0 && unicode.IsSpace(r) {
return true
}
if filterMask&FilterShellSpecial != 0 {
switch r {
case '`', '$', ';', '|', '&', '>', '<', '(', ')', '#',
'\'', '"', '\\', '*', '?', '[', ']', '{', '}', '~', '!':
return true
}
}
@@ -189,7 +196,7 @@ func applyTransform(buf *[]byte, r rune, transformMask uint64) {
var runeBytes [utf8.UTFMax]byte
n := utf8.EncodeRune(runeBytes[:], r)
*buf = append(*buf, '<')
*buf = append(*buf, hex.EncodeToString(runeBytes[:n])...)
*buf = hex.AppendEncode(*buf, runeBytes[:n])
*buf = append(*buf, '>')
case (transformMask & TransformJSONEscape) != 0:
@@ -209,7 +216,7 @@ func applyTransform(buf *[]byte, r rune, transformMask uint64) {
case '\\':
*buf = append(*buf, '\\', '\\')
default:
if r < 0x20 || r == 0x7f {
if r < 0x20 || r >= 0x7f && r <= 0x9f {
*buf = append(*buf, fmt.Sprintf("\\u%04x", r)...)
} else {
*buf = utf8.AppendRune(*buf, r)
@@ -226,6 +233,12 @@ type Serializer struct {
// NewSerializer creates a handler with format-specific behavior
func NewSerializer(format string, san *Sanitizer) *Serializer {
if san == nil {
san = New()
}
if format != "raw" && format != "json" && format != "txt" {
format = "txt"
}
return &Serializer{
format: format,
sanitizer: san,
@@ -256,50 +269,19 @@ func (se *Serializer) WriteString(buf *[]byte, s string) {
}
case "json":
// Sanitizer applied as content transform before escaping
s = se.sanitizer.Sanitize(s)
*buf = append(*buf, '"')
for i := 0; i < len(s); {
c := s[i]
// raw UTF-8 is valid in JSON strings. Only <0x20, '"', '\\', 0x7f are escaped.
if c >= 0x20 && c != '"' && c != '\\' && c != 0x7f {
start := i
for i < len(s) {
c = s[i]
if c >= 0x20 && c != '"' && c != '\\' && c != 0x7f {
i++
} else {
break
}
}
*buf = append(*buf, s[start:i]...)
} else {
switch c {
case '\\', '"':
*buf = append(*buf, '\\', c)
case '\n':
*buf = append(*buf, '\\', 'n')
case '\r':
*buf = append(*buf, '\\', 'r')
case '\t':
*buf = append(*buf, '\\', 't')
case '\b':
*buf = append(*buf, '\\', 'b')
case '\f':
*buf = append(*buf, '\\', 'f')
default:
*buf = append(*buf, fmt.Sprintf("\\u%04x", c)...)
}
i++
}
}
*buf = append(*buf, '"')
// AppendQuote always produces valid UTF-8 JSON. Its error only reports
// replacement of malformed UTF-8, which is intentional for log output.
*buf, _ = jsontext.AppendQuote(*buf, se.sanitizer.Sanitize(s))
}
}
// WriteNumber writes a number value
// WriteNumber writes a number; invalid JSON numbers are represented as strings.
func (se *Serializer) WriteNumber(buf *[]byte, n string) {
if se.format == "json" && (strings.TrimSpace(n) != n || len(n) == 0 || n[0] != '-' && (n[0] < '0' || n[0] > '9') || !json.Valid([]byte(n))) {
se.WriteString(buf, n)
return
}
*buf = append(*buf, n...)
}
@@ -320,10 +302,90 @@ func (se *Serializer) WriteNil(buf *[]byte) {
// WriteComplex writes complex types
func (se *Serializer) WriteComplex(buf *[]byte, v any) {
// fmt recursively traverses maps and slices without cycle detection. Bound
// that traversal before calling it; scalar and Stringer paths are separate.
budget := 10000
if !boundedValue(reflect.ValueOf(v), 0, &budget) {
se.WriteString(buf, "<cyclic or oversized value>")
return
}
str := fmt.Sprintf("%+v", v)
se.WriteString(buf, str)
}
// boundedValue limits depth and total traversal work without allocating a
// visited map. A cycle necessarily exceeds the depth bound. Application-defined
// formatting/marshal methods remain responsible for their own termination.
func boundedValue(v reflect.Value, depth int, budget *int) bool {
*budget -= 1
if depth > 64 || *budget < 0 {
return false
}
if !v.IsValid() {
return true
}
switch v.Kind() {
case reflect.Interface, reflect.Pointer:
return v.IsNil() || boundedValue(v.Elem(), depth+1, budget)
case reflect.Map:
iter := v.MapRange()
for iter.Next() {
if !boundedValue(iter.Key(), depth+1, budget) || !boundedValue(iter.Value(), depth+1, budget) {
return false
}
}
case reflect.Slice, reflect.Array:
for i := 0; i < v.Len(); i++ {
if !boundedValue(v.Index(i), depth+1, budget) {
return false
}
}
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
if !boundedValue(v.Field(i), depth+1, budget) {
return false
}
}
}
return true
}
// WriteJSON marshals v with encoding/json semantics, then applies the content
// policy to every JSON string (including nested keys). On error buf is unchanged.
func (se *Serializer) WriteJSON(buf *[]byte, v any) error {
data, err := json.Marshal(v)
if err != nil {
return err
}
if len(se.sanitizer.rules) == 0 {
*buf = append(*buf, data...)
return nil
}
// Marshal has already validated the JSON. Only string tokens need rewriting;
// copying the other bytes preserves numbers, booleans, and container shapes.
start := 0
for i := 0; i < len(data); i++ {
if data[i] != '"' {
continue
}
*buf = append(*buf, data[start:i]...)
end := i + 1
for data[end] != '"' {
if data[end] == '\\' {
end++
}
end++
}
var value string
_ = json.Unmarshal(data[i:end+1], &value)
quoted := Serializer{format: "json", sanitizer: se.sanitizer}
quoted.WriteString(buf, value)
i, start = end, end+1
}
*buf = append(*buf, data[start:]...)
return nil
}
// NeedsQuotes determines if quoting is needed
func (se *Serializer) NeedsQuotes(s string) bool {
switch se.format {
sanitizer/testdata/fuzz/FuzzSanitize/3120fc1ce48a8023
Vendored
+5
View File
@@ -0,0 +1,5 @@
go test fuzz v1
string("0000000000000000000\xc2 \x80")
byte('\x0f')
uint64(2)
uint64(2)
state.go
+1 -3
View File
@@ -1,7 +1,6 @@
package log
import (
"sync"
"sync/atomic"
)
@@ -23,8 +22,7 @@ type State struct {
TraceDepth atomic.Int64
// Flushing state
flushRequestChan chan chan struct{} // Channel to request a flush
flushMutex sync.Mutex // Protect concurrent Flush calls
flushRequestChan chan flushRequest // per processor; published under sendMu
// Outputs
CurrentFile atomic.Value // stores *os.File
storage.go
+60 -86
View File
@@ -5,37 +5,22 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// performSync syncs the current log file
func (l *Logger) performSync() {
c := l.getConfig()
// Skip sync if file output is disabled
enableFile := c.EnableFile
if !enableFile {
return
func (l *Logger) performSync() error {
if !l.getConfig().EnableFile {
return nil
}
cfPtr := l.state.CurrentFile.Load()
if cfPtr != nil {
if currentLogFile, isFile := cfPtr.(*os.File); isFile && currentLogFile != nil {
if err := currentLogFile.Sync(); err != nil {
// Log sync error
syncErrRecord := logRecord{
Flags: FlagDefault | FlagKV,
TimeStamp: time.Now(),
Level: LevelWarn,
Args: []any{
"msg", "log file sync failed",
"file", currentLogFile.Name(),
"error", err.Error(),
},
}
l.sendLogRecord(syncErrRecord)
}
if file, _ := l.state.CurrentFile.Load().(*os.File); file != nil {
if err := file.Sync(); err != nil {
return fmtErrorf("failed to sync log file '%s': %w", file.Name(), err)
}
return nil
}
return fmtErrorf("file output enabled without an open log file")
}
// performDiskCheck checks disk space, triggers cleanup if needed, and updates status
@@ -140,28 +125,6 @@ func (l *Logger) performDiskCheck(forceCleanup bool) bool {
}
}
// // getDiskFreeSpace retrieves available disk space for the given path
// func (l *Logger) getDiskFreeSpace(path string) (int64, error) {
// var stat syscall.Statfs_t
// info, err := os.Stat(path)
// if err != nil {
// if os.IsNotExist(err) {
// return 0, fmtErrorf("log directory '%s' does not exist for disk check: %w", path, err)
// }
// return 0, fmtErrorf("failed to stat log directory '%s': %w", path, err)
// }
// if !info.IsDir() {
// path = filepath.Dir(path)
// }
//
// if err := syscall.Statfs(path, &stat); err != nil {
// return 0, fmtErrorf("failed to get disk stats for '%s': %w", path, err)
// }
// // Explicit cast to int64 to satisfy both Linux and FreebSD
// availableBytes := int64(stat.Bavail) * int64(stat.Bsize)
// return availableBytes, nil
// }
// getLogDirSize calculates total size of log files matching the current extension
func (l *Logger) getLogDirSize(dir, ext string) (int64, error) {
var size int64
@@ -173,18 +136,15 @@ func (l *Logger) getLogDirSize(dir, ext string) (int64, error) {
return 0, fmtErrorf("failed to read log directory '%s': %w", dir, err)
}
targetExt := "." + ext
for _, entry := range entries {
if entry.IsDir() {
if !matchesLogEntry(entry, ext) {
continue
}
if filepath.Ext(entry.Name()) == targetExt {
info, errInfo := entry.Info()
if errInfo != nil {
continue
}
size += info.Size()
info, errInfo := entry.Info()
if errInfo != nil {
continue
}
size += info.Size()
}
return size, nil
}
@@ -192,6 +152,9 @@ func (l *Logger) getLogDirSize(dir, ext string) (int64, error) {
// cleanOldLogs removes oldest log files until required space is freed
func (l *Logger) cleanOldLogs(required int64) error {
c := l.getConfig()
if !c.EnableFile || required <= 0 {
return nil
}
dir := c.Directory
ext := c.Extension
name := c.Name
@@ -213,12 +176,8 @@ func (l *Logger) cleanOldLogs(required int64) error {
size int64
}
var logs []logFileMeta
targetExt := "." + ext
for _, entry := range entries {
if entry.IsDir() || entry.Name() == staticLogName {
continue
}
if ext != "" && filepath.Ext(entry.Name()) != targetExt {
if !matchesLogEntry(entry, ext) || entry.Name() == staticLogName {
continue
}
info, errInfo := entry.Info()
@@ -236,7 +195,12 @@ func (l *Logger) cleanOldLogs(required int64) error {
}
// Sort logs by modification time to delete the oldest ones first
sort.Slice(logs, func(i, j int) bool { return logs[i].modTime.Before(logs[j].modTime) })
sort.Slice(logs, func(i, j int) bool {
if logs[i].modTime.Equal(logs[j].modTime) {
return logs[i].name < logs[j].name
}
return logs[i].modTime.Before(logs[j].modTime)
})
// Iterate and remove files until enough space has been freed
var freedSpace int64
@@ -281,18 +245,14 @@ func (l *Logger) updateEarliestFileTime() {
staticLogName = name + "." + ext
}
targetExt := "." + ext
for _, entry := range entries {
if entry.IsDir() {
if !matchesLogEntry(entry, ext) {
continue
}
fname := entry.Name()
if fname == staticLogName {
continue // Skip the active log file
}
if ext != "" && filepath.Ext(fname) != targetExt {
continue
}
info, errInfo := entry.Info()
if errInfo != nil {
continue
@@ -313,7 +273,7 @@ func (l *Logger) cleanExpiredLogs(oldest time.Time) error {
retentionPeriodHrs := c.RetentionPeriodHrs
rpDuration := time.Duration(retentionPeriodHrs * float64(time.Hour))
if rpDuration <= 0 {
if !c.EnableFile || rpDuration <= 0 {
return nil
}
cutoffTime := time.Now().Add(-rpDuration)
@@ -332,14 +292,8 @@ func (l *Logger) cleanExpiredLogs(oldest time.Time) error {
staticLogName = name + "." + ext
}
targetExt := "." + ext
var deletedCount int
for _, entry := range entries {
if entry.IsDir() || entry.Name() == staticLogName {
continue
}
// Only consider files with correct extension
if ext != "" && filepath.Ext(entry.Name()) != targetExt {
if !matchesLogEntry(entry, ext) || entry.Name() == staticLogName {
continue
}
info, errInfo := entry.Info()
@@ -351,7 +305,6 @@ func (l *Logger) cleanExpiredLogs(oldest time.Time) error {
if err := os.Remove(filePath); err != nil {
l.internalLog("failed to remove expired log file '%s': %v\n", filePath, err)
} else {
deletedCount++
l.state.TotalDeletions.Add(1)
}
}
@@ -367,7 +320,7 @@ func (l *Logger) getStaticLogFilePath() string {
ext := c.Extension
name := c.Name
// Handle extension with or without dot
// Extension was validated without a leading dot
filename := name
if ext != "" {
filename = name + "." + ext
@@ -376,10 +329,10 @@ func (l *Logger) getStaticLogFilePath() string {
return filepath.Join(dir, filename)
}
// fileExists reports whether path names an existing regular file
// fileExists includes directories and symlinks, which also occupy archive names
func fileExists(path string) bool {
info, err := os.Stat(path)
return err == nil && !info.IsDir()
_, err := os.Lstat(path)
return err == nil
}
// generateArchiveLogFileName creates a second-resolution name for a rotated
@@ -394,16 +347,23 @@ func (l *Logger) generateArchiveLogFileName(timestamp time.Time) string {
base := fmt.Sprintf("%s_%s", c.Name, timestamp.Format("060102_150405"))
name := base + suffix
for i := 1; i < 1000 && fileExists(filepath.Join(c.Directory, name)); i++ {
for i := 1; fileExists(filepath.Join(c.Directory, name)); i++ {
name = fmt.Sprintf("%s_%d%s", base, i, suffix)
}
return name
}
// createNewLogFile generates a unique name and opens a new log file
// createNewLogFile opens the active log file in append mode
func (l *Logger) createNewLogFile() (*os.File, error) {
fullPath := l.getStaticLogFilePath()
return openLogFile(l.getConfig())
}
func openLogFile(c *Config) (*os.File, error) {
name := c.Name
if c.Extension != "" {
name += "." + c.Extension
}
fullPath := filepath.Join(c.Directory, name)
file, err := os.OpenFile(fullPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
return nil, fmtErrorf("failed to open/create log file '%s': %w", fullPath, err)
@@ -450,6 +410,8 @@ func (l *Logger) rotateLogFile() error {
// Continue with rotation anyway
}
l.state.CurrentFile.Store((*os.File)(nil))
// Generate a new unique name with current timestamp for the old log file
dir := c.Directory
archiveName := l.generateArchiveLogFileName(time.Now())
@@ -470,6 +432,8 @@ func (l *Logger) rotateLogFile() error {
// Create new log file at static path
newFile, err := l.createNewLogFile()
if err != nil {
l.state.LoggerDisabled.Store(true)
l.refreshLevelGate()
return fmtErrorf("failed to create new log file after rotation: %w", err)
}
@@ -495,15 +459,25 @@ func (l *Logger) getLogFileCount(dir, ext string) (int, error) {
return -1, fmtErrorf("failed to read log directory '%s': %w", dir, err)
}
targetExt := "." + ext
for _, entry := range entries {
if entry.IsDir() {
if !matchesLogEntry(entry, ext) {
continue
}
// Count all files matching the extension, including the current one if present
if filepath.Ext(entry.Name()) == targetExt {
count++
}
count++
}
return count, nil
}
// Cleanup is scoped to regular files with this extension, including files from
// earlier runs with different base names. Extensionless logs match only files
// without an extension. Each logger must own its directory exclusively.
func matchesLogEntry(entry os.DirEntry, ext string) bool {
if !entry.Type().IsRegular() {
return false
}
if ext == "" {
return filepath.Ext(entry.Name()) == ""
}
return strings.HasSuffix(entry.Name(), "."+ext)
}
stress_test.go
+153
View File
@@ -0,0 +1,153 @@
package log
import (
"bytes"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
)
// Fixed workloads assert accounting and uniqueness; they do not depend on the
// scheduler producing a particular drop rate or on a wall-clock soak duration.
func TestStressQueueAccounting(t *testing.T) {
for _, capacity := range []int64{8, 16384} {
t.Run(strconv.FormatInt(capacity, 10), func(t *testing.T) {
const workers, perWorker = 32, 512
var output bytes.Buffer
l := NewLogger()
c := DefaultConfig()
c.BufferSize = capacity
c.EnableConsole = true
c.Format = "raw"
mustNoErr(t, l.ApplyConfig(c), "apply")
l.state.StdoutWriter.Store(&sink{w: &output})
var run func()
l.SetSpawn(func(fn func()) { run = fn })
mustNoErr(t, l.Start(), "start deferred processor")
var wg sync.WaitGroup
for worker := range workers {
wg.Go(func() {
for i := range perWorker {
l.Info(strconv.Itoa(worker*perWorker+i) + "\n")
}
})
}
wg.Wait()
total := uint64(workers * perWorker)
equal(t, l.state.TotalDroppedLogs.Load(), total-uint64(capacity), "exact overflow count")
go run()
mustNoErr(t, l.Shutdown(5*time.Second), "drain")
seen := assertUniqueRecords(t, output.String(), int(total))
equal(t, len(seen), int(capacity), "accepted records all written")
equal(t, l.state.TotalLogsProcessed.Load()+l.state.TotalDroppedLogs.Load(), total, "no unaccounted records")
})
}
}
func TestStressLifecycleAccounting(t *testing.T) {
const workers, perWorker = 8, 1000
l := NewLogger()
c := DefaultConfig()
c.EnableConsole = false
c.EnableFile = true
c.Directory = t.TempDir()
c.Format = "raw"
c.BufferSize = 256
c.MaxSizeKB = 0
c.MaxTotalSizeKB = 0
c.MinDiskFreeKB = 0
mustNoErr(t, l.ApplyConfig(c), "apply")
mustNoErr(t, l.Start(), "start")
start := make(chan struct{})
var wg sync.WaitGroup
for worker := range workers {
wg.Go(func() {
<-start
for i := range perWorker {
// Exercise the enqueue boundary directly: public level-gated calls made
// while stopped intentionally never become attempted records.
l.sendLogRecord(logRecord{Args: []any{strconv.Itoa(worker*perWorker+i) + "\n"}})
}
})
}
wg.Go(func() {
<-start
for i := range 24 {
noErr(t, l.Stop(time.Second), "stop")
noErr(t, l.Start(), "start")
noErr(t, l.ApplyConfigString(fmt.Sprintf("buffer_size=%d", 128+128*(i%2))), "reconfigure")
}
})
close(start)
wg.Wait()
mustNoErr(t, l.Shutdown(5*time.Second), "final shutdown")
seen := assertUniqueRecords(t, readLog(t, c.Directory), workers*perWorker)
equal(t, uint64(len(seen)), l.state.TotalLogsProcessed.Load(), "written equals processed")
equal(t, l.state.TotalLogsProcessed.Load()+l.state.TotalDroppedLogs.Load(), uint64(workers*perWorker), "every attempted enqueue processed or dropped")
}
func TestStressRotationPreservesRecords(t *testing.T) {
const total = 4096
l := NewLogger()
c := DefaultConfig()
c.EnableConsole = false
c.EnableFile = true
c.Directory = t.TempDir()
c.BufferSize = total
c.Format = "raw"
c.MaxSizeKB = 1
c.MaxTotalSizeKB = 0
c.MinDiskFreeKB = 0
mustNoErr(t, l.ApplyConfig(c), "apply")
mustNoErr(t, l.Start(), "start")
var wg sync.WaitGroup
for worker := range 16 {
wg.Go(func() {
for i := range total / 16 {
l.Info(strconv.Itoa(worker*(total/16)+i) + "\n")
}
})
}
wg.Wait()
mustNoErr(t, l.Shutdown(5*time.Second), "shutdown")
equal(t, len(assertUniqueRecords(t, readAllLogs(t, c.Directory), total)), total, "records survive rotation")
isTrue(t, l.state.TotalRotations.Load() > 1, "multiple rotations")
var actualSize int64
files, err := os.ReadDir(c.Directory)
mustNoErr(t, err, "list")
for _, file := range files {
info, err := os.Stat(filepath.Join(c.Directory, file.Name()))
mustNoErr(t, err, "stat")
actualSize += info.Size()
}
var expectedSize int64
for i := range total {
expectedSize += int64(len(strconv.Itoa(i)) + 1)
}
equal(t, actualSize, expectedSize, "exact bytes across archives")
equal(t, l.state.TotalDroppedLogs.Load(), uint64(0), "no drops")
}
func assertUniqueRecords(t *testing.T, data string, total int) map[int]bool {
t.Helper()
seen := make(map[int]bool)
for _, line := range strings.Split(strings.TrimSuffix(data, "\n"), "\n") {
if line == "" && data == "" {
continue
}
id, err := strconv.Atoi(line)
if err != nil || id < 0 || id >= total {
t.Fatalf("invalid record %q", line)
}
if seen[id] {
t.Fatalf("duplicate record %d", id)
}
seen[id] = true
}
return seen
}
timer.go
+2 -1
View File
@@ -20,6 +20,7 @@ func (l *Logger) setupProcessingTimers() *TimerSet {
// Set up disk check timer
timers.diskCheckTicker = l.setupDiskCheckTimer()
timers.diskInterval = time.Duration(max(c.MinCheckIntervalMs, min(c.DiskCheckIntervalMs, c.MaxCheckIntervalMs))) * time.Millisecond
// Set up heartbeat timer
timers.heartbeatChan = l.setupHeartbeatTimer(timers)
@@ -35,7 +36,7 @@ func (l *Logger) setupRetentionTimer(timers *TimerSet) <-chan time.Time {
retentionDur := time.Duration(retentionPeriodHrs * float64(time.Hour))
retentionCheckInterval := time.Duration(retentionCheckMins * float64(time.Minute))
if retentionDur > 0 && retentionCheckInterval > 0 {
if c.EnableFile && retentionDur > 0 && retentionCheckInterval > 0 {
timers.retentionTicker = time.NewTicker(retentionCheckInterval)
l.updateEarliestFileTime() // Initial check
return timers.retentionTicker.C
type.go
+8
View File
@@ -15,6 +15,7 @@ const ContextSlots = formatter.ContextSlots
// logRecord represents a single log entry
type logRecord struct {
heartbeat bool
TimeStamp time.Time
Trace string
Args []any
@@ -31,9 +32,16 @@ type TimerSet struct {
heartbeatTicker *time.Ticker
retentionChan <-chan time.Time
heartbeatChan <-chan time.Time
diskInterval time.Duration
}
// sink is a wrapper around an io.Writer, atomic value type change workaround
type sink struct {
w io.Writer
}
// flushRequest belongs to one processor generation; the buffered reply permits
// completion even when the caller has already timed out.
type flushRequest struct {
done chan error
}
utility.go
+11 -7
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"path/filepath"
"runtime"
"slices"
"strings"
"unicode"
@@ -25,12 +26,14 @@ func getTrace(depth int64, skip int) string {
count := 0
for {
frame, more := frames.Next()
if !more || count >= int(depth) {
if count >= int(depth) {
break
}
funcName := filepath.Base(frame.Function)
parts := strings.Split(funcName, ".")
lastPart := parts[len(parts)-1]
prefix, lastPart, found := strings.CutLast(funcName, ".")
if !found {
lastPart = funcName
}
if strings.HasPrefix(lastPart, "func") {
isAnonymous := true
for _, r := range lastPart[4:] {
@@ -40,7 +43,7 @@ func getTrace(depth int64, skip int) string {
}
}
if isAnonymous && len(lastPart) > 4 {
funcName = fmt.Sprintf("(anonymous in %s)", strings.Join(parts[:len(parts)-1], "."))
funcName = fmt.Sprintf("(anonymous in %s)", prefix)
} else {
funcName = lastPart
}
@@ -49,14 +52,15 @@ func getTrace(depth int64, skip int) string {
}
trace = append(trace, funcName)
count++
if !more {
break
}
}
if len(trace) == 0 {
return "(unknown)"
}
// Reverse for caller -> callee order
for i, j := 0, len(trace)-1; i < j; i, j = i+1, j-1 {
trace[i], trace[j] = trace[j], trace[i]
}
slices.Reverse(trace)
return strings.Join(trace, " -> ")
}
utility_test.go
-1
View File
@@ -149,4 +149,3 @@ func TestGetTraceOrdering(t *testing.T) {
t.Errorf("frames not ordered: %q", trace)
}
}