From 2417e51da1865e82a6a9b0638c4b29328ae5713dd8d46874608a90f90ede7f16 Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sat, 26 Sep 2026 14:22:29 -0400 Subject: [PATCH] v0.1.11 harden logging lifecycle, sanitization, and formatter edge cases --- .github/workflows/test.yml | 66 +++ LICENSE | 4 +- README.md | 20 +- audit_test.go | 336 +++++++++++++++ benchmark_test.go | 8 +- builder.go | 24 +- builder_test.go | 1 - compat/builder.go | 59 +-- compat/compat_test.go | 1 - compat/fasthttp.go | 10 +- compat/fiber.go | 40 +- compat/gnet.go | 14 +- compat/structured_format_test.go | 54 +++ compat/structured_gnet.go | 97 ++--- config.go | 47 ++- config_test.go | 1 - doc/adapters.md | 394 ++++-------------- doc/api.md | 36 +- doc/builder.md | 6 +- doc/configuration.md | 37 +- doc/formatting.md | 47 ++- doc/getting-started.md | 13 +- doc/heartbeat.md | 33 +- doc/logging.md | 16 +- doc/storage.md | 45 +- doc/testing.md | 29 ++ formatter/formatter.go | 109 +++-- formatter/fuzz_test.go | 118 ++++++ go.mod | 2 +- heartbeat.go | 1 + integration_test.go | 4 +- lifecycle_test.go | 13 +- logger.go | 389 ++++++++--------- processor.go | 103 +++-- processor_test.go | 275 ++++-------- record.go | 49 +-- sanitizer/fuzz_test.go | 92 ++++ sanitizer/sanitizer.go | 202 +++++---- .../fuzz/FuzzSanitize/3120fc1ce48a8023 | 5 + state.go | 4 +- storage.go | 146 +++---- stress_test.go | 153 +++++++ timer.go | 5 +- type.go | 8 + utility.go | 18 +- utility_test.go | 1 - 46 files changed, 1846 insertions(+), 1289 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 audit_test.go create mode 100644 compat/structured_format_test.go create mode 100644 doc/testing.md create mode 100644 formatter/fuzz_test.go create mode 100644 sanitizer/fuzz_test.go create mode 100644 sanitizer/testdata/fuzz/FuzzSanitize/3120fc1ce48a8023 create mode 100644 stress_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ccfd72e --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,66 @@ +name: Test + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: test-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version-file: go.mod + cache: false # Standard-library-only module. + - name: Check formatting + run: test -z "$(gofmt -l $(git ls-files '*.go'))" + - run: go vet ./... + - name: Unit, regression, and stress tests with race detection + run: go test -race -count=1 -timeout=3m ./... + - name: Repeated lifecycle and stress tests + run: go test -race -run 'Test(Stress|StartDoesNotReplaceProcessor|ConcurrentOverridesMerge|TimedOutShutdown)' -count=5 -timeout=3m . + - name: Compile supported deployment targets + run: | + GOOS=freebsd GOARCH=amd64 go build ./... + GOOS=js GOARCH=wasm go build ./... + GOOS=windows GOARCH=amd64 go build ./... + + fuzz: + runs-on: ubuntu-latest + timeout-minutes: 5 + strategy: + fail-fast: false + matrix: + include: + - package: ./sanitizer + target: FuzzSanitize + - package: ./sanitizer + target: FuzzSerializerJSON + - package: ./formatter + target: FuzzFormatJSON + - package: ./formatter + target: FuzzFormatTextPolicy + - package: ./compat + target: FuzzParseFormat + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version-file: go.mod + cache: false + - name: Bounded fuzzing + run: go test '${{ matrix.package }}' -run '^$' -fuzz '^${{ matrix.target }}$' -fuzztime=15s -parallel=2 -timeout=2m diff --git a/LICENSE b/LICENSE index a280417..e47911e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 3-Clause License -Copyright (c) 2025, Lixen Wraith +Copyright (c) 2026, Lixen Wraith Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -25,4 +25,4 @@ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index e7088f2..7563769 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Log -[![Go](https://img.shields.io/badge/Go-1.25+-00ADD8?style=flat&logo=go)](https://golang.org) +[![Go](https://img.shields.io/badge/Go-1.27.1+-00ADD8?style=flat&logo=go)](https://go.dev) [![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Documentation](https://img.shields.io/badge/Docs-Available-green.svg)](doc/) @@ -8,10 +8,10 @@ A high-performance, buffered, rotating file logger for Go applications with buil ## Key Features -- **Lock-free async logging** with minimal application impact +- **Buffered async logging** with minimal application impact - **Automatic file rotation** and disk space management - **Operational heartbeats** for production monitoring -- **Hot reconfiguration** without data loss +- **Hot reconfiguration** with draining on restart - **Framework adapters** for gnet v2, fasthttp, Fiber v2 - **Production-grade reliability** with graceful shutdown @@ -29,7 +29,7 @@ import ( func main() { // Create and initialize logger logger := log.NewLogger() - err := logger.ApplyConfigString("directory=/var/log/myapp") + err := logger.ApplyConfigString("directory=/var/log/myapp", "enable_file=true", "format=txt") if err != nil { panic(fmt.Errorf("failed to apply logger config: %w", err)) } @@ -59,21 +59,17 @@ go get github.com/lixenwraith/log - **[Configuration Builder](doc/builder.md)** - Builder pattern guide - **[API Reference](doc/api.md)** - Complete API documentation - **[Logging Guide](doc/logging.md)** - Logging methods and best practices -+ **[Formatting & Sanitization](doc/formatting.md)** - Standalone formatter and sanitizer packages +- **[Formatting & Sanitization](doc/formatting.md)** - Standalone formatter and sanitizer packages - **[Disk Management](doc/storage.md)** - File rotation and cleanup - **[Heartbeat Monitoring](doc/heartbeat.md)** - Operational statistics - **[Compatibility Adapters](doc/adapters.md)** - Framework integrations -- **[Quick Guide](doc/quick-guide_lixenwraith_log.md)** - Quick reference guide +- **[Testing and Performance](doc/testing.md)** - Race, stress, fuzz, and benchmark commands ## Architecture Overview -The logger uses a lock-free, channel-based architecture for high performance: +Applications enqueue records into a bounded channel. A single background processor formats records, writes output, rotates files, and performs cleanup. A short read lock coordinates enqueues with lifecycle changes; producers never wait for queue capacity or output I/O. Full queues drop records and increment counters. -``` -Application → Log Methods → Buffered Channel → Background Processor → File/Console - ↓ ↓ - (non-blocking) (rotation, cleanup, monitoring) -``` +Call `ApplyConfig` (or `Build`) and then `Start` before logging. `Stop` drains accepted records and can be followed by `Start`. `Shutdown` also closes the file; after a timeout, retry it to finish cleanup. See the [API reference](doc/api.md) for ownership and concurrency rules. ## Contributing diff --git a/audit_test.go b/audit_test.go new file mode 100644 index 0000000..ed17908 --- /dev/null +++ b/audit_test.go @@ -0,0 +1,336 @@ +package log + +import ( + "bytes" + "errors" + "math" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestConfigOwnershipAndRollback(t *testing.T) { + l, dir := newTestLogger(t) + cfg := l.GetConfig() + cfg.Format = "json" + mustNoErr(t, l.ApplyConfig(cfg), "apply snapshot") + cfg.Format = "raw" + equal(t, l.GetConfig().Format, "json", "caller config is not retained") + old := *l.GetConfig() + blocker := filepath.Join(dir, "blocker") + mustNoErr(t, os.WriteFile(blocker, nil, 0600), "create blocker") + mustNoErr(t, os.Mkdir(filepath.Join(dir, "directory"), 0700), "create directory blocker") + for _, path := range []string{filepath.Join(blocker, "child"), dir} { + bad := l.GetConfig() + bad.Directory = path + bad.Name = "directory" + bad.Extension = "" + bad.Format = "txt" + bad.Level = LevelError + mustErr(t, l.ApplyConfig(bad), "failed reconfiguration") + equal(t, *l.GetConfig(), old, "configuration rolled back") + isTrue(t, l.Enabled(LevelInfo), "failed config did not close emit gate") + } + l.Info("survives") + mustNoErr(t, l.Flush(time.Second), "flush old configuration") + contains(t, readLog(t, dir), `"fields":["survives"]`, "formatter rolled back") +} + +func TestConfigBounds(t *testing.T) { + for name, mutate := range map[string]func(*Config){ + "name traversal": func(c *Config) { c.Name = "../outside" }, + "name dot": func(c *Config) { c.Name = ".." }, + "extension traversal": func(c *Config) { c.Extension = "../log" }, + "NUL": func(c *Config) { c.Name = "bad\x00name" }, + "buffer": func(c *Config) { c.BufferSize = math.MaxInt64 }, + "size": func(c *Config) { c.MaxSizeKB = math.MaxInt64 }, + "flush": func(c *Config) { c.FlushIntervalMs = math.MaxInt64 }, + "heartbeat": func(c *Config) { c.HeartbeatIntervalS = math.MaxInt64 }, + "retention nan": func(c *Config) { c.RetentionPeriodHrs = math.NaN() }, + "retention infinity": func(c *Config) { c.RetentionCheckMins = math.Inf(1) }, + "retention overflow": func(c *Config) { c.RetentionPeriodHrs = 1e30 }, + "retention underflow": func(c *Config) { c.RetentionCheckMins = 1e-30 }, + } { + t.Run(name, func(t *testing.T) { c := DefaultConfig(); mutate(c); mustErr(t, c.Validate(), "validate") }) + } + var nilConfig *Config + mustErr(t, nilConfig.Validate(), "nil config") + for _, b := range []*Builder{NewBuilder().MaxSizeMB(math.MaxInt64), NewBuilder().MaxTotalSizeMB(math.MaxInt64), NewBuilder().MinDiskFreeMB(math.MaxInt64)} { + _, err := b.Build() + mustErr(t, err, "MB overflow") + } + l := NewLogger() + isFalse(t, l.Enabled(math.MaxInt64), "closed level gate") +} + +func TestConcurrentOverridesMerge(t *testing.T) { + l, _ := newTestLogger(t) + overrides := []string{"trace_depth=2", "level=debug", "format=json", "show_level=false", "show_timestamp=false", "sanitization=txt"} + var wg sync.WaitGroup + start := make(chan struct{}) + for _, override := range overrides { + wg.Go(func() { <-start; noErr(t, l.ApplyConfigString(override), override) }) + } + close(start) + wg.Wait() + c := l.GetConfig() + equal(t, c.TraceDepth, int64(2), "depth") + equal(t, c.Level, LevelDebug, "level") + equal(t, c.Format, "json", "format") + isFalse(t, c.ShowLevel, "show level") + isFalse(t, c.ShowTimestamp, "show timestamp") + equal(t, c.Sanitization, PolicyTxt, "sanitization") +} + +func TestStartDoesNotReplaceProcessor(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + c.EnableConsole = false + mustNoErr(t, l.ApplyConfig(c), "apply") + var launches atomic.Int64 + l.SetSpawn(func(fn func()) { launches.Add(1); go fn() }) + var wg sync.WaitGroup + for range 64 { + wg.Go(func() { noErr(t, l.Start(), "concurrent start") }) + } + wg.Wait() + equal(t, launches.Load(), int64(1), "only one processor") + mustNoErr(t, l.Shutdown(time.Second), "shutdown") + mustNoErr(t, l.ApplyConfig(c), "reset after shutdown") + mustNoErr(t, l.Start(), "start reset logger") + isTrue(t, l.Enabled(LevelInfo), "reset reopens gate") + mustNoErr(t, l.Shutdown(time.Second), "shutdown reset logger") +} + +type blockedWriter struct { + entered, release chan struct{} + once sync.Once +} + +func (w *blockedWriter) Write(p []byte) (int, error) { + w.once.Do(func() { close(w.entered) }) + <-w.release + return len(p), nil +} + +func TestTimedOutShutdownKeepsProcessorResources(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + c.Directory = t.TempDir() + c.EnableFile = true + c.EnableConsole = true + mustNoErr(t, l.ApplyConfig(c), "apply") + w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})} + l.state.StdoutWriter.Store(&sink{w: w}) + mustNoErr(t, l.Start(), "start") + l.Info("accepted before timeout") + select { + case <-w.entered: + case <-time.After(time.Second): + t.Fatal("processor never reached writer") + } + // Always release the writer even when an assertion fails. + defer func() { close(w.release); noErr(t, l.Shutdown(time.Second), "retry shutdown") }() + errContains(t, l.Shutdown(time.Millisecond), "timeout", "shutdown timeout") + f := l.state.CurrentFile.Load().(*os.File) + _, err := f.Stat() + mustNoErr(t, err, "file still owned by draining processor") + mustErr(t, l.Start(), "restart blocked") + mustErr(t, l.ApplyConfig(c), "resource replacement blocked") +} + +func TestAsyncArgumentSnapshot(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + c.EnableConsole = false + c.EnableFile = true + c.Directory = t.TempDir() + c.Format = "json" + mustNoErr(t, l.ApplyConfig(c), "apply") + var run func() + l.SetSpawn(func(fn func()) { run = fn }) + mustNoErr(t, l.Start(), "start deferred processor") + args := []any{"original"} + l.Info(args...) + args[0] = "mutated" + fields := map[string]any{"value": "original"} + l.LogStructured(LevelInfo, "message", fields) + fields["value"] = "mutated" + go run() + mustNoErr(t, l.Shutdown(time.Second), "shutdown") + data := readLog(t, c.Directory) + notContains(t, data, "mutated", "snapshotted arguments and map") + equal(t, strings.Count(data, "original"), 2, "original values") +} + +func TestFlushReportsSyncError(t *testing.T) { + l, _ := newTestLogger(t) + mustNoErr(t, l.Flush(time.Second), "initial flush") + file := l.state.CurrentFile.Load().(*os.File) + mustNoErr(t, file.Close(), "inject closed file") + err := l.Flush(time.Second) + if !errors.Is(err, os.ErrClosed) { + t.Fatalf("expected sync error, got %v", err) + } +} + +func TestConsoleOnlyRetentionIsInert(t *testing.T) { + c := DefaultConfig() + c.EnableConsole = false + c.Directory = t.TempDir() + c.RetentionPeriodHrs = 1 + name := filepath.Join(c.Directory, "old.log") + mustNoErr(t, os.WriteFile(name, []byte("keep"), 0600), "write") + old := time.Now().Add(-2 * time.Hour) + mustNoErr(t, os.Chtimes(name, old, old), "set age") + l := NewLogger() + mustNoErr(t, l.ApplyConfig(c), "apply") + l.state.EarliestFileTime.Store(old) + l.handleRetentionCheck() + mustNoErr(t, l.cleanExpiredLogs(old), "explicit cleanup") + data, err := os.ReadFile(name) + mustNoErr(t, err, "file survived") + equal(t, string(data), "keep", "contents") + timers := l.setupProcessingTimers() + defer l.stopProcessingTimers(timers) + if timers.retentionChan != nil { + t.Fatal("console-only retention timer enabled") + } +} + +func TestArchiveCollisionBeyondOneThousand(t *testing.T) { + l, dir := newTestLogger(t) + stamp := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + base := "log_" + stamp.Format("060102_150405") + mustNoErr(t, os.Mkdir(filepath.Join(dir, base+".log"), 0700), "directory collision") + for i := 1; i <= 1000; i++ { + mustNoErr(t, os.WriteFile(filepath.Join(dir, base+"_"+itoa(i)+".log"), nil, 0600), "archive") + } + equal(t, l.generateArchiveLogFileName(stamp), base+"_1001.log", "no capped collision reuse") +} + +func TestExtensionlessAccountingAndCleanup(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + c.EnableConsole = false + c.EnableFile = true + c.Directory = t.TempDir() + c.Extension = "" + mustNoErr(t, l.ApplyConfig(c), "apply") + defer l.Shutdown() + for _, name := range []string{"old", "other.txt"} { + mustNoErr(t, os.WriteFile(filepath.Join(c.Directory, name), []byte("123"), 0600), "write") + } + size, err := l.getLogDirSize(c.Directory, "") + mustNoErr(t, err, "size") + equal(t, size, int64(3), "extensionless size") + count, err := l.getLogFileCount(c.Directory, "") + mustNoErr(t, err, "count") + equal(t, count, 2, "extensionless count") + mustNoErr(t, l.cleanOldLogs(0), "zero demand does not remove archives") + mustNoErr(t, l.cleanOldLogs(3), "cleanup") + _, err = os.Stat(filepath.Join(c.Directory, "other.txt")) + mustNoErr(t, err, "other extension preserved") +} + +func TestTextSanitizesMetadata(t *testing.T) { + var out bytes.Buffer + l := NewLogger() + c := DefaultConfig() + c.Format = "txt" + c.Sanitization = PolicyTxt + c.TimestampFormat = "bad\n\x1b" + c.EnableConsole = true + mustNoErr(t, l.ApplyConfig(c), "apply") + l.SetContextKeys("key\n\x1b") + l.state.StdoutWriter.Store(&sink{w: &out}) + mustNoErr(t, l.Start(), "start") + l.LogContext(Context{Tag: "value"}, FlagKV, LevelInfo, 0, "key\n\x1b", "value") + mustNoErr(t, l.Shutdown(time.Second), "shutdown") + equal(t, bytes.Count(out.Bytes(), []byte{'\n'}), 1, "one record line") + notContains(t, out.String(), "\x1b", "no terminal escape") +} + +type shortWriter struct{} + +func (shortWriter) Write(p []byte) (int, error) { return len(p) - 1, nil } + +func TestConsoleWriteFailureAccounting(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + mustNoErr(t, l.ApplyConfig(c), "apply") + l.state.StdoutWriter.Store(&sink{w: shortWriter{}}) + mustNoErr(t, l.Start(), "start") + l.Info("short write") + mustNoErr(t, l.Shutdown(time.Second), "shutdown") + equal(t, l.state.TotalDroppedLogs.Load(), uint64(1), "console drop") + equal(t, l.state.DroppedLogs.Load(), uint64(1), "interval drop") + equal(t, l.state.TotalLogsProcessed.Load(), uint64(0), "not processed successfully") +} + +func TestDrainFlushIsBounded(t *testing.T) { + // Refill after every write, reproducing a queue which never becomes empty. + // A snapshot drain must still complete after the original pending count. + l := NewLogger() + c := DefaultConfig() + c.Format = "raw" + mustNoErr(t, l.ApplyConfig(c), "apply") + ch := make(chan logRecord, 4) + for range cap(ch) { + ch <- logRecord{Args: []any{"x"}} + } + l.state.StdoutWriter.Store(&sink{w: writerFunc(func(p []byte) (int, error) { ch <- logRecord{Args: []any{"later"}}; return len(p), nil })}) + done := make(chan struct{}) + go func() { l.drain(ch); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("drain starved on refilled queue") + } + equal(t, l.state.TotalLogsProcessed.Load(), uint64(4), "bounded work") + equal(t, len(ch), 4, "later records left for next iteration") +} + +type writerFunc func([]byte) (int, error) + +func (fn writerFunc) Write(p []byte) (int, error) { return fn(p) } + +func TestReconfigureDrainsOldFileAndFormat(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + c.Directory = t.TempDir() + c.EnableFile = true + c.EnableConsole = true + c.Format = "raw" + c.MaxSizeKB = 1 + c.FlushIntervalMs = 1000 + mustNoErr(t, l.ApplyConfig(c), "apply") + w := &blockedWriter{entered: make(chan struct{}), release: make(chan struct{})} + l.state.StdoutWriter.Store(&sink{w: w}) + var diagnostics []string + l.SetErrorHandler(func(msg string) { diagnostics = append(diagnostics, msg) }) + mustNoErr(t, l.Start(), "start") + payload := strings.Repeat("x", 1500) + l.Info(payload) + <-w.entered + l.Info("old-format-tail") + updated := l.GetConfig() + updated.EnableFile = false + updated.EnableConsole = false + updated.Format = "json" + done := make(chan error, 1) + go func() { done <- l.ApplyConfig(updated) }() + mustEventually(t, time.Second, "stop began", func() bool { return !l.state.Started.Load() }) + close(w.release) + mustNoErr(t, <-done, "reconfigure") + mustNoErr(t, l.Shutdown(time.Second), "shutdown") + data := readAllLogs(t, c.Directory) + contains(t, data, payload, "original payload") + contains(t, data, "old-format-tail", "queued record used old format") + equal(t, len(data), len(payload)+len("old-format-tail"), "exact drained bytes") + equal(t, len(diagnostics), 0, "closed current handle after draining rotation") +} diff --git a/benchmark_test.go b/benchmark_test.go index abc94c0..4d20772 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -18,7 +18,7 @@ func BenchmarkLoggerInfo(b *testing.B) { } } -// BenchmarkLoggerTxt measures the txt path, which includes quote analysis. +// BenchmarkLoggerTxt measures enqueueing with txt output selected. func BenchmarkLoggerTxt(b *testing.B) { logger, _ := newTestLogger(b) @@ -46,7 +46,7 @@ func BenchmarkLoggerJSON(b *testing.B) { } } -// BenchmarkLoggerStructured measures the json.Marshal path for field maps. +// BenchmarkLoggerStructured measures enqueueing and snapshotting structured maps. func BenchmarkLoggerStructured(b *testing.B) { logger, _ := newTestLogger(b) @@ -66,8 +66,8 @@ func BenchmarkLoggerStructured(b *testing.B) { } } -// BenchmarkLoggerSanitized measures PolicyTxt overhead on control-free input, -// where the sanitizer takes its no-allocation fast path. +// BenchmarkLoggerSanitized measures enqueueing with PolicyTxt selected; +// sanitizer CPU work runs asynchronously and is benchmarked in its own package. func BenchmarkLoggerSanitized(b *testing.B) { logger, _ := newTestLogger(b) diff --git a/builder.go b/builder.go index 6a30a82..aa05ed4 100644 --- a/builder.go +++ b/builder.go @@ -1,6 +1,9 @@ package log import ( + "math" + "slices" + "github.com/lixenwraith/log/sanitizer" ) @@ -23,7 +26,7 @@ func NewBuilder() *Builder { // ContextKeys names the record keys for Context values func (b *Builder) ContextKeys(tag string, vals ...string) *Builder { b.ctxTag = tag - b.ctxVals = vals + b.ctxVals = slices.Clone(vals) return b } @@ -67,7 +70,7 @@ func (b *Builder) LevelString(level string) *Builder { return b } -// Name sets the log level +// Name sets the base filename func (b *Builder) Name(name string) *Builder { b.cfg.Name = name return b @@ -91,7 +94,7 @@ func (b *Builder) Sanitization(policy sanitizer.PolicyPreset) *Builder { return b } -// Extension sets the log level +// Extension sets the filename extension without a leading dot func (b *Builder) Extension(ext string) *Builder { b.cfg.Extension = ext return b @@ -111,6 +114,10 @@ func (b *Builder) MaxSizeKB(size int64) *Builder { // MaxSizeMB sets the maximum log file size in MB func (b *Builder) MaxSizeMB(size int64) *Builder { + if size < 0 || size > math.MaxInt64/(sizeMultiplier*sizeMultiplier) { + b.err = fmtErrorf("size in MB is out of range: %d", size) + return b + } b.cfg.MaxSizeKB = size * sizeMultiplier return b } @@ -127,7 +134,7 @@ func (b *Builder) HeartbeatLevel(level int64) *Builder { return b } -// HeartbeatIntervalS sets the heartbeat monitoring level +// HeartbeatIntervalS sets the heartbeat interval in seconds func (b *Builder) HeartbeatIntervalS(interval int64) *Builder { b.cfg.HeartbeatIntervalS = interval return b @@ -159,6 +166,10 @@ func (b *Builder) MaxTotalSizeKB(size int64) *Builder { // MaxTotalSizeMB sets the maximum total size of all log files in MB func (b *Builder) MaxTotalSizeMB(size int64) *Builder { + if size < 0 || size > math.MaxInt64/(sizeMultiplier*sizeMultiplier) { + b.err = fmtErrorf("size in MB is out of range: %d", size) + return b + } b.cfg.MaxTotalSizeKB = size * sizeMultiplier return b } @@ -171,6 +182,10 @@ func (b *Builder) MinDiskFreeKB(size int64) *Builder { // MinDiskFreeMB sets the minimum required free disk space in MB func (b *Builder) MinDiskFreeMB(size int64) *Builder { + if size < 0 || size > math.MaxInt64/(sizeMultiplier*sizeMultiplier) { + b.err = fmtErrorf("size in MB is out of range: %d", size) + return b + } b.cfg.MinDiskFreeKB = size * sizeMultiplier return b } @@ -259,6 +274,7 @@ func (b *Builder) EnableConsole(enable bool) *Builder { // if err == nil { // // defer logger.Shutdown() +// if err := logger.Start(); err != nil { return } // logger.Info("Logger initialized successfully") // // } diff --git a/builder_test.go b/builder_test.go index ab7c7a4..8bebb59 100644 --- a/builder_test.go +++ b/builder_test.go @@ -137,4 +137,3 @@ func TestBuilderDefaults(t *testing.T) { equal(t, cfg.BufferSize, def.BufferSize, "BufferSize") equal(t, cfg.Sanitization, def.Sanitization, "Sanitization") } - diff --git a/compat/builder.go b/compat/builder.go index 7b49a6c..8838f7d 100644 --- a/compat/builder.go +++ b/compat/builder.go @@ -97,7 +97,7 @@ func (b *Builder) BuildFastHTTP(opts ...FastHTTPOption) (*FastHTTPAdapter, error return NewFastHTTPAdapter(l, opts...), nil } -// BuildFiber creates a Fiber v2.54.x adapter +// BuildFiber creates a Fiber v2 adapter func (b *Builder) BuildFiber(opts ...FiberOption) (*FiberAdapter, error) { l, err := b.getLogger() if err != nil { @@ -112,57 +112,6 @@ func (b *Builder) GetLogger() (*log.Logger, error) { return b.getLogger() } -// --- Example Usage --- -// -// The following demonstrates how to integrate lixenwraith/log with gnet, fasthttp, and Fiber -// using a single, shared logger instance -// -// // 1. Create and configure application's main logger -// appLogger := log.NewLogger() -// logCfg := log.DefaultConfig() -// logCfg.Level = log.LevelDebug -// if err := appLogger.ApplyConfig(logCfg); err != nil { -// panic(fmt.Sprintf("failed to configure logger: %v", err)) -// } -// -// // 2. Create a builder and provide the existing logger -// builder := compat.NewBuilder().WithLogger(appLogger) -// -// // 3. Build the required adapters -// gnetLogger, err := builder.BuildGnet() -// if err != nil { /* handle error */ } -// -// fasthttpLogger, err := builder.BuildFastHTTP() -// if err != nil { /* handle error */ } -// -// fiberLogger, err := builder.BuildFiber() -// if err != nil { /* handle error */ } -// -// // 4. Configure your servers with the adapters -// -// // For gnet: -// var events gnet.EventHandler // your-event-handler -// // The adapter is passed directly into the gnet options -// go gnet.Run(events, "tcp://:9000", gnet.WithLogger(gnetLogger)) -// -// // For fasthttp: -// // The adapter is assigned directly to the server's Logger field -// server := &fasthttp.Server{ -// Handler: func(ctx *fasthttp.RequestCtx) { -// ctx.WriteString("Hello, world!") -// }, -// Logger: fasthttpLogger, -// } -// go server.ListenAndServe(":8080") -// -// // For Fiber v2.54.x: -// // The adapter is passed to fiber.New() via the config -// app := fiber.New(fiber.Config{ -// AppName: "My Application", -// }) -// app.UpdateConfig(fiber.Config{ -// AppName: "My Application", -// }) -// // Note: Set the logger after app creation if needed -// // fiber uses internal logging, adapter can be used in custom middleware -// go app.Listen(":3000") \ No newline at end of file +// Configure and Start the shared Logger before building adapters. WithConfig +// initializes a new Logger but does not Start it; use GetLogger for lifecycle +// control. See doc/adapters.md for framework integration examples. diff --git a/compat/compat_test.go b/compat/compat_test.go index 9038581..e1f31e6 100644 --- a/compat/compat_test.go +++ b/compat/compat_test.go @@ -501,4 +501,3 @@ func TestFiberAdapterWriter(t *testing.T) { eq(t, level, "INFO", "level") checkFields(t, fields, []any{"msg", "writer output", "source", "fiber"}, "Write") } - diff --git a/compat/fasthttp.go b/compat/fasthttp.go index e4729e0..e10eb99 100644 --- a/compat/fasthttp.go +++ b/compat/fasthttp.go @@ -59,16 +59,20 @@ func (a *FastHTTPAdapter) Printf(format string, args ...any) { } } - // Log with appropriate level switch level { case log.LevelDebug: a.logger.Debug("msg", msg, "source", "fasthttp") + case log.LevelInfo: + a.logger.Info("msg", msg, "source", "fasthttp") case log.LevelWarn: a.logger.Warn("msg", msg, "source", "fasthttp") case log.LevelError: a.logger.Error("msg", msg, "source", "fasthttp") default: - a.logger.Info("msg", msg, "source", "fasthttp") + if a.logger.Enabled(level) { + a.logger.LogContext(log.Context{}, a.logger.Flags(), level, a.logger.GetConfig().TraceDepth, + "msg", msg, "source", "fasthttp") + } } } @@ -99,4 +103,4 @@ func DetectLogLevel(msg string) int64 { // Default to info level return log.LevelInfo -} \ No newline at end of file +} diff --git a/compat/fiber.go b/compat/fiber.go index e852b8a..b4ed015 100644 --- a/compat/fiber.go +++ b/compat/fiber.go @@ -9,7 +9,7 @@ import ( ) // FiberAdapter wraps lixenwraith/log.Logger to implement Fiber's CommonLogger interface -// This provides compatibility with Fiber v2.54.x logging requirements +// This provides compatibility with Fiber v2 logging methods type FiberAdapter struct { logger *log.Logger fatalHandler func(msg string) // Customizable fatal behavior @@ -62,24 +62,36 @@ func (a *FiberAdapter) Trace(v ...any) { // Debug logs at debug level func (a *FiberAdapter) Debug(v ...any) { + if !a.logger.Enabled(log.LevelDebug) { + return + } msg := fmt.Sprint(v...) a.logger.Debug("msg", msg, "source", "fiber") } // Info logs at info level func (a *FiberAdapter) Info(v ...any) { + if !a.logger.Enabled(log.LevelInfo) { + return + } msg := fmt.Sprint(v...) a.logger.Info("msg", msg, "source", "fiber") } // Warn logs at warn level func (a *FiberAdapter) Warn(v ...any) { + if !a.logger.Enabled(log.LevelWarn) { + return + } msg := fmt.Sprint(v...) a.logger.Warn("msg", msg, "source", "fiber") } // Error logs at error level func (a *FiberAdapter) Error(v ...any) { + if !a.logger.Enabled(log.LevelError) { + return + } msg := fmt.Sprint(v...) a.logger.Error("msg", msg, "source", "fiber") } @@ -132,24 +144,36 @@ func (a *FiberAdapter) Tracef(format string, v ...any) { // Debugf logs at debug level with printf-style formatting func (a *FiberAdapter) Debugf(format string, v ...any) { + if !a.logger.Enabled(log.LevelDebug) { + return + } msg := fmt.Sprintf(format, v...) a.logger.Debug("msg", msg, "source", "fiber") } // Infof logs at info level with printf-style formatting func (a *FiberAdapter) Infof(format string, v ...any) { + if !a.logger.Enabled(log.LevelInfo) { + return + } msg := fmt.Sprintf(format, v...) a.logger.Info("msg", msg, "source", "fiber") } // Warnf logs at warn level with printf-style formatting func (a *FiberAdapter) Warnf(format string, v ...any) { + if !a.logger.Enabled(log.LevelWarn) { + return + } msg := fmt.Sprintf(format, v...) a.logger.Warn("msg", msg, "source", "fiber") } // Errorf logs at error level with printf-style formatting func (a *FiberAdapter) Errorf(format string, v ...any) { + if !a.logger.Enabled(log.LevelError) { + return + } msg := fmt.Sprintf(format, v...) a.logger.Error("msg", msg, "source", "fiber") } @@ -192,6 +216,9 @@ func (a *FiberAdapter) Tracew(msg string, keysAndValues ...any) { // Debugw logs at debug level with structured key-value pairs func (a *FiberAdapter) Debugw(msg string, keysAndValues ...any) { + if !a.logger.Enabled(log.LevelDebug) { + return + } fields := make([]any, 0, len(keysAndValues)+4) fields = append(fields, "msg", msg, "source", "fiber") fields = append(fields, keysAndValues...) @@ -200,6 +227,9 @@ func (a *FiberAdapter) Debugw(msg string, keysAndValues ...any) { // Infow logs at info level with structured key-value pairs func (a *FiberAdapter) Infow(msg string, keysAndValues ...any) { + if !a.logger.Enabled(log.LevelInfo) { + return + } fields := make([]any, 0, len(keysAndValues)+4) fields = append(fields, "msg", msg, "source", "fiber") fields = append(fields, keysAndValues...) @@ -208,6 +238,9 @@ func (a *FiberAdapter) Infow(msg string, keysAndValues ...any) { // Warnw logs at warn level with structured key-value pairs func (a *FiberAdapter) Warnw(msg string, keysAndValues ...any) { + if !a.logger.Enabled(log.LevelWarn) { + return + } fields := make([]any, 0, len(keysAndValues)+4) fields = append(fields, "msg", msg, "source", "fiber") fields = append(fields, keysAndValues...) @@ -216,6 +249,9 @@ func (a *FiberAdapter) Warnw(msg string, keysAndValues ...any) { // Errorw logs at error level with structured key-value pairs func (a *FiberAdapter) Errorw(msg string, keysAndValues ...any) { + if !a.logger.Enabled(log.LevelError) { + return + } fields := make([]any, 0, len(keysAndValues)+4) fields = append(fields, "msg", msg, "source", "fiber") fields = append(fields, keysAndValues...) @@ -250,4 +286,4 @@ func (a *FiberAdapter) Panicw(msg string, keysAndValues ...any) { if a.panicHandler != nil { a.panicHandler(msg) } -} \ No newline at end of file +} diff --git a/compat/gnet.go b/compat/gnet.go index e423f8a..3af696e 100644 --- a/compat/gnet.go +++ b/compat/gnet.go @@ -42,24 +42,36 @@ func WithFatalHandler(handler func(string)) GnetOption { // Debugf logs at debug level with printf-style formatting func (a *GnetAdapter) Debugf(format string, args ...any) { + if !a.logger.Enabled(log.LevelDebug) { + return + } msg := fmt.Sprintf(format, args...) a.logger.Debug("msg", msg, "source", "gnet") } // Infof logs at info level with printf-style formatting func (a *GnetAdapter) Infof(format string, args ...any) { + if !a.logger.Enabled(log.LevelInfo) { + return + } msg := fmt.Sprintf(format, args...) a.logger.Info("msg", msg, "source", "gnet") } // Warnf logs at warn level with printf-style formatting func (a *GnetAdapter) Warnf(format string, args ...any) { + if !a.logger.Enabled(log.LevelWarn) { + return + } msg := fmt.Sprintf(format, args...) a.logger.Warn("msg", msg, "source", "gnet") } // Errorf logs at error level with printf-style formatting func (a *GnetAdapter) Errorf(format string, args ...any) { + if !a.logger.Enabled(log.LevelError) { + return + } msg := fmt.Sprintf(format, args...) a.logger.Error("msg", msg, "source", "gnet") } @@ -75,4 +87,4 @@ func (a *GnetAdapter) Fatalf(format string, args ...any) { if a.fatalHandler != nil { a.fatalHandler(msg) } -} \ No newline at end of file +} diff --git a/compat/structured_format_test.go b/compat/structured_format_test.go new file mode 100644 index 0000000..4abaf9e --- /dev/null +++ b/compat/structured_format_test.go @@ -0,0 +1,54 @@ +package compat + +import ( + "fmt" + "reflect" + "testing" +) + +func TestParseFormatKeepsArgumentAssociation(t *testing.T) { + for _, tc := range []struct { + format string + args []any + want []any + }{ + {"prefix %s id=%d", []any{"alice", 42}, []any{"msg", "prefix alice id=42"}}, + {"id=%[2]d name=%[1]s", []any{"alice", 42}, []any{"msg", "id=42 name=alice"}}, + {"id=%d suffix", []any{42}, []any{"msg", "suffix", "id", 42}}, + {"user=%s id=%d", []any{"alice", 42}, []any{"user", "alice", "id", 42}}, + {"id=%d %s", []any{42, "tail"}, []any{"msg", "id=42 tail"}}, + {"id=%*d", []any{4, 42}, []any{"msg", "id= 42"}}, + {"literal %% id=%d", []any{42}, []any{"msg", "literal % id=42"}}, + } { + t.Run(tc.format, func(t *testing.T) { + if got := parseFormat(tc.format, tc.args); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %#v, want %#v", got, tc.want) + } + }) + } +} + +func FuzzParseFormat(f *testing.F) { + for _, format := range []string{"id=%d", "%s id=%d", "%%id=%d", "id=%[1]s"} { + f.Add(format, "value") + } + f.Fuzz(func(t *testing.T, format, value string) { + if len(format) > 1024 { + t.Skip() + } + fields := parseFormat(format, []any{value}) + if len(fields)%2 != 0 { + t.Fatalf("odd fields %#v", fields) + } + for i := 0; i < len(fields); i += 2 { + if _, ok := fields[i].(string); !ok { + t.Fatalf("non-string key %#v", fields) + } + } + if len(fields) == 2 && fields[0] == "msg" { + if fields[1] != fmt.Sprintf(format, value) { + t.Fatal("fallback changed message") + } + } + }) +} diff --git a/compat/structured_gnet.go b/compat/structured_gnet.go index 29b43fb..d9fb332 100644 --- a/compat/structured_gnet.go +++ b/compat/structured_gnet.go @@ -8,70 +8,41 @@ import ( "github.com/lixenwraith/log" ) -// parseFormat attempts to extract structured fields from printf-style format strings -// Useful for preserving structured logging semantics +// Only unindexed, single-argument verbs are eligible for extraction. Mixed +// printf expressions fall back intact, rather than attributing an argument to +// the wrong key (width, precision and explicit indexes may consume arguments). +var keyValuePattern = regexp.MustCompile(`(\w+)\s*[:=]\s*%[vsdqxXeEfFgGpbcU]`) + func parseFormat(format string, args []any) []any { - // Pattern to detect common structured patterns like "key=%v" or "key: %v" - keyValuePattern := regexp.MustCompile(`(\w+)\s*[:=]\s*%[vsdqxXeEfFgGpbcU]`) - + fallback := func() []any { return []any{"msg", fmt.Sprintf(format, args...)} } matches := keyValuePattern.FindAllStringSubmatchIndex(format, -1) - if len(matches) == 0 || len(matches) > len(args) { - // Fallback to simple message if pattern doesn't match - return []any{"msg", fmt.Sprintf(format, args...)} + if len(matches) == 0 || len(matches) != len(args) { + return fallback() } - - // Build structured fields fields := make([]any, 0, len(matches)*2+2) - lastEnd := 0 - argIndex := 0 - - for _, match := range matches { - // Add any text before this match as part of the message - if match[0] > lastEnd { - prefix := format[lastEnd:match[0]] - if strings.TrimSpace(prefix) != "" { - if len(fields) == 0 { - fields = append(fields, "msg", strings.TrimSpace(prefix)) - } - } + var text []string + end := 0 + for i, m := range matches { + gap := format[end:m[0]] + if strings.Contains(gap, "%") { + return fallback() } - - // Extract key name - keyStart := match[2] - keyEnd := match[3] - key := format[keyStart:keyEnd] - - // Get corresponding value - if argIndex < len(args) { - fields = append(fields, key, args[argIndex]) - argIndex++ + if part := strings.TrimSpace(gap); part != "" { + text = append(text, part) } - - lastEnd = match[1] + fields = append(fields, format[m[2]:m[3]], args[i]) + end = m[1] } - - // Handle remaining format string and args - if lastEnd < len(format) { - remainingFormat := format[lastEnd:] - remainingArgs := args[argIndex:] - if len(remainingArgs) > 0 { - remaining := fmt.Sprintf(remainingFormat, remainingArgs...) - if strings.TrimSpace(remaining) != "" { - if len(fields) == 0 { - fields = append(fields, "msg", strings.TrimSpace(remaining)) - } else { - // Append to existing message - for i := 0; i < len(fields); i += 2 { - if fields[i] == "msg" { - fields[i+1] = fmt.Sprintf("%v %s", fields[i+1], strings.TrimSpace(remaining)) - break - } - } - } - } - } + tail := format[end:] + if strings.Contains(tail, "%") { + return fallback() + } + if part := strings.TrimSpace(tail); part != "" { + text = append(text, part) + } + if len(text) > 0 { + fields = append([]any{"msg", strings.Join(text, " ")}, fields...) } - return fields } @@ -91,6 +62,9 @@ func NewStructuredGnetAdapter(logger *log.Logger, opts ...GnetOption) *Structure // Debugf logs with structured field extraction func (a *StructuredGnetAdapter) Debugf(format string, args ...any) { + if !a.logger.Enabled(log.LevelDebug) { + return + } if a.extractFields { fields := parseFormat(format, args) a.logger.Debug(append(fields, "source", "gnet")...) @@ -101,6 +75,9 @@ func (a *StructuredGnetAdapter) Debugf(format string, args ...any) { // Infof logs with structured field extraction func (a *StructuredGnetAdapter) Infof(format string, args ...any) { + if !a.logger.Enabled(log.LevelInfo) { + return + } if a.extractFields { fields := parseFormat(format, args) a.logger.Info(append(fields, "source", "gnet")...) @@ -111,6 +88,9 @@ func (a *StructuredGnetAdapter) Infof(format string, args ...any) { // Warnf logs with structured field extraction func (a *StructuredGnetAdapter) Warnf(format string, args ...any) { + if !a.logger.Enabled(log.LevelWarn) { + return + } if a.extractFields { fields := parseFormat(format, args) a.logger.Warn(append(fields, "source", "gnet")...) @@ -121,10 +101,13 @@ func (a *StructuredGnetAdapter) Warnf(format string, args ...any) { // Errorf logs with structured field extraction func (a *StructuredGnetAdapter) Errorf(format string, args ...any) { + if !a.logger.Enabled(log.LevelError) { + return + } if a.extractFields { fields := parseFormat(format, args) a.logger.Error(append(fields, "source", "gnet")...) } else { a.GnetAdapter.Errorf(format, args...) } -} \ No newline at end of file +} diff --git a/config.go b/config.go index b1e8c34..05d0895 100644 --- a/config.go +++ b/config.go @@ -2,6 +2,7 @@ package log import ( "fmt" + "math" "strconv" "strings" "time" @@ -117,11 +118,24 @@ func (c *Config) Clone() *Config { // Validate performs validation on the configuration func (c *Config) Validate() error { + if c == nil { + return fmtErrorf("configuration cannot be nil") + } // String validations if strings.TrimSpace(c.Name) == "" { return fmtErrorf("log name cannot be empty") } + if c.Name == "." || c.Name == ".." || strings.ContainsAny(c.Name, "/\\\x00") || strings.ContainsAny(c.Extension, "/\\\x00") { + return fmtErrorf("name and extension must be filename components, without path separators or NUL") + } + if c.EnableFile && strings.TrimSpace(c.Directory) == "" { + return fmtErrorf("directory cannot be empty when file output is enabled") + } + if c.Level == levelOff { + return fmtErrorf("level is reserved for disabling emission") + } + if c.Format != "txt" && c.Format != "json" && c.Format != "raw" { return fmtErrorf("invalid format: '%s' (use txt, json, or raw)", c.Format) } @@ -150,6 +164,14 @@ func (c *Config) Validate() error { return fmtErrorf("buffer_size must be positive: %d", c.BufferSize) } + // Bound allocation before make(chan): one million records already reserves + // over 100 MiB and is well beyond the default queue capacity. + if c.BufferSize > 1<<20 { + return fmtErrorf("buffer_size cannot exceed 1048576 records") + } + if c.MaxSizeKB > math.MaxInt64/sizeMultiplier || c.MaxTotalSizeKB > math.MaxInt64/sizeMultiplier || c.MinDiskFreeKB > math.MaxInt64/sizeMultiplier { + return fmtErrorf("size limits overflow byte counts") + } if c.MaxSizeKB < 0 || c.MaxTotalSizeKB < 0 || c.MinDiskFreeKB < 0 { return fmtErrorf("size limits cannot be negative") } @@ -159,6 +181,27 @@ func (c *Config) Validate() error { return fmtErrorf("interval settings must be positive") } + for _, ms := range []int64{c.FlushIntervalMs, c.DiskCheckIntervalMs, c.MinCheckIntervalMs, c.MaxCheckIntervalMs} { + if ms > math.MaxInt64/int64(time.Millisecond) { + return fmtErrorf("interval overflows time.Duration") + } + } + if c.FlushIntervalMs > math.MaxInt64/(2*int64(time.Millisecond)) { + return fmtErrorf("flush interval overflows default stop timeout") + } + if c.HeartbeatIntervalS > math.MaxInt64/int64(time.Second) { + return fmtErrorf("heartbeat interval overflows time.Duration") + } + for _, setting := range []struct{ value, unit float64 }{ + {c.RetentionPeriodHrs, float64(time.Hour)}, + {c.RetentionCheckMins, float64(time.Minute)}, + } { + ns := setting.value * setting.unit + if math.IsNaN(ns) || math.IsInf(ns, 0) || ns >= float64(math.MaxInt64) || setting.value > 0 && ns < 1 { + return fmtErrorf("retention settings must be finite and representable as time.Duration") + } + } + if c.TraceDepth < 0 || c.TraceDepth > 10 { return fmtErrorf("trace_depth must be between 0 and 10: %d", c.TraceDepth) } @@ -381,6 +424,8 @@ func configRequiresRestart(oldCfg, newCfg *Config) bool { if oldCfg.FlushIntervalMs != newCfg.FlushIntervalMs || oldCfg.DiskCheckIntervalMs != newCfg.DiskCheckIntervalMs || oldCfg.EnableAdaptiveInterval != newCfg.EnableAdaptiveInterval || + oldCfg.MinCheckIntervalMs != newCfg.MinCheckIntervalMs || + oldCfg.MaxCheckIntervalMs != newCfg.MaxCheckIntervalMs || oldCfg.HeartbeatIntervalS != newCfg.HeartbeatIntervalS || oldCfg.HeartbeatLevel != newCfg.HeartbeatLevel || oldCfg.RetentionCheckMins != newCfg.RetentionCheckMins || @@ -411,4 +456,4 @@ func combineConfigErrors(errors []error) error { sb.WriteString(fmt.Sprintf("\n %d. %s", i+1, errMsg)) } return fmt.Errorf("%s", sb.String()) -} \ No newline at end of file +} diff --git a/config_test.go b/config_test.go index 16adcc6..1a8f9f5 100644 --- a/config_test.go +++ b/config_test.go @@ -190,4 +190,3 @@ func TestConcurrentApplyConfig(t *testing.T) { return strings.Contains(readLog(t, tmpDir), "after concurrent config") }) } - diff --git a/doc/adapters.md b/doc/adapters.md index e0361f6..3006e9e 100644 --- a/doc/adapters.md +++ b/doc/adapters.md @@ -1,359 +1,112 @@ -# Compatibility Adapters +# Compatibility adapters -Guide to using lixenwraith/log with popular Go networking frameworks through compatibility adapters. +The `compat` package provides logging adapters for gnet v2, fasthttp, and Fiber v2. Framework dependencies are not imported by this module; applications supply them. -## Overview +## Shared logger setup -The `compat` package provides adapters that allow the lixenwraith/log logger to work seamlessly with: - -- **gnet v2**: High-performance event-driven networking framework -- **fasthttp**: Fast HTTP implementation - -### Features - -- Full interface compatibility -- Preserves structured logging -- Configurable behavior -- Shared logger instances -- Optional field extraction - -## gnet Adapter - -### Basic Usage +Configure and start the logger before passing an adapter to a framework: ```go -import ( - "github.com/lixenwraith/log" - "github.com/lixenwraith/log/compat" - "github.com/panjf2000/gnet/v2" -) - -// Create logger -logger := log.NewLogger() -cfg := log.DefaultConfig() -cfg.Directory = "/var/log/gnet" -logger.ApplyConfig(cfg) -defer logger.Shutdown() - -// Create adapter -adapter := compat.NewGnetAdapter(logger) - -// Use with gnet -gnet.Run(eventHandler, "tcp://127.0.0.1:9000", - gnet.WithLogger(adapter), -) -``` - -### gnet Interface Implementation - -The adapter implements all gnet logger methods: - -```go -type GnetAdapter struct { - logger *log.Logger +logger, err := log.NewBuilder(). + Directory("/var/log/service"). + EnableFile(true). + Format("json"). + LevelString("info"). + Build() +if err != nil { return err } +if err := logger.Start(); err != nil { + _ = logger.Shutdown() + return err } +defer logger.Shutdown(5 * time.Second) -// Methods implemented: -// - Debugf(format string, args ...any) -// - Infof(format string, args ...any) -// - Warnf(format string, args ...any) -// - Errorf(format string, args ...any) -// - Fatalf(format string, args ...any) -``` - -### Custom Fatal Behavior - -Override default fatal handling: - -```go -adapter := compat.NewGnetAdapter(logger, - compat.WithFatalHandler(func(msg string) { - // Custom cleanup - saveApplicationState() - notifyOperations(msg) - gracefulShutdown() - os.Exit(1) - }), -) -``` - -### Complete gnet Example - -```go -type echoServer struct { - gnet.BuiltinEventEngine - logger gnet.Logger -} - -func (es *echoServer) OnBoot(eng gnet.Engine) gnet.Action { - es.logger.Infof("Server started on %s", eng.Addrs) - return gnet.None -} - -func (es *echoServer) OnTraffic(c gnet.Conn) gnet.Action { - buf, _ := c.Next(-1) - es.logger.Debugf("Received %d bytes from %s", len(buf), c.RemoteAddr()) - c.Write(buf) - return gnet.None -} - -func main() { - logger := log.NewLogger() - cfg := log.DefaultConfig() - cfg.Directory = "/var/log/gnet" - cfg.Format = "json" - cfg.BufferSize = 2048 - logger.ApplyConfig(cfg) - defer logger.Shutdown() - - adapter := compat.NewGnetAdapter(logger) - - gnet.Run( - &echoServer{logger: adapter}, - "tcp://127.0.0.1:9000", - gnet.WithMulticore(true), - gnet.WithLogger(adapter), - ) -} -``` - -## fasthttp Adapter - -### Basic Usage - -```go -import ( - "github.com/lixenwraith/log" - "github.com/lixenwraith/log/compat" - "github.com/valyala/fasthttp" -) - -// Create logger -logger := log.NewLogger() -cfg := log.DefaultConfig() -cfg.Directory = "/var/log/fasthttp" -logger.ApplyConfig(cfg) -defer logger.Shutdown() - -// Create adapter -adapter := compat.NewFastHTTPAdapter(logger) - -// Configure server -server := &fasthttp.Server{ - Handler: requestHandler, - Logger: adapter, -} -``` - -### Level Detection - -The adapter automatically detects log levels from message content: - -```go -// Default detection rules: -// - Contains "error", "failed", "fatal", "panic" → ERROR -// - Contains "warn", "warning", "deprecated" → WARN -// - Contains "debug", "trace" → DEBUG -// - Otherwise → INFO -``` - -### Custom Level Detection - -```go -adapter := compat.NewFastHTTPAdapter(logger, - compat.WithDefaultLevel(log.LevelInfo), - compat.WithLevelDetector(func(msg string) int64 { - // Custom detection logic - if strings.Contains(msg, "CRITICAL") { - return log.LevelError - } - if strings.Contains(msg, "performance") { - return log.LevelWarn - } - // Return 0 to use the adapter's default log level (log.LevelInfo by default) - return 0 - }), -) -``` - -## Builder Pattern - -### Using Existing Logger (Recommended) - -Share a configured logger across adapters: - -```go -// Create and configure your main logger -logger := log.NewLogger() -cfg := log.DefaultConfig() -cfg.Level = log.LevelDebug -logger.ApplyConfig(cfg) -logger.Start() -defer logger.Shutdown() - -// Create builder with existing logger builder := compat.NewBuilder().WithLogger(logger) - -// Build adapters -gnetAdapter, _ := builder.BuildGnet() +gnetAdapter, err := builder.BuildGnet() if err != nil { return err } - -fasthttpAdapter, _ := builder.BuildFastHTTP() +fastHTTPAdapter, err := builder.BuildFastHTTP() +if err != nil { return err } +fiberAdapter, err := builder.BuildFiber() if err != nil { return err } ``` -### Creating New Logger +Alternatively, `compat.NewBuilder().WithConfig(cfg)` creates an initialized logger on the first build. Retrieve it with `GetLogger()` and call `Start()` explicitly. `WithLogger` takes precedence over `WithConfig`. Builders and option application are single-goroutine operations; completed adapters may be shared with a shared Logger. -Let the builder create a logger with config: +## gnet + +`GnetAdapter` implements `Debugf`, `Infof`, `Warnf`, `Errorf`, and `Fatalf`. Pass it through `gnet.WithLogger(adapter)`: ```go -// Option 1: With custom config -cfg := log.DefaultConfig() -cfg.Directory = "/var/log/app" -builder := compat.NewBuilder().WithConfig(cfg) - -// Option 2: Default config (created on first build) -builder := compat.NewBuilder() -if err != nil { return err } - -// Build adapters -gnetAdapter, _ := builder.BuildGnet() -logger, _ := builder.GetLogger() // Retrieve for direct use +adapter := compat.NewGnetAdapter(logger) +err := gnet.Run(eventHandler, "tcp://127.0.0.1:9000", gnet.WithLogger(adapter)) ``` -### Structured gnet Adapter +Ordinary messages include `msg` and `source` (`gnet`) as positional arguments. With JSON output, these appear in the `fields` array. Use the logger's `LogStructured` or `LogContext` with `FlagKV` directly if a keyed JSON object is required. -Extract fields from printf-style formats: +### Structured field extraction -```go -structuredAdapter, _ := builder.BuildStructuredGnet() -// "client=%s port=%d" → {"client": "...", "port": ...} -``` - -## Structured Logging - -### Field Extraction - -Structured adapters can extract fields from printf-style formats: - -```go -// Regular adapter output: -// "client=192.168.1.1 port=8080" - -// Structured adapter output: -// {"client": "192.168.1.1", "port": 8080, "source": "gnet"} -``` - -### Pattern Detection - -The structured adapter recognizes patterns like: -- `key=%v` -- `key: %v` -- `key = %v` +`NewStructuredGnetAdapter` extracts simple `key=%v` or `key: %v` expressions with unindexed, single-argument verbs: ```go adapter := compat.NewStructuredGnetAdapter(logger) - -// These will extract structured fields: adapter.Infof("client=%s port=%d", "192.168.1.1", 8080) -// → {"client": "192.168.1.1", "port": 8080} - -adapter.Errorf("user: %s, error: %s", "john", "auth failed") -// → {"user": "john", "error": "auth failed"} - -// These remain as messages: -adapter.Infof("Connected to server") -// → {"msg": "Connected to server"} +// JSON fields: ["client", "192.168.1.1", "port", 8080, "source", "gnet"] ``` -### Integration Examples +Text outside the expressions is retained in `msg`. Formats mixing unrelated conversions, argument indexes, width/precision, escaped percent signs, or a different argument count fall back to a complete `fmt.Sprintf` message. Extraction never guesses which argument belongs to a key. -#### Microservice with Both Frameworks +## fasthttp + +`FastHTTPAdapter` implements the `Printf` method accepted by `fasthttp.Server.Logger`: ```go -type Service struct { - gnetAdapter *compat.GnetAdapter - fasthttpAdapter *compat.FastHTTPAdapter - logger *log.Logger -} - -func NewService() (*Service, error) { - // Create and configure logger - logger := log.NewLogger() - cfg := log.DefaultConfig() - cfg.Directory = "/var/log/service" - cfg.Format = "json" - cfg.HeartbeatLevel = 2 - if err := logger.ApplyConfig(cfg); err != nil { - return nil, err - } - if err := logger.Start(); err != nil { - return nil, err - } - - // Create builder with the logger - builder := compat.NewBuilder().WithLogger(logger) - - // Build adapters - gnetAdapter, err := builder.BuildGnet() - if err != nil { - logger.Shutdown() - return nil, err - } - - fasthttpAdapter, err := builder.BuildFastHTTP() - if err != nil { - logger.Shutdown() - return nil, err - } - - return &Service{ - gnetAdapter: gnetAdapter, - fasthttpAdapter: fasthttpAdapter, - logger: logger, - }, nil +adapter := compat.NewFastHTTPAdapter(logger) +server := &fasthttp.Server{ + Handler: requestHandler, + Logger: adapter, } ``` -#### Middleware Integration +The default detector performs a case-insensitive substring check: + +| Message contains | Level | +|---|---| +| `error`, `failed`, `fatal`, or `panic` | Error | +| `warn` or `deprecated` | Warn | +| `debug` or `trace` | Debug | +| Otherwise | Configured default (Info initially) | + +This heuristic does not interpret HTTP status numbers. Disable it with `WithLevelDetector(nil)`, or provide a custom function. A detector result of zero selects the configured default; nonzero custom levels are preserved. ```go -// gnet middleware -func loggingMiddleware(adapter *compat.GnetAdapter) gnet.EventHandler { - return func(c gnet.Conn) gnet.Action { - start := time.Now() - addr := c.RemoteAddr() - - // Process connection - action := next(c) - - adapter.Infof("conn_duration=%v remote=%s action=%v", - time.Since(start), addr, action) - - return action - } -} - -// fasthttp middleware -func requestLogger(adapter *compat.FastHTTPAdapter) fasthttp.RequestHandler { - return func(ctx *fasthttp.RequestCtx) { - start := time.Now() - - // Process request - next(ctx) - - // Adapter will detect level from status - adapter.Printf("method=%s path=%s status=%d duration=%v", - ctx.Method(), ctx.Path(), - ctx.Response.StatusCode(), - time.Since(start)) - } -} +adapter := compat.NewFastHTTPAdapter(logger, + compat.WithDefaultLevel(log.LevelWarn), + compat.WithLevelDetector(nil), +) ``` +## Fiber + +`FiberAdapter` provides plain, printf-style (`Infof`), and key/value (`Infow`) logging methods, and implements `io.Writer` through `Write([]byte)`. Trace methods currently map to Debug with a `level`, `trace` marker. Use it directly in Fiber middleware: + +```go +adapter := compat.NewFiberAdapter(logger) +app.Use(func(c *fiber.Ctx) error { + err := c.Next() + adapter.Infow("request", "method", c.Method(), "path", c.Path(), + "status", c.Response().StatusCode()) + return err +}) +``` + +## Fatal and panic behavior + +Fatal methods enqueue an Error record, attempt a flush for 100 ms, and invoke the configured handler. Defaults call `os.Exit(1)` for fatal and `panic(msg)` for Fiber panic methods. These effects still occur when the log level suppresses the record. A timed-out or failed flush cannot guarantee that the final message was written. + +Override behavior with `WithFatalHandler`, `WithFiberFatalHandler`, or `WithFiberPanicHandler`. A nil handler disables that action. Shut down the application/framework before the shared logger so shutdown messages can still be recorded. + ### Simple integration example suite -Below simple client and server examples can be used to test the basic functionality of the adapters. They are not included in the package to avoid dependency creep. +These client and server examples can be used to test the basic functionality of the adapters. They are not included in the package to avoid dependency creep. #### gnet server @@ -391,6 +144,7 @@ func main() { // Minimal logger config logger, err := log.NewBuilder(). Directory("./logs_gnet"). + EnableFile(true). Format("json"). LevelString("info"). HeartbeatLevel(0). @@ -453,6 +207,7 @@ func main() { // Minimal logger config logger, err := log.NewBuilder(). Directory("./logs_fasthttp"). + EnableFile(true). Format("json"). LevelString("info"). HeartbeatLevel(0). @@ -521,6 +276,7 @@ func main() { // Minimal logger config logger, err := log.NewBuilder(). Directory("./logs_fiber"). + EnableFile(true). Format("json"). LevelString("info"). HeartbeatLevel(0). diff --git a/doc/api.md b/doc/api.md index 15ef933..c1bdeaa 100644 --- a/doc/api.md +++ b/doc/api.md @@ -37,7 +37,7 @@ Applies a validated configuration to the logger. This is the recommended method ```go logger := log.NewLogger() -cfg := log.GetConfig() +cfg := log.DefaultConfig() cfg.Level = log.LevelDebug cfg.Directory = "/var/log/app" err := logger.ApplyConfig(cfg) @@ -66,7 +66,7 @@ err := logger.ApplyConfigString("directory=/var/log/app", "name=app") ## Logging Methods -All logging methods accept variadic arguments, typically used as key-value pairs for structured logging. +Logging methods accept variadic arguments. Ordinary methods preserve them as positional values (a JSON array in JSON mode). Use `LogContext` with `FlagKV` for a keyed object, or `LogStructured` for a message and nested fields. ### Debug @@ -217,7 +217,7 @@ logger.Log("Checkpoint reached", "step", 5) func (l *Logger) Message(args ...any) ``` -Logs raw message without timestamp or level. +Logs using the configured format and sanitization, without timestamp or level. **Example:** ```go @@ -239,13 +239,21 @@ logger.LogTrace(2, "Function boundary", "entering", true) ## Control Methods +### Start and Stop + +`Start() error` begins processing after configuration; repeated starts are no-ops. `Stop(timeout ...time.Duration) error` closes emission, drains accepted records, and joins the processor. A timeout does not permit a second processor to start while the first still owns resources. Retry `Stop` or `Shutdown` to wait again. Lifecycle timeouts bound the processor join after acquiring the lifecycle lock. + +### Context and fast filtering + +`Enabled(level int64) bool` checks the current emit gate before constructing expensive arguments. `SetLevel(level int64)` updates the threshold without restarting. `SetContextKeys(tag string, vals ...string)` names a tag and up to three counters; `LogContext(ctx Context, flags, level, depth int64, args ...any)` stamps them onto a record. `Flags()` returns display defaults suitable for combining with `FlagKV`. + ### Shutdown ```go func (l *Logger) Shutdown(timeout ...time.Duration) error ``` -Gracefully shuts down the logger, attempting to flush pending logs. +Gracefully drains accepted records and closes the log file. If the processor times out, its resources remain open; retry `Shutdown` to finish cleanup. A fresh `ApplyConfig` permits reuse after shutdown. **Parameters:** - `timeout`: Optional timeout duration (defaults to 2x flush interval) @@ -267,7 +275,7 @@ if err != nil { func (l *Logger) Flush(timeout time.Duration) error ``` -Explicitly triggers a sync of the current log file buffer to disk. Uses **barrier semantics**: it guarantees that all log records enqueued *before* the `Flush` call are fully processed and formatted before the disk sync occurs and confirmation is returned. +Drains records enqueued before the request and syncs the file. Later arrivals do not extend the drain indefinitely. The timeout covers both queueing and confirmation. A successful return reports successful sync; previously dropped records cannot be recovered. A concurrent stop may return a stopped-processor error. **Parameters:** - `timeout`: Maximum time to wait for flush completion @@ -286,6 +294,7 @@ err := logger.Flush(1 * time.Second) ```go const ( + LevelTrace int64 = -8 LevelDebug int64 = -4 LevelInfo int64 = 0 LevelWarn int64 = 4 @@ -316,7 +325,7 @@ func Level(levelStr string) (int64, error) Converts level string to numeric constant. **Parameters:** -- `levelStr`: Level name ("debug", "info", "warn", "error", "proc", "disk", "sys") +- `levelStr`: Level name ("trace", "debug", "info", "warn", "error", "proc", "disk", "sys") **Returns:** - `int64`: Numeric level value @@ -337,6 +346,7 @@ const ( FlagStructuredJSON = formatter.FlagStructuredJSON // Structured JSON output FlagNoTimestamp = formatter.FlagNoTimestamp // Suppress timestamp FlagNoLevel = formatter.FlagNoLevel // Suppress level + FlagKV = formatter.FlagKV // Alternating string keys and values FlagDefault = formatter.FlagDefault // Default flags ) ``` @@ -348,9 +358,9 @@ Control output formatting behavior. These flags are re-exported from the formatt ```go const ( PolicyRaw = sanitizer.PolicyRaw // No sanitization - PolicyJSON = sanitizer.PolicyJSON // JSON-safe output + PolicyJSON = sanitizer.PolicyJSON // Content control-character escapes PolicyTxt = sanitizer.PolicyTxt // Text file safe - PolicyShell = sanitizer.PolicyShell // Shell-safe output + PolicyShell = sanitizer.PolicyShell // Lossy filter, not shell quoting ) ``` @@ -376,7 +386,11 @@ The logger returns errors prefixed with "log: " for easy identification: ## Thread Safety -All public methods are thread-safe and can be called concurrently from multiple goroutines. The logger uses atomic operations and channels to ensure safe concurrent access without locks in the critical path. +Logger methods support concurrent calls. Lifecycle and configuration changes are serialized. Enqueues hold a short read lock to prevent records being stranded during a stop. Logger, Formatter, Sanitizer, and Builder instances must not be copied after use. + +`ApplyConfig` takes a snapshot. Emission copies the argument slice, and `LogStructured` also copies the top-level fields map. Referenced values (byte slices, nested maps/slices, pointers, Stringers, and custom marshalers) remain caller-owned: keep them immutable until a successful `Flush` or `Stop`, or pass an application-created snapshot. Do not mutate configuration while an `ApplyConfig` call is reading it. + +Builders and standalone formatter/sanitizer configuration are not synchronized; finish configuration before sharing. `SetSpawn` callbacks must launch the function asynchronously and return promptly. Error handlers must return promptly and must not invoke lifecycle/configuration methods or `Flush`. ### Usage Pattern Example @@ -396,6 +410,10 @@ func NewService() (*Service, error) { return nil, fmt.Errorf("logger init: %w", err) } + if err := logger.Start(); err != nil { + _ = logger.Shutdown() + return nil, err + } return &Service{logger: logger}, nil } diff --git a/doc/builder.md b/doc/builder.md index 5f95e95..7b9e8e7 100644 --- a/doc/builder.md +++ b/doc/builder.md @@ -14,7 +14,7 @@ builder := log.NewBuilder() ## Builder Methods -All builder methods return `*Builder` for chaining. Errors are accumulated and returned by `Build()`. +All builder methods return `*Builder` for chaining. Validation errors are returned by `Build()`. Builders are not safe for concurrent mutation; each built logger owns a configuration snapshot. ### Common Methods @@ -25,7 +25,7 @@ All builder methods return `*Builder` for chaining. Errors are accumulated and r | `Name(name string)` | `name`: Base filename | Sets log file base name | | `Directory(dir string)` | `dir`: Path | Sets log directory | | `Format(format string)` | `format`: Output format | Sets format ("txt", "json", "raw") | -| `Sanitization(policy string)` | `policy`: Sanitization policy | Sets policy ("txt", "json", "raw", "shell") | +| `Sanitization(policy sanitizer.PolicyPreset)` | `policy`: Sanitization policy | Sets policy ("txt", "json", "raw", "shell") | | `Extension(ext string)` | `ext`: File extension | Sets log file extension | | `BufferSize(size int64)` | `size`: Buffer size | Sets channel buffer size | | `MaxSizeKB(size int64)` | `size`: Size in KB | Sets max file size in KB | @@ -36,7 +36,7 @@ All builder methods return `*Builder` for chaining. Errors are accumulated and r | `MinDiskFreeMB(size int64)` | `size`: Size in MB | Sets minimum required free disk space in MB | | `EnableConsole(enable bool)` | `enable`: Boolean | Enables console output | | `EnableFile(enable bool)` | `enable`: Boolean | Enables file output | -| `ConsoleTarget(target string)` | `target`: "stdout"/"stderr" | Sets console output target | +| `ConsoleTarget(target string)` | `target`: "stdout"/"stderr"/"split" | Sets console output target | | `ShowTimestamp(show bool)` | `show`: Boolean | Controls timestamp display | | `ShowLevel(show bool)` | `show`: Boolean | Controls log level display | | `TimestampFormat(format string)` | `format`: Time format | Sets timestamp format (Go time format) | diff --git a/doc/configuration.md b/doc/configuration.md index 700f869..ec0c587 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -17,39 +17,36 @@ logger := log.NewLogger() Direct struct configuration using the Config struct, or key-value overrides: ```go -logger := log.NewLogger() // logger instance created with DefaultConfig (using default values) - -// Note: with default config, logs only go to stderr (file output disabled by default) -logger.Start() // Required before logging -logger.Info("info raw log record written to stderr") - -// Directly change config struct -cfg := log.GetConfig() +logger := log.NewLogger() +cfg := log.DefaultConfig() cfg.Level = log.LevelDebug cfg.Name = "myapp" cfg.Directory = "/var/log/myapp" +cfg.EnableFile = true cfg.Format = "json" cfg.MaxSizeKB = 100 -err := logger.ApplyConfig(cfg) +if err := logger.ApplyConfig(cfg); err != nil { return err } +if err := logger.Start(); err != nil { return err } +defer logger.Shutdown() +logger.Info("application started") -logger.Info("info json log record written to /var/log/myapp/myapp.log") - -// Override values with key-value string -err = logger.ApplyConfigString( - "directory=/var/log/", - "extension=txt" - "format=txt") - -logger.Info("info txt log record written to /var/log/myapp.txt") +// Apply overrides to the current configuration atomically. +if err := logger.ApplyConfigString("extension=txt", "format=txt"); err != nil { + return err +} ``` +`ApplyConfig` copies the supplied configuration. The caller may reuse or modify it after the call returns. Failed validation, directory creation, or file opening leaves the active configuration unchanged. Changes requiring a restart drain accepted records using the old file and configuration before publishing the replacement. Calls made while emission is stopped are ignored; a full queue drops records. + +Names and extensions must be filename components, without path separators or NUL. The queue size must be between 1 and 1,048,576 records. Numeric settings are checked for byte-count and duration overflow; positive fractional retention settings must resolve to at least one nanosecond. Size settings use decimal units: 1 KB = 1,000 bytes and 1 MB = 1,000 KB. + ## Configuration Parameters ### Basic Settings | Parameter | Type | Description | Default | |-----------|------|-------------|------------| -| `level` | `int64` | Minimum log level (-4=Debug, 0=Info, 4=Warn, 8=Error) | `0` | +| `level` | `int64` | Minimum log level (-8=Trace, -4=Debug, 0=Info, 4=Warn, 8=Error) | `0` | | `name` | `string` | Base name for log files | `"log"` | | `extension` | `string` | Log file extension (without dot) | `"log"` | | `directory` | `string` | Directory to store log files | `"./log"` | @@ -66,7 +63,7 @@ logger.Info("info txt log record written to /var/log/myapp.txt") | `show_level` | `bool` | Include log level in entries | `true` | | `enable_console` | `bool` | Enable console output (stdout/stderr) | `true` | | `console_target` | `string` | Console target: `"stdout"`, `"stderr"`, or `"split"` | `"stderr"` | -| `enable_file` | `bool` | Enable file output (console-only) | `false` | +| `enable_file` | `bool` | Enable file output | `false` | **Note:** When `console_target="split"`, INFO/DEBUG logs go to stdout while WARN/ERROR logs go to stderr. diff --git a/doc/formatting.md b/doc/formatting.md index b4a9dd1..88556d0 100644 --- a/doc/formatting.md +++ b/doc/formatting.md @@ -65,6 +65,7 @@ const ( FlagStructuredJSON int64 = 0b1000 // Use structured JSON with message/fields FlagNoTimestamp int64 = 0b010000 // Suppress timestamp FlagNoLevel int64 = 0b100000 // Suppress level + FlagKV int64 = 0b1000000 // Alternating string keys and values FlagDefault = FlagShowTimestamp | FlagShowLevel ) ``` @@ -105,9 +106,9 @@ clean = s.Sanitize("cmd; echo test") // "cmd echo test" ```go const ( PolicyRaw PolicyPreset = "raw" // No-op passthrough - PolicyJSON PolicyPreset = "json" // JSON-safe strings + PolicyJSON PolicyPreset = "json" // Visible control-character escapes PolicyTxt PolicyPreset = "txt" // Text file safe - PolicyShell PolicyPreset = "shell" // Shell command safe + PolicyShell PolicyPreset = "shell" // Lossy filter, not shell quoting ) ``` @@ -171,7 +172,26 @@ serializer.WriteNil(&buf) // "null" The sanitizer is a content transform; JSON string escaping is transport encoding applied afterward, unconditionally. Output is valid JSON for any -sanitization policy. Multi-byte UTF-8 passes through unescaped. +sanitization policy. Multi-byte UTF-8 passes through unescaped. Malformed UTF-8 +is replaced before rule-based sanitization, preventing stripped text from joining +invalid byte fragments into control characters. A sanitizer with no rules remains +byte-preserving; the JSON transport layer always replaces malformed UTF-8. + +JSON keys and custom timestamp layouts are escaped. Non-finite scalar floats +are emitted as strings (`"NaN"`, `"+Inf"`, `"-Inf"`); non-finite values in structured +maps follow `encoding/json` and produce a marshal error. As in the existing API, +`int32` values are treated as runes; cast to `int64` when numeric output is needed. +Choose unique context and KV keys; avoid context keys named `time`, `level`, +`trace`, `message`, or `fields`. Content policies can make distinct keys identical. + +`PolicyTxt` covers text timestamps, trace, context, KV keys, and values. `PolicyRaw` +is intentionally unfiltered, and `FlagRaw` bypasses formatting and sanitization. +Generic complex-value text formatting is bounded to 64 levels and 10,000 visited +values; cycles and larger graphs produce ``. This also +applies to complex values under `FlagRaw`, which use `%+v` text formatting. +Custom formatting and marshal callbacks must terminate and be safe for shared use. +A custom rule with no recognized transform preserves its matched rune. Predicate +callbacks must be safe for concurrent use when the sanitizer is shared. - `format=json` + `sanitization=raw`: recommended; transport escaping only. - `format=json` + `sanitization=txt`: non-printables appear as `` inside @@ -179,8 +199,9 @@ sanitization policy. Multi-byte UTF-8 passes through unescaped. - `format=json` + `sanitization=json`: redundant; produces visible `\\n` double escapes. Use `raw` instead. - Structured JSON (`FlagStructuredJSON`) marshals the fields map via - `encoding/json` and bypasses the sanitizer; validity is guaranteed, - content-level sanitization is not applied to field values. + `encoding/json`, then applies the content policy to every string, including + nested values and object keys. Value types and deterministic map order are preserved. + Marshal failures produce a valid `fields._marshal_error` string. ## PolicyShell Scope @@ -215,13 +236,13 @@ logger := log.NewLogger() // Configure sanitization policy logger.ApplyConfigString( "format=json", - "sanitization=json", // Uses PolicyJSON + "sanitization=raw", // JSON transport escaping remains enabled ) // Or with custom formatter (advanced) s := sanitizer.New().Policy(sanitizer.PolicyShell) customFormatter := formatter.New(s).Type("txt") -// Note: Direct formatter injection requires using lower-level APIs +// Use customFormatter directly; Logger has no formatter injection API. ``` ## Common Patterns @@ -247,7 +268,7 @@ f := formatter.New() f.Type("json").ShowTimestamp(false).ShowLevel(false) // Create custom log entry -entry := f.FormatArgs("action", "purchase", "amount", 99.99) +entry := f.Format(formatter.FlagKV, time.Now(), 0, "", []any{"action", "purchase", "amount", 99.99}) sendToExternalSystem(entry) ``` @@ -262,17 +283,17 @@ shellSanitizer := sanitizer.New().Policy(sanitizer.PolicyShell) jsonFormatter := formatter.New(jsonSanitizer).Type("json") apiLog := jsonFormatter.Format(...) -// For shell script generation +// For a filtered text log txtFormatter := formatter.New(shellSanitizer).Type("txt") -scriptLog := txtFormatter.Format(...) +textLog := txtFormatter.Format(...) ``` ## Performance Considerations -- Both packages use pre-allocated buffers for efficiency -- Sanitizer rules are applied in a single pass +- Formatter append methods reuse caller-provided buffers +- Sanitizer avoids allocations when no rule matches valid UTF-8 - Formatter reuses internal buffers via `Reset()` -- No regex or reflection in hot paths +- Scalar formatting avoids reflection; complex values and structured marshaling may use it ## Ownership and Thread Safety diff --git a/doc/getting-started.md b/doc/getting-started.md index b8cb4f9..8543d5c 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -4,7 +4,7 @@ This guide will help you get started with the lixenwraith/log package, from inst ## Installation -Install the logger package: +Requires Go 1.27.1 or later. Install the logger package: ```bash go get github.com/lixenwraith/log @@ -74,7 +74,7 @@ func main() { Directory("/var/log/myapp"). // Log directory path LevelString("info"). // Minimum log level Format("json"). // Output format - Sanitization("json"). // Sanitization policy + Sanitization(log.PolicyRaw). // Sanitization policy EnableFile(true). // Enable file output (disabled by default) BufferSize(2048). // Channel buffer size MaxSizeMB(10). // Max file size before rotation @@ -102,8 +102,8 @@ func main() { ## Next Steps 1. **[Learn about configuration options](configuration.md)** - Customize behavior for your needs -2. **[Explore the API](api-reference.md)** - See all available methods -3. **[Logging patterns and examples](logging-guide.md)** - Write better logs +2. **[Explore the API](api.md)** - See all available methods +3. **[Logging patterns and examples](logging.md)** - Write better logs ## Common Patterns @@ -125,7 +125,10 @@ func NewService() (*Service, error) { return nil, fmt.Errorf("logger init failed: %w", err) } - logger.Start() + if err := logger.Start(); err != nil { + _ = logger.Shutdown() + return nil, err + } return &Service{ logger: logger, diff --git a/doc/heartbeat.md b/doc/heartbeat.md index 7a638fa..0fa71d7 100644 --- a/doc/heartbeat.md +++ b/doc/heartbeat.md @@ -8,7 +8,7 @@ Heartbeats are periodic log messages that provide operational statistics about t ### Key Features -- **Always Visible**: Heartbeats use special log levels that bypass filtering +- **Independent of level filtering**: Heartbeats can still be dropped on queue overflow or output failure - **Multi-Level Detail**: Choose from process, disk, or system statistics - **Production Monitoring**: Track logger health without debug logs - **Metrics Source**: Parse heartbeats for monitoring dashboards @@ -38,14 +38,15 @@ logger.ApplyConfigString( **Output:** ``` -2024-01-15T10:30:00Z PROC type="proc" sequence=1 uptime_hours="24.50" processed_logs=1847293 dropped_logs=0 +2024-01-15T10:30:00Z PROC type=proc sequence=1 uptime_hours="24.50" processed_logs=1847293 total_dropped_logs=0 ``` **Fields:** - `sequence`: Incrementing counter - `uptime_hours`: Logger uptime -- `processed_logs`: Successfully written logs -- `dropped_logs`: Logs lost due to buffer overflow +- `processed_logs`: Application records processed successfully, excluding heartbeats +- `total_dropped_logs`: Cumulative queue and output-processing drops, including heartbeats +- `dropped_since_last`: Interval drops, emitted only when non-zero; reset at each PROC heartbeat attempt ### Level 2: Process + Disk Statistics (DISK) @@ -60,7 +61,7 @@ logger.ApplyConfigString( **Additional Output:** ``` -2024-01-15T10:30:00Z DISK type="disk" sequence=1 rotated_files=12 deleted_files=5 total_log_size_mb="487.32" log_file_count=8 current_file_size_mb="23.45" disk_status_ok=true disk_free_mb="5234.67" +2024-01-15T10:30:00Z DISK type=disk sequence=1 rotated_files=12 deleted_files=5 total_log_size_mb="487.32" log_file_count=8 current_file_size_mb="23.45" disk_status_ok=true disk_free_mb="5234.67" ``` **Additional Fields:** @@ -85,7 +86,7 @@ logger.ApplyConfigString( **Additional Output:** ``` -2024-01-15T10:30:00Z SYS type="sys" sequence=1 alloc_mb="45.23" sys_mb="128.45" num_gc=1523 num_goroutine=42 +2024-01-15T10:30:00Z SYS type=sys sequence=1 alloc_mb="45.23" sys_mb="128.45" num_gc=1523 num_goroutine=42 ``` **Additional Fields:** @@ -146,13 +147,13 @@ With `format=json`, heartbeats are structured for easy parsing: { "time": "2024-01-15T10:30:00.123456789Z", "level": "PROC", - "fields": [ - "type", "proc", - "sequence", 42, - "uptime_hours", "24.50", - "processed_logs", 1847293, - "dropped_logs", 0 - ] + "fields": { + "type": "proc", + "sequence": 42, + "uptime_hours": "24.50", + "processed_logs": 1847293, + "total_dropped_logs": 0 + } } ``` @@ -161,5 +162,7 @@ With `format=json`, heartbeats are structured for easy parsing: With `format=txt`, heartbeats are human-readable: ``` -2024-01-15T10:30:00.123456789Z PROC type="proc" sequence=42 uptime_hours="24.50" processed_logs=1847293 dropped_logs=0 -``` \ No newline at end of file +2024-01-15T10:30:00.123456789Z PROC type=proc sequence=42 uptime_hours="24.50" processed_logs=1847293 total_dropped_logs=0 +``` + +PROC uptime and totals span the lifetime of the Logger instance, including stop/start cycles. DISK fields ending in `_mb` retain historical binary MiB units (1,048,576 bytes); SYS memory fields use decimal MB. Configuration size limits use decimal KB. diff --git a/doc/logging.md b/doc/logging.md index af33e98..2065c8a 100644 --- a/doc/logging.md +++ b/doc/logging.md @@ -44,7 +44,7 @@ logger.ApplyConfigString("level=8") // Error only ### Key-Value Pairs -Use structured key-value pairs for machine-parseable logs: +Ordinary methods emit positional arguments (JSON arrays). Use `LogStructured`, or `LogContext` with `FlagKV`, when named JSON fields are required: ```go logger.Info("User login", @@ -117,9 +117,9 @@ func logWithContext(ctx context.Context, logger *log.Logger, level string, msg s switch level { case "info": - logger.Info(msg, fields...) + logger.Info(append([]any{msg}, fields...)...) case "error": - logger.Error(msg, fields...) + logger.Error(append([]any{msg}, fields...)...) } } ``` @@ -130,11 +130,11 @@ The logger supports three output formats, each with configurable sanitization. T ### Txt Format (Human-Readable) -Default format for development and debugging: +Text format for development and debugging (select `format=txt`): ``` -2024-01-15T10:30:45.123456789Z INFO User login user_id=42 email=user@example.com ip=192.168.1.100 -2024-01-15T10:30:45.234567890Z WARN Rate limit approaching user_id=42 requests=95 limit=100 +2024-01-15T10:30:45.123456789Z INFO "User login" user_id 42 email user@example.com ip 192.168.1.100 +2024-01-15T10:30:45.234567890Z WARN "Rate limit approaching" user_id 42 requests 95 limit 100 ``` Note: The txt format applies the configured sanitization policy (default: raw). Non-printable characters can be hex-encoded using `sanitization=txt` configuration. @@ -204,8 +204,8 @@ func processPayment(amount float64) error { Output includes function names: ``` -2024-01-15T10:30:45.123456789Z INFO processPayment Processing payment amount=99.99 -2024-01-15T10:30:45.234567890Z ERROR validateAmount -> processPayment -> main Payment validation failed amount=-10 error="negative amount" +2024-01-15T10:30:45.123456789Z INFO processPayment "Processing payment" amount 99.99 +2024-01-15T10:30:45.234567890Z ERROR main -> processPayment "Payment validation failed" amount -10 error "negative amount" ``` ### Trace Depth Guidelines diff --git a/doc/storage.md b/doc/storage.md index e87a18d..b795395 100644 --- a/doc/storage.md +++ b/doc/storage.md @@ -10,31 +10,19 @@ Log files are automatically rotated when they reach the configured size limit in ```go logger.ApplyConfigString( - "max_size_kb=102400", // Rotate at 100MB (102400 KB) + "max_size_kb=100000", // Rotate at 100 MB (100,000 KB) ) ``` ### Rotation Behavior -1. **Size Check**: Before each write, the logger checks if the file would exceed `max_size_kb` -2. **New File Creation**: Creates a new file with timestamp: `appname_240115_103045_123456789.log` -3. **Seamless Transition**: No logs are lost during rotation -4. **Old File Closure**: Previous file is properly closed and synced +1. Before each write, the logger checks whether the active file would exceed `max_size_kb`. +2. It closes the active `{name}.{extension}` file and renames it to `{name}_{YYMMDD}_{HHMMSS}[_N].{extension}`. +3. It opens a new active file. A counter suffix avoids existing names when rotations share a second. -### File Naming Convention +For example: `myapp.log` becomes `myapp_240115_143022.log`, then `myapp_240115_143022_1.log` on the next collision. Rotation does not split a record, so a single oversized record can exceed the size limit. File errors can drop records and are counted. Periodic sync, explicit `Flush`, and shutdown provide sync points; closing a file alone does not guarantee crash durability. -``` -{name}_{YYMMDD}_{HHMMSS}_{nanoseconds}.{extension} - -Example: myapp_240115_143022_987654321.log -``` - -Components: -- `name`: Configured log name -- `YYMMDD`: Date (year, month, day) -- `HHMMSS`: Time (hour, minute, second) -- `nanoseconds`: For uniqueness -- `extension`: Configured extension +Use a directory owned exclusively by this logger. Cleanup and retention select regular files by extension, including files from earlier runs with different base names. They exclude the active filename, directories, and symlinks. With an empty extension, only extensionless files match. Console-only configurations never perform retention deletion. Total-size and free-space limits are checked periodically and can be exceeded between checks. ## Disk Space Management @@ -45,7 +33,7 @@ The logger enforces two types of space limits: ```go logger.ApplyConfigString( "max_total_size_kb=1000", // Total log directory size - "min_disk_free_kb=5000", // Minimum free disk space + "min_disk_free_mb=5000", // Minimum free disk space ) ``` @@ -55,7 +43,7 @@ When limits are exceeded, the logger: 1. Identifies oldest log files 2. Deletes them until space requirements are met 3. Preserves the current active log file -4. Logs cleanup actions for audit +4. Increments deletion counters reported in disk heartbeats ### Example Configuration @@ -64,21 +52,21 @@ When limits are exceeded, the logger: logger.ApplyConfigString( "max_size_kb=500", // 500 KB files "max_total_size_kb=5000", // 5 MB total log directory limit - "min_disk_free_kb=1048576", // 1 GB free space required on disk + "min_disk_free_mb=1000000", // 1 GB free space required on disk ) // Generous: Large files, external archival logger.ApplyConfigString( - "max_size_kb=102400", // 100 MB files + "max_size_kb=100000", // 100 MB files "max_total_size_kb=0", // No total limit - "min_disk_free_kb=10240", // 10 MB free required + "min_disk_free_mb=10000", // 10 MB free required ) // Balanced: Production defaults logger.ApplyConfigString( - "max_size_kb=102400", // 100 MB files - "max_total_size_kb=5242880", // 5 GB total limit - "min_disk_free_kb=512000", // 500 MB free required + "max_size_kb=100000", // 100 MB files + "max_total_size_kb=5000000", // 5 GB total limit + "min_disk_free_mb=500000", // 500 MB free required ) ``` @@ -161,7 +149,7 @@ logger.ApplyConfigString( Output: ``` -2024-01-15T10:30:00Z DISK type="disk" sequence=1 rotated_files=5 deleted_files=2 total_log_size_kb="487.32" log_file_count=8 current_file_size_kb="23.45" disk_status_ok=true disk_free_kb="5234.67" +2024-01-15T10:30:00Z DISK type="disk" sequence=1 rotated_files=5 deleted_files=2 total_log_size_mb="487.32" log_file_count=8 current_file_size_mb="23.45" disk_status_ok=true disk_free_mb="5234.67" ``` ## Manual Recovery @@ -176,7 +164,8 @@ df -h /var/log find /var/log/myapp -name "*.log" -size +100M # Manual cleanup (oldest first) -ls -t /var/log/myapp/*.log | tail -n 20 | xargs rm +# Review archive paths before deleting; preserve the active log file. +find /var/log/myapp -maxdepth 1 -type f -name 'myapp_*.log' -print # Verify space df -h /var/log diff --git a/doc/testing.md b/doc/testing.md new file mode 100644 index 0000000..0a76321 --- /dev/null +++ b/doc/testing.md @@ -0,0 +1,29 @@ +# Testing and performance + +Use Go 1.27.1 or later. The module has no external dependencies. + +```sh +go vet ./... +go test -race -count=1 -timeout=3m ./... +go test -race -run 'TestStress' -count=10 -timeout=3m . +``` + +`stress_test.go` uses fixed record counts and synchronized starts. It verifies exact queue overflow, unique IDs, processed-plus-dropped accounting across lifecycle changes, and byte preservation across rotations. It has no throughput pass threshold and no timed soak loop. Heartbeat and adaptive-timer tests use `testing/synctest` or explicit state transitions instead of wall-clock sleeps. Run benchmarks separately when comparing performance: + +```sh +go test ./... -run '^$' -bench . -benchmem -count=5 +``` + +Fuzz targets test malformed UTF-8, control characters, keys, custom timestamp layouts, numeric edge cases, structured marshal failures, and printf argument association. Each target runs independently: + +```sh +go test ./sanitizer -run '^$' -fuzz '^FuzzSanitize$' -fuzztime=30s -parallel=2 +go test ./sanitizer -run '^$' -fuzz '^FuzzSerializerJSON$' -fuzztime=30s -parallel=2 +go test ./formatter -run '^$' -fuzz '^FuzzFormatJSON$' -fuzztime=30s -parallel=2 +go test ./formatter -run '^$' -fuzz '^FuzzFormatTextPolicy$' -fuzztime=30s -parallel=2 +go test ./compat -run '^$' -fuzz '^FuzzParseFormat$' -fuzztime=30s -parallel=2 +``` + +Fuzzing is bounded by time in CI; execution counts depend on the runner. All committed seeds run during ordinary `go test`, including minimized failures retained under `testdata/fuzz`. + +The GitHub workflow runs on pushes and pull requests. It checks formatting, vet, race detection, repeated lifecycle/stress workloads, bounded fuzzing, and compilation for FreeBSD, Windows, and browser WASM. Runtime tests execute on Linux; cross-compilation does not validate other operating systems' runtime behavior. diff --git a/formatter/formatter.go b/formatter/formatter.go index c7ee0a2..2cffd90 100644 --- a/formatter/formatter.go +++ b/formatter/formatter.go @@ -15,8 +15,9 @@ package formatter import ( - "encoding/json" + "encoding/json/jsontext" "fmt" + "math" "strconv" "time" "unicode/utf8" @@ -50,9 +51,10 @@ type Formatter struct { // Serializers are stateless and format-fixed; built once to keep the // per-record path allocation-free - serTxt *sanitizer.Serializer - serJSON *sanitizer.Serializer - serRaw *sanitizer.Serializer + serTxt *sanitizer.Serializer + serJSON *sanitizer.Serializer + serRaw *sanitizer.Serializer + serPlain *sanitizer.Serializer } // ContextSlots is the number of correlation values a Context carries @@ -66,7 +68,8 @@ type Context struct { } // ContextKeys names the record keys for Context fields; empty names are omitted. -// Names are emitted verbatim and must be plain identifiers. +// Names are escaped for the output format. Use distinct names that do not +// collide with built-in time, level, trace, message, or fields keys. type ContextKeys struct { Tag string Vals [ContextSlots]string @@ -90,6 +93,7 @@ func New(s ...*sanitizer.Sanitizer) *Formatter { serTxt: sanitizer.NewSerializer("txt", san), serJSON: sanitizer.NewSerializer("json", san), serRaw: sanitizer.NewSerializer("raw", san), + serPlain: sanitizer.NewSerializer("raw", sanitizer.New()), } } @@ -198,16 +202,22 @@ func (f *Formatter) AppendFormatWithOptionsCtx(dst []byte, format string, ctx Co case []byte: dst = append(dst, v...) case fmt.Stringer: - dst = append(dst, v.String()...) + dst = fmt.Append(dst, v) case error: - dst = append(dst, v.Error()...) + dst = fmt.Append(dst, v) default: - dst = append(dst, fmt.Sprint(v)...) + f.serPlain.WriteComplex(&dst, v) } } return dst } + if flags&FlagNoTimestamp != 0 { + flags &^= FlagShowTimestamp + } + if flags&FlagNoLevel != 0 { + flags &^= FlagShowLevel + } format = normalizeFormat(format) serializer := f.serializerFor(format) @@ -271,10 +281,7 @@ func (f *Formatter) AppendArgs(dst []byte, args ...any) []byte { return dst } -// appendValue provides unified type conversion (was convertValue; now -// value-return style over caller buffer). Type switch body unchanged except -// buffer plumbing — replace every `serializer.WriteX(buf, ...)` with -// `serializer.WriteX(&dst, ...)` and `return dst`. +// appendValue converts one value using the selected serializer. func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Serializer, needsSpace bool) []byte { if needsSpace && len(dst) > 0 { dst = append(dst, ' ') @@ -289,17 +296,29 @@ func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Seriali n := utf8.EncodeRune(runeStr[:], val) serializer.WriteString(&dst, string(runeStr[:n])) case int: - serializer.WriteNumber(&dst, string(strconv.AppendInt(nil, int64(val), 10))) + dst = strconv.AppendInt(dst, int64(val), 10) + case int8: + dst = strconv.AppendInt(dst, int64(val), 10) + case int16: + dst = strconv.AppendInt(dst, int64(val), 10) case int64: - serializer.WriteNumber(&dst, string(strconv.AppendInt(nil, val, 10))) + dst = strconv.AppendInt(dst, val, 10) case uint: - serializer.WriteNumber(&dst, string(strconv.AppendUint(nil, uint64(val), 10))) + dst = strconv.AppendUint(dst, uint64(val), 10) + case uint8: + dst = strconv.AppendUint(dst, uint64(val), 10) + case uint16: + dst = strconv.AppendUint(dst, uint64(val), 10) + case uint32: + dst = strconv.AppendUint(dst, uint64(val), 10) + case uintptr: + dst = strconv.AppendUint(dst, uint64(val), 10) case uint64: - serializer.WriteNumber(&dst, string(strconv.AppendUint(nil, val, 10))) + dst = strconv.AppendUint(dst, val, 10) case float32: - serializer.WriteNumber(&dst, string(strconv.AppendFloat(nil, float64(val), 'f', -1, 32))) + dst = appendFloat(dst, float64(val), 32, serializer) case float64: - serializer.WriteNumber(&dst, string(strconv.AppendFloat(nil, val, 'f', -1, 64))) + dst = appendFloat(dst, val, 64, serializer) case bool: serializer.WriteBool(&dst, val) case nil: @@ -307,15 +326,25 @@ func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Seriali case time.Time: serializer.WriteString(&dst, val.Format(f.timestampFormat)) case error: - serializer.WriteString(&dst, val.Error()) + serializer.WriteString(&dst, fmt.Sprint(val)) case fmt.Stringer: - serializer.WriteString(&dst, val.String()) + serializer.WriteString(&dst, fmt.Sprint(val)) default: serializer.WriteComplex(&dst, val) } return dst } +// Non-finite floats have no JSON number representation. Keep their spelling +// as a string so a single bad measurement cannot corrupt the record. +func appendFloat(dst []byte, value float64, bits int, serializer *sanitizer.Serializer) []byte { + if math.IsNaN(value) || math.IsInf(value, 0) { + serializer.WriteString(&dst, strconv.FormatFloat(value, 'f', -1, bits)) + return dst + } + return strconv.AppendFloat(dst, value, 'f', -1, bits) +} + // LevelToString converts integer level values to string func LevelToString(level int64) string { switch level { @@ -342,10 +371,8 @@ func LevelToString(level int64) string { // appendJSONKey writes a quoted literal key followed by ':' func appendJSONKey(dst []byte, key string) []byte { - dst = append(dst, '"') - dst = append(dst, key...) - dst = append(dst, '"', ':') - return dst + dst, _ = jsontext.AppendQuote(dst, key) + return append(dst, ':') } // isKV reports whether args form an even-length list with string keys @@ -367,9 +394,9 @@ func (f *Formatter) appendJSON(dst []byte, ctx Context, flags int64, timestamp t needsComma := false if flags&FlagShowTimestamp != 0 { - dst = append(dst, `"time":"`...) - dst = timestamp.AppendFormat(dst, f.timestampFormat) - dst = append(dst, '"') + dst = append(dst, `"time":`...) + var stamp [128]byte + serializer.WriteString(&dst, string(timestamp.AppendFormat(stamp[:0], f.timestampFormat))) needsComma = true } @@ -425,13 +452,10 @@ func (f *Formatter) appendJSON(dst []byte, ctx Context, flags int64, timestamp t dst = append(dst, `,"fields":`...) - marshaledFields, err := json.Marshal(fields) - if err != nil { - dst = append(dst, `{"_marshal_error":"`...) + if err := serializer.WriteJSON(&dst, fields); err != nil { + dst = append(dst, `{"_marshal_error":`...) serializer.WriteString(&dst, err.Error()) - dst = append(dst, `"}`...) - } else { - dst = append(dst, marshaledFields...) + dst = append(dst, '}') } dst = append(dst, '}', '\n') @@ -477,7 +501,8 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti needsSpace := false if flags&FlagShowTimestamp != 0 { - dst = timestamp.AppendFormat(dst, f.timestampFormat) + var stamp [128]byte + dst = f.sanitizer.AppendSanitize(dst, string(timestamp.AppendFormat(stamp[:0], f.timestampFormat))) needsSpace = true } @@ -493,7 +518,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti if needsSpace { dst = append(dst, ' ') } - dst = append(dst, f.ctxKeys.Tag...) + serializer.WriteString(&dst, f.ctxKeys.Tag) dst = append(dst, '=') dst = f.appendValue(dst, ctx.Tag, serializer, false) needsSpace = true @@ -505,7 +530,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti if needsSpace { dst = append(dst, ' ') } - dst = append(dst, key...) + serializer.WriteString(&dst, key) dst = append(dst, '=') dst = strconv.AppendUint(dst, ctx.Vals[i], 10) needsSpace = true @@ -515,15 +540,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti if needsSpace { dst = append(dst, ' ') } - // Sanitize trace to prevent terminal control sequence injection - tempBuf := make([]byte, 0, len(trace)*2) - f.serTxt.WriteString(&tempBuf, trace) - // Extract content without quotes if added by txt serializer - if len(tempBuf) > 2 && tempBuf[0] == '"' && tempBuf[len(tempBuf)-1] == '"' { - dst = append(dst, tempBuf[1:len(tempBuf)-1]...) - } else { - dst = append(dst, tempBuf...) - } + dst = f.sanitizer.AppendSanitize(dst, trace) needsSpace = true } @@ -532,7 +549,7 @@ func (f *Formatter) appendTxt(dst []byte, ctx Context, flags int64, timestamp ti if needsSpace { dst = append(dst, ' ') } - dst = append(dst, args[i].(string)...) + serializer.WriteString(&dst, args[i].(string)) dst = append(dst, '=') dst = f.appendValue(dst, args[i+1], serializer, false) needsSpace = true diff --git a/formatter/fuzz_test.go b/formatter/fuzz_test.go new file mode 100644 index 0000000..c5193eb --- /dev/null +++ b/formatter/fuzz_test.go @@ -0,0 +1,118 @@ +package formatter + +import ( + "bytes" + "encoding/json" + "math" + "strings" + "testing" + "unicode" + "unicode/utf8" + + "github.com/lixenwraith/log/sanitizer" +) + +func FuzzFormatJSON(f *testing.F) { + for _, s := range []string{"", "safe", "\xff\n\"\\", "\u0085\x1b[0m", "世界"} { + f.Add(s, s, s, uint64(0), uint8(0)) + } + f.Fuzz(func(t *testing.T, value, key, layout string, bits uint64, mode uint8) { + policies := []sanitizer.PolicyPreset{sanitizer.PolicyRaw, sanitizer.PolicyTxt, sanitizer.PolicyJSON, sanitizer.PolicyShell} + fm := New(sanitizer.New().Policy(policies[int(mode)%4])).Type("json").TimestampFormat(layout).ContextKeys(key) + flags := FlagDefault + args := []any{value, []byte(value), math.Float64frombits(bits), uint32(bits)} + switch mode % 3 { + case 1: + flags |= FlagKV + args = []any{key, value, "number", math.Float64frombits(bits)} + case 2: + flags |= FlagStructuredJSON + args = []any{value, map[string]any{key: []any{value, math.Float64frombits(bits)}}} + } + prefix := []byte("prefix:") + out := fm.AppendFormatCtx(bytes.Clone(prefix), Context{Tag: value}, flags, testStamp, int64(bits), value, args) + if !bytes.HasPrefix(out, prefix) { + t.Fatal("prefix changed") + } + data := out[len(prefix):] + if !utf8.Valid(data) || !json.Valid(data) || bytes.Count(data, []byte{'\n'}) != 1 || data[len(data)-1] != '\n' { + t.Fatalf("invalid JSON record %q", data) + } + if second := fm.AppendFormatCtx(nil, Context{Tag: value}, flags, testStamp, int64(bits), value, args); !bytes.Equal(data, second) { + t.Fatal("non-deterministic formatter") + } + }) +} + +func FuzzFormatTextPolicy(f *testing.F) { + for _, s := range []string{"", "\n\r\x1b[31m", "\xff<00>\u0085", "a=b \"c\""} { + f.Add(s, s, s) + } + f.Fuzz(func(t *testing.T, value, key, layout string) { + fm := New(sanitizer.New().Policy(sanitizer.PolicyTxt)).Type("txt").TimestampFormat(layout).ContextKeys(key, key) + out := fm.AppendFormatCtx(nil, Context{Tag: value}, FlagDefault|FlagKV, testStamp, 0, value, []any{key, value}) + if !bytes.HasSuffix(out, []byte{'\n'}) { + t.Fatal("missing terminator") + } + for _, r := range string(out[:len(out)-1]) { + if unicode.IsControl(r) || !unicode.IsPrint(r) { + t.Fatalf("unsafe rune %U in %q", r, out) + } + } + }) +} + +type panickingStringer struct{} + +func (panickingStringer) String() string { panic("broken Stringer") } + +func TestFormatHardening(t *testing.T) { + t.Run("marshal error", func(t *testing.T) { + cycle := map[string]any{} + cycle["self"] = cycle + for _, v := range []any{math.NaN(), make(chan int), cycle, json.RawMessage(`invalid`)} { + f := New().Type("json") + record := unmarshalRecord(t, f.Format(FlagStructuredJSON, testStamp, 0, "", []any{"msg", map[string]any{"bad": v}})) + if _, ok := record["fields"].(map[string]any)["_marshal_error"].(string); !ok { + t.Fatalf("missing marshal error: %v", record) + } + } + }) + t.Run("nested sanitization", func(t *testing.T) { + f := New(sanitizer.New().Policy(sanitizer.PolicyTxt)).Type("json") + record := unmarshalRecord(t, f.Format(FlagStructuredJSON, testStamp, 0, "", []any{"msg", map[string]any{"k\n": []any{map[string]any{"v": "\x1b[31m"}, 42, true}}})) + nested := record["fields"].(map[string]any)["k<0a>"].([]any) + eq(t, nested[0].(map[string]any)["v"], any("<1b>[31m"), "nested content policy") + eq(t, nested[1], any(float64(42)), "number preserved") + }) + t.Run("explicit suppression wins", func(t *testing.T) { + f := New().Type("json") + record := unmarshalRecord(t, f.FormatWithOptions("json", FlagDefault|FlagNoLevel|FlagNoTimestamp, testStamp, 0, "", nil)) + eq(t, len(record), 0, "suppressed metadata") + }) + t.Run("bad stringer is printable", func(t *testing.T) { + for _, flags := range []int64{0, FlagRaw} { + out := New().Format(flags, testStamp, 0, "", []any{panickingStringer{}}) + if !strings.Contains(string(out), "PANIC") { + t.Fatalf("missing fmt panic marker: %q", out) + } + } + }) +} + +func TestCyclicComplexValues(t *testing.T) { + cycle := map[string]any{} + cycle["self"] = cycle + slice := make([]any, 1) + slice[0] = slice + for _, value := range []any{cycle, slice} { + for _, format := range []string{"txt", "json", "raw"} { + for _, flags := range []int64{0, FlagRaw} { + output := New().Type(format).Format(flags, testStamp, 0, "", []any{value}) + if !strings.Contains(string(output), "cyclic or oversized value") { + t.Fatalf("%s: missing cycle marker: %q", format, output) + } + } + } + } +} diff --git a/go.mod b/go.mod index 5f3e831..97b30f2 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/lixenwraith/log -go 1.26.0 +go 1.27.1 diff --git a/heartbeat.go b/heartbeat.go index 0aa6c7a..2e57cd9 100644 --- a/heartbeat.go +++ b/heartbeat.go @@ -135,6 +135,7 @@ func (l *Logger) writeHeartbeatRecord(level int64, args []any) { } record := logRecord{ + heartbeat: true, Flags: FlagDefault | FlagKV, TimeStamp: time.Now(), Level: level, diff --git a/integration_test.go b/integration_test.go index b08ce23..3e5e2d9 100644 --- a/integration_test.go +++ b/integration_test.go @@ -96,7 +96,6 @@ func TestConcurrentOperations(t *testing.T) { for i := range 3 { // Non-fatal only: Fatal outside the test goroutine is undefined behavior noErr(t, logger.ApplyConfigString(fmt.Sprintf("trace_depth=%d", i)), "ApplyConfigString") - time.Sleep(50 * time.Millisecond) } }() @@ -104,9 +103,8 @@ func TestConcurrentOperations(t *testing.T) { go func() { defer wg.Done() for range 5 { - // Timeout must exceed worst-case contention on flushMutex under load + // Allow the fixed workload to complete under race instrumentation noErr(t, logger.Flush(2*time.Second), "concurrent Flush") - time.Sleep(30 * time.Millisecond) } }() diff --git a/lifecycle_test.go b/lifecycle_test.go index 7bf7c03..a3dc424 100644 --- a/lifecycle_test.go +++ b/lifecycle_test.go @@ -123,16 +123,6 @@ func TestShutdownEdgeCases(t *testing.T) { noErr(t, logger.Shutdown(), "second Shutdown") }) - t.Run("timeout", func(t *testing.T) { - logger, _ := newTestLogger(t) - for i := range 200 { - logger.Info("flood", i) - } - // Stop may time out; terminal state transitions are unconditional - _ = logger.Shutdown(time.Millisecond) - isTrue(t, logger.state.ShutdownCalled.Load(), "ShutdownCalled") - isFalse(t, logger.state.IsInitialized.Load(), "IsInitialized") - }) } // TestFlush covers the success path and both failure modes. @@ -150,7 +140,7 @@ func TestFlush(t *testing.T) { t.Run("timeout", func(t *testing.T) { logger, _ := newTestLogger(t) - errContains(t, logger.Flush(time.Nanosecond), "timeout", "Flush") + errContains(t, logger.Flush(0), "timeout", "Flush") }) t.Run("on stopped logger", func(t *testing.T) { @@ -165,4 +155,3 @@ func TestFlush(t *testing.T) { errContains(t, logger.Flush(time.Second), "not initialized", "Flush") }) } - diff --git a/logger.go b/logger.go index 195b872..b0f8d1c 100644 --- a/logger.go +++ b/logger.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "maps" "math" "os" "sync" @@ -22,7 +23,9 @@ type Logger struct { spawner atomic.Pointer[func(func())] errHandler atomic.Pointer[func(string)] state State - initMu sync.Mutex + initMu sync.Mutex // serializes lifecycle and configuration changes + sendMu sync.RWMutex // joins enqueues before detaching a processor + processorErr error // written by processor, read only after ProcDone closes } // levelOff closes the emit gate without touching configuration @@ -42,14 +45,6 @@ func NewLogger() *Logger { l.state.Flags.Store(flagsFromConfig(defaultCfg)) l.state.TraceDepth.Store(defaultCfg.TraceDepth) - // // Initialize default formatter to prevent nil access - // defaultFormatter := formatter.New(sanitizer.New()). - // Type(defaultCfg.Format). - // TimestampFormat(defaultCfg.TimestampFormat). - // ShowLevel(defaultCfg.ShowLevel). - // ShowTimestamp(defaultCfg.ShowTimestamp) - // l.formatter.Store(defaultFormatter) - // Initialize the state l.state.IsInitialized.Store(false) l.state.LoggerDisabled.Store(false) @@ -69,14 +64,6 @@ func NewLogger() *Logger { // Typed nil: a non-blocking send on a nil channel always takes default l.state.ActiveLogChannel.Store((chan logRecord)(nil)) - l.state.flushRequestChan = make(chan chan struct{}, 1) - - // // Create a closed channel initially to prevent nil pointer issues - // initialChan := make(chan logRecord) - // close(initialChan) - // l.state.ActiveLogChannel.Store(initialChan) - // - // l.state.flushRequestChan = make(chan chan struct{}, 1) return l } @@ -88,6 +75,7 @@ func (l *Logger) ApplyConfig(cfg *Config) error { return fmt.Errorf("log: configuration cannot be nil") } + cfg = cfg.Clone() if err := cfg.Validate(); err != nil { return fmt.Errorf("log: invalid configuration: %w", err) } @@ -101,6 +89,8 @@ func (l *Logger) ApplyConfig(cfg *Config) error { // ApplyConfigString applies string key-value overrides to the logger's current configuration // Each override should be in the format "key=value" func (l *Logger) ApplyConfigString(overrides ...string) error { + l.initMu.Lock() + defer l.initMu.Unlock() cfg := l.getConfig().Clone() var errors []error @@ -121,7 +111,10 @@ func (l *Logger) ApplyConfigString(overrides ...string) error { return combineConfigErrors(errors) } - return l.ApplyConfig(cfg) + if err := cfg.Validate(); err != nil { + return fmtErrorf("invalid configuration: %w", err) + } + return l.applyConfig(cfg) } // GetConfig returns a copy of current configuration @@ -137,280 +130,239 @@ func (l *Logger) getConfig() *Config { // applyConfig is the internal implementation for applying configuration, assuming initMu is held func (l *Logger) applyConfig(cfg *Config) error { oldCfg := l.getConfig() - l.currentConfig.Store(cfg) + wasStarted := l.state.Started.Load() + needsRestart := wasStarted && configRequiresRestart(oldCfg, cfg) - // Shared formatter and sanitizer constructor with SetContextKeys - l.rebuildFormatter(cfg) + // A timed-out processor still owns its file and formatter. Do not replace + // either resource until it has actually exited. + if !wasStarted && !l.state.ProcessorExited.Load() { + return fmtErrorf("previous processor is still stopping") + } - // Emit fast-path mirrors - l.state.Flags.Store(flagsFromConfig(cfg)) - l.state.TraceDepth.Store(cfg.TraceDepth) - - // Ensure log directory exists if file output is enabled + currentFile, _ := l.state.CurrentFile.Load().(*os.File) + needsNewFile := cfg.EnableFile && (!l.state.IsInitialized.Load() || !oldCfg.EnableFile || + !wasStarted && currentFile == nil || l.state.LoggerDisabled.Load() || + oldCfg.Directory != cfg.Directory || oldCfg.Name != cfg.Name || oldCfg.Extension != cfg.Extension) + var newFile *os.File if cfg.EnableFile { if err := os.MkdirAll(cfg.Directory, 0755); err != nil { - l.state.LoggerDisabled.Store(true) - l.currentConfig.Store(oldCfg) // Rollback - l.refreshLevelGate() return fmtErrorf("failed to create log directory '%s': %w", cfg.Directory, err) } + if needsNewFile { + var err error + newFile, err = openLogFile(cfg) + if err != nil { + return fmtErrorf("failed to create log file: %w", err) + } + } } - // Get current state - wasInitialized := l.state.IsInitialized.Load() - wasStarted := l.state.Started.Load() - - // Determine if restart is needed - needsRestart := wasStarted && wasInitialized && configRequiresRestart(oldCfg, cfg) - - // Stop processor if restart needed - if needsRestart { - if err := l.Stop(); err != nil { - l.currentConfig.Store(oldCfg) // Rollback + // Drain with the old configuration and file before publishing the new one. + if needsRestart || wasStarted && needsNewFile { + needsRestart = true + if err := l.stopLocked(); err != nil { + if newFile != nil { + _ = newFile.Close() + } return fmtErrorf("failed to stop processor for restart: %w", err) } } - // Get current file handle - currentFilePtr := l.state.CurrentFile.Load() - var currentFile *os.File - if currentFilePtr != nil { - currentFile, _ = currentFilePtr.(*os.File) - } - - // Determine if we need a new file - needsNewFile := !wasInitialized || currentFile == nil || - oldCfg.Directory != cfg.Directory || - oldCfg.Name != cfg.Name || - oldCfg.Extension != cfg.Extension - - // Handle file state transitions - if !cfg.EnableFile { - // When disabling file output, close the current file + // Draining may rotate the old file; close the handle the processor left behind. + currentFile, _ = l.state.CurrentFile.Load().(*os.File) + if !cfg.EnableFile || newFile != nil { if currentFile != nil { - // Sync and close the file - _ = currentFile.Sync() if err := currentFile.Close(); err != nil { - l.internalLog("warning - failed to close log file during disable: %v\n", err) + l.internalLog("failed to close old log file: %v\n", err) } } - l.state.CurrentFile.Store((*os.File)(nil)) + l.state.CurrentFile.Store(newFile) l.state.CurrentSize.Store(0) - } else if needsNewFile { - // When enabling file output or initializing, create new file - logFile, err := l.createNewLogFile() - if err != nil { - l.state.LoggerDisabled.Store(true) - l.currentConfig.Store(oldCfg) // Rollback - return fmtErrorf("failed to create log file: %w", err) - } - - // Close old file if transitioning from one file to another - if currentFile != nil && currentFile != logFile { - _ = currentFile.Sync() - if err := currentFile.Close(); err != nil { - l.internalLog("warning - failed to close old log file: %v\n", err) + if newFile != nil { + if info, err := newFile.Stat(); err == nil { + l.state.CurrentSize.Store(info.Size()) } } - - l.state.CurrentFile.Store(logFile) - l.state.CurrentSize.Store(0) - if fi, errStat := logFile.Stat(); errStat == nil { - l.state.CurrentSize.Store(fi.Size()) - } } - // Setup console writer based on config + var writer io.Writer = io.Discard if cfg.EnableConsole { - var writer io.Writer + writer = os.Stdout if cfg.ConsoleTarget == "stderr" { writer = os.Stderr - } else { - writer = os.Stdout } - l.state.StdoutWriter.Store(&sink{w: writer}) - } else { - l.state.StdoutWriter.Store(&sink{w: io.Discard}) } - - // Mark as initialized + l.state.StdoutWriter.Store(&sink{w: writer}) + l.currentConfig.Store(cfg) + l.rebuildFormatter(cfg) + l.state.Flags.Store(flagsFromConfig(cfg)) + l.state.TraceDepth.Store(cfg.TraceDepth) l.state.IsInitialized.Store(true) l.state.ShutdownCalled.Store(false) - l.state.DiskFullLogged.Store(false) + // A running processor may have just reported an output failure. A hot + // configuration update must not revive its invalid file handle. + if !wasStarted || needsRestart { + l.state.LoggerDisabled.Store(false) + l.state.DiskFullLogged.Store(false) + } l.state.DiskStatusOK.Store(true) l.refreshLevelGate() - - // Restart processor if it was running and needs restart if needsRestart { - return l.Start() + return l.startLocked() } - return nil } -// Start begins log processing. Safe to call multiple times -// Returns error if logger is not initialized +// Start begins log processing. Repeated calls while running are no-ops. +// ApplyConfig must succeed first. A processor still stopping cannot be restarted. func (l *Logger) Start() error { - if !l.state.IsInitialized.Load() { + l.initMu.Lock() + defer l.initMu.Unlock() + return l.startLocked() +} + +func (l *Logger) startLocked() error { + if !l.state.IsInitialized.Load() || l.state.ShutdownCalled.Load() { return fmtErrorf("logger not initialized, call ApplyConfig first") } - - // Check if processor didn't exit cleanly last time - if l.state.Started.Load() && !l.state.ProcessorExited.Load() { - // Force stop to clean up - l.internalLog("warning - processor still running from previous start, forcing stop\n") - if err := l.Stop(); err != nil { - return fmtErrorf("failed to stop hung processor: %w", err) + if l.state.Started.Load() { + if l.state.ProcessorExited.Load() { + return fmtErrorf("processor exited unexpectedly; call Stop before restarting") } + return nil + } + if !l.state.ProcessorExited.Load() { + return fmtErrorf("previous processor is still stopping") } - // Only start if not already started - if l.state.Started.CompareAndSwap(false, true) { - cfg := l.getConfig() - - // Create log channels - ch := make(chan logRecord, cfg.BufferSize) - stop := make(chan struct{}) - done := make(chan struct{}) - - l.state.ActiveLogChannel.Store(ch) - l.state.ProcStop.Store(stop) - l.state.ProcDone.Store(done) - l.state.ProcessorExited.Store(false) - - // Start processor - l.spawn(func() { l.processLogs(ch, stop, done) }) - } - + ch := make(chan logRecord, l.getConfig().BufferSize) + stop, done := make(chan struct{}), make(chan struct{}) + flush := make(chan flushRequest, 1) + l.processorErr = nil + l.sendMu.Lock() + l.state.ActiveLogChannel.Store(ch) + l.state.ProcStop.Store(stop) + l.state.ProcDone.Store(done) + l.state.flushRequestChan = flush + l.state.ProcessorExited.Store(false) + l.state.Started.Store(true) l.refreshLevelGate() + l.sendMu.Unlock() + l.spawn(func() { l.processLogs(ch, stop, done, flush) }) return nil } -// Stop halts log processing. Can be restarted with Start() -// The record channel is never closed: producers are detached to a nil channel -// first, so a concurrent send falls through to the drop counter instead of -// racing a close. +// Stop detaches producers, drains accepted records, and joins the processor. +// After a timeout, Stop can be retried; Start and resource replacement remain +// blocked until that processor exits. The default timeout is 2x flush interval. func (l *Logger) Stop(timeout ...time.Duration) error { - if !l.state.Started.CompareAndSwap(true, false) { - return nil // Already stopped - } - l.refreshLevelGate() + l.initMu.Lock() + defer l.initMu.Unlock() + return l.stopLocked(timeout...) +} - // Calculate effective timeout - var effectiveTimeout time.Duration +func (l *Logger) stopLocked(timeout ...time.Duration) error { + l.sendMu.Lock() + if l.state.Started.Swap(false) { + l.refreshLevelGate() + l.state.ActiveLogChannel.Store((chan logRecord)(nil)) + close(l.state.ProcStop.Load().(chan struct{})) + } + l.sendMu.Unlock() + + done, _ := l.state.ProcDone.Load().(chan struct{}) + if done == nil { + return nil + } + effectiveTimeout := 2 * time.Duration(l.getConfig().FlushIntervalMs) * time.Millisecond if len(timeout) > 0 { effectiveTimeout = timeout[0] - } else { - effectiveTimeout = 2 * time.Duration(l.getConfig().FlushIntervalMs) * time.Millisecond - } - if effectiveTimeout < minWaitTime { - effectiveTimeout = minWaitTime - } - - // 1. Detach producers - l.state.ActiveLogChannel.Store((chan logRecord)(nil)) - - // 2. Signal the processor to drain and exit - if s, ok := l.state.ProcStop.Load().(chan struct{}); ok && s != nil { - close(s) - } - - // 3. Join - d, _ := l.state.ProcDone.Load().(chan struct{}) - if d == nil { - return nil } + effectiveTimeout = max(effectiveTimeout, minWaitTime) + // Prefer an already completed processor even with a tiny timeout. select { - case <-d: - return nil - case <-time.After(effectiveTimeout): + case <-done: + return l.processorErr + default: + } + timer := time.NewTimer(effectiveTimeout) + defer timer.Stop() + select { + case <-done: + return l.processorErr + case <-timer.C: return fmtErrorf("processor did not exit within timeout (%v)", effectiveTimeout) } } -// Shutdown gracefully closes the logger, attempting to flush pending records -// If no timeout is provided, uses a default of 2x flush interval +// Shutdown stops the logger and closes its file after the processor exits. +// If joining times out, resources remain owned by the processor; retry Shutdown +// to finish cleanup. A completed shutdown can be reset by ApplyConfig. func (l *Logger) Shutdown(timeout ...time.Duration) error { - if !l.state.ShutdownCalled.CompareAndSwap(false, true) { + l.initMu.Lock() + defer l.initMu.Unlock() + if !l.state.IsInitialized.Load() { return nil } - + l.state.ShutdownCalled.Store(true) l.state.LoggerDisabled.Store(true) l.refreshLevelGate() - - if !l.state.IsInitialized.Load() { - l.state.ShutdownCalled.Store(false) - l.state.LoggerDisabled.Store(false) - l.state.ProcessorExited.Store(true) - l.refreshLevelGate() - return nil + err := l.stopLocked(timeout...) + if !l.state.ProcessorExited.Load() { + return err } - - var stopErr error - if l.state.Started.Load() { - stopErr = l.Stop(timeout...) - } - l.state.IsInitialized.Store(false) - - var finalErr error - cfPtr := l.state.CurrentFile.Load() - if cfPtr != nil { - if currentLogFile, ok := cfPtr.(*os.File); ok && currentLogFile != nil { - if err := currentLogFile.Sync(); err != nil { - syncErr := fmtErrorf("failed to sync log file '%s' during shutdown: %w", currentLogFile.Name(), err) - finalErr = errors.Join(finalErr, syncErr) - } - if err := currentLogFile.Close(); err != nil { - closeErr := fmtErrorf("failed to close log file '%s' during shutdown: %w", currentLogFile.Name(), err) - finalErr = errors.Join(finalErr, closeErr) - } - l.state.CurrentFile.Store((*os.File)(nil)) - } + if file, _ := l.state.CurrentFile.Load().(*os.File); file != nil { + err = errors.Join(err, file.Sync(), file.Close()) + l.state.CurrentFile.Store((*os.File)(nil)) } - - if stopErr != nil { - finalErr = errors.Join(finalErr, stopErr) - } - - return finalErr + return err } -// Flush explicitly triggers a sync of the current log file buffer to disk and waits for completion or timeout +// Flush processes records queued before the request and syncs the current file. +// The timeout covers queueing and confirmation. Concurrent later records need +// not be drained, so sustained producers cannot starve the barrier. func (l *Logger) Flush(timeout time.Duration) error { - l.state.flushMutex.Lock() - defer l.state.flushMutex.Unlock() - - // State checks + l.sendMu.RLock() if !l.state.IsInitialized.Load() || l.state.ShutdownCalled.Load() { + l.sendMu.RUnlock() return fmtErrorf("logger not initialized or already shut down") } if !l.state.Started.Load() { + l.sendMu.RUnlock() return fmtErrorf("logger not started") } - - // Create a channel to wait for confirmation from the processor - confirmChan := make(chan struct{}) - - // Send the request with the confirmation channel - select { - case l.state.flushRequestChan <- confirmChan: - // Request sent - case <-time.After(minWaitTime): // Short timeout to prevent blocking if processor is stuck - return fmtErrorf("failed to send flush request to processor (possible deadlock or high load)") + requests := l.state.flushRequestChan + done := l.state.ProcDone.Load().(chan struct{}) + l.sendMu.RUnlock() + if timeout <= 0 { + return fmtErrorf("timeout waiting for flush confirmation (%v)", timeout) } + timer := time.NewTimer(timeout) + defer timer.Stop() + request := flushRequest{done: make(chan error, 1)} select { - case <-confirmChan: - return nil - case <-time.After(timeout): + case requests <- request: + case <-done: + return fmtErrorf("processor stopped during flush") + case <-timer.C: + return fmtErrorf("timeout sending flush request (%v)", timeout) + } + select { + case err := <-request.done: + return err + case <-done: + return fmtErrorf("processor stopped during flush") + case <-timer.C: return fmtErrorf("timeout waiting for flush confirmation (%v)", timeout) } } // SetSpawn installs the goroutine launcher used by Start. Hosts that own // panic recovery and terminal teardown pass their own launcher here. -// Call before Start; a nil fn restores the default. +// Call before Start; fn must launch asynchronously and return promptly. +// A nil fn restores the default. func (l *Logger) SetSpawn(fn func(func())) { if fn == nil { l.spawner.Store(nil) @@ -420,7 +372,8 @@ func (l *Logger) SetSpawn(fn func(func())) { } // SetErrorHandler routes internal diagnostics to fn instead of stderr. -// Required for TUI hosts, where stderr writes corrupt the display. +// Required for TUI hosts, where stderr writes corrupt the display. The callback +// must return promptly and must not call lifecycle/configuration methods or Flush. func (l *Logger) SetErrorHandler(fn func(string)) { if fn == nil { l.errHandler.Store(nil) @@ -450,15 +403,16 @@ func (l *Logger) SetLevel(level int64) { cfg := l.getConfig().Clone() cfg.Level = level l.currentConfig.Store(cfg) - l.initMu.Unlock() l.refreshLevelGate() + l.initMu.Unlock() } // Enabled reports whether a record at level would be emitted. Single atomic // load: the intended guard for hot call sites, where argument slices are // built before the call and would otherwise escape to the heap. func (l *Logger) Enabled(level int64) bool { - return level >= l.state.Level.Load() + threshold := l.state.Level.Load() + return threshold != levelOff && level >= threshold } // Flags returns the default record flags derived from display config @@ -468,7 +422,7 @@ func (l *Logger) Flags() int64 { // LogContext emits a record with caller-supplied context and explicit flags func (l *Logger) LogContext(ctx Context, flags, level, depth int64, args ...any) { - if level < l.state.Level.Load() { + if !l.Enabled(level) { return } l.emit(ctx, flags, level, depth, args) @@ -591,7 +545,10 @@ func (l *Logger) LogTrace(depth int, args ...any) { // LogStructured logs a message with structured fields as proper JSON func (l *Logger) LogStructured(level int64, message string, fields map[string]any) { - l.LogContext(Context{}, l.getFlags()|FlagStructuredJSON, level, 0, message, fields) + if !l.Enabled(level) { + return + } + l.LogContext(Context{}, l.getFlags()|FlagStructuredJSON, level, 0, message, maps.Clone(fields)) } // Write outputs raw, unformatted data ignoring configured format and sanitization diff --git a/processor.go b/processor.go index 58b3bb6..d622243 100644 --- a/processor.go +++ b/processor.go @@ -1,6 +1,7 @@ package log import ( + "io" "os" "time" @@ -10,7 +11,7 @@ import ( // processLogs is the main log processing loop running in a separate goroutine. // Exits on stop, draining buffered records first. No panic recovery: a fault // here is fatal by design and is surfaced by the host's spawner. -func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done chan<- struct{}) { +func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done chan<- struct{}, flush <-chan flushRequest) { defer func() { l.state.ProcessorExited.Store(true) close(done) @@ -51,7 +52,7 @@ func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done cha select { case <-stop: l.drain(ch) - l.performSync() + l.processorErr = l.performSync() return case record := <-ch: @@ -84,9 +85,9 @@ func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done cha lastCheckTime = time.Now() } - case confirmChan := <-l.state.flushRequestChan: + case request := <-flush: // Barrier: drain queued records before sync - l.handleFlushRequest(ch, confirmChan) + l.handleFlushRequest(ch, request) case <-timers.retentionChan: l.handleRetentionCheck() @@ -99,22 +100,16 @@ func (l *Logger) processLogs(ch <-chan logRecord, stop <-chan struct{}, done cha // drain processes every buffered record without blocking func (l *Logger) drain(ch <-chan logRecord) { - for { - select { - case record := <-ch: - l.processLogRecord(record) - default: - return - } + // A snapshot bounds the flush work even while producers keep the queue full. + // Stop has already joined producers, so this also drains its entire queue. + for range len(ch) { + l.processLogRecord(<-ch) } } -// handleFlushRequest drains pending records, then syncs. Gives Flush barrier -// semantics: records enqueued before the Flush call are processed first. -func (l *Logger) handleFlushRequest(ch <-chan logRecord, confirmChan chan struct{}) { +func (l *Logger) handleFlushRequest(ch <-chan logRecord, request flushRequest) { l.drain(ch) - l.performSync() - close(confirmChan) + request.done <- l.performSync() } // processLogRecord handles individual log records and returns bytes written @@ -123,8 +118,7 @@ func (l *Logger) processLogRecord(record logRecord) int64 { enableFile := c.EnableFile if enableFile && !l.state.DiskStatusOK.Load() { // Simple increment of both counters - l.state.DroppedLogs.Add(1) - l.state.TotalDroppedLogs.Add(1) + l.handleFailedSend() return 0 } @@ -147,31 +141,33 @@ func (l *Logger) processLogRecord(record logRecord) int64 { ) formattedDataLen := int64(len(formattedData)) - // Write to console if enabled - enableConsole := c.EnableConsole - if enableConsole { - if s := l.state.StdoutWriter.Load(); s != nil { - if sinkWrapper, ok := s.(*sink); ok && sinkWrapper != nil { - // Handle split mode - if c.ConsoleTarget == "split" { - if record.Level >= LevelWarn { - // Write WARN and ERROR to stderr - _, _ = os.Stderr.Write(formattedData) - } else { - // Write INFO and DEBUG to stdout - _, _ = sinkWrapper.w.Write(formattedData) - } - } else { - // Write to the configured target (stdout or stderr) - _, _ = sinkWrapper.w.Write(formattedData) - } + var consoleErr error + if c.EnableConsole { + if target, _ := l.state.StdoutWriter.Load().(*sink); target != nil { + writer := target.w + if c.ConsoleTarget == "split" && record.Level >= LevelWarn { + writer = os.Stderr + } + n, err := writer.Write(formattedData) + if err == nil && n != len(formattedData) { + err = io.ErrShortWrite + } + consoleErr = err + if err != nil { + l.internalLog("failed to write to console: %v\n", err) } } } // Skip file operations if file output is disabled if !enableFile { - l.state.TotalLogsProcessed.Add(1) + if consoleErr != nil { + l.handleFailedSend() + return 0 + } + if !record.heartbeat { + l.state.TotalLogsProcessed.Add(1) + } return formattedDataLen // Return data length for adaptive interval calculations } @@ -184,7 +180,7 @@ func (l *Logger) processLogRecord(record logRecord) int64 { if err := l.rotateLogFile(); err != nil { l.internalLog("failed to rotate log file: %v\n", err) // Account for the dropped log that triggered the failed rotation - l.state.DroppedLogs.Add(1) + l.handleFailedSend() return 0 } } @@ -195,16 +191,21 @@ func (l *Logger) processLogRecord(record logRecord) int64 { n, err := currentLogFile.Write(formattedData) if err != nil { l.internalLog("failed to write to log file: %v\n", err) - l.state.DroppedLogs.Add(1) + l.handleFailedSend() + l.state.CurrentSize.Add(int64(n)) l.performDiskCheck(true) return 0 } else { l.state.CurrentSize.Add(int64(n)) - l.state.TotalLogsProcessed.Add(1) + if consoleErr != nil { + l.handleFailedSend() + } else if !record.heartbeat { + l.state.TotalLogsProcessed.Add(1) + } return int64(n) } } else { - l.state.DroppedLogs.Add(1) + l.handleFailedSend() return 0 } } @@ -214,7 +215,9 @@ func (l *Logger) handleFlushTick() { c := l.getConfig() enableSync := c.EnablePeriodicSync if enableSync { - l.performSync() + if err := l.performSync(); err != nil { + l.internalLog("failed to sync log file: %v\n", err) + } } } @@ -224,7 +227,7 @@ func (l *Logger) handleRetentionCheck() { retentionPeriodHrs := c.RetentionPeriodHrs retentionDur := time.Duration(retentionPeriodHrs * float64(time.Hour)) - if retentionDur > 0 { + if c.EnableFile && retentionDur > 0 { etPtr := l.state.EarliestFileTime.Load() if earliest, ok := etPtr.(time.Time); ok && !earliest.IsZero() { if time.Since(earliest) > retentionDur { @@ -253,13 +256,20 @@ func (l *Logger) adjustDiskCheckInterval(timers *TimerSet, lastCheckTime time.Ti logsPerSecond := float64(logsSinceLastCheck) / elapsed.Seconds() targetLogsPerSecond := float64(100) // Baseline - diskCheckIntervalMs := c.DiskCheckIntervalMs - currentDiskCheckInterval := time.Duration(diskCheckIntervalMs) * time.Millisecond + currentDiskCheckInterval := timers.diskInterval + if currentDiskCheckInterval == 0 { + currentDiskCheckInterval = time.Duration(c.DiskCheckIntervalMs) * time.Millisecond + } // Calculate the new interval var newInterval time.Duration if logsPerSecond < targetLogsPerSecond/2 { // Load low -> increase interval - newInterval = time.Duration(float64(currentDiskCheckInterval) * adaptiveIntervalFactor) + scaled := float64(currentDiskCheckInterval) * adaptiveIntervalFactor + if scaled >= float64(time.Duration(c.MaxCheckIntervalMs)*time.Millisecond) { + newInterval = time.Duration(c.MaxCheckIntervalMs) * time.Millisecond + } else { + newInterval = time.Duration(scaled) + } } else if logsPerSecond > targetLogsPerSecond*2 { // Load high -> decrease interval newInterval = time.Duration(float64(currentDiskCheckInterval) * adaptiveSpeedUpFactor) } else { @@ -280,5 +290,6 @@ func (l *Logger) adjustDiskCheckInterval(timers *TimerSet, lastCheckTime time.Ti newInterval = maxCheckInterval } + timers.diskInterval = newInterval timers.diskCheckTicker.Reset(newInterval) } diff --git a/processor_test.go b/processor_test.go index 0909c2b..17e6478 100644 --- a/processor_test.go +++ b/processor_test.go @@ -1,40 +1,14 @@ package log import ( + "bytes" "encoding/json" "strings" "testing" + "testing/synctest" "time" ) -// // procRecords parses PROC heartbeat records out of json-formatted content. -// // Heartbeat arguments are emitted as a flat key/value array. -// func procRecords(tb testing.TB, content string) []map[string]any { -// tb.Helper() -// var out []map[string]any -// for _, line := range strings.Split(content, "\n") { -// if !strings.Contains(line, `"level":"PROC"`) { -// continue -// } -// var entry map[string]any -// if json.Unmarshal([]byte(line), &entry) != nil { -// continue -// } -// fields, ok := entry["fields"].([]any) -// if !ok { -// continue -// } -// rec := make(map[string]any, len(fields)/2) -// for i := 0; i+1 < len(fields); i += 2 { -// if key, ok := fields[i].(string); ok { -// rec[key] = fields[i+1] -// } -// } -// out = append(out, rec) -// } -// return out -// } - // procRecords parses PROC heartbeat records out of json-formatted content. // Heartbeat arguments are emitted as a keyed object (FlagKV); the flat // key/value array is still accepted for records written without the flag. @@ -100,178 +74,101 @@ func TestLoggerHeartbeat(t *testing.T) { // TestHeartbeatDisabled verifies level 0 emits nothing. func TestHeartbeatDisabled(t *testing.T) { - logger, tmpDir := newTestLogger(t) - - cfg := logger.GetConfig() - cfg.Format = "json" - cfg.HeartbeatLevel = 0 - cfg.HeartbeatIntervalS = 1 - mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig") - - logger.Info("marker") - mustNoErr(t, logger.Flush(time.Second), "Flush") - time.Sleep(1200 * time.Millisecond) // span at least one interval - - content := readLog(t, tmpDir) - contains(t, content, "marker", "regular record") - notContains(t, content, `"level":"PROC"`, "proc heartbeat") - equal(t, logger.state.HeartbeatSequence.Load(), uint64(0), "HeartbeatSequence") + synctest.Test(t, func(t *testing.T) { + l, out := memoryHeartbeatLogger(t, 0) + mustNoErr(t, l.Start(), "start") + synctest.Sleep(2 * time.Second) + l.Info("marker") + mustNoErr(t, l.Flush(time.Second), "flush") + contains(t, out.String(), "marker", "ordinary record") + notContains(t, out.String(), `"level":"PROC"`, "disabled heartbeat") + equal(t, l.state.HeartbeatSequence.Load(), uint64(0), "no heartbeat sequence") + mustNoErr(t, l.Shutdown(time.Second), "shutdown") + }) } -// TestDroppedLogs verifies buffer overflow is counted and reported by the heartbeat. +func memoryHeartbeatLogger(t *testing.T, level int64) (*Logger, *bytes.Buffer) { + t.Helper() + l := NewLogger() + c := DefaultConfig() + c.Format = "json" + c.HeartbeatLevel = level + c.HeartbeatIntervalS = 1 + c.BufferSize = 1 + mustNoErr(t, l.ApplyConfig(c), "apply") + out := new(bytes.Buffer) + l.state.StdoutWriter.Store(&sink{w: out}) + return l, out +} + +// The processor cannot start until the flood is complete, making overflow exact. func TestDroppedLogs(t *testing.T) { - logger := NewLogger() - - cfg := DefaultConfig() - cfg.Directory = t.TempDir() - cfg.EnableConsole = false - cfg.EnableFile = true - cfg.Format = "json" - cfg.BufferSize = 1 // guarantees drops under flood - cfg.FlushIntervalMs = 10 - cfg.HeartbeatLevel = 1 - cfg.HeartbeatIntervalS = 1 - - mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig") - mustNoErr(t, logger.Start(), "Start") - t.Cleanup(func() { _ = logger.Shutdown() }) - - for i := range 100 { - logger.Info("flood", i) - } - - dropped := logger.state.TotalDroppedLogs.Load() - if dropped == 0 { - t.Fatal("flood produced no drops") - } - - // The interval counter is reported only when non-zero, so wait for the - // tick-driven heartbeat that follows the flood - mustEventually(t, 5*time.Second, "heartbeat reporting interval drops", func() bool { - for _, rec := range procRecords(t, readLog(t, cfg.Directory)) { - if _, ok := rec["dropped_since_last"]; ok { - return true - } + synctest.Test(t, func(t *testing.T) { + l, out := memoryHeartbeatLogger(t, 1) + release := make(chan struct{}) + l.SetSpawn(func(fn func()) { go func() { <-release; fn() }() }) + mustNoErr(t, l.Start(), "start") + for i := range 100 { + l.Info("flood", i) } - return false + equal(t, l.state.TotalDroppedLogs.Load(), uint64(99), "exact overflow") + close(release) + synctest.Wait() + // The initial heartbeat encounters the full queue and adds one more drop. + synctest.Sleep(time.Second) + records := procRecords(t, out.String()) + mustEqual(t, len(records), 1, "tick heartbeat") + equal(t, numField(records[0], "total_dropped_logs"), float64(100), "includes dropped initial heartbeat") + equal(t, numField(records[0], "processed_logs"), float64(1), "excludes heartbeats") + mustNoErr(t, l.Shutdown(time.Second), "shutdown") }) - - records := procRecords(t, readLog(t, cfg.Directory)) - last := records[len(records)-1] - if got := numField(last, "total_dropped_logs"); got < float64(dropped) { - t.Errorf("total_dropped_logs %v below observed drops %d", got, dropped) - } } -// TestDroppedHeartbeatAccounting verifies a heartbeat discarded by the processor -// during a disk failure is still reflected in the total drop count reported by -// the next successful heartbeat. func TestDroppedHeartbeatAccounting(t *testing.T) { - logger := NewLogger() - - cfg := DefaultConfig() - cfg.Directory = t.TempDir() - cfg.EnableConsole = false - cfg.EnableFile = true - cfg.Format = "json" - cfg.BufferSize = 10 - cfg.HeartbeatLevel = 1 - cfg.HeartbeatIntervalS = 1 - cfg.InternalErrorsToStderr = false // internal logs would add drops - - mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig") - mustNoErr(t, logger.Start(), "Start") - t.Cleanup(func() { _ = logger.Shutdown() }) - - // Drops during the flood are nondeterministic; capture the actual count - for i := range int(cfg.BufferSize) + 50 { - logger.Info("flood", i) - } - floodDrops := logger.state.TotalDroppedLogs.Load() - if floodDrops == 0 { - t.Fatal("flood produced no drops") - } - - // Let the first tick-driven heartbeat consume the interval counter - mustEventually(t, 3*time.Second, "first tick heartbeat", func() bool { - return logger.state.HeartbeatSequence.Load() >= 2 - }) - - // Force the disk-unavailable state; the processor discards every record - diskFull := logger.GetConfig() - diskFull.MinDiskFreeKB = 1 << 40 - mustNoErr(t, logger.ApplyConfig(diskFull), "ApplyConfig disk full") - isFalse(t, logger.performDiskCheck(true), "performDiskCheck under disk full") - isFalse(t, logger.state.DiskStatusOK.Load(), "DiskStatusOK") - - // Hold the failure until a heartbeat has been produced and discarded - seq := logger.state.HeartbeatSequence.Load() - mustEventually(t, 3*time.Second, "heartbeat produced while disk full", func() bool { - return logger.state.HeartbeatSequence.Load() > seq - }) - droppedWithDiskFull := logger.state.TotalDroppedLogs.Load() - if droppedWithDiskFull <= floodDrops { - t.Fatalf("processor did not drop during disk failure: %d", droppedWithDiskFull) - } - - // Restore and wait for a heartbeat that reaches the file - diskOK := logger.GetConfig() - diskOK.MinDiskFreeKB = 0 - mustNoErr(t, logger.ApplyConfig(diskOK), "ApplyConfig disk ok") - isTrue(t, logger.performDiskCheck(true), "performDiskCheck after recovery") - isTrue(t, logger.state.DiskStatusOK.Load(), "DiskStatusOK after recovery") - - seq = logger.state.HeartbeatSequence.Load() - mustEventually(t, 4*time.Second, "heartbeat written after recovery", func() bool { - if logger.state.HeartbeatSequence.Load() <= seq { - return false - } - records := procRecords(t, readLog(t, cfg.Directory)) - if len(records) == 0 { - return false - } - return numField(records[len(records)-1], "sequence") > float64(seq) - }) - - records := procRecords(t, readLog(t, cfg.Directory)) - last := records[len(records)-1] - - // The dropped heartbeat is unrecoverable in the interval counter but must - // remain visible in the monotonic total - if got := numField(last, "total_dropped_logs"); got < float64(droppedWithDiskFull) { - t.Errorf("total_dropped_logs %v does not cover drops observed during failure %d", - got, droppedWithDiskFull) - } - if got := numField(last, "processed_logs"); got == 0 { - t.Error("processed_logs must be non-zero after recovery") - } + l := NewLogger() + c := DefaultConfig() + c.EnableConsole = false + c.EnableFile = true + c.Directory = t.TempDir() + c.Format = "json" + mustNoErr(t, l.ApplyConfig(c), "apply") + defer l.Shutdown() + ch := make(chan logRecord, 1) + l.state.ActiveLogChannel.Store(ch) + l.processLogRecord(logRecord{Args: []any{"application record"}}) + l.state.DiskStatusOK.Store(false) + l.logProcHeartbeat() + l.processLogRecord(<-ch) + equal(t, l.state.TotalDroppedLogs.Load(), uint64(1), "dropped heartbeat") + l.state.DiskStatusOK.Store(true) + l.logProcHeartbeat() + l.processLogRecord(<-ch) + records := procRecords(t, readLog(t, c.Directory)) + mustEqual(t, len(records), 1, "recovery heartbeat") + equal(t, numField(records[0], "total_dropped_logs"), float64(1), "persistent drops") + equal(t, numField(records[0], "dropped_since_last"), float64(1), "interval drops") + equal(t, l.state.TotalLogsProcessed.Load(), uint64(1), "heartbeat excluded from processed count") } -// TestAdaptiveDiskCheck exercises interval adjustment under varying log rates. func TestAdaptiveDiskCheck(t *testing.T) { - logger, _ := newTestLogger(t) - - cfg := logger.GetConfig() - cfg.EnableAdaptiveInterval = true - cfg.DiskCheckIntervalMs = 100 - cfg.MinCheckIntervalMs = 50 - cfg.MaxCheckIntervalMs = 500 - mustNoErr(t, logger.ApplyConfig(cfg), "ApplyConfig") - - // Low rate, then burst: both adjustment branches - for i := range 10 { - logger.Info("adaptive test", i) - time.Sleep(10 * time.Millisecond) - } - for i := range 100 { - logger.Info("burst", i) - } - mustNoErr(t, logger.Flush(2*time.Second), "Flush") - - isTrue(t, logger.state.DiskStatusOK.Load(), "DiskStatusOK") - if logger.state.TotalLogsProcessed.Load() == 0 { - t.Error("no records processed") - } + synctest.Test(t, func(t *testing.T) { + l := NewLogger() + c := DefaultConfig() + c.DiskCheckIntervalMs = 100 + c.MinCheckIntervalMs = 50 + c.MaxCheckIntervalMs = 500 + mustNoErr(t, l.ApplyConfig(c), "apply") + timers := l.setupProcessingTimers() + defer l.stopProcessingTimers(timers) + for range 8 { + l.adjustDiskCheckInterval(timers, time.Now().Add(-time.Second), 0) + } + equal(t, timers.diskInterval, 500*time.Millisecond, "repeated low load reaches maximum") + for range 16 { + l.adjustDiskCheckInterval(timers, time.Now().Add(-time.Second), 1000) + } + equal(t, timers.diskInterval, 50*time.Millisecond, "repeated high load reaches minimum") + }) } // TestFlushBarrier verifies records enqueued before Flush are written before it returns. diff --git a/record.go b/record.go index 93163a4..46bce41 100644 --- a/record.go +++ b/record.go @@ -3,6 +3,7 @@ package log import ( "fmt" "os" + "slices" "strings" "time" ) @@ -18,9 +19,11 @@ func (l *Logger) getFlags() int64 { return l.state.Flags.Load() } -// sendLogRecord queues a record without blocking. The channel is never closed, -// so no recovery is needed: a detached (nil) channel takes the default branch. +// sendLogRecord never waits for queue capacity or I/O. The short read lock +// ensures Stop cannot finish draining while a producer still holds the old queue. func (l *Logger) sendLogRecord(record logRecord) { + l.sendMu.RLock() + defer l.sendMu.RUnlock() if l.state.LoggerDisabled.Load() { l.handleFailedSend() return @@ -54,7 +57,7 @@ func (l *Logger) emit(ctx Context, flags, level, depth int64, args []any) { TimeStamp: time.Now(), Level: level, Trace: trace, - Args: args, + Args: slices.Clone(args), }) } @@ -75,43 +78,3 @@ func (l *Logger) internalLog(format string, args ...any) { } fmt.Fprintf(os.Stderr, format, args...) } - -// // log handles the core logging logic -// func (l *Logger) log(flags int64, level int64, depth int64, args ...any) { -// // State checks -// if !l.state.IsInitialized.Load() { -// return -// } -// -// if !l.state.Started.Load() { -// // Log to internal error channel if configured -// cfg := l.getConfig() -// if cfg.InternalErrorsToStderr { -// l.internalLog("warning - logger not started, dropping log entry\n") -// } -// return -// } -// -// // Discard or proceed based on level -// cfg := l.getConfig() -// if level < cfg.Level { -// return -// } -// -// // Get trace info from runtime -// // Depth filter hard-coded based on call stack of current package design -// var trace string -// if depth > 0 { -// const skipTrace = 3 // log.Info -> log -> getTrace (Adjust if call stack changes) -// trace = getTrace(depth, skipTrace) -// } -// -// record := logRecord{ -// Flags: flags, -// TimeStamp: time.Now(), -// Level: level, -// Trace: trace, -// Args: args, -// } -// l.sendLogRecord(record) -// } diff --git a/sanitizer/fuzz_test.go b/sanitizer/fuzz_test.go new file mode 100644 index 0000000..2143477 --- /dev/null +++ b/sanitizer/fuzz_test.go @@ -0,0 +1,92 @@ +package sanitizer + +import ( + "encoding/json" + "strings" + "testing" + "unicode" + "unicode/utf8" +) + +func FuzzSanitize(f *testing.F) { + for _, s := range []string{"", "plain", "\xff\xfe\n\x80", "<00>\x00", "\u0085\u009b[31m", "世界\u202e", "a'$(id)\\*"} { + f.Add(s, uint8(0), uint64(FilterControl), uint64(TransformHexEncode)) + } + f.Fuzz(func(t *testing.T, input string, policy uint8, filter, transform uint64) { + policies := []PolicyPreset{PolicyRaw, PolicyTxt, PolicyJSON, PolicyShell} + s := New().Policy(policies[int(policy)%len(policies)]).Rule(filter, transform) + want := s.Sanitize(input) + got := s.AppendSanitize([]byte("prefix:"), input) + if string(got) != "prefix:"+want { + t.Fatalf("append/string disagreement: %q != %q", got, want) + } + if want != s.Sanitize(input) { + t.Fatal("non-deterministic sanitizer") + } + if policy%4 == 3 { + for _, r := range New().Policy(PolicyShell).Sanitize(input) { + if unicode.IsControl(r) || unicode.IsSpace(r) || matchesFilter(r, FilterShellSpecial) { + t.Fatalf("shell filter leaked %U", r) + } + } + } + }) +} + +func FuzzSerializerJSON(f *testing.F) { + for _, s := range []string{"", "\xff\xfe\x00", "a\"\\\nb", "\u0085", "世界"} { + f.Add(s, uint8(0)) + } + f.Fuzz(func(t *testing.T, input string, policy uint8) { + policies := []PolicyPreset{PolicyRaw, PolicyTxt, PolicyJSON, PolicyShell} + s := New().Policy(policies[int(policy)%len(policies)]) + out := []byte("prefix:") + NewSerializer("json", s).WriteString(&out, input) + if !strings.HasPrefix(string(out), "prefix:") { + t.Fatal("prefix changed") + } + data := out[len("prefix:"):] + if !utf8.Valid(data) || !json.Valid(data) { + t.Fatalf("invalid JSON string %q", data) + } + var got, want string + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + // encoding/json is an independent oracle for malformed UTF-8 replacement. + oracle, err := json.Marshal(s.Sanitize(input)) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(oracle, &want); err != nil { + t.Fatal(err) + } + if got != want { + t.Fatalf("round trip: %q != %q", got, want) + } + }) +} + +func TestSanitizerMalformedUTF8AndTransforms(t *testing.T) { + for _, input := range []string{"\xff", "\xff\n\xfe", "\n\xff"} { + s := New().Policy(PolicyJSON) + eq(t, string(s.AppendSanitize(nil, input)), s.Sanitize(input), "malformed UTF-8 consistency") + } + eq(t, New().Rule(FilterControl, 0).Sanitize("a\x00b"), "a\x00b", "zero transform preserves") + eq(t, New().Policy(PolicyJSON).Sanitize("\u0085\u009b"), `\u0085\u009b`, "C1 controls escaped") + var out []byte + NewSerializer("unknown", nil).WriteString(&out, "a b") + eq(t, string(out), `"a b"`, "nil sanitizer and unknown format") +} + +func TestSerializerInvalidNumbers(t *testing.T) { + for _, input := range []string{"", "NaN", "+Inf", "-Inf", "01", "1,2", "null", "1\n", "1 "} { + var out []byte + NewSerializer("json", nil).WriteNumber(&out, input) + var got string + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("%q must be represented as string: %q (%v)", input, out, err) + } + eq(t, got, input, "invalid number round trip") + } +} diff --git a/sanitizer/sanitizer.go b/sanitizer/sanitizer.go index 4dcfbdf..2daf86a 100644 --- a/sanitizer/sanitizer.go +++ b/sanitizer/sanitizer.go @@ -9,8 +9,12 @@ package sanitizer import ( "encoding/hex" + "encoding/json" + "encoding/json/jsontext" "fmt" + "reflect" "strconv" + "strings" "unicode" "unicode/utf8" ) @@ -20,7 +24,7 @@ const ( FilterNonPrintable uint64 = 1 << iota // Matches runes not classified as printable by strconv.IsPrint FilterControl // Matches control characters (unicode.IsControl) FilterWhitespace // Matches whitespace characters (unicode.IsSpace) - FilterShellSpecial // Matches common shell metacharacters: '`', '$', ';', '|', '&', '>', '<', '(', ')', '#' + FilterShellSpecial // Matches shell metacharacters including quotes, backslash, globs, and: '`', '$', ';', '|', '&', '>', '<', '(', ')', '#' ) // Transform flags for character transformation @@ -38,7 +42,7 @@ const ( PolicyJSON PolicyPreset = "json" // Policy for sanitizing strings to be embedded in JSON PolicyTxt PolicyPreset = "txt" // Policy for sanitizing text written to log files // PolicyShell strips shell metacharacters, whitespace, and control characters. NOT sufficient for safe shell construction. Pass arguments via exec argv instead. - PolicyShell PolicyPreset = "shell" // Policy for sanitizing arguments passed to shell commands + PolicyShell PolicyPreset = "shell" // Lossy text filter; not shell quoting ) // rule represents a single sanitization rule @@ -62,27 +66,10 @@ var policyRules = map[PolicyPreset][]rule{ {fn: func(r rune) bool { return r == '<' }, transform: TransformHexEncode}, {filter: FilterNonPrintable, transform: TransformHexEncode}, }, - // PolicyTxt: {{filter: FilterNonPrintable, transform: TransformHexEncode}}, PolicyJSON: {{filter: FilterControl, transform: TransformJSONEscape}}, PolicyShell: {{filter: FilterShellSpecial | FilterWhitespace | FilterControl, transform: TransformStrip}}, } -// filterCheckers maps individual filter flags to their check functions -var filterCheckers = map[uint64]func(rune) bool{ - FilterNonPrintable: func(r rune) bool { return !strconv.IsPrint(r) }, - FilterControl: unicode.IsControl, - FilterWhitespace: unicode.IsSpace, - FilterShellSpecial: func(r rune) bool { - switch r { - // CHANGED: D2 — added quotes, backslash, glob, braces, '~', '!' - case '`', '$', ';', '|', '&', '>', '<', '(', ')', '#', - '\'', '"', '\\', '*', '?', '[', ']', '{', '}', '~', '!': - return true - } - return false - }, -} - // Sanitizer provides chainable text sanitization type Sanitizer struct { rules []rule @@ -122,6 +109,7 @@ func (s *Sanitizer) Sanitize(data string) string { if len(s.rules) == 0 { return data } + data = normalizeUTF8(data) i := s.firstMatch(data) if i < 0 { return data @@ -137,7 +125,16 @@ func (s *Sanitizer) AppendSanitize(dst []byte, data string) []byte { if len(s.rules) == 0 { return append(dst, data...) } - return s.appendSanitized(dst, data) + return s.appendSanitized(dst, normalizeUTF8(data)) +} + +// Repair malformed sequences before removing characters: otherwise deleting +// separators can join invalid bytes into a new control rune or metacharacter. +func normalizeUTF8(data string) string { + if utf8.ValidString(data) { + return data + } + return strings.ToValidUTF8(data, "\ufffd") } // firstMatch returns the byte index of the first rune matching any rule, -1 if none @@ -153,26 +150,36 @@ func (s *Sanitizer) firstMatch(data string) int { } func (s *Sanitizer) appendSanitized(dst []byte, data string) []byte { - for _, r := range data { - matched := false + start := 0 + for i, r := range data { for _, rl := range s.rules { // first match wins if rl.matches(r) { - applyTransform(&dst, r, rl.transform) - matched = true + // A rule without a known transform is a passthrough rule. + if rl.transform&(TransformStrip|TransformHexEncode|TransformJSONEscape) != 0 { + dst = append(dst, data[start:i]...) + applyTransform(&dst, r, rl.transform) + _, size := utf8.DecodeRuneInString(data[i:]) + start = i + size + } break } } - if !matched { - dst = utf8.AppendRune(dst, r) - } } + dst = append(dst, data[start:]...) return dst } // matchesFilter checks if a rune matches any filter in the mask func matchesFilter(r rune, filterMask uint64) bool { - for flag, checker := range filterCheckers { - if (filterMask&flag) != 0 && checker(r) { + if filterMask&FilterNonPrintable != 0 && !strconv.IsPrint(r) || + filterMask&FilterControl != 0 && unicode.IsControl(r) || + filterMask&FilterWhitespace != 0 && unicode.IsSpace(r) { + return true + } + if filterMask&FilterShellSpecial != 0 { + switch r { + case '`', '$', ';', '|', '&', '>', '<', '(', ')', '#', + '\'', '"', '\\', '*', '?', '[', ']', '{', '}', '~', '!': return true } } @@ -189,7 +196,7 @@ func applyTransform(buf *[]byte, r rune, transformMask uint64) { var runeBytes [utf8.UTFMax]byte n := utf8.EncodeRune(runeBytes[:], r) *buf = append(*buf, '<') - *buf = append(*buf, hex.EncodeToString(runeBytes[:n])...) + *buf = hex.AppendEncode(*buf, runeBytes[:n]) *buf = append(*buf, '>') case (transformMask & TransformJSONEscape) != 0: @@ -209,7 +216,7 @@ func applyTransform(buf *[]byte, r rune, transformMask uint64) { case '\\': *buf = append(*buf, '\\', '\\') default: - if r < 0x20 || r == 0x7f { + if r < 0x20 || r >= 0x7f && r <= 0x9f { *buf = append(*buf, fmt.Sprintf("\\u%04x", r)...) } else { *buf = utf8.AppendRune(*buf, r) @@ -226,6 +233,12 @@ type Serializer struct { // NewSerializer creates a handler with format-specific behavior func NewSerializer(format string, san *Sanitizer) *Serializer { + if san == nil { + san = New() + } + if format != "raw" && format != "json" && format != "txt" { + format = "txt" + } return &Serializer{ format: format, sanitizer: san, @@ -256,50 +269,19 @@ func (se *Serializer) WriteString(buf *[]byte, s string) { } case "json": - // Sanitizer applied as content transform before escaping - s = se.sanitizer.Sanitize(s) - *buf = append(*buf, '"') - for i := 0; i < len(s); { - c := s[i] - // raw UTF-8 is valid in JSON strings. Only <0x20, '"', '\\', 0x7f are escaped. - if c >= 0x20 && c != '"' && c != '\\' && c != 0x7f { - start := i - for i < len(s) { - c = s[i] - if c >= 0x20 && c != '"' && c != '\\' && c != 0x7f { - i++ - } else { - break - } - } - *buf = append(*buf, s[start:i]...) - } else { - switch c { - case '\\', '"': - *buf = append(*buf, '\\', c) - case '\n': - *buf = append(*buf, '\\', 'n') - case '\r': - *buf = append(*buf, '\\', 'r') - case '\t': - *buf = append(*buf, '\\', 't') - case '\b': - *buf = append(*buf, '\\', 'b') - case '\f': - *buf = append(*buf, '\\', 'f') - default: - *buf = append(*buf, fmt.Sprintf("\\u%04x", c)...) - } - i++ - } - } - *buf = append(*buf, '"') + // AppendQuote always produces valid UTF-8 JSON. Its error only reports + // replacement of malformed UTF-8, which is intentional for log output. + *buf, _ = jsontext.AppendQuote(*buf, se.sanitizer.Sanitize(s)) } } -// WriteNumber writes a number value +// WriteNumber writes a number; invalid JSON numbers are represented as strings. func (se *Serializer) WriteNumber(buf *[]byte, n string) { + if se.format == "json" && (strings.TrimSpace(n) != n || len(n) == 0 || n[0] != '-' && (n[0] < '0' || n[0] > '9') || !json.Valid([]byte(n))) { + se.WriteString(buf, n) + return + } *buf = append(*buf, n...) } @@ -320,10 +302,90 @@ func (se *Serializer) WriteNil(buf *[]byte) { // WriteComplex writes complex types func (se *Serializer) WriteComplex(buf *[]byte, v any) { + // fmt recursively traverses maps and slices without cycle detection. Bound + // that traversal before calling it; scalar and Stringer paths are separate. + budget := 10000 + if !boundedValue(reflect.ValueOf(v), 0, &budget) { + se.WriteString(buf, "") + return + } str := fmt.Sprintf("%+v", v) se.WriteString(buf, str) } +// boundedValue limits depth and total traversal work without allocating a +// visited map. A cycle necessarily exceeds the depth bound. Application-defined +// formatting/marshal methods remain responsible for their own termination. +func boundedValue(v reflect.Value, depth int, budget *int) bool { + *budget -= 1 + if depth > 64 || *budget < 0 { + return false + } + if !v.IsValid() { + return true + } + switch v.Kind() { + case reflect.Interface, reflect.Pointer: + return v.IsNil() || boundedValue(v.Elem(), depth+1, budget) + case reflect.Map: + iter := v.MapRange() + for iter.Next() { + if !boundedValue(iter.Key(), depth+1, budget) || !boundedValue(iter.Value(), depth+1, budget) { + return false + } + } + case reflect.Slice, reflect.Array: + for i := 0; i < v.Len(); i++ { + if !boundedValue(v.Index(i), depth+1, budget) { + return false + } + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + if !boundedValue(v.Field(i), depth+1, budget) { + return false + } + } + } + return true +} + +// WriteJSON marshals v with encoding/json semantics, then applies the content +// policy to every JSON string (including nested keys). On error buf is unchanged. +func (se *Serializer) WriteJSON(buf *[]byte, v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + if len(se.sanitizer.rules) == 0 { + *buf = append(*buf, data...) + return nil + } + // Marshal has already validated the JSON. Only string tokens need rewriting; + // copying the other bytes preserves numbers, booleans, and container shapes. + start := 0 + for i := 0; i < len(data); i++ { + if data[i] != '"' { + continue + } + *buf = append(*buf, data[start:i]...) + end := i + 1 + for data[end] != '"' { + if data[end] == '\\' { + end++ + } + end++ + } + var value string + _ = json.Unmarshal(data[i:end+1], &value) + quoted := Serializer{format: "json", sanitizer: se.sanitizer} + quoted.WriteString(buf, value) + i, start = end, end+1 + } + *buf = append(*buf, data[start:]...) + return nil +} + // NeedsQuotes determines if quoting is needed func (se *Serializer) NeedsQuotes(s string) bool { switch se.format { diff --git a/sanitizer/testdata/fuzz/FuzzSanitize/3120fc1ce48a8023 b/sanitizer/testdata/fuzz/FuzzSanitize/3120fc1ce48a8023 new file mode 100644 index 0000000..370fbed --- /dev/null +++ b/sanitizer/testdata/fuzz/FuzzSanitize/3120fc1ce48a8023 @@ -0,0 +1,5 @@ +go test fuzz v1 +string("0000000000000000000\xc2 \x80") +byte('\x0f') +uint64(2) +uint64(2) diff --git a/state.go b/state.go index c7fd306..3cef21a 100644 --- a/state.go +++ b/state.go @@ -1,7 +1,6 @@ package log import ( - "sync" "sync/atomic" ) @@ -23,8 +22,7 @@ type State struct { TraceDepth atomic.Int64 // Flushing state - flushRequestChan chan chan struct{} // Channel to request a flush - flushMutex sync.Mutex // Protect concurrent Flush calls + flushRequestChan chan flushRequest // per processor; published under sendMu // Outputs CurrentFile atomic.Value // stores *os.File diff --git a/storage.go b/storage.go index 7449f01..f670fea 100644 --- a/storage.go +++ b/storage.go @@ -5,37 +5,22 @@ import ( "os" "path/filepath" "sort" + "strings" "time" ) // performSync syncs the current log file -func (l *Logger) performSync() { - c := l.getConfig() - // Skip sync if file output is disabled - enableFile := c.EnableFile - if !enableFile { - return +func (l *Logger) performSync() error { + if !l.getConfig().EnableFile { + return nil } - - cfPtr := l.state.CurrentFile.Load() - if cfPtr != nil { - if currentLogFile, isFile := cfPtr.(*os.File); isFile && currentLogFile != nil { - if err := currentLogFile.Sync(); err != nil { - // Log sync error - syncErrRecord := logRecord{ - Flags: FlagDefault | FlagKV, - TimeStamp: time.Now(), - Level: LevelWarn, - Args: []any{ - "msg", "log file sync failed", - "file", currentLogFile.Name(), - "error", err.Error(), - }, - } - l.sendLogRecord(syncErrRecord) - } + if file, _ := l.state.CurrentFile.Load().(*os.File); file != nil { + if err := file.Sync(); err != nil { + return fmtErrorf("failed to sync log file '%s': %w", file.Name(), err) } + return nil } + return fmtErrorf("file output enabled without an open log file") } // performDiskCheck checks disk space, triggers cleanup if needed, and updates status @@ -140,28 +125,6 @@ func (l *Logger) performDiskCheck(forceCleanup bool) bool { } } -// // getDiskFreeSpace retrieves available disk space for the given path -// func (l *Logger) getDiskFreeSpace(path string) (int64, error) { -// var stat syscall.Statfs_t -// info, err := os.Stat(path) -// if err != nil { -// if os.IsNotExist(err) { -// return 0, fmtErrorf("log directory '%s' does not exist for disk check: %w", path, err) -// } -// return 0, fmtErrorf("failed to stat log directory '%s': %w", path, err) -// } -// if !info.IsDir() { -// path = filepath.Dir(path) -// } -// -// if err := syscall.Statfs(path, &stat); err != nil { -// return 0, fmtErrorf("failed to get disk stats for '%s': %w", path, err) -// } -// // Explicit cast to int64 to satisfy both Linux and FreebSD -// availableBytes := int64(stat.Bavail) * int64(stat.Bsize) -// return availableBytes, nil -// } - // getLogDirSize calculates total size of log files matching the current extension func (l *Logger) getLogDirSize(dir, ext string) (int64, error) { var size int64 @@ -173,18 +136,15 @@ func (l *Logger) getLogDirSize(dir, ext string) (int64, error) { return 0, fmtErrorf("failed to read log directory '%s': %w", dir, err) } - targetExt := "." + ext for _, entry := range entries { - if entry.IsDir() { + if !matchesLogEntry(entry, ext) { continue } - if filepath.Ext(entry.Name()) == targetExt { - info, errInfo := entry.Info() - if errInfo != nil { - continue - } - size += info.Size() + info, errInfo := entry.Info() + if errInfo != nil { + continue } + size += info.Size() } return size, nil } @@ -192,6 +152,9 @@ func (l *Logger) getLogDirSize(dir, ext string) (int64, error) { // cleanOldLogs removes oldest log files until required space is freed func (l *Logger) cleanOldLogs(required int64) error { c := l.getConfig() + if !c.EnableFile || required <= 0 { + return nil + } dir := c.Directory ext := c.Extension name := c.Name @@ -213,12 +176,8 @@ func (l *Logger) cleanOldLogs(required int64) error { size int64 } var logs []logFileMeta - targetExt := "." + ext for _, entry := range entries { - if entry.IsDir() || entry.Name() == staticLogName { - continue - } - if ext != "" && filepath.Ext(entry.Name()) != targetExt { + if !matchesLogEntry(entry, ext) || entry.Name() == staticLogName { continue } info, errInfo := entry.Info() @@ -236,7 +195,12 @@ func (l *Logger) cleanOldLogs(required int64) error { } // Sort logs by modification time to delete the oldest ones first - sort.Slice(logs, func(i, j int) bool { return logs[i].modTime.Before(logs[j].modTime) }) + sort.Slice(logs, func(i, j int) bool { + if logs[i].modTime.Equal(logs[j].modTime) { + return logs[i].name < logs[j].name + } + return logs[i].modTime.Before(logs[j].modTime) + }) // Iterate and remove files until enough space has been freed var freedSpace int64 @@ -281,18 +245,14 @@ func (l *Logger) updateEarliestFileTime() { staticLogName = name + "." + ext } - targetExt := "." + ext for _, entry := range entries { - if entry.IsDir() { + if !matchesLogEntry(entry, ext) { continue } fname := entry.Name() if fname == staticLogName { continue // Skip the active log file } - if ext != "" && filepath.Ext(fname) != targetExt { - continue - } info, errInfo := entry.Info() if errInfo != nil { continue @@ -313,7 +273,7 @@ func (l *Logger) cleanExpiredLogs(oldest time.Time) error { retentionPeriodHrs := c.RetentionPeriodHrs rpDuration := time.Duration(retentionPeriodHrs * float64(time.Hour)) - if rpDuration <= 0 { + if !c.EnableFile || rpDuration <= 0 { return nil } cutoffTime := time.Now().Add(-rpDuration) @@ -332,14 +292,8 @@ func (l *Logger) cleanExpiredLogs(oldest time.Time) error { staticLogName = name + "." + ext } - targetExt := "." + ext - var deletedCount int for _, entry := range entries { - if entry.IsDir() || entry.Name() == staticLogName { - continue - } - // Only consider files with correct extension - if ext != "" && filepath.Ext(entry.Name()) != targetExt { + if !matchesLogEntry(entry, ext) || entry.Name() == staticLogName { continue } info, errInfo := entry.Info() @@ -351,7 +305,6 @@ func (l *Logger) cleanExpiredLogs(oldest time.Time) error { if err := os.Remove(filePath); err != nil { l.internalLog("failed to remove expired log file '%s': %v\n", filePath, err) } else { - deletedCount++ l.state.TotalDeletions.Add(1) } } @@ -367,7 +320,7 @@ func (l *Logger) getStaticLogFilePath() string { ext := c.Extension name := c.Name - // Handle extension with or without dot + // Extension was validated without a leading dot filename := name if ext != "" { filename = name + "." + ext @@ -376,10 +329,10 @@ func (l *Logger) getStaticLogFilePath() string { return filepath.Join(dir, filename) } -// fileExists reports whether path names an existing regular file +// fileExists includes directories and symlinks, which also occupy archive names func fileExists(path string) bool { - info, err := os.Stat(path) - return err == nil && !info.IsDir() + _, err := os.Lstat(path) + return err == nil } // generateArchiveLogFileName creates a second-resolution name for a rotated @@ -394,16 +347,23 @@ func (l *Logger) generateArchiveLogFileName(timestamp time.Time) string { base := fmt.Sprintf("%s_%s", c.Name, timestamp.Format("060102_150405")) name := base + suffix - for i := 1; i < 1000 && fileExists(filepath.Join(c.Directory, name)); i++ { + for i := 1; fileExists(filepath.Join(c.Directory, name)); i++ { name = fmt.Sprintf("%s_%d%s", base, i, suffix) } return name } -// createNewLogFile generates a unique name and opens a new log file +// createNewLogFile opens the active log file in append mode func (l *Logger) createNewLogFile() (*os.File, error) { - fullPath := l.getStaticLogFilePath() + return openLogFile(l.getConfig()) +} +func openLogFile(c *Config) (*os.File, error) { + name := c.Name + if c.Extension != "" { + name += "." + c.Extension + } + fullPath := filepath.Join(c.Directory, name) file, err := os.OpenFile(fullPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return nil, fmtErrorf("failed to open/create log file '%s': %w", fullPath, err) @@ -450,6 +410,8 @@ func (l *Logger) rotateLogFile() error { // Continue with rotation anyway } + l.state.CurrentFile.Store((*os.File)(nil)) + // Generate a new unique name with current timestamp for the old log file dir := c.Directory archiveName := l.generateArchiveLogFileName(time.Now()) @@ -470,6 +432,8 @@ func (l *Logger) rotateLogFile() error { // Create new log file at static path newFile, err := l.createNewLogFile() if err != nil { + l.state.LoggerDisabled.Store(true) + l.refreshLevelGate() return fmtErrorf("failed to create new log file after rotation: %w", err) } @@ -495,15 +459,25 @@ func (l *Logger) getLogFileCount(dir, ext string) (int, error) { return -1, fmtErrorf("failed to read log directory '%s': %w", dir, err) } - targetExt := "." + ext for _, entry := range entries { - if entry.IsDir() { + if !matchesLogEntry(entry, ext) { continue } // Count all files matching the extension, including the current one if present - if filepath.Ext(entry.Name()) == targetExt { - count++ - } + count++ } return count, nil } + +// Cleanup is scoped to regular files with this extension, including files from +// earlier runs with different base names. Extensionless logs match only files +// without an extension. Each logger must own its directory exclusively. +func matchesLogEntry(entry os.DirEntry, ext string) bool { + if !entry.Type().IsRegular() { + return false + } + if ext == "" { + return filepath.Ext(entry.Name()) == "" + } + return strings.HasSuffix(entry.Name(), "."+ext) +} diff --git a/stress_test.go b/stress_test.go new file mode 100644 index 0000000..36aa5d9 --- /dev/null +++ b/stress_test.go @@ -0,0 +1,153 @@ +package log + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// Fixed workloads assert accounting and uniqueness; they do not depend on the +// scheduler producing a particular drop rate or on a wall-clock soak duration. +func TestStressQueueAccounting(t *testing.T) { + for _, capacity := range []int64{8, 16384} { + t.Run(strconv.FormatInt(capacity, 10), func(t *testing.T) { + const workers, perWorker = 32, 512 + var output bytes.Buffer + l := NewLogger() + c := DefaultConfig() + c.BufferSize = capacity + c.EnableConsole = true + c.Format = "raw" + mustNoErr(t, l.ApplyConfig(c), "apply") + l.state.StdoutWriter.Store(&sink{w: &output}) + var run func() + l.SetSpawn(func(fn func()) { run = fn }) + mustNoErr(t, l.Start(), "start deferred processor") + var wg sync.WaitGroup + for worker := range workers { + wg.Go(func() { + for i := range perWorker { + l.Info(strconv.Itoa(worker*perWorker+i) + "\n") + } + }) + } + wg.Wait() + total := uint64(workers * perWorker) + equal(t, l.state.TotalDroppedLogs.Load(), total-uint64(capacity), "exact overflow count") + go run() + mustNoErr(t, l.Shutdown(5*time.Second), "drain") + seen := assertUniqueRecords(t, output.String(), int(total)) + equal(t, len(seen), int(capacity), "accepted records all written") + equal(t, l.state.TotalLogsProcessed.Load()+l.state.TotalDroppedLogs.Load(), total, "no unaccounted records") + }) + } +} + +func TestStressLifecycleAccounting(t *testing.T) { + const workers, perWorker = 8, 1000 + l := NewLogger() + c := DefaultConfig() + c.EnableConsole = false + c.EnableFile = true + c.Directory = t.TempDir() + c.Format = "raw" + c.BufferSize = 256 + c.MaxSizeKB = 0 + c.MaxTotalSizeKB = 0 + c.MinDiskFreeKB = 0 + mustNoErr(t, l.ApplyConfig(c), "apply") + mustNoErr(t, l.Start(), "start") + start := make(chan struct{}) + var wg sync.WaitGroup + for worker := range workers { + wg.Go(func() { + <-start + for i := range perWorker { + // Exercise the enqueue boundary directly: public level-gated calls made + // while stopped intentionally never become attempted records. + l.sendLogRecord(logRecord{Args: []any{strconv.Itoa(worker*perWorker+i) + "\n"}}) + } + }) + } + wg.Go(func() { + <-start + for i := range 24 { + noErr(t, l.Stop(time.Second), "stop") + noErr(t, l.Start(), "start") + noErr(t, l.ApplyConfigString(fmt.Sprintf("buffer_size=%d", 128+128*(i%2))), "reconfigure") + } + }) + close(start) + wg.Wait() + mustNoErr(t, l.Shutdown(5*time.Second), "final shutdown") + seen := assertUniqueRecords(t, readLog(t, c.Directory), workers*perWorker) + equal(t, uint64(len(seen)), l.state.TotalLogsProcessed.Load(), "written equals processed") + equal(t, l.state.TotalLogsProcessed.Load()+l.state.TotalDroppedLogs.Load(), uint64(workers*perWorker), "every attempted enqueue processed or dropped") +} + +func TestStressRotationPreservesRecords(t *testing.T) { + const total = 4096 + l := NewLogger() + c := DefaultConfig() + c.EnableConsole = false + c.EnableFile = true + c.Directory = t.TempDir() + c.BufferSize = total + c.Format = "raw" + c.MaxSizeKB = 1 + c.MaxTotalSizeKB = 0 + c.MinDiskFreeKB = 0 + mustNoErr(t, l.ApplyConfig(c), "apply") + mustNoErr(t, l.Start(), "start") + var wg sync.WaitGroup + for worker := range 16 { + wg.Go(func() { + for i := range total / 16 { + l.Info(strconv.Itoa(worker*(total/16)+i) + "\n") + } + }) + } + wg.Wait() + mustNoErr(t, l.Shutdown(5*time.Second), "shutdown") + equal(t, len(assertUniqueRecords(t, readAllLogs(t, c.Directory), total)), total, "records survive rotation") + isTrue(t, l.state.TotalRotations.Load() > 1, "multiple rotations") + var actualSize int64 + files, err := os.ReadDir(c.Directory) + mustNoErr(t, err, "list") + for _, file := range files { + info, err := os.Stat(filepath.Join(c.Directory, file.Name())) + mustNoErr(t, err, "stat") + actualSize += info.Size() + } + var expectedSize int64 + for i := range total { + expectedSize += int64(len(strconv.Itoa(i)) + 1) + } + equal(t, actualSize, expectedSize, "exact bytes across archives") + equal(t, l.state.TotalDroppedLogs.Load(), uint64(0), "no drops") +} + +func assertUniqueRecords(t *testing.T, data string, total int) map[int]bool { + t.Helper() + seen := make(map[int]bool) + for _, line := range strings.Split(strings.TrimSuffix(data, "\n"), "\n") { + if line == "" && data == "" { + continue + } + id, err := strconv.Atoi(line) + if err != nil || id < 0 || id >= total { + t.Fatalf("invalid record %q", line) + } + if seen[id] { + t.Fatalf("duplicate record %d", id) + } + seen[id] = true + } + return seen +} diff --git a/timer.go b/timer.go index d9cd3b8..54a212f 100644 --- a/timer.go +++ b/timer.go @@ -20,6 +20,7 @@ func (l *Logger) setupProcessingTimers() *TimerSet { // Set up disk check timer timers.diskCheckTicker = l.setupDiskCheckTimer() + timers.diskInterval = time.Duration(max(c.MinCheckIntervalMs, min(c.DiskCheckIntervalMs, c.MaxCheckIntervalMs))) * time.Millisecond // Set up heartbeat timer timers.heartbeatChan = l.setupHeartbeatTimer(timers) @@ -35,7 +36,7 @@ func (l *Logger) setupRetentionTimer(timers *TimerSet) <-chan time.Time { retentionDur := time.Duration(retentionPeriodHrs * float64(time.Hour)) retentionCheckInterval := time.Duration(retentionCheckMins * float64(time.Minute)) - if retentionDur > 0 && retentionCheckInterval > 0 { + if c.EnableFile && retentionDur > 0 && retentionCheckInterval > 0 { timers.retentionTicker = time.NewTicker(retentionCheckInterval) l.updateEarliestFileTime() // Initial check return timers.retentionTicker.C @@ -96,4 +97,4 @@ func (l *Logger) stopProcessingTimers(timers *TimerSet) { if timers.heartbeatTicker != nil { timers.heartbeatTicker.Stop() } -} \ No newline at end of file +} diff --git a/type.go b/type.go index 5affcbc..ce69bd8 100644 --- a/type.go +++ b/type.go @@ -15,6 +15,7 @@ const ContextSlots = formatter.ContextSlots // logRecord represents a single log entry type logRecord struct { + heartbeat bool TimeStamp time.Time Trace string Args []any @@ -31,9 +32,16 @@ type TimerSet struct { heartbeatTicker *time.Ticker retentionChan <-chan time.Time heartbeatChan <-chan time.Time + diskInterval time.Duration } // sink is a wrapper around an io.Writer, atomic value type change workaround type sink struct { w io.Writer } + +// flushRequest belongs to one processor generation; the buffered reply permits +// completion even when the caller has already timed out. +type flushRequest struct { + done chan error +} diff --git a/utility.go b/utility.go index a422c87..9f5ae5e 100644 --- a/utility.go +++ b/utility.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" "runtime" + "slices" "strings" "unicode" @@ -25,12 +26,14 @@ func getTrace(depth int64, skip int) string { count := 0 for { frame, more := frames.Next() - if !more || count >= int(depth) { + if count >= int(depth) { break } funcName := filepath.Base(frame.Function) - parts := strings.Split(funcName, ".") - lastPart := parts[len(parts)-1] + prefix, lastPart, found := strings.CutLast(funcName, ".") + if !found { + lastPart = funcName + } if strings.HasPrefix(lastPart, "func") { isAnonymous := true for _, r := range lastPart[4:] { @@ -40,7 +43,7 @@ func getTrace(depth int64, skip int) string { } } if isAnonymous && len(lastPart) > 4 { - funcName = fmt.Sprintf("(anonymous in %s)", strings.Join(parts[:len(parts)-1], ".")) + funcName = fmt.Sprintf("(anonymous in %s)", prefix) } else { funcName = lastPart } @@ -49,14 +52,15 @@ func getTrace(depth int64, skip int) string { } trace = append(trace, funcName) count++ + if !more { + break + } } if len(trace) == 0 { return "(unknown)" } // Reverse for caller -> callee order - for i, j := 0, len(trace)-1; i < j; i, j = i+1, j-1 { - trace[i], trace[j] = trace[j], trace[i] - } + slices.Reverse(trace) return strings.Join(trace, " -> ") } diff --git a/utility_test.go b/utility_test.go index 29c022f..7d11958 100644 --- a/utility_test.go +++ b/utility_test.go @@ -149,4 +149,3 @@ func TestGetTraceOrdering(t *testing.T) { t.Errorf("frames not ordered: %q", trace) } } -