From 4ed618abbb779b020697db96ce768a2e4778e5504df93244f98b7a73b5f341ad Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Sat, 15 Nov 2025 16:32:27 -0500 Subject: [PATCH] v0.1.3 formatter exported, docs updated --- README.md | 3 +- config.go | 6 +- constant.go | 21 ++- doc/adapters.md | 5 +- doc/api.md | 32 +++- doc/builder.md | 72 +++++---- doc/configuration.md | 9 +- doc/formatting.md | 234 +++++++++++++++++++++++++++ doc/getting-started.md | 5 +- doc/heartbeat.md | 5 +- doc/logging.md | 23 ++- doc/quick-guide_lixenwraith_log.md | 236 ++++++++++++++-------------- doc/storage.md | 5 +- format_test.go | 174 +++++++------------- format.go => formatter/formatter.go | 130 +++++++++++---- formatter/formatter_test.go | 143 +++++++++++++++++ logger.go | 20 ++- processor.go | 4 +- sanitizer/sanitizer.go | 5 +- timer.go | 30 ++-- utility.go | 37 +---- 21 files changed, 807 insertions(+), 392 deletions(-) create mode 100644 doc/formatting.md rename format.go => formatter/formatter.go (70%) create mode 100644 formatter/formatter_test.go diff --git a/README.md b/README.md index 82a6dd5..f8cf534 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Log -[![Go](https://img.shields.io/badge/Go-1.24+-00ADD8?style=flat&logo=go)](https://golang.org) +[![Go](https://img.shields.io/badge/Go-1.25+-00ADD8?style=flat&logo=go)](https://golang.org) [![License](https://img.shields.io/badge/License-BSD_3--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Documentation](https://img.shields.io/badge/Docs-Available-green.svg)](doc/) @@ -59,6 +59,7 @@ go get github.com/lixenwraith/log - **[Configuration Builder](doc/builder.md)** - Builder pattern guide - **[API Reference](doc/api.md)** - Complete API documentation - **[Logging Guide](doc/logging.md)** - Logging methods and best practices ++ **[Formatting & Sanitization](doc/formatting.md)** - Standalone formatter and sanitizer packages - **[Disk Management](doc/storage.md)** - File rotation and cleanup - **[Heartbeat Monitoring](doc/heartbeat.md)** - Operational statistics - **[Compatibility Adapters](doc/adapters.md)** - Framework integrations diff --git a/config.go b/config.go index 3898707..0015836 100644 --- a/config.go +++ b/config.go @@ -28,7 +28,7 @@ type Config struct { ShowTimestamp bool `toml:"show_timestamp"` // Add timestamp to log records ShowLevel bool `toml:"show_level"` // Add level to log record TimestampFormat string `toml:"timestamp_format"` // Time format for log timestamps - Sanitization sanitizer.PolicyPreset `toml:"sanitization"` // "default", "json", "txt", "shell" + Sanitization sanitizer.PolicyPreset `toml:"sanitization"` // "raw", "json", "txt", "shell" // Buffer and size limits BufferSize int64 `toml:"buffer_size"` // Channel buffer size @@ -75,7 +75,7 @@ var defaultConfig = Config{ ShowTimestamp: true, ShowLevel: true, TimestampFormat: time.RFC3339Nano, - Sanitization: sanitizer.PolicyTxt, + Sanitization: PolicyTxt, // Buffer and size limits BufferSize: 1024, @@ -128,7 +128,7 @@ func (c *Config) Validate() error { } switch c.Sanitization { - case sanitizer.PolicyRaw, sanitizer.PolicyJSON, sanitizer.PolicyTxt, sanitizer.PolicyShell: + case PolicyRaw, PolicyJSON, PolicyTxt, PolicyShell: // valid policy default: return fmtErrorf("invalid sanitization policy: '%s' (use raw, json, txt, or shell)", c.Sanitization) diff --git a/constant.go b/constant.go index bad3b2d..00839e6 100644 --- a/constant.go +++ b/constant.go @@ -3,6 +3,9 @@ package log import ( "time" + + "github.com/lixenwraith/log/formatter" + "github.com/lixenwraith/log/sanitizer" ) // Log level constants @@ -22,11 +25,19 @@ const ( // Record flags for controlling output structure const ( - FlagRaw int64 = 0b0001 - FlagShowTimestamp int64 = 0b0010 - FlagShowLevel int64 = 0b0100 - FlagStructuredJSON int64 = 0b1000 - FlagDefault = FlagShowTimestamp | FlagShowLevel + FlagRaw = formatter.FlagRaw // Bypasses both formatter and sanitizer + FlagShowTimestamp = formatter.FlagShowTimestamp + FlagShowLevel = formatter.FlagShowLevel + FlagStructuredJSON = formatter.FlagStructuredJSON + FlagDefault = formatter.FlagDefault +) + +// Sanitizer policies +const ( + PolicyRaw = sanitizer.PolicyRaw + PolicyJSON = sanitizer.PolicyJSON + PolicyTxt = sanitizer.PolicyTxt + PolicyShell = sanitizer.PolicyShell ) // Storage diff --git a/doc/adapters.md b/doc/adapters.md index 6e96fc3..f617ec6 100644 --- a/doc/adapters.md +++ b/doc/adapters.md @@ -349,7 +349,4 @@ func requestLogger(adapter *compat.FastHTTPAdapter) fasthttp.RequestHandler { time.Since(start)) } } -``` - ---- -[← Heartbeat Monitoring](heartbeat-monitoring.md) | [← Back to README](../README.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/doc/api.md b/doc/api.md index 59ac764..175d935 100644 --- a/doc/api.md +++ b/doc/api.md @@ -327,6 +327,33 @@ Converts level string to numeric constant. level, err := log.Level("debug") // Returns -4 ``` +### Format Flags + +```go +const ( + FlagRaw = formatter.FlagRaw // Bypass formatting + FlagShowTimestamp = formatter.FlagShowTimestamp // Include timestamp + FlagShowLevel = formatter.FlagShowLevel // Include level + FlagStructuredJSON = formatter.FlagStructuredJSON // Structured JSON + FlagDefault = formatter.FlagDefault // Default flags +) +``` + +Control output formatting behavior. These flags are re-exported from the formatter package. + +### Sanitization Policies + +```go +const ( + PolicyRaw = sanitizer.PolicyRaw // No sanitization + PolicyJSON = sanitizer.PolicyJSON // JSON-safe output + PolicyTxt = sanitizer.PolicyTxt // Text file safe + PolicyShell = sanitizer.PolicyShell // Shell-safe output +) +``` + +Pre-configured sanitization policies. These are re-exported from the sanitizer package. + ## Error Types The logger returns errors prefixed with "log: " for easy identification: @@ -385,7 +412,4 @@ func (s *Service) ProcessRequest(id string) error { func (s *Service) Shutdown() error { return s.logger.Shutdown(5 * time.Second) } -``` - ---- -[← Configuration Builder](config-builder.md) | [← Back to README](../README.md) | [Logging Guide →](logging-guide.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/doc/builder.md b/doc/builder.md index 5705051..5f95e95 100644 --- a/doc/builder.md +++ b/doc/builder.md @@ -18,39 +18,40 @@ All builder methods return `*Builder` for chaining. Errors are accumulated and r ### Common Methods -| Method | Parameters | Description | -|--------|------------|-------------| -| `Level(level int64)` | `level`: Numeric log level | Sets log level (-4 to 8) | -| `LevelString(level string)` | `level`: Named level | Sets level by name ("debug", "info", etc.) | -| `Name(name string)` | `name`: Base filename | Sets log file base name | -| `Directory(dir string)` | `dir`: Path | Sets log directory | -| `Format(format string)` | `format`: Output format | Sets format ("txt", "json", "raw") | -| `Extension(ext string)` | `ext`: File extension | Sets log file extension | -| `BufferSize(size int64)` | `size`: Buffer size | Sets channel buffer size | -| `MaxSizeKB(size int64)` | `size`: Size in KB | Sets max file size in KB | -| `MaxSizeMB(size int64)` | `size`: Size in MB | Sets max file size in MB | -| `MaxTotalSizeKB(size int64)` | `size`: Size in KB | Sets max total log directory size in KB | -| `MaxTotalSizeMB(size int64)` | `size`: Size in MB | Sets max total log directory size in MB | -| `MinDiskFreeKB(size int64)` | `size`: Size in KB | Sets minimum required free disk space in KB | -| `MinDiskFreeMB(size int64)` | `size`: Size in MB | Sets minimum required free disk space in MB | -| `EnableConsole(enable bool)` | `enable`: Boolean | Enables console output | -| `EnableFile(enable bool)` | `enable`: Boolean | Enables file output | -| `ConsoleTarget(target string)` | `target`: "stdout"/"stderr" | Sets console output target | -| `ShowTimestamp(show bool)` | `show`: Boolean | Controls timestamp display | -| `ShowLevel(show bool)` | `show`: Boolean | Controls log level display | -| `TimestampFormat(format string)` | `format`: Time format | Sets timestamp format (Go time format) | -| `HeartbeatLevel(level int64)` | `level`: 0-3 | Sets monitoring level (0=off) | -| `HeartbeatIntervalS(interval int64)` | `interval`: Seconds | Sets heartbeat interval | -| `FlushIntervalMs(interval int64)` | `interval`: Milliseconds | Sets buffer flush interval | -| `TraceDepth(depth int64)` | `depth`: 0-10 | Sets default function trace depth | -| `DiskCheckIntervalMs(interval int64)` | `interval`: Milliseconds | Sets disk check interval | -| `EnableAdaptiveInterval(enable bool)` | `enable`: Boolean | Enables adaptive disk check intervals | -| `MinCheckIntervalMs(interval int64)` | `interval`: Milliseconds | Sets minimum adaptive interval | -| `MaxCheckIntervalMs(interval int64)` | `interval`: Milliseconds | Sets maximum adaptive interval | -| `EnablePeriodicSync(enable bool)` | `enable`: Boolean | Enables periodic disk sync | -| `RetentionPeriodHrs(hours float64)` | `hours`: Hours | Sets log retention period | -| `RetentionCheckMins(mins float64)` | `mins`: Minutes | Sets retention check interval | -| `InternalErrorsToStderr(enable bool)` | `enable`: Boolean | Send internal errors to stderr | +| Method | Parameters | Description | +|---------------------------------------|-------------------------------|---------------------------------------------| +| `Level(level int64)` | `level`: Numeric log level | Sets log level (-4 to 8) | +| `LevelString(level string)` | `level`: Named level | Sets level by name ("debug", "info", etc.) | +| `Name(name string)` | `name`: Base filename | Sets log file base name | +| `Directory(dir string)` | `dir`: Path | Sets log directory | +| `Format(format string)` | `format`: Output format | Sets format ("txt", "json", "raw") | +| `Sanitization(policy string)` | `policy`: Sanitization policy | Sets policy ("txt", "json", "raw", "shell") | +| `Extension(ext string)` | `ext`: File extension | Sets log file extension | +| `BufferSize(size int64)` | `size`: Buffer size | Sets channel buffer size | +| `MaxSizeKB(size int64)` | `size`: Size in KB | Sets max file size in KB | +| `MaxSizeMB(size int64)` | `size`: Size in MB | Sets max file size in MB | +| `MaxTotalSizeKB(size int64)` | `size`: Size in KB | Sets max total log directory size in KB | +| `MaxTotalSizeMB(size int64)` | `size`: Size in MB | Sets max total log directory size in MB | +| `MinDiskFreeKB(size int64)` | `size`: Size in KB | Sets minimum required free disk space in KB | +| `MinDiskFreeMB(size int64)` | `size`: Size in MB | Sets minimum required free disk space in MB | +| `EnableConsole(enable bool)` | `enable`: Boolean | Enables console output | +| `EnableFile(enable bool)` | `enable`: Boolean | Enables file output | +| `ConsoleTarget(target string)` | `target`: "stdout"/"stderr" | Sets console output target | +| `ShowTimestamp(show bool)` | `show`: Boolean | Controls timestamp display | +| `ShowLevel(show bool)` | `show`: Boolean | Controls log level display | +| `TimestampFormat(format string)` | `format`: Time format | Sets timestamp format (Go time format) | +| `HeartbeatLevel(level int64)` | `level`: 0-3 | Sets monitoring level (0=off) | +| `HeartbeatIntervalS(interval int64)` | `interval`: Seconds | Sets heartbeat interval | +| `FlushIntervalMs(interval int64)` | `interval`: Milliseconds | Sets buffer flush interval | +| `TraceDepth(depth int64)` | `depth`: 0-10 | Sets default function trace depth | +| `DiskCheckIntervalMs(interval int64)` | `interval`: Milliseconds | Sets disk check interval | +| `EnableAdaptiveInterval(enable bool)` | `enable`: Boolean | Enables adaptive disk check intervals | +| `MinCheckIntervalMs(interval int64)` | `interval`: Milliseconds | Sets minimum adaptive interval | +| `MaxCheckIntervalMs(interval int64)` | `interval`: Milliseconds | Sets maximum adaptive interval | +| `EnablePeriodicSync(enable bool)` | `enable`: Boolean | Enables periodic disk sync | +| `RetentionPeriodHrs(hours float64)` | `hours`: Hours | Sets log retention period | +| `RetentionCheckMins(mins float64)` | `mins`: Minutes | Sets retention check interval | +| `InternalErrorsToStderr(enable bool)` | `enable`: Boolean | Send internal errors to stderr | ## Build ```go @@ -84,7 +85,4 @@ err = logger.Start() if err != nil { return err } logger.Info("Application started") -``` - ---- -[← Configuration](configuration.md) | [← Back to README](../README.md) | [API Reference →](api-reference.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/doc/configuration.md b/doc/configuration.md index cbb9bbd..3b0064e 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -49,9 +49,11 @@ logger.Info("info txt log record written to /var/log/myapp.txt") |-----------|------|-------------|------------| | `level` | `int64` | Minimum log level (-4=Debug, 0=Info, 4=Warn, 8=Error) | `0` | | `name` | `string` | Base name for log files | `"log"` | -| `directory` | `string` | Directory to store log files | `"./log"` | -| `format` | `string` | Output format: `"txt"` or `"json"` | `"txt"` | | `extension` | `string` | Log file extension (without dot) | `"log"` | +| `directory` | `string` | Directory to store log files | `"./log"` | +| `format` | `string` | Output format: `"txt"`, `"json"`, or `"raw"` | `"txt"` | +| `sanitization` | `string` | Sanitization policy: `"raw"`, `"txt"`, `"json"`, or `"shell"` | `"raw"` | +| `timestamp_format` | `string` | Custom timestamp format (Go time format) | `time.RFC3339Nano` | | `internal_errors_to_stderr` | `bool` | Write logger's internal errors to stderr | `false` | ### Output Control @@ -101,5 +103,4 @@ logger.Info("info txt log record written to /var/log/myapp.txt") | `heartbeat_level` | `int64` | Heartbeat detail (0=off, 1=proc, 2=+disk, 3=+sys) | `0` | | `heartbeat_interval_s` | `int64` | Heartbeat interval (seconds) | `60` | ---- -[← Getting Started](getting-started.md) | [← Back to README](../README.md) | [Configuration Builder →](config-builder.md) \ No newline at end of file +--- \ No newline at end of file diff --git a/doc/formatting.md b/doc/formatting.md new file mode 100644 index 0000000..df90312 --- /dev/null +++ b/doc/formatting.md @@ -0,0 +1,234 @@ +# Formatting and Sanitization + +The logger package exports standalone `formatter` and `sanitizer` packages that can be used independently for text formatting and sanitization needs beyond logging. + +## Formatter Package + +The `formatter` package provides buffered writing and formatting of log entries with support for txt, json, and raw output formats. + +### Standalone Usage + +```go +import ( + "time" + "github.com/lixenwraith/log/formatter" + "github.com/lixenwraith/log/sanitizer" +) + +// Create formatter with optional sanitizer +s := sanitizer.New().Policy(sanitizer.PolicyTxt) +f := formatter.New(s) + +// Configure formatter +f.Type("json"). + TimestampFormat(time.RFC3339). + ShowLevel(true). + ShowTimestamp(true) + +// Format a log entry +data := f.Format( + formatter.FlagDefault, + time.Now(), + 0, // Info level + "", // No trace + []any{"User logged in", "user_id", 42}, +) +``` + +### Formatter Methods + +#### Format Configuration +- `Type(format string)` - Set output format: "txt", "json", or "raw" +- `TimestampFormat(format string)` - Set timestamp format (Go time format) +- `ShowLevel(show bool)` - Include level in output +- `ShowTimestamp(show bool)` - Include timestamp in output + +#### Formatting Methods +- `Format(flags int64, timestamp time.Time, level int64, trace string, args []any) []byte` +- `FormatWithOptions(format string, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte` +- `FormatValue(v any) []byte` - Format a single value +- `FormatArgs(args ...any) []byte` - Format multiple arguments + +### Format Flags + +```go +const ( + FlagRaw int64 = 0b0001 // Bypass formatter and sanitizer + FlagShowTimestamp int64 = 0b0010 // Include timestamp + FlagShowLevel int64 = 0b0100 // Include level + FlagStructuredJSON int64 = 0b1000 // Use structured JSON with message/fields + FlagDefault = FlagShowTimestamp | FlagShowLevel +) +``` + +### Level Constants + +```go +// Use formatter.LevelToString() to convert levels +formatter.LevelToString(0) // "INFO" +formatter.LevelToString(4) // "WARN" +formatter.LevelToString(8) // "ERROR" +``` + +## Sanitizer Package + +The `sanitizer` package provides fluent and composable string sanitization based on configurable rules using bitwise filter flags and transforms. + +### Standalone Usage + +```go +import "github.com/lixenwraith/log/sanitizer" + +// Create sanitizer with predefined policy +s := sanitizer.New().Policy(sanitizer.PolicyJSON) +clean := s.Sanitize("hello\nworld") // "hello\\nworld" + +// Custom rules +s = sanitizer.New(). + Rule(sanitizer.FilterControl, sanitizer.TransformHexEncode). + Rule(sanitizer.FilterShellSpecial, sanitizer.TransformStrip) + +clean = s.Sanitize("cmd; echo test") // "cmd echo test" +``` + +### Predefined Policies + +```go +const ( + PolicyRaw PolicyPreset = "raw" // No-op passthrough + PolicyJSON PolicyPreset = "json" // JSON-safe strings + PolicyTxt PolicyPreset = "txt" // Text file safe + PolicyShell PolicyPreset = "shell" // Shell command safe +) +``` + +- **PolicyRaw**: Pass through all characters unchanged +- **PolicyTxt**: Hex-encode non-printable characters as `` +- **PolicyJSON**: Escape control characters with JSON-style backslashes +- **PolicyShell**: Strip shell metacharacters and whitespace + +### Filter Flags + +```go +const ( + FilterNonPrintable uint64 = 1 << iota // Non-printable runes + FilterControl // Control characters + FilterWhitespace // Whitespace characters + FilterShellSpecial // Shell metacharacters +) +``` + +### Transform Flags + +```go +const ( + TransformStrip uint64 = 1 << iota // Remove character + TransformHexEncode // Encode as + TransformJSONEscape // JSON backslash escape +) +``` + +### Custom Rules + +Combine filters and transforms for custom sanitization: + +```go +// Remove control characters, hex-encode non-printable +s := sanitizer.New(). + Rule(sanitizer.FilterControl, sanitizer.TransformStrip). + Rule(sanitizer.FilterNonPrintable, sanitizer.TransformHexEncode) + +// Apply multiple policies +s = sanitizer.New(). + Policy(sanitizer.PolicyTxt). + Rule(sanitizer.FilterWhitespace, sanitizer.TransformJSONEscape) +``` + +### Serializer + +The sanitizer includes a `Serializer` for type-aware sanitization: + +```go +serializer := sanitizer.NewSerializer("json", s) + +var buf []byte +serializer.WriteString(&buf, "hello\nworld") // Adds quotes and escapes +serializer.WriteNumber(&buf, "123.45") // No quotes for numbers +serializer.WriteBool(&buf, true) // "true" +serializer.WriteNil(&buf) // "null" +``` + +## Integration with Logger + +The logger uses these packages internally but configuration remains simple: + +```go +logger := log.NewLogger() + +// Configure sanitization policy +logger.ApplyConfigString( + "format=json", + "sanitization=json", // Uses PolicyJSON +) + +// Or with custom formatter (advanced) +s := sanitizer.New().Policy(sanitizer.PolicyShell) +customFormatter := formatter.New(s).Type("txt") +// Note: Direct formatter injection requires using lower-level APIs +``` + +## Common Patterns + +### Security-Focused Sanitization + +```go +// For user input that will be logged +userInput := getUserInput() +s := sanitizer.New(). + Policy(sanitizer.PolicyShell). + Rule(sanitizer.FilterControl, sanitizer.TransformStrip) + +safeLogs := s.Sanitize(userInput) +logger.Info("User input", "data", safeLogs) +``` + +### Custom Log Formatting + +```go +// Format logs for external system +f := formatter.New() +f.Type("json").ShowTimestamp(false).ShowLevel(false) + +// Create custom log entry +entry := f.FormatArgs("action", "purchase", "amount", 99.99) +sendToExternalSystem(entry) +``` + +### Multi-Target Output + +```go +// Different sanitization for different outputs +jsonSanitizer := sanitizer.New().Policy(sanitizer.PolicyJSON) +shellSanitizer := sanitizer.New().Policy(sanitizer.PolicyShell) + +// For JSON API +jsonFormatter := formatter.New(jsonSanitizer).Type("json") +apiLog := jsonFormatter.Format(...) + +// For shell script generation +txtFormatter := formatter.New(shellSanitizer).Type("txt") +scriptLog := txtFormatter.Format(...) +``` + +## Performance Considerations + +- Both packages use pre-allocated buffers for efficiency +- Sanitizer rules are applied in a single pass +- Formatter reuses internal buffers via `Reset()` +- No regex or reflection in hot paths + +## Thread Safety + +- `Formatter` instances are **NOT** thread-safe (use separate instances per goroutine) +- `Sanitizer` instances **ARE** thread-safe (immutable after creation) +- For concurrent formatting, create a formatter per goroutine or use sync.Pool \ No newline at end of file diff --git a/doc/getting-started.md b/doc/getting-started.md index 8123a30..43ad5bd 100644 --- a/doc/getting-started.md +++ b/doc/getting-started.md @@ -114,7 +114,4 @@ func loggingMiddleware(logger *log.Logger) func(http.Handler) http.Handler { }) } } -``` - ---- -[← Back to README](../README.md) | [Configuration →](configuration.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/doc/heartbeat.md b/doc/heartbeat.md index 075de9f..7a638fa 100644 --- a/doc/heartbeat.md +++ b/doc/heartbeat.md @@ -162,7 +162,4 @@ With `format=txt`, heartbeats are human-readable: ``` 2024-01-15T10:30:00.123456789Z PROC type="proc" sequence=42 uptime_hours="24.50" processed_logs=1847293 dropped_logs=0 -``` - ---- -[← Disk Management](disk-management.md) | [← Back to README](../README.md) | [Compatibility Adapters →](compatibility-adapters.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/doc/logging.md b/doc/logging.md index 069cc86..ee07c86 100644 --- a/doc/logging.md +++ b/doc/logging.md @@ -126,6 +126,8 @@ func logWithContext(ctx context.Context, logger *log.Logger, level string, msg s ## Output Formats +The logger supports three output formats, each with configurable sanitization. For advanced formatting needs, see [Formatting & Sanitization](formatting.md) for standalone usage of the formatter and sanitizer packages. + ### Txt Format (Human-Readable) Default format for development and debugging: @@ -135,7 +137,7 @@ Default format for development and debugging: 2024-01-15T10:30:45.234567890Z WARN Rate limit approaching user_id=42 requests=95 limit=100 ``` -Note: The txt format adds quotes around non-string values (errors, stringers, complex types) when they contain spaces. Plain string arguments are not quoted. Control characters in strings are sanitized to hex representation. For logs requiring exact preservation of all values, `json` format is recommended. +Note: The txt format applies the configured sanitization policy (default: raw). Non-printable characters can be hex-encoded using `sanitization=txt` configuration. Configuration: ```go @@ -164,6 +166,20 @@ logger.ApplyConfigString( ) ``` +### Raw Format (Unstructured) + +Outputs arguments as space-separated values without any metadata: + +``` +METRIC cpu_usage 85.5 timestamp 1234567890 +``` + +Configuration: +```go +logger.ApplyConfigString("format=raw") +// Or use logger.Write() method which forces raw output +``` + ## Function Tracing ### Using Trace Methods @@ -333,7 +349,4 @@ func (m *MetricsCollector) logMetrics() { ) } } -``` - ---- -[← API Reference](api-reference.md) | [← Back to README](../README.md) | [Disk Management →](disk-management.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/doc/quick-guide_lixenwraith_log.md b/doc/quick-guide_lixenwraith_log.md index aad5d48..1a0b374 100644 --- a/doc/quick-guide_lixenwraith_log.md +++ b/doc/quick-guide_lixenwraith_log.md @@ -1,47 +1,51 @@ +I'll search the project knowledge to understand the current state of the log package and update the quick-guide documentation accordingly.# FILE: doc/quick-guide_lixenwraith_log.md + # lixenwraith/log Quick Reference Guide -This guide details the `lixenwraith/log` package, a high-performance, buffered, rotating file logger for Go with built-in disk management, operational monitoring, and framework compatibility adapters. +High-performance buffered rotating file logger with disk management, operational monitoring, and exported formatter/sanitizer packages. ## Quick Start: Recommended Usage -The recommended pattern uses the **Builder** with type-safe configuration. This provides compile-time safety and eliminates runtime errors. +Builder pattern with type-safe configuration (compile-time safety, no runtime errors): ```go package main import ( - "fmt" - "time" - - "github.com/lixenwraith/log" + "fmt" + "os" + "time" + + "github.com/lixenwraith/log" ) func main() { - // 1. Use the builder pattern for configuration (recommended). - logger, err := log.NewBuilder(). - Directory("/var/log/myapp"). // Log directory path - LevelString("info"). // Minimum log level - Format("json"). // Output format - BufferSize(2048). // Channel buffer size - MaxSizeMB(10). // Max file size before rotation - HeartbeatLevel(1). // Enable operational monitoring - HeartbeatIntervalS(300). // Every 5 minutes - Build() // Build the logger instance - if err != nil { - panic(fmt.Errorf("logger build failed: %w", err)) - } - defer logger.Shutdown(5 * time.Second) + // Build logger with configuration + logger, err := log.NewBuilder(). + Directory("/var/log/myapp"). // Log directory path + LevelString("info"). // Minimum log level + Format("json"). // Output format + Sanitization("json"). // Sanitization policy + BufferSize(2048). // Channel buffer size + MaxSizeMB(10). // Max file size before rotation + HeartbeatLevel(1). // Enable operational monitoring + HeartbeatIntervalS(300). // Every 5 minutes + Build() // Build the logger instance + if err != nil { + panic(fmt.Errorf("logger build failed: %w", err)) + } + defer logger.Shutdown(5 * time.Second) - // 2. Start the logger (required before logging). - if err := logger.Start(); err != nil { - panic(fmt.Errorf("logger start failed: %w", err)) - } + // Start the logger (required before logging) + if err := logger.Start(); err != nil { + panic(fmt.Errorf("logger start failed: %w", err)) + } - // 3. Begin logging with structured key-value pairs. - logger.Info("Application started", "version", "1.0.0", "pid", os.Getpid()) - logger.Debug("Debug information", "user_id", 12345) - logger.Warn("High memory usage", "used_mb", 1800, "limit_mb", 2048) - logger.Error("Connection failed", "host", "db.example.com", "error", err) + // Begin logging with structured key-value pairs + logger.Info("Application started", "version", "1.0.0", "pid", os.Getpid()) + logger.Debug("Debug information", "user_id", 12345) + logger.Warn("High memory usage", "used_mb", 1800, "limit_mb", 2048) + logger.Error("Connection failed", "host", "db.example.com", "error", err) } ``` @@ -52,13 +56,14 @@ func main() { ```go logger := log.NewLogger() err := logger.ApplyConfigString( - "directory=/var/log/app", - "format=json", - "level=debug", - "max_size_kb=5000", + "directory=/var/log/app", + "format=json", + "sanitization=json", + "level=debug", + "max_size_kb=5000", ) if err != nil { - return fmt.Errorf("config failed: %w", err) + return fmt.Errorf("config failed: %w", err) } defer logger.Shutdown() logger.Start() @@ -71,12 +76,13 @@ logger := log.NewLogger() cfg := log.DefaultConfig() cfg.Directory = "/var/log/app" cfg.Format = "json" +cfg.Sanitization = log.PolicyJSON cfg.Level = log.LevelDebug cfg.MaxSizeKB = 5000 cfg.HeartbeatLevel = 2 // Process + disk stats err := logger.ApplyConfig(cfg) if err != nil { - return fmt.Errorf("config failed: %w", err) + return fmt.Errorf("config failed: %w", err) } defer logger.Shutdown() logger.Start() @@ -84,12 +90,8 @@ logger.Start() ## Builder Pattern -The `Builder` is the primary way to construct a `Logger` instance with compile-time safety. - ```go -// NewBuilder creates a new logger builder. func NewBuilder() *Builder -// Build finalizes configuration and creates the logger. func (b *Builder) Build() (*Logger, error) ``` @@ -103,6 +105,7 @@ All builder methods return `*Builder` for chaining. - `Directory(dir string)`: Set log directory path - `Name(name string)`: Set base filename (default: "log") - `Format(format string)`: Set format ("txt", "json", "raw") +- `Sanitization(policy string)`: Set sanitization policy ("txt", "json", "raw", "shell") - `Extension(ext string)`: Set file extension (default: ".log") **Buffer and Performance:** @@ -222,6 +225,17 @@ const ( ) ``` +### Sanitization Policies + +```go +const ( + PolicyRaw = "raw" // No-op passthrough + PolicyJSON = "json" // JSON-safe output + PolicyTxt = "txt" // Text file safe + PolicyShell = "shell" // Shell-safe output +) +``` + ### Level Helper ```go @@ -229,88 +243,21 @@ func Level(levelStr string) (int64, error) ``` Converts level string to numeric constant: "debug", "info", "warn", "error", "proc", "disk", "sys". -## Configuration Structure - -```go -type Config struct { - // Output Settings - EnableConsole bool // Enable stdout/stderr output - ConsoleTarget string // "stdout", "stderr", or "split" - EnableFile bool // Enable file output - - // Basic Settings - Level int64 // Minimum log level - Name string // Base filename (default: "log") - Directory string // Log directory path - Format string // "txt", "json", or "raw" - Extension string // File extension (default: ".log") - - // Formatting - ShowTimestamp bool // Add timestamps - ShowLevel bool // Add level labels - TimestampFormat string // Go time format - - // Buffer and Performance - BufferSize int64 // Channel buffer size - FlushIntervalMs int64 // Buffer flush interval - TraceDepth int64 // Default trace depth (0-10) - - // File Management - MaxSizeKB int64 // Max file size (KB) - MaxTotalSizeKB int64 // Max total directory size (KB) - MinDiskFreeKB int64 // Required free disk space (KB) - RetentionPeriodHrs float64 // Hours to keep logs - RetentionCheckMins float64 // Retention check interval - - // Disk Monitoring - DiskCheckIntervalMs int64 // Base check interval - EnableAdaptiveInterval bool // Adjust based on load - MinCheckIntervalMs int64 // Minimum interval - MaxCheckIntervalMs int64 // Maximum interval - EnablePeriodicSync bool // Periodic disk sync - - // Heartbeat - HeartbeatLevel int64 // 0=off, 1=proc, 2=+disk, 3=+sys - HeartbeatIntervalS int64 // Heartbeat interval - - // Error Handling - InternalErrorsToStderr bool // Write internal errors to stderr -} -``` - -### Default Configuration - -```go -func DefaultConfig() *Config -``` - -Returns default configuration with sensible values: -- Console output enabled to stdout -- File output enabled -- Info level logging -- 1MB max file size -- 5MB max total size -- 100ms flush interval - ## Output Formats -### Text Format (default) - -Human-readable format with optional timestamps and levels: -``` -2024-01-15T10:30:00.123456Z INFO Application started version="1.0.0" pid=1234 -2024-01-15T10:30:01.456789Z ERROR Connection failed host="db.example.com" error="timeout" -``` - ### JSON Format -Structured JSON output for log aggregation: ```json -{"time":"2024-01-15T10:30:00.123456Z","level":"INFO","fields":["Application started","version","1.0.0","pid",1234]} -{"time":"2024-01-15T10:30:01.456789Z","level":"ERROR","fields":["Connection failed","host","db.example.com","error","timeout"]} +{"timestamp":"2024-01-01T12:00:00Z","level":"INFO","fields":["Application started","version","1.0.0"]} ``` -### Raw Format +### TXT Format + +``` +2024-01-01T12:00:00Z INFO Application started version="1.0.0" pid=1234 +``` + +### RAW Format Minimal format without timestamps or levels: ``` @@ -318,6 +265,47 @@ Application started version="1.0.0" pid=1234 Connection failed host="db.example.com" error="timeout" ``` +## Standalone Formatter/Sanitizer Packages + +### Formatter Package + +```go +import ( + "time" + "github.com/lixenwraith/log/formatter" + "github.com/lixenwraith/log/sanitizer" +) + +// Create formatter with sanitizer +s := sanitizer.New().Policy(sanitizer.PolicyJSON) +f := formatter.New(s) + +// Configure and format +f.Type("json").ShowTimestamp(true) +data := f.Format( + formatter.FlagDefault, + time.Now(), + 0, // Info level + "", // No trace + []any{"User action", "user_id", 42}, +) +``` + +### Sanitizer Package + +```go +import "github.com/lixenwraith/log/sanitizer" + +// Predefined policy +s := sanitizer.New().Policy(sanitizer.PolicyJSON) +clean := s.Sanitize("hello\nworld") // "hello\\nworld" + +// Custom rules +s = sanitizer.New(). + Rule(sanitizer.FilterControl, sanitizer.TransformStrip). + Rule(sanitizer.FilterNonPrintable, sanitizer.TransformHexEncode) +``` + ## Framework Adapters (compat package) ### gnet v2 Adapter @@ -450,6 +438,18 @@ logger.ApplyConfigString( ) ``` +### Security-Focused Sanitization + +```go +// User input logging with shell-safe sanitization +userInput := getUserInput() +s := sanitizer.New().Policy(sanitizer.PolicyShell) +logger.Info("User command", "input", s.Sanitize(userInput)) + +// Or configure logger-wide +logger.ApplyConfigString("sanitization=shell") +``` + ### Graceful Shutdown ```go @@ -476,7 +476,7 @@ All public methods are thread-safe. The logger uses: ## Performance Characteristics -- **Zero-allocation logging path**: Uses pre-allocated buffers +- **Zero-allocation logging path**: Pre-allocated buffers - **Lock-free async design**: Non-blocking sends to buffered channel - **Adaptive disk checks**: Adjusts I/O based on load - **Batch writes**: Flushes buffer periodically, not per-record @@ -510,4 +510,8 @@ logger.Info("User login", "id", id, "name", name) 1. **Use Builder pattern** for configuration - compile-time safety 2. **Use structured logging** - consistent key-value pairs -3. **Use appropriate levels** - filter noise in logs \ No newline at end of file +3. **Use appropriate levels** - filter noise in logs +4. **Configure sanitization** - prevent log injection attacks +5. **Monitor heartbeats** - track logger health in production +6. **Handle shutdown** - always call Shutdown() to flush logs +7. **Use standalone packages** - reuse formatter/sanitizer for other needs \ No newline at end of file diff --git a/doc/storage.md b/doc/storage.md index 63ee046..9087e51 100644 --- a/doc/storage.md +++ b/doc/storage.md @@ -180,7 +180,4 @@ ls -t /var/log/myapp/*.log | tail -n 20 | xargs rm # Verify space df -h /var/log -``` - ---- -[← Logging Guide](logging-guide.md) | [← Back to README](../README.md) | [Heartbeat Monitoring →](heartbeat-monitoring.md) \ No newline at end of file +``` \ No newline at end of file diff --git a/format_test.go b/format_test.go index 9c17e46..a7ac048 100644 --- a/format_test.go +++ b/format_test.go @@ -1,124 +1,85 @@ // FILE: lixenwraith/log/format_test.go +// This file tests the integration between log package and formatter package package log import ( - "encoding/json" - "errors" "os" "path/filepath" "strings" "testing" "time" - "github.com/lixenwraith/log/sanitizer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// TestFormatter tests the output of the formatter for txt, json, and raw formats -func TestFormatter(t *testing.T) { - f := NewFormatter("txt", 1024, time.RFC3339Nano, sanitizer.PolicyRaw) - timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) - - t.Run("txt format", func(t *testing.T) { - data := f.Format("txt", FlagDefault, timestamp, LevelInfo, "", []any{"test message", 123}) - str := string(data) - - assert.Contains(t, str, "2024-01-01") - assert.Contains(t, str, "INFO") - assert.Contains(t, str, "test message") - assert.Contains(t, str, "123") - assert.True(t, strings.HasSuffix(str, "\n")) - }) - - f = NewFormatter("json", 1024, time.RFC3339Nano, sanitizer.PolicyRaw) - t.Run("json format", func(t *testing.T) { - data := f.Format("json", FlagDefault, timestamp, LevelWarn, "trace1", []any{"warning", true}) - - var result map[string]any - err := json.Unmarshal(data[:len(data)-1], &result) // Remove trailing newline - require.NoError(t, err) - - assert.Equal(t, "WARN", result["level"]) - assert.Equal(t, "trace1", result["trace"]) - fields := result["fields"].([]any) - assert.Equal(t, "warning", fields[0]) - assert.Equal(t, true, fields[1]) - }) - - f = NewFormatter("raw", 1024, time.RFC3339Nano, sanitizer.PolicyRaw) - t.Run("raw format", func(t *testing.T) { - data := f.Format("raw", 0, timestamp, LevelInfo, "", []any{"raw", "data", 42}) - str := string(data) - - assert.Equal(t, "raw data 42", str) - assert.False(t, strings.HasSuffix(str, "\n")) - }) - - t.Run("flag override raw", func(t *testing.T) { - data := f.Format("txt", FlagRaw, timestamp, LevelInfo, "", []any{"forced", "raw"}) - str := string(data) - - assert.Equal(t, "forced raw", str) - }) - - f = NewFormatter("json", 1024, time.RFC3339Nano, sanitizer.PolicyJSON) - t.Run("structured json", func(t *testing.T) { - fields := map[string]any{"key1": "value1", "key2": 42} - data := f.Format("json", FlagStructuredJSON|FlagDefault, timestamp, LevelInfo, "", - []any{"structured message", fields}) - - var result map[string]any - err := json.Unmarshal(data[:len(data)-1], &result) - require.NoError(t, err) - - assert.Equal(t, "structured message", result["message"]) - assert.Equal(t, map[string]any{"key1": "value1", "key2": float64(42)}, result["fields"]) - }) - - f = NewFormatter("json", 1024, time.RFC3339Nano, sanitizer.PolicyJSON) - t.Run("special characters escaping", func(t *testing.T) { - data := f.Format("json", FlagDefault, timestamp, LevelInfo, "", - []any{"test\n\r\t\"\\message"}) - - str := string(data) - assert.Contains(t, str, `test\n\r\t\"\\message`) - }) - - t.Run("error type handling", func(t *testing.T) { - err := errors.New("test error") - data := f.Format("txt", FlagDefault, timestamp, LevelError, "", []any{err}) - - str := string(data) - assert.Contains(t, str, "test error") - }) -} - -// TestLevelToString verifies the conversion of log level constants to strings -func TestLevelToString(t *testing.T) { +// TestLoggerFormatterIntegration verifies logger correctly uses the new formatter package +func TestLoggerFormatterIntegration(t *testing.T) { tests := []struct { - level int64 - expected string + name string + format string + check func(t *testing.T, content string) }{ - {LevelDebug, "DEBUG"}, - {LevelInfo, "INFO"}, - {LevelWarn, "WARN"}, - {LevelError, "ERROR"}, - {LevelProc, "PROC"}, - {LevelDisk, "DISK"}, - {LevelSys, "SYS"}, - {999, "LEVEL(999)"}, + { + name: "txt format", + format: "txt", + check: func(t *testing.T, content string) { + assert.Contains(t, content, `INFO "test message"`) + }, + }, + { + name: "json format", + format: "json", + check: func(t *testing.T, content string) { + assert.Contains(t, content, `"level":"INFO"`) + assert.Contains(t, content, `"fields":["test message"]`) + }, + }, + { + name: "raw format", + format: "raw", + check: func(t *testing.T, content string) { + assert.Contains(t, content, "test message") + }, + }, } for _, tt := range tests { - t.Run(tt.expected, func(t *testing.T) { - assert.Equal(t, tt.expected, levelToString(tt.level)) + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + logger := NewLogger() + + cfg := DefaultConfig() + cfg.Directory = tmpDir + cfg.Format = tt.format + cfg.ShowTimestamp = false + cfg.ShowLevel = true + cfg.FlushIntervalMs = 10 + + err := logger.ApplyConfig(cfg) + require.NoError(t, err) + + err = logger.Start() + require.NoError(t, err) + defer logger.Shutdown() + + logger.Info("test message") + + err = logger.Flush(time.Second) + require.NoError(t, err) + + time.Sleep(50 * time.Millisecond) + + content, err := os.ReadFile(filepath.Join(tmpDir, "log.log")) + require.NoError(t, err) + + tt.check(t, string(content)) }) } } -// TestControlCharacterWrite verifies that control characters are safely handled in raw output -func TestControlCharacterWrite(t *testing.T) { +// TestControlCharacterWriteWithFormatter verifies control character handling through formatter +func TestControlCharacterWriteWithFormatter(t *testing.T) { logger, tmpDir := createTestLogger(t) defer logger.Shutdown() @@ -129,7 +90,6 @@ func TestControlCharacterWrite(t *testing.T) { err := logger.ApplyConfig(cfg) require.NoError(t, err) - // Test various control characters with expected sanitized output testCases := []struct { name string input string @@ -153,19 +113,17 @@ func TestControlCharacterWrite(t *testing.T) { content, err := os.ReadFile(filepath.Join(tmpDir, "log.log")) require.NoError(t, err) - // Verify each test case produced correct sanitized output for _, tc := range testCases { assert.Contains(t, string(content), tc.expected, "Test case '%s' should produce hex-encoded control chars", tc.name) } } -// TestRawSanitizedOutput verifies that raw output is correctly sanitized -func TestRawSanitizedOutput(t *testing.T) { +// TestRawSanitizedOutputWithFormatter verifies raw output sanitization through formatter +func TestRawSanitizedOutputWithFormatter(t *testing.T) { logger, tmpDir := createTestLogger(t) defer logger.Shutdown() - // Use raw format instead of Write() to test sanitization cfg := logger.GetConfig() cfg.Format = "raw" cfg.ShowTimestamp = false @@ -173,31 +131,21 @@ func TestRawSanitizedOutput(t *testing.T) { err := logger.ApplyConfig(cfg) require.NoError(t, err) - // 1. A string with valid multi-byte UTF-8 should be unchanged utf8String := "Hello │ 世界" - - // 2. A string with single-byte control chars should have them encoded stringWithControl := "start-\x07-end" expectedStringOutput := "start-<07>-end" - - // 3. A []byte with control chars should have them encoded, not stripped bytesWithControl := []byte("data\x00with\x08bytes") expectedBytesOutput := "data<00>with<08>bytes" - - // 4. A string with a multi-byte non-printable rune (U+0085, NEXT LINE) multiByteControl := "line1\u0085line2" expectedMultiByteOutput := "line1line2" - // Log all cases logger.Message(utf8String, stringWithControl, bytesWithControl, multiByteControl) logger.Flush(time.Second) - // Read and verify the single line of output content, err := os.ReadFile(filepath.Join(tmpDir, "log.log")) require.NoError(t, err) logOutput := string(content) - // The output should be one line with spaces between the sanitized parts expectedOutput := strings.Join([]string{ utf8String, expectedStringOutput, diff --git a/format.go b/formatter/formatter.go similarity index 70% rename from format.go rename to formatter/formatter.go index bf86c11..b4e4d15 100644 --- a/format.go +++ b/formatter/formatter.go @@ -1,5 +1,5 @@ -// FILE: lixenwraith/log/format.go -package log +// FILE: lixenwraith/log/formatter/formatter.go +package formatter import ( "encoding/json" @@ -11,42 +11,89 @@ import ( "github.com/lixenwraith/log/sanitizer" ) +// Format flags for controlling output structure +const ( + FlagRaw int64 = 0b0001 + FlagShowTimestamp int64 = 0b0010 + FlagShowLevel int64 = 0b0100 + FlagStructuredJSON int64 = 0b1000 + FlagDefault = FlagShowTimestamp | FlagShowLevel +) + // Formatter manages the buffered writing and formatting of log entries type Formatter struct { - format string - buf []byte - timestampFormat string sanitizer *sanitizer.Sanitizer + format string + timestampFormat string + showTimestamp bool + showLevel bool + buf []byte } -// NewFormatter creates a formatter instance -func NewFormatter(format string, bufferSize int64, timestampFormat string, sanitizationPolicy sanitizer.PolicyPreset) *Formatter { - if timestampFormat == "" { - timestampFormat = time.RFC3339Nano +// New creates a formatter with the provided sanitizer +func New(s ...*sanitizer.Sanitizer) *Formatter { + var san *sanitizer.Sanitizer + if len(s) > 0 && s[0] != nil { + san = s[0] + } else { + san = sanitizer.New() // Default passthrough sanitizer } - if format == "" { - format = "txt" - } - if sanitizationPolicy == "" { - sanitizationPolicy = "raw" - } - - s := (sanitizer.New()).Policy(sanitizationPolicy) return &Formatter{ - format: format, - buf: make([]byte, 0, bufferSize), - timestampFormat: timestampFormat, - sanitizer: s, + sanitizer: san, + format: "txt", + timestampFormat: time.RFC3339Nano, + showTimestamp: true, + showLevel: true, + buf: make([]byte, 0, 1024), } } -// Reset clears the formatter buffer for reuse -func (f *Formatter) Reset() { - f.buf = f.buf[:0] +// Type sets the output format ("txt", "json", or "raw") +func (f *Formatter) Type(format string) *Formatter { + f.format = format + return f } -// Format converts log entries to the configured format -func (f *Formatter) Format(format string, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte { +// TimestampFormat sets the timestamp format string +func (f *Formatter) TimestampFormat(format string) *Formatter { + if format != "" { + f.timestampFormat = format + } + return f +} + +// ShowLevel sets whether to include level in output +func (f *Formatter) ShowLevel(show bool) *Formatter { + f.showLevel = show + return f +} + +// ShowTimestamp sets whether to include timestamp in output +func (f *Formatter) ShowTimestamp(show bool) *Formatter { + f.showTimestamp = show + return f +} + +// Format formats a log entry using configured options and explicit flags +func (f *Formatter) Format(flags int64, timestamp time.Time, level int64, trace string, args []any) []byte { + // Override configured values with explicit flags + effectiveShowTimestamp := (flags&FlagShowTimestamp) != 0 || (flags == 0 && f.showTimestamp) + effectiveShowLevel := (flags&FlagShowLevel) != 0 || (flags == 0 && f.showLevel) + + // Build effective flags + effectiveFlags := flags + if effectiveShowTimestamp { + effectiveFlags |= FlagShowTimestamp + } + if effectiveShowLevel { + effectiveFlags |= FlagShowLevel + } + + return f.FormatWithOptions(f.format, effectiveFlags, timestamp, level, trace, args) +} + +// FormatWithOptions formats with explicit format and flags, ignoring configured values +func (f *Formatter) FormatWithOptions(format string, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte { f.Reset() // FlagRaw completely bypasses formatting and sanitization @@ -111,6 +158,33 @@ func (f *Formatter) FormatArgs(args ...any) []byte { return f.buf } +// Reset clears the formatter buffer for reuse +func (f *Formatter) Reset() { + f.buf = f.buf[:0] +} + +// LevelToString converts integer level values to string +func LevelToString(level int64) string { + switch level { + case -4: + return "DEBUG" + case 0: + return "INFO" + case 4: + return "WARN" + case 8: + return "ERROR" + case 12: + return "PROC" + case 16: + return "DISK" + case 20: + return "SYS" + default: + return fmt.Sprintf("LEVEL(%d)", level) + } +} + // convertValue provides unified type conversion func (f *Formatter) convertValue(buf *[]byte, v any, serializer *sanitizer.Serializer, needsSpace bool) { if needsSpace && len(*buf) > 0 { @@ -191,7 +265,7 @@ func (f *Formatter) formatJSON(flags int64, timestamp time.Time, level int64, tr f.buf = append(f.buf, ',') } f.buf = append(f.buf, `"level":"`...) - f.buf = append(f.buf, levelToString(level)...) + f.buf = append(f.buf, LevelToString(level)...) f.buf = append(f.buf, '"') needsComma = true } @@ -265,7 +339,7 @@ func (f *Formatter) formatTxt(flags int64, timestamp time.Time, level int64, tra if needsSpace { f.buf = append(f.buf, ' ') } - f.buf = append(f.buf, levelToString(level)...) + f.buf = append(f.buf, LevelToString(level)...) needsSpace = true } diff --git a/formatter/formatter_test.go b/formatter/formatter_test.go new file mode 100644 index 0000000..1e2cd32 --- /dev/null +++ b/formatter/formatter_test.go @@ -0,0 +1,143 @@ +// FILE: lixenwraith/log/formatter/formatter_test.go +package formatter + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/lixenwraith/log/sanitizer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFormatter(t *testing.T) { + timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + + t.Run("fluent API", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyRaw) + f := New(s). + Type("json"). + TimestampFormat(time.RFC3339). + ShowLevel(true). + ShowTimestamp(true) + + data := f.Format(0, timestamp, 0, "", []any{"test"}) + assert.Contains(t, string(data), `"level":"INFO"`) + assert.Contains(t, string(data), `"time":"2024-01-01T12:00:00Z"`) + }) + + t.Run("txt format", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyRaw) + f := New(s).Type("txt") + + data := f.Format(FlagDefault, timestamp, 0, "", []any{"test message", 123}) + str := string(data) + + assert.Contains(t, str, "2024-01-01") + assert.Contains(t, str, "INFO") + assert.Contains(t, str, "test message") + assert.Contains(t, str, "123") + assert.True(t, strings.HasSuffix(str, "\n")) + }) + + t.Run("json format", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyRaw) + f := New(s).Type("json") + + data := f.Format(FlagDefault, timestamp, 4, "trace1", []any{"warning", true}) + + var result map[string]any + err := json.Unmarshal(data[:len(data)-1], &result) // Remove trailing newline + require.NoError(t, err) + + assert.Equal(t, "WARN", result["level"]) + assert.Equal(t, "trace1", result["trace"]) + fields := result["fields"].([]any) + assert.Equal(t, "warning", fields[0]) + assert.Equal(t, true, fields[1]) + }) + + t.Run("raw format", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyRaw) + f := New(s).Type("raw") + + data := f.FormatWithOptions("raw", 0, timestamp, 0, "", []any{"raw", "data", 42}) + str := string(data) + + assert.Equal(t, "raw data 42", str) + assert.False(t, strings.HasSuffix(str, "\n")) + }) + + t.Run("flag override raw", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyRaw) + f := New(s).Type("json") // Configure as JSON + + data := f.Format(FlagRaw, timestamp, 0, "", []any{"forced", "raw"}) + str := string(data) + + assert.Equal(t, "forced raw", str) + }) + + t.Run("structured json", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyJSON) + f := New(s).Type("json") + + fields := map[string]any{"key1": "value1", "key2": 42} + data := f.Format(FlagStructuredJSON|FlagDefault, timestamp, 0, "", + []any{"structured message", fields}) + + var result map[string]any + err := json.Unmarshal(data[:len(data)-1], &result) + require.NoError(t, err) + + assert.Equal(t, "structured message", result["message"]) + assert.Equal(t, map[string]any{"key1": "value1", "key2": float64(42)}, result["fields"]) + }) + + t.Run("special characters escaping", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyJSON) + f := New(s).Type("json") + + data := f.Format(FlagDefault, timestamp, 0, "", + []any{"test\n\r\t\"\\message"}) + + str := string(data) + assert.Contains(t, str, `test\n\r\t\"\\message`) + }) + + t.Run("error type handling", func(t *testing.T) { + s := sanitizer.New().Policy(sanitizer.PolicyRaw) + f := New(s).Type("txt") + + err := errors.New("test error") + data := f.Format(FlagDefault, timestamp, 8, "", []any{err}) + + str := string(data) + assert.Contains(t, str, "test error") + }) +} + +func TestLevelToString(t *testing.T) { + tests := []struct { + level int64 + expected string + }{ + {-4, "DEBUG"}, + {0, "INFO"}, + {4, "WARN"}, + {8, "ERROR"}, + {12, "PROC"}, + {16, "DISK"}, + {20, "SYS"}, + {999, "LEVEL(999)"}, + } + + for _, tt := range tests { + t.Run(tt.expected, func(t *testing.T) { + assert.Equal(t, tt.expected, LevelToString(tt.level)) + }) + } +} \ No newline at end of file diff --git a/logger.go b/logger.go index c85f373..645fba1 100644 --- a/logger.go +++ b/logger.go @@ -2,12 +2,16 @@ package log import ( + "errors" "fmt" "io" "os" "sync" "sync/atomic" "time" + + "github.com/lixenwraith/log/formatter" + "github.com/lixenwraith/log/sanitizer" ) // Logger is the core struct that encapsulates all logger functionality @@ -15,7 +19,7 @@ type Logger struct { currentConfig atomic.Value // stores *Config state State initMu sync.Mutex - formatter *Formatter + formatter *formatter.Formatter } // NewLogger creates a new Logger instance with default settings @@ -205,18 +209,18 @@ func (l *Logger) Shutdown(timeout ...time.Duration) error { if currentLogFile, ok := cfPtr.(*os.File); ok && currentLogFile != nil { if err := currentLogFile.Sync(); err != nil { syncErr := fmtErrorf("failed to sync log file '%s' during shutdown: %w", currentLogFile.Name(), err) - finalErr = combineErrors(finalErr, syncErr) + finalErr = errors.Join(finalErr, syncErr) } if err := currentLogFile.Close(); err != nil { closeErr := fmtErrorf("failed to close log file '%s' during shutdown: %w", currentLogFile.Name(), err) - finalErr = combineErrors(finalErr, closeErr) + finalErr = errors.Join(finalErr, closeErr) } l.state.CurrentFile.Store((*os.File)(nil)) } } if stopErr != nil { - finalErr = combineErrors(finalErr, stopErr) + finalErr = errors.Join(finalErr, stopErr) } return finalErr @@ -341,7 +345,13 @@ func (l *Logger) applyConfig(cfg *Config) error { oldCfg := l.getConfig() l.currentConfig.Store(cfg) - l.formatter = NewFormatter(cfg.Format, cfg.BufferSize, cfg.TimestampFormat, cfg.Sanitization) + // Create formatter with sanitizer + s := sanitizer.New().Policy(cfg.Sanitization) + l.formatter = formatter.New(s). + Type(cfg.Format). + TimestampFormat(cfg.TimestampFormat). + ShowLevel(cfg.ShowLevel). + ShowTimestamp(cfg.ShowTimestamp) // Ensure log directory exists if file output is enabled if cfg.EnableFile { diff --git a/processor.go b/processor.go index 181d169..cea2647 100644 --- a/processor.go +++ b/processor.go @@ -13,7 +13,7 @@ func (l *Logger) processLogs(ch <-chan logRecord) { // Set up timers and state variables timers := l.setupProcessingTimers() - defer l.closeProcessingTimers(timers) + defer l.stopProcessingTimers(timers) c := l.getConfig() @@ -103,9 +103,7 @@ func (l *Logger) processLogRecord(record logRecord) int64 { } // Format and serialize the log entry once - format := c.Format data := l.formatter.Format( - format, record.Flags, record.TimeStamp, record.Level, diff --git a/sanitizer/sanitizer.go b/sanitizer/sanitizer.go index 5f02482..ef3b65c 100644 --- a/sanitizer/sanitizer.go +++ b/sanitizer/sanitizer.go @@ -33,7 +33,7 @@ const ( type PolicyPreset string const ( - PolicyRaw PolicyPreset = "raw" // Default is a no-op (passthrough) + PolicyRaw PolicyPreset = "raw" // Raw is a no-op (passthrough) PolicyJSON PolicyPreset = "json" // Policy for sanitizing strings to be embedded in JSON PolicyTxt PolicyPreset = "txt" // Policy for sanitizing text written to log files PolicyShell PolicyPreset = "shell" // Policy for sanitizing arguments passed to shell commands @@ -81,7 +81,7 @@ func New() *Sanitizer { } } -// Rule adds a custom rule to the sanitizer (prepended for precedence) +// Rule adds a custom rule to the sanitizer (appended, earliest rule applies first) func (s *Sanitizer) Rule(filter uint64, transform uint64) *Sanitizer { // Append rule in natural order s.rules = append(s.rules, rule{filter: filter, transform: transform}) @@ -263,6 +263,7 @@ func (se *Serializer) WriteNil(buf *[]byte) { // WriteComplex writes complex types func (se *Serializer) WriteComplex(buf *[]byte, v any) { switch se.format { + // For debugging case "raw": var b bytes.Buffer dumper := &spew.ConfigState{ diff --git a/timer.go b/timer.go index d079e66..1a92be3 100644 --- a/timer.go +++ b/timer.go @@ -1,4 +1,4 @@ -// FILE: lixenwraith/log/processor.go +// FILE: lixenwraith/log/timer.go package log import "time" @@ -28,20 +28,6 @@ func (l *Logger) setupProcessingTimers() *TimerSet { return timers } -// closeProcessingTimers stops all active timers -func (l *Logger) closeProcessingTimers(timers *TimerSet) { - timers.flushTicker.Stop() - if timers.diskCheckTicker != nil { - timers.diskCheckTicker.Stop() - } - if timers.retentionTicker != nil { - timers.retentionTicker.Stop() - } - if timers.heartbeatTicker != nil { - timers.heartbeatTicker.Stop() - } -} - // setupRetentionTimer configures the retention check timer if retention is enabled func (l *Logger) setupRetentionTimer(timers *TimerSet) <-chan time.Time { c := l.getConfig() @@ -97,4 +83,18 @@ func (l *Logger) setupHeartbeatTimer(timers *TimerSet) <-chan time.Time { return timers.heartbeatTicker.C } return nil +} + +// stopProcessingTimers stops all active timers +func (l *Logger) stopProcessingTimers(timers *TimerSet) { + timers.flushTicker.Stop() + if timers.diskCheckTicker != nil { + timers.diskCheckTicker.Stop() + } + if timers.retentionTicker != nil { + timers.retentionTicker.Stop() + } + if timers.heartbeatTicker != nil { + timers.heartbeatTicker.Stop() + } } \ No newline at end of file diff --git a/utility.go b/utility.go index 8efba20..5eff525 100644 --- a/utility.go +++ b/utility.go @@ -59,7 +59,7 @@ func getTrace(depth int64, skip int) string { return strings.Join(trace, " -> ") } -// fmtErrorf wrapper +// fmtErrorf wraps fmt.Errorf with a "log: " prefix func fmtErrorf(format string, args ...any) error { if !strings.HasPrefix(format, "log: ") { format = "log: " + format @@ -67,18 +67,7 @@ func fmtErrorf(format string, args ...any) error { return fmt.Errorf(format, args...) } -// combineErrors helper -func combineErrors(err1, err2 error) error { - if err1 == nil { - return err2 - } - if err2 == nil { - return err1 - } - return fmt.Errorf("%v; %w", err1, err2) -} - -// parseKeyValue splits a "key=value" string +// parseKeyValue splits a "key=value" string into its components func parseKeyValue(arg string) (string, string, error) { parts := strings.SplitN(strings.TrimSpace(arg), "=", 2) if len(parts) != 2 { @@ -112,26 +101,4 @@ func Level(levelStr string) (int64, error) { default: return 0, fmtErrorf("invalid level string: '%s' (use debug, info, warn, error, proc, disk, sys)", levelStr) } -} - -// levelToString converts integer level values to string -func levelToString(level int64) string { - switch level { - case LevelDebug: - return "DEBUG" - case LevelInfo: - return "INFO" - case LevelWarn: - return "WARN" - case LevelError: - return "ERROR" - case LevelProc: - return "PROC" - case LevelDisk: - return "DISK" - case LevelSys: - return "SYS" - default: - return fmt.Sprintf("LEVEL(%d)", level) - } } \ No newline at end of file