lixen/log
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.1.6 fix to formatter and sanitizer for async use, doc update

Browse Source
This commit is contained in:
lixen
2026-07-17 12:53:28 -04:00
parent c288c3790c
commit cd1ff9d4b6
15 changed files with 678 additions and 790 deletions
Download Patch File Download Diff File Expand all files Collapse all files
formatter/formatter.go
+194 -158
View File
@@ -1,3 +1,17 @@
// Package formatter provides buffered and append-style formatting of log
// entries in txt, json, and raw formats.
//
// Ownership and concurrency contract:
// - Configure via the fluent API (Type, TimestampFormat, ShowLevel,
// ShowTimestamp) before sharing an instance; configuration is not
// synchronized.
// - Buffered methods (Format, FormatWithOptions, FormatValue, FormatArgs)
// reuse an internal buffer. The returned slice is valid only until the
// next buffered call; copy before retention or async hand-off. Single
// goroutine only.
// - Append methods (AppendFormat, AppendFormatWithOptions, AppendValue,
// AppendArgs) write to a caller-provided buffer and are safe for
// concurrent use after configuration.
package formatter
import (
@@ -10,16 +24,20 @@ import (
"github.com/lixenwraith/log/sanitizer"
)
// Format flags for controlling output structure
// Format flags. Resolution in Format/AppendFormat: FlagNo* suppresses,
// FlagShow* enables, otherwise configured default applies; FlagNo* wins on conflict.
// FormatWithOptions/AppendFormatWithOptions are explicit: unset FlagShow* bits mean off.
const (
FlagRaw int64 = 0b0001
FlagShowTimestamp int64 = 0b0010
FlagShowLevel int64 = 0b0100
FlagStructuredJSON int64 = 0b1000
FlagNoTimestamp int64 = 0b010000
FlagNoLevel int64 = 0b100000
FlagDefault = FlagShowTimestamp | FlagShowLevel
)
// Formatter manages the buffered writing and formatting of log entries
// Formatter manages formatting of log entries
type Formatter struct {
sanitizer *sanitizer.Sanitizer
format string
@@ -73,93 +91,169 @@ func (f *Formatter) ShowTimestamp(show bool) *Formatter {
return f
}
// Format formats a log entry using configured options and explicit flags
// Format formats using configured options resolved against explicit flags.
// Returned slice aliases the internal buffer.
func (f *Formatter) Format(flags int64, timestamp time.Time, level int64, trace string, args []any) []byte {
// Override configured values with explicit flags
effectiveShowTimestamp := (flags&FlagShowTimestamp) != 0 || (flags == 0 && f.showTimestamp)
effectiveShowLevel := (flags&FlagShowLevel) != 0 || (flags == 0 && f.showLevel)
// Build effective flags
effectiveFlags := flags
if effectiveShowTimestamp {
effectiveFlags |= FlagShowTimestamp
}
if effectiveShowLevel {
effectiveFlags |= FlagShowLevel
}
return f.FormatWithOptions(f.format, effectiveFlags, timestamp, level, trace, args)
f.buf = f.AppendFormat(f.buf[:0], flags, timestamp, level, trace, args)
return f.buf
}
// FormatWithOptions formats with explicit format and flags, ignoring configured values
func (f *Formatter) FormatWithOptions(format string, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte {
f.Reset()
// AppendFormat appends a formatted entry to dst using configured options
// resolved against explicit flags. Safe for concurrent use.
func (f *Formatter) AppendFormat(dst []byte, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte {
// CHANGED: D3 — additive resolution replaces `flags == 0` guard
eff := flags &^ (FlagShowTimestamp | FlagShowLevel)
if resolveShow(flags, FlagShowTimestamp, FlagNoTimestamp, f.showTimestamp) {
eff |= FlagShowTimestamp
}
if resolveShow(flags, FlagShowLevel, FlagNoLevel, f.showLevel) {
eff |= FlagShowLevel
}
return f.AppendFormatWithOptions(dst, f.format, eff, timestamp, level, trace, args)
}
func resolveShow(flags, show, no int64, configured bool) bool {
switch {
case flags&no != 0:
return false
case flags&show != 0:
return true
default:
return configured
}
}
// FormatWithOptions formats with explicit format and flags, ignoring
// configured display defaults. Returned slice aliases the internal buffer.
func (f *Formatter) FormatWithOptions(format string, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte {
f.buf = f.AppendFormatWithOptions(f.buf[:0], format, flags, timestamp, level, trace, args)
return f.buf
}
// AppendFormatWithOptions is the allocation-explicit core. Safe for
// concurrent use. Unknown formats fall back to "txt".
func (f *Formatter) AppendFormatWithOptions(dst []byte, format string, flags int64, timestamp time.Time, level int64, trace string, args []any) []byte {
// FlagRaw completely bypasses formatting and sanitization
if flags&FlagRaw != 0 {
for i, arg := range args {
if i > 0 {
f.buf = append(f.buf, ' ')
dst = append(dst, ' ')
}
// Direct conversion without sanitization
switch v := arg.(type) {
case string:
f.buf = append(f.buf, v...)
dst = append(dst, v...)
case []byte:
f.buf = append(f.buf, v...)
dst = append(dst, v...)
case fmt.Stringer:
f.buf = append(f.buf, v.String()...)
dst = append(dst, v.String()...)
case error:
f.buf = append(f.buf, v.Error()...)
dst = append(dst, v.Error()...)
default:
f.buf = append(f.buf, fmt.Sprint(v)...)
dst = append(dst, fmt.Sprint(v)...)
}
}
return f.buf
return dst
}
// Create the serializer based on the effective format
// unknown formats normalize to txt instead of returning nil
format = normalizeFormat(format)
serializer := sanitizer.NewSerializer(format, f.sanitizer)
switch format {
case "raw":
// Raw formatting serializes the arguments and adds NO metadata or newlines
for i, arg := range args {
f.convertValue(&f.buf, arg, serializer, i > 0)
dst = f.appendValue(dst, arg, serializer, i > 0)
}
return f.buf
return dst
case "json":
return f.formatJSON(flags, timestamp, level, trace, args, serializer)
case "txt":
return f.formatTxt(flags, timestamp, level, trace, args, serializer)
return f.appendJSON(dst, flags, timestamp, level, trace, args, serializer)
default: // "txt"
return f.appendTxt(dst, flags, timestamp, level, trace, args, serializer)
}
return nil // forcing panic on unrecognized format
}
// FormatValue formats a single value according to the formatter's configuration
func normalizeFormat(format string) string {
switch format {
case "raw", "json", "txt":
return format
default:
return "txt"
}
}
// FormatValue formats a single value. Returned slice aliases the internal buffer.
func (f *Formatter) FormatValue(v any) []byte {
f.Reset()
serializer := sanitizer.NewSerializer(f.format, f.sanitizer)
f.convertValue(&f.buf, v, serializer, false)
f.buf = f.AppendValue(f.buf[:0], v)
return f.buf
}
// FormatArgs formats multiple arguments as space-separated values
// AppendValue appends a single formatted value to dst. Safe for concurrent use.
// ADDED: R1
func (f *Formatter) AppendValue(dst []byte, v any) []byte {
serializer := sanitizer.NewSerializer(normalizeFormat(f.format), f.sanitizer)
return f.appendValue(dst, v, serializer, false)
}
// FormatArgs formats multiple arguments. Returned slice aliases the internal buffer.
func (f *Formatter) FormatArgs(args ...any) []byte {
f.Reset()
serializer := sanitizer.NewSerializer(f.format, f.sanitizer)
for i, arg := range args {
f.convertValue(&f.buf, arg, serializer, i > 0)
}
f.buf = f.AppendArgs(f.buf[:0], args...)
return f.buf
}
// Reset clears the formatter buffer for reuse
func (f *Formatter) Reset() {
f.buf = f.buf[:0]
// AppendArgs appends multiple space-separated values to dst. Safe for
// concurrent use.
// ADDED: R1
func (f *Formatter) AppendArgs(dst []byte, args ...any) []byte {
serializer := sanitizer.NewSerializer(normalizeFormat(f.format), f.sanitizer)
for i, arg := range args {
dst = f.appendValue(dst, arg, serializer, i > 0)
}
return dst
}
// appendValue provides unified type conversion (was convertValue; now
// value-return style over caller buffer). Type switch body unchanged except
// buffer plumbing — replace every `serializer.WriteX(buf, ...)` with
// `serializer.WriteX(&dst, ...)` and `return dst`.
func (f *Formatter) appendValue(dst []byte, v any, serializer *sanitizer.Serializer, needsSpace bool) []byte {
if needsSpace && len(dst) > 0 {
dst = append(dst, ' ')
}
switch val := v.(type) {
case string:
serializer.WriteString(&dst, val)
case []byte:
serializer.WriteString(&dst, string(val))
case rune:
var runeStr [utf8.UTFMax]byte
n := utf8.EncodeRune(runeStr[:], val)
serializer.WriteString(&dst, string(runeStr[:n]))
case int:
serializer.WriteNumber(&dst, string(strconv.AppendInt(nil, int64(val), 10)))
case int64:
serializer.WriteNumber(&dst, string(strconv.AppendInt(nil, val, 10)))
case uint:
serializer.WriteNumber(&dst, string(strconv.AppendUint(nil, uint64(val), 10)))
case uint64:
serializer.WriteNumber(&dst, string(strconv.AppendUint(nil, val, 10)))
case float32:
serializer.WriteNumber(&dst, string(strconv.AppendFloat(nil, float64(val), 'f', -1, 32)))
case float64:
serializer.WriteNumber(&dst, string(strconv.AppendFloat(nil, val, 'f', -1, 64)))
case bool:
serializer.WriteBool(&dst, val)
case nil:
serializer.WriteNil(&dst)
case time.Time:
serializer.WriteString(&dst, val.Format(f.timestampFormat))
case error:
serializer.WriteString(&dst, val.Error())
case fmt.Stringer:
serializer.WriteString(&dst, val.String())
default:
serializer.WriteComplex(&dst, val)
}
return dst
}
// LevelToString converts integer level values to string
@@ -184,97 +278,34 @@ func LevelToString(level int64) string {
}
}
// convertValue provides unified type conversion
func (f *Formatter) convertValue(buf *[]byte, v any, serializer *sanitizer.Serializer, needsSpace bool) {
if needsSpace && len(*buf) > 0 {
*buf = append(*buf, ' ')
}
switch val := v.(type) {
case string:
serializer.WriteString(buf, val)
case []byte:
serializer.WriteString(buf, string(val))
case rune:
var runeStr [utf8.UTFMax]byte
n := utf8.EncodeRune(runeStr[:], val)
serializer.WriteString(buf, string(runeStr[:n]))
case int:
num := strconv.AppendInt(nil, int64(val), 10)
serializer.WriteNumber(buf, string(num))
case int64:
num := strconv.AppendInt(nil, val, 10)
serializer.WriteNumber(buf, string(num))
case uint:
num := strconv.AppendUint(nil, uint64(val), 10)
serializer.WriteNumber(buf, string(num))
case uint64:
num := strconv.AppendUint(nil, val, 10)
serializer.WriteNumber(buf, string(num))
case float32:
num := strconv.AppendFloat(nil, float64(val), 'f', -1, 32)
serializer.WriteNumber(buf, string(num))
case float64:
num := strconv.AppendFloat(nil, val, 'f', -1, 64)
serializer.WriteNumber(buf, string(num))
case bool:
serializer.WriteBool(buf, val)
case nil:
serializer.WriteNil(buf)
case time.Time:
timeStr := val.Format(f.timestampFormat)
serializer.WriteString(buf, timeStr)
case error:
serializer.WriteString(buf, val.Error())
case fmt.Stringer:
serializer.WriteString(buf, val.String())
default:
serializer.WriteComplex(buf, val)
}
}
// formatJSON unifies JSON output
func (f *Formatter) formatJSON(flags int64, timestamp time.Time, level int64, trace string, args []any, serializer *sanitizer.Serializer) []byte {
f.buf = append(f.buf, '{')
// appendJSON unifies JSON output over a caller-provided buffer
func (f *Formatter) appendJSON(dst []byte, flags int64, timestamp time.Time, level int64, trace string, args []any, serializer *sanitizer.Serializer) []byte {
dst = append(dst, '{')
needsComma := false
if flags&FlagShowTimestamp != 0 {
f.buf = append(f.buf, `"time":"`...)
f.buf = timestamp.AppendFormat(f.buf, f.timestampFormat)
f.buf = append(f.buf, '"')
dst = append(dst, `"time":"`...)
dst = timestamp.AppendFormat(dst, f.timestampFormat)
dst = append(dst, '"')
needsComma = true
}
if flags&FlagShowLevel != 0 {
if needsComma {
f.buf = append(f.buf, ',')
dst = append(dst, ',')
}
f.buf = append(f.buf, `"level":"`...)
f.buf = append(f.buf, LevelToString(level)...)
f.buf = append(f.buf, '"')
dst = append(dst, `"level":"`...)
dst = append(dst, LevelToString(level)...)
dst = append(dst, '"')
needsComma = true
}
if trace != "" {
if needsComma {
f.buf = append(f.buf, ',')
dst = append(dst, ',')
}
f.buf = append(f.buf, `"trace":`...)
serializer.WriteString(&f.buf, trace)
dst = append(dst, `"trace":`...)
serializer.WriteString(&dst, trace)
needsComma = true
}
@@ -283,25 +314,24 @@ func (f *Formatter) formatJSON(flags int64, timestamp time.Time, level int64, tr
if message, ok := args[0].(string); ok {
if fields, ok := args[1].(map[string]any); ok {
if needsComma {
f.buf = append(f.buf, ',')
dst = append(dst, ',')
}
f.buf = append(f.buf, `"message":`...)
serializer.WriteString(&f.buf, message)
dst = append(dst, `"message":`...)
serializer.WriteString(&dst, message)
f.buf = append(f.buf, ',')
f.buf = append(f.buf, `"fields":`...)
dst = append(dst, `,"fields":`...)
marshaledFields, err := json.Marshal(fields)
if err != nil {
f.buf = append(f.buf, `{"_marshal_error":"`...)
serializer.WriteString(&f.buf, err.Error())
f.buf = append(f.buf, `"}`...)
dst = append(dst, `{"_marshal_error":"`...)
serializer.WriteString(&dst, err.Error())
dst = append(dst, `"}`...)
} else {
f.buf = append(f.buf, marshaledFields...)
dst = append(dst, marshaledFields...)
}
f.buf = append(f.buf, '}', '\n')
return f.buf
dst = append(dst, '}', '\n')
return dst
}
}
}
@@ -309,42 +339,42 @@ func (f *Formatter) formatJSON(flags int64, timestamp time.Time, level int64, tr
// Regular JSON with fields array
if len(args) > 0 {
if needsComma {
f.buf = append(f.buf, ',')
dst = append(dst, ',')
}
f.buf = append(f.buf, `"fields":[`...)
dst = append(dst, `"fields":[`...)
for i, arg := range args {
if i > 0 {
f.buf = append(f.buf, ',')
dst = append(dst, ',')
}
f.convertValue(&f.buf, arg, serializer, false)
dst = f.appendValue(dst, arg, serializer, false)
}
f.buf = append(f.buf, ']')
dst = append(dst, ']')
}
f.buf = append(f.buf, '}', '\n')
return f.buf
dst = append(dst, '}', '\n')
return dst
}
// formatTxt handles txt format output
func (f *Formatter) formatTxt(flags int64, timestamp time.Time, level int64, trace string, args []any, serializer *sanitizer.Serializer) []byte {
// appendTxt handles txt format output over a caller-provided buffer
func (f *Formatter) appendTxt(dst []byte, flags int64, timestamp time.Time, level int64, trace string, args []any, serializer *sanitizer.Serializer) []byte {
needsSpace := false
if flags&FlagShowTimestamp != 0 {
f.buf = timestamp.AppendFormat(f.buf, f.timestampFormat)
dst = timestamp.AppendFormat(dst, f.timestampFormat)
needsSpace = true
}
if flags&FlagShowLevel != 0 {
if needsSpace {
f.buf = append(f.buf, ' ')
dst = append(dst, ' ')
}
f.buf = append(f.buf, LevelToString(level)...)
dst = append(dst, LevelToString(level)...)
needsSpace = true
}
if trace != "" {
if needsSpace {
f.buf = append(f.buf, ' ')
dst = append(dst, ' ')
}
// Sanitize trace to prevent terminal control sequence injection
traceHandler := sanitizer.NewSerializer("txt", f.sanitizer)
@@ -352,18 +382,24 @@ func (f *Formatter) formatTxt(flags int64, timestamp time.Time, level int64, tra
traceHandler.WriteString(&tempBuf, trace)
// Extract content without quotes if added by txt serializer
if len(tempBuf) > 2 && tempBuf[0] == '"' && tempBuf[len(tempBuf)-1] == '"' {
f.buf = append(f.buf, tempBuf[1:len(tempBuf)-1]...)
dst = append(dst, tempBuf[1:len(tempBuf)-1]...)
} else {
f.buf = append(f.buf, tempBuf...)
dst = append(dst, tempBuf...)
}
needsSpace = true
}
for _, arg := range args {
f.convertValue(&f.buf, arg, serializer, needsSpace)
dst = f.appendValue(dst, arg, serializer, needsSpace)
needsSpace = true
}
f.buf = append(f.buf, '\n')
return f.buf
}
dst = append(dst, '\n')
return dst
}
// Reset clears the internal buffer for reuse
func (f *Formatter) Reset() {
f.buf = f.buf[:0]
}
formatter/formatter_test.go
+111 -2
View File
@@ -1,9 +1,11 @@
package formatter
import (
"bytes"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
@@ -97,7 +99,9 @@ func TestFormatter(t *testing.T) {
})
t.Run("special characters escaping", func(t *testing.T) {
s := sanitizer.New().Policy(sanitizer.PolicyJSON)
// PolicyRaw — transport escaping applies exactly once.
// PolicyJSON + json format double-escapes (see TestJSONSanitizerLayering).
s := sanitizer.New().Policy(sanitizer.PolicyRaw)
f := New(s).Type("json")
data := f.Format(FlagDefault, timestamp, 0, "",
@@ -119,6 +123,111 @@ func TestFormatter(t *testing.T) {
})
}
func TestJSONUTF8Passthrough(t *testing.T) {
timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
f := New(sanitizer.New()).Type("json")
in := "héllo 世界 ✓"
data := f.Format(FlagDefault, timestamp, 0, "", []any{in})
var result map[string]any
require.NoError(t, json.Unmarshal(bytes.TrimSuffix(data, []byte("\n")), &result))
assert.Equal(t, in, result["fields"].([]any)[0])
assert.NotContains(t, string(data), `\u00`, "no per-byte escapes of UTF-8")
}
func TestJSONSanitizerLayering(t *testing.T) {
timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
// Content transform (PolicyTxt) applied before transport escaping
f := New(sanitizer.New().Policy(sanitizer.PolicyTxt)).Type("json")
data := f.Format(FlagDefault, timestamp, 0, "", []any{"a\x07b"})
var result map[string]any
require.NoError(t, json.Unmarshal(bytes.TrimSuffix(data, []byte("\n")), &result))
assert.Equal(t, "a<07>b", result["fields"].([]any)[0])
// PolicyJSON + json format: content transform emits literal backslash
// sequences; transport escaping preserves them (double-escape by design)
f2 := New(sanitizer.New().Policy(sanitizer.PolicyJSON)).Type("json")
data2 := f2.Format(FlagDefault, timestamp, 0, "", []any{"a\nb"})
require.NoError(t, json.Unmarshal(bytes.TrimSuffix(data2, []byte("\n")), &result))
assert.Equal(t, `a\nb`, result["fields"].([]any)[0])
}
func TestFlagResolution(t *testing.T) {
timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
f := New(sanitizer.New()).Type("txt").ShowTimestamp(true).ShowLevel(true)
// Non-display flags alone inherit configured defaults
str := string(f.Format(FlagStructuredJSON, timestamp, 0, "", []any{"m"}))
assert.Contains(t, str, "2024-01-01")
assert.Contains(t, str, "INFO")
// Explicit suppression
str = string(f.Format(FlagNoLevel, timestamp, 0, "", []any{"m"}))
assert.Contains(t, str, "2024-01-01")
assert.NotContains(t, str, "INFO")
str = string(f.Format(FlagNoTimestamp|FlagNoLevel, timestamp, 0, "", []any{"m"}))
assert.NotContains(t, str, "2024-01-01")
assert.NotContains(t, str, "INFO")
// FormatWithOptions is fully explicit: unset Show bits mean off
str = string(f.FormatWithOptions("txt", 0, timestamp, 0, "", []any{"m"}))
assert.NotContains(t, str, "INFO")
}
func TestUnknownFormatFallback(t *testing.T) {
timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
f := New(sanitizer.New()).Type("txt")
data := f.FormatWithOptions("xml", FlagShowLevel, timestamp, 8, "", []any{"boom"})
require.NotNil(t, data)
assert.Contains(t, string(data), "ERROR")
assert.Contains(t, string(data), "boom")
}
func TestReturnedSliceInvalidation(t *testing.T) {
timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
f := New(sanitizer.New()).Type("txt").ShowTimestamp(false).ShowLevel(false)
first := f.Format(0, timestamp, 0, "", []any{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
snapshot := string(first)
_ = f.Format(0, timestamp, 0, "", []any{"b"})
assert.NotEqual(t, snapshot, string(first),
"buffered Format output is invalidated by the next buffered call")
}
func TestAppendFormatStable(t *testing.T) {
timestamp := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
f := New(sanitizer.New()).Type("txt").ShowTimestamp(false).ShowLevel(false)
first := f.AppendFormat(nil, 0, timestamp, 0, "", []any{"first-payload"})
snapshot := string(first)
_ = f.Format(0, timestamp, 0, "", []any{"interleaved-buffered-call"})
second := f.AppendFormat(nil, 0, timestamp, 0, "", []any{"second"})
assert.Equal(t, snapshot, string(first), "caller-owned buffer unaffected by buffered calls")
assert.Equal(t, "second\n", string(second))
}
func TestFormatterConcurrentAppend(t *testing.T) {
f := New(sanitizer.New().Policy(sanitizer.PolicyTxt)).Type("json")
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for j := 0; j < 200; j++ {
out := f.AppendFormat(nil, FlagDefault, time.Now(), 0, "", []any{"w", id, "i", j, "s", "x\x00y"})
if !json.Valid(bytes.TrimSuffix(out, []byte("\n"))) {
t.Errorf("invalid JSON: %s", out)
return
}
}
}(i)
}
wg.Wait()
}
func TestLevelToString(t *testing.T) {
tests := []struct {
level int64
@@ -139,4 +248,4 @@ func TestLevelToString(t *testing.T) {
assert.Equal(t, tt.expected, LevelToString(tt.level))
})
}
}
}