lixen/logwisp
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.18.0 raw and json formatter improvements, dockerfile, go bump to 1.27.1

Browse Source
This commit is contained in:
lixen
2026-09-08 23:33:39 -04:00
parent dd665bb339
commit 5934a2e35f
12 changed files with 306 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+2
View File
@@ -198,6 +198,8 @@ type FileSourceOptions struct {
Directory string `toml:"directory"`
Pattern string `toml:"pattern"` // glob pattern
CheckIntervalMS int64 `toml:"check_interval_ms"`
Raw bool `toml:"raw"` // keep the whole line as the message, never parse it
From string `toml:"from"` // "end" (default) or "start" of a newly discovered file
}
// ConsoleSourceOptions defines settings for a stdin-based source
internal/format/adapter.go
+30 -43
View File
@@ -90,57 +90,44 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
// Format implements Formatter interface
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
// Map logwisp LogEntry to formatter args
level := mapLevel(entry.Level)
// syslog-style origin prefix for chained entries
src := sourceLabel(entry)
// Build args based on whether we have structured fields
var args []any
effectiveFlags := a.flags
if len(entry.Fields) > 0 {
// Parse fields JSON
var fields map[string]any
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
// Use structured JSON format for fields
args = []any{entry.Message, fields}
// Add structured flag to properly format fields as JSON object
effectiveFlags |= formatter.FlagStructuredJSON
return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil
}
}
if args == nil {
args = []any{entry.Message}
}
a.mu.Lock()
out := bytes.Clone(a.formatter.Format(effectiveFlags, entry.Time, level, src, args))
a.mu.Unlock()
return out, nil
return a.serialize(entry, a.flags), nil
}
// FormatWithFlags allows custom flags for specific formatting needs
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
level := mapLevel(entry.Level)
src := sourceLabel(entry)
return a.serialize(entry, customFlags), nil
}
var args []any
if len(entry.Fields) > 0 {
var fields map[string]any
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
args = []any{entry.Message, fields}
customFlags |= formatter.FlagStructuredJSON
}
}
if args == nil {
args = []any{entry.Message}
}
// serialize renders an entry under the given flags. The returned slice is a
// copy: the underlying formatter reuses one buffer and sinks retain payloads.
func (a *FormatterAdapter) serialize(entry core.LogEntry, flags int64) []byte {
args, flags := formatArgs(entry, flags)
a.mu.Lock()
out := bytes.Clone(a.formatter.Format(customFlags, entry.Time, level, src, args))
out := bytes.Clone(a.formatter.Format(flags, entry.Time, mapLevel(entry.Level), sourceLabel(entry), args))
a.mu.Unlock()
return out, nil
return out
}
// formatArgs pairs the entry with its flags. FlagRaw keeps the fields JSON
// verbatim beside the message rather than silently overriding the caller's
// choice of passthrough; every other mode renders it as a JSON object.
func formatArgs(entry core.LogEntry, flags int64) ([]any, int64) {
if len(entry.Fields) == 0 {
return []any{entry.Message}, flags
}
if flags&formatter.FlagRaw != 0 {
if entry.Message == "" {
return []any{[]byte(entry.Fields)}, flags
}
return []any{entry.Message, []byte(entry.Fields)}, flags
}
var fields map[string]any
if err := json.Unmarshal(entry.Fields, &fields); err != nil || len(fields) == 0 {
return []any{entry.Message}, flags
}
return []any{entry.Message, fields}, flags | formatter.FlagStructuredJSON
}
// Name returns formatter type
internal/source/file/file.go
+10 -2
View File
@@ -61,6 +61,7 @@ const (
DefaultFileSourcePattern = "*"
DefaultFileSourceCheckIntervalMS = 100
MinFileSourceCheckIntervalMS = 10
DefaultFileSourceFrom = "end"
)
// NewFileSourcePlugin creates a file source through plugin factory
@@ -90,6 +91,11 @@ func NewFileSourcePlugin(
} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
}
if opts.From == "" {
opts.From = DefaultFileSourceFrom
} else if err := lconfig.OneOf("start", "end")(opts.From); err != nil {
return nil, fmt.Errorf("from: %w", err)
}
// Create and return plugin instance
fs := &FileSource{
@@ -116,7 +122,9 @@ func NewFileSourcePlugin(
"component", "file_source",
"instance_id", id,
"directory", opts.Directory,
"pattern", opts.Pattern)
"pattern", opts.Pattern,
"raw", opts.Raw,
"from", opts.From)
return fs, nil
}
@@ -261,7 +269,7 @@ func (fs *FileSource) ensureWatcher(path string) {
return
}
w := newFileWatcher(path, fs.publish, fs.logger)
w := newFileWatcher(path, fs.config.Raw, fs.config.From == "start", fs.publish, fs.logger)
fs.watchers[path] = w
fs.logger.Debug("msg", "Created file watcher",
internal/source/file/file_watcher.go
+67 -30
View File
@@ -34,6 +34,7 @@ type WatcherInfo struct {
type fileWatcher struct {
directory string
callback func(core.LogEntry)
raw bool
position int64
size int64
inode uint64
@@ -46,12 +47,18 @@ type fileWatcher struct {
logger *log.Logger
}
// newFileWatcher creates a new watcher for a specific file path
func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
// newFileWatcher creates a new watcher for a specific file path.
// A start position of 0 reads an existing file whole; -1 seeks to its end.
func newFileWatcher(directory string, raw, fromStart bool, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
position := int64(-1)
if fromStart {
position = 0
}
w := &fileWatcher{
directory: directory,
callback: callback,
position: -1,
raw: raw,
position: position,
logger: logger,
}
w.lastReadTime.Store(time.Time{})
@@ -60,8 +67,8 @@ func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.
// watch starts the main monitoring loop for the file
func (w *fileWatcher) watch(ctx context.Context) error {
if err := w.seekToEnd(); err != nil {
return fmt.Errorf("seekToEnd failed: %w", err)
if err := w.initPosition(); err != nil {
return fmt.Errorf("initPosition failed: %w", err)
}
ticker := time.NewTicker(core.FileWatcherPollInterval)
@@ -297,8 +304,9 @@ func (w *fileWatcher) checkFile() error {
return nil
}
// seekToEnd sets the initial read position to the end of the file
func (w *fileWatcher) seekToEnd() error {
// initPosition records the file's metadata and, unless the watcher was created
// to read from the start, sets the initial read position to the end
func (w *fileWatcher) initPosition() error {
file, err := os.Open(w.directory)
if err != nil {
if os.IsNotExist(err) {
@@ -322,8 +330,6 @@ func (w *fileWatcher) seekToEnd() error {
w.mu.Lock()
defer w.mu.Unlock()
// Keep existing position (including 0)
// First time initialization seeks to the end of the file
if w.position == -1 {
pos, err := file.Seek(0, io.SeekEnd)
if err != nil {
@@ -348,36 +354,67 @@ func (w *fileWatcher) isStopped() bool {
return w.stopped
}
// parseLine attempts to parse a line as JSON, falling back to plain text
// parseLine converts a line into an entry, as JSON when nothing would be lost
func (w *fileWatcher) parseLine(line string) core.LogEntry {
var jsonLog struct {
Time string `json:"time"`
Level string `json:"level"`
Message string `json:"msg"`
Fields json.RawMessage `json:"fields"`
}
if err := json.Unmarshal([]byte(line), &jsonLog); err == nil {
timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time)
if err != nil {
timestamp = time.Now()
}
if w.raw {
// Newline restored: sinks write the payload as it stands
return core.LogEntry{
Time: timestamp,
Time: time.Now(),
Source: filepath.Base(w.directory),
Level: jsonLog.Level,
Message: jsonLog.Message,
Fields: jsonLog.Fields,
Level: source.ExtractLogLevel(line),
Message: line + "\n",
}
}
level := source.ExtractLogLevel(line)
if entry, ok := w.parseJSON(line); ok {
return entry
}
return core.LogEntry{
Time: time.Now(),
Source: filepath.Base(w.directory),
Level: level,
Level: source.ExtractLogLevel(line),
Message: line,
}
}
}
// parseJSON decodes a line into the entry envelope. A top-level key LogEntry
// cannot carry refuses the whole line, so a richer record reaches the pipeline
// as text rather than silently reduced to the four keys kept here.
func (w *fileWatcher) parseJSON(line string) (core.LogEntry, bool) {
if len(line) == 0 || line[0] != '{' {
return core.LogEntry{}, false
}
var obj map[string]json.RawMessage
if err := json.Unmarshal([]byte(line), &obj); err != nil || len(obj) == 0 {
return core.LogEntry{}, false
}
entry := core.LogEntry{Time: time.Now(), Source: filepath.Base(w.directory)}
for key, val := range obj {
var err error
switch key {
case "time":
var ts string
if json.Unmarshal(val, &ts) == nil {
if t, terr := time.Parse(time.RFC3339Nano, ts); terr == nil {
entry.Time = t
}
}
case "level":
err = json.Unmarshal(val, &entry.Level)
case "msg":
err = json.Unmarshal(val, &entry.Message)
case "fields":
entry.Fields = val
default:
return core.LogEntry{}, false
}
if err != nil {
return core.LogEntry{}, false
}
}
return entry, true
}