lixen/logwisp
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.7.1 default config and documentation update, refactor

Browse Source
This commit is contained in:
lixen
2025-10-10 13:03:03 -04:00
parent 89e6a4ea05
commit 33bf36f27e
34 changed files with 2877 additions and 2794 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cmd/logwisp/reload.go
+13 -15
View File
@@ -62,18 +62,11 @@ func (rm *ReloadManager) Start(ctx context.Context) error {
rm.startStatusReporter(ctx, svc)
}
// Create lconfig instance for file watching, logwisp config is always TOML
lcfg, err := lconfig.NewBuilder().
WithFile(rm.configPath).
WithTarget(rm.cfg).
WithFileFormat("toml").
WithSecurityOptions(lconfig.SecurityOptions{
PreventPathTraversal: true,
MaxFileSize: 10 * 1024 * 1024,
}).
Build()
if err != nil {
return fmt.Errorf("failed to create config watcher: %w", err)
// Use the same lconfig instance from initial load
lcfg := config.GetConfigManager()
if lcfg == nil {
// Config manager not initialized - potential for config bypass
return fmt.Errorf("config manager not initialized - cannot enable hot reload")
}
rm.lcfg = lcfg
@@ -83,7 +76,7 @@ func (rm *ReloadManager) Start(ctx context.Context) error {
PollInterval: time.Second,
Debounce: 500 * time.Millisecond,
ReloadTimeout: 30 * time.Second,
VerifyPermissions: true, // TODO: Prevent malicious config replacement, to be implemented
VerifyPermissions: true,
}
lcfg.AutoUpdateWithOptions(watchOpts)
@@ -243,8 +236,14 @@ func (rm *ReloadManager) performReload(ctx context.Context) error {
return fmt.Errorf("failed to get updated config: %w", err)
}
// AsStruct returns the target pointer, not a new instance
newCfg := updatedCfg.(*config.Config)
// Validate the new config
if err := config.ValidateConfig(newCfg); err != nil {
return fmt.Errorf("updated config validation failed: %w", err)
}
// Get current service snapshot
rm.mu.RLock()
oldService := rm.service
@@ -267,8 +266,7 @@ func (rm *ReloadManager) performReload(ctx context.Context) error {
// Stop old status reporter and start new one
rm.restartStatusReporter(ctx, newService)
// Gracefully shutdown old services
// This happens after the swap to minimize downtime
// Gracefully shutdown old services after swap to minimize downtime
go rm.shutdownOldServices(oldService)
return nil
src/internal/auth/authenticator.go
+2
View File
@@ -29,6 +29,8 @@ type Authenticator struct {
sessionMu sync.RWMutex
}
// TODO: only one connection per user, token, mtls
// TODO: implement tracker logic
// Represents an authenticated connection
type Session struct {
ID string
src/internal/config/config.go
+22 -29
View File
@@ -13,11 +13,11 @@ type Config struct {
DisableStatusReporter bool `toml:"disable_status_reporter"`
ConfigAutoReload bool `toml:"config_auto_reload"`
// Internal flag indicating demonized child process
BackgroundDaemon bool `toml:"background-daemon"`
// Internal flag indicating demonized child process (DO NOT SET IN CONFIG FILE)
BackgroundDaemon bool
// Configuration file path
ConfigFile string `toml:"config"`
ConfigFile string `toml:"config_file"`
// Existing fields
Logging *LogConfig `toml:"logging"`
@@ -83,18 +83,16 @@ type PipelineConfig struct {
// Common configuration structs used across components
type NetLimitConfig struct {
Enabled bool `toml:"enabled"`
MaxConnections int64 `toml:"max_connections"`
RequestsPerSecond float64 `toml:"requests_per_second"`
BurstSize int64 `toml:"burst_size"`
ResponseMessage string `toml:"response_message"`
ResponseCode int64 `toml:"response_code"` // Default: 429
MaxConnectionsPerIP int64 `toml:"max_connections_per_ip"`
MaxConnectionsPerUser int64 `toml:"max_connections_per_user"`
MaxConnectionsPerToken int64 `toml:"max_connections_per_token"`
MaxConnectionsTotal int64 `toml:"max_connections_total"`
IPWhitelist []string `toml:"ip_whitelist"`
IPBlacklist []string `toml:"ip_blacklist"`
Enabled bool `toml:"enabled"`
MaxConnections int64 `toml:"max_connections"`
RequestsPerSecond float64 `toml:"requests_per_second"`
BurstSize int64 `toml:"burst_size"`
ResponseMessage string `toml:"response_message"`
ResponseCode int64 `toml:"response_code"` // Default: 429
MaxConnectionsPerIP int64 `toml:"max_connections_per_ip"`
MaxConnectionsTotal int64 `toml:"max_connections_total"`
IPWhitelist []string `toml:"ip_whitelist"`
IPBlacklist []string `toml:"ip_blacklist"`
}
type TLSConfig struct {
@@ -120,7 +118,7 @@ type TLSConfig struct {
type HeartbeatConfig struct {
Enabled bool `toml:"enabled"`
Interval int64 `toml:"interval_ms"`
IntervalMS int64 `toml:"interval_ms"`
IncludeTimestamp bool `toml:"include_timestamp"`
IncludeStats bool `toml:"include_stats"`
Format string `toml:"format"`
@@ -149,10 +147,7 @@ type DirectorySourceOptions struct {
Path string `toml:"path"`
Pattern string `toml:"pattern"` // glob pattern
CheckIntervalMS int64 `toml:"check_interval_ms"`
Recursive bool `toml:"recursive"`
FollowSymlinks bool `toml:"follow_symlinks"`
DeleteAfterRead bool `toml:"delete_after_read"`
MoveToDirectory string `toml:"move_to_directory"` // move after processing
Recursive bool `toml:"recursive"` // TODO: implement logic
}
type StdinSourceOptions struct {
@@ -204,9 +199,8 @@ type ConsoleSinkOptions struct {
}
type FileSinkOptions struct {
Directory string `toml:"directory"`
Name string `toml:"name"`
// Extension string `toml:"extension"`
Directory string `toml:"directory"`
Name string `toml:"name"`
MaxSizeMB int64 `toml:"max_size_mb"`
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
MinDiskFreeMB int64 `toml:"min_disk_free_mb"`
@@ -242,7 +236,6 @@ type TCPSinkOptions struct {
type HTTPClientSinkOptions struct {
URL string `toml:"url"`
Headers map[string]string `toml:"headers"`
BufferSize int64 `toml:"buffer_size"`
BatchSize int64 `toml:"batch_size"`
BatchDelayMS int64 `toml:"batch_delay_ms"`
@@ -322,12 +315,12 @@ type FilterConfig struct {
type FormatConfig struct {
// Format configuration - polymorphic like sources/sinks
Type string `toml:"type"` // "json", "text", "raw"
Type string `toml:"type"` // "json", "txt", "raw"
// Only one will be populated based on format type
JSONFormatOptions *JSONFormatterOptions `toml:"json_format,omitempty"`
TextFormatOptions *TextFormatterOptions `toml:"text_format,omitempty"`
RawFormatOptions *RawFormatterOptions `toml:"raw_format,omitempty"`
JSONFormatOptions *JSONFormatterOptions `toml:"json,omitempty"`
TxtFormatOptions *TxtFormatterOptions `toml:"txt,omitempty"`
RawFormatOptions *RawFormatterOptions `toml:"raw,omitempty"`
}
type JSONFormatterOptions struct {
@@ -338,7 +331,7 @@ type JSONFormatterOptions struct {
SourceField string `toml:"source_field"`
}
type TextFormatterOptions struct {
type TxtFormatterOptions struct {
Template string `toml:"template"`
TimestampFormat string `toml:"timestamp_format"`
}
src/internal/config/loader.go
+8 -5
View File
@@ -13,6 +13,11 @@ import (
var configManager *lconfig.Config
// Hot reload access
func GetConfigManager() *lconfig.Config {
return configManager
}
func defaults() *Config {
return &Config{
// Top-level flag defaults
@@ -79,7 +84,7 @@ func Load(args []string) (*Config, error) {
// Create target config instance that will be populated
finalConfig := &Config{}
// The builder now handles loading, populating the target struct, and validation
// Builder handles loading, populating the target struct, and validation
cfg, err := lconfig.NewBuilder().
WithTarget(finalConfig). // Typed target struct
WithDefaults(defaults()). // Default values
@@ -94,7 +99,7 @@ func Load(args []string) (*Config, error) {
WithArgs(args). // Command-line arguments
WithFile(configPath). // TOML config file
WithFileFormat("toml"). // Explicit format
WithTypedValidator(validateConfig). // Centralized validation
WithTypedValidator(ValidateConfig). // Centralized validation
WithSecurityOptions(lconfig.SecurityOptions{
PreventPathTraversal: true,
MaxFileSize: 10 * 1024 * 1024, // 10MB max config
@@ -117,9 +122,7 @@ func Load(args []string) (*Config, error) {
finalConfig.ConfigFile = configPath
// Store the manager for hot reload
if cfg != nil {
configManager = cfg
}
configManager = cfg
return finalConfig, nil
}
src/internal/config/validation.go
+8 -16
View File
@@ -13,7 +13,7 @@ import (
// validateConfig is the centralized validator for the entire configuration
// This replaces the old (c *Config) validate() method
func validateConfig(cfg *Config) error {
func ValidateConfig(cfg *Config) error {
if cfg == nil {
return fmt.Errorf("config is nil")
}
@@ -599,14 +599,6 @@ func validateHTTPClientSink(pipelineName string, index int, opts *HTTPClientSink
if opts.RetryBackoff < 1.0 {
opts.RetryBackoff = 2.0
}
if opts.Headers == nil {
opts.Headers = make(map[string]string)
}
// Set default Content-Type if not specified
if _, exists := opts.Headers["Content-Type"]; !exists {
opts.Headers["Content-Type"] = "application/json"
}
// Validate auth configuration
if opts.Auth != nil {
@@ -748,20 +740,20 @@ func validateFormatterConfig(p *PipelineConfig) error {
}
case "txt":
if p.Format.TextFormatOptions == nil {
p.Format.TextFormatOptions = &TextFormatterOptions{}
if p.Format.TxtFormatOptions == nil {
p.Format.TxtFormatOptions = &TxtFormatterOptions{}
}
// Default template format
templateStr := "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}{{ if .Fields }} {{.Fields}}{{ end }}"
if p.Format.TextFormatOptions.Template != "" {
p.Format.TextFormatOptions.Template = templateStr
if p.Format.TxtFormatOptions.Template != "" {
p.Format.TxtFormatOptions.Template = templateStr
}
// Default timestamp format
timestampFormat := time.RFC3339
if p.Format.TextFormatOptions.TimestampFormat != "" {
p.Format.TextFormatOptions.TimestampFormat = timestampFormat
if p.Format.TxtFormatOptions.TimestampFormat != "" {
p.Format.TxtFormatOptions.TimestampFormat = timestampFormat
}
case "json":
@@ -810,7 +802,7 @@ func validateHeartbeat(pipelineName, location string, hb *HeartbeatConfig) error
return nil // Skip validation if disabled
}
if hb.Interval < 1000 { // At least 1 second
if hb.IntervalMS < 1000 { // At least 1 second
return fmt.Errorf("pipeline '%s' %s: heartbeat interval must be at least 1000ms", pipelineName, location)
}
src/internal/format/format.go
+2 -2
View File
@@ -3,8 +3,8 @@ package format
import (
"fmt"
"logwisp/src/internal/config"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"github.com/lixenwraith/log"
@@ -25,7 +25,7 @@ func NewFormatter(cfg *config.FormatConfig, logger *log.Logger) (Formatter, erro
case "json":
return NewJSONFormatter(cfg.JSONFormatOptions, logger)
case "txt":
return NewTextFormatter(cfg.TextFormatOptions, logger)
return NewTxtFormatter(cfg.TxtFormatOptions, logger)
case "raw", "":
return NewRawFormatter(cfg.RawFormatOptions, logger)
default:
src/internal/format/text.go → src/internal/format/txt.go
+9 -9
View File
@@ -1,29 +1,29 @@
// FILE: logwisp/src/internal/format/text.go
// FILE: logwisp/src/internal/format/txt.go
package format
import (
"bytes"
"fmt"
"logwisp/src/internal/config"
"strings"
"text/template"
"time"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"github.com/lixenwraith/log"
)
// Produces human-readable text logs using templates
type TextFormatter struct {
config *config.TextFormatterOptions
type TxtFormatter struct {
config *config.TxtFormatterOptions
template *template.Template
logger *log.Logger
}
// Creates a new text formatter
func NewTextFormatter(opts *config.TextFormatterOptions, logger *log.Logger) (*TextFormatter, error) {
f := &TextFormatter{
func NewTxtFormatter(opts *config.TxtFormatterOptions, logger *log.Logger) (*TxtFormatter, error) {
f := &TxtFormatter{
config: opts,
logger: logger,
}
@@ -48,7 +48,7 @@ func NewTextFormatter(opts *config.TextFormatterOptions, logger *log.Logger) (*T
}
// Formats the log entry using the template
func (f *TextFormatter) Format(entry core.LogEntry) ([]byte, error) {
func (f *TxtFormatter) Format(entry core.LogEntry) ([]byte, error) {
// Prepare data for template
data := map[string]any{
"Timestamp": entry.Time,
@@ -71,7 +71,7 @@ func (f *TextFormatter) Format(entry core.LogEntry) ([]byte, error) {
if err := f.template.Execute(&buf, data); err != nil {
// Fallback: return a basic formatted message
f.logger.Debug("msg", "Template execution failed, using fallback",
"component", "text_formatter",
"component", "txt_formatter",
"error", err)
fallback := fmt.Sprintf("[%s] [%s] %s - %s\n",
@@ -92,6 +92,6 @@ func (f *TextFormatter) Format(entry core.LogEntry) ([]byte, error) {
}
// Returns the formatter name
func (f *TextFormatter) Name() string {
func (f *TxtFormatter) Name() string {
return "txt"
}
src/internal/limit/net.go
+3 -61
View File
@@ -140,8 +140,6 @@ func NewNetLimiter(cfg *config.NetLimitConfig, logger *log.Logger) *NetLimiter {
"requests_per_second", cfg.RequestsPerSecond,
"burst_size", cfg.BurstSize,
"max_connections_per_ip", cfg.MaxConnectionsPerIP,
"max_connections_per_user", cfg.MaxConnectionsPerUser,
"max_connections_per_token", cfg.MaxConnectionsPerToken,
"max_connections_total", cfg.MaxConnectionsTotal)
return l
@@ -609,10 +607,8 @@ func (l *NetLimiter) GetStats() map[string]any {
"tracked_tokens": tokenConnTrackers,
// Configuration limits (0 = disabled)
"limit_per_ip": l.config.MaxConnectionsPerIP,
"limit_per_user": l.config.MaxConnectionsPerUser,
"limit_per_token": l.config.MaxConnectionsPerToken,
"limit_total": l.config.MaxConnectionsTotal,
"limit_per_ip": l.config.MaxConnectionsPerIP,
"limit_total": l.config.MaxConnectionsTotal,
},
}
}
@@ -807,7 +803,7 @@ func (l *NetLimiter) TrackConnection(ip string, user string, token string) bool
l.logger.Debug("msg", "TCP connection blocked by total limit",
"component", "netlimit",
"current_total", currentTotal,
"max_total", l.config.MaxConnectionsTotal)
"max_connections_total", l.config.MaxConnectionsTotal)
return false
}
}
@@ -830,42 +826,6 @@ func (l *NetLimiter) TrackConnection(ip string, user string, token string) bool
}
}
// Check per-user connection limit (0 = disabled)
if l.config.MaxConnectionsPerUser > 0 && user != "" {
tracker, exists := l.userConnections[user]
if !exists {
tracker = &connTracker{lastSeen: time.Now()}
l.userConnections[user] = tracker
}
if tracker.connections.Load() >= l.config.MaxConnectionsPerUser {
l.blockedByConnLimit.Add(1)
l.logger.Debug("msg", "TCP connection blocked by user limit",
"component", "netlimit",
"user", user,
"current", tracker.connections.Load(),
"max", l.config.MaxConnectionsPerUser)
return false
}
}
// Check per-token connection limit (0 = disabled)
if l.config.MaxConnectionsPerToken > 0 && token != "" {
tracker, exists := l.tokenConnections[token]
if !exists {
tracker = &connTracker{lastSeen: time.Now()}
l.tokenConnections[token] = tracker
}
if tracker.connections.Load() >= l.config.MaxConnectionsPerToken {
l.blockedByConnLimit.Add(1)
l.logger.Debug("msg", "TCP connection blocked by token limit",
"component", "netlimit",
"token", token,
"current", tracker.connections.Load(),
"max", l.config.MaxConnectionsPerToken)
return false
}
}
// All checks passed, increment counters
l.totalConnections.Add(1)
@@ -878,24 +838,6 @@ func (l *NetLimiter) TrackConnection(ip string, user string, token string) bool
}
}
if user != "" && l.config.MaxConnectionsPerUser > 0 {
if tracker, exists := l.userConnections[user]; exists {
tracker.connections.Add(1)
tracker.mu.Lock()
tracker.lastSeen = time.Now()
tracker.mu.Unlock()
}
}
if token != "" && l.config.MaxConnectionsPerToken > 0 {
if tracker, exists := l.tokenConnections[token]; exists {
tracker.connections.Add(1)
tracker.mu.Lock()
tracker.lastSeen = time.Now()
tracker.mu.Unlock()
}
}
return true
}
src/internal/sink/http.go
+5 -5
View File
@@ -205,7 +205,7 @@ func (h *HTTPSink) brokerLoop(ctx context.Context) {
var tickerChan <-chan time.Time
if h.config.Heartbeat != nil && h.config.Heartbeat.Enabled {
ticker = time.NewTicker(time.Duration(h.config.Heartbeat.Interval) * time.Second)
ticker = time.NewTicker(time.Duration(h.config.Heartbeat.IntervalMS) * time.Millisecond)
tickerChan = ticker.C
defer ticker.Stop()
}
@@ -545,7 +545,7 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx, session *auth.Session)
var tickerChan <-chan time.Time
if h.config.Heartbeat != nil && h.config.Heartbeat.Enabled {
ticker = time.NewTicker(time.Duration(h.config.Heartbeat.Interval) * time.Second)
ticker = time.NewTicker(time.Duration(h.config.Heartbeat.IntervalMS) * time.Millisecond)
tickerChan = ticker.C
defer ticker.Stop()
}
@@ -698,9 +698,9 @@ func (h *HTTPSink) handleStatus(ctx *fasthttp.RequestCtx) {
},
"features": map[string]any{
"heartbeat": map[string]any{
"enabled": h.config.Heartbeat.Enabled,
"interval": h.config.Heartbeat.Interval,
"format": h.config.Heartbeat.Format,
"enabled": h.config.Heartbeat.Enabled,
"interval_ms": h.config.Heartbeat.IntervalMS,
"format": h.config.Heartbeat.Format,
},
"tls": tlsStats,
"auth": authStats,
src/internal/sink/http_client.go
+1 -5
View File
@@ -24,6 +24,7 @@ import (
"github.com/valyala/fasthttp"
)
// TODO: implement heartbeat for HTTP Client Sink, similar to HTTP Sink
// Forwards log entries to a remote HTTP endpoint
type HTTPClientSink struct {
input chan core.LogEntry
@@ -340,11 +341,6 @@ func (h *HTTPClientSink) sendBatch(batch []core.LogEntry) {
// No authentication
}
// Set headers
for k, v := range h.config.Headers {
req.Header.Set(k, v)
}
// Send request
err := h.client.DoTimeout(req, resp, time.Duration(h.config.Timeout)*time.Second)
src/internal/sink/tcp.go
+1 -1
View File
@@ -205,7 +205,7 @@ func (t *TCPSink) broadcastLoop(ctx context.Context) {
var tickerChan <-chan time.Time
if t.config.Heartbeat != nil && t.config.Heartbeat.Enabled {
ticker = time.NewTicker(time.Duration(t.config.Heartbeat.Interval) * time.Second)
ticker = time.NewTicker(time.Duration(t.config.Heartbeat.IntervalMS) * time.Millisecond)
tickerChan = ticker.C
defer ticker.Stop()
}
src/internal/sink/tcp_client.go
+2 -1
View File
@@ -7,7 +7,6 @@ import (
"encoding/json"
"errors"
"fmt"
"logwisp/src/internal/auth"
"net"
"strconv"
"strings"
@@ -15,6 +14,7 @@ import (
"sync/atomic"
"time"
"logwisp/src/internal/auth"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
@@ -22,6 +22,7 @@ import (
"github.com/lixenwraith/log"
)
// TODO: implement heartbeat for TCP Client Sink, similar to TCP Sink
// Forwards log entries to a remote TCP endpoint
type TCPClientSink struct {
input chan core.LogEntry