v0.13.1 folder restructure, test script added, format adapter async fix
This commit is contained in:
@@ -0,0 +1,98 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"logwisp/internal/core"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProtocolVersion is declared in the hello preamble
|
||||
const ProtocolVersion = 1
|
||||
|
||||
// Hello is the first NDJSON line sent by the dialing side after connect.
|
||||
// Reserved for future revisions: auth credential, feature flags (ack, compression).
|
||||
type Hello struct {
|
||||
LogWisp int `json:"logwisp"`
|
||||
Node string `json:"node,omitempty"`
|
||||
// Auth string `json:"auth,omitempty"`
|
||||
// Features []string `json:"features,omitempty"`
|
||||
}
|
||||
|
||||
// HTTP transport mapping of the chain protocol.
|
||||
// Hello preamble equivalent: protocol + node carried as request headers.
|
||||
// Reserved extension point: Authorization header for auth, TLS at transport.
|
||||
const (
|
||||
HeaderProtocol = "X-Logwisp-Protocol"
|
||||
HeaderNode = "X-Logwisp-Node"
|
||||
HeaderAccepted = "X-Logwisp-Accepted"
|
||||
ContentTypeNDJSON = "application/x-ndjson"
|
||||
)
|
||||
|
||||
// EncodeHello serializes a newline-terminated hello preamble
|
||||
func EncodeHello(node string) ([]byte, error) {
|
||||
b, err := json.Marshal(Hello{LogWisp: ProtocolVersion, Node: node})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(b, '\n'), nil
|
||||
}
|
||||
|
||||
// DecodeHello parses and validates a hello preamble line
|
||||
func DecodeHello(line []byte) (Hello, error) {
|
||||
var h Hello
|
||||
if err := json.Unmarshal(line, &h); err != nil {
|
||||
return h, fmt.Errorf("malformed hello: %w", err)
|
||||
}
|
||||
if h.LogWisp != ProtocolVersion {
|
||||
return h, fmt.Errorf("unsupported protocol version: %d", h.LogWisp)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// DecodeEntry parses a canonical LogEntry line and applies the node trust policy
|
||||
func DecodeEntry(line []byte, connNode string, trustNode bool) (core.LogEntry, error) {
|
||||
var entry core.LogEntry
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
return core.LogEntry{}, err
|
||||
}
|
||||
if entry.Time.IsZero() {
|
||||
entry.Time = time.Now()
|
||||
}
|
||||
if entry.Node == "" || !trustNode {
|
||||
entry.Node = connNode
|
||||
}
|
||||
entry.RawSize = int64(len(line))
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// EntryFromEvent extracts the structured entry, stamping node identity at
|
||||
// first hop. Second return is true when synthesized from a formatted payload.
|
||||
func EntryFromEvent(event core.TransportEvent, node, fallbackSource string) (core.LogEntry, bool) {
|
||||
entry := event.Entry
|
||||
synthesized := false
|
||||
if entry.Time.IsZero() {
|
||||
synthesized = true
|
||||
entry = core.LogEntry{
|
||||
Time: event.Time,
|
||||
Source: fallbackSource,
|
||||
Message: string(event.Payload),
|
||||
}
|
||||
}
|
||||
if entry.Node == "" {
|
||||
entry.Node = node
|
||||
}
|
||||
return entry, synthesized
|
||||
}
|
||||
|
||||
// BackoffDelay computes exponential backoff with ±20% jitter
|
||||
func BackoffDelay(minD, maxD time.Duration, failures int) time.Duration {
|
||||
d := maxD
|
||||
if failures < 63 {
|
||||
if v := minD << uint(failures-1); v > 0 && v < maxD {
|
||||
d = v
|
||||
}
|
||||
}
|
||||
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package config
|
||||
|
||||
// --- LogWisp Configuration Options ---
|
||||
|
||||
// Config is the top-level configuration structure for the LogWisp application
|
||||
type Config struct {
|
||||
// Top-level flags for application control
|
||||
ShowVersion bool `toml:"version"`
|
||||
Quiet bool `toml:"quiet"`
|
||||
|
||||
// Runtime behavior flags
|
||||
StatusReporter bool `toml:"status_reporter"`
|
||||
ConfigAutoReload bool `toml:"auto_reload"`
|
||||
|
||||
// Configuration file path
|
||||
ConfigFile string `toml:"config_file"`
|
||||
|
||||
// Existing fields
|
||||
Logging *LogConfig `toml:"logging"`
|
||||
Pipelines []PipelineConfig `toml:"pipelines"`
|
||||
}
|
||||
|
||||
// --- Logging Options ---
|
||||
|
||||
// LogConfig represents the logging configuration for the LogWisp application itself
|
||||
type LogConfig struct {
|
||||
// Output mode: "file", "stdout", "stderr", "split", "all", "none"
|
||||
Output string `toml:"output"`
|
||||
|
||||
// Log level: "debug", "info", "warn", "error"
|
||||
Level string `toml:"level"`
|
||||
|
||||
// Format: "raw", "txt", "json"
|
||||
Format string `toml:"format"`
|
||||
|
||||
// Sanitization policy for console output
|
||||
Sanitization string `toml:"sanitization"`
|
||||
|
||||
// File output settings (when Output includes "file" or "all")
|
||||
File *LogFileConfig `toml:"file"`
|
||||
|
||||
// Console output settings
|
||||
Console *LogConsoleConfig `toml:"console"`
|
||||
}
|
||||
|
||||
// LogFileConfig defines settings for file-based application logging
|
||||
type LogFileConfig struct {
|
||||
// Directory for log files
|
||||
Directory string `toml:"directory"`
|
||||
|
||||
// Base name for log files
|
||||
Name string `toml:"name"`
|
||||
|
||||
// Maximum size per log file in MB
|
||||
MaxSizeMB int64 `toml:"max_size_mb"`
|
||||
|
||||
// Maximum total size of all logs in MB
|
||||
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
|
||||
|
||||
// Log retention in hours (0 = disabled)
|
||||
RetentionHours float64 `toml:"retention_hours"`
|
||||
}
|
||||
|
||||
// LogConsoleConfig defines settings for console-based application logging
|
||||
type LogConsoleConfig struct {
|
||||
// Target for console output: "stdout", "stderr"
|
||||
Target string `toml:"target"`
|
||||
}
|
||||
|
||||
// --- Pipeline ---
|
||||
|
||||
// PipelineConfig defines a complete data flow from sources to sinks
|
||||
type PipelineConfig struct {
|
||||
Name string `toml:"name"`
|
||||
Flow *FlowConfig `toml:"flow"`
|
||||
|
||||
PluginSources []PluginSourceConfig `toml:"plugin_sources,omitempty"`
|
||||
PluginSinks []PluginSinkConfig `toml:"plugin_sinks,omitempty"`
|
||||
}
|
||||
|
||||
// --- Flow ---
|
||||
|
||||
// FlowConfig consolidates all processing stages between sources and sinks
|
||||
type FlowConfig struct {
|
||||
Heartbeat *HeartbeatConfig `toml:"heartbeat"`
|
||||
RateLimit *RateLimitConfig `toml:"rate_limit"`
|
||||
Filters []FilterConfig `toml:"filters"`
|
||||
Format *FormatConfig `toml:"format"`
|
||||
}
|
||||
|
||||
// --- Heartbeat Options ---
|
||||
|
||||
// HeartbeatConfig defines settings for periodic keep-alive or status messages
|
||||
type HeartbeatConfig struct {
|
||||
Enabled bool `toml:"enabled"`
|
||||
IntervalMS int64 `toml:"interval_ms"`
|
||||
IncludeTimestamp bool `toml:"include_timestamp"`
|
||||
IncludeStats bool `toml:"include_stats"`
|
||||
Format string `toml:"format"`
|
||||
}
|
||||
|
||||
// --- Formatter Options ---
|
||||
|
||||
// FormatConfig is a polymorphic struct representing log entry formatting options
|
||||
type FormatConfig struct {
|
||||
Type string `toml:"type"` // "json", "txt", "raw"
|
||||
Flags int64 `toml:"flags"`
|
||||
TimestampFormat string `toml:"timestamp_format"`
|
||||
SanitizerPolicy string `toml:"sanitizer_policy"` // "raw", "json", "txt", "shell"
|
||||
}
|
||||
|
||||
// --- Rate Limit Options ---
|
||||
|
||||
// RateLimitPolicy defines the action to take when a rate limit is exceeded
|
||||
type RateLimitPolicy int
|
||||
|
||||
const (
|
||||
// PolicyPass allows all logs through, effectively disabling the limiter
|
||||
PolicyPass RateLimitPolicy = iota
|
||||
// PolicyDrop drops logs that exceed the rate limit
|
||||
PolicyDrop
|
||||
)
|
||||
|
||||
// RateLimitConfig defines the configuration for pipeline-level rate limiting
|
||||
type RateLimitConfig struct {
|
||||
// Rate is the number of log entries allowed per second. Default: 0 (disabled)
|
||||
Rate float64 `toml:"rate"`
|
||||
// Burst is the maximum number of log entries that can be sent in a short burst. Defaults to the Rate
|
||||
Burst float64 `toml:"burst"`
|
||||
// Policy defines the action to take when the limit is exceeded. "pass" or "drop"
|
||||
Policy string `toml:"policy"`
|
||||
// MaxEntrySizeBytes is the maximum allowed size for a single log entry. 0 = no limit
|
||||
MaxEntrySizeBytes int64 `toml:"max_entry_size_bytes"`
|
||||
}
|
||||
|
||||
// --- Filter Options ---
|
||||
|
||||
// FilterType represents the filter's behavior (include or exclude)
|
||||
type FilterType string
|
||||
|
||||
const (
|
||||
// FilterTypeInclude specifies that only matching logs will pass
|
||||
FilterTypeInclude FilterType = "include" // Whitelist - only matching logs pass
|
||||
// FilterTypeExclude specifies that matching logs will be dropped
|
||||
FilterTypeExclude FilterType = "exclude" // Blacklist - matching logs are dropped
|
||||
)
|
||||
|
||||
// FilterLogic represents how multiple filter patterns are combined
|
||||
type FilterLogic string
|
||||
|
||||
const (
|
||||
// FilterLogicOr specifies that a match on any pattern is sufficient
|
||||
FilterLogicOr FilterLogic = "or" // Match any pattern
|
||||
// FilterLogicAnd specifies that all patterns must match
|
||||
FilterLogicAnd FilterLogic = "and" // Match all patterns
|
||||
)
|
||||
|
||||
// FilterConfig represents the configuration for a single filter
|
||||
type FilterConfig struct {
|
||||
Type FilterType `toml:"type"`
|
||||
Logic FilterLogic `toml:"logic"`
|
||||
Patterns []string `toml:"patterns"`
|
||||
}
|
||||
|
||||
// --- Source Options ---
|
||||
|
||||
// PluginSourceConfig represents a source plugin instance configuration
|
||||
type PluginSourceConfig struct {
|
||||
ID string `toml:"id"`
|
||||
Type string `toml:"type"`
|
||||
Config map[string]any `toml:"config"`
|
||||
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
|
||||
}
|
||||
|
||||
// // SourceConfig is a polymorphic struct representing a single data source
|
||||
// type SourceConfig struct {
|
||||
// Type string `toml:"type"`
|
||||
//
|
||||
// // Polymorphic - only one populated based on type
|
||||
// File *FileSourceOptions `toml:"file,omitempty"`
|
||||
// Console *ConsoleSourceOptions `toml:"console,omitempty"`
|
||||
// }
|
||||
|
||||
// NullSourceOptions defines settings for a null source (no configuration needed)
|
||||
type NullSourceOptions struct{}
|
||||
|
||||
// RandomSourceOptions defines settings for a random log generator source
|
||||
type RandomSourceOptions struct {
|
||||
IntervalMS int64 `toml:"interval_ms"`
|
||||
JitterMS int64 `toml:"jitter_ms"`
|
||||
Format string `toml:"format"`
|
||||
Length int64 `toml:"length"`
|
||||
Special bool `toml:"special"`
|
||||
}
|
||||
|
||||
// FileSourceOptions defines settings for a file-based source
|
||||
type FileSourceOptions struct {
|
||||
Directory string `toml:"directory"`
|
||||
Pattern string `toml:"pattern"` // glob pattern
|
||||
CheckIntervalMS int64 `toml:"check_interval_ms"`
|
||||
}
|
||||
|
||||
// ConsoleSourceOptions defines settings for a stdin-based source
|
||||
type ConsoleSourceOptions struct {
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
}
|
||||
|
||||
// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting
|
||||
// NDJSON entries from upstream logwisp tcp_chain sinks
|
||||
type TCPChainSourceOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
|
||||
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // per-connection idle deadline, 0 = none
|
||||
HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
|
||||
// NDJSON batches from upstream logwisp http_chain sinks
|
||||
type HTTPChainSourceOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
IngestPath string `toml:"ingest_path"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
MaxBodyBytes int64 `toml:"max_body_bytes"` // per-request cap
|
||||
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
// --- Sink Options ---
|
||||
|
||||
// PluginSinkConfig represents a sink plugin instance configuration
|
||||
type PluginSinkConfig struct {
|
||||
ID string `toml:"id"`
|
||||
Type string `toml:"type"`
|
||||
Config map[string]any `toml:"config"`
|
||||
ConfigFile string `toml:"config_file,omitempty"` // TODO: support for include/source mechanism for nested config
|
||||
}
|
||||
|
||||
// // SinkConfig is a polymorphic struct representing a single data sink
|
||||
// type SinkConfig struct {
|
||||
// Type string `toml:"type"`
|
||||
//
|
||||
// // Polymorphic - only one populated based on type
|
||||
// Console *ConsoleSinkOptions `toml:"console,omitempty"`
|
||||
// File *FileSinkOptions `toml:"file,omitempty"`
|
||||
// }
|
||||
|
||||
// NullSinkOptions defines settings for a null sink (no configuration needed)
|
||||
type NullSinkOptions struct{}
|
||||
|
||||
// ConsoleSinkOptions defines settings for a console-based sink
|
||||
type ConsoleSinkOptions struct {
|
||||
Target string `toml:"target"` // "stdout", "stderr"
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
}
|
||||
|
||||
// FileSinkOptions defines settings for a file-based sink
|
||||
type FileSinkOptions struct {
|
||||
Directory string `toml:"directory"`
|
||||
Name string `toml:"name"`
|
||||
MaxSizeMB int64 `toml:"max_size_mb"`
|
||||
MaxTotalSizeMB int64 `toml:"max_total_size_mb"`
|
||||
MinDiskFreeMB int64 `toml:"min_disk_free_mb"`
|
||||
RetentionHours float64 `toml:"retention_hours"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
FlushIntervalMs int64 `toml:"flush_interval_ms"`
|
||||
}
|
||||
|
||||
// TCPSinkOptions defines settings for a TCP server sink
|
||||
type TCPSinkOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
WriteTimeout int64 `toml:"write_timeout_ms"`
|
||||
KeepAlivePeriod int64 `toml:"keep_alive_period_ms"`
|
||||
KeepAlive bool `toml:"keep_alive"`
|
||||
}
|
||||
|
||||
// HTTPSinkOptions defines settings for an HTTP SSE server sink
|
||||
type HTTPSinkOptions struct {
|
||||
StreamPath string `toml:"stream_path"`
|
||||
StatusPath string `toml:"status_path"`
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
WriteTimeout int64 `toml:"write_timeout_ms"`
|
||||
}
|
||||
|
||||
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
|
||||
// entries to a downstream logwisp tcp_chain source
|
||||
type TCPChainSinkOptions struct {
|
||||
Node string `toml:"node"` // origin label, default: os.Hostname()
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
DialTimeoutMS int64 `toml:"dial_timeout_ms"`
|
||||
WriteTimeoutMS int64 `toml:"write_timeout_ms"`
|
||||
BackoffMinMS int64 `toml:"backoff_min_ms"`
|
||||
BackoffMaxMS int64 `toml:"backoff_max_ms"`
|
||||
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
|
||||
KeepAlive bool `toml:"keep_alive"`
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting
|
||||
// NDJSON batches to a downstream logwisp http_chain source
|
||||
type HTTPChainSinkOptions struct {
|
||||
Node string `toml:"node"` // origin label, default: os.Hostname()
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
IngestPath string `toml:"ingest_path"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
MaxBatchCount int64 `toml:"max_batch_count"`
|
||||
MaxBatchBytes int64 `toml:"max_batch_bytes"`
|
||||
FlushIntervalMS int64 `toml:"flush_interval_ms"`
|
||||
RequestTimeoutMS int64 `toml:"request_timeout_ms"` // covers dial + write + response
|
||||
BackoffMinMS int64 `toml:"backoff_min_ms"`
|
||||
BackoffMaxMS int64 `toml:"backoff_max_ms"`
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"logwisp/internal/core"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
)
|
||||
|
||||
// configManager holds the global instance of the configuration manager
|
||||
var configManager *lconfig.Config
|
||||
|
||||
// Load is the single entry point for loading all application configuration
|
||||
func Load(args []string) (*Config, error) {
|
||||
configPath, isExplicit := resolveConfigPath(args)
|
||||
// Build configuration with all sources
|
||||
|
||||
// Create target config instance that will be populated
|
||||
finalConfig := &Config{}
|
||||
|
||||
// Builder handles loading, populating the target struct, and validation
|
||||
cfg, err := lconfig.NewBuilder().
|
||||
WithTarget(finalConfig). // Typed target struct
|
||||
WithDefaults(defaults()). // Default values
|
||||
WithSources(
|
||||
lconfig.SourceCLI,
|
||||
lconfig.SourceEnv,
|
||||
lconfig.SourceFile,
|
||||
lconfig.SourceDefault,
|
||||
).
|
||||
WithEnvTransform(customEnvTransform). // Convert '.' to '_' in env separation
|
||||
WithEnvPrefix("LOGWISP_"). // Environment variable prefix
|
||||
WithArgs(args). // Command-line arguments
|
||||
WithFile(configPath). // TOML config file
|
||||
WithFileFormat("toml"). // Explicit format
|
||||
WithTypedValidator(ValidateConfig). // Centralized validation
|
||||
WithSecurityOptions(lconfig.SecurityOptions{
|
||||
PreventPathTraversal: true,
|
||||
MaxFileSize: 10 * 1024 * 1024, // 10MB max config
|
||||
}).
|
||||
Build()
|
||||
|
||||
if err != nil {
|
||||
// Handle file not found errors - maintain existing behavior
|
||||
if errors.Is(err, lconfig.ErrConfigNotFound) {
|
||||
if isExplicit {
|
||||
// Return empty config with file path
|
||||
finalConfig.ConfigFile = configPath
|
||||
return finalConfig, fmt.Errorf("config file not found: %s", configPath)
|
||||
}
|
||||
// If the default config file is not found, it's not an error, default/cli/env will be used
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to load or validate config: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Store the config file path for hot reload
|
||||
finalConfig.ConfigFile = configPath
|
||||
|
||||
// Store the manager for hot reload
|
||||
configManager = cfg
|
||||
|
||||
// Surface typo'd flags (e.g. --status-reporter vs --status_reporter);
|
||||
// pre-logger phase, stderr only, suppressed in quiet mode
|
||||
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet {
|
||||
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
|
||||
}
|
||||
|
||||
// Start watcher if auto-reload is enabled
|
||||
if finalConfig.ConfigAutoReload {
|
||||
watchOpts := lconfig.WatchOptions{
|
||||
PollInterval: core.ReloadWatchPollInterval,
|
||||
Debounce: core.ReloadWatchDebounce,
|
||||
ReloadTimeout: core.ReloadWatchTimeout,
|
||||
VerifyPermissions: true,
|
||||
}
|
||||
cfg.AutoUpdateWithOptions(watchOpts)
|
||||
}
|
||||
|
||||
return finalConfig, nil
|
||||
}
|
||||
|
||||
// GetConfigManager returns the global configuration manager instance for hot-reloading
|
||||
func GetConfigManager() *lconfig.Config {
|
||||
return configManager
|
||||
}
|
||||
|
||||
// defaults provides the default configuration values for the application
|
||||
func defaults() *Config {
|
||||
return &Config{
|
||||
// Top-level flag defaults
|
||||
ShowVersion: false,
|
||||
Quiet: false,
|
||||
|
||||
// Runtime behavior defaults
|
||||
StatusReporter: true,
|
||||
ConfigAutoReload: false,
|
||||
|
||||
// Existing defaults
|
||||
Logging: &LogConfig{
|
||||
Output: "stdout",
|
||||
Level: "info",
|
||||
Format: "txt",
|
||||
File: &LogFileConfig{
|
||||
Directory: "./log",
|
||||
Name: "logwisp",
|
||||
MaxSizeMB: 100,
|
||||
MaxTotalSizeMB: 1000,
|
||||
RetentionHours: 168, // 7 days
|
||||
},
|
||||
Console: &LogConsoleConfig{
|
||||
Target: "stdout",
|
||||
},
|
||||
},
|
||||
Pipelines: []PipelineConfig{
|
||||
{
|
||||
Name: "default_pipeline",
|
||||
Flow: &FlowConfig{
|
||||
RateLimit: &RateLimitConfig{
|
||||
Rate: 5,
|
||||
Burst: 10,
|
||||
Policy: "drop",
|
||||
MaxEntrySizeBytes: 65536,
|
||||
},
|
||||
Format: &FormatConfig{
|
||||
Type: "json",
|
||||
SanitizerPolicy: "json",
|
||||
},
|
||||
},
|
||||
PluginSources: []PluginSourceConfig{
|
||||
{
|
||||
ID: "default_source",
|
||||
Type: "random",
|
||||
Config: map[string]any{
|
||||
"special": true,
|
||||
},
|
||||
// Config: &FileSourceOptions{
|
||||
// Directory: "./",
|
||||
// Pattern: "*.log",
|
||||
// CheckIntervalMS: int64(100),
|
||||
},
|
||||
},
|
||||
PluginSinks: []PluginSinkConfig{
|
||||
{
|
||||
ID: "default_sink",
|
||||
Type: "console",
|
||||
Config: map[string]any{
|
||||
"target": "stdout",
|
||||
"buffer_size": 100,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// resolveConfigPath determines the configuration file path based on CLI args, env vars, and default locations
|
||||
func resolveConfigPath(args []string) (path string, isExplicit bool) {
|
||||
// 1. Check for --config flag in command-line arguments (highest precedence)
|
||||
for i, arg := range args {
|
||||
if arg == "-c" {
|
||||
return args[i+1], true
|
||||
}
|
||||
if strings.HasPrefix(arg, "--config=") {
|
||||
return strings.TrimPrefix(arg, "--config="), true
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Check environment variables
|
||||
if configFile := os.Getenv("LOGWISP_CONFIG_FILE"); configFile != "" {
|
||||
path = configFile
|
||||
if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
|
||||
path = filepath.Join(configDir, configFile)
|
||||
}
|
||||
return path, true
|
||||
}
|
||||
if configDir := os.Getenv("LOGWISP_CONFIG_DIR"); configDir != "" {
|
||||
return filepath.Join(configDir, "logwisp.toml"), true
|
||||
}
|
||||
|
||||
// 3. Check default user config location
|
||||
if homeDir, err := os.UserHomeDir(); err == nil {
|
||||
configPath := filepath.Join(homeDir, ".config", "logwisp", "logwisp.toml")
|
||||
if _, err := os.Stat(configPath); err == nil {
|
||||
return configPath, false // Found a default, but not explicitly set by user
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Fallback to default in current directory
|
||||
return "logwisp.toml", false
|
||||
}
|
||||
|
||||
// customEnvTransform converts TOML-style config paths (e.g., logging.level) to environment variable format (LOGGING_LEVEL)
|
||||
func customEnvTransform(path string) string {
|
||||
env := strings.ReplaceAll(path, ".", "_")
|
||||
env = strings.ToUpper(env)
|
||||
// env = "LOGWISP_" + env // already added by WithEnvPrefix
|
||||
return env
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
)
|
||||
|
||||
// ValidateConfig validates top-level structure only
|
||||
// Value range validation is delegated to component constructors
|
||||
func ValidateConfig(cfg *Config) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("config is nil")
|
||||
}
|
||||
|
||||
if len(cfg.Pipelines) == 0 {
|
||||
return fmt.Errorf("no pipelines configured")
|
||||
}
|
||||
|
||||
// Reject duplicate pipeline names (service map is keyed by name)
|
||||
names := make(map[string]struct{}, len(cfg.Pipelines))
|
||||
for i, p := range cfg.Pipelines {
|
||||
if _, dup := names[p.Name]; dup {
|
||||
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
|
||||
}
|
||||
names[p.Name] = struct{}{}
|
||||
}
|
||||
|
||||
if err := validateLogConfig(cfg.Logging); err != nil {
|
||||
return fmt.Errorf("logging: %w", err)
|
||||
}
|
||||
|
||||
for i, p := range cfg.Pipelines {
|
||||
if err := lconfig.NonEmpty(p.Name); err != nil {
|
||||
return fmt.Errorf("pipeline[%d].name: %w", i, err)
|
||||
}
|
||||
if len(p.PluginSources) == 0 {
|
||||
return fmt.Errorf("pipeline[%d]: no sources defined", i)
|
||||
}
|
||||
if len(p.PluginSinks) == 0 {
|
||||
return fmt.Errorf("pipeline[%d]: no sinks defined", i)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateLogConfig validates application logging settings
|
||||
func validateLogConfig(cfg *LogConfig) error {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
validateOutput := lconfig.OneOf("file", "stdout", "stderr", "split", "all", "none")
|
||||
if err := validateOutput(cfg.Output); err != nil {
|
||||
return fmt.Errorf("output: %w", err)
|
||||
}
|
||||
|
||||
validateLevel := lconfig.OneOf("debug", "info", "warn", "error")
|
||||
if err := validateLevel(cfg.Level); err != nil {
|
||||
return fmt.Errorf("level: %w", err)
|
||||
}
|
||||
|
||||
if cfg.Format != "" {
|
||||
if err := lconfig.OneOf("raw", "txt", "json")(cfg.Format); err != nil {
|
||||
return fmt.Errorf("format: %w", err)
|
||||
}
|
||||
}
|
||||
if cfg.Sanitization != "" {
|
||||
if err := lconfig.OneOf("raw", "json", "txt", "shell")(cfg.Sanitization); err != nil {
|
||||
return fmt.Errorf("sanitization: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Console != nil {
|
||||
validateTarget := lconfig.OneOf("stdout", "stderr", "split")
|
||||
if err := validateTarget(cfg.Console.Target); err != nil {
|
||||
return fmt.Errorf("console.target: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package core
|
||||
|
||||
// Capability represents a plugin feature
|
||||
type Capability string
|
||||
|
||||
const (
|
||||
// Network capabilities
|
||||
CapNetLimit Capability = "netlimit"
|
||||
CapTLS Capability = "tls"
|
||||
CapAuth Capability = "auth"
|
||||
|
||||
// Session capabilities
|
||||
CapSessionAware Capability = "session_aware"
|
||||
CapMultiSession Capability = "multi_session"
|
||||
CapSingleInstance Capability = "single_instance"
|
||||
|
||||
// Stream capabilities
|
||||
CapBidirectional Capability = "bidirectional"
|
||||
CapCompression Capability = "compression"
|
||||
)
|
||||
@@ -0,0 +1,29 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxLogEntryBytes = 1024 * 1024
|
||||
|
||||
FileWatcherPollInterval = 100 * time.Millisecond
|
||||
|
||||
SessionDefaultMaxIdleTime = 30 * time.Minute
|
||||
|
||||
SessionCleanupInterval = 5 * time.Minute
|
||||
|
||||
ServiceStatsUpdateInterval = 1 * time.Second
|
||||
|
||||
ShutdownTimeout = 10 * time.Second
|
||||
|
||||
ConfigReloadTimeout = 30 * time.Second
|
||||
|
||||
LoggerShutdownTimeout = 2 * time.Second
|
||||
|
||||
ReloadWatchPollInterval = time.Second
|
||||
|
||||
ReloadWatchDebounce = 500 * time.Millisecond
|
||||
|
||||
ReloadWatchTimeout = 30 * time.Second
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LogEntry represents a single log record flowing through the pipeline
|
||||
type LogEntry struct {
|
||||
Time time.Time `json:"time"`
|
||||
Node string `json:"node,omitempty"` // origin node identity for chained topologies; first hop stamps, relays preserve
|
||||
Source string `json:"source"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Message string `json:"message"`
|
||||
Fields json.RawMessage `json:"fields,omitempty"`
|
||||
RawSize int64 `json:"-"`
|
||||
}
|
||||
|
||||
// TransportEvent contains the final payload and minimal metadata needed by sinks
|
||||
type TransportEvent struct {
|
||||
Time time.Time
|
||||
// Formatted, serialized log payload
|
||||
Payload []byte
|
||||
// Structured entry for re-serializing sinks (chain links). Zero Time => absent
|
||||
Entry LogEntry
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Chain manages a sequence of filters, applying them in order
|
||||
type Chain struct {
|
||||
filters []*Filter
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
totalPassed atomic.Uint64
|
||||
}
|
||||
|
||||
// NewChain creates a new filter chain from a slice of filter configurations
|
||||
func NewChain(configs []config.FilterConfig, logger *log.Logger) (*Chain, error) {
|
||||
chain := &Chain{
|
||||
filters: make([]*Filter, 0, len(configs)),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
for i, cfg := range configs {
|
||||
filter, err := NewFilter(cfg, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("filter[%d]: %w", i, err)
|
||||
}
|
||||
chain.filters = append(chain.filters, filter)
|
||||
}
|
||||
|
||||
logger.Info("msg", "Filter chain created",
|
||||
"component", "filter_chain",
|
||||
"filter_count", len(configs))
|
||||
return chain, nil
|
||||
}
|
||||
|
||||
// Apply runs a log entry through all filters in the chain
|
||||
func (c *Chain) Apply(entry core.LogEntry) bool {
|
||||
c.totalProcessed.Add(1)
|
||||
|
||||
// No filters means pass everything
|
||||
if len(c.filters) == 0 {
|
||||
c.totalPassed.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
// All filters must pass
|
||||
for i, filter := range c.filters {
|
||||
if !filter.Apply(entry) {
|
||||
c.logger.Debug("msg", "Entry filtered out",
|
||||
"component", "filter_chain",
|
||||
"filter_index", i,
|
||||
"filter_type", filter.config.Type)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
c.totalPassed.Add(1)
|
||||
return true
|
||||
}
|
||||
|
||||
// GetStats returns aggregated statistics for the entire chain
|
||||
func (c *Chain) GetStats() map[string]any {
|
||||
filterStats := make([]map[string]any, len(c.filters))
|
||||
for i, filter := range c.filters {
|
||||
filterStats[i] = filter.GetStats()
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"filter_count": len(c.filters),
|
||||
"total_processed": c.totalProcessed.Load(),
|
||||
"total_passed": c.totalPassed.Load(),
|
||||
"filters": filterStats,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package filter
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Filter applies regex-based filtering to log entries
|
||||
type Filter struct {
|
||||
config config.FilterConfig
|
||||
patterns []*regexp.Regexp
|
||||
mu sync.RWMutex
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
totalMatched atomic.Uint64
|
||||
totalDropped atomic.Uint64
|
||||
}
|
||||
|
||||
// NewFilter creates a new filter from a configuration
|
||||
func NewFilter(cfg config.FilterConfig, logger *log.Logger) (*Filter, error) {
|
||||
// Validate enums before setting defaults
|
||||
if cfg.Type != "" {
|
||||
validateType := lconfig.OneOf(config.FilterTypeInclude, config.FilterTypeExclude)
|
||||
if err := validateType(cfg.Type); err != nil {
|
||||
return nil, fmt.Errorf("type: %w", err)
|
||||
}
|
||||
}
|
||||
if cfg.Logic != "" {
|
||||
validateLogic := lconfig.OneOf(config.FilterLogicOr, config.FilterLogicAnd)
|
||||
if err := validateLogic(cfg.Logic); err != nil {
|
||||
return nil, fmt.Errorf("logic: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Set defaults
|
||||
if cfg.Type == "" {
|
||||
cfg.Type = config.FilterTypeInclude
|
||||
}
|
||||
if cfg.Logic == "" {
|
||||
cfg.Logic = config.FilterLogicOr
|
||||
}
|
||||
|
||||
f := &Filter{
|
||||
config: cfg,
|
||||
patterns: make([]*regexp.Regexp, 0, len(cfg.Patterns)),
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
// Compile patterns
|
||||
for i, pattern := range cfg.Patterns {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("pattern[%d] '%s': %w", i, pattern, err)
|
||||
}
|
||||
f.patterns = append(f.patterns, re)
|
||||
}
|
||||
|
||||
logger.Debug("msg", "Filter created",
|
||||
"component", "filter",
|
||||
"type", cfg.Type,
|
||||
"logic", cfg.Logic,
|
||||
"pattern_count", len(cfg.Patterns))
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Apply determines if a log entry should be passed through based on the filter's rules
|
||||
func (f *Filter) Apply(entry core.LogEntry) bool {
|
||||
f.totalProcessed.Add(1)
|
||||
|
||||
// No patterns means pass everything
|
||||
if len(f.patterns) == 0 {
|
||||
f.logger.Debug("msg", "No patterns configured, passing entry",
|
||||
"component", "filter",
|
||||
"type", f.config.Type)
|
||||
return true
|
||||
}
|
||||
|
||||
// Check against all fields that might contain the log content
|
||||
text := entry.Message
|
||||
if entry.Level != "" {
|
||||
text = entry.Level + " " + text
|
||||
}
|
||||
if entry.Source != "" {
|
||||
text = entry.Source + " " + text
|
||||
}
|
||||
|
||||
f.logger.Debug("msg", "Filter checking entry",
|
||||
"component", "filter",
|
||||
"type", f.config.Type,
|
||||
"logic", f.config.Logic,
|
||||
"entry_level", entry.Level,
|
||||
"entry_source", entry.Source,
|
||||
"entry_message", entry.Message[:min(100, len(entry.Message))], // First 100 chars
|
||||
"text_to_match", text[:min(150, len(text))], // First 150 chars
|
||||
"patterns", f.config.Patterns)
|
||||
|
||||
for i, pattern := range f.config.Patterns {
|
||||
isMatch := f.patterns[i].MatchString(text)
|
||||
f.logger.Debug("msg", "Pattern match result",
|
||||
"component", "filter",
|
||||
"pattern_index", i,
|
||||
"pattern", pattern,
|
||||
"matched", isMatch)
|
||||
}
|
||||
|
||||
matched := f.matches(text)
|
||||
if matched {
|
||||
f.totalMatched.Add(1)
|
||||
}
|
||||
f.logger.Debug("msg", "Filter final match result",
|
||||
"component", "filter",
|
||||
"matched", matched)
|
||||
|
||||
// Determine if we should pass or drop
|
||||
shouldPass := false
|
||||
switch f.config.Type {
|
||||
case config.FilterTypeInclude:
|
||||
shouldPass = matched
|
||||
case config.FilterTypeExclude:
|
||||
shouldPass = !matched
|
||||
}
|
||||
|
||||
f.logger.Debug("msg", "Filter decision",
|
||||
"component", "filter",
|
||||
"type", f.config.Type,
|
||||
"matched", matched,
|
||||
"should_pass", shouldPass)
|
||||
|
||||
if !shouldPass {
|
||||
f.totalDropped.Add(1)
|
||||
}
|
||||
|
||||
return shouldPass
|
||||
}
|
||||
|
||||
// GetStats returns the filter's current statistics
|
||||
func (f *Filter) GetStats() map[string]any {
|
||||
return map[string]any{
|
||||
"type": f.config.Type,
|
||||
"logic": f.config.Logic,
|
||||
"pattern_count": len(f.patterns),
|
||||
"total_processed": f.totalProcessed.Load(),
|
||||
"total_matched": f.totalMatched.Load(),
|
||||
"total_dropped": f.totalDropped.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
// UpdatePatterns allows for dynamic, thread-safe updates to the filter's regex patterns
|
||||
func (f *Filter) UpdatePatterns(patterns []string) error {
|
||||
compiled := make([]*regexp.Regexp, 0, len(patterns))
|
||||
|
||||
// Compile all patterns first
|
||||
for i, pattern := range patterns {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid regex pattern[%d] '%s': %w", i, pattern, err)
|
||||
}
|
||||
compiled = append(compiled, re)
|
||||
}
|
||||
|
||||
// Update atomically
|
||||
f.mu.Lock()
|
||||
f.patterns = compiled
|
||||
f.config.Patterns = patterns
|
||||
f.mu.Unlock()
|
||||
|
||||
f.logger.Info("msg", "Filter patterns updated",
|
||||
"component", "filter",
|
||||
"pattern_count", len(patterns))
|
||||
return nil
|
||||
}
|
||||
|
||||
// matches checks if the given text matches the filter's patterns according to its logic
|
||||
func (f *Filter) matches(text string) bool {
|
||||
switch f.config.Logic {
|
||||
case config.FilterLogicOr:
|
||||
// Match any pattern
|
||||
for _, re := range f.patterns {
|
||||
if re.MatchString(text) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
|
||||
case config.FilterLogicAnd:
|
||||
// Must match all patterns
|
||||
for _, re := range f.patterns {
|
||||
if !re.MatchString(text) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
default:
|
||||
// Shouldn't happen after validation
|
||||
f.logger.Warn("msg", "Unknown filter logic",
|
||||
"component", "filter",
|
||||
"logic", f.config.Logic)
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/filter"
|
||||
"logwisp/internal/format"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Flow manages the complete processing pipeline for log entries:
|
||||
// LogEntry -> Rate Limiter -> Filters -> Formatter (with Sanitizer) -> TransportEvent
|
||||
type Flow struct {
|
||||
rateLimiter *RateLimiter
|
||||
filterChain *filter.Chain
|
||||
formatter format.Formatter
|
||||
heartbeat *HeartbeatGenerator
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
totalDropped atomic.Uint64
|
||||
totalFormatted atomic.Uint64
|
||||
}
|
||||
|
||||
// NewFlow creates a flow processor from configuration
|
||||
func NewFlow(cfg *config.FlowConfig, logger *log.Logger) (*Flow, error) {
|
||||
if cfg == nil {
|
||||
cfg = &config.FlowConfig{}
|
||||
}
|
||||
|
||||
f := &Flow{
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
// Create rate limiter if configured
|
||||
if cfg.RateLimit != nil {
|
||||
limiter, err := NewRateLimiter(*cfg.RateLimit, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create rate limiter: %w", err)
|
||||
}
|
||||
f.rateLimiter = limiter
|
||||
}
|
||||
|
||||
// Create filter chain if configured
|
||||
if len(cfg.Filters) > 0 {
|
||||
chain, err := filter.NewChain(cfg.Filters, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create filter chain: %w", err)
|
||||
}
|
||||
f.filterChain = chain
|
||||
}
|
||||
|
||||
// Create formatter with sanitizer integration
|
||||
formatter, err := format.NewFormatter(cfg.Format)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create formatter: %w", err)
|
||||
}
|
||||
f.formatter = formatter
|
||||
|
||||
// Create heartbeat generator with the same formatter if configured
|
||||
if cfg.Heartbeat != nil {
|
||||
hb, err := NewHeartbeatGenerator(cfg.Heartbeat, formatter, logger)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("heartbeat: %w", err)
|
||||
}
|
||||
f.heartbeat = hb
|
||||
}
|
||||
|
||||
logger.Info("msg", "Flow processor created",
|
||||
"component", "flow",
|
||||
"rate_limiter", f.rateLimiter != nil,
|
||||
"filter_chain", f.filterChain != nil,
|
||||
"formatter", formatter.Name(),
|
||||
"heartbeat", f.heartbeat != nil)
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Process applies all flow stages to a log entry
|
||||
// Returns TransportEvent and whether entry passed all stages
|
||||
func (f *Flow) Process(entry core.LogEntry) (core.TransportEvent, bool) {
|
||||
f.totalProcessed.Add(1)
|
||||
|
||||
// Stage 1: Rate limiting
|
||||
if f.rateLimiter != nil {
|
||||
if !f.rateLimiter.Allow(entry) {
|
||||
f.totalDropped.Add(1)
|
||||
return core.TransportEvent{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 2: Filtering
|
||||
if f.filterChain != nil {
|
||||
if !f.filterChain.Apply(entry) {
|
||||
f.totalDropped.Add(1)
|
||||
return core.TransportEvent{}, false
|
||||
}
|
||||
}
|
||||
|
||||
// Stage 3: Formatting
|
||||
formatted, err := f.formatter.Format(entry)
|
||||
if err != nil {
|
||||
f.logger.Error("msg", "Failed to format log entry",
|
||||
"component", "flow",
|
||||
"error", err)
|
||||
f.totalDropped.Add(1)
|
||||
return core.TransportEvent{}, false
|
||||
}
|
||||
|
||||
f.totalFormatted.Add(1)
|
||||
|
||||
// Create transport event
|
||||
event := core.TransportEvent{
|
||||
Time: entry.Time,
|
||||
Payload: formatted,
|
||||
Entry: entry, // Carry structured entry so chain sinks are format-independent
|
||||
}
|
||||
|
||||
return event, true
|
||||
}
|
||||
|
||||
// StartHeartbeat starts the heartbeat generator if configured
|
||||
// Returns channel that emits heartbeat events
|
||||
func (f *Flow) StartHeartbeat(ctx context.Context) <-chan core.TransportEvent {
|
||||
if f.heartbeat == nil {
|
||||
return nil
|
||||
}
|
||||
return f.heartbeat.Start(ctx)
|
||||
}
|
||||
|
||||
// GetStats returns flow statistics
|
||||
func (f *Flow) GetStats() map[string]any {
|
||||
stats := map[string]any{
|
||||
"total_processed": f.totalProcessed.Load(),
|
||||
"total_dropped": f.totalDropped.Load(),
|
||||
"total_formatted": f.totalFormatted.Load(),
|
||||
}
|
||||
|
||||
if f.rateLimiter != nil {
|
||||
stats["rate_limiter"] = f.rateLimiter.GetStats()
|
||||
}
|
||||
|
||||
if f.filterChain != nil {
|
||||
stats["filters"] = f.filterChain.GetStats()
|
||||
}
|
||||
|
||||
if f.formatter != nil {
|
||||
stats["formatter"] = f.formatter.Name()
|
||||
}
|
||||
|
||||
if f.heartbeat != nil {
|
||||
stats["heartbeat_enabled"] = true
|
||||
stats["heartbeat_interval_ms"] = f.heartbeat.IntervalMS()
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
package flow
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/format"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/lixenwraith/log/formatter"
|
||||
)
|
||||
|
||||
const (
|
||||
MinHeartbeatIntervalMS = 100
|
||||
DefaultHeartbeatIntervalMS = 1000
|
||||
DefaultHeartbeatFormat = "txt"
|
||||
)
|
||||
|
||||
// HeartbeatGenerator produces periodic heartbeat events
|
||||
type HeartbeatGenerator struct {
|
||||
config *config.HeartbeatConfig
|
||||
formatter format.Formatter // Use flow's formatter
|
||||
logger *log.Logger
|
||||
beatCount atomic.Uint64
|
||||
lastBeat atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewHeartbeatGenerator creates a new heartbeat generator
|
||||
func NewHeartbeatGenerator(cfg *config.HeartbeatConfig, formatter format.Formatter, logger *log.Logger) (*HeartbeatGenerator, error) {
|
||||
if cfg == nil || !cfg.Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Validate
|
||||
if cfg.IntervalMS == 0 {
|
||||
cfg.IntervalMS = DefaultHeartbeatIntervalMS
|
||||
} else if cfg.IntervalMS < MinHeartbeatIntervalMS {
|
||||
return nil, fmt.Errorf("interval_ms: must be >= %d, got %d", MinHeartbeatIntervalMS, cfg.IntervalMS)
|
||||
}
|
||||
|
||||
validateFormat := lconfig.OneOf("txt", "json", "raw", "")
|
||||
if err := validateFormat(cfg.Format); err != nil {
|
||||
return nil, fmt.Errorf("format: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if cfg.Format == "" {
|
||||
cfg.Format = DefaultHeartbeatFormat
|
||||
}
|
||||
|
||||
hg := &HeartbeatGenerator{
|
||||
config: cfg,
|
||||
formatter: formatter,
|
||||
logger: logger,
|
||||
}
|
||||
hg.lastBeat.Store(time.Time{})
|
||||
return hg, nil
|
||||
}
|
||||
|
||||
// Start begins generating heartbeat events
|
||||
func (hg *HeartbeatGenerator) Start(ctx context.Context) <-chan core.TransportEvent {
|
||||
ch := make(chan core.TransportEvent)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
ticker := time.NewTicker(time.Duration(hg.config.IntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case t := <-ticker.C:
|
||||
event := hg.generateHeartbeat(t)
|
||||
select {
|
||||
case ch <- event:
|
||||
hg.beatCount.Add(1)
|
||||
hg.lastBeat.Store(t)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// generateHeartbeat creates a heartbeat transport event
|
||||
func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent {
|
||||
// Create heartbeat as LogEntry for consistent formatting
|
||||
entry := core.LogEntry{
|
||||
Time: t,
|
||||
Source: "heartbeat",
|
||||
Level: "INFO",
|
||||
Message: "heartbeat",
|
||||
}
|
||||
|
||||
// Add stats if configured
|
||||
if hg.config.IncludeStats {
|
||||
fields := map[string]any{
|
||||
"type": "heartbeat",
|
||||
"beat_count": hg.beatCount.Load(),
|
||||
}
|
||||
|
||||
if last, ok := hg.lastBeat.Load().(time.Time); ok && !last.IsZero() {
|
||||
fields["interval_ms"] = t.Sub(last).Milliseconds()
|
||||
}
|
||||
|
||||
fieldsJSON, _ := json.Marshal(fields)
|
||||
entry.Fields = fieldsJSON
|
||||
}
|
||||
|
||||
// Use formatter to generate payload
|
||||
var payload []byte
|
||||
var err error
|
||||
|
||||
// Check if we need special formatting for heartbeat
|
||||
if hg.config.Format == "comment" {
|
||||
// SSE comment format - bypass formatter for this special case
|
||||
if hg.config.IncludeStats {
|
||||
beatNum := hg.beatCount.Load()
|
||||
payload = []byte(": heartbeat " + t.Format(time.RFC3339) +
|
||||
" [#" + strconv.FormatUint(beatNum, 10) + "]\n")
|
||||
} else {
|
||||
payload = []byte(": heartbeat " + t.Format(time.RFC3339) + "\n")
|
||||
}
|
||||
} else {
|
||||
// Use flow's formatter for consistent formatting
|
||||
if adapter, ok := hg.formatter.(*format.FormatterAdapter); ok {
|
||||
// Customize flags for heartbeat if needed
|
||||
customFlags := int64(0)
|
||||
if !hg.config.IncludeTimestamp {
|
||||
// Remove timestamp flag if not wanted
|
||||
customFlags = formatter.FlagShowLevel
|
||||
} else {
|
||||
customFlags = formatter.FlagDefault
|
||||
}
|
||||
payload, err = adapter.FormatWithFlags(entry, customFlags)
|
||||
} else {
|
||||
// Fallback to standard format
|
||||
payload, err = hg.formatter.Format(entry)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
hg.logger.Error("msg", "Failed to format heartbeat",
|
||||
"error", err)
|
||||
// Fallback to simple text
|
||||
payload = []byte("heartbeat: " + t.Format(time.RFC3339) + "\n")
|
||||
}
|
||||
}
|
||||
|
||||
return core.TransportEvent{
|
||||
Time: t,
|
||||
Payload: payload,
|
||||
Entry: entry, // heartbeats traverse chain links as structured entries
|
||||
}
|
||||
}
|
||||
|
||||
// IntervalMS returns the heartbeat interval in milliseconds
|
||||
func (hg *HeartbeatGenerator) IntervalMS() int64 {
|
||||
return hg.config.IntervalMS
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package flow
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/tokenbucket"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// RateLimiter enforces rate limits on log entries flowing through a pipeline
|
||||
type RateLimiter struct {
|
||||
bucket *tokenbucket.TokenBucket
|
||||
policy config.RateLimitPolicy
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
maxEntrySizeBytes int64
|
||||
droppedBySizeCount atomic.Uint64
|
||||
droppedCount atomic.Uint64
|
||||
}
|
||||
|
||||
// NewRateLimiter creates a new pipeline-level rate limiter from configuration
|
||||
func NewRateLimiter(cfg config.RateLimitConfig, logger *log.Logger) (*RateLimiter, error) {
|
||||
// Rate <= 0 means disabled
|
||||
if cfg.Rate <= 0 {
|
||||
return nil, nil // No rate limit
|
||||
}
|
||||
|
||||
// Validate
|
||||
if err := lconfig.NonNegative(cfg.Rate); err != nil {
|
||||
return nil, fmt.Errorf("rate: %w", err)
|
||||
}
|
||||
if err := lconfig.NonNegative(cfg.Burst); err != nil {
|
||||
return nil, fmt.Errorf("burst: %w", err)
|
||||
}
|
||||
if err := lconfig.NonNegative(cfg.MaxEntrySizeBytes); err != nil {
|
||||
return nil, fmt.Errorf("max_entry_size_bytes: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
burst := cfg.Burst
|
||||
if burst <= 0 {
|
||||
burst = cfg.Rate
|
||||
}
|
||||
|
||||
var policy config.RateLimitPolicy
|
||||
switch strings.ToLower(cfg.Policy) {
|
||||
case "drop":
|
||||
policy = config.PolicyDrop
|
||||
case "pass", "":
|
||||
policy = config.PolicyPass
|
||||
default:
|
||||
return nil, fmt.Errorf("policy: must be one of [drop, pass], got %s", cfg.Policy)
|
||||
}
|
||||
|
||||
l := &RateLimiter{
|
||||
bucket: tokenbucket.New(burst, cfg.Rate),
|
||||
policy: policy,
|
||||
logger: logger,
|
||||
maxEntrySizeBytes: cfg.MaxEntrySizeBytes,
|
||||
}
|
||||
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Allow checks if a log entry is permitted to pass based on the rate limit
|
||||
func (l *RateLimiter) Allow(entry core.LogEntry) bool {
|
||||
if l == nil || l.policy == config.PolicyPass {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check size limit first
|
||||
if l.maxEntrySizeBytes > 0 && entry.RawSize > l.maxEntrySizeBytes {
|
||||
l.droppedBySizeCount.Add(1)
|
||||
return false
|
||||
}
|
||||
|
||||
// Check rate limit if configured
|
||||
if l.bucket != nil {
|
||||
if l.bucket.Allow() {
|
||||
return true
|
||||
}
|
||||
// Not enough tokens, drop the entry
|
||||
l.droppedCount.Add(1)
|
||||
return false
|
||||
}
|
||||
|
||||
// No rate limit configured, size check passed
|
||||
return true
|
||||
}
|
||||
|
||||
// GetStats returns statistics for the rate limiter
|
||||
func (l *RateLimiter) GetStats() map[string]any {
|
||||
if l == nil {
|
||||
return map[string]any{
|
||||
"enabled": false,
|
||||
}
|
||||
}
|
||||
|
||||
stats := map[string]any{
|
||||
"enabled": true,
|
||||
"rate": l.bucket.Rate(),
|
||||
"burst": l.bucket.Capacity(),
|
||||
"dropped_total": l.droppedCount.Load(),
|
||||
"dropped_by_size_total": l.droppedBySizeCount.Load(),
|
||||
"policy": policyString(l.policy),
|
||||
"max_entry_size_bytes": l.maxEntrySizeBytes,
|
||||
}
|
||||
|
||||
if l.bucket != nil {
|
||||
stats["available_tokens"] = l.bucket.Tokens()
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// policyString returns the string representation of a rate limit policy
|
||||
func policyString(p config.RateLimitPolicy) string {
|
||||
switch p {
|
||||
case config.PolicyDrop:
|
||||
return "drop"
|
||||
case config.PolicyPass:
|
||||
return "pass"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log/formatter"
|
||||
"github.com/lixenwraith/log/sanitizer"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultFormatType = "raw"
|
||||
)
|
||||
|
||||
// FormatterAdapter wraps log/formatter for logwisp compatibility
|
||||
type FormatterAdapter struct {
|
||||
formatter *formatter.Formatter
|
||||
format string
|
||||
flags int64
|
||||
mu sync.Mutex // formatter reuses internal buffer, not goroutine-safe
|
||||
}
|
||||
|
||||
// NewFormatterAdapter creates adapter from config
|
||||
func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
|
||||
// Validate
|
||||
if cfg.Type != "" {
|
||||
validateType := lconfig.OneOf("json", "txt", "text", "raw")
|
||||
if err := validateType(cfg.Type); err != nil {
|
||||
return nil, fmt.Errorf("type: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.SanitizerPolicy != "" {
|
||||
validatePolicy := lconfig.OneOf("raw", "json", "txt", "shell")
|
||||
if err := validatePolicy(cfg.SanitizerPolicy); err != nil {
|
||||
return nil, fmt.Errorf("sanitizer_policy: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if cfg.Type == "" {
|
||||
cfg.Type = DefaultFormatType
|
||||
}
|
||||
|
||||
// Create sanitizer based on policy
|
||||
var s *sanitizer.Sanitizer
|
||||
if cfg.SanitizerPolicy != "" {
|
||||
s = sanitizer.New().Policy(sanitizer.PolicyPreset(cfg.SanitizerPolicy))
|
||||
} else {
|
||||
// Default sanitizer policy based on format type
|
||||
switch cfg.Type {
|
||||
case "json":
|
||||
s = sanitizer.New().Policy(sanitizer.PolicyJSON)
|
||||
case "txt", "text":
|
||||
s = sanitizer.New().Policy(sanitizer.PolicyTxt)
|
||||
default:
|
||||
s = sanitizer.New().Policy(sanitizer.PolicyRaw)
|
||||
}
|
||||
}
|
||||
|
||||
// Create formatter with sanitizer
|
||||
f := formatter.New(s).Type(cfg.Type)
|
||||
|
||||
if cfg.TimestampFormat != "" {
|
||||
f.TimestampFormat(cfg.TimestampFormat)
|
||||
}
|
||||
|
||||
// Build flags from config
|
||||
flags := cfg.Flags
|
||||
if flags == 0 {
|
||||
if cfg.Type == "raw" {
|
||||
flags = formatter.FlagRaw
|
||||
} else {
|
||||
flags = formatter.FlagDefault
|
||||
}
|
||||
}
|
||||
|
||||
return &FormatterAdapter{
|
||||
formatter: f,
|
||||
format: cfg.Type,
|
||||
flags: flags,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Format implements Formatter interface
|
||||
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
|
||||
// Map logwisp LogEntry to formatter args
|
||||
level := mapLevel(entry.Level)
|
||||
// syslog-style origin prefix for chained entries
|
||||
src := sourceLabel(entry)
|
||||
|
||||
// Build args based on whether we have structured fields
|
||||
var args []any
|
||||
effectiveFlags := a.flags
|
||||
|
||||
if len(entry.Fields) > 0 {
|
||||
// Parse fields JSON
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
|
||||
// Use structured JSON format for fields
|
||||
args = []any{entry.Message, fields}
|
||||
// Add structured flag to properly format fields as JSON object
|
||||
effectiveFlags |= formatter.FlagStructuredJSON
|
||||
return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil
|
||||
}
|
||||
}
|
||||
if args == nil {
|
||||
args = []any{entry.Message}
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
out := bytes.Clone(a.formatter.Format(effectiveFlags, entry.Time, level, src, args))
|
||||
a.mu.Unlock()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// FormatWithFlags allows custom flags for specific formatting needs
|
||||
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
|
||||
level := mapLevel(entry.Level)
|
||||
src := sourceLabel(entry)
|
||||
|
||||
var args []any
|
||||
if len(entry.Fields) > 0 {
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(entry.Fields, &fields); err == nil && len(fields) > 0 {
|
||||
args = []any{entry.Message, fields}
|
||||
customFlags |= formatter.FlagStructuredJSON
|
||||
}
|
||||
}
|
||||
if args == nil {
|
||||
args = []any{entry.Message}
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
out := bytes.Clone(a.formatter.Format(customFlags, entry.Time, level, src, args))
|
||||
a.mu.Unlock()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Name returns formatter type
|
||||
func (a *FormatterAdapter) Name() string {
|
||||
return a.format
|
||||
}
|
||||
|
||||
// mapLevel maps string level to int64
|
||||
func mapLevel(level string) int64 {
|
||||
switch level {
|
||||
case "DEBUG", "debug":
|
||||
return -4
|
||||
case "INFO", "info":
|
||||
return 0
|
||||
case "WARN", "warn", "WARNING", "warning":
|
||||
return 4
|
||||
case "ERROR", "error":
|
||||
return 8
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// sourceLabel prefixes origin node onto source (syslog HOSTNAME + TAG convention)
|
||||
func sourceLabel(entry core.LogEntry) string {
|
||||
if entry.Node == "" {
|
||||
return entry.Source
|
||||
}
|
||||
return entry.Node + "/" + entry.Source
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
)
|
||||
|
||||
// Formatter defines the interface for transforming a LogEntry into a byte slice
|
||||
type Formatter interface {
|
||||
// Format takes a LogEntry and returns the formatted log as a byte slice
|
||||
Format(entry core.LogEntry) ([]byte, error)
|
||||
|
||||
// Name returns the formatter's type name (e.g., "json", "raw")
|
||||
Name() string
|
||||
}
|
||||
|
||||
// NewFormatter creates a Formatter using formatter/sanitizer packages
|
||||
func NewFormatter(cfg *config.FormatConfig) (Formatter, error) {
|
||||
if cfg == nil {
|
||||
cfg = &config.FormatConfig{
|
||||
Type: DefaultFormatType,
|
||||
Flags: 0,
|
||||
SanitizerPolicy: "raw",
|
||||
}
|
||||
}
|
||||
|
||||
return NewFormatterAdapter(cfg)
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/flow"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
"logwisp/internal/source"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Pipeline manages the flow of data from sources, through filters, to sinks
|
||||
type Pipeline struct {
|
||||
Config *config.PipelineConfig
|
||||
|
||||
// Components
|
||||
Registry *Registry
|
||||
Sources map[string]source.Source // Track instances by ID
|
||||
Sinks map[string]sink.Sink
|
||||
Sessions *session.Manager
|
||||
|
||||
// Pipeline flow
|
||||
Flow *flow.Flow
|
||||
Stats *PipelineStats
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
running atomic.Bool
|
||||
}
|
||||
|
||||
// PipelineStats contains runtime statistics for a pipeline
|
||||
type PipelineStats struct {
|
||||
StartTime time.Time
|
||||
TotalEntriesDroppedBySink atomic.Uint64
|
||||
SourceStats []source.SourceStats
|
||||
SinkStats []sink.SinkStats
|
||||
FlowStats map[string]any
|
||||
}
|
||||
|
||||
// NewPipeline creates a new pipeline with registry support
|
||||
func NewPipeline(
|
||||
cfg *config.PipelineConfig,
|
||||
logger *log.Logger,
|
||||
) (*Pipeline, error) {
|
||||
// Create pipeline context
|
||||
pipelineCtx, pipelineCancel := context.WithCancel(context.Background())
|
||||
|
||||
// Create session manager with default timeout
|
||||
sessionManager := session.NewManager(core.SessionDefaultMaxIdleTime)
|
||||
|
||||
// Create pipeline instance with registry
|
||||
pipeline := &Pipeline{
|
||||
Config: cfg,
|
||||
Registry: NewRegistry(cfg.Name, logger),
|
||||
Sessions: sessionManager,
|
||||
Sources: make(map[string]source.Source),
|
||||
Sinks: make(map[string]sink.Sink),
|
||||
Stats: &PipelineStats{},
|
||||
logger: logger,
|
||||
ctx: pipelineCtx,
|
||||
cancel: pipelineCancel,
|
||||
}
|
||||
|
||||
// Create flow processor
|
||||
flowProcessor, err := flow.NewFlow(cfg.Flow, logger)
|
||||
if err != nil {
|
||||
// If flow fails, stop session manager
|
||||
sessionManager.Stop()
|
||||
return nil, fmt.Errorf("failed to create flow processor: %w", err)
|
||||
}
|
||||
pipeline.Flow = flowProcessor
|
||||
|
||||
// Initialize sources and sinks
|
||||
if err := pipeline.initializeComponents(); err != nil {
|
||||
pipelineCancel()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pipeline, nil
|
||||
}
|
||||
|
||||
func (p *Pipeline) initializeComponents() error {
|
||||
// Create sources based on plugin config if available
|
||||
if len(p.Config.PluginSources) > 0 {
|
||||
for _, srcCfg := range p.Config.PluginSources {
|
||||
// Create session proxy for this source instance
|
||||
sessionProxy := session.NewProxy(p.Sessions, srcCfg.ID)
|
||||
|
||||
src, err := p.Registry.CreateSource(
|
||||
srcCfg.ID,
|
||||
srcCfg.Type,
|
||||
srcCfg.Config,
|
||||
p.logger,
|
||||
sessionProxy,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create source %s: %w", srcCfg.ID, err)
|
||||
}
|
||||
|
||||
// Check and inject capabilities using core interfaces
|
||||
if err := p.initSourceCapabilities(src, srcCfg); err != nil {
|
||||
return fmt.Errorf("failed to initiate capabilities for source %s: %w", srcCfg.ID, err)
|
||||
}
|
||||
|
||||
p.Sources[srcCfg.ID] = src
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("no plugin sources defined")
|
||||
}
|
||||
|
||||
// Create sinks based on plugin config if available
|
||||
if len(p.Config.PluginSinks) > 0 {
|
||||
for _, sinkCfg := range p.Config.PluginSinks {
|
||||
// Create session proxy for this sink instance
|
||||
sessionProxy := session.NewProxy(p.Sessions, sinkCfg.ID)
|
||||
|
||||
snk, err := p.Registry.CreateSink(
|
||||
sinkCfg.ID,
|
||||
sinkCfg.Type,
|
||||
sinkCfg.Config,
|
||||
p.logger,
|
||||
sessionProxy,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create sink %s: %w", sinkCfg.ID, err)
|
||||
}
|
||||
|
||||
// Check and inject capabilities using core interfaces
|
||||
if err := p.initSinkCapabilities(snk, sinkCfg); err != nil {
|
||||
return fmt.Errorf("failed to initiate capabilities for sink %s: %w", sinkCfg.ID, err)
|
||||
}
|
||||
|
||||
p.Sinks[sinkCfg.ID] = snk
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("no plugin sinks defined")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initSourceCapabilities checks and injects optional capabilities
|
||||
func (p *Pipeline) initSourceCapabilities(s source.Source, cfg config.PluginSourceConfig) error {
|
||||
// Initiate and activate source capabilities
|
||||
for _, c := range s.Capabilities() {
|
||||
switch c {
|
||||
// Network capabilities
|
||||
case core.CapNetLimit, core.CapTLS, core.CapAuth:
|
||||
continue // No-op for now, placeholder
|
||||
|
||||
// Session capabilities
|
||||
case core.CapSessionAware:
|
||||
case core.CapMultiSession:
|
||||
continue // TODO
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown capability type: %s", c)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initSinkCapabilities checks and injects optional capabilities
|
||||
func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig) error {
|
||||
// Initiate and activate sink capabilities
|
||||
for _, c := range s.Capabilities() {
|
||||
switch c {
|
||||
// Network capabilities
|
||||
case core.CapNetLimit, core.CapTLS, core.CapAuth:
|
||||
continue // No-op for now, placeholder
|
||||
|
||||
// Session capabilities
|
||||
case core.CapSessionAware:
|
||||
case core.CapMultiSession:
|
||||
continue // TODO
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown capability type: %s", c)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// run is the central processing loop that connects sources, flow, and sinks
|
||||
func (p *Pipeline) run() {
|
||||
defer p.wg.Done()
|
||||
defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name)
|
||||
|
||||
var componentWg sync.WaitGroup
|
||||
// Start a goroutine for each source to fan-in data
|
||||
for _, src := range p.Sources {
|
||||
componentWg.Add(1)
|
||||
go func(s source.Source) {
|
||||
defer componentWg.Done()
|
||||
ch := s.Subscribe()
|
||||
// Range allows in-flight data to drain cleanly once Source.Stop() closes the channel
|
||||
for entry := range ch {
|
||||
if event, passed := p.Flow.Process(entry); passed {
|
||||
// Use non-blocking dispatcher
|
||||
p.dispatch(event)
|
||||
}
|
||||
}
|
||||
}(src)
|
||||
}
|
||||
|
||||
var hbWg sync.WaitGroup
|
||||
// Start heartbeat generator if enabled
|
||||
if heartbeatCh := p.Flow.StartHeartbeat(p.ctx); heartbeatCh != nil {
|
||||
hbWg.Add(1)
|
||||
go func() {
|
||||
defer hbWg.Done()
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-heartbeatCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Use non-blocking dispatcher
|
||||
p.dispatch(event)
|
||||
case <-p.ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
componentWg.Wait()
|
||||
|
||||
// Terminate internal contexts (heartbeat) once flow is complete
|
||||
p.cancel()
|
||||
hbWg.Wait()
|
||||
}
|
||||
|
||||
// dispatch performs a non-blocking send to all sinks.
|
||||
// A full/stalled sink must never block the run loop or starve sibling sinks.
|
||||
func (p *Pipeline) dispatch(event core.TransportEvent) {
|
||||
for _, snk := range p.Sinks {
|
||||
select {
|
||||
case snk.Input() <- event:
|
||||
default:
|
||||
// Buffer full - drop to avoid deadlocking the pipeline
|
||||
p.Stats.TotalEntriesDroppedBySink.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the pipeline operation and all its components including flow, sources, and sinks
|
||||
func (p *Pipeline) Start() error {
|
||||
if !p.running.CompareAndSwap(false, true) {
|
||||
return fmt.Errorf("pipeline %s is already running", p.Config.Name)
|
||||
}
|
||||
|
||||
p.logger.Info("msg", "Starting pipeline", "pipeline", p.Config.Name)
|
||||
p.ctx, p.cancel = context.WithCancel(context.Background())
|
||||
|
||||
// Start all sinks
|
||||
for id, s := range p.Sinks {
|
||||
if err := s.Start(p.ctx); err != nil {
|
||||
return fmt.Errorf("failed to start sink %s: %w", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start all sources
|
||||
for id, src := range p.Sources {
|
||||
if err := src.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start source %s: %w", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Start the central processing loop
|
||||
p.Stats.StartTime = time.Now()
|
||||
p.wg.Add(1)
|
||||
go p.run()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop stops the pipeline operation and all its components including flow, sources, and sinks
|
||||
func (p *Pipeline) Stop() error {
|
||||
if !p.running.CompareAndSwap(true, false) {
|
||||
return fmt.Errorf("pipeline %s is not running", p.Config.Name)
|
||||
}
|
||||
|
||||
p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name)
|
||||
|
||||
// 1. Stop all sources concurrently to halt new data ingress and close their channels
|
||||
var sourceWg sync.WaitGroup
|
||||
for _, src := range p.Sources {
|
||||
sourceWg.Add(1)
|
||||
go func(s source.Source) {
|
||||
defer sourceWg.Done()
|
||||
s.Stop()
|
||||
}(src)
|
||||
}
|
||||
sourceWg.Wait()
|
||||
|
||||
// 2. Wait for the run loop to finish processing and sending all in-flight data
|
||||
// run() inherently calls p.cancel() when the source channels are empty
|
||||
p.wg.Wait()
|
||||
|
||||
// 3. Stop all sinks concurrently now that no new data will be sent
|
||||
var sinkWg sync.WaitGroup
|
||||
for _, s := range p.Sinks {
|
||||
sinkWg.Add(1)
|
||||
go func(snk sink.Sink) {
|
||||
defer sinkWg.Done()
|
||||
snk.Stop()
|
||||
}(s)
|
||||
}
|
||||
sinkWg.Wait()
|
||||
|
||||
p.logger.Info("msg", "Pipeline stopped", "pipeline", p.Config.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown gracefully stops the pipeline and all its components, deinitializing them for app shutdown or complete pipeline removal by service
|
||||
func (p *Pipeline) Shutdown() {
|
||||
p.logger.Info("msg", "Shutting down pipeline",
|
||||
"component", "pipeline",
|
||||
"pipeline", p.Config.Name)
|
||||
|
||||
// Ensure the pipeline is stopped before shutting down
|
||||
if p.running.Load() {
|
||||
if err := p.Stop(); err != nil {
|
||||
p.logger.Error("msg", "Error stopping pipeline during shutdown", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop long-running components
|
||||
if p.Sessions != nil {
|
||||
p.Sessions.Stop()
|
||||
}
|
||||
|
||||
p.logger.Info("msg", "Pipeline shutdown complete",
|
||||
"component", "pipeline",
|
||||
"pipeline", p.Config.Name)
|
||||
}
|
||||
|
||||
// GetStats returns a map of pipeline statistics
|
||||
func (p *Pipeline) GetStats() map[string]any {
|
||||
// Recovery to handle concurrent access during shutdown
|
||||
// When service is shutting down, sources/sinks might be nil or partially stopped
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
p.logger.Error("msg", "Panic getting pipeline stats",
|
||||
"pipeline", p.Config.Name,
|
||||
"panic", r)
|
||||
}
|
||||
}()
|
||||
|
||||
// 1. Live collect source stats
|
||||
sources := make([]map[string]any, 0, len(p.Sources))
|
||||
for _, src := range p.Sources {
|
||||
if src == nil {
|
||||
continue
|
||||
}
|
||||
s := src.GetStats()
|
||||
sources = append(sources, map[string]any{
|
||||
"id": s.ID,
|
||||
"type": s.Type,
|
||||
"total_entries": s.TotalEntries,
|
||||
"dropped_entries": s.DroppedEntries,
|
||||
"start_time": s.StartTime,
|
||||
"last_entry_time": s.LastEntryTime,
|
||||
"details": s.Details,
|
||||
})
|
||||
}
|
||||
|
||||
// 2. Live collect sink stats
|
||||
sinks := make([]map[string]any, 0, len(p.Sinks))
|
||||
for _, snk := range p.Sinks {
|
||||
if snk == nil {
|
||||
continue
|
||||
}
|
||||
s := snk.GetStats()
|
||||
sinks = append(sinks, map[string]any{
|
||||
"id": s.ID,
|
||||
"type": s.Type,
|
||||
"total_processed": s.TotalProcessed,
|
||||
"active_connections": s.ActiveConnections,
|
||||
"start_time": s.StartTime,
|
||||
"last_processed": s.LastProcessed,
|
||||
"details": s.Details,
|
||||
})
|
||||
}
|
||||
|
||||
// 3. Collect flow stats and calculate filtered total
|
||||
var flowStats map[string]any
|
||||
var totalFiltered uint64
|
||||
var totalProcessed uint64
|
||||
|
||||
if p.Flow != nil {
|
||||
flowStats = p.Flow.GetStats()
|
||||
|
||||
// Map the top-level processed counter directly from Flow's source of truth
|
||||
if tp, ok := flowStats["total_processed"].(uint64); ok {
|
||||
totalProcessed = tp
|
||||
}
|
||||
|
||||
// Calculate total dropped specifically by the filter chain
|
||||
if filters, ok := flowStats["filters"].(map[string]any); ok {
|
||||
if totalPassed, ok := filters["total_passed"].(uint64); ok {
|
||||
if tProc, ok := filters["total_processed"].(uint64); ok {
|
||||
totalFiltered = tProc - totalPassed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Calculate Uptime
|
||||
var uptime int
|
||||
if p.running.Load() && !p.Stats.StartTime.IsZero() {
|
||||
uptime = int(time.Since(p.Stats.StartTime).Seconds())
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"name": p.Config.Name,
|
||||
"running": p.running.Load(),
|
||||
"uptime_seconds": uptime,
|
||||
"total_processed": totalProcessed,
|
||||
"total_filtered": totalFiltered,
|
||||
"total_dropped_by_sink": p.Stats.TotalEntriesDroppedBySink.Load(),
|
||||
"source_count": len(p.Sources),
|
||||
"sources": sources,
|
||||
"sink_count": len(p.Sinks),
|
||||
"sinks": sinks,
|
||||
"flow": flowStats,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
"logwisp/internal/source"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// SourceFactory creates source instances with required dependencies
|
||||
type SourceFactory func(
|
||||
id string,
|
||||
config map[string]any,
|
||||
logger *log.Logger,
|
||||
sessions *session.Proxy,
|
||||
) (source.Source, error)
|
||||
|
||||
// SinkFactory creates sink instances with required dependencies
|
||||
type SinkFactory func(
|
||||
id string,
|
||||
config map[string]any,
|
||||
logger *log.Logger,
|
||||
sessions *session.Proxy,
|
||||
) (sink.Sink, error)
|
||||
|
||||
// Registry manages plugin instances for a single pipeline
|
||||
type Registry struct {
|
||||
pipelineName string
|
||||
|
||||
// Instance tracking
|
||||
sourceInstances map[string]source.Source
|
||||
sinkInstances map[string]sink.Sink
|
||||
// Type count tracking (for single instance enforcement)
|
||||
sourceTypeCounts map[string]int
|
||||
sinkTypeCounts map[string]int
|
||||
|
||||
mu sync.RWMutex
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewRegistry creates a new registry for a pipeline
|
||||
func NewRegistry(pipelineName string, logger *log.Logger) *Registry {
|
||||
return &Registry{
|
||||
pipelineName: pipelineName,
|
||||
sourceInstances: make(map[string]source.Source),
|
||||
sinkInstances: make(map[string]sink.Sink),
|
||||
sourceTypeCounts: make(map[string]int),
|
||||
sinkTypeCounts: make(map[string]int),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateSource creates and tracks a source instance
|
||||
func (r *Registry) CreateSource(
|
||||
id string,
|
||||
pluginType string,
|
||||
config map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
// Check for duplicate instance ID
|
||||
if _, exists := r.sourceInstances[id]; exists {
|
||||
return nil, fmt.Errorf("source instance with ID %s already exists", id)
|
||||
}
|
||||
|
||||
// Check single instance constraint
|
||||
if meta, ok := plugin.GetSourceMetadata(pluginType); ok {
|
||||
if meta.MaxInstances == 1 && r.sourceTypeCounts[pluginType] >= 1 {
|
||||
return nil, fmt.Errorf("source type %s only allows single instance", pluginType)
|
||||
}
|
||||
}
|
||||
|
||||
// Get source constructor
|
||||
constructor, ok := plugin.GetSource(pluginType)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown source type: %s", pluginType)
|
||||
}
|
||||
|
||||
// Create instance
|
||||
src, err := constructor(id, config, logger, proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create source %s: %w", id, err)
|
||||
}
|
||||
|
||||
// Track instance
|
||||
r.sourceInstances[id] = src
|
||||
r.sourceTypeCounts[pluginType]++
|
||||
|
||||
r.logger.Info("msg", "Created source instance",
|
||||
"pipeline", r.pipelineName,
|
||||
"id", id,
|
||||
"type", pluginType)
|
||||
|
||||
return src, nil
|
||||
}
|
||||
|
||||
// CreateSink creates and tracks a sink instance
|
||||
func (r *Registry) CreateSink(
|
||||
id string,
|
||||
pluginType string,
|
||||
config map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
// Check for duplicate instance ID
|
||||
if _, exists := r.sinkInstances[id]; exists {
|
||||
return nil, fmt.Errorf("sink instance with ID %s already exists", id)
|
||||
}
|
||||
|
||||
// Check single instance constraint
|
||||
if meta, ok := plugin.GetSinkMetadata(pluginType); ok {
|
||||
if meta.MaxInstances == 1 && r.sinkTypeCounts[pluginType] >= 1 {
|
||||
return nil, fmt.Errorf("sink type %s only allows single instance", pluginType)
|
||||
}
|
||||
}
|
||||
|
||||
// Get sink constructor
|
||||
constructor, ok := plugin.GetSink(pluginType)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown sink type: %s", pluginType)
|
||||
}
|
||||
|
||||
// Create instance
|
||||
snk, err := constructor(id, config, logger, proxy)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create sink %s: %w", id, err)
|
||||
}
|
||||
|
||||
// Track instance
|
||||
r.sinkInstances[id] = snk
|
||||
r.sinkTypeCounts[pluginType]++
|
||||
|
||||
r.logger.Info("msg", "Created sink instance",
|
||||
"pipeline", r.pipelineName,
|
||||
"id", id,
|
||||
"type", pluginType)
|
||||
|
||||
return snk, nil
|
||||
}
|
||||
|
||||
// GetSourceInstance retrieves a source instance by ID
|
||||
func (r *Registry) GetSourceInstance(id string) (source.Source, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
src, exists := r.sourceInstances[id]
|
||||
return src, exists
|
||||
}
|
||||
|
||||
// GetSinkInstance retrieves a sink instance by ID
|
||||
func (r *Registry) GetSinkInstance(id string) (sink.Sink, bool) {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
snk, exists := r.sinkInstances[id]
|
||||
return snk, exists
|
||||
}
|
||||
|
||||
// GetAllSources returns all source instances
|
||||
func (r *Registry) GetAllSources() map[string]source.Source {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
sources := make(map[string]source.Source, len(r.sourceInstances))
|
||||
for k, v := range r.sourceInstances {
|
||||
sources[k] = v
|
||||
}
|
||||
return sources
|
||||
}
|
||||
|
||||
// GetAllSinks returns all sink instances
|
||||
func (r *Registry) GetAllSinks() map[string]sink.Sink {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
sinks := make(map[string]sink.Sink, len(r.sinkInstances))
|
||||
for k, v := range r.sinkInstances {
|
||||
sinks[k] = v
|
||||
}
|
||||
return sinks
|
||||
}
|
||||
|
||||
// RemoveSource removes a source instance
|
||||
func (r *Registry) RemoveSource(id string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
// Decrement type count
|
||||
if src, exists := r.sourceInstances[id]; exists {
|
||||
stats := src.GetStats()
|
||||
if pluginType, ok := stats.Details["type"].(string); ok {
|
||||
r.sourceTypeCounts[pluginType]--
|
||||
}
|
||||
}
|
||||
|
||||
delete(r.sourceInstances, id)
|
||||
}
|
||||
|
||||
// RemoveSink removes a sink instance
|
||||
func (r *Registry) RemoveSink(id string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
// Decrement type count
|
||||
if snk, exists := r.sinkInstances[id]; exists {
|
||||
stats := snk.GetStats()
|
||||
if pluginType, ok := stats.Details["type"].(string); ok {
|
||||
r.sinkTypeCounts[pluginType]--
|
||||
}
|
||||
}
|
||||
|
||||
delete(r.sinkInstances, id)
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
"logwisp/internal/source"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// SourceFactory creates source instances
|
||||
type SourceFactory func(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
sessions *session.Proxy,
|
||||
) (source.Source, error)
|
||||
|
||||
// SinkFactory creates sink instances
|
||||
type SinkFactory func(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
sessions *session.Proxy,
|
||||
) (sink.Sink, error)
|
||||
|
||||
// PluginMetadata stores metadata about a plugin type
|
||||
type PluginMetadata struct {
|
||||
Capabilities []core.Capability
|
||||
MaxInstances int // 0 = unlimited, 1 = single instance only
|
||||
}
|
||||
|
||||
// registry encapsulates all plugin factories with lazy initialization
|
||||
type registry struct {
|
||||
sourceFactories map[string]SourceFactory
|
||||
sinkFactories map[string]SinkFactory
|
||||
sourceMetadata map[string]*PluginMetadata
|
||||
sinkMetadata map[string]*PluginMetadata
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
var (
|
||||
globalRegistry *registry
|
||||
once sync.Once
|
||||
)
|
||||
|
||||
// getRegistry returns the singleton registry, initializing on first access
|
||||
func getRegistry() *registry {
|
||||
once.Do(func() {
|
||||
globalRegistry = ®istry{
|
||||
sourceFactories: make(map[string]SourceFactory),
|
||||
sinkFactories: make(map[string]SinkFactory),
|
||||
sourceMetadata: make(map[string]*PluginMetadata),
|
||||
sinkMetadata: make(map[string]*PluginMetadata),
|
||||
}
|
||||
})
|
||||
return globalRegistry
|
||||
}
|
||||
|
||||
// RegisterSource registers a source factory function
|
||||
func RegisterSource(name string, constructor SourceFactory) error {
|
||||
r := getRegistry()
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if _, exists := r.sourceFactories[name]; exists {
|
||||
return fmt.Errorf("source type %s already registered", name)
|
||||
}
|
||||
r.sourceFactories[name] = constructor
|
||||
|
||||
// Set default metadata
|
||||
r.sourceMetadata[name] = &PluginMetadata{
|
||||
MaxInstances: 0, // Unlimited by default
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RegisterSink registers a sink factory function
|
||||
func RegisterSink(name string, constructor SinkFactory) error {
|
||||
r := getRegistry()
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if _, exists := r.sinkFactories[name]; exists {
|
||||
return fmt.Errorf("sink type %s already registered", name)
|
||||
}
|
||||
r.sinkFactories[name] = constructor
|
||||
|
||||
// Set default metadata
|
||||
r.sinkMetadata[name] = &PluginMetadata{
|
||||
MaxInstances: 0, // Unlimited by default
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSourceMetadata sets metadata for a source type (call after RegisterSource)
|
||||
func SetSourceMetadata(name string, metadata *PluginMetadata) error {
|
||||
r := getRegistry()
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if _, exists := r.sourceFactories[name]; !exists {
|
||||
return fmt.Errorf("source type %s not registered", name)
|
||||
}
|
||||
r.sourceMetadata[name] = metadata
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetSinkMetadata sets metadata for a sink type (call after RegisterSink)
|
||||
func SetSinkMetadata(name string, metadata *PluginMetadata) error {
|
||||
r := getRegistry()
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if _, exists := r.sinkFactories[name]; !exists {
|
||||
return fmt.Errorf("sink type %s not registered", name)
|
||||
}
|
||||
r.sinkMetadata[name] = metadata
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSource retrieves a source factory function
|
||||
func GetSource(name string) (SourceFactory, bool) {
|
||||
r := getRegistry()
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
constructor, exists := r.sourceFactories[name]
|
||||
return constructor, exists
|
||||
}
|
||||
|
||||
// GetSink retrieves a sink factory function
|
||||
func GetSink(name string) (SinkFactory, bool) {
|
||||
r := getRegistry()
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
constructor, exists := r.sinkFactories[name]
|
||||
return constructor, exists
|
||||
}
|
||||
|
||||
// GetSourceMetadata retrieves metadata for a source type
|
||||
func GetSourceMetadata(name string) (*PluginMetadata, bool) {
|
||||
r := getRegistry()
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
meta, exists := r.sourceMetadata[name]
|
||||
return meta, exists
|
||||
}
|
||||
|
||||
// GetSinkMetadata retrieves metadata for a sink type
|
||||
func GetSinkMetadata(name string) (*PluginMetadata, bool) {
|
||||
r := getRegistry()
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
meta, exists := r.sinkMetadata[name]
|
||||
return meta, exists
|
||||
}
|
||||
|
||||
// ListSources returns all registered source types
|
||||
func ListSources() []string {
|
||||
r := getRegistry()
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
types := make([]string, 0, len(r.sourceFactories))
|
||||
for t := range r.sourceFactories {
|
||||
types = append(types, t)
|
||||
}
|
||||
return types
|
||||
}
|
||||
|
||||
// ListSinks returns all registered sink types
|
||||
func ListSinks() []string {
|
||||
r := getRegistry()
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
|
||||
types := make([]string, 0, len(r.sinkFactories))
|
||||
for t := range r.sinkFactories {
|
||||
types = append(types, t)
|
||||
}
|
||||
return types
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// String sanitizes a string by replacing non-printable characters with hex encoding
|
||||
// Non-printable characters are encoded as <hex> (e.g., newline becomes <0a>)
|
||||
func String(data string) string {
|
||||
// Fast path: check if sanitization is needed
|
||||
needsSanitization := false
|
||||
for _, r := range data {
|
||||
if !strconv.IsPrint(r) {
|
||||
needsSanitization = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !needsSanitization {
|
||||
return data
|
||||
}
|
||||
|
||||
// Pre-allocate builder for efficiency
|
||||
var builder strings.Builder
|
||||
builder.Grow(len(data))
|
||||
|
||||
for _, r := range data {
|
||||
if strconv.IsPrint(r) {
|
||||
builder.WriteRune(r)
|
||||
} else {
|
||||
// Encode non-printable rune as <hex>
|
||||
var runeBytes [utf8.UTFMax]byte
|
||||
n := utf8.EncodeRune(runeBytes[:], r)
|
||||
builder.WriteByte('<')
|
||||
builder.WriteString(hex.EncodeToString(runeBytes[:n]))
|
||||
builder.WriteByte('>')
|
||||
}
|
||||
}
|
||||
|
||||
return builder.String()
|
||||
}
|
||||
|
||||
// Bytes sanitizes a byte slice by converting to string and sanitizing
|
||||
func Bytes(data []byte) []byte {
|
||||
return []byte(String(string(data)))
|
||||
}
|
||||
|
||||
// Rune sanitizes a single rune, returning its string representation
|
||||
func Rune(r rune) string {
|
||||
if strconv.IsPrint(r) {
|
||||
return string(r)
|
||||
}
|
||||
|
||||
var runeBytes [utf8.UTFMax]byte
|
||||
n := utf8.EncodeRune(runeBytes[:], r)
|
||||
return "<" + hex.EncodeToString(runeBytes[:n]) + ">"
|
||||
}
|
||||
|
||||
// IsSafe checks if a string contains only printable characters
|
||||
func IsSafe(data string) bool {
|
||||
for _, r := range data {
|
||||
if !strconv.IsPrint(r) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/pipeline"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// Service manages a collection of log processing pipelines
|
||||
type Service struct {
|
||||
pipelines map[string]*pipeline.Pipeline
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// NewService creates a new, empty service
|
||||
func NewService(ctx context.Context, cfg *config.Config, logger *log.Logger) (*Service, error) {
|
||||
serviceCtx, cancel := context.WithCancel(ctx)
|
||||
svc := &Service{
|
||||
pipelines: make(map[string]*pipeline.Pipeline),
|
||||
ctx: serviceCtx,
|
||||
cancel: cancel,
|
||||
logger: logger,
|
||||
}
|
||||
|
||||
var errs error
|
||||
// Initialize pipelines
|
||||
for _, pipelineCfg := range cfg.Pipelines {
|
||||
pipelineName := pipelineCfg.Name
|
||||
logger.Info("msg", "Initializing pipeline", "pipeline", pipelineName)
|
||||
|
||||
// Create the pipeline
|
||||
if pl, err := pipeline.NewPipeline(&pipelineCfg, logger); err != nil {
|
||||
logger.Error("msg", "Failed to create pipeline",
|
||||
"pipeline", pipelineCfg.Name,
|
||||
"error", err)
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to initialize pipeline %s: %w", pipelineName, err))
|
||||
} else {
|
||||
svc.pipelines[pipelineName] = pl
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("msg", "Service initialization completed", "pipelines", len(svc.pipelines))
|
||||
|
||||
return svc, errs
|
||||
}
|
||||
|
||||
// Start starts all or specific pipelines
|
||||
func (svc *Service) Start(names ...string) error {
|
||||
svc.mu.RLock()
|
||||
defer svc.mu.RUnlock()
|
||||
|
||||
var errs error
|
||||
// If no names are provided, start all pipelines
|
||||
if len(names) == 0 {
|
||||
svc.logger.Info("msg", "Starting all pipelines")
|
||||
for name, p := range svc.pipelines {
|
||||
if err := p.Start(); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to start pipeline %s: %w", name, err))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Start only the specified pipelines
|
||||
svc.logger.Info("msg", "Starting specified pipelines", "pipelines", names)
|
||||
for _, name := range names {
|
||||
if p, exists := svc.pipelines[name]; exists {
|
||||
if err := p.Start(); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to start pipeline %s: %w", name, err))
|
||||
}
|
||||
} else {
|
||||
errs = errors.Join(errs, fmt.Errorf("pipeline %s not found", name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
svc.logger.Debug("msg", "Finished starting pipeline(s)", "pipelines", names)
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// Stop stops all or specific pipeline
|
||||
func (svc *Service) Stop(names ...string) error {
|
||||
svc.mu.RLock()
|
||||
defer svc.mu.RUnlock()
|
||||
|
||||
var errs error
|
||||
|
||||
// If no names are provided, stop all pipelines
|
||||
if len(names) == 0 {
|
||||
svc.logger.Info("msg", "Stopping all pipelines")
|
||||
for name, p := range svc.pipelines {
|
||||
if err := p.Stop(); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to stop pipeline %s: %w", name, err))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Stop only the specified pipelines
|
||||
svc.logger.Info("msg", "Stopping specified pipelines", "pipelines", names)
|
||||
for _, name := range names {
|
||||
if p, exists := svc.pipelines[name]; exists {
|
||||
if err := p.Stop(); err != nil {
|
||||
errs = errors.Join(errs, fmt.Errorf("failed to stop pipeline %s: %w", name, err))
|
||||
}
|
||||
} else {
|
||||
errs = errors.Join(errs, fmt.Errorf("pipeline %s not found", name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
svc.logger.Debug("msg", "Finished stopping pipeline(s)", "pipelines", names)
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
// GetPipeline returns a pipeline by its name
|
||||
func (svc *Service) GetPipeline(name string) (*pipeline.Pipeline, error) {
|
||||
svc.mu.RLock()
|
||||
defer svc.mu.RUnlock()
|
||||
|
||||
pipeline, exists := svc.pipelines[name]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("pipeline '%s' not found", name)
|
||||
}
|
||||
return pipeline, nil
|
||||
}
|
||||
|
||||
// ListPipelines returns the names of all currently managed pipelines
|
||||
func (svc *Service) ListPipelines() []string {
|
||||
svc.mu.RLock()
|
||||
defer svc.mu.RUnlock()
|
||||
|
||||
names := make([]string, 0, len(svc.pipelines))
|
||||
for name := range svc.pipelines {
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// RemovePipeline stops and removes a pipeline from the service
|
||||
func (svc *Service) RemovePipeline(name string) error {
|
||||
svc.mu.Lock()
|
||||
defer svc.mu.Unlock()
|
||||
|
||||
pl, exists := svc.pipelines[name]
|
||||
if !exists {
|
||||
err := fmt.Errorf("pipeline '%s' not found", name)
|
||||
svc.logger.Warn("msg", "Cannot remove non-existent pipeline",
|
||||
"component", "service",
|
||||
"pipeline", name,
|
||||
"error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
svc.logger.Info("msg", "Removing pipeline", "pipeline", name)
|
||||
pl.Shutdown()
|
||||
delete(svc.pipelines, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown gracefully stops all pipelines managed by the service
|
||||
func (svc *Service) Shutdown() {
|
||||
svc.logger.Info("msg", "Service shutdown initiated")
|
||||
|
||||
svc.mu.Lock()
|
||||
pipelines := make([]*pipeline.Pipeline, 0, len(svc.pipelines))
|
||||
for _, pl := range svc.pipelines {
|
||||
pipelines = append(pipelines, pl)
|
||||
}
|
||||
svc.mu.Unlock()
|
||||
|
||||
// Stop all pipelines concurrently
|
||||
var wg sync.WaitGroup
|
||||
for _, pl := range pipelines {
|
||||
wg.Add(1)
|
||||
go func(p *pipeline.Pipeline) {
|
||||
defer wg.Done()
|
||||
p.Shutdown()
|
||||
}(pl)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
svc.cancel()
|
||||
svc.wg.Wait()
|
||||
|
||||
svc.logger.Info("msg", "Service shutdown complete")
|
||||
}
|
||||
|
||||
// GetGlobalStats returns statistics for all pipelines
|
||||
func (svc *Service) GetGlobalStats() map[string]any {
|
||||
svc.mu.RLock()
|
||||
defer svc.mu.RUnlock()
|
||||
|
||||
stats := map[string]any{
|
||||
"pipelines": make(map[string]any),
|
||||
"total_pipelines": len(svc.pipelines),
|
||||
}
|
||||
|
||||
for name, pl := range svc.pipelines {
|
||||
stats["pipelines"].(map[string]any)[name] = pl.GetStats()
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Proxy provides filtered access to session management for a specific plugin instance
|
||||
type Proxy struct {
|
||||
manager *Manager
|
||||
instanceID string
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewProxy creates a session proxy for a specific plugin instance
|
||||
func NewProxy(manager *Manager, instanceID string) *Proxy {
|
||||
return &Proxy{
|
||||
manager: manager,
|
||||
instanceID: instanceID,
|
||||
}
|
||||
}
|
||||
|
||||
// CreateSession creates a new session scoped to this instance
|
||||
func (p *Proxy) CreateSession(remoteAddr string, metadata map[string]any) *Session {
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]any)
|
||||
}
|
||||
|
||||
// Add instance ID to metadata
|
||||
metadata["instance_id"] = p.instanceID
|
||||
|
||||
// Create session with instance-scoped source
|
||||
session := p.manager.CreateSession(remoteAddr, p.instanceID, metadata)
|
||||
session.InstanceID = p.instanceID
|
||||
|
||||
return session
|
||||
}
|
||||
|
||||
// GetSession retrieves a session if it belongs to this instance
|
||||
func (p *Proxy) GetSession(sessionID string) (*Session, bool) {
|
||||
session, exists := p.manager.GetSession(sessionID)
|
||||
if !exists || session.InstanceID != p.instanceID {
|
||||
return nil, false
|
||||
}
|
||||
return session, true
|
||||
}
|
||||
|
||||
// RemoveSession removes a session if it belongs to this instance
|
||||
func (p *Proxy) RemoveSession(sessionID string) bool {
|
||||
if session, exists := p.GetSession(sessionID); exists {
|
||||
p.manager.RemoveSession(session.ID)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetActiveSessions returns all active sessions for this instance
|
||||
func (p *Proxy) GetActiveSessions() []*Session {
|
||||
allSessions := p.manager.GetSessionsBySource(p.instanceID)
|
||||
|
||||
// Filter by instance ID
|
||||
var filtered []*Session
|
||||
for _, session := range allSessions {
|
||||
if session.InstanceID == p.instanceID {
|
||||
filtered = append(filtered, session)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
// UpdateActivity updates activity for a session if it belongs to this instance
|
||||
func (p *Proxy) UpdateActivity(sessionID string) bool {
|
||||
if session, exists := p.GetSession(sessionID); exists {
|
||||
p.manager.UpdateActivity(session.ID)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetInstanceID returns the instance ID this proxy is bound to
|
||||
func (p *Proxy) GetInstanceID() string {
|
||||
return p.instanceID
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
package session
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
)
|
||||
|
||||
// Session represents a connection session
|
||||
type Session struct {
|
||||
InstanceID string // Plugin instance identifier
|
||||
ID string // Unique session identifier
|
||||
RemoteAddr string // Client address
|
||||
CreatedAt time.Time // Session creation time
|
||||
LastActivity time.Time // Last activity timestamp
|
||||
Metadata map[string]any // Optional metadata (e.g., TLS info)
|
||||
|
||||
// Connection context
|
||||
Source string // Source type: "tcp_source", "http_source", "tcp_sink", etc.
|
||||
}
|
||||
|
||||
// Manager handles the lifecycle of sessions
|
||||
type Manager struct {
|
||||
sessions map[string]*Session
|
||||
mu sync.RWMutex
|
||||
|
||||
// Cleanup configuration
|
||||
maxIdleTime time.Duration
|
||||
cleanupTicker *time.Ticker
|
||||
done chan struct{}
|
||||
|
||||
// Expiry callbacks by source type
|
||||
expiryCallbacks map[string]func(sessionID, remoteAddr string)
|
||||
callbacksMu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewManager creates a new session manager with a specified idle timeout
|
||||
func NewManager(maxIdleTime time.Duration) *Manager {
|
||||
if maxIdleTime == 0 {
|
||||
maxIdleTime = core.SessionDefaultMaxIdleTime
|
||||
}
|
||||
|
||||
m := &Manager{
|
||||
sessions: make(map[string]*Session),
|
||||
maxIdleTime: maxIdleTime,
|
||||
done: make(chan struct{}),
|
||||
expiryCallbacks: make(map[string]func(sessionID, remoteAddr string)),
|
||||
}
|
||||
|
||||
// Start cleanup routine
|
||||
m.startCleanup()
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// CreateSession creates and stores a new session for a connection
|
||||
func (m *Manager) CreateSession(remoteAddr string, source string, metadata map[string]any) *Session {
|
||||
session := &Session{
|
||||
ID: generateSessionID(),
|
||||
RemoteAddr: remoteAddr,
|
||||
CreatedAt: time.Now(),
|
||||
LastActivity: time.Now(),
|
||||
Source: source,
|
||||
Metadata: metadata,
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
session.Metadata = make(map[string]any)
|
||||
}
|
||||
|
||||
m.StoreSession(session)
|
||||
return session
|
||||
}
|
||||
|
||||
// StoreSession adds a session to the manager
|
||||
func (m *Manager) StoreSession(session *Session) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.sessions[session.ID] = session
|
||||
}
|
||||
|
||||
// GetSession retrieves a session by its unique ID
|
||||
func (m *Manager) GetSession(sessionID string) (*Session, bool) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
session, exists := m.sessions[sessionID]
|
||||
return session, exists
|
||||
}
|
||||
|
||||
// RemoveSession removes a session from the manager
|
||||
func (m *Manager) RemoveSession(sessionID string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.sessions, sessionID)
|
||||
}
|
||||
|
||||
// UpdateActivity updates the last activity timestamp for a session
|
||||
func (m *Manager) UpdateActivity(sessionID string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if session, exists := m.sessions[sessionID]; exists {
|
||||
session.LastActivity = time.Now()
|
||||
}
|
||||
}
|
||||
|
||||
// IsSessionActive checks if a session exists and has not been idle for too long
|
||||
func (m *Manager) IsSessionActive(sessionID string) bool {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
if session, exists := m.sessions[sessionID]; exists {
|
||||
// Session exists and hasn't exceeded idle timeout
|
||||
return time.Since(session.LastActivity) < m.maxIdleTime
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetActiveSessions returns a snapshot of all currently active sessions
|
||||
func (m *Manager) GetActiveSessions() []*Session {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
sessions := make([]*Session, 0, len(m.sessions))
|
||||
for _, session := range m.sessions {
|
||||
sessions = append(sessions, session)
|
||||
}
|
||||
return sessions
|
||||
}
|
||||
|
||||
// GetSessionCount returns the number of active sessions
|
||||
func (m *Manager) GetSessionCount() int {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return len(m.sessions)
|
||||
}
|
||||
|
||||
// GetSessionsBySource returns all sessions matching a specific source type
|
||||
func (m *Manager) GetSessionsBySource(source string) []*Session {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
var sessions []*Session
|
||||
for _, session := range m.sessions {
|
||||
if session.Source == source {
|
||||
sessions = append(sessions, session)
|
||||
}
|
||||
}
|
||||
return sessions
|
||||
}
|
||||
|
||||
// GetActiveSessionsBySource returns all active sessions for a given source
|
||||
func (m *Manager) GetActiveSessionsBySource(source string) []*Session {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
var sessions []*Session
|
||||
now := time.Now()
|
||||
|
||||
for _, session := range m.sessions {
|
||||
if session.Source == source && now.Sub(session.LastActivity) < m.maxIdleTime {
|
||||
sessions = append(sessions, session)
|
||||
}
|
||||
}
|
||||
return sessions
|
||||
}
|
||||
|
||||
// GetStats returns statistics about the session manager
|
||||
func (m *Manager) GetStats() map[string]any {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
sourceCounts := make(map[string]int)
|
||||
var totalSessions int
|
||||
var oldestSession time.Time
|
||||
var newestSession time.Time
|
||||
|
||||
for _, session := range m.sessions {
|
||||
totalSessions++
|
||||
sourceCounts[session.Source]++
|
||||
|
||||
if oldestSession.IsZero() || session.CreatedAt.Before(oldestSession) {
|
||||
oldestSession = session.CreatedAt
|
||||
}
|
||||
if newestSession.IsZero() || session.CreatedAt.After(newestSession) {
|
||||
newestSession = session.CreatedAt
|
||||
}
|
||||
}
|
||||
|
||||
stats := map[string]any{
|
||||
"total_sessions": totalSessions,
|
||||
"sessions_by_type": sourceCounts,
|
||||
"max_idle_time": m.maxIdleTime.String(),
|
||||
}
|
||||
|
||||
if !oldestSession.IsZero() {
|
||||
stats["oldest_session_age"] = time.Since(oldestSession).String()
|
||||
}
|
||||
if !newestSession.IsZero() {
|
||||
stats["newest_session_age"] = time.Since(newestSession).String()
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// Stop gracefully stops the session manager and its cleanup goroutine
|
||||
func (m *Manager) Stop() {
|
||||
close(m.done)
|
||||
if m.cleanupTicker != nil {
|
||||
m.cleanupTicker.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterExpiryCallback registers a callback function to be executed when a session expires
|
||||
func (m *Manager) RegisterExpiryCallback(source string, callback func(sessionID, remoteAddr string)) {
|
||||
m.callbacksMu.Lock()
|
||||
defer m.callbacksMu.Unlock()
|
||||
|
||||
if m.expiryCallbacks == nil {
|
||||
m.expiryCallbacks = make(map[string]func(sessionID, remoteAddr string))
|
||||
}
|
||||
m.expiryCallbacks[source] = callback
|
||||
}
|
||||
|
||||
// UnregisterExpiryCallback removes an expiry callback for a given source type
|
||||
func (m *Manager) UnregisterExpiryCallback(source string) {
|
||||
m.callbacksMu.Lock()
|
||||
defer m.callbacksMu.Unlock()
|
||||
|
||||
delete(m.expiryCallbacks, source)
|
||||
}
|
||||
|
||||
// startCleanup initializes the periodic cleanup of idle sessions
|
||||
func (m *Manager) startCleanup() {
|
||||
m.cleanupTicker = time.NewTicker(core.SessionCleanupInterval)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-m.cleanupTicker.C:
|
||||
m.cleanupIdleSessions()
|
||||
case <-m.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// cleanupIdleSessions removes sessions that have exceeded the maximum idle time.
|
||||
func (m *Manager) cleanupIdleSessions() {
|
||||
now := time.Now()
|
||||
expiredSessions := make([]*Session, 0)
|
||||
|
||||
m.mu.Lock()
|
||||
for id, session := range m.sessions {
|
||||
idleTime := now.Sub(session.LastActivity)
|
||||
|
||||
if idleTime > m.maxIdleTime {
|
||||
expiredSessions = append(expiredSessions, session)
|
||||
delete(m.sessions, id)
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
if len(expiredSessions) > 0 {
|
||||
m.callbacksMu.RLock()
|
||||
callbacks := make(map[string]func(sessionID, remoteAddr string))
|
||||
for k, v := range m.expiryCallbacks {
|
||||
callbacks[k] = v
|
||||
}
|
||||
m.callbacksMu.RUnlock()
|
||||
|
||||
for _, session := range expiredSessions {
|
||||
if callback, exists := callbacks[session.Source]; exists {
|
||||
// Call callback to notify owner
|
||||
go callback(session.ID, session.RemoteAddr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generateSessionID creates a unique, random session identifier.
|
||||
func generateSessionID() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
// Fallback to timestamp-based ID
|
||||
return fmt.Sprintf("session_%d", time.Now().UnixNano())
|
||||
}
|
||||
return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(b)
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("console", NewConsoleSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register console sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// ConsoleSink writes log entries to the console (stdout/stderr) using an dedicated logger instance
|
||||
type ConsoleSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.ConsoleSinkOptions
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
output io.Writer
|
||||
logger *log.Logger // application logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
// Defaults
|
||||
DefaultConsoleTarget = "stdout"
|
||||
DefaultConsoleBufferSize = 1000
|
||||
)
|
||||
|
||||
// NewConsoleSinkPlugin creates a console sink through plugin factory
|
||||
func NewConsoleSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.ConsoleSinkOptions{}
|
||||
|
||||
// Scan config map into struct
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate and apply defaults
|
||||
if opts.Target == "" {
|
||||
opts.Target = DefaultConsoleTarget
|
||||
} else {
|
||||
validateTarget := lconfig.OneOf("stdout", "stderr")
|
||||
if err := validateTarget(opts.Target); err != nil {
|
||||
return nil, fmt.Errorf("target: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var output io.Writer
|
||||
switch opts.Target {
|
||||
case "stdout":
|
||||
output = os.Stdout
|
||||
case "stderr":
|
||||
output = os.Stderr
|
||||
}
|
||||
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultConsoleBufferSize
|
||||
}
|
||||
|
||||
// Create and return plugin instance
|
||||
cs := &ConsoleSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
output: output,
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
cs.lastProcessed.Store(time.Time{})
|
||||
|
||||
// Create session for output
|
||||
cs.session = proxy.CreateSession(
|
||||
fmt.Sprintf("console:%s", opts.Target),
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "console",
|
||||
"target": opts.Target,
|
||||
},
|
||||
)
|
||||
|
||||
cs.logger.Info("msg", "Console sink initialized",
|
||||
"component", "console_sink",
|
||||
"instance_id", id,
|
||||
"target", opts.Target,
|
||||
)
|
||||
|
||||
return cs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (cs *ConsoleSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware, // Single output session
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (cs *ConsoleSink) Input() chan<- core.TransportEvent {
|
||||
return cs.input
|
||||
}
|
||||
|
||||
// Start begins the processing loop
|
||||
func (cs *ConsoleSink) Start(ctx context.Context) error {
|
||||
cs.startTime = time.Now()
|
||||
go cs.processLoop(ctx)
|
||||
cs.logger.Info("msg", "Console sink started",
|
||||
"component", "console_sink",
|
||||
"target", cs.config.Target)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (cs *ConsoleSink) Stop() {
|
||||
cs.logger.Info("msg", "Stopping console sink", "target", cs.config.Target)
|
||||
|
||||
// Remove session
|
||||
if cs.session != nil {
|
||||
cs.proxy.RemoveSession(cs.session.ID)
|
||||
}
|
||||
|
||||
close(cs.done)
|
||||
|
||||
cs.logger.Info("msg", "Console sink stopped",
|
||||
"instance_id", cs.id,
|
||||
"target", cs.config.Target,
|
||||
"instance_id", cs.id,
|
||||
)
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (cs *ConsoleSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := cs.lastProcessed.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: cs.id,
|
||||
Type: "console",
|
||||
TotalProcessed: cs.totalProcessed.Load(),
|
||||
StartTime: cs.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": cs.config.Target,
|
||||
"buffer_size": cs.config.BufferSize,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop reads transport events and writes to console
|
||||
func (cs *ConsoleSink) processLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-cs.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Write pre-formatted payload directly to output
|
||||
if _, err := cs.output.Write(event.Payload); err != nil {
|
||||
cs.logger.Error("msg", "Failed to write to console",
|
||||
"component", "console_sink",
|
||||
"target", cs.config.Target,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
cs.totalProcessed.Add(1)
|
||||
cs.lastProcessed.Store(time.Now())
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-cs.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("file", NewFileSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register file sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// FileSink writes log entries to files with rotation
|
||||
type FileSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.FileSinkOptions
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
writer *log.Logger // internal logger for file writing
|
||||
logger *log.Logger // application logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
// Defaults
|
||||
DefaultFileMaxSizeMB = 100
|
||||
DefaultFileMaxTotalSizeMB = 1000
|
||||
DefaultFileMinDiskFreeMB = 100
|
||||
DefaultFileRetentionHours = 168 // 7 days
|
||||
DefaultFileBufferSize = 1000
|
||||
DefaultFileFlushIntervalMs = 100
|
||||
)
|
||||
|
||||
// NewFileSinkPlugin creates a file sink through plugin factory
|
||||
func NewFileSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
// Create empty config struct
|
||||
opts := &config.FileSinkOptions{}
|
||||
|
||||
// Scan config map into struct
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate
|
||||
if err := lconfig.NonEmpty(opts.Directory); err != nil {
|
||||
return nil, fmt.Errorf("directory: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Name); err != nil {
|
||||
return nil, fmt.Errorf("name: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.MaxSizeMB <= 0 {
|
||||
opts.MaxSizeMB = DefaultFileMaxSizeMB
|
||||
}
|
||||
if opts.MaxTotalSizeMB <= 0 {
|
||||
opts.MaxTotalSizeMB = DefaultFileMaxTotalSizeMB
|
||||
}
|
||||
if opts.MinDiskFreeMB < 0 {
|
||||
opts.MinDiskFreeMB = DefaultFileMinDiskFreeMB
|
||||
}
|
||||
if opts.RetentionHours <= 0 {
|
||||
opts.RetentionHours = DefaultFileRetentionHours
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultFileBufferSize
|
||||
}
|
||||
if opts.FlushIntervalMs <= 0 {
|
||||
opts.FlushIntervalMs = DefaultFileFlushIntervalMs
|
||||
}
|
||||
|
||||
// Create configuration for the internal log writer
|
||||
writerConfig := log.DefaultConfig()
|
||||
writerConfig.Directory = opts.Directory
|
||||
writerConfig.Name = opts.Name
|
||||
writerConfig.MaxSizeKB = opts.MaxSizeMB * 1000
|
||||
writerConfig.MaxTotalSizeKB = opts.MaxTotalSizeMB * 1000
|
||||
writerConfig.MinDiskFreeKB = opts.MinDiskFreeMB * 1000
|
||||
writerConfig.RetentionPeriodHrs = opts.RetentionHours
|
||||
writerConfig.BufferSize = opts.BufferSize
|
||||
writerConfig.FlushIntervalMs = opts.FlushIntervalMs
|
||||
// Sink logic
|
||||
writerConfig.EnableConsole = false
|
||||
writerConfig.EnableFile = true
|
||||
writerConfig.ShowTimestamp = false
|
||||
writerConfig.ShowLevel = false
|
||||
writerConfig.Format = "raw"
|
||||
|
||||
// Create internal logger for file writing
|
||||
writer := log.NewLogger()
|
||||
if err := writer.ApplyConfig(writerConfig); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize file writer: %w", err)
|
||||
}
|
||||
|
||||
fs := &FileSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
writer: writer,
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
fs.lastProcessed.Store(time.Time{})
|
||||
|
||||
// Create session for file output
|
||||
fs.session = proxy.CreateSession(
|
||||
fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Name),
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "file",
|
||||
"directory": opts.Directory,
|
||||
"name": opts.Name,
|
||||
},
|
||||
)
|
||||
|
||||
fs.logger.Info("msg", "File sink initialized",
|
||||
"component", "file_sink",
|
||||
"instance_id", id,
|
||||
"directory", opts.Directory,
|
||||
"name", opts.Name)
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (fs *FileSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware, // Single output session
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (fs *FileSink) Input() chan<- core.TransportEvent {
|
||||
return fs.input
|
||||
}
|
||||
|
||||
// Start begins the processing loop for the sink
|
||||
func (fs *FileSink) Start(ctx context.Context) error {
|
||||
// Start the internal file writer
|
||||
if err := fs.writer.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start file writer: %w", err)
|
||||
}
|
||||
|
||||
fs.startTime = time.Now()
|
||||
go fs.processLoop(ctx)
|
||||
|
||||
fs.logger.Info("msg", "File sink started",
|
||||
"component", "file_sink",
|
||||
)
|
||||
fs.logger.Debug("msg", "File sink config",
|
||||
"component", "file_sink",
|
||||
"directory", fs.config.Directory,
|
||||
"name", fs.config.Name,
|
||||
"max_size_mb", fs.config.MaxSizeMB,
|
||||
"max_total_size_mb", fs.config.MaxTotalSizeMB,
|
||||
"min_disk_free_mb", fs.config.MinDiskFreeMB,
|
||||
"retention_hours", fs.config.RetentionHours,
|
||||
"buffer_size", fs.config.BufferSize,
|
||||
"flush_interval_ms", fs.config.FlushIntervalMs,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (fs *FileSink) Stop() {
|
||||
fs.logger.Info("msg", "Stopping file sink",
|
||||
"component", "file_sink",
|
||||
"directory", fs.config.Directory,
|
||||
"name", fs.config.Name)
|
||||
|
||||
close(fs.done)
|
||||
|
||||
// Remove session
|
||||
if fs.session != nil {
|
||||
fs.proxy.RemoveSession(fs.session.ID)
|
||||
}
|
||||
|
||||
// Shutdown the writer with timeout
|
||||
if err := fs.writer.Shutdown(core.LoggerShutdownTimeout); err != nil {
|
||||
fs.logger.Error("msg", "Error shutting down file writer",
|
||||
"component", "file_sink",
|
||||
"error", err)
|
||||
}
|
||||
|
||||
fs.logger.Info("msg", "File sink stopped",
|
||||
"component", "file_sink",
|
||||
"instance_id", fs.id,
|
||||
"total_processed", fs.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns the sink's statistics
|
||||
func (fs *FileSink) GetStats() sink.SinkStats {
|
||||
return sink.SinkStats{
|
||||
ID: fs.id,
|
||||
Type: "file",
|
||||
TotalProcessed: fs.totalProcessed.Load(),
|
||||
StartTime: fs.startTime,
|
||||
LastProcessed: fs.lastProcessed.Load().(time.Time),
|
||||
Details: map[string]any{
|
||||
"directory": fs.config.Directory,
|
||||
"name": fs.config.Name,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop reads transport events and writes to file
|
||||
func (fs *FileSink) processLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-fs.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Write the pre-formatted payload directly
|
||||
// The writer handles rotation automatically based on configuration
|
||||
fs.writer.Write(string(event.Payload))
|
||||
|
||||
fs.totalProcessed.Add(1)
|
||||
fs.lastProcessed.Store(time.Now())
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
case <-fs.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
"logwisp/internal/version"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/lixenwraith/log/compat"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("http", NewHTTPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPSink streams log entries via Server-Sent Events (SSE)
|
||||
type HTTPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.HTTPSinkOptions
|
||||
|
||||
// Network
|
||||
server *fasthttp.Server
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
// Broker
|
||||
clients map[uint64]chan []byte
|
||||
clientsMu sync.RWMutex
|
||||
unregister chan uint64
|
||||
nextClientID atomic.Uint64
|
||||
|
||||
// Client session tracking
|
||||
clientSessions map[uint64]string // clientID -> sessionID
|
||||
sessionsMu sync.RWMutex
|
||||
|
||||
// Statistics
|
||||
activeClients atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
// Server lifecycle
|
||||
HttpServerStartTimeout = 100 * time.Millisecond
|
||||
HttpServerShutdownTimeout = 2 * time.Second
|
||||
|
||||
// Defaults
|
||||
DefaultHTTPHost = "0.0.0.0"
|
||||
DefaultHTTPBufferSize = 1000
|
||||
DefaultHTTPStreamPath = "/stream"
|
||||
DefaultHTTPStatusPath = "/status"
|
||||
HTTPMaxPort = 65535
|
||||
)
|
||||
|
||||
// NewHTTPSinkPlugin creates an HTTP sink through plugin factory
|
||||
func NewHTTPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.HTTPSinkOptions{
|
||||
Host: DefaultHTTPHost,
|
||||
Port: 0,
|
||||
WriteTimeout: 0, // SSE indefinite streaming
|
||||
}
|
||||
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate
|
||||
if opts.Port <= 0 || opts.Port > HTTPMaxPort {
|
||||
return nil, fmt.Errorf("port must be between 1 and %d", HTTPMaxPort)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPBufferSize
|
||||
}
|
||||
if opts.StreamPath == "" {
|
||||
opts.StreamPath = DefaultHTTPStreamPath
|
||||
}
|
||||
if opts.StatusPath == "" {
|
||||
opts.StatusPath = DefaultHTTPStatusPath
|
||||
}
|
||||
|
||||
h := &HTTPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
clients: make(map[uint64]chan []byte),
|
||||
unregister: make(chan uint64),
|
||||
clientSessions: make(map[uint64]string),
|
||||
}
|
||||
h.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "HTTP sink initialized",
|
||||
"component", "http_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port,
|
||||
"stream_path", opts.StreamPath,
|
||||
"status_path", opts.StatusPath)
|
||||
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (h *HTTPSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (h *HTTPSink) Input() chan<- core.TransportEvent {
|
||||
return h.input
|
||||
}
|
||||
|
||||
// Start initializes the HTTP server and begins the broker loop
|
||||
func (h *HTTPSink) Start(ctx context.Context) error {
|
||||
h.startTime = time.Now()
|
||||
|
||||
// Start central broker goroutine
|
||||
h.wg.Add(1)
|
||||
go h.brokerLoop(ctx)
|
||||
|
||||
fasthttpLogger := compat.NewFastHTTPAdapter(h.logger)
|
||||
|
||||
h.server = &fasthttp.Server{
|
||||
Name: fmt.Sprintf("LogWisp/%s", version.Short()),
|
||||
Handler: h.requestHandler,
|
||||
DisableKeepalive: false,
|
||||
StreamRequestBody: true,
|
||||
Logger: fasthttpLogger,
|
||||
WriteTimeout: time.Duration(h.config.WriteTimeout) * time.Millisecond,
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", h.config.Host, h.config.Port)
|
||||
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http sink bind %s: %w", addr, err)
|
||||
}
|
||||
go func() {
|
||||
if err := h.server.Serve(ln); err != nil {
|
||||
h.logger.Error("msg", "HTTP server terminated",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Monitor context for shutdown
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
if h.server != nil {
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), HttpServerShutdownTimeout)
|
||||
defer cancel()
|
||||
h.server.ShutdownWithContext(shutdownCtx)
|
||||
}
|
||||
}()
|
||||
|
||||
h.logger.Info("msg", "HTTP server started",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"host", h.config.Host,
|
||||
"port", h.config.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the HTTP server and all client connections
|
||||
func (h *HTTPSink) Stop() {
|
||||
h.logger.Info("msg", "Stopping HTTP sink",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id)
|
||||
|
||||
close(h.done)
|
||||
|
||||
if h.server != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), HttpServerShutdownTimeout)
|
||||
defer cancel()
|
||||
h.server.ShutdownWithContext(ctx)
|
||||
}
|
||||
|
||||
h.wg.Wait()
|
||||
|
||||
close(h.unregister)
|
||||
|
||||
h.clientsMu.Lock()
|
||||
for _, ch := range h.clients {
|
||||
close(ch)
|
||||
}
|
||||
h.clients = make(map[uint64]chan []byte)
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
h.logger.Info("msg", "HTTP sink stopped",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"total_processed", h.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (h *HTTPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := h.lastProcessed.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: h.id,
|
||||
Type: "http",
|
||||
TotalProcessed: h.totalProcessed.Load(),
|
||||
ActiveConnections: h.activeClients.Load(),
|
||||
StartTime: h.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": h.config.Host,
|
||||
"port": h.config.Port,
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"endpoints": map[string]string{
|
||||
"stream": h.config.StreamPath,
|
||||
"status": h.config.StatusPath,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// brokerLoop manages client connections and broadcasts transport events
|
||||
func (h *HTTPSink) brokerLoop(ctx context.Context) {
|
||||
defer h.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
h.logger.Debug("msg", "Broker loop stopping due to context cancellation",
|
||||
"component", "http_sink")
|
||||
return
|
||||
|
||||
case <-h.done:
|
||||
h.logger.Debug("msg", "Broker loop stopping due to shutdown signal",
|
||||
"component", "http_sink")
|
||||
return
|
||||
|
||||
case clientID := <-h.unregister:
|
||||
h.clientsMu.Lock()
|
||||
if clientChan, exists := h.clients[clientID]; exists {
|
||||
delete(h.clients, clientID)
|
||||
close(clientChan)
|
||||
h.logger.Debug("msg", "Unregistered client",
|
||||
"component", "http_sink",
|
||||
"client_id", clientID)
|
||||
}
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
h.sessionsMu.Lock()
|
||||
delete(h.clientSessions, clientID)
|
||||
h.sessionsMu.Unlock()
|
||||
|
||||
case event, ok := <-h.input:
|
||||
if !ok {
|
||||
h.logger.Debug("msg", "Input channel closed, broker stopping",
|
||||
"component", "http_sink")
|
||||
return
|
||||
}
|
||||
|
||||
h.totalProcessed.Add(1)
|
||||
h.lastProcessed.Store(time.Now())
|
||||
|
||||
h.clientsMu.RLock()
|
||||
clientCount := len(h.clients)
|
||||
if clientCount > 0 {
|
||||
var staleClients []uint64
|
||||
|
||||
for id, ch := range h.clients {
|
||||
h.sessionsMu.RLock()
|
||||
sessionID, hasSession := h.clientSessions[id]
|
||||
h.sessionsMu.RUnlock()
|
||||
|
||||
if !hasSession {
|
||||
staleClients = append(staleClients, id)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check session still exists via proxy
|
||||
if _, exists := h.proxy.GetSession(sessionID); !exists {
|
||||
staleClients = append(staleClients, id)
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- event.Payload:
|
||||
h.proxy.UpdateActivity(sessionID)
|
||||
default:
|
||||
h.logger.Debug("msg", "Dropped event for slow client",
|
||||
"component", "http_sink",
|
||||
"client_id", id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(staleClients) > 0 {
|
||||
go func() {
|
||||
for _, clientID := range staleClients {
|
||||
select {
|
||||
case h.unregister <- clientID:
|
||||
case <-h.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
h.clientsMu.RUnlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// requestHandler is the main entry point for all incoming HTTP requests
|
||||
func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
// IPv4-only enforcement - silent drop IPv6
|
||||
remoteAddr := ctx.RemoteAddr()
|
||||
if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok {
|
||||
if tcpAddr.IP.To4() == nil {
|
||||
h.logger.Debug("msg", "IPv6 connection rejected",
|
||||
"component", "http_sink", "remote_addr", remoteAddr.String())
|
||||
ctx.SetConnectionClose()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
path := string(ctx.Path())
|
||||
|
||||
switch path {
|
||||
case h.config.StatusPath:
|
||||
h.handleStatus(ctx)
|
||||
case h.config.StreamPath:
|
||||
h.handleStream(ctx)
|
||||
default:
|
||||
ctx.SetStatusCode(fasthttp.StatusNotFound)
|
||||
ctx.SetContentType("application/json")
|
||||
json.NewEncoder(ctx).Encode(map[string]any{
|
||||
"error": "Not Found",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleStream manages a client's Server-Sent Events (SSE) stream
|
||||
func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
|
||||
remoteAddrStr := ctx.RemoteAddr().String()
|
||||
|
||||
// Create session via proxy
|
||||
sess := h.proxy.CreateSession(remoteAddrStr, map[string]any{
|
||||
"type": "http_client",
|
||||
})
|
||||
|
||||
// Set SSE headers
|
||||
ctx.Response.Header.Set("Content-Type", "text/event-stream")
|
||||
ctx.Response.Header.Set("Cache-Control", "no-cache")
|
||||
ctx.Response.Header.Set("Connection", "keep-alive")
|
||||
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
|
||||
ctx.Response.Header.Set("X-Accel-Buffering", "no")
|
||||
|
||||
// Register client with broker
|
||||
clientID := h.nextClientID.Add(1)
|
||||
clientChan := make(chan []byte, h.config.BufferSize)
|
||||
|
||||
h.clientsMu.Lock()
|
||||
h.clients[clientID] = clientChan
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
h.sessionsMu.Lock()
|
||||
h.clientSessions[clientID] = sess.ID
|
||||
h.sessionsMu.Unlock()
|
||||
|
||||
streamFunc := func(w *bufio.Writer) {
|
||||
connectCount := h.activeClients.Add(1)
|
||||
h.logger.Debug("msg", "HTTP client connected",
|
||||
"component", "http_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"session_id", sess.ID,
|
||||
"client_id", clientID,
|
||||
"active_clients", connectCount)
|
||||
|
||||
defer func() {
|
||||
disconnectCount := h.activeClients.Add(-1)
|
||||
h.logger.Debug("msg", "HTTP client disconnected",
|
||||
"component", "http_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"session_id", sess.ID,
|
||||
"client_id", clientID,
|
||||
"active_clients", disconnectCount)
|
||||
|
||||
select {
|
||||
case h.unregister <- clientID:
|
||||
case <-h.done:
|
||||
}
|
||||
|
||||
h.proxy.RemoveSession(sess.ID)
|
||||
}()
|
||||
|
||||
// Send connected event with metadata
|
||||
connectionInfo := map[string]any{
|
||||
"client_id": fmt.Sprintf("%d", clientID),
|
||||
"session_id": sess.ID,
|
||||
"instance_id": h.id,
|
||||
"stream_path": h.config.StreamPath,
|
||||
"status_path": h.config.StatusPath,
|
||||
"buffer_size": h.config.BufferSize,
|
||||
}
|
||||
data, _ := json.Marshal(connectionInfo)
|
||||
fmt.Fprintf(w, "event: connected\ndata: %s\n\n", data)
|
||||
if err := w.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case payload, ok := <-clientChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.writeSSE(w, payload); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := w.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.UpdateActivity(sess.ID)
|
||||
|
||||
case <-h.done:
|
||||
fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
|
||||
w.Flush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ctx.SetBodyStreamWriter(streamFunc)
|
||||
}
|
||||
|
||||
// handleStatus provides a JSON status report
|
||||
func (h *HTTPSink) handleStatus(ctx *fasthttp.RequestCtx) {
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
status := map[string]any{
|
||||
"service": "LogWisp",
|
||||
"version": version.Short(),
|
||||
"instance_id": h.id,
|
||||
"server": map[string]any{
|
||||
"type": "http",
|
||||
"host": h.config.Host,
|
||||
"port": h.config.Port,
|
||||
"active_clients": h.activeClients.Load(),
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"uptime_seconds": int(time.Since(h.startTime).Seconds()),
|
||||
},
|
||||
"endpoints": map[string]string{
|
||||
"stream": h.config.StreamPath,
|
||||
"status": h.config.StatusPath,
|
||||
},
|
||||
"statistics": map[string]any{
|
||||
"total_processed": h.totalProcessed.Load(),
|
||||
},
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(status)
|
||||
ctx.SetBody(data)
|
||||
}
|
||||
|
||||
// writeSSE formats payload into SSE data format
|
||||
func (h *HTTPSink) writeSSE(w *bufio.Writer, payload []byte) error {
|
||||
// Handle multi-line payloads per W3C SSE spec
|
||||
lines := splitLines(payload)
|
||||
for _, line := range lines {
|
||||
if _, err := fmt.Fprintf(w, "data: %s\n", line); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Empty line terminates event
|
||||
if _, err := w.WriteString("\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitLines splits payload by newlines, handling different line endings
|
||||
func splitLines(data []byte) [][]byte {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Trim trailing newline if present
|
||||
if data[len(data)-1] == '\n' {
|
||||
data = data[:len(data)-1]
|
||||
}
|
||||
|
||||
var lines [][]byte
|
||||
start := 0
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] == '\n' {
|
||||
lines = append(lines, data[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
if start < len(data) {
|
||||
lines = append(lines, data[start:])
|
||||
}
|
||||
|
||||
if len(lines) == 0 {
|
||||
return [][]byte{data}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/chain"
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("http_chain", NewHTTPChainSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http_chain sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultHTTPChainSinkBufferSize = 1000
|
||||
DefaultHTTPChainSinkIngestPath = "/ingest"
|
||||
DefaultHTTPChainSinkMaxBatchCount = 100
|
||||
DefaultHTTPChainSinkMaxBatchBytes = 1024 * 1024
|
||||
DefaultHTTPChainSinkFlushIntervalMS = 1000
|
||||
DefaultHTTPChainSinkRequestTimeoutMS = 10000
|
||||
DefaultHTTPChainSinkBackoffMinMS = 500
|
||||
DefaultHTTPChainSinkBackoffMaxMS = 30000
|
||||
)
|
||||
|
||||
// HTTPChainSink batches structured entries and posts NDJSON to a downstream
|
||||
// http_chain source. Delivery is at-least-once per batch.
|
||||
type HTTPChainSink struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
config *config.HTTPChainSinkOptions
|
||||
|
||||
node string
|
||||
url string
|
||||
|
||||
client *http.Client
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Batch state owned exclusively by run loop goroutine
|
||||
batch bytes.Buffer
|
||||
batchCount int64
|
||||
|
||||
reqTimeout time.Duration
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
totalProcessed atomic.Uint64
|
||||
batchesSent atomic.Uint64
|
||||
requestErrors atomic.Uint64
|
||||
droppedBatches atomic.Uint64
|
||||
synthesized atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewHTTPChainSinkPlugin creates an http_chain sink through plugin factory
|
||||
func NewHTTPChainSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.HTTPChainSinkOptions{}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Host); err != nil {
|
||||
return nil, fmt.Errorf("host: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.IngestPath == "" {
|
||||
opts.IngestPath = DefaultHTTPChainSinkIngestPath
|
||||
} else if !strings.HasPrefix(opts.IngestPath, "/") {
|
||||
return nil, fmt.Errorf("ingest_path: must start with '/'")
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPChainSinkBufferSize
|
||||
}
|
||||
if opts.MaxBatchCount <= 0 {
|
||||
opts.MaxBatchCount = DefaultHTTPChainSinkMaxBatchCount
|
||||
}
|
||||
if opts.MaxBatchBytes <= 0 {
|
||||
opts.MaxBatchBytes = DefaultHTTPChainSinkMaxBatchBytes
|
||||
}
|
||||
if opts.FlushIntervalMS <= 0 {
|
||||
opts.FlushIntervalMS = DefaultHTTPChainSinkFlushIntervalMS
|
||||
}
|
||||
if opts.RequestTimeoutMS <= 0 {
|
||||
opts.RequestTimeoutMS = DefaultHTTPChainSinkRequestTimeoutMS
|
||||
}
|
||||
if opts.BackoffMinMS <= 0 {
|
||||
opts.BackoffMinMS = DefaultHTTPChainSinkBackoffMinMS
|
||||
}
|
||||
if opts.BackoffMaxMS < opts.BackoffMinMS {
|
||||
opts.BackoffMaxMS = DefaultHTTPChainSinkBackoffMaxMS
|
||||
}
|
||||
|
||||
node := opts.Node
|
||||
if node == "" {
|
||||
if hn, err := os.Hostname(); err == nil {
|
||||
node = hn
|
||||
} else {
|
||||
node = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
|
||||
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
// IPv4-only, aligns with tcp/http sinks
|
||||
d := net.Dialer{}
|
||||
return d.DialContext(ctx, "tcp4", address)
|
||||
},
|
||||
MaxIdleConnsPerHost: 2,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: true,
|
||||
// Future: TLSClientConfig; HTTP/2 via ALPN once TLS lands
|
||||
}
|
||||
|
||||
t := &HTTPChainSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
node: node,
|
||||
// Future: "https" scheme with TLS
|
||||
url: "http://" + addr + opts.IngestPath,
|
||||
client: &http.Client{Transport: transport},
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
reqTimeout: time.Duration(opts.RequestTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
t.session = proxy.CreateSession(
|
||||
"http_chain://"+addr,
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "http_chain",
|
||||
"target": t.url,
|
||||
"node": node,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Info("msg", "HTTP chain sink initialized",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", id,
|
||||
"target", t.url,
|
||||
"node", node)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *HTTPChainSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *HTTPChainSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start launches the batching loop; downstream availability is not required
|
||||
func (t *HTTPChainSink) Start(ctx context.Context) error {
|
||||
t.startTime = time.Now()
|
||||
t.wg.Add(1)
|
||||
go t.runLoop(ctx)
|
||||
|
||||
t.logger.Info("msg", "HTTP chain sink started",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"target", t.url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the loop. Worst case: one in-flight request timeout plus
|
||||
// one final-flush request timeout.
|
||||
func (t *HTTPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping HTTP chain sink",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
t.client.CloseIdleConnections()
|
||||
|
||||
if t.session != nil {
|
||||
t.proxy.RemoveSession(t.session.ID)
|
||||
}
|
||||
|
||||
t.logger.Info("msg", "HTTP chain sink stopped",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *HTTPChainSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "http_chain",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": t.url,
|
||||
"node": t.node,
|
||||
"batches_sent": t.batchesSent.Load(),
|
||||
"request_errors": t.requestErrors.Load(),
|
||||
"dropped_batches": t.droppedBatches.Load(),
|
||||
"synthesized": t.synthesized.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runLoop batches events and flushes on size or interval
|
||||
func (t *HTTPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
|
||||
// Fold done channel into a context for request/backoff interruption
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-runCtx.Done():
|
||||
t.finalFlush()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if t.batchCount > 0 && !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
t.append(event)
|
||||
if t.batchCount >= t.config.MaxBatchCount ||
|
||||
int64(t.batch.Len()) >= t.config.MaxBatchBytes {
|
||||
if !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// append serializes one event into the pending batch
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop entry
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "http_chain_sink",
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batch.Write(line)
|
||||
t.batch.WriteByte('\n')
|
||||
t.batchCount++
|
||||
}
|
||||
|
||||
// flush delivers the pending batch, retrying transient failures with backoff.
|
||||
// Returns false when shutdown interrupts delivery; undelivered batch is dropped
|
||||
// by finalFlush semantics (batch already consumed here).
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
body := bytes.Clone(t.batch.Bytes())
|
||||
count := t.batchCount
|
||||
t.batch.Reset()
|
||||
t.batchCount = 0
|
||||
|
||||
failures := 0
|
||||
for {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
}
|
||||
transient, err := t.post(ctx, body)
|
||||
if err == nil {
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
return true
|
||||
}
|
||||
t.requestErrors.Add(1)
|
||||
if !transient {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Error("msg", "Chain batch rejected, dropping",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return true
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
}
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain batch delivery failed",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
|
||||
func (t *HTTPChainSink) finalFlush() {
|
||||
if t.batchCount == 0 {
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
count := t.batchCount
|
||||
if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Warn("msg", "Final chain batch dropped on shutdown",
|
||||
"component", "http_chain_sink",
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
}
|
||||
|
||||
// post sends one NDJSON batch; transient=true marks retryable failures
|
||||
func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool, err error) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, t.url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("Content-Type", chain.ContentTypeNDJSON)
|
||||
req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion))
|
||||
req.Header.Set(chain.HeaderNode, t.node)
|
||||
// Future: Authorization header for auth
|
||||
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Drain for connection reuse
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return false, nil
|
||||
case resp.StatusCode == http.StatusRequestTimeout,
|
||||
resp.StatusCode == http.StatusTooManyRequests,
|
||||
resp.StatusCode >= 500:
|
||||
return true, fmt.Errorf("status %s", resp.Status)
|
||||
default:
|
||||
return false, fmt.Errorf("status %s", resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// waitBackoff sleeps for the computed delay, interruptible by shutdown
|
||||
func (t *HTTPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package null
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("null", NewNullSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register null sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// NullSink discards all received transport events, used for testing
|
||||
type NullSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
totalReceived atomic.Uint64
|
||||
totalBytes atomic.Uint64
|
||||
lastReceived atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewNullSinkPlugin creates a null sink through plugin factory
|
||||
func NewNullSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
ns := &NullSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
input: make(chan core.TransportEvent, 1000),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
ns.lastReceived.Store(time.Time{})
|
||||
|
||||
// Create session for null sink
|
||||
ns.session = proxy.CreateSession(
|
||||
"null://devnull",
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "null",
|
||||
},
|
||||
)
|
||||
|
||||
logger.Debug("msg", "Null sink initialized",
|
||||
"component", "null_sink",
|
||||
"instance_id", id)
|
||||
|
||||
return ns, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (ns *NullSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (ns *NullSink) Input() chan<- core.TransportEvent {
|
||||
return ns.input
|
||||
}
|
||||
|
||||
// Start begins the processing loop
|
||||
func (ns *NullSink) Start(ctx context.Context) error {
|
||||
|
||||
ns.startTime = time.Now()
|
||||
go ns.processLoop(ctx)
|
||||
ns.logger.Debug("msg", "Null sink started",
|
||||
"component", "null_sink",
|
||||
"instance_id", ns.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (ns *NullSink) Stop() {
|
||||
if ns.session != nil {
|
||||
ns.proxy.RemoveSession(ns.session.ID)
|
||||
}
|
||||
close(ns.done)
|
||||
ns.logger.Debug("msg", "Null sink stopped",
|
||||
"instance_id", ns.id,
|
||||
"total_received", ns.totalReceived.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (ns *NullSink) GetStats() sink.SinkStats {
|
||||
lastRcv, _ := ns.lastReceived.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: ns.id,
|
||||
Type: "null",
|
||||
TotalProcessed: ns.totalReceived.Load(),
|
||||
StartTime: ns.startTime,
|
||||
LastProcessed: lastRcv,
|
||||
Details: map[string]any{
|
||||
"total_bytes": ns.totalBytes.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop reads transport events and discards them
|
||||
func (ns *NullSink) processLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-ns.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Discard the event, only update stats
|
||||
ns.totalReceived.Add(1)
|
||||
ns.totalBytes.Add(uint64(len(event.Payload)))
|
||||
ns.lastReceived.Store(time.Now())
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ns.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package sink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
)
|
||||
|
||||
// Sink represents an output data stream.
|
||||
type Sink interface {
|
||||
// Capabilities returns a slice of supported Source capabilities
|
||||
Capabilities() []core.Capability
|
||||
|
||||
// Input returns the channel for sending transport events to this sink.
|
||||
Input() chan<- core.TransportEvent
|
||||
|
||||
// Start begins processing transport events.
|
||||
Start(ctx context.Context) error
|
||||
|
||||
// Stop gracefully shuts down the sink.
|
||||
Stop()
|
||||
|
||||
// GetStats returns sink statistics.
|
||||
GetStats() SinkStats
|
||||
}
|
||||
|
||||
// SinkStats contains statistics about a sink.
|
||||
type SinkStats struct {
|
||||
ID string
|
||||
Type string
|
||||
TotalProcessed uint64
|
||||
ActiveConnections int64
|
||||
StartTime time.Time
|
||||
LastProcessed time.Time
|
||||
Details map[string]any
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/lixenwraith/log/compat"
|
||||
"github.com/panjf2000/gnet/v2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("tcp", NewTCPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// TCPSink streams log entries to connected TCP clients
|
||||
type TCPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.TCPSinkOptions
|
||||
|
||||
// Network
|
||||
server *tcpServer
|
||||
engine *gnet.Engine
|
||||
engineMu sync.Mutex
|
||||
booted chan struct{}
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
activeConns atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
|
||||
// Error tracking
|
||||
writeErrors atomic.Uint64
|
||||
consecutiveWriteErrors map[gnet.Conn]int
|
||||
errorMu sync.Mutex
|
||||
}
|
||||
|
||||
const (
|
||||
// Server lifecycle
|
||||
TCPServerStartTimeout = 2 * time.Second
|
||||
TCPServerShutdownTimeout = 2 * time.Second
|
||||
|
||||
// Connection management
|
||||
TCPMaxConsecutiveWriteErrors = 3
|
||||
TCPMaxPort = 65535
|
||||
|
||||
// Defaults
|
||||
DefaultTCPHost = "0.0.0.0"
|
||||
DefaultTCPBufferSize = 1000
|
||||
DefaultTCPWriteTimeoutMS = 5000
|
||||
DefaultTCPKeepAlivePeriod = 30000
|
||||
)
|
||||
|
||||
// NewTCPSinkPlugin creates a TCP sink through plugin factory
|
||||
func NewTCPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
// Create config struct with defaults
|
||||
opts := &config.TCPSinkOptions{
|
||||
Host: DefaultTCPHost,
|
||||
Port: 0,
|
||||
KeepAlive: true,
|
||||
}
|
||||
|
||||
// Parse config map into struct
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultTCPBufferSize
|
||||
}
|
||||
if opts.WriteTimeout <= 0 {
|
||||
opts.WriteTimeout = DefaultTCPWriteTimeoutMS
|
||||
}
|
||||
if opts.KeepAlivePeriod <= 0 {
|
||||
opts.KeepAlivePeriod = DefaultTCPKeepAlivePeriod
|
||||
}
|
||||
|
||||
t := &TCPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
consecutiveWriteErrors: make(map[gnet.Conn]int),
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "TCP sink initialized",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port)
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *TCPSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *TCPSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start initializes the TCP server and begins the broadcast loop
|
||||
func (t *TCPSink) Start(ctx context.Context) error {
|
||||
t.server = &tcpServer{
|
||||
sink: t,
|
||||
clients: make(map[gnet.Conn]*tcpClient),
|
||||
}
|
||||
// Fresh channel per Start
|
||||
t.booted = make(chan struct{})
|
||||
|
||||
t.startTime = time.Now()
|
||||
|
||||
// Start broadcast loop
|
||||
t.wg.Add(1)
|
||||
go func() {
|
||||
defer t.wg.Done()
|
||||
t.broadcastLoop(ctx)
|
||||
}()
|
||||
|
||||
// Configure gnet
|
||||
addr := fmt.Sprintf("tcp://%s:%d", t.config.Host, t.config.Port)
|
||||
gnetLogger := compat.NewGnetAdapter(t.logger)
|
||||
|
||||
opts := []gnet.Option{
|
||||
gnet.WithLogger(gnetLogger),
|
||||
gnet.WithMulticore(true),
|
||||
gnet.WithReusePort(true),
|
||||
}
|
||||
|
||||
// Apply TCP keep-alive settings from config
|
||||
if t.config.KeepAlive {
|
||||
opts = append(opts,
|
||||
gnet.WithTCPKeepAlive(time.Duration(t.config.KeepAlivePeriod)*time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
// Start gnet server
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
t.logger.Info("msg", "Starting TCP server",
|
||||
"component", "tcp_sink",
|
||||
"host", t.config.Host,
|
||||
"port", t.config.Port)
|
||||
|
||||
err := gnet.Run(t.server, addr, opts...)
|
||||
if err != nil {
|
||||
t.logger.Error("msg", "TCP server failed",
|
||||
"component", "tcp_sink",
|
||||
"error", err)
|
||||
}
|
||||
errChan <- err
|
||||
}()
|
||||
|
||||
// Monitor context for shutdown
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
t.engineMu.Lock()
|
||||
if t.engine != nil {
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
(*t.engine).Stop(shutdownCtx)
|
||||
}
|
||||
t.engineMu.Unlock()
|
||||
}()
|
||||
|
||||
// Wait briefly for server to start or fail
|
||||
select {
|
||||
case err := <-errChan:
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
return err
|
||||
// Bind confirmation via OnBoot
|
||||
case <-t.booted:
|
||||
t.logger.Info("msg", "TCP server started",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id,
|
||||
"port", t.config.Port)
|
||||
return nil
|
||||
// Timeout failure
|
||||
case <-time.After(TCPServerStartTimeout):
|
||||
t.engineMu.Lock()
|
||||
if t.engine != nil {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout)
|
||||
(*t.engine).Stop(stopCtx)
|
||||
cancel()
|
||||
}
|
||||
t.engineMu.Unlock()
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
return fmt.Errorf("tcp sink start timeout on %s", addr)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the TCP sink
|
||||
func (t *TCPSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP sink",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
|
||||
// Stop gnet engine
|
||||
t.engineMu.Lock()
|
||||
engine := t.engine
|
||||
t.engineMu.Unlock()
|
||||
|
||||
if engine != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout)
|
||||
defer cancel()
|
||||
(*engine).Stop(ctx)
|
||||
}
|
||||
|
||||
t.wg.Wait()
|
||||
|
||||
t.logger.Info("msg", "TCP sink stopped",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *TCPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "tcp",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: t.activeConns.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": t.config.Host,
|
||||
"port": t.config.Port,
|
||||
"buffer_size": t.config.BufferSize,
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// tcpServer implements gnet.EventHandler
|
||||
type tcpServer struct {
|
||||
gnet.BuiltinEventEngine
|
||||
sink *TCPSink
|
||||
clients map[gnet.Conn]*tcpClient
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// tcpClient represents a connected TCP client
|
||||
type tcpClient struct {
|
||||
conn gnet.Conn
|
||||
buffer bytes.Buffer
|
||||
sessionID string
|
||||
}
|
||||
|
||||
// broadcastLoop sends transport events to all connected clients
|
||||
func (t *TCPSink) broadcastLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.broadcastData(event.Payload)
|
||||
case <-t.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OnBoot is called when the server starts
|
||||
func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
|
||||
s.sink.engineMu.Lock()
|
||||
s.sink.engine = &eng
|
||||
s.sink.engineMu.Unlock()
|
||||
|
||||
// Listener is bound at this point; unblock Start
|
||||
close(s.sink.booted)
|
||||
|
||||
s.sink.logger.Debug("msg", "TCP server booted",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", s.sink.id)
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
// OnOpen is called when a new connection is established
|
||||
func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
|
||||
remoteAddr := c.RemoteAddr()
|
||||
remoteAddrStr := remoteAddr.String()
|
||||
|
||||
s.sink.logger.Debug("msg", "TCP connection attempt",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr)
|
||||
|
||||
// Reject IPv6 connections
|
||||
if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok {
|
||||
if tcpAddr.IP.To4() == nil {
|
||||
s.sink.logger.Warn("msg", "IPv6 connection rejected",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr)
|
||||
return []byte("IPv4-only (IPv6 not supported)\n"), gnet.Close
|
||||
}
|
||||
}
|
||||
|
||||
// Apply write timeout from config
|
||||
if s.sink.config.WriteTimeout > 0 {
|
||||
c.SetWriteDeadline(time.Now().Add(time.Duration(s.sink.config.WriteTimeout) * time.Millisecond))
|
||||
}
|
||||
|
||||
// Create session via proxy
|
||||
sess := s.sink.proxy.CreateSession(remoteAddrStr, map[string]any{
|
||||
"type": "tcp_client",
|
||||
"remote_addr": remoteAddrStr,
|
||||
})
|
||||
|
||||
client := &tcpClient{
|
||||
conn: c,
|
||||
sessionID: sess.ID,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.clients[c] = client
|
||||
s.mu.Unlock()
|
||||
|
||||
newCount := s.sink.activeConns.Add(1)
|
||||
s.sink.logger.Debug("msg", "TCP connection opened",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"session_id", sess.ID,
|
||||
"active_connections", newCount)
|
||||
|
||||
return nil, gnet.None
|
||||
}
|
||||
|
||||
// OnClose is called when a connection is closed
|
||||
func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
|
||||
remoteAddrStr := c.RemoteAddr().String()
|
||||
|
||||
s.mu.RLock()
|
||||
client, exists := s.clients[c]
|
||||
s.mu.RUnlock()
|
||||
|
||||
if exists && client.sessionID != "" {
|
||||
s.sink.proxy.RemoveSession(client.sessionID)
|
||||
s.sink.logger.Debug("msg", "Session removed",
|
||||
"component", "tcp_sink",
|
||||
"session_id", client.sessionID,
|
||||
"remote_addr", remoteAddrStr)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
delete(s.clients, c)
|
||||
s.mu.Unlock()
|
||||
|
||||
s.sink.errorMu.Lock()
|
||||
delete(s.sink.consecutiveWriteErrors, c)
|
||||
s.sink.errorMu.Unlock()
|
||||
|
||||
newCount := s.sink.activeConns.Add(-1)
|
||||
s.sink.logger.Debug("msg", "TCP connection closed",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"active_connections", newCount,
|
||||
"error", err)
|
||||
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
// OnTraffic is called when data is received from a connection
|
||||
func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
|
||||
s.mu.RLock()
|
||||
client, exists := s.clients[c]
|
||||
s.mu.RUnlock()
|
||||
|
||||
// Update session activity
|
||||
if exists && client.sessionID != "" {
|
||||
s.sink.proxy.UpdateActivity(client.sessionID)
|
||||
}
|
||||
|
||||
// TCP sink doesn't expect data from clients, discard safely
|
||||
if bufLen := c.InboundBuffered(); bufLen > 0 {
|
||||
c.Next(bufLen)
|
||||
}
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
// broadcastData sends data to all connected clients
|
||||
func (t *TCPSink) broadcastData(data []byte) {
|
||||
t.server.mu.RLock()
|
||||
defer t.server.mu.RUnlock()
|
||||
|
||||
for conn, client := range t.server.clients {
|
||||
// Update session activity
|
||||
if client.sessionID != "" {
|
||||
t.proxy.UpdateActivity(client.sessionID)
|
||||
}
|
||||
|
||||
// Refresh write deadline on each write if configured
|
||||
if t.config.WriteTimeout > 0 {
|
||||
conn.SetWriteDeadline(time.Now().Add(time.Duration(t.config.WriteTimeout) * time.Millisecond))
|
||||
}
|
||||
|
||||
conn.AsyncWrite(data, func(c gnet.Conn, err error) error {
|
||||
if err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
t.handleWriteError(c, err)
|
||||
} else {
|
||||
t.errorMu.Lock()
|
||||
delete(t.consecutiveWriteErrors, c)
|
||||
t.errorMu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleWriteError manages errors during async writes
|
||||
func (t *TCPSink) handleWriteError(c gnet.Conn, err error) {
|
||||
remoteAddrStr := c.RemoteAddr().String()
|
||||
|
||||
t.errorMu.Lock()
|
||||
defer t.errorMu.Unlock()
|
||||
|
||||
t.consecutiveWriteErrors[c]++
|
||||
errorCount := t.consecutiveWriteErrors[c]
|
||||
|
||||
t.logger.Debug("msg", "AsyncWrite error",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"error", err,
|
||||
"consecutive_errors", errorCount)
|
||||
|
||||
// Close connection max consecutive write errors
|
||||
if errorCount >= TCPMaxConsecutiveWriteErrors {
|
||||
t.logger.Warn("msg", "Closing connection due to repeated write errors",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"error_count", errorCount)
|
||||
delete(t.consecutiveWriteErrors, c)
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package tcpchain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/chain"
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("tcp_chain", NewTCPChainSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp_chain sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultChainSinkBufferSize = 1000
|
||||
DefaultChainSinkDialTimeoutMS = 5000
|
||||
DefaultChainSinkWriteTimeoutMS = 5000
|
||||
DefaultChainSinkBackoffMinMS = 500
|
||||
DefaultChainSinkBackoffMaxMS = 30000
|
||||
DefaultChainSinkKeepAlivePeriodMS = 30000
|
||||
)
|
||||
|
||||
// TCPChainSink forwards structured entries to a downstream tcp_chain source
|
||||
type TCPChainSink struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
config *config.TCPChainSinkOptions
|
||||
|
||||
node string
|
||||
addr string
|
||||
helloLine []byte
|
||||
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// conn owned exclusively by run loop goroutine
|
||||
conn net.Conn
|
||||
everConnected bool
|
||||
dialTimeout time.Duration
|
||||
writeTimeout time.Duration
|
||||
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
totalProcessed atomic.Uint64
|
||||
writeErrors atomic.Uint64
|
||||
reconnects atomic.Uint64
|
||||
synthesized atomic.Uint64
|
||||
connected atomic.Bool
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewTCPChainSinkPlugin creates a tcp_chain sink through plugin factory
|
||||
func NewTCPChainSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.TCPChainSinkOptions{
|
||||
KeepAlive: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Host); err != nil {
|
||||
return nil, fmt.Errorf("host: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultChainSinkBufferSize
|
||||
}
|
||||
if opts.DialTimeoutMS <= 0 {
|
||||
opts.DialTimeoutMS = DefaultChainSinkDialTimeoutMS
|
||||
}
|
||||
if opts.WriteTimeoutMS <= 0 {
|
||||
opts.WriteTimeoutMS = DefaultChainSinkWriteTimeoutMS
|
||||
}
|
||||
if opts.BackoffMinMS <= 0 {
|
||||
opts.BackoffMinMS = DefaultChainSinkBackoffMinMS
|
||||
}
|
||||
if opts.BackoffMaxMS < opts.BackoffMinMS {
|
||||
opts.BackoffMaxMS = DefaultChainSinkBackoffMaxMS
|
||||
}
|
||||
if opts.KeepAlivePeriodMS <= 0 {
|
||||
opts.KeepAlivePeriodMS = DefaultChainSinkKeepAlivePeriodMS
|
||||
}
|
||||
|
||||
node := opts.Node
|
||||
if node == "" {
|
||||
if hn, err := os.Hostname(); err == nil {
|
||||
node = hn
|
||||
} else {
|
||||
node = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
helloLine, err := chain.EncodeHello(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hello: %w", err)
|
||||
}
|
||||
|
||||
t := &TCPChainSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
node: node,
|
||||
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
|
||||
helloLine: helloLine,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
dialTimeout: time.Duration(opts.DialTimeoutMS) * time.Millisecond,
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
t.session = proxy.CreateSession(
|
||||
"tcp_chain://"+t.addr,
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "tcp_chain",
|
||||
"target": t.addr,
|
||||
"node": node,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Info("msg", "TCP chain sink initialized",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", id,
|
||||
"target", t.addr,
|
||||
"node", node)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *TCPChainSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *TCPChainSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start launches the forwarding loop; connection is established lazily so
|
||||
// pipeline start does not depend on downstream availability
|
||||
func (t *TCPChainSink) Start(ctx context.Context) error {
|
||||
t.startTime = time.Now()
|
||||
t.wg.Add(1)
|
||||
go t.runLoop(ctx)
|
||||
|
||||
t.logger.Info("msg", "TCP chain sink started",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"target", t.addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the forwarding loop. Worst-case latency: one write timeout
|
||||
// plus one backoff wait (both interruptible or bounded).
|
||||
func (t *TCPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP chain sink",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
|
||||
if t.session != nil {
|
||||
t.proxy.RemoveSession(t.session.ID)
|
||||
}
|
||||
|
||||
t.logger.Info("msg", "TCP chain sink stopped",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *TCPChainSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
var active int64
|
||||
if t.connected.Load() {
|
||||
active = 1
|
||||
}
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "tcp_chain",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: active,
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": t.addr,
|
||||
"node": t.node,
|
||||
"connected": t.connected.Load(),
|
||||
"reconnects": t.reconnects.Load(),
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
"synthesized": t.synthesized.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runLoop consumes transport events and forwards them downstream
|
||||
func (t *TCPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
defer t.closeConn()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "tcp_chain_sink",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
if !t.deliver(ctx, append(line, '\n')) {
|
||||
return // shutdown during retry
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toEntry extracts the structured entry, stamping node identity at first hop
|
||||
func (t *TCPChainSink) toEntry(event core.TransportEvent) core.LogEntry {
|
||||
entry := event.Entry
|
||||
if entry.Time.IsZero() {
|
||||
// Defensive: event without structured entry, wrap formatted payload
|
||||
t.synthesized.Add(1)
|
||||
entry = core.LogEntry{
|
||||
Time: event.Time,
|
||||
Source: t.id,
|
||||
Message: string(event.Payload),
|
||||
}
|
||||
}
|
||||
if entry.Node == "" {
|
||||
entry.Node = t.node
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
// deliver writes one line, holding it across reconnects until sent or shutdown.
|
||||
// Backpressure during outage propagates to the pipeline dispatch drop counter.
|
||||
func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
|
||||
failures := 0
|
||||
for {
|
||||
if t.conn == nil {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
return false
|
||||
}
|
||||
if err := t.connect(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
failures++
|
||||
t.logger.Debug("msg", "Chain connect failed",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
t.conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
if _, err := t.conn.Write(line); err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain write failed",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"error", err)
|
||||
t.closeConn()
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// connect performs a single dial + hello attempt
|
||||
func (t *TCPChainSink) connect(ctx context.Context) error {
|
||||
d := net.Dialer{Timeout: t.dialTimeout}
|
||||
if t.config.KeepAlive {
|
||||
d.KeepAliveConfig = net.KeepAliveConfig{
|
||||
Enable: true,
|
||||
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
// IPv4-only
|
||||
conn, err := d.DialContext(ctx, "tcp4", t.addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
if _, err := conn.Write(t.helloLine); err != nil {
|
||||
conn.Close()
|
||||
return fmt.Errorf("hello: %w", err)
|
||||
}
|
||||
|
||||
t.conn = conn
|
||||
t.connected.Store(true)
|
||||
if t.everConnected {
|
||||
t.reconnects.Add(1)
|
||||
}
|
||||
t.everConnected = true
|
||||
|
||||
t.logger.Info("msg", "Chain link established",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"node", t.node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// closeConn tears down the current connection (run loop goroutine only)
|
||||
func (t *TCPChainSink) closeConn() {
|
||||
if t.conn != nil {
|
||||
t.conn.Close()
|
||||
t.conn = nil
|
||||
}
|
||||
t.connected.Store(false)
|
||||
}
|
||||
|
||||
// waitBackoff sleeps for the computed delay, interruptible by shutdown
|
||||
func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.done:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// backoffDelay computes exponential backoff with ±20% jitter
|
||||
func (t *TCPChainSink) backoffDelay(failures int) time.Duration {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
|
||||
d := maxD
|
||||
if failures < 63 {
|
||||
if v := minD << uint(failures-1); v > 0 && v < maxD {
|
||||
d = v
|
||||
}
|
||||
}
|
||||
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("console", NewConsoleSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register console source: %v", err))
|
||||
}
|
||||
|
||||
// Console stdin can only have one reader
|
||||
if err := plugin.SetSourceMetadata("console", &plugin.PluginMetadata{
|
||||
Capabilities: []core.Capability{core.CapSessionAware, core.CapSingleInstance},
|
||||
MaxInstances: 1,
|
||||
}); err != nil {
|
||||
panic(fmt.Sprintf("failed to set console source metadata: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// ConsoleSource reads log entries from the standard input stream
|
||||
type ConsoleSource struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.ConsoleSourceOptions
|
||||
|
||||
// Application
|
||||
subscribers []chan core.LogEntry
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
|
||||
// Statistics
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
startTime time.Time
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultConsoleSourceBufferSize = 1000
|
||||
)
|
||||
|
||||
// NewConsoleSourcePlugin creates a console source through plugin factory
|
||||
func NewConsoleSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
opts := &config.ConsoleSourceOptions{}
|
||||
|
||||
// Scan config map
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate and apply defaults
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultConsoleSourceBufferSize
|
||||
}
|
||||
|
||||
// Create and return plugin instance
|
||||
cs := &ConsoleSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
cs.lastEntryTime.Store(time.Time{})
|
||||
|
||||
// Create session
|
||||
cs.session = proxy.CreateSession(
|
||||
"console_stdin",
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "console",
|
||||
},
|
||||
)
|
||||
|
||||
cs.logger.Info("msg", "Console source initialized",
|
||||
"component", "console_source",
|
||||
"instance_id", id)
|
||||
|
||||
return cs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (s *ConsoleSource) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware, // Single console session
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries.
|
||||
func (s *ConsoleSource) Subscribe() <-chan core.LogEntry {
|
||||
ch := make(chan core.LogEntry, s.config.BufferSize)
|
||||
s.subscribers = append(s.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start begins reading from the standard input.
|
||||
func (s *ConsoleSource) Start() error {
|
||||
s.startTime = time.Now()
|
||||
go s.readLoop()
|
||||
|
||||
// Update session activity
|
||||
s.proxy.UpdateActivity(s.session.ID)
|
||||
|
||||
s.logger.Info("msg", "Console source started",
|
||||
"component", "console_source",
|
||||
"instance_id", s.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop signals the source to stop reading.
|
||||
func (s *ConsoleSource) Stop() {
|
||||
close(s.done)
|
||||
|
||||
// Remove session
|
||||
if s.session != nil {
|
||||
s.proxy.RemoveSession(s.session.ID)
|
||||
}
|
||||
|
||||
// Close subscriber channels
|
||||
for _, ch := range s.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
|
||||
s.logger.Info("msg", "Console source stopped",
|
||||
"component", "console_source",
|
||||
"instance_id", s.id)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (s *ConsoleSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
|
||||
|
||||
return source.SourceStats{
|
||||
Type: "console",
|
||||
TotalEntries: s.totalEntries.Load(),
|
||||
DroppedEntries: s.droppedEntries.Load(),
|
||||
StartTime: s.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{},
|
||||
}
|
||||
}
|
||||
|
||||
// readLoop continuously reads lines from stdin and publishes them
|
||||
func (s *ConsoleSource) readLoop() {
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
for scanner.Scan() {
|
||||
select {
|
||||
case <-s.done:
|
||||
return
|
||||
default:
|
||||
// Update session activity on each read
|
||||
s.proxy.UpdateActivity(s.session.ID)
|
||||
|
||||
// Get raw line
|
||||
lineBytes := scanner.Bytes()
|
||||
if len(lineBytes) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Add newline back (scanner strips it)
|
||||
lineWithNewline := append(lineBytes, '\n')
|
||||
|
||||
entry := core.LogEntry{
|
||||
Time: time.Now(),
|
||||
Source: "console",
|
||||
Message: string(lineWithNewline), // Keep newline
|
||||
Level: source.ExtractLogLevel(string(lineBytes)),
|
||||
RawSize: int64(len(lineWithNewline)),
|
||||
}
|
||||
|
||||
s.publish(entry)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
s.logger.Error("msg", "Scanner error reading stdin",
|
||||
"component", "console_source",
|
||||
"instance_id", s.id,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers
|
||||
func (s *ConsoleSource) publish(entry core.LogEntry) {
|
||||
s.totalEntries.Add(1)
|
||||
s.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range s.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
s.droppedEntries.Add(1)
|
||||
s.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
|
||||
"component", "console_source")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("file", NewFileSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register file source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// FileSource monitors log files and tails them
|
||||
type FileSource struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.FileSourceOptions
|
||||
|
||||
// Application
|
||||
subscribers []chan core.LogEntry
|
||||
watchers map[string]*fileWatcher
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
// Statistics
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
startTime time.Time
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultFileSourcePattern = "*"
|
||||
DefaultFileSourceCheckIntervalMS = 100
|
||||
MinFileSourceCheckIntervalMS = 10
|
||||
)
|
||||
|
||||
// NewFileSourcePlugin creates a file source through plugin factory
|
||||
func NewFileSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
opts := &config.FileSourceOptions{}
|
||||
|
||||
// Use lconfig to scan map into struct (overriding defaults)
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate and apply defaults
|
||||
if err := lconfig.NonEmpty(opts.Directory); err != nil {
|
||||
return nil, fmt.Errorf("directory: %w", err)
|
||||
}
|
||||
|
||||
if opts.Pattern == "" {
|
||||
opts.Pattern = DefaultFileSourcePattern
|
||||
}
|
||||
if opts.CheckIntervalMS <= 0 {
|
||||
opts.CheckIntervalMS = DefaultFileSourceCheckIntervalMS
|
||||
} else if opts.CheckIntervalMS < MinFileSourceCheckIntervalMS {
|
||||
return nil, fmt.Errorf("check_interval_ms: must be >= %d", MinFileSourceCheckIntervalMS)
|
||||
}
|
||||
|
||||
// Create and return plugin instance
|
||||
fs := &FileSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
watchers: make(map[string]*fileWatcher),
|
||||
logger: logger,
|
||||
}
|
||||
fs.lastEntryTime.Store(time.Time{})
|
||||
|
||||
fs.session = proxy.CreateSession(
|
||||
fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Pattern),
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "file",
|
||||
"directory": opts.Directory,
|
||||
"pattern": opts.Pattern,
|
||||
},
|
||||
)
|
||||
|
||||
fs.logger.Info("msg", "File source initialized",
|
||||
"component", "file_source",
|
||||
"instance_id", id,
|
||||
"directory", opts.Directory,
|
||||
"pattern", opts.Pattern)
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (fs *FileSource) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware, // Tracks sessions per file
|
||||
core.CapMultiSession, // Multiple file sessions
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (fs *FileSource) Subscribe() <-chan core.LogEntry {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
|
||||
ch := make(chan core.LogEntry, 1000)
|
||||
fs.subscribers = append(fs.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start begins the file monitoring loop
|
||||
func (fs *FileSource) Start() error {
|
||||
fs.ctx, fs.cancel = context.WithCancel(context.Background())
|
||||
fs.startTime = time.Now()
|
||||
fs.wg.Add(1)
|
||||
go fs.monitorLoop()
|
||||
|
||||
fs.logger.Info("msg", "File source started",
|
||||
"component", "File_source",
|
||||
"path", fs.config.Directory,
|
||||
"pattern", fs.config.Pattern,
|
||||
"check_interval_ms", fs.config.CheckIntervalMS)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the file source and all file watchers
|
||||
func (fs *FileSource) Stop() {
|
||||
if fs.cancel != nil {
|
||||
fs.cancel()
|
||||
}
|
||||
fs.wg.Wait()
|
||||
|
||||
fs.proxy.RemoveSession(fs.session.ID)
|
||||
|
||||
fs.mu.Lock()
|
||||
for _, w := range fs.watchers {
|
||||
w.stop()
|
||||
}
|
||||
for _, ch := range fs.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
fs.mu.Unlock()
|
||||
|
||||
fs.logger.Info("msg", "File source stopped",
|
||||
"component", "file_source",
|
||||
"instance_id", fs.id,
|
||||
"path", fs.config.Directory)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics, including active watchers.
|
||||
func (fs *FileSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := fs.lastEntryTime.Load().(time.Time)
|
||||
|
||||
fs.mu.RLock()
|
||||
watcherCount := int64(len(fs.watchers))
|
||||
details := make(map[string]any)
|
||||
|
||||
// Add watcher details
|
||||
watchers := make([]map[string]any, 0, watcherCount)
|
||||
for _, w := range fs.watchers {
|
||||
info := w.getInfo()
|
||||
watchers = append(watchers, map[string]any{
|
||||
"directory": info.Directory,
|
||||
"size": info.Size,
|
||||
"position": info.Position,
|
||||
"entries_read": info.EntriesRead,
|
||||
"rotations": info.Rotations,
|
||||
"last_read": info.LastReadTime,
|
||||
})
|
||||
}
|
||||
details["watchers"] = watchers
|
||||
details["active_watchers"] = watcherCount
|
||||
fs.mu.RUnlock()
|
||||
|
||||
return source.SourceStats{
|
||||
ID: fs.id,
|
||||
Type: "file",
|
||||
TotalEntries: fs.totalEntries.Load(),
|
||||
DroppedEntries: fs.droppedEntries.Load(),
|
||||
StartTime: fs.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: details,
|
||||
}
|
||||
}
|
||||
|
||||
// monitorLoop periodically scans path for new or changed files.
|
||||
func (fs *FileSource) monitorLoop() {
|
||||
defer fs.wg.Done()
|
||||
|
||||
fs.checkTargets()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(fs.config.CheckIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-fs.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
fs.checkTargets()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// checkTargets finds matching files and ensures watchers are running for them.
|
||||
func (fs *FileSource) checkTargets() {
|
||||
files, err := fs.scanFile()
|
||||
if err != nil {
|
||||
fs.logger.Warn("msg", "Failed to scan file",
|
||||
"component", "file_source",
|
||||
"path", fs.config.Directory,
|
||||
"pattern", fs.config.Pattern,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
fs.ensureWatcher(file)
|
||||
}
|
||||
|
||||
fs.cleanupWatchers()
|
||||
}
|
||||
|
||||
// ensureWatcher creates and starts a new file watcher if one doesn't exist for the given path.
|
||||
func (fs *FileSource) ensureWatcher(path string) {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
|
||||
if _, exists := fs.watchers[path]; exists {
|
||||
return
|
||||
}
|
||||
|
||||
w := newFileWatcher(path, fs.publish, fs.logger)
|
||||
fs.watchers[path] = w
|
||||
|
||||
fs.logger.Debug("msg", "Created file watcher",
|
||||
"component", "file_source",
|
||||
"path", path)
|
||||
|
||||
fs.wg.Add(1)
|
||||
go func() {
|
||||
defer fs.wg.Done()
|
||||
if err := w.watch(fs.ctx); err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
fs.logger.Debug("msg", "Watcher cancelled",
|
||||
"component", "file_source",
|
||||
"path", path)
|
||||
} else {
|
||||
fs.logger.Error("msg", "Watcher failed",
|
||||
"component", "file_source",
|
||||
"path", path,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
fs.mu.Lock()
|
||||
delete(fs.watchers, path)
|
||||
fs.mu.Unlock()
|
||||
}()
|
||||
}
|
||||
|
||||
// cleanupWatchers stops and removes watchers for files that no longer exist.
|
||||
func (fs *FileSource) cleanupWatchers() {
|
||||
fs.mu.Lock()
|
||||
defer fs.mu.Unlock()
|
||||
|
||||
for path, w := range fs.watchers {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
w.stop()
|
||||
delete(fs.watchers, path)
|
||||
fs.logger.Debug("msg", "Cleaned up watcher for non-existent file",
|
||||
"component", "file_source",
|
||||
"path", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers.
|
||||
func (fs *FileSource) publish(entry core.LogEntry) {
|
||||
fs.mu.RLock()
|
||||
defer fs.mu.RUnlock()
|
||||
|
||||
fs.totalEntries.Add(1)
|
||||
fs.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range fs.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
fs.droppedEntries.Add(1)
|
||||
fs.logger.Debug("msg", "Dropped log entry - subscriber buffer full",
|
||||
"component", "file_source")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// scanFile finds all files in the configured path that match the pattern.
|
||||
func (fs *FileSource) scanFile() ([]string, error) {
|
||||
entries, err := os.ReadDir(fs.config.Directory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Convert glob pattern to regex
|
||||
regexPattern := globToRegex(fs.config.Pattern)
|
||||
re, err := regexp.Compile(regexPattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern regex: %w", err)
|
||||
}
|
||||
|
||||
var files []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
name := entry.Name()
|
||||
if re.MatchString(name) {
|
||||
files = append(files, filepath.Join(fs.config.Directory, name))
|
||||
}
|
||||
}
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// globToRegex converts a simple glob pattern to a regular expression.
|
||||
func globToRegex(glob string) string {
|
||||
regex := regexp.QuoteMeta(glob)
|
||||
regex = strings.ReplaceAll(regex, `\*`, `.*`)
|
||||
regex = strings.ReplaceAll(regex, `\?`, `.`)
|
||||
return "^" + regex + "$"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,383 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/source"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// WatcherInfo contains snapshot information about a file watcher's state
|
||||
type WatcherInfo struct {
|
||||
Directory string
|
||||
Size int64
|
||||
Position int64
|
||||
ModTime time.Time
|
||||
EntriesRead uint64
|
||||
LastReadTime time.Time
|
||||
Rotations int64
|
||||
}
|
||||
|
||||
// fileWatcher tails a single file, handles rotations, and sends new lines to a callback
|
||||
type fileWatcher struct {
|
||||
directory string
|
||||
callback func(core.LogEntry)
|
||||
position int64
|
||||
size int64
|
||||
inode uint64
|
||||
modTime time.Time
|
||||
mu sync.Mutex
|
||||
stopped bool
|
||||
rotationSeq int64
|
||||
entriesRead atomic.Uint64
|
||||
lastReadTime atomic.Value // time.Time
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
// newFileWatcher creates a new watcher for a specific file path
|
||||
func newFileWatcher(directory string, callback func(core.LogEntry), logger *log.Logger) *fileWatcher {
|
||||
w := &fileWatcher{
|
||||
directory: directory,
|
||||
callback: callback,
|
||||
position: -1,
|
||||
logger: logger,
|
||||
}
|
||||
w.lastReadTime.Store(time.Time{})
|
||||
return w
|
||||
}
|
||||
|
||||
// watch starts the main monitoring loop for the file
|
||||
func (w *fileWatcher) watch(ctx context.Context) error {
|
||||
if err := w.seekToEnd(); err != nil {
|
||||
return fmt.Errorf("seekToEnd failed: %w", err)
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(core.FileWatcherPollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if w.isStopped() {
|
||||
return fmt.Errorf("watcher stopped")
|
||||
}
|
||||
if err := w.checkFile(); err != nil {
|
||||
// Log error but continue watching
|
||||
w.logger.Warn("msg", "checkFile error", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stop signals the watcher to terminate its loop
|
||||
func (w *fileWatcher) stop() {
|
||||
w.mu.Lock()
|
||||
w.stopped = true
|
||||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
// getInfo returns a snapshot of the watcher's current statistics
|
||||
func (w *fileWatcher) getInfo() WatcherInfo {
|
||||
w.mu.Lock()
|
||||
info := WatcherInfo{
|
||||
Directory: w.directory,
|
||||
Size: w.size,
|
||||
Position: w.position,
|
||||
ModTime: w.modTime,
|
||||
EntriesRead: w.entriesRead.Load(),
|
||||
Rotations: w.rotationSeq,
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
if lastRead, ok := w.lastReadTime.Load().(time.Time); ok {
|
||||
info.LastReadTime = lastRead
|
||||
}
|
||||
|
||||
return info
|
||||
}
|
||||
|
||||
// checkFile examines the file for changes, rotations, or new content
|
||||
func (w *fileWatcher) checkFile() error {
|
||||
file, err := os.Open(w.directory)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// File doesn't exist yet, keep watching
|
||||
return nil
|
||||
}
|
||||
w.logger.Error("msg", "Failed to open file for checking",
|
||||
"component", "file_watcher",
|
||||
"directory", w.directory,
|
||||
"error", err)
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
w.logger.Error("msg", "Failed to stat file",
|
||||
"component", "file_watcher",
|
||||
"directory", w.directory,
|
||||
"error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
oldPos := w.position
|
||||
oldSize := w.size
|
||||
oldInode := w.inode
|
||||
oldModTime := w.modTime
|
||||
w.mu.Unlock()
|
||||
|
||||
currentSize := info.Size()
|
||||
currentModTime := info.ModTime()
|
||||
var currentInode uint64
|
||||
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
currentInode = stat.Ino
|
||||
}
|
||||
|
||||
// Handle first time seeing a file that didn't exist before
|
||||
if oldInode == 0 && currentInode != 0 {
|
||||
// File just appeared, don't treat as rotation
|
||||
w.mu.Lock()
|
||||
w.inode = currentInode
|
||||
w.size = currentSize
|
||||
w.modTime = currentModTime
|
||||
// Position stays at 0 for new files
|
||||
w.mu.Unlock()
|
||||
// Don't return here - continue to read content
|
||||
}
|
||||
|
||||
// Check for rotation
|
||||
rotated := false
|
||||
rotationReason := ""
|
||||
startPos := oldPos
|
||||
|
||||
// Rotation detection
|
||||
if currentSize < oldSize {
|
||||
// File was truncated
|
||||
rotated = true
|
||||
rotationReason = "size decrease"
|
||||
} else if currentModTime.Before(oldModTime) && currentSize <= oldSize {
|
||||
// Modification time went backwards (logrotate behavior)
|
||||
rotated = true
|
||||
rotationReason = "modification time reset"
|
||||
} else if oldPos > currentSize+1024 {
|
||||
// Our position is way beyond file size
|
||||
rotated = true
|
||||
rotationReason = "position beyond file size"
|
||||
} else if oldInode != 0 && currentInode != 0 && currentInode != oldInode {
|
||||
// Inode changed - distinguish between rotation and atomic save
|
||||
if currentSize == 0 {
|
||||
// Empty file with new inode = likely rotation
|
||||
rotated = true
|
||||
rotationReason = "inode change with empty file"
|
||||
} else if currentSize < oldPos {
|
||||
// New file is smaller than our position = rotation
|
||||
rotated = true
|
||||
rotationReason = "inode change with size less than position"
|
||||
} else {
|
||||
// Inode changed but file has content and size >= position
|
||||
// This is likely an atomic save by an editor
|
||||
// Update inode but keep position
|
||||
w.mu.Lock()
|
||||
w.inode = currentInode
|
||||
w.mu.Unlock()
|
||||
|
||||
w.logger.Debug("msg", "Atomic file update detected",
|
||||
"component", "file_watcher",
|
||||
"directory", w.directory,
|
||||
"old_inode", oldInode,
|
||||
"new_inode", currentInode,
|
||||
"position", oldPos,
|
||||
"size", currentSize)
|
||||
}
|
||||
}
|
||||
|
||||
if rotated {
|
||||
startPos = 0
|
||||
w.mu.Lock()
|
||||
w.rotationSeq++
|
||||
seq := w.rotationSeq
|
||||
w.inode = currentInode
|
||||
w.position = 0 // Reset position on rotation
|
||||
w.mu.Unlock()
|
||||
|
||||
w.callback(core.LogEntry{
|
||||
Time: time.Now(),
|
||||
Source: filepath.Base(w.directory),
|
||||
Level: "INFO",
|
||||
Message: fmt.Sprintf("Log rotation detected (#%d): %s", seq, rotationReason),
|
||||
})
|
||||
|
||||
w.logger.Info("msg", "Log rotation detected",
|
||||
"component", "file_watcher",
|
||||
"directory", w.directory,
|
||||
"sequence", seq,
|
||||
"reason", rotationReason)
|
||||
}
|
||||
|
||||
// Read if there's new content OR if we need to continue from position
|
||||
if currentSize > startPos {
|
||||
if _, err := file.Seek(startPos, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
rawSize := int64(len(line))
|
||||
entry := w.parseLine(line)
|
||||
entry.RawSize = rawSize
|
||||
|
||||
w.callback(entry)
|
||||
w.entriesRead.Add(1)
|
||||
w.lastReadTime.Store(time.Now())
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
w.logger.Error("msg", "Scanner error while reading file",
|
||||
"component", "file_watcher",
|
||||
"directory", w.directory,
|
||||
"position", startPos,
|
||||
"error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Update position after successful read
|
||||
currentPos, err := file.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
// Log error but don't fail - best effort position tracking
|
||||
w.logger.Warn("msg", "Failed to get file position", "error", err)
|
||||
// Use size as fallback position
|
||||
currentPos = currentSize
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
w.position = currentPos
|
||||
w.size = currentSize
|
||||
w.modTime = currentModTime
|
||||
if !rotated && currentInode != 0 {
|
||||
w.inode = currentInode
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
return scanner.Err()
|
||||
}
|
||||
|
||||
// Update metadata even if no new content
|
||||
w.mu.Lock()
|
||||
w.size = currentSize
|
||||
w.modTime = currentModTime
|
||||
if currentInode != 0 {
|
||||
w.inode = currentInode
|
||||
}
|
||||
w.mu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// seekToEnd sets the initial read position to the end of the file
|
||||
func (w *fileWatcher) seekToEnd() error {
|
||||
file, err := os.Open(w.directory)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
w.mu.Lock()
|
||||
w.position = 0
|
||||
w.size = 0
|
||||
w.modTime = time.Now()
|
||||
w.inode = 0
|
||||
w.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
// Keep existing position (including 0)
|
||||
// First time initialization seeks to the end of the file
|
||||
if w.position == -1 {
|
||||
pos, err := file.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.position = pos
|
||||
}
|
||||
|
||||
w.size = info.Size()
|
||||
w.modTime = info.ModTime()
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
w.inode = stat.Ino
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isStopped checks if the watcher has been instructed to stop
|
||||
func (w *fileWatcher) isStopped() bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.stopped
|
||||
}
|
||||
|
||||
// parseLine attempts to parse a line as JSON, falling back to plain text
|
||||
func (w *fileWatcher) parseLine(line string) core.LogEntry {
|
||||
var jsonLog struct {
|
||||
Time string `json:"time"`
|
||||
Level string `json:"level"`
|
||||
Message string `json:"msg"`
|
||||
Fields json.RawMessage `json:"fields"`
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(line), &jsonLog); err == nil {
|
||||
timestamp, err := time.Parse(time.RFC3339Nano, jsonLog.Time)
|
||||
if err != nil {
|
||||
timestamp = time.Now()
|
||||
}
|
||||
|
||||
return core.LogEntry{
|
||||
Time: timestamp,
|
||||
Source: filepath.Base(w.directory),
|
||||
Level: jsonLog.Level,
|
||||
Message: jsonLog.Message,
|
||||
Fields: jsonLog.Fields,
|
||||
}
|
||||
}
|
||||
|
||||
level := source.ExtractLogLevel(line)
|
||||
|
||||
return core.LogEntry{
|
||||
Time: time.Now(),
|
||||
Source: filepath.Base(w.directory),
|
||||
Level: level,
|
||||
Message: line,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/chain"
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("http_chain", NewHTTPChainSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http_chain source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultHTTPChainSourceBufferSize = 1000
|
||||
DefaultHTTPChainSourceIngestPath = "/ingest"
|
||||
DefaultHTTPChainSourceMaxBodyBytes = 8 * 1024 * 1024
|
||||
DefaultHTTPChainSourceReadTimeoutMS = 30000
|
||||
HTTPChainReadHeaderTimeout = 10 * time.Second
|
||||
HTTPChainServerShutdownTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// HTTPChainSource accepts NDJSON batches from upstream http_chain sinks
|
||||
type HTTPChainSource struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
config *config.HTTPChainSourceOptions
|
||||
|
||||
subscribers []chan core.LogEntry
|
||||
server *http.Server
|
||||
logger *log.Logger
|
||||
|
||||
// Session cache: one session per remote host + declared node
|
||||
sessions map[string]string // key -> sessionID
|
||||
sessionsMu sync.Mutex
|
||||
|
||||
mu sync.RWMutex
|
||||
|
||||
startTime time.Time
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
parseErrors atomic.Uint64
|
||||
totalRequests atomic.Uint64
|
||||
rejectedRequests atomic.Uint64
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewHTTPChainSourcePlugin creates an http_chain source through plugin factory
|
||||
func NewHTTPChainSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
opts := &config.HTTPChainSourceOptions{
|
||||
Host: "0.0.0.0",
|
||||
TrustNode: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.IngestPath == "" {
|
||||
opts.IngestPath = DefaultHTTPChainSourceIngestPath
|
||||
} else if !strings.HasPrefix(opts.IngestPath, "/") {
|
||||
return nil, fmt.Errorf("ingest_path: must start with '/'")
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPChainSourceBufferSize
|
||||
}
|
||||
if opts.MaxBodyBytes <= 0 {
|
||||
opts.MaxBodyBytes = DefaultHTTPChainSourceMaxBodyBytes
|
||||
}
|
||||
if opts.ReadTimeoutMS <= 0 {
|
||||
opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS
|
||||
}
|
||||
|
||||
s := &HTTPChainSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
sessions: make(map[string]string),
|
||||
logger: logger,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "HTTP chain source initialized",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port,
|
||||
"ingest_path", opts.IngestPath)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (s *HTTPChainSource) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (s *HTTPChainSource) Subscribe() <-chan core.LogEntry {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ch := make(chan core.LogEntry, s.config.BufferSize)
|
||||
s.subscribers = append(s.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start binds the listener and serves the ingest endpoint
|
||||
func (s *HTTPChainSource) Start() error {
|
||||
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
|
||||
// IPv4-only, aligns with tcp/http sinks
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped pattern: mux answers 405 with Allow header on non-POST
|
||||
mux.HandleFunc(http.MethodPost+" "+s.config.IngestPath, s.handleIngest)
|
||||
|
||||
s.server = &http.Server{
|
||||
Handler: mux,
|
||||
ReadTimeout: time.Duration(s.config.ReadTimeoutMS) * time.Millisecond,
|
||||
ReadHeaderTimeout: HTTPChainReadHeaderTimeout,
|
||||
// Future: TLSConfig for transport security
|
||||
}
|
||||
s.startTime = time.Now()
|
||||
|
||||
go func() {
|
||||
if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
s.logger.Error("msg", "HTTP chain server terminated",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", s.id,
|
||||
"error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
s.logger.Info("msg", "HTTP chain source started",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", s.id,
|
||||
"addr", addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop shuts down the server, sessions, and subscriber channels
|
||||
func (s *HTTPChainSource) Stop() {
|
||||
if s.server != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), HTTPChainServerShutdownTimeout)
|
||||
defer cancel()
|
||||
s.server.Shutdown(ctx)
|
||||
}
|
||||
|
||||
s.sessionsMu.Lock()
|
||||
for _, id := range s.sessions {
|
||||
s.proxy.RemoveSession(id)
|
||||
}
|
||||
s.sessions = make(map[string]string)
|
||||
s.sessionsMu.Unlock()
|
||||
|
||||
s.mu.Lock()
|
||||
for _, ch := range s.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.logger.Info("msg", "HTTP chain source stopped",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", s.id)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (s *HTTPChainSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
|
||||
|
||||
s.sessionsMu.Lock()
|
||||
cachedSessions := len(s.sessions)
|
||||
s.sessionsMu.Unlock()
|
||||
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
Type: "http_chain",
|
||||
TotalEntries: s.totalEntries.Load(),
|
||||
DroppedEntries: s.droppedEntries.Load(),
|
||||
StartTime: s.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{
|
||||
"host": s.config.Host,
|
||||
"port": s.config.Port,
|
||||
"ingest_path": s.config.IngestPath,
|
||||
"total_requests": s.totalRequests.Load(),
|
||||
"rejected_requests": s.rejectedRequests.Load(),
|
||||
"parse_errors": s.parseErrors.Load(),
|
||||
"cached_sessions": cachedSessions,
|
||||
"trust_node": s.config.TrustNode,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// handleIngest validates protocol headers and ingests one NDJSON batch.
|
||||
// Batch acceptance is atomic: entries publish only after a clean full read.
|
||||
func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) {
|
||||
s.totalRequests.Add(1)
|
||||
|
||||
if r.Header.Get(chain.HeaderProtocol) != strconv.Itoa(chain.ProtocolVersion) {
|
||||
s.rejectedRequests.Add(1)
|
||||
http.Error(w, "unsupported protocol version", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remoteHost := r.RemoteAddr
|
||||
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
||||
remoteHost = host
|
||||
}
|
||||
connNode := r.Header.Get(chain.HeaderNode)
|
||||
if connNode == "" || !s.config.TrustNode {
|
||||
connNode = remoteHost
|
||||
}
|
||||
|
||||
body := http.MaxBytesReader(w, r.Body, s.config.MaxBodyBytes)
|
||||
scanner := bufio.NewScanner(body)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
|
||||
|
||||
entries := make([]core.LogEntry, 0, 128)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode)
|
||||
if err != nil {
|
||||
// Content error within a clean transfer: skip line, keep batch
|
||||
s.parseErrors.Add(1)
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
// Transfer error: reject batch without partial ingestion, sender retries
|
||||
s.rejectedRequests.Add(1)
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) {
|
||||
http.Error(w, "body too large", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
s.logger.Debug("msg", "Chain batch read failed",
|
||||
"component", "http_chain_source",
|
||||
"remote_addr", r.RemoteAddr,
|
||||
"error", err)
|
||||
http.Error(w, "malformed body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
s.publish(entry)
|
||||
}
|
||||
s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode))
|
||||
|
||||
w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries)))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// sessionFor returns the cached session for a remote+node, recreating after idle expiry
|
||||
func (s *HTTPChainSource) sessionFor(remoteHost, node string) string {
|
||||
key := remoteHost + "|" + node
|
||||
s.sessionsMu.Lock()
|
||||
defer s.sessionsMu.Unlock()
|
||||
|
||||
if id, ok := s.sessions[key]; ok {
|
||||
if _, exists := s.proxy.GetSession(id); exists {
|
||||
return id
|
||||
}
|
||||
}
|
||||
sess := s.proxy.CreateSession(remoteHost, map[string]any{
|
||||
"type": "http_chain",
|
||||
"node": node,
|
||||
})
|
||||
s.sessions[key] = sess.ID
|
||||
return sess.ID
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers
|
||||
func (s *HTTPChainSource) publish(entry core.LogEntry) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
s.totalEntries.Add(1)
|
||||
s.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range s.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
s.droppedEntries.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package null
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/source"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("null", NewNullSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register null source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// NullSource generates no log entries, used for testing
|
||||
type NullSource struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Application
|
||||
subscribers []chan core.LogEntry
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
|
||||
// Statistics
|
||||
totalEntries atomic.Uint64
|
||||
startTime time.Time
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewNullSourcePlugin creates a null source through plugin factory
|
||||
func NewNullSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
ns := &NullSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
ns.lastEntryTime.Store(time.Time{})
|
||||
|
||||
// Create session for null source
|
||||
ns.session = proxy.CreateSession(
|
||||
"null://void",
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "null",
|
||||
},
|
||||
)
|
||||
|
||||
logger.Debug("msg", "Null source initialized",
|
||||
"component", "null_source",
|
||||
"instance_id", id)
|
||||
|
||||
return ns, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (ns *NullSource) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (ns *NullSource) Subscribe() <-chan core.LogEntry {
|
||||
ch := make(chan core.LogEntry, 1000)
|
||||
ns.subscribers = append(ns.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start begins the source operation (no-op for null source)
|
||||
func (ns *NullSource) Start() error {
|
||||
ns.startTime = time.Now()
|
||||
ns.proxy.UpdateActivity(ns.session.ID)
|
||||
ns.logger.Debug("msg", "Null source started",
|
||||
"component", "null_source",
|
||||
"instance_id", ns.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop signals the source to stop
|
||||
func (ns *NullSource) Stop() {
|
||||
close(ns.done)
|
||||
if ns.session != nil {
|
||||
ns.proxy.RemoveSession(ns.session.ID)
|
||||
}
|
||||
for _, ch := range ns.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
ns.logger.Debug("msg", "Null source stopped",
|
||||
"component", "null_source",
|
||||
"instance_id", ns.id)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (ns *NullSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := ns.lastEntryTime.Load().(time.Time)
|
||||
|
||||
return source.SourceStats{
|
||||
ID: ns.id,
|
||||
Type: "null",
|
||||
TotalEntries: ns.totalEntries.Load(),
|
||||
StartTime: ns.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,358 @@
|
||||
package random
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("random", NewRandomSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register random source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// RandomSource generates random log entries for testing
|
||||
type RandomSource struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.RandomSourceOptions
|
||||
|
||||
// Application
|
||||
subscribers []chan core.LogEntry
|
||||
logger *log.Logger
|
||||
rng *rand.Rand
|
||||
mu sync.RWMutex
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
cancel chan struct{}
|
||||
|
||||
// Statistics
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
startTime time.Time
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultRandomSourceIntervalMS = 500
|
||||
DefaultRandomSourceFormat = "txt"
|
||||
DefaultRandomSourceLength = 20
|
||||
)
|
||||
|
||||
// NewRandomSourcePlugin creates a random source through plugin factory
|
||||
func NewRandomSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
// Step 1: Create empty config struct with defaults
|
||||
opts := &config.RandomSourceOptions{
|
||||
IntervalMS: 500,
|
||||
JitterMS: 0,
|
||||
Format: "txt",
|
||||
Length: 20,
|
||||
Special: false,
|
||||
}
|
||||
|
||||
// Scan config map
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.IntervalMS <= 0 {
|
||||
opts.IntervalMS = DefaultRandomSourceIntervalMS
|
||||
}
|
||||
if opts.Format == "" {
|
||||
opts.Format = DefaultRandomSourceFormat
|
||||
}
|
||||
if opts.Length <= 0 {
|
||||
opts.Length = DefaultRandomSourceLength
|
||||
}
|
||||
|
||||
// Validate
|
||||
if opts.JitterMS < 0 {
|
||||
return nil, fmt.Errorf("jitter_ms cannot be negative")
|
||||
}
|
||||
if opts.JitterMS > opts.IntervalMS {
|
||||
opts.JitterMS = opts.IntervalMS
|
||||
}
|
||||
|
||||
validateFormat := lconfig.OneOf("raw", "txt", "json")
|
||||
if err := validateFormat(opts.Format); err != nil {
|
||||
return nil, fmt.Errorf("format: %w", err)
|
||||
}
|
||||
|
||||
rs := &RandomSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
done: make(chan struct{}),
|
||||
cancel: make(chan struct{}),
|
||||
logger: logger,
|
||||
rng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||
}
|
||||
rs.lastEntryTime.Store(time.Time{})
|
||||
|
||||
// Create session for random source
|
||||
rs.session = proxy.CreateSession(
|
||||
fmt.Sprintf("random://%s", id),
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "random",
|
||||
"format": opts.Format,
|
||||
"interval_ms": opts.IntervalMS,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Debug("msg", "Random source initialized",
|
||||
"component", "random_source",
|
||||
"instance_id", id,
|
||||
"format", opts.Format,
|
||||
"interval_ms", opts.IntervalMS,
|
||||
"jitter_ms", opts.JitterMS)
|
||||
|
||||
return rs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (rs *RandomSource) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (rs *RandomSource) Subscribe() <-chan core.LogEntry {
|
||||
rs.mu.Lock()
|
||||
defer rs.mu.Unlock()
|
||||
ch := make(chan core.LogEntry, 1000)
|
||||
rs.subscribers = append(rs.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start begins generating random log entries
|
||||
func (rs *RandomSource) Start() error {
|
||||
rs.startTime = time.Now()
|
||||
rs.wg.Add(1)
|
||||
go rs.generateLoop()
|
||||
|
||||
rs.proxy.UpdateActivity(rs.session.ID)
|
||||
rs.logger.Debug("msg", "Random source started",
|
||||
"component", "random_source",
|
||||
"instance_id", rs.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop signals the source to stop generating
|
||||
func (rs *RandomSource) Stop() {
|
||||
close(rs.cancel)
|
||||
rs.wg.Wait()
|
||||
|
||||
if rs.session != nil {
|
||||
rs.proxy.RemoveSession(rs.session.ID)
|
||||
}
|
||||
|
||||
rs.mu.Lock()
|
||||
for _, ch := range rs.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
rs.mu.Unlock()
|
||||
|
||||
rs.logger.Debug("msg", "Random source stopped",
|
||||
"component", "random_source",
|
||||
"instance_id", rs.id,
|
||||
"total_entries", rs.totalEntries.Load())
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (rs *RandomSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := rs.lastEntryTime.Load().(time.Time)
|
||||
|
||||
return source.SourceStats{
|
||||
ID: rs.id,
|
||||
Type: "random",
|
||||
TotalEntries: rs.totalEntries.Load(),
|
||||
DroppedEntries: rs.droppedEntries.Load(),
|
||||
StartTime: rs.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{
|
||||
"format": rs.config.Format,
|
||||
"interval_ms": rs.config.IntervalMS,
|
||||
"jitter_ms": rs.config.JitterMS,
|
||||
"length": rs.config.Length,
|
||||
"special": rs.config.Special,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// generateLoop continuously generates random log entries at configured intervals
|
||||
func (rs *RandomSource) generateLoop() {
|
||||
defer rs.wg.Done()
|
||||
|
||||
for {
|
||||
// Calculate next interval with jitter
|
||||
interval := time.Duration(rs.config.IntervalMS) * time.Millisecond
|
||||
if rs.config.JitterMS > 0 {
|
||||
jitter := time.Duration(rs.rng.Intn(int(rs.config.JitterMS))) * time.Millisecond
|
||||
interval = interval - time.Duration(rs.config.JitterMS/2)*time.Millisecond + jitter
|
||||
}
|
||||
|
||||
select {
|
||||
case <-time.After(interval):
|
||||
entry := rs.generateEntry()
|
||||
rs.publish(entry)
|
||||
rs.proxy.UpdateActivity(rs.session.ID)
|
||||
case <-rs.cancel:
|
||||
return
|
||||
case <-rs.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generateEntry creates a random log entry based on configured format
|
||||
func (rs *RandomSource) generateEntry() core.LogEntry {
|
||||
now := time.Now()
|
||||
|
||||
switch rs.config.Format {
|
||||
case "raw":
|
||||
message := rs.generateRandomString(int(rs.config.Length))
|
||||
return core.LogEntry{
|
||||
Time: now,
|
||||
Source: fmt.Sprintf("random_%s", rs.id),
|
||||
Message: message,
|
||||
RawSize: int64(len(message) + 1), // +1 for newline
|
||||
}
|
||||
|
||||
case "txt":
|
||||
level := rs.randomLogLevel()
|
||||
message := rs.generateRandomString(int(rs.config.Length))
|
||||
formatted := fmt.Sprintf("[%s] [%s] random_%s - %s",
|
||||
now.Format(time.RFC3339),
|
||||
level,
|
||||
rs.id,
|
||||
message)
|
||||
return core.LogEntry{
|
||||
Time: now,
|
||||
Source: fmt.Sprintf("random_%s", rs.id),
|
||||
Level: level,
|
||||
Message: formatted,
|
||||
RawSize: int64(len(formatted) + 1),
|
||||
}
|
||||
|
||||
case "json":
|
||||
level := rs.randomLogLevel()
|
||||
message := rs.generateRandomString(int(rs.config.Length))
|
||||
data := map[string]any{
|
||||
"time": now.Format(time.RFC3339Nano),
|
||||
"level": level,
|
||||
"source": fmt.Sprintf("random_%s", rs.id),
|
||||
"message": message,
|
||||
}
|
||||
jsonBytes, _ := json.Marshal(data)
|
||||
return core.LogEntry{
|
||||
Time: now,
|
||||
Source: fmt.Sprintf("random_%s", rs.id),
|
||||
Level: level,
|
||||
Message: string(jsonBytes),
|
||||
RawSize: int64(len(jsonBytes) + 1),
|
||||
}
|
||||
|
||||
default:
|
||||
return core.LogEntry{}
|
||||
}
|
||||
}
|
||||
|
||||
// generateRandomString creates a random string of specified length
|
||||
func (rs *RandomSource) generateRandomString(length int) string {
|
||||
const normalChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 "
|
||||
const specialChars = "\t\n\r\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F"
|
||||
const unicodeChars = "™€¢£¥§©®°±µ¶·ÀÉÑÖÜßäëïöü←↑→↓∀∃∅∇∈∉∪∩≈≠≤≥"
|
||||
|
||||
result := make([]byte, 0, length)
|
||||
|
||||
if rs.config.Special && length >= 3 {
|
||||
// Reserve space for at least one special and one unicode char
|
||||
normalLength := length - 2
|
||||
|
||||
// Generate normal characters
|
||||
for i := 0; i < normalLength; i++ {
|
||||
result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
|
||||
}
|
||||
|
||||
// Insert special character at random position
|
||||
specialPos := rs.rng.Intn(len(result) + 1)
|
||||
specialChar := specialChars[rs.rng.Intn(len(specialChars))]
|
||||
result = append(result[:specialPos], append([]byte{specialChar}, result[specialPos:]...)...)
|
||||
|
||||
// Insert unicode character at random position
|
||||
unicodePos := rs.rng.Intn(len(result) + 1)
|
||||
unicodeChar := unicodeChars[rs.rng.Intn(len(unicodeChars)/3)*3:]
|
||||
if len(unicodeChar) >= 3 {
|
||||
unicodeBytes := []byte(unicodeChar[:3])
|
||||
if unicodePos == len(result) {
|
||||
result = append(result, unicodeBytes...)
|
||||
} else {
|
||||
result = append(result[:unicodePos], append(unicodeBytes, result[unicodePos:]...)...)
|
||||
}
|
||||
}
|
||||
|
||||
// Trim to exact length if needed
|
||||
if len(result) > length {
|
||||
result = result[:length]
|
||||
}
|
||||
} else {
|
||||
// Normal generation without special characters
|
||||
for i := 0; i < length; i++ {
|
||||
result = append(result, normalChars[rs.rng.Intn(len(normalChars))])
|
||||
}
|
||||
}
|
||||
|
||||
return string(result)
|
||||
}
|
||||
|
||||
// randomLogLevel returns a random log level
|
||||
func (rs *RandomSource) randomLogLevel() string {
|
||||
levels := []string{"DEBUG", "INFO", "WARN", "ERROR"}
|
||||
return levels[rs.rng.Intn(len(levels))]
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers
|
||||
func (rs *RandomSource) publish(entry core.LogEntry) {
|
||||
rs.mu.RLock()
|
||||
defer rs.mu.RUnlock()
|
||||
|
||||
rs.totalEntries.Add(1)
|
||||
rs.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range rs.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
rs.droppedEntries.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
)
|
||||
|
||||
// Source represents an input data stream for log entries
|
||||
type Source interface {
|
||||
// Capabilities returns a slice of supported Source capabilities
|
||||
Capabilities() []core.Capability
|
||||
|
||||
// Subscribe returns a channel that receives log entries from the source
|
||||
Subscribe() <-chan core.LogEntry
|
||||
|
||||
// Start begins reading from the source
|
||||
Start() error
|
||||
|
||||
// Stop gracefully shuts down the source
|
||||
Stop()
|
||||
|
||||
// SourceStats contains statistics about a source
|
||||
GetStats() SourceStats
|
||||
}
|
||||
|
||||
// SourceStats contains statistics about a source
|
||||
type SourceStats struct {
|
||||
ID string
|
||||
Type string
|
||||
TotalEntries uint64
|
||||
DroppedEntries uint64
|
||||
StartTime time.Time
|
||||
LastEntryTime time.Time
|
||||
Details map[string]any
|
||||
}
|
||||
|
||||
// ExtractLogLevel heuristically determines the log level from a line of text
|
||||
func ExtractLogLevel(line string) string {
|
||||
patterns := []struct {
|
||||
patterns []string
|
||||
level string
|
||||
}{
|
||||
{[]string{"[ERROR]", "ERROR:", " ERROR ", "ERR:", "[ERR]", "FATAL:", "[FATAL]"}, "ERROR"},
|
||||
{[]string{"[WARN]", "WARN:", " WARN ", "WARNING:", "[WARNING]"}, "WARN"},
|
||||
{[]string{"[INFO]", "INFO:", " INFO ", "[INF]", "INF:"}, "INFO"},
|
||||
{[]string{"[DEBUG]", "DEBUG:", " DEBUG ", "[DBG]", "DBG:"}, "DEBUG"},
|
||||
{[]string{"[TRACE]", "TRACE:", " TRACE "}, "TRACE"},
|
||||
}
|
||||
|
||||
upperLine := strings.ToUpper(line)
|
||||
for _, group := range patterns {
|
||||
for _, pattern := range group.patterns {
|
||||
if strings.Contains(upperLine, pattern) {
|
||||
return group.level
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package tcpchain
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/chain"
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("tcp_chain", NewTCPChainSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp_chain source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultChainSourceBufferSize = 1000
|
||||
DefaultChainSourceHelloTimeoutMS = 10000
|
||||
)
|
||||
|
||||
// TCPChainSource accepts connections from upstream tcp_chain sinks and ingests NDJSON entries
|
||||
type TCPChainSource struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
config *config.TCPChainSourceOptions
|
||||
|
||||
subscribers []chan core.LogEntry
|
||||
listener net.Listener
|
||||
conns map[net.Conn]struct{}
|
||||
logger *log.Logger
|
||||
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
startTime time.Time
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
parseErrors atomic.Uint64
|
||||
rejectedConns atomic.Uint64
|
||||
activeConns atomic.Int64
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewTCPChainSourcePlugin creates a tcp_chain source through plugin factory
|
||||
func NewTCPChainSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
opts := &config.TCPChainSourceOptions{
|
||||
Host: "0.0.0.0",
|
||||
TrustNode: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultChainSourceBufferSize
|
||||
}
|
||||
if opts.HelloTimeoutMS <= 0 {
|
||||
opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS
|
||||
}
|
||||
|
||||
s := &TCPChainSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
conns: make(map[net.Conn]struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "TCP chain source initialized",
|
||||
"component", "tcp_chain_source",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (s *TCPChainSource) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (s *TCPChainSource) Subscribe() <-chan core.LogEntry {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ch := make(chan core.LogEntry, s.config.BufferSize)
|
||||
s.subscribers = append(s.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start binds the listener and begins accepting connections
|
||||
func (s *TCPChainSource) Start() error {
|
||||
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
|
||||
// IPv4-only
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
s.listener = ln
|
||||
s.ctx, s.cancel = context.WithCancel(context.Background())
|
||||
s.startTime = time.Now()
|
||||
|
||||
s.wg.Add(1)
|
||||
go s.acceptLoop()
|
||||
|
||||
s.logger.Info("msg", "TCP chain source started",
|
||||
"component", "tcp_chain_source",
|
||||
"instance_id", s.id,
|
||||
"addr", addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop closes the listener, all connections, and subscriber channels
|
||||
func (s *TCPChainSource) Stop() {
|
||||
if s.cancel != nil {
|
||||
s.cancel()
|
||||
}
|
||||
if s.listener != nil {
|
||||
s.listener.Close()
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
for conn := range s.conns {
|
||||
conn.Close() // unblocks per-connection reads
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.wg.Wait()
|
||||
|
||||
s.mu.Lock()
|
||||
for _, ch := range s.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.logger.Info("msg", "TCP chain source stopped",
|
||||
"component", "tcp_chain_source",
|
||||
"instance_id", s.id)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (s *TCPChainSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
Type: "tcp_chain",
|
||||
TotalEntries: s.totalEntries.Load(),
|
||||
DroppedEntries: s.droppedEntries.Load(),
|
||||
StartTime: s.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{
|
||||
"host": s.config.Host,
|
||||
"port": s.config.Port,
|
||||
"active_connections": s.activeConns.Load(),
|
||||
"rejected_conns": s.rejectedConns.Load(),
|
||||
"parse_errors": s.parseErrors.Load(),
|
||||
"trust_node": s.config.TrustNode,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// acceptLoop accepts upstream connections until listener close
|
||||
func (s *TCPChainSource) acceptLoop() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
conn, err := s.listener.Accept()
|
||||
if err != nil {
|
||||
if errors.Is(err, net.ErrClosed) || s.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
s.logger.Warn("msg", "Accept error",
|
||||
"component", "tcp_chain_source",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if s.config.MaxConnections > 0 && s.activeConns.Load() >= s.config.MaxConnections {
|
||||
s.rejectedConns.Add(1)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.conns[conn] = struct{}{}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.wg.Add(1)
|
||||
go s.handleConn(conn)
|
||||
}
|
||||
}
|
||||
|
||||
// handleConn validates the hello preamble, then streams entries until EOF/error
|
||||
func (s *TCPChainSource) handleConn(conn net.Conn) {
|
||||
defer s.wg.Done()
|
||||
remote := conn.RemoteAddr().String()
|
||||
s.activeConns.Add(1)
|
||||
|
||||
var sessID string
|
||||
defer func() {
|
||||
conn.Close()
|
||||
s.mu.Lock()
|
||||
delete(s.conns, conn)
|
||||
s.mu.Unlock()
|
||||
if sessID != "" {
|
||||
s.proxy.RemoveSession(sessID)
|
||||
}
|
||||
s.activeConns.Add(-1)
|
||||
}()
|
||||
|
||||
scanner := bufio.NewScanner(conn)
|
||||
// Oversized line (> MaxLogEntryBytes) is a protocol violation; scanner is
|
||||
// unrecoverable after ErrTooLong, connection terminates
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
|
||||
|
||||
// Hello preamble
|
||||
conn.SetReadDeadline(time.Now().Add(time.Duration(s.config.HelloTimeoutMS) * time.Millisecond))
|
||||
if !scanner.Scan() {
|
||||
s.logger.Warn("msg", "Connection closed before hello",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"error", scanner.Err())
|
||||
return
|
||||
}
|
||||
hello, err := chain.DecodeHello(scanner.Bytes())
|
||||
if err != nil {
|
||||
s.logger.Warn("msg", "Rejected chain connection",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
|
||||
connNode := hello.Node
|
||||
if connNode == "" || !s.config.TrustNode {
|
||||
if host, _, splitErr := net.SplitHostPort(remote); splitErr == nil {
|
||||
connNode = host
|
||||
} else {
|
||||
connNode = remote
|
||||
}
|
||||
}
|
||||
|
||||
sess := s.proxy.CreateSession(remote, map[string]any{
|
||||
"type": "tcp_chain",
|
||||
"node": connNode,
|
||||
})
|
||||
sessID = sess.ID
|
||||
|
||||
s.logger.Info("msg", "Chain connection established",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"node", connNode)
|
||||
|
||||
idle := time.Duration(s.config.ReadTimeoutMS) * time.Millisecond
|
||||
for {
|
||||
if idle > 0 {
|
||||
conn.SetReadDeadline(time.Now().Add(idle))
|
||||
} else {
|
||||
conn.SetReadDeadline(time.Time{})
|
||||
}
|
||||
if !scanner.Scan() {
|
||||
if err := scanner.Err(); err != nil && !errors.Is(err, net.ErrClosed) {
|
||||
s.logger.Debug("msg", "Chain read terminated",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
line := scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
s.proxy.UpdateActivity(sessID)
|
||||
|
||||
entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode)
|
||||
if err != nil {
|
||||
s.parseErrors.Add(1)
|
||||
s.logger.Debug("msg", "Dropped malformed chain entry",
|
||||
"component", "tcp_chain_source",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
s.publish(entry)
|
||||
}
|
||||
}
|
||||
|
||||
// parseEntry decodes a canonical LogEntry line and applies the node policy
|
||||
func (s *TCPChainSource) parseEntry(line []byte, connNode string) (core.LogEntry, bool) {
|
||||
var entry core.LogEntry
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
s.parseErrors.Add(1)
|
||||
s.logger.Debug("msg", "Dropped malformed chain entry",
|
||||
"component", "tcp_chain_source",
|
||||
"error", err)
|
||||
return core.LogEntry{}, false
|
||||
}
|
||||
if entry.Time.IsZero() {
|
||||
entry.Time = time.Now()
|
||||
}
|
||||
if entry.Node == "" || !s.config.TrustNode {
|
||||
entry.Node = connNode
|
||||
}
|
||||
entry.RawSize = int64(len(line))
|
||||
return entry, true
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers
|
||||
func (s *TCPChainSource) publish(entry core.LogEntry) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
s.totalEntries.Add(1)
|
||||
s.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range s.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
s.droppedEntries.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package tokenbucket
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TokenBucket implements a thread-safe token bucket rate limiter
|
||||
type TokenBucket struct {
|
||||
capacity float64
|
||||
tokens float64
|
||||
refillRate float64 // tokens per second
|
||||
lastRefill time.Time
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// New creates a new token bucket with given capacity and refill rate
|
||||
func New(capacity float64, refillRate float64) *TokenBucket {
|
||||
return &TokenBucket{
|
||||
capacity: capacity,
|
||||
tokens: capacity, // Start full
|
||||
refillRate: refillRate,
|
||||
lastRefill: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// Allow attempts to consume one token, returns true if allowed
|
||||
func (tb *TokenBucket) Allow() bool {
|
||||
return tb.AllowN(1)
|
||||
}
|
||||
|
||||
// AllowN attempts to consume n tokens, returns true if allowed
|
||||
func (tb *TokenBucket) AllowN(n float64) bool {
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
|
||||
tb.refill()
|
||||
|
||||
if tb.tokens >= n {
|
||||
tb.tokens -= n
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Tokens returns the current number of available tokens
|
||||
func (tb *TokenBucket) Tokens() float64 {
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
|
||||
tb.refill()
|
||||
return tb.tokens
|
||||
}
|
||||
|
||||
// refill adds tokens based on time elapsed since last refill
|
||||
// MUST be called with mutex held
|
||||
func (tb *TokenBucket) refill() {
|
||||
now := time.Now()
|
||||
elapsed := now.Sub(tb.lastRefill).Seconds()
|
||||
|
||||
// Handle time sync issues causing negative elapsed time
|
||||
if elapsed < 0 {
|
||||
// Clock went backwards, reset to current time but don't add tokens
|
||||
tb.lastRefill = now
|
||||
elapsed = 0
|
||||
}
|
||||
|
||||
tb.tokens += elapsed * tb.refillRate
|
||||
if tb.tokens > tb.capacity {
|
||||
tb.tokens = tb.capacity
|
||||
}
|
||||
tb.lastRefill = now
|
||||
}
|
||||
|
||||
// Rate returns the refill rate in tokens per second
|
||||
func (tb *TokenBucket) Rate() float64 {
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
return tb.refillRate
|
||||
}
|
||||
|
||||
// Capacity returns the bucket capacity
|
||||
func (tb *TokenBucket) Capacity() float64 {
|
||||
tb.mu.Lock()
|
||||
defer tb.mu.Unlock()
|
||||
return tb.capacity
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package version
|
||||
|
||||
import "fmt"
|
||||
|
||||
var (
|
||||
// Version is the application version, set at compile time via -ldflags
|
||||
Version = "dev"
|
||||
// GitCommit is the git commit hash, set at compile time
|
||||
GitCommit = "unknown"
|
||||
// BuildTime is the application build time, set at compile time
|
||||
BuildTime = "unknown"
|
||||
)
|
||||
|
||||
// String returns a detailed, formatted version string including commit and build time
|
||||
func String() string {
|
||||
if Version == "dev" {
|
||||
return fmt.Sprintf("dev (commit: %s, built: %s)", GitCommit, BuildTime)
|
||||
}
|
||||
return fmt.Sprintf("%s (commit: %s, built: %s)", Version, GitCommit, BuildTime)
|
||||
}
|
||||
|
||||
// Short returns just the version tag
|
||||
func Short() string {
|
||||
return Version
|
||||
}
|
||||
Reference in New Issue
Block a user