v0.13.0 doc update, refactor, http/tcp chain source and sink added
This commit is contained in:
@@ -6,20 +6,25 @@ import (
|
||||
|
||||
_ "logwisp/src/internal/source/console"
|
||||
_ "logwisp/src/internal/source/file"
|
||||
_ "logwisp/src/internal/source/httpchain"
|
||||
_ "logwisp/src/internal/source/null"
|
||||
_ "logwisp/src/internal/source/random"
|
||||
_ "logwisp/src/internal/source/tcpchain"
|
||||
|
||||
_ "logwisp/src/internal/sink/console"
|
||||
_ "logwisp/src/internal/sink/file"
|
||||
_ "logwisp/src/internal/sink/http"
|
||||
_ "logwisp/src/internal/sink/httpchain"
|
||||
_ "logwisp/src/internal/sink/null"
|
||||
_ "logwisp/src/internal/sink/tcp"
|
||||
_ "logwisp/src/internal/sink/tcpchain"
|
||||
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/service"
|
||||
"logwisp/src/internal/version"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/lixenwraith/log/sanitizer"
|
||||
)
|
||||
|
||||
// bootstrapInitial handles initial service startup with status reporter
|
||||
@@ -132,6 +137,14 @@ func initializeLogger(cfg *config.Config) error {
|
||||
}
|
||||
logCfg.Level = levelValue
|
||||
|
||||
// Configure log format
|
||||
if cfg.Logging.Format != "" {
|
||||
logCfg.Format = cfg.Logging.Format
|
||||
}
|
||||
if cfg.Logging.Sanitization != "" {
|
||||
logCfg.Sanitization = sanitizer.PolicyPreset(cfg.Logging.Sanitization)
|
||||
}
|
||||
|
||||
// Configure based on output mode
|
||||
switch cfg.Logging.Output {
|
||||
case "none":
|
||||
@@ -176,4 +189,4 @@ func configureFileLogging(logCfg *log.Config, cfg *config.Config) {
|
||||
logCfg.RetentionPeriodHrs = cfg.Logging.File.RetentionHours
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"logwisp/src/internal/version"
|
||||
"os"
|
||||
)
|
||||
|
||||
// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths.
|
||||
const helpText = `LogWisp %s - log collection, processing, and distribution
|
||||
|
||||
Usage:
|
||||
logwisp [options]
|
||||
logwisp help | -h | --help
|
||||
logwisp --version
|
||||
|
||||
Any configuration key is settable as a flag using its TOML path:
|
||||
--<path>=<value> e.g. --logging.level=debug
|
||||
|
||||
Common options:
|
||||
-c, --config <path> Configuration file (default: ./logwisp.toml)
|
||||
--quiet Suppress console output
|
||||
--status_reporter=<bool> Periodic status logging (default: true)
|
||||
--auto_reload=<bool> Config hot reload on file change (default: false)
|
||||
|
||||
Logging:
|
||||
--logging.output=<mode> file|stdout|stderr|split|all|none
|
||||
--logging.level=<level> debug|info|warn|error
|
||||
--logging.file.directory=<path>
|
||||
--logging.console.target=<target> stdout|stderr|split
|
||||
|
||||
Pipelines (N = 0-based index):
|
||||
--pipelines.N.name=<name>
|
||||
--pipelines.N.plugin_sources.N.type=<type> file|console|random|null
|
||||
--pipelines.N.plugin_sinks.N.type=<type> console|file|http|tcp|null
|
||||
--pipelines.N.flow.filters.N.patterns='["ERROR","WARN"]'
|
||||
|
||||
Environment:
|
||||
LOGWISP_<PATH> Config path, '.' -> '_', uppercase
|
||||
e.g. LOGWISP_LOGGING_LEVEL=debug
|
||||
LOGWISP_CONFIG_FILE Configuration file path
|
||||
LOGWISP_CONFIG_DIR Configuration directory
|
||||
|
||||
Signals:
|
||||
SIGINT, SIGTERM Graceful shutdown
|
||||
SIGHUP, SIGUSR1 Reload configuration
|
||||
|
||||
Exit codes:
|
||||
0 success
|
||||
1 general error
|
||||
2 configuration file not found
|
||||
`
|
||||
|
||||
// handleHelp prints usage and exits if a help request is present in args
|
||||
func handleHelp(args []string) {
|
||||
if len(args) > 0 && args[0] == "help" {
|
||||
printHelp()
|
||||
}
|
||||
for _, arg := range args {
|
||||
if arg == "--" {
|
||||
break // end of flags
|
||||
}
|
||||
if arg == "-h" || arg == "--help" {
|
||||
printHelp()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// printHelp writes usage to stdout and exits with success
|
||||
func printHelp() {
|
||||
fmt.Printf(helpText, version.Short())
|
||||
os.Exit(0)
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/core"
|
||||
@@ -25,6 +24,10 @@ func main() {
|
||||
// Emulates nohup
|
||||
signal.Ignore(syscall.SIGHUP)
|
||||
|
||||
// Help handled before config parsing; loader has no help flag.
|
||||
// Also the future dispatch point for subcommands (tls, etc.)
|
||||
handleHelp(os.Args[1:])
|
||||
|
||||
// Load configuration with automatic CLI parsing
|
||||
cfg, err := config.Load(os.Args[1:])
|
||||
if err != nil {
|
||||
@@ -64,8 +67,6 @@ func main() {
|
||||
"status_reporter", cfg.StatusReporter,
|
||||
"auto_reload", cfg.ConfigAutoReload)
|
||||
|
||||
time.Sleep(time.Second)
|
||||
|
||||
// Create context for shutdown
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
@@ -152,4 +153,4 @@ func shutdownLogger() {
|
||||
Error("Logger shutdown error: %v\n", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package chain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"logwisp/src/internal/core"
|
||||
"math/rand/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProtocolVersion is declared in the hello preamble
|
||||
const ProtocolVersion = 1
|
||||
|
||||
// Hello is the first NDJSON line sent by the dialing side after connect.
|
||||
// Reserved for future revisions: auth credential, feature flags (ack, compression).
|
||||
type Hello struct {
|
||||
LogWisp int `json:"logwisp"`
|
||||
Node string `json:"node,omitempty"`
|
||||
// Auth string `json:"auth,omitempty"`
|
||||
// Features []string `json:"features,omitempty"`
|
||||
}
|
||||
|
||||
// HTTP transport mapping of the chain protocol.
|
||||
// Hello preamble equivalent: protocol + node carried as request headers.
|
||||
// Reserved extension point: Authorization header for auth, TLS at transport.
|
||||
const (
|
||||
HeaderProtocol = "X-Logwisp-Protocol"
|
||||
HeaderNode = "X-Logwisp-Node"
|
||||
HeaderAccepted = "X-Logwisp-Accepted"
|
||||
ContentTypeNDJSON = "application/x-ndjson"
|
||||
)
|
||||
|
||||
// EncodeHello serializes a newline-terminated hello preamble
|
||||
func EncodeHello(node string) ([]byte, error) {
|
||||
b, err := json.Marshal(Hello{LogWisp: ProtocolVersion, Node: node})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(b, '\n'), nil
|
||||
}
|
||||
|
||||
// DecodeHello parses and validates a hello preamble line
|
||||
func DecodeHello(line []byte) (Hello, error) {
|
||||
var h Hello
|
||||
if err := json.Unmarshal(line, &h); err != nil {
|
||||
return h, fmt.Errorf("malformed hello: %w", err)
|
||||
}
|
||||
if h.LogWisp != ProtocolVersion {
|
||||
return h, fmt.Errorf("unsupported protocol version: %d", h.LogWisp)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// DecodeEntry parses a canonical LogEntry line and applies the node trust policy
|
||||
func DecodeEntry(line []byte, connNode string, trustNode bool) (core.LogEntry, error) {
|
||||
var entry core.LogEntry
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
return core.LogEntry{}, err
|
||||
}
|
||||
if entry.Time.IsZero() {
|
||||
entry.Time = time.Now()
|
||||
}
|
||||
if entry.Node == "" || !trustNode {
|
||||
entry.Node = connNode
|
||||
}
|
||||
entry.RawSize = int64(len(line))
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
// EntryFromEvent extracts the structured entry, stamping node identity at
|
||||
// first hop. Second return is true when synthesized from a formatted payload.
|
||||
func EntryFromEvent(event core.TransportEvent, node, fallbackSource string) (core.LogEntry, bool) {
|
||||
entry := event.Entry
|
||||
synthesized := false
|
||||
if entry.Time.IsZero() {
|
||||
synthesized = true
|
||||
entry = core.LogEntry{
|
||||
Time: event.Time,
|
||||
Source: fallbackSource,
|
||||
Message: string(event.Payload),
|
||||
}
|
||||
}
|
||||
if entry.Node == "" {
|
||||
entry.Node = node
|
||||
}
|
||||
return entry, synthesized
|
||||
}
|
||||
|
||||
// BackoffDelay computes exponential backoff with ±20% jitter
|
||||
func BackoffDelay(minD, maxD time.Duration, failures int) time.Duration {
|
||||
d := maxD
|
||||
if failures < 63 {
|
||||
if v := minD << uint(failures-1); v > 0 && v < maxD {
|
||||
d = v
|
||||
}
|
||||
}
|
||||
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
|
||||
}
|
||||
@@ -205,6 +205,32 @@ type ConsoleSourceOptions struct {
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
}
|
||||
|
||||
// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting
|
||||
// NDJSON entries from upstream logwisp tcp_chain sinks
|
||||
type TCPChainSourceOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
|
||||
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // per-connection idle deadline, 0 = none
|
||||
HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
|
||||
// NDJSON batches from upstream logwisp http_chain sinks
|
||||
type HTTPChainSourceOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
IngestPath string `toml:"ingest_path"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
MaxBodyBytes int64 `toml:"max_body_bytes"` // per-request cap
|
||||
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
// --- Sink Options ---
|
||||
|
||||
// PluginSinkConfig represents a sink plugin instance configuration
|
||||
@@ -251,16 +277,49 @@ type TCPSinkOptions struct {
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
WriteTimeout int64 `toml:"write_timeout_ms"`
|
||||
KeepAlive bool `toml:"keep_alive"`
|
||||
KeepAlivePeriod int64 `toml:"keep_alive_period_ms"`
|
||||
KeepAlive bool `toml:"keep_alive"`
|
||||
}
|
||||
|
||||
// HTTPSinkOptions defines settings for an HTTP SSE server sink
|
||||
type HTTPSinkOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
StreamPath string `toml:"stream_path"`
|
||||
StatusPath string `toml:"status_path"`
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
WriteTimeout int64 `toml:"write_timeout_ms"`
|
||||
}
|
||||
}
|
||||
|
||||
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
|
||||
// entries to a downstream logwisp tcp_chain source
|
||||
type TCPChainSinkOptions struct {
|
||||
Node string `toml:"node"` // origin label, default: os.Hostname()
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
DialTimeoutMS int64 `toml:"dial_timeout_ms"`
|
||||
WriteTimeoutMS int64 `toml:"write_timeout_ms"`
|
||||
BackoffMinMS int64 `toml:"backoff_min_ms"`
|
||||
BackoffMaxMS int64 `toml:"backoff_max_ms"`
|
||||
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
|
||||
KeepAlive bool `toml:"keep_alive"`
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting
|
||||
// NDJSON batches to a downstream logwisp http_chain source
|
||||
type HTTPChainSinkOptions struct {
|
||||
Node string `toml:"node"` // origin label, default: os.Hostname()
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
IngestPath string `toml:"ingest_path"`
|
||||
BufferSize int64 `toml:"buffer_size"`
|
||||
MaxBatchCount int64 `toml:"max_batch_count"`
|
||||
MaxBatchBytes int64 `toml:"max_batch_bytes"`
|
||||
FlushIntervalMS int64 `toml:"flush_interval_ms"`
|
||||
RequestTimeoutMS int64 `toml:"request_timeout_ms"` // covers dial + write + response
|
||||
BackoffMinMS int64 `toml:"backoff_min_ms"`
|
||||
BackoffMaxMS int64 `toml:"backoff_max_ms"`
|
||||
// Future: TLS/auth options
|
||||
}
|
||||
|
||||
@@ -65,6 +65,12 @@ func Load(args []string) (*Config, error) {
|
||||
// Store the manager for hot reload
|
||||
configManager = cfg
|
||||
|
||||
// Surface typo'd flags (e.g. --status-reporter vs --status_reporter);
|
||||
// pre-logger phase, stderr only, suppressed in quiet mode
|
||||
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet {
|
||||
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
|
||||
}
|
||||
|
||||
// Start watcher if auto-reload is enabled
|
||||
if finalConfig.ConfigAutoReload {
|
||||
watchOpts := lconfig.WatchOptions{
|
||||
@@ -99,6 +105,7 @@ func defaults() *Config {
|
||||
Logging: &LogConfig{
|
||||
Output: "stdout",
|
||||
Level: "info",
|
||||
Format: "txt",
|
||||
File: &LogFileConfig{
|
||||
Directory: "./log",
|
||||
Name: "logwisp",
|
||||
@@ -195,4 +202,4 @@ func customEnvTransform(path string) string {
|
||||
env = strings.ToUpper(env)
|
||||
// env = "LOGWISP_" + env // already added by WithEnvPrefix
|
||||
return env
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,15 @@ func ValidateConfig(cfg *Config) error {
|
||||
return fmt.Errorf("no pipelines configured")
|
||||
}
|
||||
|
||||
// Reject duplicate pipeline names (service map is keyed by name)
|
||||
names := make(map[string]struct{}, len(cfg.Pipelines))
|
||||
for i, p := range cfg.Pipelines {
|
||||
if _, dup := names[p.Name]; dup {
|
||||
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
|
||||
}
|
||||
names[p.Name] = struct{}{}
|
||||
}
|
||||
|
||||
if err := validateLogConfig(cfg.Logging); err != nil {
|
||||
return fmt.Errorf("logging: %w", err)
|
||||
}
|
||||
@@ -52,6 +61,17 @@ func validateLogConfig(cfg *LogConfig) error {
|
||||
return fmt.Errorf("level: %w", err)
|
||||
}
|
||||
|
||||
if cfg.Format != "" {
|
||||
if err := lconfig.OneOf("raw", "txt", "json")(cfg.Format); err != nil {
|
||||
return fmt.Errorf("format: %w", err)
|
||||
}
|
||||
}
|
||||
if cfg.Sanitization != "" {
|
||||
if err := lconfig.OneOf("raw", "json", "txt", "shell")(cfg.Sanitization); err != nil {
|
||||
return fmt.Errorf("sanitization: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Console != nil {
|
||||
validateTarget := lconfig.OneOf("stdout", "stderr", "split")
|
||||
if err := validateTarget(cfg.Console.Target); err != nil {
|
||||
@@ -60,4 +80,4 @@ func validateLogConfig(cfg *LogConfig) error {
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,9 +5,10 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// Represents a single log record flowing through the pipeline
|
||||
// LogEntry represents a single log record flowing through the pipeline
|
||||
type LogEntry struct {
|
||||
Time time.Time `json:"time"`
|
||||
Node string `json:"node,omitempty"` // origin node identity for chained topologies; first hop stamps, relays preserve
|
||||
Source string `json:"source"`
|
||||
Level string `json:"level,omitempty"`
|
||||
Message string `json:"message"`
|
||||
@@ -20,4 +21,7 @@ type TransportEvent struct {
|
||||
Time time.Time
|
||||
// Formatted, serialized log payload
|
||||
Payload []byte
|
||||
}
|
||||
// Structured entry for re-serializing sinks (chain links). Zero Time => absent
|
||||
Entry LogEntry
|
||||
}
|
||||
|
||||
|
||||
@@ -119,6 +119,7 @@ func (f *Flow) Process(entry core.LogEntry) (core.TransportEvent, bool) {
|
||||
event := core.TransportEvent{
|
||||
Time: entry.Time,
|
||||
Payload: formatted,
|
||||
Entry: entry, // Carry structured entry so chain sinks are format-independent
|
||||
}
|
||||
|
||||
return event, true
|
||||
@@ -159,4 +160,5 @@ func (f *Flow) GetStats() map[string]any {
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -127,7 +128,8 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent
|
||||
// SSE comment format - bypass formatter for this special case
|
||||
if hg.config.IncludeStats {
|
||||
beatNum := hg.beatCount.Load()
|
||||
payload = []byte(": heartbeat " + t.Format(time.RFC3339) + " [#" + string(beatNum) + "]\n")
|
||||
payload = []byte(": heartbeat " + t.Format(time.RFC3339) +
|
||||
" [#" + strconv.FormatUint(beatNum, 10) + "]\n")
|
||||
} else {
|
||||
payload = []byte(": heartbeat " + t.Format(time.RFC3339) + "\n")
|
||||
}
|
||||
@@ -159,10 +161,11 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent
|
||||
return core.TransportEvent{
|
||||
Time: t,
|
||||
Payload: payload,
|
||||
Entry: entry, // heartbeats traverse chain links as structured entries
|
||||
}
|
||||
}
|
||||
|
||||
// IntervalMS returns the heartbeat interval in milliseconds
|
||||
func (hg *HeartbeatGenerator) IntervalMS() int64 {
|
||||
return hg.config.IntervalMS
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +89,8 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
|
||||
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
|
||||
// Map logwisp LogEntry to formatter args
|
||||
level := mapLevel(entry.Level)
|
||||
// syslog-style origin prefix for chained entries
|
||||
src := sourceLabel(entry)
|
||||
|
||||
// Build args based on whether we have structured fields
|
||||
var args []any
|
||||
@@ -101,18 +103,19 @@ func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
|
||||
args = []any{entry.Message, fields}
|
||||
// Add structured flag to properly format fields as JSON object
|
||||
effectiveFlags := a.flags | formatter.FlagStructuredJSON
|
||||
return a.formatter.Format(effectiveFlags, entry.Time, level, entry.Source, args), nil
|
||||
return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Simple message without fields
|
||||
args = []any{entry.Message}
|
||||
return a.formatter.Format(a.flags, entry.Time, level, entry.Source, args), nil
|
||||
return a.formatter.Format(a.flags, entry.Time, level, src, args), nil
|
||||
}
|
||||
|
||||
// FormatWithFlags allows custom flags for specific formatting needs
|
||||
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
|
||||
level := mapLevel(entry.Level)
|
||||
src := sourceLabel(entry)
|
||||
|
||||
var args []any
|
||||
if len(entry.Fields) > 0 {
|
||||
@@ -127,7 +130,7 @@ func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int6
|
||||
args = []any{entry.Message}
|
||||
}
|
||||
|
||||
return a.formatter.Format(customFlags, entry.Time, level, entry.Source, args), nil
|
||||
return a.formatter.Format(customFlags, entry.Time, level, src, args), nil
|
||||
}
|
||||
|
||||
// Name returns formatter type
|
||||
@@ -149,4 +152,13 @@ func mapLevel(level string) int64 {
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// sourceLabel prefixes origin node onto source (syslog HOSTNAME + TAG convention)
|
||||
func sourceLabel(entry core.LogEntry) string {
|
||||
if entry.Node == "" {
|
||||
return entry.Source
|
||||
}
|
||||
return entry.Node + "/" + entry.Source
|
||||
}
|
||||
|
||||
|
||||
@@ -41,13 +41,11 @@ type Pipeline struct {
|
||||
|
||||
// PipelineStats contains runtime statistics for a pipeline
|
||||
type PipelineStats struct {
|
||||
StartTime time.Time
|
||||
TotalEntriesProcessed atomic.Uint64
|
||||
TotalEntriesDroppedByRateLimit atomic.Uint64
|
||||
TotalEntriesFiltered atomic.Uint64
|
||||
SourceStats []source.SourceStats
|
||||
SinkStats []sink.SinkStats
|
||||
FlowStats map[string]any
|
||||
StartTime time.Time
|
||||
TotalEntriesDroppedBySink atomic.Uint64
|
||||
SourceStats []source.SourceStats
|
||||
SinkStats []sink.SinkStats
|
||||
FlowStats map[string]any
|
||||
}
|
||||
|
||||
// NewPipeline creates a new pipeline with registry support
|
||||
@@ -74,7 +72,6 @@ func NewPipeline(
|
||||
cancel: pipelineCancel,
|
||||
}
|
||||
|
||||
// Create flow processor
|
||||
// Create flow processor
|
||||
flowProcessor, err := flow.NewFlow(cfg.Flow, logger)
|
||||
if err != nil {
|
||||
@@ -177,7 +174,7 @@ func (p *Pipeline) initSourceCapabilities(s source.Source, cfg config.PluginSour
|
||||
|
||||
// initSinkCapabilities checks and injects optional capabilities
|
||||
func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig) error {
|
||||
// Initiate and activate source capabilities
|
||||
// Initiate and activate sink capabilities
|
||||
for _, c := range s.Capabilities() {
|
||||
switch c {
|
||||
// Network capabilities
|
||||
@@ -203,48 +200,36 @@ func (p *Pipeline) run() {
|
||||
defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name)
|
||||
|
||||
var componentWg sync.WaitGroup
|
||||
|
||||
// Start a goroutine for each source to fan-in data
|
||||
for _, src := range p.Sources {
|
||||
componentWg.Add(1)
|
||||
go func(s source.Source) {
|
||||
defer componentWg.Done()
|
||||
ch := s.Subscribe()
|
||||
for {
|
||||
select {
|
||||
case entry, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Process and distribute the log entry
|
||||
if event, passed := p.Flow.Process(entry); passed {
|
||||
// Fan-out to all sinks
|
||||
for _, snk := range p.Sinks {
|
||||
snk.Input() <- event
|
||||
}
|
||||
}
|
||||
case <-p.ctx.Done():
|
||||
return
|
||||
// Range allows in-flight data to drain cleanly once Source.Stop() closes the channel
|
||||
for entry := range ch {
|
||||
if event, passed := p.Flow.Process(entry); passed {
|
||||
// Use non-blocking dispatcher
|
||||
p.dispatch(event)
|
||||
}
|
||||
}
|
||||
}(src)
|
||||
}
|
||||
|
||||
var hbWg sync.WaitGroup
|
||||
// Start heartbeat generator if enabled
|
||||
if heartbeatCh := p.Flow.StartHeartbeat(p.ctx); heartbeatCh != nil {
|
||||
componentWg.Add(1)
|
||||
hbWg.Add(1)
|
||||
go func() {
|
||||
defer componentWg.Done()
|
||||
defer hbWg.Done()
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-heartbeatCh:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Fan-out heartbeat to all sinks
|
||||
for _, snk := range p.Sinks {
|
||||
snk.Input() <- event
|
||||
}
|
||||
// Use non-blocking dispatcher
|
||||
p.dispatch(event)
|
||||
case <-p.ctx.Done():
|
||||
return
|
||||
}
|
||||
@@ -253,6 +238,23 @@ func (p *Pipeline) run() {
|
||||
}
|
||||
|
||||
componentWg.Wait()
|
||||
|
||||
// Terminate internal contexts (heartbeat) once flow is complete
|
||||
p.cancel()
|
||||
hbWg.Wait()
|
||||
}
|
||||
|
||||
// dispatch performs a non-blocking send to all sinks.
|
||||
// A full/stalled sink must never block the run loop or starve sibling sinks.
|
||||
func (p *Pipeline) dispatch(event core.TransportEvent) {
|
||||
for _, snk := range p.Sinks {
|
||||
select {
|
||||
case snk.Input() <- event:
|
||||
default:
|
||||
// Buffer full - drop to avoid deadlocking the pipeline
|
||||
p.Stats.TotalEntriesDroppedBySink.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the pipeline operation and all its components including flow, sources, and sinks
|
||||
@@ -294,10 +296,7 @@ func (p *Pipeline) Stop() error {
|
||||
|
||||
p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name)
|
||||
|
||||
// Signal all components and the run loop to stop
|
||||
p.cancel()
|
||||
|
||||
// Stop all sources concurrently to halt new data ingress
|
||||
// 1. Stop all sources concurrently to halt new data ingress and close their channels
|
||||
var sourceWg sync.WaitGroup
|
||||
for _, src := range p.Sources {
|
||||
sourceWg.Add(1)
|
||||
@@ -308,10 +307,11 @@ func (p *Pipeline) Stop() error {
|
||||
}
|
||||
sourceWg.Wait()
|
||||
|
||||
// Wait for the run loop to finish processing and sending all in-flight data
|
||||
// 2. Wait for the run loop to finish processing and sending all in-flight data
|
||||
// run() inherently calls p.cancel() when the source channels are empty
|
||||
p.wg.Wait()
|
||||
|
||||
// Stop all sinks concurrently now that no new data will be sent
|
||||
// 3. Stop all sinks concurrently now that no new data will be sent
|
||||
var sinkWg sync.WaitGroup
|
||||
for _, s := range p.Sinks {
|
||||
sinkWg.Add(1)
|
||||
@@ -361,92 +361,82 @@ func (p *Pipeline) GetStats() map[string]any {
|
||||
}
|
||||
}()
|
||||
|
||||
// Collect source stats
|
||||
sourceStats := make([]map[string]any, 0, len(p.Sources))
|
||||
// 1. Live collect source stats
|
||||
sources := make([]map[string]any, 0, len(p.Sources))
|
||||
for _, src := range p.Sources {
|
||||
if src == nil {
|
||||
continue // Skip nil sources
|
||||
continue
|
||||
}
|
||||
|
||||
stats := src.GetStats()
|
||||
sourceStats = append(sourceStats, map[string]any{
|
||||
"id": stats.ID,
|
||||
"type": stats.Type,
|
||||
"total_entries": stats.TotalEntries,
|
||||
"dropped_entries": stats.DroppedEntries,
|
||||
"start_time": stats.StartTime,
|
||||
"last_entry_time": stats.LastEntryTime,
|
||||
"details": stats.Details,
|
||||
s := src.GetStats()
|
||||
sources = append(sources, map[string]any{
|
||||
"id": s.ID,
|
||||
"type": s.Type,
|
||||
"total_entries": s.TotalEntries,
|
||||
"dropped_entries": s.DroppedEntries,
|
||||
"start_time": s.StartTime,
|
||||
"last_entry_time": s.LastEntryTime,
|
||||
"details": s.Details,
|
||||
})
|
||||
}
|
||||
|
||||
// Collect sink stats
|
||||
sinkStats := make([]map[string]any, 0, len(p.Sinks))
|
||||
for _, s := range p.Sinks {
|
||||
if s == nil {
|
||||
continue // Skip nil sinks
|
||||
// 2. Live collect sink stats
|
||||
sinks := make([]map[string]any, 0, len(p.Sinks))
|
||||
for _, snk := range p.Sinks {
|
||||
if snk == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
stats := s.GetStats()
|
||||
sinkStats = append(sinkStats, map[string]any{
|
||||
"id": stats.ID,
|
||||
"type": stats.Type,
|
||||
"total_processed": stats.TotalProcessed,
|
||||
"active_connections": stats.ActiveConnections,
|
||||
"start_time": stats.StartTime,
|
||||
"last_processed": stats.LastProcessed,
|
||||
"details": stats.Details,
|
||||
s := snk.GetStats()
|
||||
sinks = append(sinks, map[string]any{
|
||||
"id": s.ID,
|
||||
"type": s.Type,
|
||||
"total_processed": s.TotalProcessed,
|
||||
"active_connections": s.ActiveConnections,
|
||||
"start_time": s.StartTime,
|
||||
"last_processed": s.LastProcessed,
|
||||
"details": s.Details,
|
||||
})
|
||||
}
|
||||
|
||||
// Get flow stats
|
||||
// 3. Collect flow stats and calculate filtered total
|
||||
var flowStats map[string]any
|
||||
var totalFiltered uint64
|
||||
var totalProcessed uint64
|
||||
|
||||
if p.Flow != nil {
|
||||
flowStats = p.Flow.GetStats()
|
||||
// Extract total_filtered from flow for top-level visibility
|
||||
|
||||
// Map the top-level processed counter directly from Flow's source of truth
|
||||
if tp, ok := flowStats["total_processed"].(uint64); ok {
|
||||
totalProcessed = tp
|
||||
}
|
||||
|
||||
// Calculate total dropped specifically by the filter chain
|
||||
if filters, ok := flowStats["filters"].(map[string]any); ok {
|
||||
if totalPassed, ok := filters["total_passed"].(uint64); ok {
|
||||
if totalProcessed, ok := filters["total_processed"].(uint64); ok {
|
||||
totalFiltered = totalProcessed - totalPassed
|
||||
if tProc, ok := filters["total_processed"].(uint64); ok {
|
||||
totalFiltered = tProc - totalPassed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Calculate Uptime
|
||||
var uptime int
|
||||
if p.running.Load() && !p.Stats.StartTime.IsZero() {
|
||||
uptime = int(time.Since(p.Stats.StartTime).Seconds())
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"name": p.Config.Name,
|
||||
"running": p.running.Load(),
|
||||
"uptime_seconds": uptime,
|
||||
"total_processed": p.Stats.TotalEntriesProcessed.Load(),
|
||||
"total_filtered": totalFiltered,
|
||||
"source_count": len(p.Sources),
|
||||
"sources": sourceStats,
|
||||
"sink_count": len(p.Sinks),
|
||||
"sinks": sinkStats,
|
||||
"flow": flowStats,
|
||||
"name": p.Config.Name,
|
||||
"running": p.running.Load(),
|
||||
"uptime_seconds": uptime,
|
||||
"total_processed": totalProcessed,
|
||||
"total_filtered": totalFiltered,
|
||||
"total_dropped_by_sink": p.Stats.TotalEntriesDroppedBySink.Load(),
|
||||
"source_count": len(p.Sources),
|
||||
"sources": sources,
|
||||
"sink_count": len(p.Sinks),
|
||||
"sinks": sinks,
|
||||
"flow": flowStats,
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: incomplete implementation
|
||||
// startStatsUpdater runs a periodic stats updater
|
||||
func (p *Pipeline) startStatsUpdater(ctx context.Context) {
|
||||
go func() {
|
||||
ticker := time.NewTicker(core.ServiceStatsUpdateInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
// Periodic stats updates if needed
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -34,16 +34,6 @@ type PluginMetadata struct {
|
||||
MaxInstances int // 0 = unlimited, 1 = single instance only
|
||||
}
|
||||
|
||||
// // global variables holding available source and sink plugins
|
||||
// var (
|
||||
// sourceFactories map[string]SourceFactory
|
||||
// sinkFactories map[string]SinkFactory
|
||||
// sourceMetadata map[string]*PluginMetadata
|
||||
// sinkMetadata map[string]*PluginMetadata
|
||||
// mu sync.RWMutex
|
||||
// // once sync.Once
|
||||
// )
|
||||
|
||||
// registry encapsulates all plugin factories with lazy initialization
|
||||
type registry struct {
|
||||
sourceFactories map[string]SourceFactory
|
||||
@@ -71,11 +61,6 @@ func getRegistry() *registry {
|
||||
return globalRegistry
|
||||
}
|
||||
|
||||
// func init() {
|
||||
// sourceFactories = make(map[string]SourceFactory)
|
||||
// sinkFactories = make(map[string]SinkFactory)
|
||||
// }
|
||||
|
||||
// RegisterSource registers a source factory function
|
||||
func RegisterSource(name string, constructor SourceFactory) error {
|
||||
r := getRegistry()
|
||||
|
||||
@@ -170,16 +170,16 @@ func (h *HTTPSink) Start(ctx context.Context) error {
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", h.config.Host, h.config.Port)
|
||||
|
||||
errChan := make(chan error, 1)
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http sink bind %s: %w", addr, err)
|
||||
}
|
||||
go func() {
|
||||
h.logger.Info("msg", "HTTP server starting",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"address", addr)
|
||||
|
||||
err := h.server.ListenAndServe(addr)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
if err := h.server.Serve(ln); err != nil {
|
||||
h.logger.Error("msg", "HTTP server terminated",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -193,18 +193,12 @@ func (h *HTTPSink) Start(ctx context.Context) error {
|
||||
}
|
||||
}()
|
||||
|
||||
// Check if server started
|
||||
select {
|
||||
case err := <-errChan:
|
||||
return err
|
||||
case <-time.After(HttpServerStartTimeout):
|
||||
h.logger.Info("msg", "HTTP server started",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"host", h.config.Host,
|
||||
"port", h.config.Port)
|
||||
return nil
|
||||
}
|
||||
h.logger.Info("msg", "HTTP server started",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"host", h.config.Host,
|
||||
"port", h.config.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the HTTP server and all client connections
|
||||
@@ -356,6 +350,8 @@ func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
remoteAddr := ctx.RemoteAddr()
|
||||
if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok {
|
||||
if tcpAddr.IP.To4() == nil {
|
||||
h.logger.Debug("msg", "IPv6 connection rejected",
|
||||
"component", "http_sink", "remote_addr", remoteAddr.String())
|
||||
ctx.SetConnectionClose()
|
||||
return
|
||||
}
|
||||
@@ -414,8 +410,6 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
|
||||
"client_id", clientID,
|
||||
"active_clients", connectCount)
|
||||
|
||||
h.wg.Add(1)
|
||||
|
||||
defer func() {
|
||||
disconnectCount := h.activeClients.Add(-1)
|
||||
h.logger.Debug("msg", "HTTP client disconnected",
|
||||
@@ -431,7 +425,6 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
|
||||
}
|
||||
|
||||
h.proxy.RemoveSession(sess.ID)
|
||||
h.wg.Done()
|
||||
}()
|
||||
|
||||
// Send connected event with metadata
|
||||
@@ -549,4 +542,4 @@ func splitLines(data []byte) [][]byte {
|
||||
return [][]byte{data}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/chain"
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/core"
|
||||
"logwisp/src/internal/plugin"
|
||||
"logwisp/src/internal/session"
|
||||
"logwisp/src/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("http_chain", NewHTTPChainSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http_chain sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultHTTPChainSinkBufferSize = 1000
|
||||
DefaultHTTPChainSinkIngestPath = "/ingest"
|
||||
DefaultHTTPChainSinkMaxBatchCount = 100
|
||||
DefaultHTTPChainSinkMaxBatchBytes = 1024 * 1024
|
||||
DefaultHTTPChainSinkFlushIntervalMS = 1000
|
||||
DefaultHTTPChainSinkRequestTimeoutMS = 10000
|
||||
DefaultHTTPChainSinkBackoffMinMS = 500
|
||||
DefaultHTTPChainSinkBackoffMaxMS = 30000
|
||||
)
|
||||
|
||||
// HTTPChainSink batches structured entries and posts NDJSON to a downstream
|
||||
// http_chain source. Delivery is at-least-once per batch.
|
||||
type HTTPChainSink struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
config *config.HTTPChainSinkOptions
|
||||
|
||||
node string
|
||||
url string
|
||||
|
||||
client *http.Client
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Batch state owned exclusively by run loop goroutine
|
||||
batch bytes.Buffer
|
||||
batchCount int64
|
||||
|
||||
reqTimeout time.Duration
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
totalProcessed atomic.Uint64
|
||||
batchesSent atomic.Uint64
|
||||
requestErrors atomic.Uint64
|
||||
droppedBatches atomic.Uint64
|
||||
synthesized atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewHTTPChainSinkPlugin creates an http_chain sink through plugin factory
|
||||
func NewHTTPChainSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.HTTPChainSinkOptions{}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Host); err != nil {
|
||||
return nil, fmt.Errorf("host: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.IngestPath == "" {
|
||||
opts.IngestPath = DefaultHTTPChainSinkIngestPath
|
||||
} else if !strings.HasPrefix(opts.IngestPath, "/") {
|
||||
return nil, fmt.Errorf("ingest_path: must start with '/'")
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPChainSinkBufferSize
|
||||
}
|
||||
if opts.MaxBatchCount <= 0 {
|
||||
opts.MaxBatchCount = DefaultHTTPChainSinkMaxBatchCount
|
||||
}
|
||||
if opts.MaxBatchBytes <= 0 {
|
||||
opts.MaxBatchBytes = DefaultHTTPChainSinkMaxBatchBytes
|
||||
}
|
||||
if opts.FlushIntervalMS <= 0 {
|
||||
opts.FlushIntervalMS = DefaultHTTPChainSinkFlushIntervalMS
|
||||
}
|
||||
if opts.RequestTimeoutMS <= 0 {
|
||||
opts.RequestTimeoutMS = DefaultHTTPChainSinkRequestTimeoutMS
|
||||
}
|
||||
if opts.BackoffMinMS <= 0 {
|
||||
opts.BackoffMinMS = DefaultHTTPChainSinkBackoffMinMS
|
||||
}
|
||||
if opts.BackoffMaxMS < opts.BackoffMinMS {
|
||||
opts.BackoffMaxMS = DefaultHTTPChainSinkBackoffMaxMS
|
||||
}
|
||||
|
||||
node := opts.Node
|
||||
if node == "" {
|
||||
if hn, err := os.Hostname(); err == nil {
|
||||
node = hn
|
||||
} else {
|
||||
node = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
|
||||
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
// IPv4-only, aligns with tcp/http sinks
|
||||
d := net.Dialer{}
|
||||
return d.DialContext(ctx, "tcp4", address)
|
||||
},
|
||||
MaxIdleConnsPerHost: 2,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: true,
|
||||
// Future: TLSClientConfig; HTTP/2 via ALPN once TLS lands
|
||||
}
|
||||
|
||||
t := &HTTPChainSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
node: node,
|
||||
// Future: "https" scheme with TLS
|
||||
url: "http://" + addr + opts.IngestPath,
|
||||
client: &http.Client{Transport: transport},
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
reqTimeout: time.Duration(opts.RequestTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
t.session = proxy.CreateSession(
|
||||
"http_chain://"+addr,
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "http_chain",
|
||||
"target": t.url,
|
||||
"node": node,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Info("msg", "HTTP chain sink initialized",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", id,
|
||||
"target", t.url,
|
||||
"node", node)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *HTTPChainSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *HTTPChainSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start launches the batching loop; downstream availability is not required
|
||||
func (t *HTTPChainSink) Start(ctx context.Context) error {
|
||||
t.startTime = time.Now()
|
||||
t.wg.Add(1)
|
||||
go t.runLoop(ctx)
|
||||
|
||||
t.logger.Info("msg", "HTTP chain sink started",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"target", t.url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the loop. Worst case: one in-flight request timeout plus
|
||||
// one final-flush request timeout.
|
||||
func (t *HTTPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping HTTP chain sink",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
t.client.CloseIdleConnections()
|
||||
|
||||
if t.session != nil {
|
||||
t.proxy.RemoveSession(t.session.ID)
|
||||
}
|
||||
|
||||
t.logger.Info("msg", "HTTP chain sink stopped",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *HTTPChainSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "http_chain",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": t.url,
|
||||
"node": t.node,
|
||||
"batches_sent": t.batchesSent.Load(),
|
||||
"request_errors": t.requestErrors.Load(),
|
||||
"dropped_batches": t.droppedBatches.Load(),
|
||||
"synthesized": t.synthesized.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runLoop batches events and flushes on size or interval
|
||||
func (t *HTTPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
|
||||
// Fold done channel into a context for request/backoff interruption
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-runCtx.Done():
|
||||
t.finalFlush()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if t.batchCount > 0 && !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
t.append(event)
|
||||
if t.batchCount >= t.config.MaxBatchCount ||
|
||||
int64(t.batch.Len()) >= t.config.MaxBatchBytes {
|
||||
if !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// append serializes one event into the pending batch
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop entry
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "http_chain_sink",
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batch.Write(line)
|
||||
t.batch.WriteByte('\n')
|
||||
t.batchCount++
|
||||
}
|
||||
|
||||
// flush delivers the pending batch, retrying transient failures with backoff.
|
||||
// Returns false when shutdown interrupts delivery; undelivered batch is dropped
|
||||
// by finalFlush semantics (batch already consumed here).
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
body := bytes.Clone(t.batch.Bytes())
|
||||
count := t.batchCount
|
||||
t.batch.Reset()
|
||||
t.batchCount = 0
|
||||
|
||||
failures := 0
|
||||
for {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
}
|
||||
transient, err := t.post(ctx, body)
|
||||
if err == nil {
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
return true
|
||||
}
|
||||
t.requestErrors.Add(1)
|
||||
if !transient {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Error("msg", "Chain batch rejected, dropping",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return true
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
}
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain batch delivery failed",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
|
||||
func (t *HTTPChainSink) finalFlush() {
|
||||
if t.batchCount == 0 {
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
count := t.batchCount
|
||||
if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Warn("msg", "Final chain batch dropped on shutdown",
|
||||
"component", "http_chain_sink",
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
}
|
||||
|
||||
// post sends one NDJSON batch; transient=true marks retryable failures
|
||||
func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool, err error) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, t.url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("Content-Type", chain.ContentTypeNDJSON)
|
||||
req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion))
|
||||
req.Header.Set(chain.HeaderNode, t.node)
|
||||
// Future: Authorization header for auth
|
||||
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Drain for connection reuse
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return false, nil
|
||||
case resp.StatusCode == http.StatusRequestTimeout,
|
||||
resp.StatusCode == http.StatusTooManyRequests,
|
||||
resp.StatusCode >= 500:
|
||||
return true, fmt.Errorf("status %s", resp.Status)
|
||||
default:
|
||||
return false, fmt.Errorf("status %s", resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// waitBackoff sleeps for the computed delay, interruptible by shutdown
|
||||
func (t *HTTPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ type TCPSink struct {
|
||||
server *tcpServer
|
||||
engine *gnet.Engine
|
||||
engineMu sync.Mutex
|
||||
booted chan struct{}
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
@@ -63,7 +64,7 @@ type TCPSink struct {
|
||||
|
||||
const (
|
||||
// Server lifecycle
|
||||
TCPServerStartTimeout = 100 * time.Millisecond
|
||||
TCPServerStartTimeout = 2 * time.Second
|
||||
TCPServerShutdownTimeout = 2 * time.Second
|
||||
|
||||
// Connection management
|
||||
@@ -151,6 +152,8 @@ func (t *TCPSink) Start(ctx context.Context) error {
|
||||
sink: t,
|
||||
clients: make(map[gnet.Conn]*tcpClient),
|
||||
}
|
||||
// Fresh channel per Start
|
||||
t.booted = make(chan struct{})
|
||||
|
||||
t.startTime = time.Now()
|
||||
|
||||
@@ -213,12 +216,25 @@ func (t *TCPSink) Start(ctx context.Context) error {
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
return err
|
||||
case <-time.After(TCPServerStartTimeout):
|
||||
// Bind confirmation via OnBoot
|
||||
case <-t.booted:
|
||||
t.logger.Info("msg", "TCP server started",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id,
|
||||
"port", t.config.Port)
|
||||
return nil
|
||||
// Timeout failure
|
||||
case <-time.After(TCPServerStartTimeout):
|
||||
t.engineMu.Lock()
|
||||
if t.engine != nil {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout)
|
||||
(*t.engine).Stop(stopCtx)
|
||||
cancel()
|
||||
}
|
||||
t.engineMu.Unlock()
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
return fmt.Errorf("tcp sink start timeout on %s", addr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -309,6 +325,9 @@ func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
|
||||
s.sink.engine = &eng
|
||||
s.sink.engineMu.Unlock()
|
||||
|
||||
// Listener is bound at this point; unblock Start
|
||||
close(s.sink.booted)
|
||||
|
||||
s.sink.logger.Debug("msg", "TCP server booted",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", s.sink.id)
|
||||
@@ -409,8 +428,10 @@ func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
|
||||
s.sink.proxy.UpdateActivity(client.sessionID)
|
||||
}
|
||||
|
||||
// TCP sink doesn't expect data from clients, discard
|
||||
c.Discard(-1)
|
||||
// TCP sink doesn't expect data from clients, discard safely
|
||||
if bufLen := c.InboundBuffered(); bufLen > 0 {
|
||||
c.Next(bufLen)
|
||||
}
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
@@ -469,4 +490,4 @@ func (t *TCPSink) handleWriteError(c gnet.Conn, err error) {
|
||||
delete(t.consecutiveWriteErrors, c)
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
package tcpchain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/chain"
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/core"
|
||||
"logwisp/src/internal/plugin"
|
||||
"logwisp/src/internal/session"
|
||||
"logwisp/src/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("tcp_chain", NewTCPChainSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp_chain sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultChainSinkBufferSize = 1000
|
||||
DefaultChainSinkDialTimeoutMS = 5000
|
||||
DefaultChainSinkWriteTimeoutMS = 5000
|
||||
DefaultChainSinkBackoffMinMS = 500
|
||||
DefaultChainSinkBackoffMaxMS = 30000
|
||||
DefaultChainSinkKeepAlivePeriodMS = 30000
|
||||
)
|
||||
|
||||
// TCPChainSink forwards structured entries to a downstream tcp_chain source
|
||||
type TCPChainSink struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
config *config.TCPChainSinkOptions
|
||||
|
||||
node string
|
||||
addr string
|
||||
helloLine []byte
|
||||
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// conn owned exclusively by run loop goroutine
|
||||
conn net.Conn
|
||||
everConnected bool
|
||||
dialTimeout time.Duration
|
||||
writeTimeout time.Duration
|
||||
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
totalProcessed atomic.Uint64
|
||||
writeErrors atomic.Uint64
|
||||
reconnects atomic.Uint64
|
||||
synthesized atomic.Uint64
|
||||
connected atomic.Bool
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewTCPChainSinkPlugin creates a tcp_chain sink through plugin factory
|
||||
func NewTCPChainSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.TCPChainSinkOptions{
|
||||
KeepAlive: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Host); err != nil {
|
||||
return nil, fmt.Errorf("host: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultChainSinkBufferSize
|
||||
}
|
||||
if opts.DialTimeoutMS <= 0 {
|
||||
opts.DialTimeoutMS = DefaultChainSinkDialTimeoutMS
|
||||
}
|
||||
if opts.WriteTimeoutMS <= 0 {
|
||||
opts.WriteTimeoutMS = DefaultChainSinkWriteTimeoutMS
|
||||
}
|
||||
if opts.BackoffMinMS <= 0 {
|
||||
opts.BackoffMinMS = DefaultChainSinkBackoffMinMS
|
||||
}
|
||||
if opts.BackoffMaxMS < opts.BackoffMinMS {
|
||||
opts.BackoffMaxMS = DefaultChainSinkBackoffMaxMS
|
||||
}
|
||||
if opts.KeepAlivePeriodMS <= 0 {
|
||||
opts.KeepAlivePeriodMS = DefaultChainSinkKeepAlivePeriodMS
|
||||
}
|
||||
|
||||
node := opts.Node
|
||||
if node == "" {
|
||||
if hn, err := os.Hostname(); err == nil {
|
||||
node = hn
|
||||
} else {
|
||||
node = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
helloLine, err := chain.EncodeHello(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hello: %w", err)
|
||||
}
|
||||
|
||||
t := &TCPChainSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
node: node,
|
||||
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
|
||||
helloLine: helloLine,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
dialTimeout: time.Duration(opts.DialTimeoutMS) * time.Millisecond,
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
t.session = proxy.CreateSession(
|
||||
"tcp_chain://"+t.addr,
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "tcp_chain",
|
||||
"target": t.addr,
|
||||
"node": node,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Info("msg", "TCP chain sink initialized",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", id,
|
||||
"target", t.addr,
|
||||
"node", node)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *TCPChainSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *TCPChainSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start launches the forwarding loop; connection is established lazily so
|
||||
// pipeline start does not depend on downstream availability
|
||||
func (t *TCPChainSink) Start(ctx context.Context) error {
|
||||
t.startTime = time.Now()
|
||||
t.wg.Add(1)
|
||||
go t.runLoop(ctx)
|
||||
|
||||
t.logger.Info("msg", "TCP chain sink started",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"target", t.addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the forwarding loop. Worst-case latency: one write timeout
|
||||
// plus one backoff wait (both interruptible or bounded).
|
||||
func (t *TCPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP chain sink",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
|
||||
if t.session != nil {
|
||||
t.proxy.RemoveSession(t.session.ID)
|
||||
}
|
||||
|
||||
t.logger.Info("msg", "TCP chain sink stopped",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *TCPChainSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
var active int64
|
||||
if t.connected.Load() {
|
||||
active = 1
|
||||
}
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "tcp_chain",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: active,
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": t.addr,
|
||||
"node": t.node,
|
||||
"connected": t.connected.Load(),
|
||||
"reconnects": t.reconnects.Load(),
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
"synthesized": t.synthesized.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runLoop consumes transport events and forwards them downstream
|
||||
func (t *TCPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
defer t.closeConn()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "tcp_chain_sink",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
if !t.deliver(ctx, append(line, '\n')) {
|
||||
return // shutdown during retry
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toEntry extracts the structured entry, stamping node identity at first hop
|
||||
func (t *TCPChainSink) toEntry(event core.TransportEvent) core.LogEntry {
|
||||
entry := event.Entry
|
||||
if entry.Time.IsZero() {
|
||||
// Defensive: event without structured entry, wrap formatted payload
|
||||
t.synthesized.Add(1)
|
||||
entry = core.LogEntry{
|
||||
Time: event.Time,
|
||||
Source: t.id,
|
||||
Message: string(event.Payload),
|
||||
}
|
||||
}
|
||||
if entry.Node == "" {
|
||||
entry.Node = t.node
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
// deliver writes one line, holding it across reconnects until sent or shutdown.
|
||||
// Backpressure during outage propagates to the pipeline dispatch drop counter.
|
||||
func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
|
||||
failures := 0
|
||||
for {
|
||||
if t.conn == nil {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
return false
|
||||
}
|
||||
if err := t.connect(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
failures++
|
||||
t.logger.Debug("msg", "Chain connect failed",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
t.conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
if _, err := t.conn.Write(line); err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain write failed",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"error", err)
|
||||
t.closeConn()
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// connect performs a single dial + hello attempt
|
||||
func (t *TCPChainSink) connect(ctx context.Context) error {
|
||||
d := net.Dialer{Timeout: t.dialTimeout}
|
||||
if t.config.KeepAlive {
|
||||
d.KeepAliveConfig = net.KeepAliveConfig{
|
||||
Enable: true,
|
||||
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
// IPv4-only
|
||||
conn, err := d.DialContext(ctx, "tcp4", t.addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
if _, err := conn.Write(t.helloLine); err != nil {
|
||||
conn.Close()
|
||||
return fmt.Errorf("hello: %w", err)
|
||||
}
|
||||
|
||||
t.conn = conn
|
||||
t.connected.Store(true)
|
||||
if t.everConnected {
|
||||
t.reconnects.Add(1)
|
||||
}
|
||||
t.everConnected = true
|
||||
|
||||
t.logger.Info("msg", "Chain link established",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"node", t.node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// closeConn tears down the current connection (run loop goroutine only)
|
||||
func (t *TCPChainSink) closeConn() {
|
||||
if t.conn != nil {
|
||||
t.conn.Close()
|
||||
t.conn = nil
|
||||
}
|
||||
t.connected.Store(false)
|
||||
}
|
||||
|
||||
// waitBackoff sleeps for the computed delay, interruptible by shutdown
|
||||
func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.done:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// backoffDelay computes exponential backoff with ±20% jitter
|
||||
func (t *TCPChainSink) backoffDelay(failures int) time.Duration {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
|
||||
d := maxD
|
||||
if failures < 63 {
|
||||
if v := minD << uint(failures-1); v > 0 && v < maxD {
|
||||
d = v
|
||||
}
|
||||
}
|
||||
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func (fs *FileSource) Stop() {
|
||||
}
|
||||
fs.wg.Wait()
|
||||
|
||||
fs.proxy.RemoveSession(fs.id)
|
||||
fs.proxy.RemoveSession(fs.session.ID)
|
||||
|
||||
fs.mu.Lock()
|
||||
for _, w := range fs.watchers {
|
||||
@@ -360,4 +360,5 @@ func globToRegex(glob string) string {
|
||||
regex = strings.ReplaceAll(regex, `\*`, `.*`)
|
||||
regex = strings.ReplaceAll(regex, `\?`, `.`)
|
||||
return "^" + regex + "$"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/chain"
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/core"
|
||||
"logwisp/src/internal/plugin"
|
||||
"logwisp/src/internal/session"
|
||||
"logwisp/src/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("http_chain", NewHTTPChainSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http_chain source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultHTTPChainSourceBufferSize = 1000
|
||||
DefaultHTTPChainSourceIngestPath = "/ingest"
|
||||
DefaultHTTPChainSourceMaxBodyBytes = 8 * 1024 * 1024
|
||||
DefaultHTTPChainSourceReadTimeoutMS = 30000
|
||||
HTTPChainReadHeaderTimeout = 10 * time.Second
|
||||
HTTPChainServerShutdownTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// HTTPChainSource accepts NDJSON batches from upstream http_chain sinks
|
||||
type HTTPChainSource struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
config *config.HTTPChainSourceOptions
|
||||
|
||||
subscribers []chan core.LogEntry
|
||||
server *http.Server
|
||||
logger *log.Logger
|
||||
|
||||
// Session cache: one session per remote host + declared node
|
||||
sessions map[string]string // key -> sessionID
|
||||
sessionsMu sync.Mutex
|
||||
|
||||
mu sync.RWMutex
|
||||
|
||||
startTime time.Time
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
parseErrors atomic.Uint64
|
||||
totalRequests atomic.Uint64
|
||||
rejectedRequests atomic.Uint64
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewHTTPChainSourcePlugin creates an http_chain source through plugin factory
|
||||
func NewHTTPChainSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
opts := &config.HTTPChainSourceOptions{
|
||||
Host: "0.0.0.0",
|
||||
TrustNode: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.IngestPath == "" {
|
||||
opts.IngestPath = DefaultHTTPChainSourceIngestPath
|
||||
} else if !strings.HasPrefix(opts.IngestPath, "/") {
|
||||
return nil, fmt.Errorf("ingest_path: must start with '/'")
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPChainSourceBufferSize
|
||||
}
|
||||
if opts.MaxBodyBytes <= 0 {
|
||||
opts.MaxBodyBytes = DefaultHTTPChainSourceMaxBodyBytes
|
||||
}
|
||||
if opts.ReadTimeoutMS <= 0 {
|
||||
opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS
|
||||
}
|
||||
|
||||
s := &HTTPChainSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
sessions: make(map[string]string),
|
||||
logger: logger,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "HTTP chain source initialized",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port,
|
||||
"ingest_path", opts.IngestPath)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (s *HTTPChainSource) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (s *HTTPChainSource) Subscribe() <-chan core.LogEntry {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ch := make(chan core.LogEntry, s.config.BufferSize)
|
||||
s.subscribers = append(s.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start binds the listener and serves the ingest endpoint
|
||||
func (s *HTTPChainSource) Start() error {
|
||||
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
|
||||
// IPv4-only, aligns with tcp/http sinks
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped pattern: mux answers 405 with Allow header on non-POST
|
||||
mux.HandleFunc(http.MethodPost+" "+s.config.IngestPath, s.handleIngest)
|
||||
|
||||
s.server = &http.Server{
|
||||
Handler: mux,
|
||||
ReadTimeout: time.Duration(s.config.ReadTimeoutMS) * time.Millisecond,
|
||||
ReadHeaderTimeout: HTTPChainReadHeaderTimeout,
|
||||
// Future: TLSConfig for transport security
|
||||
}
|
||||
s.startTime = time.Now()
|
||||
|
||||
go func() {
|
||||
if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
s.logger.Error("msg", "HTTP chain server terminated",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", s.id,
|
||||
"error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
s.logger.Info("msg", "HTTP chain source started",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", s.id,
|
||||
"addr", addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop shuts down the server, sessions, and subscriber channels
|
||||
func (s *HTTPChainSource) Stop() {
|
||||
if s.server != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), HTTPChainServerShutdownTimeout)
|
||||
defer cancel()
|
||||
s.server.Shutdown(ctx)
|
||||
}
|
||||
|
||||
s.sessionsMu.Lock()
|
||||
for _, id := range s.sessions {
|
||||
s.proxy.RemoveSession(id)
|
||||
}
|
||||
s.sessions = make(map[string]string)
|
||||
s.sessionsMu.Unlock()
|
||||
|
||||
s.mu.Lock()
|
||||
for _, ch := range s.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.logger.Info("msg", "HTTP chain source stopped",
|
||||
"component", "http_chain_source",
|
||||
"instance_id", s.id)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (s *HTTPChainSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
|
||||
|
||||
s.sessionsMu.Lock()
|
||||
cachedSessions := len(s.sessions)
|
||||
s.sessionsMu.Unlock()
|
||||
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
Type: "http_chain",
|
||||
TotalEntries: s.totalEntries.Load(),
|
||||
DroppedEntries: s.droppedEntries.Load(),
|
||||
StartTime: s.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{
|
||||
"host": s.config.Host,
|
||||
"port": s.config.Port,
|
||||
"ingest_path": s.config.IngestPath,
|
||||
"total_requests": s.totalRequests.Load(),
|
||||
"rejected_requests": s.rejectedRequests.Load(),
|
||||
"parse_errors": s.parseErrors.Load(),
|
||||
"cached_sessions": cachedSessions,
|
||||
"trust_node": s.config.TrustNode,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// handleIngest validates protocol headers and ingests one NDJSON batch.
|
||||
// Batch acceptance is atomic: entries publish only after a clean full read.
|
||||
func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) {
|
||||
s.totalRequests.Add(1)
|
||||
|
||||
if r.Header.Get(chain.HeaderProtocol) != strconv.Itoa(chain.ProtocolVersion) {
|
||||
s.rejectedRequests.Add(1)
|
||||
http.Error(w, "unsupported protocol version", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
remoteHost := r.RemoteAddr
|
||||
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
|
||||
remoteHost = host
|
||||
}
|
||||
connNode := r.Header.Get(chain.HeaderNode)
|
||||
if connNode == "" || !s.config.TrustNode {
|
||||
connNode = remoteHost
|
||||
}
|
||||
|
||||
body := http.MaxBytesReader(w, r.Body, s.config.MaxBodyBytes)
|
||||
scanner := bufio.NewScanner(body)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
|
||||
|
||||
entries := make([]core.LogEntry, 0, 128)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode)
|
||||
if err != nil {
|
||||
// Content error within a clean transfer: skip line, keep batch
|
||||
s.parseErrors.Add(1)
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
// Transfer error: reject batch without partial ingestion, sender retries
|
||||
s.rejectedRequests.Add(1)
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) {
|
||||
http.Error(w, "body too large", http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
s.logger.Debug("msg", "Chain batch read failed",
|
||||
"component", "http_chain_source",
|
||||
"remote_addr", r.RemoteAddr,
|
||||
"error", err)
|
||||
http.Error(w, "malformed body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
s.publish(entry)
|
||||
}
|
||||
s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode))
|
||||
|
||||
w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries)))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// sessionFor returns the cached session for a remote+node, recreating after idle expiry
|
||||
func (s *HTTPChainSource) sessionFor(remoteHost, node string) string {
|
||||
key := remoteHost + "|" + node
|
||||
s.sessionsMu.Lock()
|
||||
defer s.sessionsMu.Unlock()
|
||||
|
||||
if id, ok := s.sessions[key]; ok {
|
||||
if _, exists := s.proxy.GetSession(id); exists {
|
||||
return id
|
||||
}
|
||||
}
|
||||
sess := s.proxy.CreateSession(remoteHost, map[string]any{
|
||||
"type": "http_chain",
|
||||
"node": node,
|
||||
})
|
||||
s.sessions[key] = sess.ID
|
||||
return sess.ID
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers
|
||||
func (s *HTTPChainSource) publish(entry core.LogEntry) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
s.totalEntries.Add(1)
|
||||
s.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range s.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
s.droppedEntries.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
package tcpchain
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/chain"
|
||||
"logwisp/src/internal/config"
|
||||
"logwisp/src/internal/core"
|
||||
"logwisp/src/internal/plugin"
|
||||
"logwisp/src/internal/session"
|
||||
"logwisp/src/internal/source"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSource("tcp_chain", NewTCPChainSourcePlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp_chain source: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultChainSourceBufferSize = 1000
|
||||
DefaultChainSourceHelloTimeoutMS = 10000
|
||||
)
|
||||
|
||||
// TCPChainSource accepts connections from upstream tcp_chain sinks and ingests NDJSON entries
|
||||
type TCPChainSource struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
config *config.TCPChainSourceOptions
|
||||
|
||||
subscribers []chan core.LogEntry
|
||||
listener net.Listener
|
||||
conns map[net.Conn]struct{}
|
||||
logger *log.Logger
|
||||
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
|
||||
startTime time.Time
|
||||
totalEntries atomic.Uint64
|
||||
droppedEntries atomic.Uint64
|
||||
parseErrors atomic.Uint64
|
||||
rejectedConns atomic.Uint64
|
||||
activeConns atomic.Int64
|
||||
lastEntryTime atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewTCPChainSourcePlugin creates a tcp_chain source through plugin factory
|
||||
func NewTCPChainSourcePlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (source.Source, error) {
|
||||
opts := &config.TCPChainSourceOptions{
|
||||
Host: "0.0.0.0",
|
||||
TrustNode: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultChainSourceBufferSize
|
||||
}
|
||||
if opts.HelloTimeoutMS <= 0 {
|
||||
opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS
|
||||
}
|
||||
|
||||
s := &TCPChainSource{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
subscribers: make([]chan core.LogEntry, 0),
|
||||
conns: make(map[net.Conn]struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "TCP chain source initialized",
|
||||
"component", "tcp_chain_source",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (s *TCPChainSource) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe returns a channel for receiving log entries
|
||||
func (s *TCPChainSource) Subscribe() <-chan core.LogEntry {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
ch := make(chan core.LogEntry, s.config.BufferSize)
|
||||
s.subscribers = append(s.subscribers, ch)
|
||||
return ch
|
||||
}
|
||||
|
||||
// Start binds the listener and begins accepting connections
|
||||
func (s *TCPChainSource) Start() error {
|
||||
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
|
||||
// IPv4-only
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
s.listener = ln
|
||||
s.ctx, s.cancel = context.WithCancel(context.Background())
|
||||
s.startTime = time.Now()
|
||||
|
||||
s.wg.Add(1)
|
||||
go s.acceptLoop()
|
||||
|
||||
s.logger.Info("msg", "TCP chain source started",
|
||||
"component", "tcp_chain_source",
|
||||
"instance_id", s.id,
|
||||
"addr", addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop closes the listener, all connections, and subscriber channels
|
||||
func (s *TCPChainSource) Stop() {
|
||||
if s.cancel != nil {
|
||||
s.cancel()
|
||||
}
|
||||
if s.listener != nil {
|
||||
s.listener.Close()
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
for conn := range s.conns {
|
||||
conn.Close() // unblocks per-connection reads
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.wg.Wait()
|
||||
|
||||
s.mu.Lock()
|
||||
for _, ch := range s.subscribers {
|
||||
close(ch)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.logger.Info("msg", "TCP chain source stopped",
|
||||
"component", "tcp_chain_source",
|
||||
"instance_id", s.id)
|
||||
}
|
||||
|
||||
// GetStats returns the source's statistics
|
||||
func (s *TCPChainSource) GetStats() source.SourceStats {
|
||||
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
Type: "tcp_chain",
|
||||
TotalEntries: s.totalEntries.Load(),
|
||||
DroppedEntries: s.droppedEntries.Load(),
|
||||
StartTime: s.startTime,
|
||||
LastEntryTime: lastEntry,
|
||||
Details: map[string]any{
|
||||
"host": s.config.Host,
|
||||
"port": s.config.Port,
|
||||
"active_connections": s.activeConns.Load(),
|
||||
"rejected_conns": s.rejectedConns.Load(),
|
||||
"parse_errors": s.parseErrors.Load(),
|
||||
"trust_node": s.config.TrustNode,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// acceptLoop accepts upstream connections until listener close
|
||||
func (s *TCPChainSource) acceptLoop() {
|
||||
defer s.wg.Done()
|
||||
for {
|
||||
conn, err := s.listener.Accept()
|
||||
if err != nil {
|
||||
if errors.Is(err, net.ErrClosed) || s.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
s.logger.Warn("msg", "Accept error",
|
||||
"component", "tcp_chain_source",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if s.config.MaxConnections > 0 && s.activeConns.Load() >= s.config.MaxConnections {
|
||||
s.rejectedConns.Add(1)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.conns[conn] = struct{}{}
|
||||
s.mu.Unlock()
|
||||
|
||||
s.wg.Add(1)
|
||||
go s.handleConn(conn)
|
||||
}
|
||||
}
|
||||
|
||||
// handleConn validates the hello preamble, then streams entries until EOF/error
|
||||
func (s *TCPChainSource) handleConn(conn net.Conn) {
|
||||
defer s.wg.Done()
|
||||
remote := conn.RemoteAddr().String()
|
||||
s.activeConns.Add(1)
|
||||
|
||||
var sessID string
|
||||
defer func() {
|
||||
conn.Close()
|
||||
s.mu.Lock()
|
||||
delete(s.conns, conn)
|
||||
s.mu.Unlock()
|
||||
if sessID != "" {
|
||||
s.proxy.RemoveSession(sessID)
|
||||
}
|
||||
s.activeConns.Add(-1)
|
||||
}()
|
||||
|
||||
scanner := bufio.NewScanner(conn)
|
||||
// Oversized line (> MaxLogEntryBytes) is a protocol violation; scanner is
|
||||
// unrecoverable after ErrTooLong, connection terminates
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
|
||||
|
||||
// Hello preamble
|
||||
conn.SetReadDeadline(time.Now().Add(time.Duration(s.config.HelloTimeoutMS) * time.Millisecond))
|
||||
if !scanner.Scan() {
|
||||
s.logger.Warn("msg", "Connection closed before hello",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"error", scanner.Err())
|
||||
return
|
||||
}
|
||||
hello, err := chain.DecodeHello(scanner.Bytes())
|
||||
if err != nil {
|
||||
s.logger.Warn("msg", "Rejected chain connection",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
|
||||
connNode := hello.Node
|
||||
if connNode == "" || !s.config.TrustNode {
|
||||
if host, _, splitErr := net.SplitHostPort(remote); splitErr == nil {
|
||||
connNode = host
|
||||
} else {
|
||||
connNode = remote
|
||||
}
|
||||
}
|
||||
|
||||
sess := s.proxy.CreateSession(remote, map[string]any{
|
||||
"type": "tcp_chain",
|
||||
"node": connNode,
|
||||
})
|
||||
sessID = sess.ID
|
||||
|
||||
s.logger.Info("msg", "Chain connection established",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"node", connNode)
|
||||
|
||||
idle := time.Duration(s.config.ReadTimeoutMS) * time.Millisecond
|
||||
for {
|
||||
if idle > 0 {
|
||||
conn.SetReadDeadline(time.Now().Add(idle))
|
||||
} else {
|
||||
conn.SetReadDeadline(time.Time{})
|
||||
}
|
||||
if !scanner.Scan() {
|
||||
if err := scanner.Err(); err != nil && !errors.Is(err, net.ErrClosed) {
|
||||
s.logger.Debug("msg", "Chain read terminated",
|
||||
"component", "tcp_chain_source",
|
||||
"remote_addr", remote,
|
||||
"error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
line := scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
s.proxy.UpdateActivity(sessID)
|
||||
|
||||
entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode)
|
||||
if err != nil {
|
||||
s.parseErrors.Add(1)
|
||||
s.logger.Debug("msg", "Dropped malformed chain entry",
|
||||
"component", "tcp_chain_source",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
s.publish(entry)
|
||||
}
|
||||
}
|
||||
|
||||
// parseEntry decodes a canonical LogEntry line and applies the node policy
|
||||
func (s *TCPChainSource) parseEntry(line []byte, connNode string) (core.LogEntry, bool) {
|
||||
var entry core.LogEntry
|
||||
if err := json.Unmarshal(line, &entry); err != nil {
|
||||
s.parseErrors.Add(1)
|
||||
s.logger.Debug("msg", "Dropped malformed chain entry",
|
||||
"component", "tcp_chain_source",
|
||||
"error", err)
|
||||
return core.LogEntry{}, false
|
||||
}
|
||||
if entry.Time.IsZero() {
|
||||
entry.Time = time.Now()
|
||||
}
|
||||
if entry.Node == "" || !s.config.TrustNode {
|
||||
entry.Node = connNode
|
||||
}
|
||||
entry.RawSize = int64(len(line))
|
||||
return entry, true
|
||||
}
|
||||
|
||||
// publish sends a log entry to all subscribers
|
||||
func (s *TCPChainSource) publish(entry core.LogEntry) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
s.totalEntries.Add(1)
|
||||
s.lastEntryTime.Store(entry.Time)
|
||||
|
||||
for _, ch := range s.subscribers {
|
||||
select {
|
||||
case ch <- entry:
|
||||
default:
|
||||
s.droppedEntries.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user