v0.13.0 doc update, refactor, http/tcp chain source and sink added

This commit is contained in:
2026-07-17 05:58:44 -04:00
parent ebb5aa3bfe
commit 87e57784da
35 changed files with 2211 additions and 1292 deletions
+14 -1
View File
@@ -6,20 +6,25 @@ import (
_ "logwisp/src/internal/source/console"
_ "logwisp/src/internal/source/file"
_ "logwisp/src/internal/source/httpchain"
_ "logwisp/src/internal/source/null"
_ "logwisp/src/internal/source/random"
_ "logwisp/src/internal/source/tcpchain"
_ "logwisp/src/internal/sink/console"
_ "logwisp/src/internal/sink/file"
_ "logwisp/src/internal/sink/http"
_ "logwisp/src/internal/sink/httpchain"
_ "logwisp/src/internal/sink/null"
_ "logwisp/src/internal/sink/tcp"
_ "logwisp/src/internal/sink/tcpchain"
"logwisp/src/internal/config"
"logwisp/src/internal/service"
"logwisp/src/internal/version"
"github.com/lixenwraith/log"
"github.com/lixenwraith/log/sanitizer"
)
// bootstrapInitial handles initial service startup with status reporter
@@ -132,6 +137,14 @@ func initializeLogger(cfg *config.Config) error {
}
logCfg.Level = levelValue
// Configure log format
if cfg.Logging.Format != "" {
logCfg.Format = cfg.Logging.Format
}
if cfg.Logging.Sanitization != "" {
logCfg.Sanitization = sanitizer.PolicyPreset(cfg.Logging.Sanitization)
}
// Configure based on output mode
switch cfg.Logging.Output {
case "none":
@@ -176,4 +189,4 @@ func configureFileLogging(logCfg *log.Config, cfg *config.Config) {
logCfg.RetentionPeriodHrs = cfg.Logging.File.RetentionHours
}
}
}
}
+73
View File
@@ -0,0 +1,73 @@
package main
import (
"fmt"
"logwisp/src/internal/version"
"os"
)
// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths.
const helpText = `LogWisp %s - log collection, processing, and distribution
Usage:
logwisp [options]
logwisp help | -h | --help
logwisp --version
Any configuration key is settable as a flag using its TOML path:
--<path>=<value> e.g. --logging.level=debug
Common options:
-c, --config <path> Configuration file (default: ./logwisp.toml)
--quiet Suppress console output
--status_reporter=<bool> Periodic status logging (default: true)
--auto_reload=<bool> Config hot reload on file change (default: false)
Logging:
--logging.output=<mode> file|stdout|stderr|split|all|none
--logging.level=<level> debug|info|warn|error
--logging.file.directory=<path>
--logging.console.target=<target> stdout|stderr|split
Pipelines (N = 0-based index):
--pipelines.N.name=<name>
--pipelines.N.plugin_sources.N.type=<type> file|console|random|null
--pipelines.N.plugin_sinks.N.type=<type> console|file|http|tcp|null
--pipelines.N.flow.filters.N.patterns='["ERROR","WARN"]'
Environment:
LOGWISP_<PATH> Config path, '.' -> '_', uppercase
e.g. LOGWISP_LOGGING_LEVEL=debug
LOGWISP_CONFIG_FILE Configuration file path
LOGWISP_CONFIG_DIR Configuration directory
Signals:
SIGINT, SIGTERM Graceful shutdown
SIGHUP, SIGUSR1 Reload configuration
Exit codes:
0 success
1 general error
2 configuration file not found
`
// handleHelp prints usage and exits if a help request is present in args
func handleHelp(args []string) {
if len(args) > 0 && args[0] == "help" {
printHelp()
}
for _, arg := range args {
if arg == "--" {
break // end of flags
}
if arg == "-h" || arg == "--help" {
printHelp()
}
}
}
// printHelp writes usage to stdout and exits with success
func printHelp() {
fmt.Printf(helpText, version.Short())
os.Exit(0)
}
+5 -4
View File
@@ -7,7 +7,6 @@ import (
"os/signal"
"strings"
"syscall"
"time"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
@@ -25,6 +24,10 @@ func main() {
// Emulates nohup
signal.Ignore(syscall.SIGHUP)
// Help handled before config parsing; loader has no help flag.
// Also the future dispatch point for subcommands (tls, etc.)
handleHelp(os.Args[1:])
// Load configuration with automatic CLI parsing
cfg, err := config.Load(os.Args[1:])
if err != nil {
@@ -64,8 +67,6 @@ func main() {
"status_reporter", cfg.StatusReporter,
"auto_reload", cfg.ConfigAutoReload)
time.Sleep(time.Second)
// Create context for shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -152,4 +153,4 @@ func shutdownLogger() {
Error("Logger shutdown error: %v\n", err)
}
}
}
}
+98
View File
@@ -0,0 +1,98 @@
package chain
import (
"encoding/json"
"fmt"
"logwisp/src/internal/core"
"math/rand/v2"
"time"
)
// ProtocolVersion is declared in the hello preamble
const ProtocolVersion = 1
// Hello is the first NDJSON line sent by the dialing side after connect.
// Reserved for future revisions: auth credential, feature flags (ack, compression).
type Hello struct {
LogWisp int `json:"logwisp"`
Node string `json:"node,omitempty"`
// Auth string `json:"auth,omitempty"`
// Features []string `json:"features,omitempty"`
}
// HTTP transport mapping of the chain protocol.
// Hello preamble equivalent: protocol + node carried as request headers.
// Reserved extension point: Authorization header for auth, TLS at transport.
const (
HeaderProtocol = "X-Logwisp-Protocol"
HeaderNode = "X-Logwisp-Node"
HeaderAccepted = "X-Logwisp-Accepted"
ContentTypeNDJSON = "application/x-ndjson"
)
// EncodeHello serializes a newline-terminated hello preamble
func EncodeHello(node string) ([]byte, error) {
b, err := json.Marshal(Hello{LogWisp: ProtocolVersion, Node: node})
if err != nil {
return nil, err
}
return append(b, '\n'), nil
}
// DecodeHello parses and validates a hello preamble line
func DecodeHello(line []byte) (Hello, error) {
var h Hello
if err := json.Unmarshal(line, &h); err != nil {
return h, fmt.Errorf("malformed hello: %w", err)
}
if h.LogWisp != ProtocolVersion {
return h, fmt.Errorf("unsupported protocol version: %d", h.LogWisp)
}
return h, nil
}
// DecodeEntry parses a canonical LogEntry line and applies the node trust policy
func DecodeEntry(line []byte, connNode string, trustNode bool) (core.LogEntry, error) {
var entry core.LogEntry
if err := json.Unmarshal(line, &entry); err != nil {
return core.LogEntry{}, err
}
if entry.Time.IsZero() {
entry.Time = time.Now()
}
if entry.Node == "" || !trustNode {
entry.Node = connNode
}
entry.RawSize = int64(len(line))
return entry, nil
}
// EntryFromEvent extracts the structured entry, stamping node identity at
// first hop. Second return is true when synthesized from a formatted payload.
func EntryFromEvent(event core.TransportEvent, node, fallbackSource string) (core.LogEntry, bool) {
entry := event.Entry
synthesized := false
if entry.Time.IsZero() {
synthesized = true
entry = core.LogEntry{
Time: event.Time,
Source: fallbackSource,
Message: string(event.Payload),
}
}
if entry.Node == "" {
entry.Node = node
}
return entry, synthesized
}
// BackoffDelay computes exponential backoff with ±20% jitter
func BackoffDelay(minD, maxD time.Duration, failures int) time.Duration {
d := maxD
if failures < 63 {
if v := minD << uint(failures-1); v > 0 && v < maxD {
d = v
}
}
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
}
+63 -4
View File
@@ -205,6 +205,32 @@ type ConsoleSourceOptions struct {
BufferSize int64 `toml:"buffer_size"`
}
// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting
// NDJSON entries from upstream logwisp tcp_chain sinks
type TCPChainSourceOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // per-connection idle deadline, 0 = none
HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
// Future: TLS/auth options
}
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
// NDJSON batches from upstream logwisp http_chain sinks
type HTTPChainSourceOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
IngestPath string `toml:"ingest_path"`
BufferSize int64 `toml:"buffer_size"`
MaxBodyBytes int64 `toml:"max_body_bytes"` // per-request cap
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
// Future: TLS/auth options
}
// --- Sink Options ---
// PluginSinkConfig represents a sink plugin instance configuration
@@ -251,16 +277,49 @@ type TCPSinkOptions struct {
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
WriteTimeout int64 `toml:"write_timeout_ms"`
KeepAlive bool `toml:"keep_alive"`
KeepAlivePeriod int64 `toml:"keep_alive_period_ms"`
KeepAlive bool `toml:"keep_alive"`
}
// HTTPSinkOptions defines settings for an HTTP SSE server sink
type HTTPSinkOptions struct {
Host string `toml:"host"`
Port int64 `toml:"port"`
StreamPath string `toml:"stream_path"`
StatusPath string `toml:"status_path"`
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
WriteTimeout int64 `toml:"write_timeout_ms"`
}
}
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
// entries to a downstream logwisp tcp_chain source
type TCPChainSinkOptions struct {
Node string `toml:"node"` // origin label, default: os.Hostname()
Host string `toml:"host"`
Port int64 `toml:"port"`
BufferSize int64 `toml:"buffer_size"`
DialTimeoutMS int64 `toml:"dial_timeout_ms"`
WriteTimeoutMS int64 `toml:"write_timeout_ms"`
BackoffMinMS int64 `toml:"backoff_min_ms"`
BackoffMaxMS int64 `toml:"backoff_max_ms"`
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
KeepAlive bool `toml:"keep_alive"`
// Future: TLS/auth options
}
// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting
// NDJSON batches to a downstream logwisp http_chain source
type HTTPChainSinkOptions struct {
Node string `toml:"node"` // origin label, default: os.Hostname()
Host string `toml:"host"`
Port int64 `toml:"port"`
IngestPath string `toml:"ingest_path"`
BufferSize int64 `toml:"buffer_size"`
MaxBatchCount int64 `toml:"max_batch_count"`
MaxBatchBytes int64 `toml:"max_batch_bytes"`
FlushIntervalMS int64 `toml:"flush_interval_ms"`
RequestTimeoutMS int64 `toml:"request_timeout_ms"` // covers dial + write + response
BackoffMinMS int64 `toml:"backoff_min_ms"`
BackoffMaxMS int64 `toml:"backoff_max_ms"`
// Future: TLS/auth options
}
+8 -1
View File
@@ -65,6 +65,12 @@ func Load(args []string) (*Config, error) {
// Store the manager for hot reload
configManager = cfg
// Surface typo'd flags (e.g. --status-reporter vs --status_reporter);
// pre-logger phase, stderr only, suppressed in quiet mode
if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet {
fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown)
}
// Start watcher if auto-reload is enabled
if finalConfig.ConfigAutoReload {
watchOpts := lconfig.WatchOptions{
@@ -99,6 +105,7 @@ func defaults() *Config {
Logging: &LogConfig{
Output: "stdout",
Level: "info",
Format: "txt",
File: &LogFileConfig{
Directory: "./log",
Name: "logwisp",
@@ -195,4 +202,4 @@ func customEnvTransform(path string) string {
env = strings.ToUpper(env)
// env = "LOGWISP_" + env // already added by WithEnvPrefix
return env
}
}
+21 -1
View File
@@ -17,6 +17,15 @@ func ValidateConfig(cfg *Config) error {
return fmt.Errorf("no pipelines configured")
}
// Reject duplicate pipeline names (service map is keyed by name)
names := make(map[string]struct{}, len(cfg.Pipelines))
for i, p := range cfg.Pipelines {
if _, dup := names[p.Name]; dup {
return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name)
}
names[p.Name] = struct{}{}
}
if err := validateLogConfig(cfg.Logging); err != nil {
return fmt.Errorf("logging: %w", err)
}
@@ -52,6 +61,17 @@ func validateLogConfig(cfg *LogConfig) error {
return fmt.Errorf("level: %w", err)
}
if cfg.Format != "" {
if err := lconfig.OneOf("raw", "txt", "json")(cfg.Format); err != nil {
return fmt.Errorf("format: %w", err)
}
}
if cfg.Sanitization != "" {
if err := lconfig.OneOf("raw", "json", "txt", "shell")(cfg.Sanitization); err != nil {
return fmt.Errorf("sanitization: %w", err)
}
}
if cfg.Console != nil {
validateTarget := lconfig.OneOf("stdout", "stderr", "split")
if err := validateTarget(cfg.Console.Target); err != nil {
@@ -60,4 +80,4 @@ func validateLogConfig(cfg *LogConfig) error {
}
return nil
}
}
+6 -2
View File
@@ -5,9 +5,10 @@ import (
"time"
)
// Represents a single log record flowing through the pipeline
// LogEntry represents a single log record flowing through the pipeline
type LogEntry struct {
Time time.Time `json:"time"`
Node string `json:"node,omitempty"` // origin node identity for chained topologies; first hop stamps, relays preserve
Source string `json:"source"`
Level string `json:"level,omitempty"`
Message string `json:"message"`
@@ -20,4 +21,7 @@ type TransportEvent struct {
Time time.Time
// Formatted, serialized log payload
Payload []byte
}
// Structured entry for re-serializing sinks (chain links). Zero Time => absent
Entry LogEntry
}
+3 -1
View File
@@ -119,6 +119,7 @@ func (f *Flow) Process(entry core.LogEntry) (core.TransportEvent, bool) {
event := core.TransportEvent{
Time: entry.Time,
Payload: formatted,
Entry: entry, // Carry structured entry so chain sinks are format-independent
}
return event, true
@@ -159,4 +160,5 @@ func (f *Flow) GetStats() map[string]any {
}
return stats
}
}
+5 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync/atomic"
"time"
@@ -127,7 +128,8 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent
// SSE comment format - bypass formatter for this special case
if hg.config.IncludeStats {
beatNum := hg.beatCount.Load()
payload = []byte(": heartbeat " + t.Format(time.RFC3339) + " [#" + string(beatNum) + "]\n")
payload = []byte(": heartbeat " + t.Format(time.RFC3339) +
" [#" + strconv.FormatUint(beatNum, 10) + "]\n")
} else {
payload = []byte(": heartbeat " + t.Format(time.RFC3339) + "\n")
}
@@ -159,10 +161,11 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent
return core.TransportEvent{
Time: t,
Payload: payload,
Entry: entry, // heartbeats traverse chain links as structured entries
}
}
// IntervalMS returns the heartbeat interval in milliseconds
func (hg *HeartbeatGenerator) IntervalMS() int64 {
return hg.config.IntervalMS
}
}
+16 -4
View File
@@ -89,6 +89,8 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) {
func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
// Map logwisp LogEntry to formatter args
level := mapLevel(entry.Level)
// syslog-style origin prefix for chained entries
src := sourceLabel(entry)
// Build args based on whether we have structured fields
var args []any
@@ -101,18 +103,19 @@ func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) {
args = []any{entry.Message, fields}
// Add structured flag to properly format fields as JSON object
effectiveFlags := a.flags | formatter.FlagStructuredJSON
return a.formatter.Format(effectiveFlags, entry.Time, level, entry.Source, args), nil
return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil
}
}
// Simple message without fields
args = []any{entry.Message}
return a.formatter.Format(a.flags, entry.Time, level, entry.Source, args), nil
return a.formatter.Format(a.flags, entry.Time, level, src, args), nil
}
// FormatWithFlags allows custom flags for specific formatting needs
func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) {
level := mapLevel(entry.Level)
src := sourceLabel(entry)
var args []any
if len(entry.Fields) > 0 {
@@ -127,7 +130,7 @@ func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int6
args = []any{entry.Message}
}
return a.formatter.Format(customFlags, entry.Time, level, entry.Source, args), nil
return a.formatter.Format(customFlags, entry.Time, level, src, args), nil
}
// Name returns formatter type
@@ -149,4 +152,13 @@ func mapLevel(level string) int64 {
default:
return 0
}
}
}
// sourceLabel prefixes origin node onto source (syslog HOSTNAME + TAG convention)
func sourceLabel(entry core.LogEntry) string {
if entry.Node == "" {
return entry.Source
}
return entry.Node + "/" + entry.Source
}
+87 -97
View File
@@ -41,13 +41,11 @@ type Pipeline struct {
// PipelineStats contains runtime statistics for a pipeline
type PipelineStats struct {
StartTime time.Time
TotalEntriesProcessed atomic.Uint64
TotalEntriesDroppedByRateLimit atomic.Uint64
TotalEntriesFiltered atomic.Uint64
SourceStats []source.SourceStats
SinkStats []sink.SinkStats
FlowStats map[string]any
StartTime time.Time
TotalEntriesDroppedBySink atomic.Uint64
SourceStats []source.SourceStats
SinkStats []sink.SinkStats
FlowStats map[string]any
}
// NewPipeline creates a new pipeline with registry support
@@ -74,7 +72,6 @@ func NewPipeline(
cancel: pipelineCancel,
}
// Create flow processor
// Create flow processor
flowProcessor, err := flow.NewFlow(cfg.Flow, logger)
if err != nil {
@@ -177,7 +174,7 @@ func (p *Pipeline) initSourceCapabilities(s source.Source, cfg config.PluginSour
// initSinkCapabilities checks and injects optional capabilities
func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig) error {
// Initiate and activate source capabilities
// Initiate and activate sink capabilities
for _, c := range s.Capabilities() {
switch c {
// Network capabilities
@@ -203,48 +200,36 @@ func (p *Pipeline) run() {
defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name)
var componentWg sync.WaitGroup
// Start a goroutine for each source to fan-in data
for _, src := range p.Sources {
componentWg.Add(1)
go func(s source.Source) {
defer componentWg.Done()
ch := s.Subscribe()
for {
select {
case entry, ok := <-ch:
if !ok {
return
}
// Process and distribute the log entry
if event, passed := p.Flow.Process(entry); passed {
// Fan-out to all sinks
for _, snk := range p.Sinks {
snk.Input() <- event
}
}
case <-p.ctx.Done():
return
// Range allows in-flight data to drain cleanly once Source.Stop() closes the channel
for entry := range ch {
if event, passed := p.Flow.Process(entry); passed {
// Use non-blocking dispatcher
p.dispatch(event)
}
}
}(src)
}
var hbWg sync.WaitGroup
// Start heartbeat generator if enabled
if heartbeatCh := p.Flow.StartHeartbeat(p.ctx); heartbeatCh != nil {
componentWg.Add(1)
hbWg.Add(1)
go func() {
defer componentWg.Done()
defer hbWg.Done()
for {
select {
case event, ok := <-heartbeatCh:
if !ok {
return
}
// Fan-out heartbeat to all sinks
for _, snk := range p.Sinks {
snk.Input() <- event
}
// Use non-blocking dispatcher
p.dispatch(event)
case <-p.ctx.Done():
return
}
@@ -253,6 +238,23 @@ func (p *Pipeline) run() {
}
componentWg.Wait()
// Terminate internal contexts (heartbeat) once flow is complete
p.cancel()
hbWg.Wait()
}
// dispatch performs a non-blocking send to all sinks.
// A full/stalled sink must never block the run loop or starve sibling sinks.
func (p *Pipeline) dispatch(event core.TransportEvent) {
for _, snk := range p.Sinks {
select {
case snk.Input() <- event:
default:
// Buffer full - drop to avoid deadlocking the pipeline
p.Stats.TotalEntriesDroppedBySink.Add(1)
}
}
}
// Start starts the pipeline operation and all its components including flow, sources, and sinks
@@ -294,10 +296,7 @@ func (p *Pipeline) Stop() error {
p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name)
// Signal all components and the run loop to stop
p.cancel()
// Stop all sources concurrently to halt new data ingress
// 1. Stop all sources concurrently to halt new data ingress and close their channels
var sourceWg sync.WaitGroup
for _, src := range p.Sources {
sourceWg.Add(1)
@@ -308,10 +307,11 @@ func (p *Pipeline) Stop() error {
}
sourceWg.Wait()
// Wait for the run loop to finish processing and sending all in-flight data
// 2. Wait for the run loop to finish processing and sending all in-flight data
// run() inherently calls p.cancel() when the source channels are empty
p.wg.Wait()
// Stop all sinks concurrently now that no new data will be sent
// 3. Stop all sinks concurrently now that no new data will be sent
var sinkWg sync.WaitGroup
for _, s := range p.Sinks {
sinkWg.Add(1)
@@ -361,92 +361,82 @@ func (p *Pipeline) GetStats() map[string]any {
}
}()
// Collect source stats
sourceStats := make([]map[string]any, 0, len(p.Sources))
// 1. Live collect source stats
sources := make([]map[string]any, 0, len(p.Sources))
for _, src := range p.Sources {
if src == nil {
continue // Skip nil sources
continue
}
stats := src.GetStats()
sourceStats = append(sourceStats, map[string]any{
"id": stats.ID,
"type": stats.Type,
"total_entries": stats.TotalEntries,
"dropped_entries": stats.DroppedEntries,
"start_time": stats.StartTime,
"last_entry_time": stats.LastEntryTime,
"details": stats.Details,
s := src.GetStats()
sources = append(sources, map[string]any{
"id": s.ID,
"type": s.Type,
"total_entries": s.TotalEntries,
"dropped_entries": s.DroppedEntries,
"start_time": s.StartTime,
"last_entry_time": s.LastEntryTime,
"details": s.Details,
})
}
// Collect sink stats
sinkStats := make([]map[string]any, 0, len(p.Sinks))
for _, s := range p.Sinks {
if s == nil {
continue // Skip nil sinks
// 2. Live collect sink stats
sinks := make([]map[string]any, 0, len(p.Sinks))
for _, snk := range p.Sinks {
if snk == nil {
continue
}
stats := s.GetStats()
sinkStats = append(sinkStats, map[string]any{
"id": stats.ID,
"type": stats.Type,
"total_processed": stats.TotalProcessed,
"active_connections": stats.ActiveConnections,
"start_time": stats.StartTime,
"last_processed": stats.LastProcessed,
"details": stats.Details,
s := snk.GetStats()
sinks = append(sinks, map[string]any{
"id": s.ID,
"type": s.Type,
"total_processed": s.TotalProcessed,
"active_connections": s.ActiveConnections,
"start_time": s.StartTime,
"last_processed": s.LastProcessed,
"details": s.Details,
})
}
// Get flow stats
// 3. Collect flow stats and calculate filtered total
var flowStats map[string]any
var totalFiltered uint64
var totalProcessed uint64
if p.Flow != nil {
flowStats = p.Flow.GetStats()
// Extract total_filtered from flow for top-level visibility
// Map the top-level processed counter directly from Flow's source of truth
if tp, ok := flowStats["total_processed"].(uint64); ok {
totalProcessed = tp
}
// Calculate total dropped specifically by the filter chain
if filters, ok := flowStats["filters"].(map[string]any); ok {
if totalPassed, ok := filters["total_passed"].(uint64); ok {
if totalProcessed, ok := filters["total_processed"].(uint64); ok {
totalFiltered = totalProcessed - totalPassed
if tProc, ok := filters["total_processed"].(uint64); ok {
totalFiltered = tProc - totalPassed
}
}
}
}
// 4. Calculate Uptime
var uptime int
if p.running.Load() && !p.Stats.StartTime.IsZero() {
uptime = int(time.Since(p.Stats.StartTime).Seconds())
}
return map[string]any{
"name": p.Config.Name,
"running": p.running.Load(),
"uptime_seconds": uptime,
"total_processed": p.Stats.TotalEntriesProcessed.Load(),
"total_filtered": totalFiltered,
"source_count": len(p.Sources),
"sources": sourceStats,
"sink_count": len(p.Sinks),
"sinks": sinkStats,
"flow": flowStats,
"name": p.Config.Name,
"running": p.running.Load(),
"uptime_seconds": uptime,
"total_processed": totalProcessed,
"total_filtered": totalFiltered,
"total_dropped_by_sink": p.Stats.TotalEntriesDroppedBySink.Load(),
"source_count": len(p.Sources),
"sources": sources,
"sink_count": len(p.Sinks),
"sinks": sinks,
"flow": flowStats,
}
}
// TODO: incomplete implementation
// startStatsUpdater runs a periodic stats updater
func (p *Pipeline) startStatsUpdater(ctx context.Context) {
go func() {
ticker := time.NewTicker(core.ServiceStatsUpdateInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
// Periodic stats updates if needed
}
}
}()
}
-15
View File
@@ -34,16 +34,6 @@ type PluginMetadata struct {
MaxInstances int // 0 = unlimited, 1 = single instance only
}
// // global variables holding available source and sink plugins
// var (
// sourceFactories map[string]SourceFactory
// sinkFactories map[string]SinkFactory
// sourceMetadata map[string]*PluginMetadata
// sinkMetadata map[string]*PluginMetadata
// mu sync.RWMutex
// // once sync.Once
// )
// registry encapsulates all plugin factories with lazy initialization
type registry struct {
sourceFactories map[string]SourceFactory
@@ -71,11 +61,6 @@ func getRegistry() *registry {
return globalRegistry
}
// func init() {
// sourceFactories = make(map[string]SourceFactory)
// sinkFactories = make(map[string]SinkFactory)
// }
// RegisterSource registers a source factory function
func RegisterSource(name string, constructor SourceFactory) error {
r := getRegistry()
+18 -25
View File
@@ -170,16 +170,16 @@ func (h *HTTPSink) Start(ctx context.Context) error {
addr := fmt.Sprintf("%s:%d", h.config.Host, h.config.Port)
errChan := make(chan error, 1)
ln, err := net.Listen("tcp4", addr)
if err != nil {
return fmt.Errorf("http sink bind %s: %w", addr, err)
}
go func() {
h.logger.Info("msg", "HTTP server starting",
"component", "http_sink",
"instance_id", h.id,
"address", addr)
err := h.server.ListenAndServe(addr)
if err != nil {
errChan <- err
if err := h.server.Serve(ln); err != nil {
h.logger.Error("msg", "HTTP server terminated",
"component", "http_sink",
"instance_id", h.id,
"error", err)
}
}()
@@ -193,18 +193,12 @@ func (h *HTTPSink) Start(ctx context.Context) error {
}
}()
// Check if server started
select {
case err := <-errChan:
return err
case <-time.After(HttpServerStartTimeout):
h.logger.Info("msg", "HTTP server started",
"component", "http_sink",
"instance_id", h.id,
"host", h.config.Host,
"port", h.config.Port)
return nil
}
h.logger.Info("msg", "HTTP server started",
"component", "http_sink",
"instance_id", h.id,
"host", h.config.Host,
"port", h.config.Port)
return nil
}
// Stop gracefully shuts down the HTTP server and all client connections
@@ -356,6 +350,8 @@ func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) {
remoteAddr := ctx.RemoteAddr()
if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok {
if tcpAddr.IP.To4() == nil {
h.logger.Debug("msg", "IPv6 connection rejected",
"component", "http_sink", "remote_addr", remoteAddr.String())
ctx.SetConnectionClose()
return
}
@@ -414,8 +410,6 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
"client_id", clientID,
"active_clients", connectCount)
h.wg.Add(1)
defer func() {
disconnectCount := h.activeClients.Add(-1)
h.logger.Debug("msg", "HTTP client disconnected",
@@ -431,7 +425,6 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
}
h.proxy.RemoveSession(sess.ID)
h.wg.Done()
}()
// Send connected event with metadata
@@ -549,4 +542,4 @@ func splitLines(data []byte) [][]byte {
return [][]byte{data}
}
return lines
}
}
+422
View File
@@ -0,0 +1,422 @@
package httpchain
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"logwisp/src/internal/chain"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/plugin"
"logwisp/src/internal/session"
"logwisp/src/internal/sink"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
)
func init() {
if err := plugin.RegisterSink("http_chain", NewHTTPChainSinkPlugin); err != nil {
panic(fmt.Sprintf("failed to register http_chain sink: %v", err))
}
}
const (
DefaultHTTPChainSinkBufferSize = 1000
DefaultHTTPChainSinkIngestPath = "/ingest"
DefaultHTTPChainSinkMaxBatchCount = 100
DefaultHTTPChainSinkMaxBatchBytes = 1024 * 1024
DefaultHTTPChainSinkFlushIntervalMS = 1000
DefaultHTTPChainSinkRequestTimeoutMS = 10000
DefaultHTTPChainSinkBackoffMinMS = 500
DefaultHTTPChainSinkBackoffMaxMS = 30000
)
// HTTPChainSink batches structured entries and posts NDJSON to a downstream
// http_chain source. Delivery is at-least-once per batch.
type HTTPChainSink struct {
id string
proxy *session.Proxy
session *session.Session
config *config.HTTPChainSinkOptions
node string
url string
client *http.Client
input chan core.TransportEvent
logger *log.Logger
// Batch state owned exclusively by run loop goroutine
batch bytes.Buffer
batchCount int64
reqTimeout time.Duration
done chan struct{}
wg sync.WaitGroup
startTime time.Time
totalProcessed atomic.Uint64
batchesSent atomic.Uint64
requestErrors atomic.Uint64
droppedBatches atomic.Uint64
synthesized atomic.Uint64
lastProcessed atomic.Value // time.Time
}
// NewHTTPChainSinkPlugin creates an http_chain sink through plugin factory
func NewHTTPChainSinkPlugin(
id string,
configMap map[string]any,
logger *log.Logger,
proxy *session.Proxy,
) (sink.Sink, error) {
opts := &config.HTTPChainSinkOptions{}
if err := lconfig.ScanMap(configMap, opts); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
if err := lconfig.NonEmpty(opts.Host); err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
if opts.IngestPath == "" {
opts.IngestPath = DefaultHTTPChainSinkIngestPath
} else if !strings.HasPrefix(opts.IngestPath, "/") {
return nil, fmt.Errorf("ingest_path: must start with '/'")
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultHTTPChainSinkBufferSize
}
if opts.MaxBatchCount <= 0 {
opts.MaxBatchCount = DefaultHTTPChainSinkMaxBatchCount
}
if opts.MaxBatchBytes <= 0 {
opts.MaxBatchBytes = DefaultHTTPChainSinkMaxBatchBytes
}
if opts.FlushIntervalMS <= 0 {
opts.FlushIntervalMS = DefaultHTTPChainSinkFlushIntervalMS
}
if opts.RequestTimeoutMS <= 0 {
opts.RequestTimeoutMS = DefaultHTTPChainSinkRequestTimeoutMS
}
if opts.BackoffMinMS <= 0 {
opts.BackoffMinMS = DefaultHTTPChainSinkBackoffMinMS
}
if opts.BackoffMaxMS < opts.BackoffMinMS {
opts.BackoffMaxMS = DefaultHTTPChainSinkBackoffMaxMS
}
node := opts.Node
if node == "" {
if hn, err := os.Hostname(); err == nil {
node = hn
} else {
node = "unknown"
}
}
addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
transport := &http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
// IPv4-only, aligns with tcp/http sinks
d := net.Dialer{}
return d.DialContext(ctx, "tcp4", address)
},
MaxIdleConnsPerHost: 2,
IdleConnTimeout: 90 * time.Second,
DisableCompression: true,
// Future: TLSClientConfig; HTTP/2 via ALPN once TLS lands
}
t := &HTTPChainSink{
id: id,
proxy: proxy,
config: opts,
node: node,
// Future: "https" scheme with TLS
url: "http://" + addr + opts.IngestPath,
client: &http.Client{Transport: transport},
input: make(chan core.TransportEvent, opts.BufferSize),
done: make(chan struct{}),
logger: logger,
reqTimeout: time.Duration(opts.RequestTimeoutMS) * time.Millisecond,
}
t.lastProcessed.Store(time.Time{})
t.session = proxy.CreateSession(
"http_chain://"+addr,
map[string]any{
"instance_id": id,
"type": "http_chain",
"target": t.url,
"node": node,
},
)
logger.Info("msg", "HTTP chain sink initialized",
"component", "http_chain_sink",
"instance_id", id,
"target", t.url,
"node", node)
return t, nil
}
// Capabilities returns supported capabilities
func (t *HTTPChainSink) Capabilities() []core.Capability {
// CapTLS/CapAuth added when transport security lands
return []core.Capability{
core.CapSessionAware,
}
}
// Input returns the channel for sending transport events
func (t *HTTPChainSink) Input() chan<- core.TransportEvent {
return t.input
}
// Start launches the batching loop; downstream availability is not required
func (t *HTTPChainSink) Start(ctx context.Context) error {
t.startTime = time.Now()
t.wg.Add(1)
go t.runLoop(ctx)
t.logger.Info("msg", "HTTP chain sink started",
"component", "http_chain_sink",
"instance_id", t.id,
"target", t.url)
return nil
}
// Stop terminates the loop. Worst case: one in-flight request timeout plus
// one final-flush request timeout.
func (t *HTTPChainSink) Stop() {
t.logger.Info("msg", "Stopping HTTP chain sink",
"component", "http_chain_sink",
"instance_id", t.id)
close(t.done)
t.wg.Wait()
t.client.CloseIdleConnections()
if t.session != nil {
t.proxy.RemoveSession(t.session.ID)
}
t.logger.Info("msg", "HTTP chain sink stopped",
"component", "http_chain_sink",
"instance_id", t.id,
"total_processed", t.totalProcessed.Load())
}
// GetStats returns sink statistics
func (t *HTTPChainSink) GetStats() sink.SinkStats {
lastProc, _ := t.lastProcessed.Load().(time.Time)
return sink.SinkStats{
ID: t.id,
Type: "http_chain",
TotalProcessed: t.totalProcessed.Load(),
StartTime: t.startTime,
LastProcessed: lastProc,
Details: map[string]any{
"target": t.url,
"node": t.node,
"batches_sent": t.batchesSent.Load(),
"request_errors": t.requestErrors.Load(),
"dropped_batches": t.droppedBatches.Load(),
"synthesized": t.synthesized.Load(),
},
}
}
// runLoop batches events and flushes on size or interval
func (t *HTTPChainSink) runLoop(ctx context.Context) {
defer t.wg.Done()
// Fold done channel into a context for request/backoff interruption
runCtx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
select {
case <-t.done:
cancel()
case <-runCtx.Done():
}
}()
ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-runCtx.Done():
t.finalFlush()
return
case <-ticker.C:
if t.batchCount > 0 && !t.flush(runCtx) {
t.finalFlush()
return
}
case event, ok := <-t.input:
if !ok {
t.finalFlush()
return
}
t.append(event)
if t.batchCount >= t.config.MaxBatchCount ||
int64(t.batch.Len()) >= t.config.MaxBatchBytes {
if !t.flush(runCtx) {
t.finalFlush()
return
}
}
}
}
}
// append serializes one event into the pending batch
func (t *HTTPChainSink) append(event core.TransportEvent) {
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
if synthesized {
t.synthesized.Add(1)
}
line, err := json.Marshal(entry)
if err != nil {
// Non-transient: drop entry
t.logger.Error("msg", "Failed to marshal chain entry",
"component", "http_chain_sink",
"error", err)
return
}
t.batch.Write(line)
t.batch.WriteByte('\n')
t.batchCount++
}
// flush delivers the pending batch, retrying transient failures with backoff.
// Returns false when shutdown interrupts delivery; undelivered batch is dropped
// by finalFlush semantics (batch already consumed here).
func (t *HTTPChainSink) flush(ctx context.Context) bool {
body := bytes.Clone(t.batch.Bytes())
count := t.batchCount
t.batch.Reset()
t.batchCount = 0
failures := 0
for {
if failures > 0 && !t.waitBackoff(ctx, failures) {
t.droppedBatches.Add(1)
return false
}
transient, err := t.post(ctx, body)
if err == nil {
t.batchesSent.Add(1)
t.totalProcessed.Add(uint64(count))
t.lastProcessed.Store(time.Now())
t.proxy.UpdateActivity(t.session.ID)
return true
}
t.requestErrors.Add(1)
if !transient {
t.droppedBatches.Add(1)
t.logger.Error("msg", "Chain batch rejected, dropping",
"component", "http_chain_sink",
"target", t.url,
"entries", count,
"error", err)
return true
}
if ctx.Err() != nil {
t.droppedBatches.Add(1)
return false
}
failures++
t.logger.Warn("msg", "Chain batch delivery failed",
"component", "http_chain_sink",
"target", t.url,
"attempt", failures,
"error", err)
}
}
// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
func (t *HTTPChainSink) finalFlush() {
if t.batchCount == 0 {
return
}
fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
defer cancel()
count := t.batchCount
if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
t.droppedBatches.Add(1)
t.logger.Warn("msg", "Final chain batch dropped on shutdown",
"component", "http_chain_sink",
"entries", count,
"error", err)
return
}
t.batchesSent.Add(1)
t.totalProcessed.Add(uint64(count))
}
// post sends one NDJSON batch; transient=true marks retryable failures
func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool, err error) {
reqCtx, cancel := context.WithTimeout(ctx, t.reqTimeout)
defer cancel()
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, t.url, bytes.NewReader(body))
if err != nil {
return false, err
}
req.Header.Set("Content-Type", chain.ContentTypeNDJSON)
req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion))
req.Header.Set(chain.HeaderNode, t.node)
// Future: Authorization header for auth
resp, err := t.client.Do(req)
if err != nil {
return true, err
}
defer resp.Body.Close()
// Drain for connection reuse
io.Copy(io.Discard, resp.Body)
switch {
case resp.StatusCode >= 200 && resp.StatusCode < 300:
return false, nil
case resp.StatusCode == http.StatusRequestTimeout,
resp.StatusCode == http.StatusTooManyRequests,
resp.StatusCode >= 500:
return true, fmt.Errorf("status %s", resp.Status)
default:
return false, fmt.Errorf("status %s", resp.Status)
}
}
// waitBackoff sleeps for the computed delay, interruptible by shutdown
func (t *HTTPChainSink) waitBackoff(ctx context.Context, failures int) bool {
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
defer timer.Stop()
select {
case <-timer.C:
return true
case <-ctx.Done():
return false
}
}
+26 -5
View File
@@ -40,6 +40,7 @@ type TCPSink struct {
server *tcpServer
engine *gnet.Engine
engineMu sync.Mutex
booted chan struct{}
// Application
input chan core.TransportEvent
@@ -63,7 +64,7 @@ type TCPSink struct {
const (
// Server lifecycle
TCPServerStartTimeout = 100 * time.Millisecond
TCPServerStartTimeout = 2 * time.Second
TCPServerShutdownTimeout = 2 * time.Second
// Connection management
@@ -151,6 +152,8 @@ func (t *TCPSink) Start(ctx context.Context) error {
sink: t,
clients: make(map[gnet.Conn]*tcpClient),
}
// Fresh channel per Start
t.booted = make(chan struct{})
t.startTime = time.Now()
@@ -213,12 +216,25 @@ func (t *TCPSink) Start(ctx context.Context) error {
close(t.done)
t.wg.Wait()
return err
case <-time.After(TCPServerStartTimeout):
// Bind confirmation via OnBoot
case <-t.booted:
t.logger.Info("msg", "TCP server started",
"component", "tcp_sink",
"instance_id", t.id,
"port", t.config.Port)
return nil
// Timeout failure
case <-time.After(TCPServerStartTimeout):
t.engineMu.Lock()
if t.engine != nil {
stopCtx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout)
(*t.engine).Stop(stopCtx)
cancel()
}
t.engineMu.Unlock()
close(t.done)
t.wg.Wait()
return fmt.Errorf("tcp sink start timeout on %s", addr)
}
}
@@ -309,6 +325,9 @@ func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
s.sink.engine = &eng
s.sink.engineMu.Unlock()
// Listener is bound at this point; unblock Start
close(s.sink.booted)
s.sink.logger.Debug("msg", "TCP server booted",
"component", "tcp_sink",
"instance_id", s.sink.id)
@@ -409,8 +428,10 @@ func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
s.sink.proxy.UpdateActivity(client.sessionID)
}
// TCP sink doesn't expect data from clients, discard
c.Discard(-1)
// TCP sink doesn't expect data from clients, discard safely
if bufLen := c.InboundBuffered(); bufLen > 0 {
c.Next(bufLen)
}
return gnet.None
}
@@ -469,4 +490,4 @@ func (t *TCPSink) handleWriteError(c gnet.Conn, err error) {
delete(t.consecutiveWriteErrors, c)
c.Close()
}
}
}
+398
View File
@@ -0,0 +1,398 @@
package tcpchain
import (
"context"
"encoding/json"
"fmt"
"math/rand/v2"
"net"
"os"
"strconv"
"sync"
"sync/atomic"
"time"
"logwisp/src/internal/chain"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/plugin"
"logwisp/src/internal/session"
"logwisp/src/internal/sink"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
)
func init() {
if err := plugin.RegisterSink("tcp_chain", NewTCPChainSinkPlugin); err != nil {
panic(fmt.Sprintf("failed to register tcp_chain sink: %v", err))
}
}
const (
DefaultChainSinkBufferSize = 1000
DefaultChainSinkDialTimeoutMS = 5000
DefaultChainSinkWriteTimeoutMS = 5000
DefaultChainSinkBackoffMinMS = 500
DefaultChainSinkBackoffMaxMS = 30000
DefaultChainSinkKeepAlivePeriodMS = 30000
)
// TCPChainSink forwards structured entries to a downstream tcp_chain source
type TCPChainSink struct {
id string
proxy *session.Proxy
session *session.Session
config *config.TCPChainSinkOptions
node string
addr string
helloLine []byte
input chan core.TransportEvent
logger *log.Logger
// conn owned exclusively by run loop goroutine
conn net.Conn
everConnected bool
dialTimeout time.Duration
writeTimeout time.Duration
done chan struct{}
wg sync.WaitGroup
startTime time.Time
totalProcessed atomic.Uint64
writeErrors atomic.Uint64
reconnects atomic.Uint64
synthesized atomic.Uint64
connected atomic.Bool
lastProcessed atomic.Value // time.Time
}
// NewTCPChainSinkPlugin creates a tcp_chain sink through plugin factory
func NewTCPChainSinkPlugin(
id string,
configMap map[string]any,
logger *log.Logger,
proxy *session.Proxy,
) (sink.Sink, error) {
opts := &config.TCPChainSinkOptions{
KeepAlive: true,
}
if err := lconfig.ScanMap(configMap, opts); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
if err := lconfig.NonEmpty(opts.Host); err != nil {
return nil, fmt.Errorf("host: %w", err)
}
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultChainSinkBufferSize
}
if opts.DialTimeoutMS <= 0 {
opts.DialTimeoutMS = DefaultChainSinkDialTimeoutMS
}
if opts.WriteTimeoutMS <= 0 {
opts.WriteTimeoutMS = DefaultChainSinkWriteTimeoutMS
}
if opts.BackoffMinMS <= 0 {
opts.BackoffMinMS = DefaultChainSinkBackoffMinMS
}
if opts.BackoffMaxMS < opts.BackoffMinMS {
opts.BackoffMaxMS = DefaultChainSinkBackoffMaxMS
}
if opts.KeepAlivePeriodMS <= 0 {
opts.KeepAlivePeriodMS = DefaultChainSinkKeepAlivePeriodMS
}
node := opts.Node
if node == "" {
if hn, err := os.Hostname(); err == nil {
node = hn
} else {
node = "unknown"
}
}
helloLine, err := chain.EncodeHello(node)
if err != nil {
return nil, fmt.Errorf("hello: %w", err)
}
t := &TCPChainSink{
id: id,
proxy: proxy,
config: opts,
node: node,
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
helloLine: helloLine,
input: make(chan core.TransportEvent, opts.BufferSize),
done: make(chan struct{}),
logger: logger,
dialTimeout: time.Duration(opts.DialTimeoutMS) * time.Millisecond,
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
}
t.lastProcessed.Store(time.Time{})
t.session = proxy.CreateSession(
"tcp_chain://"+t.addr,
map[string]any{
"instance_id": id,
"type": "tcp_chain",
"target": t.addr,
"node": node,
},
)
logger.Info("msg", "TCP chain sink initialized",
"component", "tcp_chain_sink",
"instance_id", id,
"target", t.addr,
"node", node)
return t, nil
}
// Capabilities returns supported capabilities
func (t *TCPChainSink) Capabilities() []core.Capability {
// CapTLS/CapAuth added when transport security lands
return []core.Capability{
core.CapSessionAware,
}
}
// Input returns the channel for sending transport events
func (t *TCPChainSink) Input() chan<- core.TransportEvent {
return t.input
}
// Start launches the forwarding loop; connection is established lazily so
// pipeline start does not depend on downstream availability
func (t *TCPChainSink) Start(ctx context.Context) error {
t.startTime = time.Now()
t.wg.Add(1)
go t.runLoop(ctx)
t.logger.Info("msg", "TCP chain sink started",
"component", "tcp_chain_sink",
"instance_id", t.id,
"target", t.addr)
return nil
}
// Stop terminates the forwarding loop. Worst-case latency: one write timeout
// plus one backoff wait (both interruptible or bounded).
func (t *TCPChainSink) Stop() {
t.logger.Info("msg", "Stopping TCP chain sink",
"component", "tcp_chain_sink",
"instance_id", t.id)
close(t.done)
t.wg.Wait()
if t.session != nil {
t.proxy.RemoveSession(t.session.ID)
}
t.logger.Info("msg", "TCP chain sink stopped",
"component", "tcp_chain_sink",
"instance_id", t.id,
"total_processed", t.totalProcessed.Load())
}
// GetStats returns sink statistics
func (t *TCPChainSink) GetStats() sink.SinkStats {
lastProc, _ := t.lastProcessed.Load().(time.Time)
var active int64
if t.connected.Load() {
active = 1
}
return sink.SinkStats{
ID: t.id,
Type: "tcp_chain",
TotalProcessed: t.totalProcessed.Load(),
ActiveConnections: active,
StartTime: t.startTime,
LastProcessed: lastProc,
Details: map[string]any{
"target": t.addr,
"node": t.node,
"connected": t.connected.Load(),
"reconnects": t.reconnects.Load(),
"write_errors": t.writeErrors.Load(),
"synthesized": t.synthesized.Load(),
},
}
}
// runLoop consumes transport events and forwards them downstream
func (t *TCPChainSink) runLoop(ctx context.Context) {
defer t.wg.Done()
defer t.closeConn()
for {
select {
case <-ctx.Done():
return
case <-t.done:
return
case event, ok := <-t.input:
if !ok {
return
}
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
if synthesized {
t.synthesized.Add(1)
}
line, err := json.Marshal(entry)
if err != nil {
// Non-transient: drop
t.logger.Error("msg", "Failed to marshal chain entry",
"component", "tcp_chain_sink",
"error", err)
continue
}
if !t.deliver(ctx, append(line, '\n')) {
return // shutdown during retry
}
t.totalProcessed.Add(1)
t.lastProcessed.Store(time.Now())
t.proxy.UpdateActivity(t.session.ID)
}
}
}
// toEntry extracts the structured entry, stamping node identity at first hop
func (t *TCPChainSink) toEntry(event core.TransportEvent) core.LogEntry {
entry := event.Entry
if entry.Time.IsZero() {
// Defensive: event without structured entry, wrap formatted payload
t.synthesized.Add(1)
entry = core.LogEntry{
Time: event.Time,
Source: t.id,
Message: string(event.Payload),
}
}
if entry.Node == "" {
entry.Node = t.node
}
return entry
}
// deliver writes one line, holding it across reconnects until sent or shutdown.
// Backpressure during outage propagates to the pipeline dispatch drop counter.
func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
failures := 0
for {
if t.conn == nil {
if failures > 0 && !t.waitBackoff(ctx, failures) {
return false
}
if err := t.connect(ctx); err != nil {
if ctx.Err() != nil {
return false
}
failures++
t.logger.Debug("msg", "Chain connect failed",
"component", "tcp_chain_sink",
"target", t.addr,
"attempt", failures,
"error", err)
continue
}
}
t.conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
if _, err := t.conn.Write(line); err != nil {
t.writeErrors.Add(1)
failures++
t.logger.Warn("msg", "Chain write failed",
"component", "tcp_chain_sink",
"target", t.addr,
"error", err)
t.closeConn()
continue
}
return true
}
}
// connect performs a single dial + hello attempt
func (t *TCPChainSink) connect(ctx context.Context) error {
d := net.Dialer{Timeout: t.dialTimeout}
if t.config.KeepAlive {
d.KeepAliveConfig = net.KeepAliveConfig{
Enable: true,
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
}
}
// IPv4-only
conn, err := d.DialContext(ctx, "tcp4", t.addr)
if err != nil {
return err
}
conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
if _, err := conn.Write(t.helloLine); err != nil {
conn.Close()
return fmt.Errorf("hello: %w", err)
}
t.conn = conn
t.connected.Store(true)
if t.everConnected {
t.reconnects.Add(1)
}
t.everConnected = true
t.logger.Info("msg", "Chain link established",
"component", "tcp_chain_sink",
"target", t.addr,
"node", t.node)
return nil
}
// closeConn tears down the current connection (run loop goroutine only)
func (t *TCPChainSink) closeConn() {
if t.conn != nil {
t.conn.Close()
t.conn = nil
}
t.connected.Store(false)
}
// waitBackoff sleeps for the computed delay, interruptible by shutdown
func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
defer timer.Stop()
select {
case <-timer.C:
return true
case <-ctx.Done():
return false
case <-t.done:
return false
}
}
// backoffDelay computes exponential backoff with ±20% jitter
func (t *TCPChainSink) backoffDelay(failures int) time.Duration {
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
d := maxD
if failures < 63 {
if v := minD << uint(failures-1); v > 0 && v < maxD {
d = v
}
}
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
}
+3 -2
View File
@@ -161,7 +161,7 @@ func (fs *FileSource) Stop() {
}
fs.wg.Wait()
fs.proxy.RemoveSession(fs.id)
fs.proxy.RemoveSession(fs.session.ID)
fs.mu.Lock()
for _, w := range fs.watchers {
@@ -360,4 +360,5 @@ func globToRegex(glob string) string {
regex = strings.ReplaceAll(regex, `\*`, `.*`)
regex = strings.ReplaceAll(regex, `\?`, `.`)
return "^" + regex + "$"
}
}
+324
View File
@@ -0,0 +1,324 @@
package httpchain
import (
"bufio"
"context"
"errors"
"fmt"
"net"
"net/http"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"logwisp/src/internal/chain"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/plugin"
"logwisp/src/internal/session"
"logwisp/src/internal/source"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
)
func init() {
if err := plugin.RegisterSource("http_chain", NewHTTPChainSourcePlugin); err != nil {
panic(fmt.Sprintf("failed to register http_chain source: %v", err))
}
}
const (
DefaultHTTPChainSourceBufferSize = 1000
DefaultHTTPChainSourceIngestPath = "/ingest"
DefaultHTTPChainSourceMaxBodyBytes = 8 * 1024 * 1024
DefaultHTTPChainSourceReadTimeoutMS = 30000
HTTPChainReadHeaderTimeout = 10 * time.Second
HTTPChainServerShutdownTimeout = 2 * time.Second
)
// HTTPChainSource accepts NDJSON batches from upstream http_chain sinks
type HTTPChainSource struct {
id string
proxy *session.Proxy
config *config.HTTPChainSourceOptions
subscribers []chan core.LogEntry
server *http.Server
logger *log.Logger
// Session cache: one session per remote host + declared node
sessions map[string]string // key -> sessionID
sessionsMu sync.Mutex
mu sync.RWMutex
startTime time.Time
totalEntries atomic.Uint64
droppedEntries atomic.Uint64
parseErrors atomic.Uint64
totalRequests atomic.Uint64
rejectedRequests atomic.Uint64
lastEntryTime atomic.Value // time.Time
}
// NewHTTPChainSourcePlugin creates an http_chain source through plugin factory
func NewHTTPChainSourcePlugin(
id string,
configMap map[string]any,
logger *log.Logger,
proxy *session.Proxy,
) (source.Source, error) {
opts := &config.HTTPChainSourceOptions{
Host: "0.0.0.0",
TrustNode: true,
}
if err := lconfig.ScanMap(configMap, opts); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
if opts.IngestPath == "" {
opts.IngestPath = DefaultHTTPChainSourceIngestPath
} else if !strings.HasPrefix(opts.IngestPath, "/") {
return nil, fmt.Errorf("ingest_path: must start with '/'")
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultHTTPChainSourceBufferSize
}
if opts.MaxBodyBytes <= 0 {
opts.MaxBodyBytes = DefaultHTTPChainSourceMaxBodyBytes
}
if opts.ReadTimeoutMS <= 0 {
opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS
}
s := &HTTPChainSource{
id: id,
proxy: proxy,
config: opts,
subscribers: make([]chan core.LogEntry, 0),
sessions: make(map[string]string),
logger: logger,
}
s.lastEntryTime.Store(time.Time{})
logger.Info("msg", "HTTP chain source initialized",
"component", "http_chain_source",
"instance_id", id,
"host", opts.Host,
"port", opts.Port,
"ingest_path", opts.IngestPath)
return s, nil
}
// Capabilities returns supported capabilities
func (s *HTTPChainSource) Capabilities() []core.Capability {
// CapTLS/CapAuth added when transport security lands
return []core.Capability{
core.CapSessionAware,
core.CapMultiSession,
}
}
// Subscribe returns a channel for receiving log entries
func (s *HTTPChainSource) Subscribe() <-chan core.LogEntry {
s.mu.Lock()
defer s.mu.Unlock()
ch := make(chan core.LogEntry, s.config.BufferSize)
s.subscribers = append(s.subscribers, ch)
return ch
}
// Start binds the listener and serves the ingest endpoint
func (s *HTTPChainSource) Start() error {
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
// IPv4-only, aligns with tcp/http sinks
ln, err := net.Listen("tcp4", addr)
if err != nil {
return fmt.Errorf("listen %s: %w", addr, err)
}
mux := http.NewServeMux()
// Method-scoped pattern: mux answers 405 with Allow header on non-POST
mux.HandleFunc(http.MethodPost+" "+s.config.IngestPath, s.handleIngest)
s.server = &http.Server{
Handler: mux,
ReadTimeout: time.Duration(s.config.ReadTimeoutMS) * time.Millisecond,
ReadHeaderTimeout: HTTPChainReadHeaderTimeout,
// Future: TLSConfig for transport security
}
s.startTime = time.Now()
go func() {
if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
s.logger.Error("msg", "HTTP chain server terminated",
"component", "http_chain_source",
"instance_id", s.id,
"error", err)
}
}()
s.logger.Info("msg", "HTTP chain source started",
"component", "http_chain_source",
"instance_id", s.id,
"addr", addr)
return nil
}
// Stop shuts down the server, sessions, and subscriber channels
func (s *HTTPChainSource) Stop() {
if s.server != nil {
ctx, cancel := context.WithTimeout(context.Background(), HTTPChainServerShutdownTimeout)
defer cancel()
s.server.Shutdown(ctx)
}
s.sessionsMu.Lock()
for _, id := range s.sessions {
s.proxy.RemoveSession(id)
}
s.sessions = make(map[string]string)
s.sessionsMu.Unlock()
s.mu.Lock()
for _, ch := range s.subscribers {
close(ch)
}
s.mu.Unlock()
s.logger.Info("msg", "HTTP chain source stopped",
"component", "http_chain_source",
"instance_id", s.id)
}
// GetStats returns the source's statistics
func (s *HTTPChainSource) GetStats() source.SourceStats {
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
s.sessionsMu.Lock()
cachedSessions := len(s.sessions)
s.sessionsMu.Unlock()
return source.SourceStats{
ID: s.id,
Type: "http_chain",
TotalEntries: s.totalEntries.Load(),
DroppedEntries: s.droppedEntries.Load(),
StartTime: s.startTime,
LastEntryTime: lastEntry,
Details: map[string]any{
"host": s.config.Host,
"port": s.config.Port,
"ingest_path": s.config.IngestPath,
"total_requests": s.totalRequests.Load(),
"rejected_requests": s.rejectedRequests.Load(),
"parse_errors": s.parseErrors.Load(),
"cached_sessions": cachedSessions,
"trust_node": s.config.TrustNode,
},
}
}
// handleIngest validates protocol headers and ingests one NDJSON batch.
// Batch acceptance is atomic: entries publish only after a clean full read.
func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) {
s.totalRequests.Add(1)
if r.Header.Get(chain.HeaderProtocol) != strconv.Itoa(chain.ProtocolVersion) {
s.rejectedRequests.Add(1)
http.Error(w, "unsupported protocol version", http.StatusBadRequest)
return
}
remoteHost := r.RemoteAddr
if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
remoteHost = host
}
connNode := r.Header.Get(chain.HeaderNode)
if connNode == "" || !s.config.TrustNode {
connNode = remoteHost
}
body := http.MaxBytesReader(w, r.Body, s.config.MaxBodyBytes)
scanner := bufio.NewScanner(body)
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
entries := make([]core.LogEntry, 0, 128)
for scanner.Scan() {
line := scanner.Bytes()
if len(line) == 0 {
continue
}
entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode)
if err != nil {
// Content error within a clean transfer: skip line, keep batch
s.parseErrors.Add(1)
continue
}
entries = append(entries, entry)
}
if err := scanner.Err(); err != nil {
// Transfer error: reject batch without partial ingestion, sender retries
s.rejectedRequests.Add(1)
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
http.Error(w, "body too large", http.StatusRequestEntityTooLarge)
return
}
s.logger.Debug("msg", "Chain batch read failed",
"component", "http_chain_source",
"remote_addr", r.RemoteAddr,
"error", err)
http.Error(w, "malformed body", http.StatusBadRequest)
return
}
for _, entry := range entries {
s.publish(entry)
}
s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode))
w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries)))
w.WriteHeader(http.StatusNoContent)
}
// sessionFor returns the cached session for a remote+node, recreating after idle expiry
func (s *HTTPChainSource) sessionFor(remoteHost, node string) string {
key := remoteHost + "|" + node
s.sessionsMu.Lock()
defer s.sessionsMu.Unlock()
if id, ok := s.sessions[key]; ok {
if _, exists := s.proxy.GetSession(id); exists {
return id
}
}
sess := s.proxy.CreateSession(remoteHost, map[string]any{
"type": "http_chain",
"node": node,
})
s.sessions[key] = sess.ID
return sess.ID
}
// publish sends a log entry to all subscribers
func (s *HTTPChainSource) publish(entry core.LogEntry) {
s.mu.RLock()
defer s.mu.RUnlock()
s.totalEntries.Add(1)
s.lastEntryTime.Store(entry.Time)
for _, ch := range s.subscribers {
select {
case ch <- entry:
default:
s.droppedEntries.Add(1)
}
}
}
+353
View File
@@ -0,0 +1,353 @@
package tcpchain
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"strconv"
"sync"
"sync/atomic"
"time"
"logwisp/src/internal/chain"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/plugin"
"logwisp/src/internal/session"
"logwisp/src/internal/source"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
)
func init() {
if err := plugin.RegisterSource("tcp_chain", NewTCPChainSourcePlugin); err != nil {
panic(fmt.Sprintf("failed to register tcp_chain source: %v", err))
}
}
const (
DefaultChainSourceBufferSize = 1000
DefaultChainSourceHelloTimeoutMS = 10000
)
// TCPChainSource accepts connections from upstream tcp_chain sinks and ingests NDJSON entries
type TCPChainSource struct {
id string
proxy *session.Proxy
config *config.TCPChainSourceOptions
subscribers []chan core.LogEntry
listener net.Listener
conns map[net.Conn]struct{}
logger *log.Logger
mu sync.RWMutex
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
startTime time.Time
totalEntries atomic.Uint64
droppedEntries atomic.Uint64
parseErrors atomic.Uint64
rejectedConns atomic.Uint64
activeConns atomic.Int64
lastEntryTime atomic.Value // time.Time
}
// NewTCPChainSourcePlugin creates a tcp_chain source through plugin factory
func NewTCPChainSourcePlugin(
id string,
configMap map[string]any,
logger *log.Logger,
proxy *session.Proxy,
) (source.Source, error) {
opts := &config.TCPChainSourceOptions{
Host: "0.0.0.0",
TrustNode: true,
}
if err := lconfig.ScanMap(configMap, opts); err != nil {
return nil, fmt.Errorf("failed to parse config: %w", err)
}
if err := lconfig.Port(opts.Port); err != nil {
return nil, fmt.Errorf("port: %w", err)
}
if opts.BufferSize <= 0 {
opts.BufferSize = DefaultChainSourceBufferSize
}
if opts.HelloTimeoutMS <= 0 {
opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS
}
s := &TCPChainSource{
id: id,
proxy: proxy,
config: opts,
subscribers: make([]chan core.LogEntry, 0),
conns: make(map[net.Conn]struct{}),
logger: logger,
}
s.lastEntryTime.Store(time.Time{})
logger.Info("msg", "TCP chain source initialized",
"component", "tcp_chain_source",
"instance_id", id,
"host", opts.Host,
"port", opts.Port)
return s, nil
}
// Capabilities returns supported capabilities
func (s *TCPChainSource) Capabilities() []core.Capability {
// CapTLS/CapAuth added when transport security lands
return []core.Capability{
core.CapSessionAware,
core.CapMultiSession,
}
}
// Subscribe returns a channel for receiving log entries
func (s *TCPChainSource) Subscribe() <-chan core.LogEntry {
s.mu.Lock()
defer s.mu.Unlock()
ch := make(chan core.LogEntry, s.config.BufferSize)
s.subscribers = append(s.subscribers, ch)
return ch
}
// Start binds the listener and begins accepting connections
func (s *TCPChainSource) Start() error {
addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10))
// IPv4-only
ln, err := net.Listen("tcp4", addr)
if err != nil {
return fmt.Errorf("listen %s: %w", addr, err)
}
s.listener = ln
s.ctx, s.cancel = context.WithCancel(context.Background())
s.startTime = time.Now()
s.wg.Add(1)
go s.acceptLoop()
s.logger.Info("msg", "TCP chain source started",
"component", "tcp_chain_source",
"instance_id", s.id,
"addr", addr)
return nil
}
// Stop closes the listener, all connections, and subscriber channels
func (s *TCPChainSource) Stop() {
if s.cancel != nil {
s.cancel()
}
if s.listener != nil {
s.listener.Close()
}
s.mu.Lock()
for conn := range s.conns {
conn.Close() // unblocks per-connection reads
}
s.mu.Unlock()
s.wg.Wait()
s.mu.Lock()
for _, ch := range s.subscribers {
close(ch)
}
s.mu.Unlock()
s.logger.Info("msg", "TCP chain source stopped",
"component", "tcp_chain_source",
"instance_id", s.id)
}
// GetStats returns the source's statistics
func (s *TCPChainSource) GetStats() source.SourceStats {
lastEntry, _ := s.lastEntryTime.Load().(time.Time)
return source.SourceStats{
ID: s.id,
Type: "tcp_chain",
TotalEntries: s.totalEntries.Load(),
DroppedEntries: s.droppedEntries.Load(),
StartTime: s.startTime,
LastEntryTime: lastEntry,
Details: map[string]any{
"host": s.config.Host,
"port": s.config.Port,
"active_connections": s.activeConns.Load(),
"rejected_conns": s.rejectedConns.Load(),
"parse_errors": s.parseErrors.Load(),
"trust_node": s.config.TrustNode,
},
}
}
// acceptLoop accepts upstream connections until listener close
func (s *TCPChainSource) acceptLoop() {
defer s.wg.Done()
for {
conn, err := s.listener.Accept()
if err != nil {
if errors.Is(err, net.ErrClosed) || s.ctx.Err() != nil {
return
}
s.logger.Warn("msg", "Accept error",
"component", "tcp_chain_source",
"error", err)
continue
}
if s.config.MaxConnections > 0 && s.activeConns.Load() >= s.config.MaxConnections {
s.rejectedConns.Add(1)
conn.Close()
continue
}
s.mu.Lock()
s.conns[conn] = struct{}{}
s.mu.Unlock()
s.wg.Add(1)
go s.handleConn(conn)
}
}
// handleConn validates the hello preamble, then streams entries until EOF/error
func (s *TCPChainSource) handleConn(conn net.Conn) {
defer s.wg.Done()
remote := conn.RemoteAddr().String()
s.activeConns.Add(1)
var sessID string
defer func() {
conn.Close()
s.mu.Lock()
delete(s.conns, conn)
s.mu.Unlock()
if sessID != "" {
s.proxy.RemoveSession(sessID)
}
s.activeConns.Add(-1)
}()
scanner := bufio.NewScanner(conn)
// Oversized line (> MaxLogEntryBytes) is a protocol violation; scanner is
// unrecoverable after ErrTooLong, connection terminates
scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes)
// Hello preamble
conn.SetReadDeadline(time.Now().Add(time.Duration(s.config.HelloTimeoutMS) * time.Millisecond))
if !scanner.Scan() {
s.logger.Warn("msg", "Connection closed before hello",
"component", "tcp_chain_source",
"remote_addr", remote,
"error", scanner.Err())
return
}
hello, err := chain.DecodeHello(scanner.Bytes())
if err != nil {
s.logger.Warn("msg", "Rejected chain connection",
"component", "tcp_chain_source",
"remote_addr", remote,
"error", err)
return
}
connNode := hello.Node
if connNode == "" || !s.config.TrustNode {
if host, _, splitErr := net.SplitHostPort(remote); splitErr == nil {
connNode = host
} else {
connNode = remote
}
}
sess := s.proxy.CreateSession(remote, map[string]any{
"type": "tcp_chain",
"node": connNode,
})
sessID = sess.ID
s.logger.Info("msg", "Chain connection established",
"component", "tcp_chain_source",
"remote_addr", remote,
"node", connNode)
idle := time.Duration(s.config.ReadTimeoutMS) * time.Millisecond
for {
if idle > 0 {
conn.SetReadDeadline(time.Now().Add(idle))
} else {
conn.SetReadDeadline(time.Time{})
}
if !scanner.Scan() {
if err := scanner.Err(); err != nil && !errors.Is(err, net.ErrClosed) {
s.logger.Debug("msg", "Chain read terminated",
"component", "tcp_chain_source",
"remote_addr", remote,
"error", err)
}
return
}
line := scanner.Bytes()
if len(line) == 0 {
continue
}
s.proxy.UpdateActivity(sessID)
entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode)
if err != nil {
s.parseErrors.Add(1)
s.logger.Debug("msg", "Dropped malformed chain entry",
"component", "tcp_chain_source",
"error", err)
continue
}
s.publish(entry)
}
}
// parseEntry decodes a canonical LogEntry line and applies the node policy
func (s *TCPChainSource) parseEntry(line []byte, connNode string) (core.LogEntry, bool) {
var entry core.LogEntry
if err := json.Unmarshal(line, &entry); err != nil {
s.parseErrors.Add(1)
s.logger.Debug("msg", "Dropped malformed chain entry",
"component", "tcp_chain_source",
"error", err)
return core.LogEntry{}, false
}
if entry.Time.IsZero() {
entry.Time = time.Now()
}
if entry.Node == "" || !s.config.TrustNode {
entry.Node = connNode
}
entry.RawSize = int64(len(line))
return entry, true
}
// publish sends a log entry to all subscribers
func (s *TCPChainSource) publish(entry core.LogEntry) {
s.mu.RLock()
defer s.mu.RUnlock()
s.totalEntries.Add(1)
s.lastEntryTime.Store(entry.Time)
for _, ch := range s.subscribers {
select {
case ch <- entry:
default:
s.droppedEntries.Add(1)
}
}
}