v0.13.1 folder restructure, test script added, format adapter async fix
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
package console
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("console", NewConsoleSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register console sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// ConsoleSink writes log entries to the console (stdout/stderr) using an dedicated logger instance
|
||||
type ConsoleSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.ConsoleSinkOptions
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
output io.Writer
|
||||
logger *log.Logger // application logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
// Defaults
|
||||
DefaultConsoleTarget = "stdout"
|
||||
DefaultConsoleBufferSize = 1000
|
||||
)
|
||||
|
||||
// NewConsoleSinkPlugin creates a console sink through plugin factory
|
||||
func NewConsoleSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.ConsoleSinkOptions{}
|
||||
|
||||
// Scan config map into struct
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate and apply defaults
|
||||
if opts.Target == "" {
|
||||
opts.Target = DefaultConsoleTarget
|
||||
} else {
|
||||
validateTarget := lconfig.OneOf("stdout", "stderr")
|
||||
if err := validateTarget(opts.Target); err != nil {
|
||||
return nil, fmt.Errorf("target: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var output io.Writer
|
||||
switch opts.Target {
|
||||
case "stdout":
|
||||
output = os.Stdout
|
||||
case "stderr":
|
||||
output = os.Stderr
|
||||
}
|
||||
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultConsoleBufferSize
|
||||
}
|
||||
|
||||
// Create and return plugin instance
|
||||
cs := &ConsoleSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
output: output,
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
cs.lastProcessed.Store(time.Time{})
|
||||
|
||||
// Create session for output
|
||||
cs.session = proxy.CreateSession(
|
||||
fmt.Sprintf("console:%s", opts.Target),
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "console",
|
||||
"target": opts.Target,
|
||||
},
|
||||
)
|
||||
|
||||
cs.logger.Info("msg", "Console sink initialized",
|
||||
"component", "console_sink",
|
||||
"instance_id", id,
|
||||
"target", opts.Target,
|
||||
)
|
||||
|
||||
return cs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (cs *ConsoleSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware, // Single output session
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (cs *ConsoleSink) Input() chan<- core.TransportEvent {
|
||||
return cs.input
|
||||
}
|
||||
|
||||
// Start begins the processing loop
|
||||
func (cs *ConsoleSink) Start(ctx context.Context) error {
|
||||
cs.startTime = time.Now()
|
||||
go cs.processLoop(ctx)
|
||||
cs.logger.Info("msg", "Console sink started",
|
||||
"component", "console_sink",
|
||||
"target", cs.config.Target)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (cs *ConsoleSink) Stop() {
|
||||
cs.logger.Info("msg", "Stopping console sink", "target", cs.config.Target)
|
||||
|
||||
// Remove session
|
||||
if cs.session != nil {
|
||||
cs.proxy.RemoveSession(cs.session.ID)
|
||||
}
|
||||
|
||||
close(cs.done)
|
||||
|
||||
cs.logger.Info("msg", "Console sink stopped",
|
||||
"instance_id", cs.id,
|
||||
"target", cs.config.Target,
|
||||
"instance_id", cs.id,
|
||||
)
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (cs *ConsoleSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := cs.lastProcessed.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: cs.id,
|
||||
Type: "console",
|
||||
TotalProcessed: cs.totalProcessed.Load(),
|
||||
StartTime: cs.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": cs.config.Target,
|
||||
"buffer_size": cs.config.BufferSize,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop reads transport events and writes to console
|
||||
func (cs *ConsoleSink) processLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-cs.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Write pre-formatted payload directly to output
|
||||
if _, err := cs.output.Write(event.Payload); err != nil {
|
||||
cs.logger.Error("msg", "Failed to write to console",
|
||||
"component", "console_sink",
|
||||
"target", cs.config.Target,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
cs.totalProcessed.Add(1)
|
||||
cs.lastProcessed.Store(time.Now())
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-cs.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("file", NewFileSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register file sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// FileSink writes log entries to files with rotation
|
||||
type FileSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Configuration
|
||||
config *config.FileSinkOptions
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
writer *log.Logger // internal logger for file writing
|
||||
logger *log.Logger // application logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
// Defaults
|
||||
DefaultFileMaxSizeMB = 100
|
||||
DefaultFileMaxTotalSizeMB = 1000
|
||||
DefaultFileMinDiskFreeMB = 100
|
||||
DefaultFileRetentionHours = 168 // 7 days
|
||||
DefaultFileBufferSize = 1000
|
||||
DefaultFileFlushIntervalMs = 100
|
||||
)
|
||||
|
||||
// NewFileSinkPlugin creates a file sink through plugin factory
|
||||
func NewFileSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
// Create empty config struct
|
||||
opts := &config.FileSinkOptions{}
|
||||
|
||||
// Scan config map into struct
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate
|
||||
if err := lconfig.NonEmpty(opts.Directory); err != nil {
|
||||
return nil, fmt.Errorf("directory: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Name); err != nil {
|
||||
return nil, fmt.Errorf("name: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.MaxSizeMB <= 0 {
|
||||
opts.MaxSizeMB = DefaultFileMaxSizeMB
|
||||
}
|
||||
if opts.MaxTotalSizeMB <= 0 {
|
||||
opts.MaxTotalSizeMB = DefaultFileMaxTotalSizeMB
|
||||
}
|
||||
if opts.MinDiskFreeMB < 0 {
|
||||
opts.MinDiskFreeMB = DefaultFileMinDiskFreeMB
|
||||
}
|
||||
if opts.RetentionHours <= 0 {
|
||||
opts.RetentionHours = DefaultFileRetentionHours
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultFileBufferSize
|
||||
}
|
||||
if opts.FlushIntervalMs <= 0 {
|
||||
opts.FlushIntervalMs = DefaultFileFlushIntervalMs
|
||||
}
|
||||
|
||||
// Create configuration for the internal log writer
|
||||
writerConfig := log.DefaultConfig()
|
||||
writerConfig.Directory = opts.Directory
|
||||
writerConfig.Name = opts.Name
|
||||
writerConfig.MaxSizeKB = opts.MaxSizeMB * 1000
|
||||
writerConfig.MaxTotalSizeKB = opts.MaxTotalSizeMB * 1000
|
||||
writerConfig.MinDiskFreeKB = opts.MinDiskFreeMB * 1000
|
||||
writerConfig.RetentionPeriodHrs = opts.RetentionHours
|
||||
writerConfig.BufferSize = opts.BufferSize
|
||||
writerConfig.FlushIntervalMs = opts.FlushIntervalMs
|
||||
// Sink logic
|
||||
writerConfig.EnableConsole = false
|
||||
writerConfig.EnableFile = true
|
||||
writerConfig.ShowTimestamp = false
|
||||
writerConfig.ShowLevel = false
|
||||
writerConfig.Format = "raw"
|
||||
|
||||
// Create internal logger for file writing
|
||||
writer := log.NewLogger()
|
||||
if err := writer.ApplyConfig(writerConfig); err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize file writer: %w", err)
|
||||
}
|
||||
|
||||
fs := &FileSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
writer: writer,
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
fs.lastProcessed.Store(time.Time{})
|
||||
|
||||
// Create session for file output
|
||||
fs.session = proxy.CreateSession(
|
||||
fmt.Sprintf("file:///%s/%s", opts.Directory, opts.Name),
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "file",
|
||||
"directory": opts.Directory,
|
||||
"name": opts.Name,
|
||||
},
|
||||
)
|
||||
|
||||
fs.logger.Info("msg", "File sink initialized",
|
||||
"component", "file_sink",
|
||||
"instance_id", id,
|
||||
"directory", opts.Directory,
|
||||
"name", opts.Name)
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (fs *FileSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware, // Single output session
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (fs *FileSink) Input() chan<- core.TransportEvent {
|
||||
return fs.input
|
||||
}
|
||||
|
||||
// Start begins the processing loop for the sink
|
||||
func (fs *FileSink) Start(ctx context.Context) error {
|
||||
// Start the internal file writer
|
||||
if err := fs.writer.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start file writer: %w", err)
|
||||
}
|
||||
|
||||
fs.startTime = time.Now()
|
||||
go fs.processLoop(ctx)
|
||||
|
||||
fs.logger.Info("msg", "File sink started",
|
||||
"component", "file_sink",
|
||||
)
|
||||
fs.logger.Debug("msg", "File sink config",
|
||||
"component", "file_sink",
|
||||
"directory", fs.config.Directory,
|
||||
"name", fs.config.Name,
|
||||
"max_size_mb", fs.config.MaxSizeMB,
|
||||
"max_total_size_mb", fs.config.MaxTotalSizeMB,
|
||||
"min_disk_free_mb", fs.config.MinDiskFreeMB,
|
||||
"retention_hours", fs.config.RetentionHours,
|
||||
"buffer_size", fs.config.BufferSize,
|
||||
"flush_interval_ms", fs.config.FlushIntervalMs,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (fs *FileSink) Stop() {
|
||||
fs.logger.Info("msg", "Stopping file sink",
|
||||
"component", "file_sink",
|
||||
"directory", fs.config.Directory,
|
||||
"name", fs.config.Name)
|
||||
|
||||
close(fs.done)
|
||||
|
||||
// Remove session
|
||||
if fs.session != nil {
|
||||
fs.proxy.RemoveSession(fs.session.ID)
|
||||
}
|
||||
|
||||
// Shutdown the writer with timeout
|
||||
if err := fs.writer.Shutdown(core.LoggerShutdownTimeout); err != nil {
|
||||
fs.logger.Error("msg", "Error shutting down file writer",
|
||||
"component", "file_sink",
|
||||
"error", err)
|
||||
}
|
||||
|
||||
fs.logger.Info("msg", "File sink stopped",
|
||||
"component", "file_sink",
|
||||
"instance_id", fs.id,
|
||||
"total_processed", fs.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns the sink's statistics
|
||||
func (fs *FileSink) GetStats() sink.SinkStats {
|
||||
return sink.SinkStats{
|
||||
ID: fs.id,
|
||||
Type: "file",
|
||||
TotalProcessed: fs.totalProcessed.Load(),
|
||||
StartTime: fs.startTime,
|
||||
LastProcessed: fs.lastProcessed.Load().(time.Time),
|
||||
Details: map[string]any{
|
||||
"directory": fs.config.Directory,
|
||||
"name": fs.config.Name,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop reads transport events and writes to file
|
||||
func (fs *FileSink) processLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-fs.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Write the pre-formatted payload directly
|
||||
// The writer handles rotation automatically based on configuration
|
||||
fs.writer.Write(string(event.Payload))
|
||||
|
||||
fs.totalProcessed.Add(1)
|
||||
fs.lastProcessed.Store(time.Now())
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
case <-fs.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,545 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
"logwisp/internal/version"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/lixenwraith/log/compat"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("http", NewHTTPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// HTTPSink streams log entries via Server-Sent Events (SSE)
|
||||
type HTTPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.HTTPSinkOptions
|
||||
|
||||
// Network
|
||||
server *fasthttp.Server
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
// Broker
|
||||
clients map[uint64]chan []byte
|
||||
clientsMu sync.RWMutex
|
||||
unregister chan uint64
|
||||
nextClientID atomic.Uint64
|
||||
|
||||
// Client session tracking
|
||||
clientSessions map[uint64]string // clientID -> sessionID
|
||||
sessionsMu sync.RWMutex
|
||||
|
||||
// Statistics
|
||||
activeClients atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
// Server lifecycle
|
||||
HttpServerStartTimeout = 100 * time.Millisecond
|
||||
HttpServerShutdownTimeout = 2 * time.Second
|
||||
|
||||
// Defaults
|
||||
DefaultHTTPHost = "0.0.0.0"
|
||||
DefaultHTTPBufferSize = 1000
|
||||
DefaultHTTPStreamPath = "/stream"
|
||||
DefaultHTTPStatusPath = "/status"
|
||||
HTTPMaxPort = 65535
|
||||
)
|
||||
|
||||
// NewHTTPSinkPlugin creates an HTTP sink through plugin factory
|
||||
func NewHTTPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.HTTPSinkOptions{
|
||||
Host: DefaultHTTPHost,
|
||||
Port: 0,
|
||||
WriteTimeout: 0, // SSE indefinite streaming
|
||||
}
|
||||
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate
|
||||
if opts.Port <= 0 || opts.Port > HTTPMaxPort {
|
||||
return nil, fmt.Errorf("port must be between 1 and %d", HTTPMaxPort)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPBufferSize
|
||||
}
|
||||
if opts.StreamPath == "" {
|
||||
opts.StreamPath = DefaultHTTPStreamPath
|
||||
}
|
||||
if opts.StatusPath == "" {
|
||||
opts.StatusPath = DefaultHTTPStatusPath
|
||||
}
|
||||
|
||||
h := &HTTPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
clients: make(map[uint64]chan []byte),
|
||||
unregister: make(chan uint64),
|
||||
clientSessions: make(map[uint64]string),
|
||||
}
|
||||
h.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "HTTP sink initialized",
|
||||
"component", "http_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port,
|
||||
"stream_path", opts.StreamPath,
|
||||
"status_path", opts.StatusPath)
|
||||
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (h *HTTPSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (h *HTTPSink) Input() chan<- core.TransportEvent {
|
||||
return h.input
|
||||
}
|
||||
|
||||
// Start initializes the HTTP server and begins the broker loop
|
||||
func (h *HTTPSink) Start(ctx context.Context) error {
|
||||
h.startTime = time.Now()
|
||||
|
||||
// Start central broker goroutine
|
||||
h.wg.Add(1)
|
||||
go h.brokerLoop(ctx)
|
||||
|
||||
fasthttpLogger := compat.NewFastHTTPAdapter(h.logger)
|
||||
|
||||
h.server = &fasthttp.Server{
|
||||
Name: fmt.Sprintf("LogWisp/%s", version.Short()),
|
||||
Handler: h.requestHandler,
|
||||
DisableKeepalive: false,
|
||||
StreamRequestBody: true,
|
||||
Logger: fasthttpLogger,
|
||||
WriteTimeout: time.Duration(h.config.WriteTimeout) * time.Millisecond,
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", h.config.Host, h.config.Port)
|
||||
|
||||
ln, err := net.Listen("tcp4", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("http sink bind %s: %w", addr, err)
|
||||
}
|
||||
go func() {
|
||||
if err := h.server.Serve(ln); err != nil {
|
||||
h.logger.Error("msg", "HTTP server terminated",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Monitor context for shutdown
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
if h.server != nil {
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), HttpServerShutdownTimeout)
|
||||
defer cancel()
|
||||
h.server.ShutdownWithContext(shutdownCtx)
|
||||
}
|
||||
}()
|
||||
|
||||
h.logger.Info("msg", "HTTP server started",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"host", h.config.Host,
|
||||
"port", h.config.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the HTTP server and all client connections
|
||||
func (h *HTTPSink) Stop() {
|
||||
h.logger.Info("msg", "Stopping HTTP sink",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id)
|
||||
|
||||
close(h.done)
|
||||
|
||||
if h.server != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), HttpServerShutdownTimeout)
|
||||
defer cancel()
|
||||
h.server.ShutdownWithContext(ctx)
|
||||
}
|
||||
|
||||
h.wg.Wait()
|
||||
|
||||
close(h.unregister)
|
||||
|
||||
h.clientsMu.Lock()
|
||||
for _, ch := range h.clients {
|
||||
close(ch)
|
||||
}
|
||||
h.clients = make(map[uint64]chan []byte)
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
h.logger.Info("msg", "HTTP sink stopped",
|
||||
"component", "http_sink",
|
||||
"instance_id", h.id,
|
||||
"total_processed", h.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (h *HTTPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := h.lastProcessed.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: h.id,
|
||||
Type: "http",
|
||||
TotalProcessed: h.totalProcessed.Load(),
|
||||
ActiveConnections: h.activeClients.Load(),
|
||||
StartTime: h.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": h.config.Host,
|
||||
"port": h.config.Port,
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"endpoints": map[string]string{
|
||||
"stream": h.config.StreamPath,
|
||||
"status": h.config.StatusPath,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// brokerLoop manages client connections and broadcasts transport events
|
||||
func (h *HTTPSink) brokerLoop(ctx context.Context) {
|
||||
defer h.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
h.logger.Debug("msg", "Broker loop stopping due to context cancellation",
|
||||
"component", "http_sink")
|
||||
return
|
||||
|
||||
case <-h.done:
|
||||
h.logger.Debug("msg", "Broker loop stopping due to shutdown signal",
|
||||
"component", "http_sink")
|
||||
return
|
||||
|
||||
case clientID := <-h.unregister:
|
||||
h.clientsMu.Lock()
|
||||
if clientChan, exists := h.clients[clientID]; exists {
|
||||
delete(h.clients, clientID)
|
||||
close(clientChan)
|
||||
h.logger.Debug("msg", "Unregistered client",
|
||||
"component", "http_sink",
|
||||
"client_id", clientID)
|
||||
}
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
h.sessionsMu.Lock()
|
||||
delete(h.clientSessions, clientID)
|
||||
h.sessionsMu.Unlock()
|
||||
|
||||
case event, ok := <-h.input:
|
||||
if !ok {
|
||||
h.logger.Debug("msg", "Input channel closed, broker stopping",
|
||||
"component", "http_sink")
|
||||
return
|
||||
}
|
||||
|
||||
h.totalProcessed.Add(1)
|
||||
h.lastProcessed.Store(time.Now())
|
||||
|
||||
h.clientsMu.RLock()
|
||||
clientCount := len(h.clients)
|
||||
if clientCount > 0 {
|
||||
var staleClients []uint64
|
||||
|
||||
for id, ch := range h.clients {
|
||||
h.sessionsMu.RLock()
|
||||
sessionID, hasSession := h.clientSessions[id]
|
||||
h.sessionsMu.RUnlock()
|
||||
|
||||
if !hasSession {
|
||||
staleClients = append(staleClients, id)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check session still exists via proxy
|
||||
if _, exists := h.proxy.GetSession(sessionID); !exists {
|
||||
staleClients = append(staleClients, id)
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case ch <- event.Payload:
|
||||
h.proxy.UpdateActivity(sessionID)
|
||||
default:
|
||||
h.logger.Debug("msg", "Dropped event for slow client",
|
||||
"component", "http_sink",
|
||||
"client_id", id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(staleClients) > 0 {
|
||||
go func() {
|
||||
for _, clientID := range staleClients {
|
||||
select {
|
||||
case h.unregister <- clientID:
|
||||
case <-h.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
h.clientsMu.RUnlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// requestHandler is the main entry point for all incoming HTTP requests
|
||||
func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) {
|
||||
// IPv4-only enforcement - silent drop IPv6
|
||||
remoteAddr := ctx.RemoteAddr()
|
||||
if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok {
|
||||
if tcpAddr.IP.To4() == nil {
|
||||
h.logger.Debug("msg", "IPv6 connection rejected",
|
||||
"component", "http_sink", "remote_addr", remoteAddr.String())
|
||||
ctx.SetConnectionClose()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
path := string(ctx.Path())
|
||||
|
||||
switch path {
|
||||
case h.config.StatusPath:
|
||||
h.handleStatus(ctx)
|
||||
case h.config.StreamPath:
|
||||
h.handleStream(ctx)
|
||||
default:
|
||||
ctx.SetStatusCode(fasthttp.StatusNotFound)
|
||||
ctx.SetContentType("application/json")
|
||||
json.NewEncoder(ctx).Encode(map[string]any{
|
||||
"error": "Not Found",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleStream manages a client's Server-Sent Events (SSE) stream
|
||||
func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
|
||||
remoteAddrStr := ctx.RemoteAddr().String()
|
||||
|
||||
// Create session via proxy
|
||||
sess := h.proxy.CreateSession(remoteAddrStr, map[string]any{
|
||||
"type": "http_client",
|
||||
})
|
||||
|
||||
// Set SSE headers
|
||||
ctx.Response.Header.Set("Content-Type", "text/event-stream")
|
||||
ctx.Response.Header.Set("Cache-Control", "no-cache")
|
||||
ctx.Response.Header.Set("Connection", "keep-alive")
|
||||
ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
|
||||
ctx.Response.Header.Set("X-Accel-Buffering", "no")
|
||||
|
||||
// Register client with broker
|
||||
clientID := h.nextClientID.Add(1)
|
||||
clientChan := make(chan []byte, h.config.BufferSize)
|
||||
|
||||
h.clientsMu.Lock()
|
||||
h.clients[clientID] = clientChan
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
h.sessionsMu.Lock()
|
||||
h.clientSessions[clientID] = sess.ID
|
||||
h.sessionsMu.Unlock()
|
||||
|
||||
streamFunc := func(w *bufio.Writer) {
|
||||
connectCount := h.activeClients.Add(1)
|
||||
h.logger.Debug("msg", "HTTP client connected",
|
||||
"component", "http_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"session_id", sess.ID,
|
||||
"client_id", clientID,
|
||||
"active_clients", connectCount)
|
||||
|
||||
defer func() {
|
||||
disconnectCount := h.activeClients.Add(-1)
|
||||
h.logger.Debug("msg", "HTTP client disconnected",
|
||||
"component", "http_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"session_id", sess.ID,
|
||||
"client_id", clientID,
|
||||
"active_clients", disconnectCount)
|
||||
|
||||
select {
|
||||
case h.unregister <- clientID:
|
||||
case <-h.done:
|
||||
}
|
||||
|
||||
h.proxy.RemoveSession(sess.ID)
|
||||
}()
|
||||
|
||||
// Send connected event with metadata
|
||||
connectionInfo := map[string]any{
|
||||
"client_id": fmt.Sprintf("%d", clientID),
|
||||
"session_id": sess.ID,
|
||||
"instance_id": h.id,
|
||||
"stream_path": h.config.StreamPath,
|
||||
"status_path": h.config.StatusPath,
|
||||
"buffer_size": h.config.BufferSize,
|
||||
}
|
||||
data, _ := json.Marshal(connectionInfo)
|
||||
fmt.Fprintf(w, "event: connected\ndata: %s\n\n", data)
|
||||
if err := w.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case payload, ok := <-clientChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.writeSSE(w, payload); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := w.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.proxy.UpdateActivity(sess.ID)
|
||||
|
||||
case <-h.done:
|
||||
fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
|
||||
w.Flush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ctx.SetBodyStreamWriter(streamFunc)
|
||||
}
|
||||
|
||||
// handleStatus provides a JSON status report
|
||||
func (h *HTTPSink) handleStatus(ctx *fasthttp.RequestCtx) {
|
||||
ctx.SetContentType("application/json")
|
||||
|
||||
status := map[string]any{
|
||||
"service": "LogWisp",
|
||||
"version": version.Short(),
|
||||
"instance_id": h.id,
|
||||
"server": map[string]any{
|
||||
"type": "http",
|
||||
"host": h.config.Host,
|
||||
"port": h.config.Port,
|
||||
"active_clients": h.activeClients.Load(),
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"uptime_seconds": int(time.Since(h.startTime).Seconds()),
|
||||
},
|
||||
"endpoints": map[string]string{
|
||||
"stream": h.config.StreamPath,
|
||||
"status": h.config.StatusPath,
|
||||
},
|
||||
"statistics": map[string]any{
|
||||
"total_processed": h.totalProcessed.Load(),
|
||||
},
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(status)
|
||||
ctx.SetBody(data)
|
||||
}
|
||||
|
||||
// writeSSE formats payload into SSE data format
|
||||
func (h *HTTPSink) writeSSE(w *bufio.Writer, payload []byte) error {
|
||||
// Handle multi-line payloads per W3C SSE spec
|
||||
lines := splitLines(payload)
|
||||
for _, line := range lines {
|
||||
if _, err := fmt.Fprintf(w, "data: %s\n", line); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Empty line terminates event
|
||||
if _, err := w.WriteString("\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// splitLines splits payload by newlines, handling different line endings
|
||||
func splitLines(data []byte) [][]byte {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Trim trailing newline if present
|
||||
if data[len(data)-1] == '\n' {
|
||||
data = data[:len(data)-1]
|
||||
}
|
||||
|
||||
var lines [][]byte
|
||||
start := 0
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] == '\n' {
|
||||
lines = append(lines, data[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
if start < len(data) {
|
||||
lines = append(lines, data[start:])
|
||||
}
|
||||
|
||||
if len(lines) == 0 {
|
||||
return [][]byte{data}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/chain"
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("http_chain", NewHTTPChainSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register http_chain sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultHTTPChainSinkBufferSize = 1000
|
||||
DefaultHTTPChainSinkIngestPath = "/ingest"
|
||||
DefaultHTTPChainSinkMaxBatchCount = 100
|
||||
DefaultHTTPChainSinkMaxBatchBytes = 1024 * 1024
|
||||
DefaultHTTPChainSinkFlushIntervalMS = 1000
|
||||
DefaultHTTPChainSinkRequestTimeoutMS = 10000
|
||||
DefaultHTTPChainSinkBackoffMinMS = 500
|
||||
DefaultHTTPChainSinkBackoffMaxMS = 30000
|
||||
)
|
||||
|
||||
// HTTPChainSink batches structured entries and posts NDJSON to a downstream
|
||||
// http_chain source. Delivery is at-least-once per batch.
|
||||
type HTTPChainSink struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
config *config.HTTPChainSinkOptions
|
||||
|
||||
node string
|
||||
url string
|
||||
|
||||
client *http.Client
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Batch state owned exclusively by run loop goroutine
|
||||
batch bytes.Buffer
|
||||
batchCount int64
|
||||
|
||||
reqTimeout time.Duration
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
totalProcessed atomic.Uint64
|
||||
batchesSent atomic.Uint64
|
||||
requestErrors atomic.Uint64
|
||||
droppedBatches atomic.Uint64
|
||||
synthesized atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewHTTPChainSinkPlugin creates an http_chain sink through plugin factory
|
||||
func NewHTTPChainSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.HTTPChainSinkOptions{}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Host); err != nil {
|
||||
return nil, fmt.Errorf("host: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.IngestPath == "" {
|
||||
opts.IngestPath = DefaultHTTPChainSinkIngestPath
|
||||
} else if !strings.HasPrefix(opts.IngestPath, "/") {
|
||||
return nil, fmt.Errorf("ingest_path: must start with '/'")
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultHTTPChainSinkBufferSize
|
||||
}
|
||||
if opts.MaxBatchCount <= 0 {
|
||||
opts.MaxBatchCount = DefaultHTTPChainSinkMaxBatchCount
|
||||
}
|
||||
if opts.MaxBatchBytes <= 0 {
|
||||
opts.MaxBatchBytes = DefaultHTTPChainSinkMaxBatchBytes
|
||||
}
|
||||
if opts.FlushIntervalMS <= 0 {
|
||||
opts.FlushIntervalMS = DefaultHTTPChainSinkFlushIntervalMS
|
||||
}
|
||||
if opts.RequestTimeoutMS <= 0 {
|
||||
opts.RequestTimeoutMS = DefaultHTTPChainSinkRequestTimeoutMS
|
||||
}
|
||||
if opts.BackoffMinMS <= 0 {
|
||||
opts.BackoffMinMS = DefaultHTTPChainSinkBackoffMinMS
|
||||
}
|
||||
if opts.BackoffMaxMS < opts.BackoffMinMS {
|
||||
opts.BackoffMaxMS = DefaultHTTPChainSinkBackoffMaxMS
|
||||
}
|
||||
|
||||
node := opts.Node
|
||||
if node == "" {
|
||||
if hn, err := os.Hostname(); err == nil {
|
||||
node = hn
|
||||
} else {
|
||||
node = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
|
||||
|
||||
transport := &http.Transport{
|
||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
// IPv4-only, aligns with tcp/http sinks
|
||||
d := net.Dialer{}
|
||||
return d.DialContext(ctx, "tcp4", address)
|
||||
},
|
||||
MaxIdleConnsPerHost: 2,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
DisableCompression: true,
|
||||
// Future: TLSClientConfig; HTTP/2 via ALPN once TLS lands
|
||||
}
|
||||
|
||||
t := &HTTPChainSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
node: node,
|
||||
// Future: "https" scheme with TLS
|
||||
url: "http://" + addr + opts.IngestPath,
|
||||
client: &http.Client{Transport: transport},
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
reqTimeout: time.Duration(opts.RequestTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
t.session = proxy.CreateSession(
|
||||
"http_chain://"+addr,
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "http_chain",
|
||||
"target": t.url,
|
||||
"node": node,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Info("msg", "HTTP chain sink initialized",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", id,
|
||||
"target", t.url,
|
||||
"node", node)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *HTTPChainSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *HTTPChainSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start launches the batching loop; downstream availability is not required
|
||||
func (t *HTTPChainSink) Start(ctx context.Context) error {
|
||||
t.startTime = time.Now()
|
||||
t.wg.Add(1)
|
||||
go t.runLoop(ctx)
|
||||
|
||||
t.logger.Info("msg", "HTTP chain sink started",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"target", t.url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the loop. Worst case: one in-flight request timeout plus
|
||||
// one final-flush request timeout.
|
||||
func (t *HTTPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping HTTP chain sink",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
t.client.CloseIdleConnections()
|
||||
|
||||
if t.session != nil {
|
||||
t.proxy.RemoveSession(t.session.ID)
|
||||
}
|
||||
|
||||
t.logger.Info("msg", "HTTP chain sink stopped",
|
||||
"component", "http_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *HTTPChainSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "http_chain",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": t.url,
|
||||
"node": t.node,
|
||||
"batches_sent": t.batchesSent.Load(),
|
||||
"request_errors": t.requestErrors.Load(),
|
||||
"dropped_batches": t.droppedBatches.Load(),
|
||||
"synthesized": t.synthesized.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runLoop batches events and flushes on size or interval
|
||||
func (t *HTTPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
|
||||
// Fold done channel into a context for request/backoff interruption
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-runCtx.Done():
|
||||
t.finalFlush()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if t.batchCount > 0 && !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
t.append(event)
|
||||
if t.batchCount >= t.config.MaxBatchCount ||
|
||||
int64(t.batch.Len()) >= t.config.MaxBatchBytes {
|
||||
if !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// append serializes one event into the pending batch
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop entry
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "http_chain_sink",
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batch.Write(line)
|
||||
t.batch.WriteByte('\n')
|
||||
t.batchCount++
|
||||
}
|
||||
|
||||
// flush delivers the pending batch, retrying transient failures with backoff.
|
||||
// Returns false when shutdown interrupts delivery; undelivered batch is dropped
|
||||
// by finalFlush semantics (batch already consumed here).
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
body := bytes.Clone(t.batch.Bytes())
|
||||
count := t.batchCount
|
||||
t.batch.Reset()
|
||||
t.batchCount = 0
|
||||
|
||||
failures := 0
|
||||
for {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
}
|
||||
transient, err := t.post(ctx, body)
|
||||
if err == nil {
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
return true
|
||||
}
|
||||
t.requestErrors.Add(1)
|
||||
if !transient {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Error("msg", "Chain batch rejected, dropping",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return true
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
}
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain batch delivery failed",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
|
||||
func (t *HTTPChainSink) finalFlush() {
|
||||
if t.batchCount == 0 {
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
count := t.batchCount
|
||||
if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Warn("msg", "Final chain batch dropped on shutdown",
|
||||
"component", "http_chain_sink",
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
}
|
||||
|
||||
// post sends one NDJSON batch; transient=true marks retryable failures
|
||||
func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool, err error) {
|
||||
reqCtx, cancel := context.WithTimeout(ctx, t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, t.url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("Content-Type", chain.ContentTypeNDJSON)
|
||||
req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion))
|
||||
req.Header.Set(chain.HeaderNode, t.node)
|
||||
// Future: Authorization header for auth
|
||||
|
||||
resp, err := t.client.Do(req)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
// Drain for connection reuse
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
|
||||
switch {
|
||||
case resp.StatusCode >= 200 && resp.StatusCode < 300:
|
||||
return false, nil
|
||||
case resp.StatusCode == http.StatusRequestTimeout,
|
||||
resp.StatusCode == http.StatusTooManyRequests,
|
||||
resp.StatusCode >= 500:
|
||||
return true, fmt.Errorf("status %s", resp.Status)
|
||||
default:
|
||||
return false, fmt.Errorf("status %s", resp.Status)
|
||||
}
|
||||
}
|
||||
|
||||
// waitBackoff sleeps for the computed delay, interruptible by shutdown
|
||||
func (t *HTTPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package null
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// init registers the component in plugin factory
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("null", NewNullSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register null sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// NullSink discards all received transport events, used for testing
|
||||
type NullSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
totalReceived atomic.Uint64
|
||||
totalBytes atomic.Uint64
|
||||
lastReceived atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewNullSinkPlugin creates a null sink through plugin factory
|
||||
func NewNullSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
ns := &NullSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
input: make(chan core.TransportEvent, 1000),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
}
|
||||
ns.lastReceived.Store(time.Time{})
|
||||
|
||||
// Create session for null sink
|
||||
ns.session = proxy.CreateSession(
|
||||
"null://devnull",
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "null",
|
||||
},
|
||||
)
|
||||
|
||||
logger.Debug("msg", "Null sink initialized",
|
||||
"component", "null_sink",
|
||||
"instance_id", id)
|
||||
|
||||
return ns, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (ns *NullSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (ns *NullSink) Input() chan<- core.TransportEvent {
|
||||
return ns.input
|
||||
}
|
||||
|
||||
// Start begins the processing loop
|
||||
func (ns *NullSink) Start(ctx context.Context) error {
|
||||
|
||||
ns.startTime = time.Now()
|
||||
go ns.processLoop(ctx)
|
||||
ns.logger.Debug("msg", "Null sink started",
|
||||
"component", "null_sink",
|
||||
"instance_id", ns.id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (ns *NullSink) Stop() {
|
||||
if ns.session != nil {
|
||||
ns.proxy.RemoveSession(ns.session.ID)
|
||||
}
|
||||
close(ns.done)
|
||||
ns.logger.Debug("msg", "Null sink stopped",
|
||||
"instance_id", ns.id,
|
||||
"total_received", ns.totalReceived.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (ns *NullSink) GetStats() sink.SinkStats {
|
||||
lastRcv, _ := ns.lastReceived.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: ns.id,
|
||||
Type: "null",
|
||||
TotalProcessed: ns.totalReceived.Load(),
|
||||
StartTime: ns.startTime,
|
||||
LastProcessed: lastRcv,
|
||||
Details: map[string]any{
|
||||
"total_bytes": ns.totalBytes.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// processLoop reads transport events and discards them
|
||||
func (ns *NullSink) processLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-ns.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
// Discard the event, only update stats
|
||||
ns.totalReceived.Add(1)
|
||||
ns.totalBytes.Add(uint64(len(event.Payload)))
|
||||
ns.lastReceived.Store(time.Now())
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ns.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package sink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/core"
|
||||
)
|
||||
|
||||
// Sink represents an output data stream.
|
||||
type Sink interface {
|
||||
// Capabilities returns a slice of supported Source capabilities
|
||||
Capabilities() []core.Capability
|
||||
|
||||
// Input returns the channel for sending transport events to this sink.
|
||||
Input() chan<- core.TransportEvent
|
||||
|
||||
// Start begins processing transport events.
|
||||
Start(ctx context.Context) error
|
||||
|
||||
// Stop gracefully shuts down the sink.
|
||||
Stop()
|
||||
|
||||
// GetStats returns sink statistics.
|
||||
GetStats() SinkStats
|
||||
}
|
||||
|
||||
// SinkStats contains statistics about a sink.
|
||||
type SinkStats struct {
|
||||
ID string
|
||||
Type string
|
||||
TotalProcessed uint64
|
||||
ActiveConnections int64
|
||||
StartTime time.Time
|
||||
LastProcessed time.Time
|
||||
Details map[string]any
|
||||
}
|
||||
@@ -0,0 +1,493 @@
|
||||
package tcp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/lixenwraith/log/compat"
|
||||
"github.com/panjf2000/gnet/v2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("tcp", NewTCPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
// TCPSink streams log entries to connected TCP clients
|
||||
type TCPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.TCPSinkOptions
|
||||
|
||||
// Network
|
||||
server *tcpServer
|
||||
engine *gnet.Engine
|
||||
engineMu sync.Mutex
|
||||
booted chan struct{}
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
// Statistics
|
||||
activeConns atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
|
||||
// Error tracking
|
||||
writeErrors atomic.Uint64
|
||||
consecutiveWriteErrors map[gnet.Conn]int
|
||||
errorMu sync.Mutex
|
||||
}
|
||||
|
||||
const (
|
||||
// Server lifecycle
|
||||
TCPServerStartTimeout = 2 * time.Second
|
||||
TCPServerShutdownTimeout = 2 * time.Second
|
||||
|
||||
// Connection management
|
||||
TCPMaxConsecutiveWriteErrors = 3
|
||||
TCPMaxPort = 65535
|
||||
|
||||
// Defaults
|
||||
DefaultTCPHost = "0.0.0.0"
|
||||
DefaultTCPBufferSize = 1000
|
||||
DefaultTCPWriteTimeoutMS = 5000
|
||||
DefaultTCPKeepAlivePeriod = 30000
|
||||
)
|
||||
|
||||
// NewTCPSinkPlugin creates a TCP sink through plugin factory
|
||||
func NewTCPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
// Create config struct with defaults
|
||||
opts := &config.TCPSinkOptions{
|
||||
Host: DefaultTCPHost,
|
||||
Port: 0,
|
||||
KeepAlive: true,
|
||||
}
|
||||
|
||||
// Parse config map into struct
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
|
||||
// Validate
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
|
||||
// Defaults
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultTCPBufferSize
|
||||
}
|
||||
if opts.WriteTimeout <= 0 {
|
||||
opts.WriteTimeout = DefaultTCPWriteTimeoutMS
|
||||
}
|
||||
if opts.KeepAlivePeriod <= 0 {
|
||||
opts.KeepAlivePeriod = DefaultTCPKeepAlivePeriod
|
||||
}
|
||||
|
||||
t := &TCPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
consecutiveWriteErrors: make(map[gnet.Conn]int),
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "TCP sink initialized",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port)
|
||||
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *TCPSink) Capabilities() []core.Capability {
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *TCPSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start initializes the TCP server and begins the broadcast loop
|
||||
func (t *TCPSink) Start(ctx context.Context) error {
|
||||
t.server = &tcpServer{
|
||||
sink: t,
|
||||
clients: make(map[gnet.Conn]*tcpClient),
|
||||
}
|
||||
// Fresh channel per Start
|
||||
t.booted = make(chan struct{})
|
||||
|
||||
t.startTime = time.Now()
|
||||
|
||||
// Start broadcast loop
|
||||
t.wg.Add(1)
|
||||
go func() {
|
||||
defer t.wg.Done()
|
||||
t.broadcastLoop(ctx)
|
||||
}()
|
||||
|
||||
// Configure gnet
|
||||
addr := fmt.Sprintf("tcp://%s:%d", t.config.Host, t.config.Port)
|
||||
gnetLogger := compat.NewGnetAdapter(t.logger)
|
||||
|
||||
opts := []gnet.Option{
|
||||
gnet.WithLogger(gnetLogger),
|
||||
gnet.WithMulticore(true),
|
||||
gnet.WithReusePort(true),
|
||||
}
|
||||
|
||||
// Apply TCP keep-alive settings from config
|
||||
if t.config.KeepAlive {
|
||||
opts = append(opts,
|
||||
gnet.WithTCPKeepAlive(time.Duration(t.config.KeepAlivePeriod)*time.Millisecond),
|
||||
)
|
||||
}
|
||||
|
||||
// Start gnet server
|
||||
errChan := make(chan error, 1)
|
||||
go func() {
|
||||
t.logger.Info("msg", "Starting TCP server",
|
||||
"component", "tcp_sink",
|
||||
"host", t.config.Host,
|
||||
"port", t.config.Port)
|
||||
|
||||
err := gnet.Run(t.server, addr, opts...)
|
||||
if err != nil {
|
||||
t.logger.Error("msg", "TCP server failed",
|
||||
"component", "tcp_sink",
|
||||
"error", err)
|
||||
}
|
||||
errChan <- err
|
||||
}()
|
||||
|
||||
// Monitor context for shutdown
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
t.engineMu.Lock()
|
||||
if t.engine != nil {
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
(*t.engine).Stop(shutdownCtx)
|
||||
}
|
||||
t.engineMu.Unlock()
|
||||
}()
|
||||
|
||||
// Wait briefly for server to start or fail
|
||||
select {
|
||||
case err := <-errChan:
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
return err
|
||||
// Bind confirmation via OnBoot
|
||||
case <-t.booted:
|
||||
t.logger.Info("msg", "TCP server started",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id,
|
||||
"port", t.config.Port)
|
||||
return nil
|
||||
// Timeout failure
|
||||
case <-time.After(TCPServerStartTimeout):
|
||||
t.engineMu.Lock()
|
||||
if t.engine != nil {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout)
|
||||
(*t.engine).Stop(stopCtx)
|
||||
cancel()
|
||||
}
|
||||
t.engineMu.Unlock()
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
return fmt.Errorf("tcp sink start timeout on %s", addr)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the TCP sink
|
||||
func (t *TCPSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP sink",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
|
||||
// Stop gnet engine
|
||||
t.engineMu.Lock()
|
||||
engine := t.engine
|
||||
t.engineMu.Unlock()
|
||||
|
||||
if engine != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout)
|
||||
defer cancel()
|
||||
(*engine).Stop(ctx)
|
||||
}
|
||||
|
||||
t.wg.Wait()
|
||||
|
||||
t.logger.Info("msg", "TCP sink stopped",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *TCPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "tcp",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: t.activeConns.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": t.config.Host,
|
||||
"port": t.config.Port,
|
||||
"buffer_size": t.config.BufferSize,
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// tcpServer implements gnet.EventHandler
|
||||
type tcpServer struct {
|
||||
gnet.BuiltinEventEngine
|
||||
sink *TCPSink
|
||||
clients map[gnet.Conn]*tcpClient
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// tcpClient represents a connected TCP client
|
||||
type tcpClient struct {
|
||||
conn gnet.Conn
|
||||
buffer bytes.Buffer
|
||||
sessionID string
|
||||
}
|
||||
|
||||
// broadcastLoop sends transport events to all connected clients
|
||||
func (t *TCPSink) broadcastLoop(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.broadcastData(event.Payload)
|
||||
case <-t.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// OnBoot is called when the server starts
|
||||
func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action {
|
||||
s.sink.engineMu.Lock()
|
||||
s.sink.engine = &eng
|
||||
s.sink.engineMu.Unlock()
|
||||
|
||||
// Listener is bound at this point; unblock Start
|
||||
close(s.sink.booted)
|
||||
|
||||
s.sink.logger.Debug("msg", "TCP server booted",
|
||||
"component", "tcp_sink",
|
||||
"instance_id", s.sink.id)
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
// OnOpen is called when a new connection is established
|
||||
func (s *tcpServer) OnOpen(c gnet.Conn) (out []byte, action gnet.Action) {
|
||||
remoteAddr := c.RemoteAddr()
|
||||
remoteAddrStr := remoteAddr.String()
|
||||
|
||||
s.sink.logger.Debug("msg", "TCP connection attempt",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr)
|
||||
|
||||
// Reject IPv6 connections
|
||||
if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok {
|
||||
if tcpAddr.IP.To4() == nil {
|
||||
s.sink.logger.Warn("msg", "IPv6 connection rejected",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr)
|
||||
return []byte("IPv4-only (IPv6 not supported)\n"), gnet.Close
|
||||
}
|
||||
}
|
||||
|
||||
// Apply write timeout from config
|
||||
if s.sink.config.WriteTimeout > 0 {
|
||||
c.SetWriteDeadline(time.Now().Add(time.Duration(s.sink.config.WriteTimeout) * time.Millisecond))
|
||||
}
|
||||
|
||||
// Create session via proxy
|
||||
sess := s.sink.proxy.CreateSession(remoteAddrStr, map[string]any{
|
||||
"type": "tcp_client",
|
||||
"remote_addr": remoteAddrStr,
|
||||
})
|
||||
|
||||
client := &tcpClient{
|
||||
conn: c,
|
||||
sessionID: sess.ID,
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.clients[c] = client
|
||||
s.mu.Unlock()
|
||||
|
||||
newCount := s.sink.activeConns.Add(1)
|
||||
s.sink.logger.Debug("msg", "TCP connection opened",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"session_id", sess.ID,
|
||||
"active_connections", newCount)
|
||||
|
||||
return nil, gnet.None
|
||||
}
|
||||
|
||||
// OnClose is called when a connection is closed
|
||||
func (s *tcpServer) OnClose(c gnet.Conn, err error) gnet.Action {
|
||||
remoteAddrStr := c.RemoteAddr().String()
|
||||
|
||||
s.mu.RLock()
|
||||
client, exists := s.clients[c]
|
||||
s.mu.RUnlock()
|
||||
|
||||
if exists && client.sessionID != "" {
|
||||
s.sink.proxy.RemoveSession(client.sessionID)
|
||||
s.sink.logger.Debug("msg", "Session removed",
|
||||
"component", "tcp_sink",
|
||||
"session_id", client.sessionID,
|
||||
"remote_addr", remoteAddrStr)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
delete(s.clients, c)
|
||||
s.mu.Unlock()
|
||||
|
||||
s.sink.errorMu.Lock()
|
||||
delete(s.sink.consecutiveWriteErrors, c)
|
||||
s.sink.errorMu.Unlock()
|
||||
|
||||
newCount := s.sink.activeConns.Add(-1)
|
||||
s.sink.logger.Debug("msg", "TCP connection closed",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"active_connections", newCount,
|
||||
"error", err)
|
||||
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
// OnTraffic is called when data is received from a connection
|
||||
func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
|
||||
s.mu.RLock()
|
||||
client, exists := s.clients[c]
|
||||
s.mu.RUnlock()
|
||||
|
||||
// Update session activity
|
||||
if exists && client.sessionID != "" {
|
||||
s.sink.proxy.UpdateActivity(client.sessionID)
|
||||
}
|
||||
|
||||
// TCP sink doesn't expect data from clients, discard safely
|
||||
if bufLen := c.InboundBuffered(); bufLen > 0 {
|
||||
c.Next(bufLen)
|
||||
}
|
||||
return gnet.None
|
||||
}
|
||||
|
||||
// broadcastData sends data to all connected clients
|
||||
func (t *TCPSink) broadcastData(data []byte) {
|
||||
t.server.mu.RLock()
|
||||
defer t.server.mu.RUnlock()
|
||||
|
||||
for conn, client := range t.server.clients {
|
||||
// Update session activity
|
||||
if client.sessionID != "" {
|
||||
t.proxy.UpdateActivity(client.sessionID)
|
||||
}
|
||||
|
||||
// Refresh write deadline on each write if configured
|
||||
if t.config.WriteTimeout > 0 {
|
||||
conn.SetWriteDeadline(time.Now().Add(time.Duration(t.config.WriteTimeout) * time.Millisecond))
|
||||
}
|
||||
|
||||
conn.AsyncWrite(data, func(c gnet.Conn, err error) error {
|
||||
if err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
t.handleWriteError(c, err)
|
||||
} else {
|
||||
t.errorMu.Lock()
|
||||
delete(t.consecutiveWriteErrors, c)
|
||||
t.errorMu.Unlock()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// handleWriteError manages errors during async writes
|
||||
func (t *TCPSink) handleWriteError(c gnet.Conn, err error) {
|
||||
remoteAddrStr := c.RemoteAddr().String()
|
||||
|
||||
t.errorMu.Lock()
|
||||
defer t.errorMu.Unlock()
|
||||
|
||||
t.consecutiveWriteErrors[c]++
|
||||
errorCount := t.consecutiveWriteErrors[c]
|
||||
|
||||
t.logger.Debug("msg", "AsyncWrite error",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"error", err,
|
||||
"consecutive_errors", errorCount)
|
||||
|
||||
// Close connection max consecutive write errors
|
||||
if errorCount >= TCPMaxConsecutiveWriteErrors {
|
||||
t.logger.Warn("msg", "Closing connection due to repeated write errors",
|
||||
"component", "tcp_sink",
|
||||
"remote_addr", remoteAddrStr,
|
||||
"error_count", errorCount)
|
||||
delete(t.consecutiveWriteErrors, c)
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
package tcpchain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/chain"
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("tcp_chain", NewTCPChainSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register tcp_chain sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultChainSinkBufferSize = 1000
|
||||
DefaultChainSinkDialTimeoutMS = 5000
|
||||
DefaultChainSinkWriteTimeoutMS = 5000
|
||||
DefaultChainSinkBackoffMinMS = 500
|
||||
DefaultChainSinkBackoffMaxMS = 30000
|
||||
DefaultChainSinkKeepAlivePeriodMS = 30000
|
||||
)
|
||||
|
||||
// TCPChainSink forwards structured entries to a downstream tcp_chain source
|
||||
type TCPChainSink struct {
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
session *session.Session
|
||||
config *config.TCPChainSinkOptions
|
||||
|
||||
node string
|
||||
addr string
|
||||
helloLine []byte
|
||||
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// conn owned exclusively by run loop goroutine
|
||||
conn net.Conn
|
||||
everConnected bool
|
||||
dialTimeout time.Duration
|
||||
writeTimeout time.Duration
|
||||
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
totalProcessed atomic.Uint64
|
||||
writeErrors atomic.Uint64
|
||||
reconnects atomic.Uint64
|
||||
synthesized atomic.Uint64
|
||||
connected atomic.Bool
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// NewTCPChainSinkPlugin creates a tcp_chain sink through plugin factory
|
||||
func NewTCPChainSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.TCPChainSinkOptions{
|
||||
KeepAlive: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.NonEmpty(opts.Host); err != nil {
|
||||
return nil, fmt.Errorf("host: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultChainSinkBufferSize
|
||||
}
|
||||
if opts.DialTimeoutMS <= 0 {
|
||||
opts.DialTimeoutMS = DefaultChainSinkDialTimeoutMS
|
||||
}
|
||||
if opts.WriteTimeoutMS <= 0 {
|
||||
opts.WriteTimeoutMS = DefaultChainSinkWriteTimeoutMS
|
||||
}
|
||||
if opts.BackoffMinMS <= 0 {
|
||||
opts.BackoffMinMS = DefaultChainSinkBackoffMinMS
|
||||
}
|
||||
if opts.BackoffMaxMS < opts.BackoffMinMS {
|
||||
opts.BackoffMaxMS = DefaultChainSinkBackoffMaxMS
|
||||
}
|
||||
if opts.KeepAlivePeriodMS <= 0 {
|
||||
opts.KeepAlivePeriodMS = DefaultChainSinkKeepAlivePeriodMS
|
||||
}
|
||||
|
||||
node := opts.Node
|
||||
if node == "" {
|
||||
if hn, err := os.Hostname(); err == nil {
|
||||
node = hn
|
||||
} else {
|
||||
node = "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
helloLine, err := chain.EncodeHello(node)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hello: %w", err)
|
||||
}
|
||||
|
||||
t := &TCPChainSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
node: node,
|
||||
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
|
||||
helloLine: helloLine,
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
dialTimeout: time.Duration(opts.DialTimeoutMS) * time.Millisecond,
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
t.session = proxy.CreateSession(
|
||||
"tcp_chain://"+t.addr,
|
||||
map[string]any{
|
||||
"instance_id": id,
|
||||
"type": "tcp_chain",
|
||||
"target": t.addr,
|
||||
"node": node,
|
||||
},
|
||||
)
|
||||
|
||||
logger.Info("msg", "TCP chain sink initialized",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", id,
|
||||
"target", t.addr,
|
||||
"node", node)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *TCPChainSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth added when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *TCPChainSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// Start launches the forwarding loop; connection is established lazily so
|
||||
// pipeline start does not depend on downstream availability
|
||||
func (t *TCPChainSink) Start(ctx context.Context) error {
|
||||
t.startTime = time.Now()
|
||||
t.wg.Add(1)
|
||||
go t.runLoop(ctx)
|
||||
|
||||
t.logger.Info("msg", "TCP chain sink started",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"target", t.addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the forwarding loop. Worst-case latency: one write timeout
|
||||
// plus one backoff wait (both interruptible or bounded).
|
||||
func (t *TCPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP chain sink",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
close(t.done)
|
||||
t.wg.Wait()
|
||||
|
||||
if t.session != nil {
|
||||
t.proxy.RemoveSession(t.session.ID)
|
||||
}
|
||||
|
||||
t.logger.Info("msg", "TCP chain sink stopped",
|
||||
"component", "tcp_chain_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *TCPChainSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
var active int64
|
||||
if t.connected.Load() {
|
||||
active = 1
|
||||
}
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "tcp_chain",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: active,
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"target": t.addr,
|
||||
"node": t.node,
|
||||
"connected": t.connected.Load(),
|
||||
"reconnects": t.reconnects.Load(),
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
"synthesized": t.synthesized.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// runLoop consumes transport events and forwards them downstream
|
||||
func (t *TCPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
defer t.closeConn()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "tcp_chain_sink",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
if !t.deliver(ctx, append(line, '\n')) {
|
||||
return // shutdown during retry
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// toEntry extracts the structured entry, stamping node identity at first hop
|
||||
func (t *TCPChainSink) toEntry(event core.TransportEvent) core.LogEntry {
|
||||
entry := event.Entry
|
||||
if entry.Time.IsZero() {
|
||||
// Defensive: event without structured entry, wrap formatted payload
|
||||
t.synthesized.Add(1)
|
||||
entry = core.LogEntry{
|
||||
Time: event.Time,
|
||||
Source: t.id,
|
||||
Message: string(event.Payload),
|
||||
}
|
||||
}
|
||||
if entry.Node == "" {
|
||||
entry.Node = t.node
|
||||
}
|
||||
return entry
|
||||
}
|
||||
|
||||
// deliver writes one line, holding it across reconnects until sent or shutdown.
|
||||
// Backpressure during outage propagates to the pipeline dispatch drop counter.
|
||||
func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
|
||||
failures := 0
|
||||
for {
|
||||
if t.conn == nil {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
return false
|
||||
}
|
||||
if err := t.connect(ctx); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return false
|
||||
}
|
||||
failures++
|
||||
t.logger.Debug("msg", "Chain connect failed",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
t.conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
if _, err := t.conn.Write(line); err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain write failed",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"error", err)
|
||||
t.closeConn()
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// connect performs a single dial + hello attempt
|
||||
func (t *TCPChainSink) connect(ctx context.Context) error {
|
||||
d := net.Dialer{Timeout: t.dialTimeout}
|
||||
if t.config.KeepAlive {
|
||||
d.KeepAliveConfig = net.KeepAliveConfig{
|
||||
Enable: true,
|
||||
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
// IPv4-only
|
||||
conn, err := d.DialContext(ctx, "tcp4", t.addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
if _, err := conn.Write(t.helloLine); err != nil {
|
||||
conn.Close()
|
||||
return fmt.Errorf("hello: %w", err)
|
||||
}
|
||||
|
||||
t.conn = conn
|
||||
t.connected.Store(true)
|
||||
if t.everConnected {
|
||||
t.reconnects.Add(1)
|
||||
}
|
||||
t.everConnected = true
|
||||
|
||||
t.logger.Info("msg", "Chain link established",
|
||||
"component", "tcp_chain_sink",
|
||||
"target", t.addr,
|
||||
"node", t.node)
|
||||
return nil
|
||||
}
|
||||
|
||||
// closeConn tears down the current connection (run loop goroutine only)
|
||||
func (t *TCPChainSink) closeConn() {
|
||||
if t.conn != nil {
|
||||
t.conn.Close()
|
||||
t.conn = nil
|
||||
}
|
||||
t.connected.Store(false)
|
||||
}
|
||||
|
||||
// waitBackoff sleeps for the computed delay, interruptible by shutdown
|
||||
func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures))
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.done:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// backoffDelay computes exponential backoff with ±20% jitter
|
||||
func (t *TCPChainSink) backoffDelay(failures int) time.Duration {
|
||||
minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond
|
||||
maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond
|
||||
|
||||
d := maxD
|
||||
if failures < 63 {
|
||||
if v := minD << uint(failures-1); v > 0 && v < maxD {
|
||||
d = v
|
||||
}
|
||||
}
|
||||
return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1))
|
||||
}
|
||||
Reference in New Issue
Block a user