v0.14.0 tcp/http sinks with standard library added, chain aggregate test script added
This commit is contained in:
@@ -0,0 +1,399 @@
|
||||
package stdtcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("stdtcp", NewStdTCPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register stdtcp sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultStdTCPHost = "0.0.0.0"
|
||||
DefaultStdTCPBufferSize = 1000
|
||||
DefaultStdTCPClientBufferSize = 256
|
||||
DefaultStdTCPWriteTimeoutMS = 5000
|
||||
DefaultStdTCPKeepAlivePeriodMS = 30000
|
||||
)
|
||||
|
||||
// StdTCPSink streams formatted log entries to connected TCP clients using
|
||||
// only the standard library. Functional peer of the gnet-based tcp sink.
|
||||
//
|
||||
// Concurrency model: one broadcast loop fans out into bounded per-client
|
||||
// queues; each connection owns a writer goroutine (drains queue) and a
|
||||
// reader goroutine (disconnect detection). A stalled client drops events,
|
||||
// never the pipeline.
|
||||
type StdTCPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.StdTCPSinkOptions
|
||||
addr string
|
||||
|
||||
// Network
|
||||
listener net.Listener
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Client registry
|
||||
clients map[uint64]*tcpClient
|
||||
clientsMu sync.Mutex
|
||||
nextClientID atomic.Uint64
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
stopOnce sync.Once
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
writeTimeout time.Duration
|
||||
|
||||
// Statistics
|
||||
activeConns atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
writeErrors atomic.Uint64
|
||||
droppedWrites atomic.Uint64
|
||||
rejectedConns atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// tcpClient pairs a connection with its bounded send queue.
|
||||
// send is written by the broadcast loop (non-blocking) and drained by the
|
||||
// writer goroutine; closed signals reader-detected disconnect.
|
||||
type tcpClient struct {
|
||||
conn net.Conn
|
||||
send chan []byte
|
||||
sessionID string
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
// NewStdTCPSinkPlugin creates a stdtcp sink through plugin factory
|
||||
func NewStdTCPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.StdTCPSinkOptions{
|
||||
Host: DefaultStdTCPHost,
|
||||
KeepAlive: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultStdTCPBufferSize
|
||||
}
|
||||
if opts.ClientBufferSize <= 0 {
|
||||
opts.ClientBufferSize = DefaultStdTCPClientBufferSize
|
||||
}
|
||||
if opts.WriteTimeoutMS <= 0 {
|
||||
opts.WriteTimeoutMS = DefaultStdTCPWriteTimeoutMS
|
||||
}
|
||||
if opts.KeepAlivePeriodMS <= 0 {
|
||||
opts.KeepAlivePeriodMS = DefaultStdTCPKeepAlivePeriodMS
|
||||
}
|
||||
|
||||
t := &StdTCPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
clients: make(map[uint64]*tcpClient),
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "Std TCP sink initialized",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *StdTCPSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth appended when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *StdTCPSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// listen creates the server listener.
|
||||
// TLS extension point: wrap the returned listener with tls.NewListener here
|
||||
// once cert config lands; no other code path changes. mTLS peer identity is
|
||||
// then available via conn.(*tls.Conn).ConnectionState() in the auth hook.
|
||||
func (t *StdTCPSink) listen() (net.Listener, error) {
|
||||
lc := net.ListenConfig{}
|
||||
if t.config.KeepAlive {
|
||||
lc.KeepAliveConfig = net.KeepAliveConfig{
|
||||
Enable: true,
|
||||
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
// IPv4-only, parity with existing network sinks
|
||||
return lc.Listen(context.Background(), "tcp4", t.addr)
|
||||
}
|
||||
|
||||
// Start binds the listener and launches accept and broadcast loops
|
||||
func (t *StdTCPSink) Start(ctx context.Context) error {
|
||||
ln, err := t.listen()
|
||||
if err != nil {
|
||||
return fmt.Errorf("stdtcp sink bind %s: %w", t.addr, err)
|
||||
}
|
||||
t.listener = ln
|
||||
t.startTime = time.Now()
|
||||
|
||||
t.wg.Add(2)
|
||||
go t.acceptLoop()
|
||||
go t.broadcastLoop(ctx)
|
||||
|
||||
// Pipeline context cancellation mirrors gnet engine stop: cease accepting
|
||||
// and tear down existing connections
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.shutdown()
|
||||
case <-t.done:
|
||||
}
|
||||
}()
|
||||
|
||||
t.logger.Info("msg", "Std TCP server started",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", t.id,
|
||||
"addr", t.addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (t *StdTCPSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping std TCP sink",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
t.shutdown()
|
||||
t.wg.Wait()
|
||||
|
||||
t.logger.Info("msg", "Std TCP sink stopped",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// shutdown funnels ctx-cancel and Stop() teardown through a single path
|
||||
func (t *StdTCPSink) shutdown() {
|
||||
t.stopOnce.Do(func() {
|
||||
close(t.done)
|
||||
if t.listener != nil {
|
||||
t.listener.Close() // unblocks acceptLoop
|
||||
}
|
||||
t.clientsMu.Lock()
|
||||
for _, c := range t.clients {
|
||||
c.conn.Close() // unblocks per-connection readers
|
||||
}
|
||||
t.clientsMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// acceptLoop accepts client connections until listener close
|
||||
func (t *StdTCPSink) acceptLoop() {
|
||||
defer t.wg.Done()
|
||||
for {
|
||||
conn, err := t.listener.Accept()
|
||||
if err != nil {
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
return
|
||||
}
|
||||
t.logger.Warn("msg", "Accept error",
|
||||
"component", "stdtcp_sink",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if t.config.MaxConnections > 0 && t.activeConns.Load() >= t.config.MaxConnections {
|
||||
// Load/admit race can over-admit by a conn under burst; acceptable
|
||||
t.rejectedConns.Add(1)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
// Auth extension point: credential/peer verification runs here,
|
||||
// pre-registration (password preamble read or TLS peer cert check)
|
||||
|
||||
t.wg.Add(1)
|
||||
go t.handleConn(conn)
|
||||
}
|
||||
}
|
||||
|
||||
// handleConn registers the client and runs its writer; a companion reader
|
||||
// goroutine drains inbound bytes for disconnect detection
|
||||
func (t *StdTCPSink) handleConn(conn net.Conn) {
|
||||
defer t.wg.Done()
|
||||
remote := conn.RemoteAddr().String()
|
||||
|
||||
sess := t.proxy.CreateSession(remote, map[string]any{
|
||||
"type": "stdtcp_client",
|
||||
"remote_addr": remote,
|
||||
})
|
||||
|
||||
c := &tcpClient{
|
||||
conn: conn,
|
||||
send: make(chan []byte, t.config.ClientBufferSize),
|
||||
sessionID: sess.ID,
|
||||
closed: make(chan struct{}),
|
||||
}
|
||||
id := t.nextClientID.Add(1)
|
||||
|
||||
t.clientsMu.Lock()
|
||||
t.clients[id] = c
|
||||
t.clientsMu.Unlock()
|
||||
|
||||
count := t.activeConns.Add(1)
|
||||
t.logger.Debug("msg", "TCP connection opened",
|
||||
"component", "stdtcp_sink",
|
||||
"remote_addr", remote,
|
||||
"session_id", sess.ID,
|
||||
"active_connections", count)
|
||||
|
||||
defer func() {
|
||||
t.clientsMu.Lock()
|
||||
delete(t.clients, id)
|
||||
t.clientsMu.Unlock()
|
||||
conn.Close()
|
||||
<-c.closed // reader has exited
|
||||
t.proxy.RemoveSession(sess.ID)
|
||||
newCount := t.activeConns.Add(-1)
|
||||
t.logger.Debug("msg", "TCP connection closed",
|
||||
"component", "stdtcp_sink",
|
||||
"remote_addr", remote,
|
||||
"active_connections", newCount)
|
||||
}()
|
||||
|
||||
// Reader: sink is write-only; drain and discard inbound bytes to detect
|
||||
// disconnect and refresh session activity on client traffic
|
||||
go func() {
|
||||
defer close(c.closed)
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := conn.Read(buf)
|
||||
if n > 0 {
|
||||
t.proxy.UpdateActivity(sess.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Writer: synchronous stdlib write with deadline. A failed write means
|
||||
// the kernel buffer stayed full for the full deadline - connection is
|
||||
// dead or hopelessly stalled, so disconnect immediately (no gnet-style
|
||||
// consecutive-error counter needed for transient async callback errors).
|
||||
for {
|
||||
select {
|
||||
case data := <-c.send:
|
||||
if t.writeTimeout > 0 {
|
||||
conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
}
|
||||
if _, err := conn.Write(data); err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
t.logger.Debug("msg", "Write failed, closing client",
|
||||
"component", "stdtcp_sink",
|
||||
"remote_addr", remote,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.proxy.UpdateActivity(sess.ID)
|
||||
case <-c.closed:
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// broadcastLoop fans out transport events to all client queues, non-blocking
|
||||
func (t *StdTCPSink) broadcastLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
|
||||
t.clientsMu.Lock()
|
||||
for _, c := range t.clients {
|
||||
select {
|
||||
case c.send <- event.Payload:
|
||||
default:
|
||||
// Slow client: drop its event, never stall siblings
|
||||
t.droppedWrites.Add(1)
|
||||
}
|
||||
}
|
||||
t.clientsMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *StdTCPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "stdtcp",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: t.activeConns.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": t.config.Host,
|
||||
"port": t.config.Port,
|
||||
"buffer_size": t.config.BufferSize,
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
"dropped_writes": t.droppedWrites.Load(),
|
||||
"rejected_conns": t.rejectedConns.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user