v0.14.0 tcp/http sinks with standard library added, chain aggregate test script added
This commit is contained in:
@@ -16,6 +16,8 @@ import (
|
||||
_ "logwisp/internal/sink/http"
|
||||
_ "logwisp/internal/sink/httpchain"
|
||||
_ "logwisp/internal/sink/null"
|
||||
_ "logwisp/internal/sink/stdhttp"
|
||||
_ "logwisp/internal/sink/stdtcp"
|
||||
_ "logwisp/internal/sink/tcp"
|
||||
_ "logwisp/internal/sink/tcpchain"
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98
|
||||
github.com/lixenwraith/log v0.1.1-0.20251117213308-9ae1b6669bf3
|
||||
github.com/lixenwraith/log v0.1.1-0.20260717175128-82eea9846ccd
|
||||
github.com/panjf2000/gnet/v2 v2.10.0
|
||||
github.com/valyala/fasthttp v1.72.0
|
||||
)
|
||||
|
||||
@@ -12,6 +12,8 @@ github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98 h1:VEFo2WsYgM
|
||||
github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98/go.mod h1:J9ydxY7he4Dz+S59xKs/kDQ2YAv+zOLF05b3Rr2/ogE=
|
||||
github.com/lixenwraith/log v0.1.1-0.20251117213308-9ae1b6669bf3 h1:brSUhER7EZ28aMRFTSovZskiIoobUiAWlx+DYXYvWXQ=
|
||||
github.com/lixenwraith/log v0.1.1-0.20251117213308-9ae1b6669bf3/go.mod h1:MY59N65ltw/9uTqJKwCRKjxO6w3CApXsLXCVuaekbu0=
|
||||
github.com/lixenwraith/log v0.1.1-0.20260717175128-82eea9846ccd h1:06Rk4DLvJW1kdeclH5HwywizgtMI64PHNE/2sBeuiwA=
|
||||
github.com/lixenwraith/log v0.1.1-0.20260717175128-82eea9846ccd/go.mod h1:2+qURSVdWcX7REFH+3jUtIbtc+NtsAPecSvXLeGyK6U=
|
||||
github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10=
|
||||
github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY=
|
||||
github.com/panjf2000/gnet/v2 v2.10.0 h1:rC4jNF+jtXj/FH+8JOIQ3XxjD+yBunYBLKg9TE3dc4g=
|
||||
|
||||
@@ -291,6 +291,34 @@ type HTTPSinkOptions struct {
|
||||
WriteTimeout int64 `toml:"write_timeout_ms"`
|
||||
}
|
||||
|
||||
// StdTCPSinkOptions defines settings for a stdlib TCP streaming server sink.
|
||||
// TOML keys mirror TCPSinkOptions for drop-in type swap ("tcp" -> "stdtcp").
|
||||
type StdTCPSinkOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
BufferSize int64 `toml:"buffer_size"` // sink input queue
|
||||
ClientBufferSize int64 `toml:"client_buffer_size"` // per-client send queue
|
||||
WriteTimeoutMS int64 `toml:"write_timeout_ms"` // per-write deadline
|
||||
KeepAlive bool `toml:"keep_alive"`
|
||||
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
|
||||
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
|
||||
// Future: TLS (cert_file/key_file/client_ca), auth (token/mTLS) blocks
|
||||
}
|
||||
|
||||
// StdHTTPSinkOptions defines settings for a stdlib HTTP SSE streaming sink.
|
||||
// TOML keys mirror HTTPSinkOptions for drop-in type swap ("http" -> "stdhttp").
|
||||
type StdHTTPSinkOptions struct {
|
||||
Host string `toml:"host"`
|
||||
Port int64 `toml:"port"`
|
||||
StreamPath string `toml:"stream_path"`
|
||||
StatusPath string `toml:"status_path"`
|
||||
BufferSize int64 `toml:"buffer_size"` // sink input queue
|
||||
ClientBufferSize int64 `toml:"client_buffer_size"` // per-client send queue
|
||||
WriteTimeoutMS int64 `toml:"write_timeout_ms"` // per-SSE-write deadline, 0 = none
|
||||
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
|
||||
// Future: TLS (server.TLSConfig), auth middleware options
|
||||
}
|
||||
|
||||
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
|
||||
// entries to a downstream logwisp tcp_chain source
|
||||
type TCPChainSinkOptions struct {
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
package stdhttp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
"logwisp/internal/version"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("stdhttp", NewStdHTTPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register stdhttp sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultStdHTTPHost = "0.0.0.0"
|
||||
DefaultStdHTTPBufferSize = 1000
|
||||
DefaultStdHTTPClientBufferSize = 256
|
||||
DefaultStdHTTPStreamPath = "/stream"
|
||||
DefaultStdHTTPStatusPath = "/status"
|
||||
StdHTTPReadHeaderTimeout = 10 * time.Second
|
||||
StdHTTPShutdownTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// StdHTTPSink streams log entries via Server-Sent Events using only the
|
||||
// standard library. Functional peer of the fasthttp-based http sink.
|
||||
//
|
||||
// Server.WriteTimeout is deliberately unset (it would terminate long-lived
|
||||
// SSE streams); per-write deadlines are applied via http.ResponseController.
|
||||
type StdHTTPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.StdHTTPSinkOptions
|
||||
addr string
|
||||
|
||||
// Network
|
||||
server *http.Server
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Client registry
|
||||
clients map[uint64]*sseClient
|
||||
clientsMu sync.Mutex
|
||||
nextClientID atomic.Uint64
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
stopOnce sync.Once
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
writeTimeout time.Duration
|
||||
|
||||
// Statistics
|
||||
activeClients atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
droppedWrites atomic.Uint64
|
||||
rejectedClients atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// sseClient is a registered stream consumer with a bounded send queue
|
||||
type sseClient struct {
|
||||
send chan []byte
|
||||
sessionID string
|
||||
}
|
||||
|
||||
// NewStdHTTPSinkPlugin creates a stdhttp sink through plugin factory
|
||||
func NewStdHTTPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.StdHTTPSinkOptions{
|
||||
Host: DefaultStdHTTPHost,
|
||||
WriteTimeoutMS: 0, // SSE indefinite streaming
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.StreamPath == "" {
|
||||
opts.StreamPath = DefaultStdHTTPStreamPath
|
||||
} else if !strings.HasPrefix(opts.StreamPath, "/") {
|
||||
return nil, fmt.Errorf("stream_path: must start with '/'")
|
||||
}
|
||||
if opts.StatusPath == "" {
|
||||
opts.StatusPath = DefaultStdHTTPStatusPath
|
||||
} else if !strings.HasPrefix(opts.StatusPath, "/") {
|
||||
return nil, fmt.Errorf("status_path: must start with '/'")
|
||||
}
|
||||
if opts.StreamPath == opts.StatusPath {
|
||||
return nil, fmt.Errorf("stream_path and status_path must differ")
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultStdHTTPBufferSize
|
||||
}
|
||||
if opts.ClientBufferSize <= 0 {
|
||||
opts.ClientBufferSize = DefaultStdHTTPClientBufferSize
|
||||
}
|
||||
|
||||
h := &StdHTTPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
clients: make(map[uint64]*sseClient),
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
h.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "Std HTTP sink initialized",
|
||||
"component", "stdhttp_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port,
|
||||
"stream_path", opts.StreamPath,
|
||||
"status_path", opts.StatusPath)
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (h *StdHTTPSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth appended when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (h *StdHTTPSink) Input() chan<- core.TransportEvent {
|
||||
return h.input
|
||||
}
|
||||
|
||||
// Start binds the listener and serves stream/status endpoints
|
||||
func (h *StdHTTPSink) Start(ctx context.Context) error {
|
||||
// IPv4-only, parity with existing network sinks.
|
||||
// TLS extension point: wrap ln with tls.NewListener (or set
|
||||
// server.TLSConfig and use ServeTLS); single seam, handlers unchanged.
|
||||
ln, err := net.Listen("tcp4", h.addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stdhttp sink bind %s: %w", h.addr, err)
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped patterns: mux answers 405 with Allow header on non-GET
|
||||
mux.HandleFunc(http.MethodGet+" "+h.config.StreamPath, h.handleStream)
|
||||
mux.HandleFunc(http.MethodGet+" "+h.config.StatusPath, h.handleStatus)
|
||||
|
||||
// Auth extension point: wrap mux with auth middleware once credentials
|
||||
// land, e.g. handler = authMiddleware(cfg)(handler)
|
||||
var handler http.Handler = mux
|
||||
|
||||
h.server = &http.Server{
|
||||
Handler: handler,
|
||||
ReadHeaderTimeout: StdHTTPReadHeaderTimeout,
|
||||
// WriteTimeout unset by design: SSE responses are long-lived.
|
||||
// Per-write deadlines via ResponseController in handleStream.
|
||||
}
|
||||
h.startTime = time.Now()
|
||||
|
||||
h.wg.Add(1)
|
||||
go h.brokerLoop(ctx)
|
||||
|
||||
go func() {
|
||||
if err := h.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
h.logger.Error("msg", "HTTP server terminated",
|
||||
"component", "stdhttp_sink",
|
||||
"instance_id", h.id,
|
||||
"error", err)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
h.shutdown()
|
||||
case <-h.done:
|
||||
}
|
||||
}()
|
||||
|
||||
h.logger.Info("msg", "Std HTTP server started",
|
||||
"component", "stdhttp_sink",
|
||||
"instance_id", h.id,
|
||||
"addr", h.addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (h *StdHTTPSink) Stop() {
|
||||
h.logger.Info("msg", "Stopping std HTTP sink",
|
||||
"component", "stdhttp_sink",
|
||||
"instance_id", h.id)
|
||||
|
||||
h.shutdown()
|
||||
h.wg.Wait()
|
||||
|
||||
h.logger.Info("msg", "Std HTTP sink stopped",
|
||||
"component", "stdhttp_sink",
|
||||
"instance_id", h.id,
|
||||
"total_processed", h.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// shutdown funnels ctx-cancel and Stop() teardown through a single path.
|
||||
// done is closed first so SSE handlers exit and Shutdown can complete;
|
||||
// Server.Close force-closes any handler stalled in a deadline-free write.
|
||||
func (h *StdHTTPSink) shutdown() {
|
||||
h.stopOnce.Do(func() {
|
||||
close(h.done)
|
||||
if h.server != nil {
|
||||
sctx, cancel := context.WithTimeout(context.Background(), StdHTTPShutdownTimeout)
|
||||
defer cancel()
|
||||
if err := h.server.Shutdown(sctx); err != nil {
|
||||
h.server.Close()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// removeClient unregisters a client; the first caller closes the send
|
||||
// channel and removes the session. Broker (stale-session eviction) and
|
||||
// stream handler (disconnect) may race here safely.
|
||||
func (h *StdHTTPSink) removeClient(id uint64) {
|
||||
h.clientsMu.Lock()
|
||||
c, ok := h.clients[id]
|
||||
if ok {
|
||||
delete(h.clients, id)
|
||||
}
|
||||
h.clientsMu.Unlock()
|
||||
if ok {
|
||||
close(c.send)
|
||||
h.proxy.RemoveSession(c.sessionID)
|
||||
}
|
||||
}
|
||||
|
||||
// brokerLoop fans out transport events to all client queues, non-blocking,
|
||||
// and evicts clients whose sessions were idle-expired by the session manager
|
||||
func (h *StdHTTPSink) brokerLoop(ctx context.Context) {
|
||||
defer h.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-h.done:
|
||||
return
|
||||
case event, ok := <-h.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
h.totalProcessed.Add(1)
|
||||
h.lastProcessed.Store(time.Now())
|
||||
|
||||
var stale []uint64
|
||||
h.clientsMu.Lock()
|
||||
for id, c := range h.clients {
|
||||
if _, exists := h.proxy.GetSession(c.sessionID); !exists {
|
||||
stale = append(stale, id)
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case c.send <- event.Payload:
|
||||
h.proxy.UpdateActivity(c.sessionID)
|
||||
default:
|
||||
h.droppedWrites.Add(1)
|
||||
}
|
||||
}
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
for _, id := range stale {
|
||||
h.removeClient(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleStream serves one client's SSE stream
|
||||
func (h *StdHTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
|
||||
if h.config.MaxConnections > 0 && h.activeClients.Load() >= h.config.MaxConnections {
|
||||
h.rejectedClients.Add(1)
|
||||
http.Error(w, "too many clients", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
rc := http.NewResponseController(w)
|
||||
remote := r.RemoteAddr
|
||||
|
||||
sess := h.proxy.CreateSession(remote, map[string]any{
|
||||
"type": "stdhttp_client",
|
||||
})
|
||||
|
||||
c := &sseClient{
|
||||
send: make(chan []byte, h.config.ClientBufferSize),
|
||||
sessionID: sess.ID,
|
||||
}
|
||||
id := h.nextClientID.Add(1)
|
||||
|
||||
h.clientsMu.Lock()
|
||||
h.clients[id] = c
|
||||
h.clientsMu.Unlock()
|
||||
|
||||
count := h.activeClients.Add(1)
|
||||
h.logger.Debug("msg", "HTTP client connected",
|
||||
"component", "stdhttp_sink",
|
||||
"remote_addr", remote,
|
||||
"session_id", sess.ID,
|
||||
"client_id", id,
|
||||
"active_clients", count)
|
||||
|
||||
defer func() {
|
||||
h.removeClient(id)
|
||||
newCount := h.activeClients.Add(-1)
|
||||
h.logger.Debug("msg", "HTTP client disconnected",
|
||||
"component", "stdhttp_sink",
|
||||
"remote_addr", remote,
|
||||
"session_id", sess.ID,
|
||||
"client_id", id,
|
||||
"active_clients", newCount)
|
||||
}()
|
||||
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
// Connected event with metadata, parity with fasthttp sink
|
||||
info, _ := json.Marshal(map[string]any{
|
||||
"client_id": strconv.FormatUint(id, 10),
|
||||
"session_id": sess.ID,
|
||||
"instance_id": h.id,
|
||||
"stream_path": h.config.StreamPath,
|
||||
"status_path": h.config.StatusPath,
|
||||
"buffer_size": h.config.ClientBufferSize,
|
||||
})
|
||||
fmt.Fprintf(w, "event: connected\ndata: %s\n\n", info)
|
||||
if err := rc.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
clientGone := r.Context().Done()
|
||||
for {
|
||||
select {
|
||||
case payload, ok := <-c.send:
|
||||
if !ok {
|
||||
return // broker evicted (stale session)
|
||||
}
|
||||
if h.writeTimeout > 0 {
|
||||
_ = rc.SetWriteDeadline(time.Now().Add(h.writeTimeout))
|
||||
}
|
||||
if err := writeSSE(w, payload); err != nil {
|
||||
return
|
||||
}
|
||||
if err := rc.Flush(); err != nil {
|
||||
return
|
||||
}
|
||||
h.proxy.UpdateActivity(sess.ID)
|
||||
case <-clientGone:
|
||||
return
|
||||
case <-h.done:
|
||||
fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
|
||||
rc.Flush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleStatus provides a JSON status report
|
||||
func (h *StdHTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
status := map[string]any{
|
||||
"service": "LogWisp",
|
||||
"version": version.Short(),
|
||||
"instance_id": h.id,
|
||||
"server": map[string]any{
|
||||
"type": "stdhttp",
|
||||
"host": h.config.Host,
|
||||
"port": h.config.Port,
|
||||
"active_clients": h.activeClients.Load(),
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"uptime_seconds": int(time.Since(h.startTime).Seconds()),
|
||||
},
|
||||
"endpoints": map[string]string{
|
||||
"stream": h.config.StreamPath,
|
||||
"status": h.config.StatusPath,
|
||||
},
|
||||
"statistics": map[string]any{
|
||||
"total_processed": h.totalProcessed.Load(),
|
||||
"dropped_writes": h.droppedWrites.Load(),
|
||||
"rejected_clients": h.rejectedClients.Load(),
|
||||
},
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(status)
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (h *StdHTTPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := h.lastProcessed.Load().(time.Time)
|
||||
return sink.SinkStats{
|
||||
ID: h.id,
|
||||
Type: "stdhttp",
|
||||
TotalProcessed: h.totalProcessed.Load(),
|
||||
ActiveConnections: h.activeClients.Load(),
|
||||
StartTime: h.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": h.config.Host,
|
||||
"port": h.config.Port,
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"dropped_writes": h.droppedWrites.Load(),
|
||||
"rejected_clients": h.rejectedClients.Load(),
|
||||
"endpoints": map[string]string{
|
||||
"stream": h.config.StreamPath,
|
||||
"status": h.config.StatusPath,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// writeSSE frames a payload per the W3C SSE spec (multi-line safe)
|
||||
func writeSSE(w http.ResponseWriter, payload []byte) error {
|
||||
for _, line := range splitLines(payload) {
|
||||
if _, err := fmt.Fprintf(w, "data: %s\n", line); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err := fmt.Fprint(w, "\n")
|
||||
return err
|
||||
}
|
||||
|
||||
// splitLines splits payload by newlines, trimming a single trailing newline
|
||||
func splitLines(data []byte) [][]byte {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if data[len(data)-1] == '\n' {
|
||||
data = data[:len(data)-1]
|
||||
}
|
||||
var lines [][]byte
|
||||
start := 0
|
||||
for i := 0; i < len(data); i++ {
|
||||
if data[i] == '\n' {
|
||||
lines = append(lines, data[start:i])
|
||||
start = i + 1
|
||||
}
|
||||
}
|
||||
if start < len(data) {
|
||||
lines = append(lines, data[start:])
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
return [][]byte{data}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
@@ -0,0 +1,399 @@
|
||||
package stdtcp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/internal/config"
|
||||
"logwisp/internal/core"
|
||||
"logwisp/internal/plugin"
|
||||
"logwisp/internal/session"
|
||||
"logwisp/internal/sink"
|
||||
|
||||
lconfig "github.com/lixenwraith/config"
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func init() {
|
||||
if err := plugin.RegisterSink("stdtcp", NewStdTCPSinkPlugin); err != nil {
|
||||
panic(fmt.Sprintf("failed to register stdtcp sink: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultStdTCPHost = "0.0.0.0"
|
||||
DefaultStdTCPBufferSize = 1000
|
||||
DefaultStdTCPClientBufferSize = 256
|
||||
DefaultStdTCPWriteTimeoutMS = 5000
|
||||
DefaultStdTCPKeepAlivePeriodMS = 30000
|
||||
)
|
||||
|
||||
// StdTCPSink streams formatted log entries to connected TCP clients using
|
||||
// only the standard library. Functional peer of the gnet-based tcp sink.
|
||||
//
|
||||
// Concurrency model: one broadcast loop fans out into bounded per-client
|
||||
// queues; each connection owns a writer goroutine (drains queue) and a
|
||||
// reader goroutine (disconnect detection). A stalled client drops events,
|
||||
// never the pipeline.
|
||||
type StdTCPSink struct {
|
||||
// Plugin identity and session management
|
||||
id string
|
||||
proxy *session.Proxy
|
||||
|
||||
// Configuration
|
||||
config *config.StdTCPSinkOptions
|
||||
addr string
|
||||
|
||||
// Network
|
||||
listener net.Listener
|
||||
|
||||
// Application
|
||||
input chan core.TransportEvent
|
||||
logger *log.Logger
|
||||
|
||||
// Client registry
|
||||
clients map[uint64]*tcpClient
|
||||
clientsMu sync.Mutex
|
||||
nextClientID atomic.Uint64
|
||||
|
||||
// Runtime
|
||||
done chan struct{}
|
||||
stopOnce sync.Once
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
|
||||
writeTimeout time.Duration
|
||||
|
||||
// Statistics
|
||||
activeConns atomic.Int64
|
||||
totalProcessed atomic.Uint64
|
||||
writeErrors atomic.Uint64
|
||||
droppedWrites atomic.Uint64
|
||||
rejectedConns atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
}
|
||||
|
||||
// tcpClient pairs a connection with its bounded send queue.
|
||||
// send is written by the broadcast loop (non-blocking) and drained by the
|
||||
// writer goroutine; closed signals reader-detected disconnect.
|
||||
type tcpClient struct {
|
||||
conn net.Conn
|
||||
send chan []byte
|
||||
sessionID string
|
||||
closed chan struct{}
|
||||
}
|
||||
|
||||
// NewStdTCPSinkPlugin creates a stdtcp sink through plugin factory
|
||||
func NewStdTCPSinkPlugin(
|
||||
id string,
|
||||
configMap map[string]any,
|
||||
logger *log.Logger,
|
||||
proxy *session.Proxy,
|
||||
) (sink.Sink, error) {
|
||||
opts := &config.StdTCPSinkOptions{
|
||||
Host: DefaultStdTCPHost,
|
||||
KeepAlive: true,
|
||||
}
|
||||
if err := lconfig.ScanMap(configMap, opts); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config: %w", err)
|
||||
}
|
||||
if err := lconfig.Port(opts.Port); err != nil {
|
||||
return nil, fmt.Errorf("port: %w", err)
|
||||
}
|
||||
if opts.BufferSize <= 0 {
|
||||
opts.BufferSize = DefaultStdTCPBufferSize
|
||||
}
|
||||
if opts.ClientBufferSize <= 0 {
|
||||
opts.ClientBufferSize = DefaultStdTCPClientBufferSize
|
||||
}
|
||||
if opts.WriteTimeoutMS <= 0 {
|
||||
opts.WriteTimeoutMS = DefaultStdTCPWriteTimeoutMS
|
||||
}
|
||||
if opts.KeepAlivePeriodMS <= 0 {
|
||||
opts.KeepAlivePeriodMS = DefaultStdTCPKeepAlivePeriodMS
|
||||
}
|
||||
|
||||
t := &StdTCPSink{
|
||||
id: id,
|
||||
proxy: proxy,
|
||||
config: opts,
|
||||
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
|
||||
input: make(chan core.TransportEvent, opts.BufferSize),
|
||||
done: make(chan struct{}),
|
||||
logger: logger,
|
||||
clients: make(map[uint64]*tcpClient),
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
logger.Info("msg", "Std TCP sink initialized",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", id,
|
||||
"host", opts.Host,
|
||||
"port", opts.Port)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Capabilities returns supported capabilities
|
||||
func (t *StdTCPSink) Capabilities() []core.Capability {
|
||||
// CapTLS/CapAuth appended when transport security lands
|
||||
return []core.Capability{
|
||||
core.CapSessionAware,
|
||||
core.CapMultiSession,
|
||||
}
|
||||
}
|
||||
|
||||
// Input returns the channel for sending transport events
|
||||
func (t *StdTCPSink) Input() chan<- core.TransportEvent {
|
||||
return t.input
|
||||
}
|
||||
|
||||
// listen creates the server listener.
|
||||
// TLS extension point: wrap the returned listener with tls.NewListener here
|
||||
// once cert config lands; no other code path changes. mTLS peer identity is
|
||||
// then available via conn.(*tls.Conn).ConnectionState() in the auth hook.
|
||||
func (t *StdTCPSink) listen() (net.Listener, error) {
|
||||
lc := net.ListenConfig{}
|
||||
if t.config.KeepAlive {
|
||||
lc.KeepAliveConfig = net.KeepAliveConfig{
|
||||
Enable: true,
|
||||
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
|
||||
}
|
||||
}
|
||||
// IPv4-only, parity with existing network sinks
|
||||
return lc.Listen(context.Background(), "tcp4", t.addr)
|
||||
}
|
||||
|
||||
// Start binds the listener and launches accept and broadcast loops
|
||||
func (t *StdTCPSink) Start(ctx context.Context) error {
|
||||
ln, err := t.listen()
|
||||
if err != nil {
|
||||
return fmt.Errorf("stdtcp sink bind %s: %w", t.addr, err)
|
||||
}
|
||||
t.listener = ln
|
||||
t.startTime = time.Now()
|
||||
|
||||
t.wg.Add(2)
|
||||
go t.acceptLoop()
|
||||
go t.broadcastLoop(ctx)
|
||||
|
||||
// Pipeline context cancellation mirrors gnet engine stop: cease accepting
|
||||
// and tear down existing connections
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.shutdown()
|
||||
case <-t.done:
|
||||
}
|
||||
}()
|
||||
|
||||
t.logger.Info("msg", "Std TCP server started",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", t.id,
|
||||
"addr", t.addr)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the sink
|
||||
func (t *StdTCPSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping std TCP sink",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", t.id)
|
||||
|
||||
t.shutdown()
|
||||
t.wg.Wait()
|
||||
|
||||
t.logger.Info("msg", "Std TCP sink stopped",
|
||||
"component", "stdtcp_sink",
|
||||
"instance_id", t.id,
|
||||
"total_processed", t.totalProcessed.Load())
|
||||
}
|
||||
|
||||
// shutdown funnels ctx-cancel and Stop() teardown through a single path
|
||||
func (t *StdTCPSink) shutdown() {
|
||||
t.stopOnce.Do(func() {
|
||||
close(t.done)
|
||||
if t.listener != nil {
|
||||
t.listener.Close() // unblocks acceptLoop
|
||||
}
|
||||
t.clientsMu.Lock()
|
||||
for _, c := range t.clients {
|
||||
c.conn.Close() // unblocks per-connection readers
|
||||
}
|
||||
t.clientsMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
// acceptLoop accepts client connections until listener close
|
||||
func (t *StdTCPSink) acceptLoop() {
|
||||
defer t.wg.Done()
|
||||
for {
|
||||
conn, err := t.listener.Accept()
|
||||
if err != nil {
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
return
|
||||
}
|
||||
t.logger.Warn("msg", "Accept error",
|
||||
"component", "stdtcp_sink",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if t.config.MaxConnections > 0 && t.activeConns.Load() >= t.config.MaxConnections {
|
||||
// Load/admit race can over-admit by a conn under burst; acceptable
|
||||
t.rejectedConns.Add(1)
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
// Auth extension point: credential/peer verification runs here,
|
||||
// pre-registration (password preamble read or TLS peer cert check)
|
||||
|
||||
t.wg.Add(1)
|
||||
go t.handleConn(conn)
|
||||
}
|
||||
}
|
||||
|
||||
// handleConn registers the client and runs its writer; a companion reader
|
||||
// goroutine drains inbound bytes for disconnect detection
|
||||
func (t *StdTCPSink) handleConn(conn net.Conn) {
|
||||
defer t.wg.Done()
|
||||
remote := conn.RemoteAddr().String()
|
||||
|
||||
sess := t.proxy.CreateSession(remote, map[string]any{
|
||||
"type": "stdtcp_client",
|
||||
"remote_addr": remote,
|
||||
})
|
||||
|
||||
c := &tcpClient{
|
||||
conn: conn,
|
||||
send: make(chan []byte, t.config.ClientBufferSize),
|
||||
sessionID: sess.ID,
|
||||
closed: make(chan struct{}),
|
||||
}
|
||||
id := t.nextClientID.Add(1)
|
||||
|
||||
t.clientsMu.Lock()
|
||||
t.clients[id] = c
|
||||
t.clientsMu.Unlock()
|
||||
|
||||
count := t.activeConns.Add(1)
|
||||
t.logger.Debug("msg", "TCP connection opened",
|
||||
"component", "stdtcp_sink",
|
||||
"remote_addr", remote,
|
||||
"session_id", sess.ID,
|
||||
"active_connections", count)
|
||||
|
||||
defer func() {
|
||||
t.clientsMu.Lock()
|
||||
delete(t.clients, id)
|
||||
t.clientsMu.Unlock()
|
||||
conn.Close()
|
||||
<-c.closed // reader has exited
|
||||
t.proxy.RemoveSession(sess.ID)
|
||||
newCount := t.activeConns.Add(-1)
|
||||
t.logger.Debug("msg", "TCP connection closed",
|
||||
"component", "stdtcp_sink",
|
||||
"remote_addr", remote,
|
||||
"active_connections", newCount)
|
||||
}()
|
||||
|
||||
// Reader: sink is write-only; drain and discard inbound bytes to detect
|
||||
// disconnect and refresh session activity on client traffic
|
||||
go func() {
|
||||
defer close(c.closed)
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := conn.Read(buf)
|
||||
if n > 0 {
|
||||
t.proxy.UpdateActivity(sess.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Writer: synchronous stdlib write with deadline. A failed write means
|
||||
// the kernel buffer stayed full for the full deadline - connection is
|
||||
// dead or hopelessly stalled, so disconnect immediately (no gnet-style
|
||||
// consecutive-error counter needed for transient async callback errors).
|
||||
for {
|
||||
select {
|
||||
case data := <-c.send:
|
||||
if t.writeTimeout > 0 {
|
||||
conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
|
||||
}
|
||||
if _, err := conn.Write(data); err != nil {
|
||||
t.writeErrors.Add(1)
|
||||
t.logger.Debug("msg", "Write failed, closing client",
|
||||
"component", "stdtcp_sink",
|
||||
"remote_addr", remote,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.proxy.UpdateActivity(sess.ID)
|
||||
case <-c.closed:
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// broadcastLoop fans out transport events to all client queues, non-blocking
|
||||
func (t *StdTCPSink) broadcastLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
|
||||
t.clientsMu.Lock()
|
||||
for _, c := range t.clients {
|
||||
select {
|
||||
case c.send <- event.Payload:
|
||||
default:
|
||||
// Slow client: drop its event, never stall siblings
|
||||
t.droppedWrites.Add(1)
|
||||
}
|
||||
}
|
||||
t.clientsMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetStats returns sink statistics
|
||||
func (t *StdTCPSink) GetStats() sink.SinkStats {
|
||||
lastProc, _ := t.lastProcessed.Load().(time.Time)
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
Type: "stdtcp",
|
||||
TotalProcessed: t.totalProcessed.Load(),
|
||||
ActiveConnections: t.activeConns.Load(),
|
||||
StartTime: t.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"host": t.config.Host,
|
||||
"port": t.config.Port,
|
||||
"buffer_size": t.config.BufferSize,
|
||||
"write_errors": t.writeErrors.Load(),
|
||||
"dropped_writes": t.droppedWrites.Load(),
|
||||
"rejected_conns": t.rejectedConns.Load(),
|
||||
},
|
||||
}
|
||||
}
|
||||
Executable
+246
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env bash
|
||||
# logwisp chain topology test
|
||||
#
|
||||
# random --> tcp_chain sink --> :15801 tcp_chain src --> :15803 stdtcp sink
|
||||
# random --> http_chain sink --> :15802 http_chain src --> :15804 stdhttp sink (SSE)
|
||||
#
|
||||
# Usage:
|
||||
# ./chain_aggregate_test.sh manual mode: 2 edge daemons + relay foreground
|
||||
# ./chain_aggregate_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown
|
||||
# ./chain_aggregate_test.sh --keep (with --auto) skip teardown on success
|
||||
#
|
||||
# The relay aggregates both sinks and fans out aggregated streams into both http and tcp sinks
|
||||
#
|
||||
# Requires: bash 5+, coreutils (timeout), curl. Linux dev host only.
|
||||
|
||||
set -u
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
|
||||
RUN="$SCRIPT_DIR/run"
|
||||
CONF="$RUN/conf"
|
||||
LOG="$RUN/log"
|
||||
|
||||
PORT_TCP_CHAIN=15801
|
||||
PORT_HTTP_CHAIN=15802
|
||||
PORT_TCP_SINK=15803
|
||||
PORT_HTTP_SINK=15804
|
||||
|
||||
AUTO=0; KEEP=0
|
||||
for a in "$@"; do case "$a" in
|
||||
--auto) AUTO=1 ;;
|
||||
--keep) KEEP=1 ;;
|
||||
*) echo "unknown arg: $a" >&2; exit 1 ;;
|
||||
esac; done
|
||||
|
||||
PIDS=()
|
||||
cleanup() {
|
||||
local rc=$?
|
||||
trap - EXIT INT TERM
|
||||
if (( ${#PIDS[@]} )); then
|
||||
echo "--- teardown: stopping ${#PIDS[@]} daemon(s)"
|
||||
kill -TERM "${PIDS[@]}" 2>/dev/null
|
||||
local deadline=$(( SECONDS + 10 ))
|
||||
for pid in "${PIDS[@]}"; do
|
||||
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
|
||||
kill -KILL "$pid" 2>/dev/null
|
||||
done
|
||||
fi
|
||||
exit "$rc"
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
|
||||
|
||||
wait_port() { # port timeout_s
|
||||
local i; for (( i=0; i < $2 * 10; i++ )); do
|
||||
port_open "$1" && return 0
|
||||
sleep 0.1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Reads TCP sink stream for N seconds; connection counts as a sink client
|
||||
tcp_read() { # port secs
|
||||
timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true
|
||||
}
|
||||
|
||||
start_daemon() { # name conf
|
||||
"$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 &
|
||||
PIDS+=($!)
|
||||
echo "started $1 (pid $!)"
|
||||
}
|
||||
|
||||
# --- Preflight ---
|
||||
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
|
||||
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
|
||||
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
|
||||
done
|
||||
mkdir -p "$CONF" "$LOG"
|
||||
|
||||
# --- Config generation ---
|
||||
cat > "$CONF/edge_tcp.toml" <<EOF
|
||||
status_reporter = false
|
||||
[logging]
|
||||
output = "file"
|
||||
level = "info"
|
||||
[logging.file]
|
||||
directory = "$LOG"
|
||||
name = "edge_tcp"
|
||||
|
||||
[[pipelines]]
|
||||
name = "edge_tcp"
|
||||
[[pipelines.plugin_sources]]
|
||||
id = "rand"
|
||||
type = "random"
|
||||
[pipelines.plugin_sources.config]
|
||||
interval_ms = 200
|
||||
format = "txt"
|
||||
length = 24
|
||||
[[pipelines.plugin_sinks]]
|
||||
id = "to_relay"
|
||||
type = "tcp_chain"
|
||||
[pipelines.plugin_sinks.config]
|
||||
host = "127.0.0.1"
|
||||
port = $PORT_TCP_CHAIN
|
||||
node = "edge-tcp"
|
||||
EOF
|
||||
|
||||
cat > "$CONF/edge_http.toml" <<EOF
|
||||
status_reporter = false
|
||||
[logging]
|
||||
output = "file"
|
||||
level = "info"
|
||||
[logging.file]
|
||||
directory = "$LOG"
|
||||
name = "edge_http"
|
||||
|
||||
[[pipelines]]
|
||||
name = "edge_http"
|
||||
[[pipelines.plugin_sources]]
|
||||
id = "rand"
|
||||
type = "random"
|
||||
[pipelines.plugin_sources.config]
|
||||
interval_ms = 200
|
||||
format = "txt"
|
||||
length = 24
|
||||
[[pipelines.plugin_sinks]]
|
||||
id = "to_relay"
|
||||
type = "http_chain"
|
||||
[pipelines.plugin_sinks.config]
|
||||
host = "127.0.0.1"
|
||||
port = $PORT_HTTP_CHAIN
|
||||
node = "edge-http"
|
||||
flush_interval_ms = 500
|
||||
EOF
|
||||
|
||||
cat > "$CONF/relay.toml" <<EOF
|
||||
status_reporter = false
|
||||
[logging]
|
||||
output = "stdout"
|
||||
level = "info"
|
||||
|
||||
# Single pipeline: fan-in both chain sources, fan-out to both sinks
|
||||
[[pipelines]]
|
||||
name = "aggregate"
|
||||
[pipelines.flow.format]
|
||||
type = "json"
|
||||
sanitizer_policy = "json"
|
||||
|
||||
[[pipelines.plugin_sources]]
|
||||
id = "in_tcp"
|
||||
type = "tcp_chain"
|
||||
[pipelines.plugin_sources.config]
|
||||
host = "127.0.0.1"
|
||||
port = $PORT_TCP_CHAIN
|
||||
|
||||
[[pipelines.plugin_sources]]
|
||||
id = "in_http"
|
||||
type = "http_chain"
|
||||
[pipelines.plugin_sources.config]
|
||||
host = "127.0.0.1"
|
||||
port = $PORT_HTTP_CHAIN
|
||||
|
||||
[[pipelines.plugin_sinks]]
|
||||
id = "out_tcp"
|
||||
type = "stdtcp"
|
||||
[pipelines.plugin_sinks.config]
|
||||
host = "127.0.0.1"
|
||||
port = $PORT_TCP_SINK
|
||||
|
||||
[[pipelines.plugin_sinks]]
|
||||
id = "out_http"
|
||||
type = "stdhttp"
|
||||
[pipelines.plugin_sinks.config]
|
||||
host = "127.0.0.1"
|
||||
port = $PORT_HTTP_SINK
|
||||
EOF
|
||||
|
||||
# --- Guide ---
|
||||
cat <<EOF
|
||||
================================================================
|
||||
logwisp chain test — port map
|
||||
$PORT_TCP_CHAIN relay ingest (tcp_chain, from edge-tcp)
|
||||
$PORT_HTTP_CHAIN relay ingest (http_chain, from edge-http, POST /ingest)
|
||||
$PORT_TCP_SINK TCP sink -> live check: nc 127.0.0.1 $PORT_TCP_SINK
|
||||
$PORT_HTTP_SINK HTTP sink -> live check: browser/curl:
|
||||
http://127.0.0.1:$PORT_HTTP_SINK/stream (SSE)
|
||||
http://127.0.0.1:$PORT_HTTP_SINK/status (JSON stats)
|
||||
Use 127.0.0.1, not localhost — sinks reject IPv6.
|
||||
Expected: json entries from BOTH nodes ("edge-tcp", "edge-http")
|
||||
interleaved on :$PORT_TCP_SINK and :$PORT_HTTP_SINK.
|
||||
Logs: $LOG/
|
||||
================================================================
|
||||
EOF
|
||||
|
||||
if (( AUTO == 0 )); then
|
||||
# Manual mode: edges daemonized first (chain sinks backoff-retry), relay foreground
|
||||
start_daemon edge_tcp edge_tcp.toml
|
||||
start_daemon edge_http edge_http.toml
|
||||
echo "--- relay starting in FOREGROUND; Ctrl-C stops relay and both edges"
|
||||
"$BIN" -c "$CONF/relay.toml"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- Auto mode ---
|
||||
start_daemon relay relay.toml
|
||||
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
|
||||
wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; }
|
||||
done
|
||||
start_daemon edge_tcp edge_tcp.toml
|
||||
start_daemon edge_http edge_http.toml
|
||||
|
||||
echo "--- settling 3s (connect + first http_chain flush)"
|
||||
sleep 3
|
||||
|
||||
fail=0
|
||||
check() { # label condition_result
|
||||
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
|
||||
}
|
||||
|
||||
# 1. TCP chain: edge-tcp -> relay -> tcp sink
|
||||
tcp_out="$(tcp_read "$PORT_TCP_SINK" 4)"
|
||||
nt=$(grep -c '"node":"edge-tcp"' <<< "$tcp_out")
|
||||
nh=$(grep -c '"node":"edge-http"' <<< "$tcp_out")
|
||||
check "tcp sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $(( nt >= 1 && nh >= 1 ))
|
||||
|
||||
# 2. HTTP chain: edge-http -> relay -> SSE sink
|
||||
sse_out="$(curl -sN --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
|
||||
nt=$(grep -c '^data:.*"node":"edge-tcp"' <<< "$sse_out")
|
||||
nh=$(grep -c '^data:.*"node":"edge-http"' <<< "$sse_out")
|
||||
check "http sink: aggregated edge-tcp ($nt) + edge-http ($nh)" $(( nt >= 1 && nh >= 1 ))
|
||||
|
||||
# 3. HTTP sink status endpoint
|
||||
status="$(curl -s --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
|
||||
proc=$(grep -o '"total_processed":[0-9]*' <<< "$status" | grep -o '[0-9]*' || echo 0)
|
||||
check "status endpoint: total_processed=$proc > 0" $(( proc > 0 ))
|
||||
|
||||
echo "================================================================"
|
||||
if (( fail == 0 )); then
|
||||
echo "RESULT: ALL PASS"
|
||||
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
|
||||
else
|
||||
echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log"
|
||||
fi
|
||||
exit "$fail"
|
||||
|
||||
@@ -9,6 +9,11 @@
|
||||
# ./chain_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown
|
||||
# ./chain_test.sh --keep (with --auto) skip teardown on success
|
||||
#
|
||||
# relay runs TWO independent pipelines; no cross-path aggregation:
|
||||
# relay_tcp: tcp_chain src :15801 -> tcp sink :15803 (edge-tcp only)
|
||||
# relay_http: http_chain src :15802 -> http sink :15804 (edge-http only)
|
||||
# For fan-in aggregation (both edges -> both sinks) see chain_agg_test.sh.
|
||||
#
|
||||
# Requires: bash 5+, coreutils (timeout), curl. Linux dev host only.
|
||||
|
||||
set -u
|
||||
|
||||
Reference in New Issue
Block a user