lixen/logwisp
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.16.0 tls added to tcp/http sources and sinks

Browse Source
This commit is contained in:
lixen
2026-07-18 04:10:22 -04:00
parent f0aca019f3
commit 325b51d840
8 changed files with 526 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/sink/http/http.go
+44 -13
View File
@@ -2,6 +2,7 @@ package http
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
@@ -18,6 +19,7 @@ import (
"logwisp/internal/plugin"
"logwisp/internal/session"
"logwisp/internal/sink"
"logwisp/internal/tlsx"
"logwisp/internal/version"
lconfig "github.com/lixenwraith/config"
@@ -63,6 +65,10 @@ type HTTPSink struct {
clients map[uint64]*sseClient
clientsMu sync.Mutex
nextClientID atomic.Uint64
writeTimeout time.Duration
// TLS
tlsConfig *tls.Config
// Runtime
done chan struct{}
@@ -70,8 +76,6 @@ type HTTPSink struct {
wg sync.WaitGroup
startTime time.Time
writeTimeout time.Duration
// Statistics
activeClients atomic.Int64
totalProcessed atomic.Uint64
@@ -122,6 +126,10 @@ func NewHTTPSinkPlugin(
if opts.ClientBufferSize <= 0 {
opts.ClientBufferSize = DefaultHTTPClientBufferSize
}
tlsCfg, err := tlsx.Server(opts.TLS)
if err != nil {
return nil, err
}
h := &HTTPSink{
id: id,
@@ -133,6 +141,7 @@ func NewHTTPSinkPlugin(
logger: logger,
clients: make(map[uint64]*sseClient),
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
tlsConfig: tlsCfg,
}
h.lastProcessed.Store(time.Time{})
@@ -142,17 +151,22 @@ func NewHTTPSinkPlugin(
"host", opts.Host,
"port", opts.Port,
"stream_path", opts.StreamPath,
"status_path", opts.StatusPath)
"status_path", opts.StatusPath,
"tls", tlsCfg != nil,
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert)
return h, nil
}
// Capabilities returns supported capabilities
func (h *HTTPSink) Capabilities() []core.Capability {
// CapTLS/CapAuth appended when transport security lands
return []core.Capability{
core.CapSessionAware,
core.CapMultiSession,
caps := []core.Capability{core.CapSessionAware, core.CapMultiSession}
if h.tlsConfig != nil {
caps = append(caps, core.CapTLS)
if h.tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert {
caps = append(caps, core.CapAuth) // mTLS is authentication
}
}
return caps
}
// Input returns the channel for sending transport events
@@ -163,8 +177,8 @@ func (h *HTTPSink) Input() chan<- core.TransportEvent {
// Start binds the listener and serves stream/status endpoints
func (h *HTTPSink) Start(ctx context.Context) error {
// IPv4-only, parity with existing network sinks.
// TLS extension point: wrap ln with tls.NewListener (or set
// server.TLSConfig and use ServeTLS); single seam, handlers unchanged.
// TLS is applied via server.TLSConfig + ServeTLS below, not by wrapping
// ln; net/http then owns handshake, ALPN (h2), and per-conn errors.
ln, err := net.Listen("tcp4", h.addr)
if err != nil {
return fmt.Errorf("http sink bind %s: %w", h.addr, err)
@@ -183,15 +197,23 @@ func (h *HTTPSink) Start(ctx context.Context) error {
Handler: handler,
ReadHeaderTimeout: HTTPReadHeaderTimeout,
// WriteTimeout unset by design: SSE responses are long-lived.
// Per-write deadlines via ResponseController in handleStream.
// net/http bounds the TLS handshake by min(ReadHeaderTimeout,
// ReadTimeout, WriteTimeout), so ReadHeaderTimeout covers it here.
ErrorLog: tlsx.HTTPErrorLog(h.logger, "http_sink"),
}
h.startTime = time.Now()
h.wg.Add(1)
go h.brokerLoop(ctx)
serve := h.server.Serve
if h.tlsConfig != nil {
h.server.TLSConfig = h.tlsConfig
serve = func(l net.Listener) error { return h.server.ServeTLS(l, "", "") }
}
go func() {
if err := h.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
if err := serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
h.logger.Error("msg", "HTTP server terminated",
"component", "http_sink",
"instance_id", h.id,
@@ -312,9 +334,16 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
rc := http.NewResponseController(w)
remote := r.RemoteAddr
sess := h.proxy.CreateSession(remote, map[string]any{
meta := map[string]any{
"type": "http_client",
})
}
if r.TLS != nil {
meta["tls"] = true
if cn := tlsx.PeerCN(*r.TLS); cn != "" {
meta["tls_peer_cn"] = cn
}
}
sess := h.proxy.CreateSession(remote, meta)
c := &sseClient{
send: make(chan []byte, h.config.ClientBufferSize),
@@ -402,6 +431,7 @@ func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
"type": "http",
"host": h.config.Host,
"port": h.config.Port,
"tls": h.tlsConfig != nil,
"active_clients": h.activeClients.Load(),
"buffer_size": h.config.BufferSize,
"uptime_seconds": int(time.Since(h.startTime).Seconds()),
@@ -435,6 +465,7 @@ func (h *HTTPSink) GetStats() sink.SinkStats {
"host": h.config.Host,
"port": h.config.Port,
"buffer_size": h.config.BufferSize,
"tls": h.tlsConfig != nil,
"dropped_writes": h.droppedWrites.Load(),
"rejected_clients": h.rejectedClients.Load(),
"endpoints": map[string]string{
internal/sink/httpchain/httpchain.go
+36 -11
View File
@@ -21,6 +21,7 @@ import (
"logwisp/internal/plugin"
"logwisp/internal/session"
"logwisp/internal/sink"
"logwisp/internal/tlsx"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
@@ -54,6 +55,9 @@ type HTTPChainSink struct {
node string
url string
tlsEnabled bool
mtls bool
client *http.Client
input chan core.TransportEvent
logger *log.Logger
@@ -128,6 +132,11 @@ func NewHTTPChainSinkPlugin(
}
}
tlsCfg, err := tlsx.Client(opts.TLS, opts.Host)
if err != nil {
return nil, err
}
addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10))
transport := &http.Transport{
@@ -139,16 +148,25 @@ func NewHTTPChainSinkPlugin(
MaxIdleConnsPerHost: 2,
IdleConnTimeout: 90 * time.Second,
DisableCompression: true,
// Future: TLSClientConfig; HTTP/2 via ALPN once TLS lands
TLSClientConfig: tlsCfg, // nil = plaintext
TLSHandshakeTimeout: tlsx.HandshakeTimeout,
// h2 stays off: custom DialContext disables auto-ALPN and batched
// NDJSON POSTs gain nothing from it
}
scheme := "http"
if tlsCfg != nil {
scheme = "https"
}
t := &HTTPChainSink{
id: id,
proxy: proxy,
config: opts,
node: node,
// Future: "https" scheme with TLS
url: "http://" + addr + opts.IngestPath,
id: id,
proxy: proxy,
config: opts,
node: node,
tlsEnabled: tlsCfg != nil,
mtls: tlsCfg != nil && len(tlsCfg.Certificates) > 0,
url: scheme + "://" + addr + opts.IngestPath,
client: &http.Client{Transport: transport},
input: make(chan core.TransportEvent, opts.BufferSize),
done: make(chan struct{}),
@@ -171,16 +189,22 @@ func NewHTTPChainSinkPlugin(
"component", "http_chain_sink",
"instance_id", id,
"target", t.url,
"node", node)
"node", node,
"tls", t.tlsEnabled,
"mtls", t.mtls)
return t, nil
}
// Capabilities returns supported capabilities
func (t *HTTPChainSink) Capabilities() []core.Capability {
// CapTLS/CapAuth added when transport security lands
return []core.Capability{
core.CapSessionAware,
caps := []core.Capability{core.CapSessionAware}
if t.tlsEnabled {
caps = append(caps, core.CapTLS)
if t.mtls {
caps = append(caps, core.CapAuth) // presents client identity (mTLS)
}
}
return caps
}
// Input returns the channel for sending transport events
@@ -234,6 +258,7 @@ func (t *HTTPChainSink) GetStats() sink.SinkStats {
Details: map[string]any{
"target": t.url,
"node": t.node,
"tls": t.tlsEnabled,
"batches_sent": t.batchesSent.Load(),
"request_errors": t.requestErrors.Load(),
"dropped_batches": t.droppedBatches.Load(),
internal/sink/tcp/tcp.go
+74 -28
View File
@@ -2,6 +2,7 @@ package tcp
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
@@ -15,6 +16,7 @@ import (
"logwisp/internal/plugin"
"logwisp/internal/session"
"logwisp/internal/sink"
"logwisp/internal/tlsx"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
@@ -58,6 +60,11 @@ type TCPSink struct {
clients map[uint64]*tcpClient
clientsMu sync.Mutex
nextClientID atomic.Uint64
writeTimeout time.Duration
// TLS
tlsConfig *tls.Config
tlsHandshakeErrors atomic.Uint64
// Runtime
done chan struct{}
@@ -65,8 +72,6 @@ type TCPSink struct {
wg sync.WaitGroup
startTime time.Time
writeTimeout time.Duration
// Statistics
activeConns atomic.Int64
totalProcessed atomic.Uint64
@@ -115,6 +120,10 @@ func NewTCPSinkPlugin(
if opts.KeepAlivePeriodMS <= 0 {
opts.KeepAlivePeriodMS = DefaultTCPKeepAlivePeriodMS
}
tlsCfg, err := tlsx.Server(opts.TLS)
if err != nil {
return nil, err
}
t := &TCPSink{
id: id,
@@ -126,6 +135,7 @@ func NewTCPSinkPlugin(
logger: logger,
clients: make(map[uint64]*tcpClient),
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
tlsConfig: tlsCfg,
}
t.lastProcessed.Store(time.Time{})
@@ -133,17 +143,22 @@ func NewTCPSinkPlugin(
"component", "tcp_sink",
"instance_id", id,
"host", opts.Host,
"port", opts.Port)
"port", opts.Port,
"tls", tlsCfg != nil,
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert)
return t, nil
}
// Capabilities returns supported capabilities
func (t *TCPSink) Capabilities() []core.Capability {
// CapTLS/CapAuth appended when transport security lands
return []core.Capability{
core.CapSessionAware,
core.CapMultiSession,
caps := []core.Capability{core.CapSessionAware, core.CapMultiSession}
if t.tlsConfig != nil {
caps = append(caps, core.CapTLS)
if t.tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert {
caps = append(caps, core.CapAuth) // mTLS is authentication
}
}
return caps
}
// Input returns the channel for sending transport events
@@ -151,10 +166,9 @@ func (t *TCPSink) Input() chan<- core.TransportEvent {
return t.input
}
// listen creates the server listener.
// TLS extension point: wrap the returned listener with tls.NewListener here
// once cert config lands; no other code path changes. mTLS peer identity is
// then available via conn.(*tls.Conn).ConnectionState() in the auth hook.
// listen creates the server listener, TLS-wrapped when configured.
// Handshake is deferred: tls.NewListener conns handshake explicitly in
// handleConn under tlsx.HandshakeTimeout, post max_connections admission.
func (t *TCPSink) listen() (net.Listener, error) {
lc := net.ListenConfig{}
if t.config.KeepAlive {
@@ -164,7 +178,14 @@ func (t *TCPSink) listen() (net.Listener, error) {
}
}
// IPv4-only, parity with existing network sinks
return lc.Listen(context.Background(), "tcp4", t.addr)
ln, err := lc.Listen(context.Background(), "tcp4", t.addr)
if err != nil {
return nil, err
}
if t.tlsConfig != nil {
ln = tls.NewListener(ln, t.tlsConfig)
}
return ln, nil
}
// Start binds the listener and launches accept and broadcast loops
@@ -249,8 +270,8 @@ func (t *TCPSink) acceptLoop() {
continue
}
// Auth extension point: credential/peer verification runs here,
// pre-registration (password preamble read or TLS peer cert check)
// Password-auth extension point: preamble verification runs in
// handleConn post-handshake, pre-registration
t.wg.Add(1)
go t.handleConn(conn)
@@ -263,11 +284,40 @@ func (t *TCPSink) handleConn(conn net.Conn) {
defer t.wg.Done()
remote := conn.RemoteAddr().String()
sess := t.proxy.CreateSession(remote, map[string]any{
// Counted from accept: max_connections bounds concurrent handshakes too
count := t.activeConns.Add(1)
defer func() {
newCount := t.activeConns.Add(-1)
t.logger.Debug("msg", "TCP connection closed",
"component", "tcp_sink",
"remote_addr", remote,
"active_connections", newCount)
}()
meta := map[string]any{
"type": "tcp_client",
"remote_addr": remote,
})
}
if tc, ok := conn.(*tls.Conn); ok {
hctx, cancel := context.WithTimeout(context.Background(), tlsx.HandshakeTimeout)
err := tc.HandshakeContext(hctx)
cancel()
if err != nil {
t.tlsHandshakeErrors.Add(1)
t.logger.Debug("msg", "TLS handshake failed",
"component", "tcp_sink",
"remote_addr", remote,
"error", err)
conn.Close()
return
}
meta["tls"] = true
if cn := tlsx.PeerCN(tc.ConnectionState()); cn != "" {
meta["tls_peer_cn"] = cn
}
}
sess := t.proxy.CreateSession(remote, meta)
c := &tcpClient{
conn: conn,
send: make(chan []byte, t.config.ClientBufferSize),
@@ -280,7 +330,6 @@ func (t *TCPSink) handleConn(conn net.Conn) {
t.clients[id] = c
t.clientsMu.Unlock()
count := t.activeConns.Add(1)
t.logger.Debug("msg", "TCP connection opened",
"component", "tcp_sink",
"remote_addr", remote,
@@ -294,11 +343,6 @@ func (t *TCPSink) handleConn(conn net.Conn) {
conn.Close()
<-c.closed // reader has exited
t.proxy.RemoveSession(sess.ID)
newCount := t.activeConns.Add(-1)
t.logger.Debug("msg", "TCP connection closed",
"component", "tcp_sink",
"remote_addr", remote,
"active_connections", newCount)
}()
// Reader: sink is write-only; drain and discard inbound bytes to detect
@@ -385,12 +429,14 @@ func (t *TCPSink) GetStats() sink.SinkStats {
StartTime: t.startTime,
LastProcessed: lastProc,
Details: map[string]any{
"host": t.config.Host,
"port": t.config.Port,
"buffer_size": t.config.BufferSize,
"write_errors": t.writeErrors.Load(),
"dropped_writes": t.droppedWrites.Load(),
"rejected_conns": t.rejectedConns.Load(),
"host": t.config.Host,
"port": t.config.Port,
"buffer_size": t.config.BufferSize,
"write_errors": t.writeErrors.Load(),
"dropped_writes": t.droppedWrites.Load(),
"rejected_conns": t.rejectedConns.Load(),
"tls": t.tlsConfig != nil,
"tls_handshake_errors": t.tlsHandshakeErrors.Load(),
},
}
}
internal/sink/tcpchain/tcpchain.go
+36 -10
View File
@@ -2,6 +2,7 @@ package tcpchain
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"math/rand/v2"
@@ -18,6 +19,7 @@ import (
"logwisp/internal/plugin"
"logwisp/internal/session"
"logwisp/internal/sink"
"logwisp/internal/tlsx"
lconfig "github.com/lixenwraith/config"
"github.com/lixenwraith/log"
@@ -48,6 +50,7 @@ type TCPChainSink struct {
node string
addr string
helloLine []byte
tlsConfig *tls.Config
input chan core.TransportEvent
logger *log.Logger
@@ -122,6 +125,10 @@ func NewTCPChainSinkPlugin(
if err != nil {
return nil, fmt.Errorf("hello: %w", err)
}
tlsCfg, err := tlsx.Client(opts.TLS, opts.Host)
if err != nil {
return nil, err
}
t := &TCPChainSink{
id: id,
@@ -130,6 +137,7 @@ func NewTCPChainSinkPlugin(
node: node,
addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)),
helloLine: helloLine,
tlsConfig: tlsCfg,
input: make(chan core.TransportEvent, opts.BufferSize),
done: make(chan struct{}),
logger: logger,
@@ -152,16 +160,22 @@ func NewTCPChainSinkPlugin(
"component", "tcp_chain_sink",
"instance_id", id,
"target", t.addr,
"node", node)
"node", node,
"tls", tlsCfg != nil,
"mtls", tlsCfg != nil && len(tlsCfg.Certificates) > 0)
return t, nil
}
// Capabilities returns supported capabilities
func (t *TCPChainSink) Capabilities() []core.Capability {
// CapTLS/CapAuth added when transport security lands
return []core.Capability{
core.CapSessionAware,
caps := []core.Capability{core.CapSessionAware}
if t.tlsConfig != nil {
caps = append(caps, core.CapTLS)
if len(t.tlsConfig.Certificates) > 0 {
caps = append(caps, core.CapAuth) // presents client identity (mTLS)
}
}
return caps
}
// Input returns the channel for sending transport events
@@ -220,6 +234,7 @@ func (t *TCPChainSink) GetStats() sink.SinkStats {
Details: map[string]any{
"target": t.addr,
"node": t.node,
"tls": t.tlsConfig != nil,
"connected": t.connected.Load(),
"reconnects": t.reconnects.Load(),
"write_errors": t.writeErrors.Load(),
@@ -321,18 +336,28 @@ func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
}
}
// connect performs a single dial + hello attempt
// connect performs a single dial (+ TLS handshake) + hello attempt
func (t *TCPChainSink) connect(ctx context.Context) error {
d := net.Dialer{Timeout: t.dialTimeout}
nd := net.Dialer{Timeout: t.dialTimeout}
if t.config.KeepAlive {
d.KeepAliveConfig = net.KeepAliveConfig{
nd.KeepAliveConfig = net.KeepAliveConfig{
Enable: true,
Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond,
}
}
// IPv4-only
conn, err := d.DialContext(ctx, "tcp4", t.addr)
var conn net.Conn
var err error
if t.tlsConfig != nil {
// nd.Timeout only bounds the TCP connect; tls.Dialer runs the
// handshake under ctx, so bound dial + handshake together here
dctx, cancel := context.WithTimeout(ctx, t.dialTimeout+tlsx.HandshakeTimeout)
td := tls.Dialer{NetDialer: &nd, Config: t.tlsConfig}
conn, err = td.DialContext(dctx, "tcp4", t.addr) // IPv4-only
cancel()
} else {
conn, err = nd.DialContext(ctx, "tcp4", t.addr) // IPv4-only
}
if err != nil {
return err
}
@@ -353,7 +378,8 @@ func (t *TCPChainSink) connect(ctx context.Context) error {
t.logger.Info("msg", "Chain link established",
"component", "tcp_chain_sink",
"target", t.addr,
"node", t.node)
"node", t.node,
"tls", t.tlsConfig != nil)
return nil
}