v0.23.0 color, presets, certificates made at startup, cli.go, sink flush

internal/sink/console/console.go

@@ -18,8 +18,10 @@ import (
 	"github.com/lixenwraith/logwisp/internal/session"
 	"github.com/lixenwraith/logwisp/internal/sink"
 
+	"github.com/lixenwraith/color"
 	lconfig "github.com/lixenwraith/config"
 	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/terminal/inline"
 	"golang.org/x/term"
 )
 
@@ -38,8 +40,9 @@ type ConsoleSink struct {
 	session *session.Session
 
 	// Configuration
-	config *config.ConsoleSinkOptions
-	escape bool
+	config  *config.ConsoleSinkOptions
+	escape  bool
+	painter *inline.Printer // paints level names; nil when color is off
 
 	// Application
 	input  chan core.TransportEvent
@@ -63,8 +66,23 @@ const (
 	DefaultConsoleTarget     = "stdout"
 	DefaultConsoleBufferSize = 1000
 	DefaultConsoleEscape     = "auto"
+	DefaultConsoleColor      = "auto"
 )
 
+// levelColors paints the first of a level's names in an entry, ANSI 16 so
+// the terminal's theme picks the shades and a text console renders them.
+// Names run longest first; cyan, not blue, stays readable on black.
+var levelColors = map[string]struct {
+	names []string
+	style inline.Style
+}{
+	"ERROR": {[]string{"ERROR", "FATAL", "ERR"}, inline.FgANSI(color.ANSIRed).Bold()},
+	"WARN":  {[]string{"WARNING", "WARN"}, inline.FgANSI(color.ANSIYellow)},
+	"INFO":  {[]string{"INFO", "INF"}, inline.FgANSI(color.ANSIGreen)},
+	"DEBUG": {[]string{"DEBUG", "DBG"}, inline.FgANSI(color.ANSICyan)},
+	"TRACE": {[]string{"TRACE"}, inline.FgANSI(color.ANSIBrightBlack)},
+}
+
 // NewConsoleSinkPlugin creates a console sink through plugin factory
 func NewConsoleSinkPlugin(
 	id string,
@@ -72,7 +90,7 @@ func NewConsoleSinkPlugin(
 	logger *log.Logger,
 	proxy *session.Proxy,
 ) (sink.Sink, error) {
-	opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape}
+	opts := &config.ConsoleSinkOptions{Target: DefaultConsoleTarget, Escape: DefaultConsoleEscape, Color: DefaultConsoleColor}
 
 	// Scan config map into struct
 	if err := config.Scan(configMap, opts); err != nil {
@@ -85,6 +103,9 @@ func NewConsoleSinkPlugin(
 	if err := lconfig.OneOf("auto", "always", "never")(opts.Escape); err != nil {
 		return nil, fmt.Errorf("escape: %w", err)
 	}
+	if err := lconfig.OneOf("auto", "always", "never")(opts.Color); err != nil {
+		return nil, fmt.Errorf("color: %w", err)
+	}
 
 	output := os.Stdout
 	if opts.Target == "stderr" {
@@ -95,17 +116,29 @@ func NewConsoleSinkPlugin(
 		opts.BufferSize = DefaultConsoleBufferSize
 	}
 
+	// inline decides auto: a terminal, NO_COLOR unset, TERM not dumb. Paint
+	// returns its text unchanged exactly when color is off.
+	printer := inline.New(output)
+	if opts.Color != "auto" {
+		printer.SetColor(opts.Color == "always")
+	}
+	var painter *inline.Printer
+	if printer.Paint("x", inline.Style{}) != "x" {
+		painter = printer
+	}
+
 	// Create and return plugin instance
 	cs := &ConsoleSink{
-		id:     id,
-		proxy:  proxy,
-		config: opts,
-		escape: opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())),
-		input:  make(chan core.TransportEvent, opts.BufferSize),
-		output: output,
-		done:   make(chan struct{}),
-		exited: make(chan struct{}),
-		logger: logger,
+		id:      id,
+		proxy:   proxy,
+		config:  opts,
+		escape:  opts.Escape == "always" || opts.Escape == "auto" && term.IsTerminal(int(output.Fd())),
+		painter: painter,
+		input:   make(chan core.TransportEvent, opts.BufferSize),
+		output:  output,
+		done:    make(chan struct{}),
+		exited:  make(chan struct{}),
+		logger:  logger,
 	}
 	cs.lastProcessed.Store(time.Time{})
 
@@ -124,6 +157,7 @@ func NewConsoleSinkPlugin(
 		"instance_id", id,
 		"target", opts.Target,
 		"escape", cs.escape,
+		"color", painter != nil,
 	)
 
 	return cs, nil
@@ -187,6 +221,7 @@ func (cs *ConsoleSink) GetStats() sink.SinkStats {
 			"target":      cs.config.Target,
 			"buffer_size": cs.config.BufferSize,
 			"escape":      cs.escape,
+			"color":       cs.painter != nil,
 		},
 	}
 }
@@ -224,6 +259,9 @@ func (cs *ConsoleSink) write(event core.TransportEvent) {
 	if cs.escape {
 		payload = escapeControls(payload)
 	}
+	if cs.painter != nil {
+		payload = paintLevel(cs.painter, payload, event.Entry.Level)
+	}
 	if _, err := cs.output.Write(payload); err != nil {
 		cs.logger.Error("msg", "Failed to write to console",
 			"component", "console_sink",
@@ -235,6 +273,42 @@ func (cs *ConsoleSink) write(event core.TransportEvent) {
 	cs.lastProcessed.Store(time.Now())
 }
 
+// paintLevel colors the first whole-word, case-insensitive occurrence of a
+// name of level, after escaping: the escape sequences are the sink's own.
+func paintLevel(p *inline.Printer, payload []byte, level string) []byte {
+	lc, ok := levelColors[level]
+	if !ok {
+		return payload
+	}
+	for _, name := range lc.names {
+		for i := 0; i+len(name) <= len(payload); i++ {
+			j := i + len(name)
+			if !nameAt(payload[i:j], name) || i > 0 && word(payload[i-1]) || j < len(payload) && word(payload[j]) {
+				continue
+			}
+			out := make([]byte, 0, len(payload)+16)
+			out = append(out, payload[:i]...)
+			out = append(out, p.Paint(string(payload[i:j]), lc.style)...)
+			return append(out, payload[j:]...)
+		}
+	}
+	return payload
+}
+
+// nameAt compares ASCII letters without case; name is upper case
+func nameAt(b []byte, name string) bool {
+	for k := range len(name) {
+		if c := b[k]; c != name[k] && c != name[k]+'a'-'A' {
+			return false
+		}
+	}
+	return true
+}
+
+func word(c byte) bool {
+	return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '_'
+}
+
 // escapeControls writes what a terminal would act on or reorder (C0 and C1
 // controls but tab, DEL, bidi controls, line separators, invalid UTF-8) as
 // <hex>, keeping the final newline: a log line cannot drive the terminal.

internal/sink/console/console_test.go

@@ -1,6 +1,11 @@
 package console
 
-import "testing"
+import (
+	"io"
+	"testing"
+
+	"github.com/lixenwraith/terminal/inline"
+)
 
 // Everything a terminal would act on is escaped; tabs and the final newline,
 // which it only lays out, are kept.
@@ -23,3 +28,22 @@ func TestEscapeControlsKeepsOnlyLayout(t *testing.T) {
 		}
 	}
 }
+
+// The entry's level is painted where its line first names it as a whole
+// word, in any case, longest name first; other words and levels stay plain.
+func TestPaintLevelColorsTheFirstWholeName(t *testing.T) {
+	p := inline.New(io.Discard)
+	p.SetColor(true)
+	red, yellow := "\x1b[0;1;31m", "\x1b[0;33m"
+	for _, c := range []struct{ in, level, want string }{
+		{"2026 ERROR boom ERROR\n", "ERROR", "2026 " + red + "ERROR\x1b[0m boom ERROR\n"},
+		{"[warning] disk\n", "WARN", "[" + yellow + "warning\x1b[0m] disk\n"},
+		{"terror_x ERR:1\n", "ERROR", "terror_x " + red + "ERR\x1b[0m:1\n"},
+		{"INFOS informal\n", "INFO", "INFOS informal\n"},
+		{"ERROR without level\n", "", "ERROR without level\n"},
+	} {
+		if got := string(paintLevel(p, []byte(c.in), c.level)); got != c.want {
+			t.Errorf("%q %s: got %q, want %q", c.in, c.level, got, c.want)
+		}
+	}
+}

internal/sink/http/http.go

@@ -43,7 +43,6 @@ const (
 	DefaultHTTPStreamPath       = "/stream"
 	DefaultHTTPStatusPath       = "/status"
 	HTTPReadHeaderTimeout       = 10 * time.Second
-	HTTPShutdownTimeout         = 2 * time.Second
 )
 
 // HTTPSink streams log entries via Server-Sent Events
@@ -80,8 +79,14 @@ type HTTPSink struct {
 	auth *authz.Policy
 	web  map[string]http.Handler // proxy mode: GET /auth/... browser files
 
-	// Runtime
+	// Runtime. Stop closes flush; the broker then moves what remains of its
+	// input to tail and closes flushed; each stream writes its queue, tail
+	// and the disconnect event on its own, until flushBy.
 	done      chan struct{}
+	flush     chan struct{}
+	flushed   chan struct{}
+	flushBy   time.Time
+	tail      [][]byte // read only once flushed is closed
 	stopOnce  sync.Once
 	wg        sync.WaitGroup
 	startTime time.Time
@@ -145,7 +150,7 @@ func NewHTTPSinkPlugin(
 	if opts.ClientBufferSize <= 0 {
 		opts.ClientBufferSize = DefaultHTTPClientBufferSize
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -174,6 +179,8 @@ func NewHTTPSinkPlugin(
 		network:      network,
 		input:        make(chan core.TransportEvent, opts.BufferSize),
 		done:         make(chan struct{}),
+		flush:        make(chan struct{}),
+		flushed:      make(chan struct{}),
 		logger:       logger,
 		clients:      make(map[uint64]*sseClient),
 		writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
@@ -276,7 +283,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error {
 	h.startTime = time.Now()
 
 	h.wg.Add(1)
-	go h.brokerLoop(ctx)
+	go h.brokerLoop()
 
 	serve := h.server.Serve
 	if h.tlsConfig != nil {
@@ -325,18 +332,21 @@ func (h *HTTPSink) Stop() {
 }
 
 // shutdown funnels ctx-cancel and Stop() teardown through a single path.
-// done is closed first so SSE handlers exit and Shutdown can complete;
-// Server.Close force-closes any handler stalled in a deadline-free write.
+// Server.Shutdown refuses new streams and waits for the open ones, which end
+// once they wrote what is queued, within sink.FlushBound; Server.Close cuts a
+// stream stalled past it.
 func (h *HTTPSink) shutdown() {
 	h.stopOnce.Do(func() {
-		close(h.done)
 		if h.server != nil {
-			sctx, cancel := context.WithTimeout(context.Background(), HTTPShutdownTimeout)
+			h.flushBy = time.Now().Add(sink.FlushBound(h.writeTimeout))
+			ctx, cancel := context.WithDeadline(context.Background(), h.flushBy)
 			defer cancel()
-			if err := h.server.Shutdown(sctx); err != nil {
+			close(h.flush)
+			if err := h.server.Shutdown(ctx); err != nil {
 				h.server.Close()
 			}
 		}
+		close(h.done)
 	})
 }
 
@@ -356,19 +366,22 @@ func (h *HTTPSink) removeClient(id uint64) {
 }
 
 // brokerLoop fans out transport events to all client queues, non-blocking,
-// and evicts clients whose sessions were idle-expired by the session manager
-func (h *HTTPSink) brokerLoop(ctx context.Context) {
+// and evicts clients whose sessions were idle-expired by the session manager,
+// until flush, which shutdown closes on Stop and on context cancellation.
+func (h *HTTPSink) brokerLoop() {
 	defer h.wg.Done()
 	for {
 		select {
-		case <-ctx.Done():
+		case <-h.flush: // first: a select with both ready picks at random
+			h.flushInput()
 			return
-		case <-h.done:
+		default:
+		}
+		select {
+		case <-h.flush:
+			h.flushInput()
 			return
-		case event, ok := <-h.input:
-			if !ok {
-				return
-			}
+		case event := <-h.input:
 			h.totalProcessed.Add(1)
 			h.lastProcessed.Store(time.Now())
 
@@ -395,6 +408,13 @@ func (h *HTTPSink) brokerLoop(ctx context.Context) {
 	}
 }
 
+// flushInput moves what remains of the input to tail for the streams
+func (h *HTTPSink) flushInput() {
+	h.tail = sink.Drain(h.input)
+	h.totalProcessed.Add(uint64(len(h.tail)))
+	close(h.flushed)
+}
+
 // handleStream serves one client's SSE stream
 func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 	if h.config.MaxConnections > 0 && h.activeClients.Load() >= h.config.MaxConnections {
@@ -483,6 +503,13 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 	idle := time.NewTicker(h.keepalive)
 	defer idle.Stop()
 
+	send := func(payload []byte) bool {
+		if writeSSE(w, payload) != nil || rc.Flush() != nil {
+			return false
+		}
+		h.proxy.UpdateActivity(sess.ID)
+		return true
+	}
 	clientGone := r.Context().Done()
 	for {
 		select {
@@ -491,13 +518,9 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 				return // broker evicted (stale session)
 			}
 			h.armWrite(rc)
-			if err := writeSSE(w, payload); err != nil {
+			if !send(payload) {
 				return
 			}
-			if err := rc.Flush(); err != nil {
-				return
-			}
-			h.proxy.UpdateActivity(sess.ID)
 		case <-idle.C:
 			h.armWrite(rc)
 			if _, err := fmt.Fprint(w, ":\n\n"); err != nil {
@@ -509,7 +532,20 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 			h.proxy.UpdateActivity(sess.ID)
 		case <-clientGone:
 			return
-		case <-h.done:
+		case <-h.flushed:
+			// Nothing more will be queued: write the queue and the tail, then
+			// say why the stream ends
+			_ = rc.SetWriteDeadline(h.flushBy)
+			for len(c.send) > 0 {
+				if payload, ok := <-c.send; !ok || !send(payload) {
+					return
+				}
+			}
+			for _, payload := range h.tail {
+				if !send(payload) {
+					return
+				}
+			}
 			fmt.Fprintf(w, "event: disconnect\ndata: {\"reason\":\"server_shutdown\"}\n\n")
 			rc.Flush()
 			return

internal/sink/http/http_test.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"fmt"
 	"io"
 	"maps"
 	"net/http"
@@ -319,3 +320,73 @@ func TestSSEFramesEveryLineAsData(t *testing.T) {
 		t.Fatalf("framed %q, want %q", got, want)
 	}
 }
+
+// Stop writes what is queued, in the sink and per stream, then the
+// disconnect event, to a client that keeps reading, whatever another that
+// stopped reading does; Stop returns at the flush bound.
+func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) {
+	const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold
+	httpSink, _ := newTestHTTPSink(t, map[string]any{"buffer_size": int64(n), "client_buffer_size": int64(64), "write_timeout_ms": int64(500)})
+	client, baseURL := serveTestHTTPSink(t, httpSink)
+	var scanners []*bufio.Scanner
+	for range 2 {
+		resp, err := client.Get(baseURL + "/stream")
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { _ = resp.Body.Close() })
+		scanner := bufio.NewScanner(resp.Body)
+		scanner.Buffer(nil, 64*1024)
+		for scanner.Scan() && scanner.Text() != "event: connected" {
+		}
+		scanners = append(scanners, scanner)
+	}
+	for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+		httpSink.clientsMu.Lock()
+		registered := len(httpSink.clients)
+		httpSink.clientsMu.Unlock()
+		if registered == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("streams never registered")
+		}
+	}
+	type result struct {
+		lines      int
+		disconnect bool
+	}
+	done := make(chan result)
+	go func() {
+		var r result
+		for scanners[0].Scan() {
+			switch line := scanners[0].Text(); {
+			case strings.HasPrefix(line, "data: line "):
+				r.lines++
+			case line == "event: disconnect":
+				r.disconnect = true
+			}
+		}
+		done <- r
+	}()
+	// Held, the lock keeps the input queued until Stop has begun its flush
+	httpSink.clientsMu.Lock()
+	padding := strings.Repeat("x", 8192)
+	for i := range n {
+		httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)}
+	}
+	stopped := make(chan time.Duration)
+	go func() {
+		start := time.Now()
+		httpSink.Stop()
+		stopped <- time.Since(start)
+	}()
+	time.Sleep(100 * time.Millisecond)
+	httpSink.clientsMu.Unlock()
+	if d := <-stopped; d > 1500*time.Millisecond {
+		t.Errorf("Stop took %v past a 500 ms flush bound", d)
+	}
+	if r := <-done; r.lines != n || !r.disconnect {
+		t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect)
+	}
+}

internal/sink/sink.go

@@ -35,3 +35,27 @@ type SinkStats struct {
 	LastProcessed     time.Time
 	Details           map[string]any
 }
+
+// FlushBound is how long a network sink's Stop gives its clients to take
+// what is queued: write_timeout_ms, at most core.SinkFlushTimeout, which a
+// reload also waits out for a client that stopped reading.
+func FlushBound(writeTimeout time.Duration) time.Duration {
+	if writeTimeout > 0 && writeTimeout < core.SinkFlushTimeout {
+		return writeTimeout
+	}
+	return core.SinkFlushTimeout
+}
+
+// Drain takes the payloads queued in input without waiting: a network sink's
+// last events, which each client then writes on its own.
+func Drain(input <-chan core.TransportEvent) [][]byte {
+	var tail [][]byte
+	for {
+		select {
+		case event := <-input:
+			tail = append(tail, event.Payload)
+		default:
+			return tail
+		}
+	}
+}

internal/sink/tcp/tcp.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"maps"
 	"net"
+	"slices"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -74,8 +75,15 @@ type TCPSink struct {
 	// Authorization
 	auth *authz.Policy
 
-	// Runtime
+	// Runtime. Stop closes flush; the broadcast loop then moves what remains
+	// of its input to tail and closes flushed; each writer writes its queue
+	// and tail on its own, until flushBy. done tears down what is left.
 	done      chan struct{}
+	flush     chan struct{}
+	flushed   chan struct{}
+	flushBy   time.Time
+	tail      [][]byte // read only once flushed is closed
+	flushing  bool     // under clientsMu: no client registers once set
 	stopOnce  sync.Once
 	wg        sync.WaitGroup
 	startTime time.Time
@@ -91,11 +99,13 @@ type TCPSink struct {
 
 // tcpClient is a registered connection's bounded send queue.
 // send is written by the broadcast loop (non-blocking) and drained by the
-// writer goroutine; closed signals reader-detected disconnect.
+// writer goroutine; closed signals reader-detected disconnect, exited the
+// writer's end.
 type tcpClient struct {
 	send      chan []byte
 	sessionID string
 	closed    chan struct{}
+	exited    chan struct{}
 }
 
 // NewTCPSinkPlugin creates a tcp sink through plugin factory
@@ -131,7 +141,7 @@ func NewTCPSinkPlugin(
 	if opts.KeepAlivePeriodMS <= 0 {
 		opts.KeepAlivePeriodMS = DefaultTCPKeepAlivePeriodMS
 	}
-	tlsCfg, err := tlsx.Server(opts.TLS)
+	tlsCfg, err := tlsx.Server(opts.TLS, opts.Host)
 	if err != nil {
 		return nil, err
 	}
@@ -148,6 +158,8 @@ func NewTCPSinkPlugin(
 		network:      network,
 		input:        make(chan core.TransportEvent, opts.BufferSize),
 		done:         make(chan struct{}),
+		flush:        make(chan struct{}),
+		flushed:      make(chan struct{}),
 		logger:       logger,
 		clients:      make(map[uint64]*tcpClient),
 		conns:        make(map[net.Conn]struct{}),
@@ -223,7 +235,7 @@ func (t *TCPSink) Start(ctx context.Context) error {
 
 	t.wg.Add(2)
 	go t.acceptLoop()
-	go t.broadcastLoop(ctx)
+	go t.broadcastLoop()
 
 	// Pipeline context cancellation mirrors gnet engine stop: cease accepting
 	// and tear down existing connections
@@ -258,13 +270,27 @@ func (t *TCPSink) Stop() {
 		"total_processed", t.totalProcessed.Load())
 }
 
-// shutdown funnels ctx-cancel and Stop() teardown through a single path
+// shutdown funnels ctx-cancel and Stop() teardown through a single path.
+// Clients first receive what is queued, within sink.FlushBound, so a finite
+// input reaches them whole.
 func (t *TCPSink) shutdown() {
 	t.stopOnce.Do(func() {
-		close(t.done)
 		if t.listener != nil {
 			t.listener.Close() // unblocks acceptLoop
+			t.flushBy = time.Now().Add(sink.FlushBound(t.writeTimeout))
+			close(t.flush)
+			t.clientsMu.Lock()
+			t.flushing = true
+			clients := slices.Collect(maps.Values(t.clients))
+			t.clientsMu.Unlock()
+			for _, c := range clients {
+				select {
+				case <-c.exited:
+				case <-time.After(time.Until(t.flushBy)):
+				}
+			}
 		}
+		close(t.done)
 		t.clientsMu.Lock()
 		for conn := range t.conns {
 			conn.Close() // unblocks handshakes and per-connection readers
@@ -381,10 +407,17 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 		send:      make(chan []byte, t.config.ClientBufferSize),
 		sessionID: sess.ID,
 		closed:    make(chan struct{}),
+		exited:    make(chan struct{}),
 	}
+	defer close(c.exited)
 	id := t.nextClientID.Add(1)
 
 	t.clientsMu.Lock()
+	if t.flushing {
+		t.clientsMu.Unlock()
+		t.proxy.RemoveSession(sess.ID)
+		return
+	}
 	t.clients[id] = c
 	t.clientsMu.Unlock()
 
@@ -422,23 +455,45 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 
 	// Writer: synchronous lib write with deadline. A failed write means
 	// the kernel buffer stayed full for the full deadline - connection is
-	// dead or hopelessly stalled, so disconnect immediately (no gnet-style
-	// consecutive-error counter needed for transient async callback errors).
+	// dead or hopelessly stalled, so disconnect immediately.
+	write := func(data []byte, deadline time.Time) bool {
+		if !deadline.IsZero() {
+			conn.SetWriteDeadline(deadline)
+		}
+		if _, err := conn.Write(data); err != nil {
+			t.writeErrors.Add(1)
+			t.logger.Debug("msg", "Write failed, closing client",
+				"component", "tcp_sink",
+				"remote_addr", remote,
+				"error", err)
+			return false
+		}
+		t.proxy.UpdateActivity(sess.ID)
+		return true
+	}
 	for {
 		select {
 		case data := <-c.send:
+			var deadline time.Time
 			if t.writeTimeout > 0 {
-				conn.SetWriteDeadline(time.Now().Add(t.writeTimeout))
+				deadline = time.Now().Add(t.writeTimeout)
 			}
-			if _, err := conn.Write(data); err != nil {
-				t.writeErrors.Add(1)
-				t.logger.Debug("msg", "Write failed, closing client",
-					"component", "tcp_sink",
-					"remote_addr", remote,
-					"error", err)
+			if !write(data, deadline) {
 				return
 			}
-			t.proxy.UpdateActivity(sess.ID)
+		case <-t.flushed:
+			// Nothing more will be queued: write the queue, then the tail
+			for len(c.send) > 0 {
+				if !write(<-c.send, t.flushBy) {
+					return
+				}
+			}
+			for _, data := range t.tail {
+				if !write(data, t.flushBy) {
+					return
+				}
+			}
+			return
 		case <-c.closed:
 			return
 		case <-t.done:
@@ -447,19 +502,22 @@ func (t *TCPSink) handleConn(conn net.Conn) {
 	}
 }
 
-// broadcastLoop fans out transport events to all client queues, non-blocking
-func (t *TCPSink) broadcastLoop(ctx context.Context) {
+// broadcastLoop fans out transport events to all client queues until flush,
+// which shutdown closes on Stop and on context cancellation alike.
+func (t *TCPSink) broadcastLoop() {
 	defer t.wg.Done()
 	for {
 		select {
-		case <-ctx.Done():
+		case <-t.flush: // first: a select with both ready picks at random
+			t.flushInput()
 			return
-		case <-t.done:
+		default:
+		}
+		select {
+		case <-t.flush:
+			t.flushInput()
 			return
-		case event, ok := <-t.input:
-			if !ok {
-				return
-			}
+		case event := <-t.input:
 			t.totalProcessed.Add(1)
 			t.lastProcessed.Store(time.Now())
 
@@ -477,6 +535,13 @@ func (t *TCPSink) broadcastLoop(ctx context.Context) {
 	}
 }
 
+// flushInput moves what remains of the input to tail for the writers
+func (t *TCPSink) flushInput() {
+	t.tail = sink.Drain(t.input)
+	t.totalProcessed.Add(uint64(len(t.tail)))
+	close(t.flushed)
+}
+
 // GetStats returns sink statistics
 func (t *TCPSink) GetStats() sink.SinkStats {
 	lastProc, _ := t.lastProcessed.Load().(time.Time)

internal/sink/tcp/tcp_test.go

@@ -1,11 +1,15 @@
 package tcp
 
 import (
+	"bufio"
+	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/lixenwraith/logwisp/internal/core"
 	"github.com/lixenwraith/logwisp/internal/session"
 	"github.com/lixenwraith/logwisp/internal/testutil"
 
@@ -57,3 +61,81 @@ func TestStopDoesNotWaitForSilentHandshake(t *testing.T) {
 		t.Fatalf("Stop took %v with a silent peer in the handshake", d)
 	}
 }
+
+// Stop writes what is queued, in the sink and per client, to a client that
+// keeps reading, whatever another that stopped reading does: the end of a
+// finite input reaches it whole, and Stop returns at the flush bound.
+func TestStopWritesQueuedEventsToReadingClients(t *testing.T) {
+	const n = 2000 // of 8 KiB: more than a silent peer's socket buffers hold
+	probe, err := net.Listen("tcp4", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	addr := probe.Addr().String()
+	probe.Close()
+	_, port, _ := net.SplitHostPort(addr)
+	p, _ := strconv.ParseInt(port, 10, 64)
+
+	manager := session.NewManager(time.Hour)
+	t.Cleanup(manager.Stop)
+	created, err := NewTCPSinkPlugin("out", map[string]any{
+		"host": "127.0.0.1", "port": p, "buffer_size": int64(n), "client_buffer_size": int64(64),
+		"write_timeout_ms": int64(500),
+	}, log.NewLogger(), session.NewProxy(manager, "out"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sink := created.(*TCPSink)
+	if err := sink.Start(t.Context()); err != nil {
+		t.Fatal(err)
+	}
+	var conns []net.Conn
+	for range 2 {
+		conn, err := net.Dial("tcp4", addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { conn.Close() })
+		conns = append(conns, conn)
+	}
+	for deadline := time.Now().Add(2 * time.Second); ; time.Sleep(10 * time.Millisecond) {
+		sink.clientsMu.Lock()
+		registered := len(sink.clients)
+		sink.clientsMu.Unlock()
+		if registered == 2 {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("clients never registered")
+		}
+	}
+	lines := make(chan int)
+	go func() {
+		count, scanner := 0, bufio.NewScanner(conns[0])
+		scanner.Buffer(nil, 64*1024)
+		for scanner.Scan() {
+			count++
+		}
+		lines <- count
+	}()
+	// Held, the lock keeps the input queued until Stop has begun its flush
+	sink.clientsMu.Lock()
+	padding := strings.Repeat("x", 8192)
+	for i := range n {
+		sink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d %s\n", i, padding)}
+	}
+	stopped := make(chan time.Duration)
+	go func() {
+		start := time.Now()
+		sink.Stop()
+		stopped <- time.Since(start)
+	}()
+	time.Sleep(100 * time.Millisecond)
+	sink.clientsMu.Unlock()
+	if d := <-stopped; d > 1500*time.Millisecond {
+		t.Errorf("Stop took %v past a 500 ms flush bound", d)
+	}
+	if got := <-lines; got != n {
+		t.Fatalf("reading client received %d of %d lines", got, n)
+	}
+}

internal/sink/tcpchain/tcpchain.go

@@ -329,8 +329,9 @@ func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
 					return false
 				}
 				t.failures++
-				if errors.Is(err, authz.ErrRefused) {
-					// A refusal is configuration, not weather: show it by default
+				if errors.Is(err, authz.ErrRefused) || errors.As(err, new(*tls.CertificateVerificationError)) {
+					// A refusal, or a server failing its CA or pin, is
+					// configuration, not weather: show it by default
 					t.logger.Warn("msg", "Chain connect refused",
 						"component", "tcp_chain_sink",
 						"target", t.addr,