v0.26.2 http sink backlog for late streams, web updates

internal/config/config.go

@@ -293,6 +293,7 @@ type HTTPSinkOptions struct {
 	ClientBufferSize int64        `toml:"client_buffer_size"` // per-client send queue
 	WriteTimeoutMS   int64        `toml:"write_timeout_ms"`   // per-SSE-write deadline, 0 = none
 	MaxConnections   int64        `toml:"max_connections"`    // 0 = unlimited
+	ReplayLines      int64        `toml:"replay_lines"`       // entries a new stream gets first, 0 = none
 	LoginPage        bool         `toml:"login_page"`         // GET /auth/login, needs auth.trusted_proxies
 	ViewerPage       bool         `toml:"viewer_page"`        // GET /auth/view, needs login_page
 	Auth             *AuthOptions `toml:"auth"`

internal/config/preset.go

@@ -97,6 +97,7 @@ var presets = []Preset{
 			default:
 				return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"])
 			}
+			sink["replay_lines"] = int64(1000) // a browser opened late sees the recent entries
 			p.Flow.Format = &FormatConfig{Type: v["format"]}
 			p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "http", Type: "http", Config: sink})
 			return nil

internal/sink/http/http.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -67,8 +68,9 @@ type HTTPSink struct {
 	input  chan core.TransportEvent
 	logger *log.Logger
 
-	// Client registry
+	// Client registry, and the last replay_lines entries, under clientsMu
 	clients      map[uint64]*sseClient
+	replay       [][]byte
 	clientsMu    sync.Mutex
 	nextClientID atomic.Uint64
 	writeTimeout time.Duration
@@ -399,6 +401,9 @@ func (h *HTTPSink) brokerLoop() {
 
 			var stale []uint64
 			h.clientsMu.Lock()
+			if n := int(h.config.ReplayLines); n > 0 {
+				h.replay = append(h.replay[max(len(h.replay)+1-n, 0):], event.Payload)
+			}
 			for id, c := range h.clients {
 				if _, exists := h.proxy.GetSession(c.sessionID); !exists {
 					stale = append(stale, id)
@@ -506,18 +511,6 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Registered only now: a client the broker can queue into before its reader
-	// reaches the loop below loses a burst to a buffer nobody is draining.
-	h.clientsMu.Lock()
-	h.clients[id] = c
-	h.clientsMu.Unlock()
-
-	// A stream with nothing to carry still has to prove the peer is there. The
-	// comment refreshes the session the broker evicts on, and fails on a peer
-	// that stopped reading.
-	idle := time.NewTicker(h.keepalive)
-	defer idle.Stop()
-
 	send := func(payload []byte) bool {
 		if writeSSE(w, payload) != nil || rc.Flush() != nil {
 			return false
@@ -525,6 +518,27 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 		h.proxy.UpdateActivity(sess.ID)
 		return true
 	}
+
+	// Registered only now: a client the broker can queue into before its reader
+	// reaches the loop below loses a burst to a buffer nobody is draining. The
+	// backlog is taken under the same lock, so it ends where the queue begins.
+	h.clientsMu.Lock()
+	h.clients[id] = c
+	backlog := slices.Clone(h.replay)
+	h.clientsMu.Unlock()
+	for _, payload := range backlog {
+		h.armWrite(rc)
+		if !send(payload) {
+			return
+		}
+	}
+
+	// A stream with nothing to carry still has to prove the peer is there. The
+	// comment refreshes the session the broker evicts on, and fails on a peer
+	// that stopped reading.
+	idle := time.NewTicker(h.keepalive)
+	defer idle.Stop()
+
 	clientGone := r.Context().Done()
 	for {
 		select {

internal/sink/http/http_test.go

@@ -229,7 +229,7 @@ func TestProxyModeServesBrowserFiles(t *testing.T) {
 	client, baseURL := serveTestHTTPSink(t, h)
 	for file, ctype := range map[string]string{
 		"scram.js": "text/javascript", "login": "text/html", "login.js": "text/javascript",
-		"style.css": "text/css", "view": "text/html", "view.js": "text/javascript",
+		"style.css": "text/css", "view": "text/html", "view.js": "text/javascript", "favicon.svg": "image/svg+xml",
 	} {
 		req, _ := http.NewRequest(http.MethodGet, baseURL+"/auth/"+file, nil)
 		resp, err := client.Do(proxied(req))
@@ -239,7 +239,7 @@ func TestProxyModeServesBrowserFiles(t *testing.T) {
 		body, _ := io.ReadAll(resp.Body)
 		resp.Body.Close()
 		if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), ctype) ||
-			!strings.Contains(resp.Header.Get("Content-Security-Policy"), "script-src 'self'") {
+			!strings.Contains(resp.Header.Get("Content-Security-Policy"), "script-src 'self'; connect-src 'self'; style-src 'self'; img-src 'self';") {
 			t.Errorf("/auth/%s: %d %q %q", file, resp.StatusCode, resp.Header.Get("Content-Type"), resp.Header.Get("Content-Security-Policy"))
 		}
 		if file == "view" && !bytes.Contains(body, []byte(`content="api/status"`)) {
@@ -474,3 +474,45 @@ func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) {
 		t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect)
 	}
 }
+
+// A stream that connects late gets the last replay_lines entries after its
+// connected event, then the live ones, none twice
+func TestLateStreamGetsTheBacklog(t *testing.T) {
+	httpSink, _ := newTestHTTPSink(t, map[string]any{"replay_lines": int64(2)})
+	client, baseURL := serveTestHTTPSink(t, httpSink)
+	until := func(cond func() bool) {
+		for deadline := time.Now().Add(2 * time.Second); !cond(); time.Sleep(10 * time.Millisecond) {
+			if time.Now().After(deadline) {
+				t.Fatal("timed out")
+			}
+		}
+	}
+	for i := range 3 {
+		httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d\n", i)}
+	}
+	until(func() bool { return httpSink.totalProcessed.Load() == 3 })
+	resp, err := client.Get(baseURL + "/stream")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = resp.Body.Close() })
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() && scanner.Text() != "event: connected" {
+	}
+	scanner.Scan() // its data
+	until(func() bool {
+		httpSink.clientsMu.Lock()
+		defer httpSink.clientsMu.Unlock()
+		return len(httpSink.clients) == 1
+	})
+	httpSink.Input() <- core.TransportEvent{Payload: []byte("line 3\n")}
+	var got []string
+	for len(got) < 3 && scanner.Scan() {
+		if line, ok := strings.CutPrefix(scanner.Text(), "data: "); ok {
+			got = append(got, line)
+		}
+	}
+	if !slices.Equal(got, []string{"line 1", "line 2", "line 3"}) {
+		t.Fatalf("stream got %q", got)
+	}
+}

internal/sink/http/web.go

@@ -16,18 +16,19 @@ import (
 	"github.com/lixenwraith/logwisp/internal/netacl"
 )
 
-//go:embed web/*.js web/*.html web/*.css
+//go:embed web/*.js web/*.html web/*.css web/*.svg
 var webFiles embed.FS
 
-// pageCSP confines the pages to their own scripts and styles: no inline code,
-// no other origin, no framing
+// pageCSP confines the pages to their own scripts, styles and icon: no inline
+// code, no other origin, no framing
 const pageCSP = "default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; " +
-	"form-action 'self'; frame-ancestors 'none'; base-uri 'none'"
+	"img-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'"
 
 var webTypes = map[string]string{
 	".js":   "text/javascript; charset=utf-8",
 	".css":  "text/css; charset=utf-8",
 	".html": "text/html; charset=utf-8",
+	".svg":  "image/svg+xml",
 }
 
 // webRoutes are the browser files under /auth/, each with the page that needs
@@ -38,6 +39,7 @@ var webRoutes = []struct {
 }{
 	{"scram.js", "scram.js", ""},
 	{"style.css", "style.css", ""},
+	{"favicon.svg", "favicon.svg", ""},
 	{"login", "login.html", "login"},
 	{"login.js", "login.js", "login"},
 	{"view", "view.html", "view"},

internal/sink/http/web/favicon.svg

@@ -0,0 +1,12 @@
+<svg viewBox="1 1 98 98" xmlns="http://www.w3.org/2000/svg">
+  <!-- The wisps of logwisp-logo.svg, broadened to read at 16 px -->
+  <defs>
+    <path id="wisp" d="M50,50 C50,37 59.2,27 71,25.8 C83.5,24.6 92,33.5 97.2,48
+      A3.6,3.6 0 0 1 90.6,51.8 C86.6,46.2 80.6,43.6 74,43.9 C65.5,44.3 57,46.6 50,50 Z"/>
+  </defs>
+  <use href="#wisp" fill="#E0409A"/>
+  <use href="#wisp" fill="#D4640A" transform="rotate(72 50 50)"/>
+  <use href="#wisp" fill="#169B58" transform="rotate(144 50 50)"/>
+  <use href="#wisp" fill="#0891B2" transform="rotate(216 50 50)"/>
+  <use href="#wisp" fill="#8E5CF0" transform="rotate(288 50 50)"/>
+</svg>

internal/sink/http/web/login.html

@@ -5,6 +5,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>LogWisp sign-in</title>
 <link rel="stylesheet" href="style.css">
+<link rel="icon" href="favicon.svg" type="image/svg+xml">
 <script type="module" src="login.js"></script>
 </head>
 <body>

internal/sink/http/web/view.html

@@ -7,6 +7,7 @@
 <meta name="logwisp-status" content="status">
 <meta name="logwisp-login" content="scram">
 <link rel="stylesheet" href="style.css">
+<link rel="icon" href="favicon.svg" type="image/svg+xml">
 <script type="module" src="view.js"></script>
 </head>
 <body class="viewer">

internal/sink/http/web/view.js

@@ -36,8 +36,9 @@ function append(line) {
   if (atBottom) log.scrollTop = log.scrollHeight;
 }
 
-// The status answer names the stream path; its 401 is the only way to tell an
-// ended session apart, since EventSource hides the status of a failure.
+// The status answer names the stream path and whether the stream has room; its
+// 401 is the only way to tell an ended session apart, since EventSource hides
+// the status of a failure.
 async function connect() {
   if (tokenMode && !token) return showLogin("");
   let res;
@@ -52,10 +53,14 @@ async function connect() {
   // An open page has no session to end: a 401 there is a proxy's, so it retries
   if (res.status === 401 && !open) return sessionEnded();
   fresh = false;
-  const path = res.ok ? (await res.json().catch(() => ({})))?.endpoints?.stream : null;
+  const body = res.ok ? await res.json().catch(() => ({})) : {};
+  const path = body?.endpoints?.stream;
   if (typeof path !== "string" || !path.startsWith("/")) {
     return retry(`status unavailable (${res.status})`);
   }
+  // The stream would answer 503, which EventSource hides
+  const { active_clients: active, max_connections: max } = body?.server ?? {};
+  if (max > 0 && active >= max) return retry(`server full (${active} of ${max} streams)`);
   // Stream paths are absolute in logwisp; the proxy may mount it under a prefix
   const url = new URL("./" + path.replace(/^\/+/, ""), base);
   if (tokenMode) streamWithToken(url);