v0.26.2 http sink backlog for late streams, web updates

config/logwisp.toml

@@ -335,6 +335,7 @@ target = "stdout"                       # stdout|stderr ("split" NOT supported)
 # client_buffer_size = 256              # Per-client send queue
 # write_timeout_ms = 0                  # Per-event deadline, 0 = none
 # max_connections = 0                   # 0 = unlimited
+# replay_lines = 0                      # Last entries a new stream gets first, 0 = none
 # login_page = false                    # GET /auth/login; scram behind auth.trusted_proxies
 # viewer_page = false                   # GET /auth/view, needs login_page; scram + trusted_proxies only
                                         # (without auth or under mtls the viewer is always on)

doc/cli.md

@@ -177,7 +177,8 @@ unknown key fails and lists the valid ones. A `path` is a file, a directory
 - `tail`: follow files to stdout, like `tail -F`
   - `path` (required), `from`, `format` (`raw`)
 - `serve`: a live stream, an [http sink](sinks.md#http): a browser at
-  `listen` gets the viewer, `/stream` is SSE
+  `listen` gets the viewer, `/stream` is SSE, each stream starting with the
+  last 1000 entries
   - `path`, `from`, `format` (`json`), `listen` (`127.0.0.1:8080`)
   - `tls`: `off` (default), `self` (a self-signed certificate made at
     startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or

doc/lw.1

@@ -304,7 +304,8 @@ Follow files to standard output, like
 .It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer , allow , deny Oc
 Serve a live HTTP stream on
 .Cm listen
-.Pq default Li 127.0.0.1:8080 .
+.Pq default Li 127.0.0.1:8080 ,
+each stream starting with the last 1000 entries.
 .Cm tls
 is
 .Cm off ,

doc/sinks.md

@@ -170,6 +170,9 @@ Options, each as type and default:
 - `client_buffer_size` (int, `256`): per-client send queue depth.
 - `write_timeout_ms` (int, `0`): per-event write deadline; `0` = none.
 - `max_connections` (int, `0`): concurrent stream cap; `0` = unlimited.
+- `replay_lines` (int, `0`; `1000` from the `serve` preset): the last entries
+  a new stream gets after its `connected` event, before the live ones, so a
+  browser opened late sees a backlog; `0` = none.
 - `login_page` (bool, `false`): `scram` in proxy mode only (an error
   elsewhere): the browser login page at `/auth/login`.
 - `viewer_page` (bool, `false`): `scram` in proxy mode only (an error
@@ -212,8 +215,8 @@ Options, each as type and default:
   - never under `scram` on the sink's own TLS, which a browser cannot log in
     to: `/` is refused like any other path;
   - an endpoint at `/` keeps the root, and the viewer stays at `/auth/view`.
-  The viewer shows entries from when it connects (the sink keeps no backlog)
+  The viewer starts with the `replay_lines` backlog, holds one stream against
-  and holds one stream against `max_connections`.
+  `max_connections`, and says so when all are taken (`server full`).
 - Refusals are logged at WARN and counted in `auth_rejected`. The authorized
   identity is recorded in the client's session as `auth_method` /
   `auth_identity`.

doc/todo.md

@@ -65,16 +65,11 @@ identity, and both skeletons download the tagged source.
 
 Smaller items found along the way, each independent of the others.
 
-1. A late browser sees no backlog: the `http` sink keeps none, so the viewer
+1. Level detection. `source.ExtractLogLevel` needs a delimiter after the name
-   shows entries from when it connects. A bounded replay (a `replay_lines`
-   option, sent after `event: connected`) would fill it. The viewer also hides
-   a `503` from `max_connections` behind "retrying"; the status JSON it already
-   fetches could name it.
-2. Level detection. `source.ExtractLogLevel` needs a delimiter after the name
    (`ERROR:`, `[WARN]`, ` INFO `), so `warning disk full` or `DBG x` get no
    level, hence no color. Match the names as words, as the console sink's
    painter does, with one table for both.
-3. Command line:
+2. Command line:
    - a positional word at the top level (`lw tail`, `lw -t FILE`) still reaches
      config with an unclear message; name it ("unexpected argument X; the file
      is -c FILE");
@@ -84,11 +79,9 @@ Smaller items found along the way, each independent of the others.
    - `lw tls cert --host` could become `--hosts` (keeping `--host`), to match
      `tls.hosts` and the preset key;
    - top-level usage errors exit 1, subcommand ones 2.
-4. `GET /favicon.ico` answers 404; the logo could serve as the icon once the
+3. TLS: the generated-certificate key is one per process, so a pin taken from a
-   pages' CSP allows `img-src 'self'`.
-5. TLS: the generated-certificate key is one per process, so a pin taken from a
    `self_signed` listener also matches an `issuer` listener of the same
    process; document it or key per listener. Pins are checked in
    `VerifyPeerCertificate`, which Go skips on resumption: guard it if a dialer
    ever keeps a `ClientSessionCache`.
-6. `core.ShutdownTimeout` is unused.
+4. `core.ShutdownTimeout` is unused.

internal/config/config.go

@@ -293,6 +293,7 @@ type HTTPSinkOptions struct {
 	ClientBufferSize int64        `toml:"client_buffer_size"` // per-client send queue
 	WriteTimeoutMS   int64        `toml:"write_timeout_ms"`   // per-SSE-write deadline, 0 = none
 	MaxConnections   int64        `toml:"max_connections"`    // 0 = unlimited
+	ReplayLines      int64        `toml:"replay_lines"`       // entries a new stream gets first, 0 = none
 	LoginPage        bool         `toml:"login_page"`         // GET /auth/login, needs auth.trusted_proxies
 	ViewerPage       bool         `toml:"viewer_page"`        // GET /auth/view, needs login_page
 	Auth             *AuthOptions `toml:"auth"`

internal/config/preset.go

@@ -97,6 +97,7 @@ var presets = []Preset{
 			default:
 				return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"])
 			}
+			sink["replay_lines"] = int64(1000) // a browser opened late sees the recent entries
 			p.Flow.Format = &FormatConfig{Type: v["format"]}
 			p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "http", Type: "http", Config: sink})
 			return nil

internal/sink/http/http.go

@@ -11,6 +11,7 @@ import (
 	"net"
 	"net/http"
 	"path"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -67,8 +68,9 @@ type HTTPSink struct {
 	input  chan core.TransportEvent
 	logger *log.Logger
 
-	// Client registry
+	// Client registry, and the last replay_lines entries, under clientsMu
 	clients      map[uint64]*sseClient
+	replay       [][]byte
 	clientsMu    sync.Mutex
 	nextClientID atomic.Uint64
 	writeTimeout time.Duration
@@ -399,6 +401,9 @@ func (h *HTTPSink) brokerLoop() {
 
 			var stale []uint64
 			h.clientsMu.Lock()
+			if n := int(h.config.ReplayLines); n > 0 {
+				h.replay = append(h.replay[max(len(h.replay)+1-n, 0):], event.Payload)
+			}
 			for id, c := range h.clients {
 				if _, exists := h.proxy.GetSession(c.sessionID); !exists {
 					stale = append(stale, id)
@@ -506,18 +511,6 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Registered only now: a client the broker can queue into before its reader
-	// reaches the loop below loses a burst to a buffer nobody is draining.
-	h.clientsMu.Lock()
-	h.clients[id] = c
-	h.clientsMu.Unlock()
-
-	// A stream with nothing to carry still has to prove the peer is there. The
-	// comment refreshes the session the broker evicts on, and fails on a peer
-	// that stopped reading.
-	idle := time.NewTicker(h.keepalive)
-	defer idle.Stop()
-
 	send := func(payload []byte) bool {
 		if writeSSE(w, payload) != nil || rc.Flush() != nil {
 			return false
@@ -525,6 +518,27 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
 		h.proxy.UpdateActivity(sess.ID)
 		return true
 	}
+
+	// Registered only now: a client the broker can queue into before its reader
+	// reaches the loop below loses a burst to a buffer nobody is draining. The
+	// backlog is taken under the same lock, so it ends where the queue begins.
+	h.clientsMu.Lock()
+	h.clients[id] = c
+	backlog := slices.Clone(h.replay)
+	h.clientsMu.Unlock()
+	for _, payload := range backlog {
+		h.armWrite(rc)
+		if !send(payload) {
+			return
+		}
+	}
+
+	// A stream with nothing to carry still has to prove the peer is there. The
+	// comment refreshes the session the broker evicts on, and fails on a peer
+	// that stopped reading.
+	idle := time.NewTicker(h.keepalive)
+	defer idle.Stop()
+
 	clientGone := r.Context().Done()
 	for {
 		select {

internal/sink/http/http_test.go

@@ -229,7 +229,7 @@ func TestProxyModeServesBrowserFiles(t *testing.T) {
 	client, baseURL := serveTestHTTPSink(t, h)
 	for file, ctype := range map[string]string{
 		"scram.js": "text/javascript", "login": "text/html", "login.js": "text/javascript",
-		"style.css": "text/css", "view": "text/html", "view.js": "text/javascript",
+		"style.css": "text/css", "view": "text/html", "view.js": "text/javascript", "favicon.svg": "image/svg+xml",
 	} {
 		req, _ := http.NewRequest(http.MethodGet, baseURL+"/auth/"+file, nil)
 		resp, err := client.Do(proxied(req))
@@ -239,7 +239,7 @@ func TestProxyModeServesBrowserFiles(t *testing.T) {
 		body, _ := io.ReadAll(resp.Body)
 		resp.Body.Close()
 		if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), ctype) ||
-			!strings.Contains(resp.Header.Get("Content-Security-Policy"), "script-src 'self'") {
+			!strings.Contains(resp.Header.Get("Content-Security-Policy"), "script-src 'self'; connect-src 'self'; style-src 'self'; img-src 'self';") {
 			t.Errorf("/auth/%s: %d %q %q", file, resp.StatusCode, resp.Header.Get("Content-Type"), resp.Header.Get("Content-Security-Policy"))
 		}
 		if file == "view" && !bytes.Contains(body, []byte(`content="api/status"`)) {
@@ -474,3 +474,45 @@ func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) {
 		t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect)
 	}
 }
+
+// A stream that connects late gets the last replay_lines entries after its
+// connected event, then the live ones, none twice
+func TestLateStreamGetsTheBacklog(t *testing.T) {
+	httpSink, _ := newTestHTTPSink(t, map[string]any{"replay_lines": int64(2)})
+	client, baseURL := serveTestHTTPSink(t, httpSink)
+	until := func(cond func() bool) {
+		for deadline := time.Now().Add(2 * time.Second); !cond(); time.Sleep(10 * time.Millisecond) {
+			if time.Now().After(deadline) {
+				t.Fatal("timed out")
+			}
+		}
+	}
+	for i := range 3 {
+		httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d\n", i)}
+	}
+	until(func() bool { return httpSink.totalProcessed.Load() == 3 })
+	resp, err := client.Get(baseURL + "/stream")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = resp.Body.Close() })
+	scanner := bufio.NewScanner(resp.Body)
+	for scanner.Scan() && scanner.Text() != "event: connected" {
+	}
+	scanner.Scan() // its data
+	until(func() bool {
+		httpSink.clientsMu.Lock()
+		defer httpSink.clientsMu.Unlock()
+		return len(httpSink.clients) == 1
+	})
+	httpSink.Input() <- core.TransportEvent{Payload: []byte("line 3\n")}
+	var got []string
+	for len(got) < 3 && scanner.Scan() {
+		if line, ok := strings.CutPrefix(scanner.Text(), "data: "); ok {
+			got = append(got, line)
+		}
+	}
+	if !slices.Equal(got, []string{"line 1", "line 2", "line 3"}) {
+		t.Fatalf("stream got %q", got)
+	}
+}

internal/sink/http/web.go

@@ -16,18 +16,19 @@ import (
 	"github.com/lixenwraith/logwisp/internal/netacl"
 )
 
-//go:embed web/*.js web/*.html web/*.css
+//go:embed web/*.js web/*.html web/*.css web/*.svg
 var webFiles embed.FS
 
-// pageCSP confines the pages to their own scripts and styles: no inline code,
+// pageCSP confines the pages to their own scripts, styles and icon: no inline
-// no other origin, no framing
+// code, no other origin, no framing
 const pageCSP = "default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; " +
-	"form-action 'self'; frame-ancestors 'none'; base-uri 'none'"
+	"img-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'"
 
 var webTypes = map[string]string{
 	".js":   "text/javascript; charset=utf-8",
 	".css":  "text/css; charset=utf-8",
 	".html": "text/html; charset=utf-8",
+	".svg":  "image/svg+xml",
 }
 
 // webRoutes are the browser files under /auth/, each with the page that needs
@@ -38,6 +39,7 @@ var webRoutes = []struct {
 }{
 	{"scram.js", "scram.js", ""},
 	{"style.css", "style.css", ""},
+	{"favicon.svg", "favicon.svg", ""},
 	{"login", "login.html", "login"},
 	{"login.js", "login.js", "login"},
 	{"view", "view.html", "view"},

internal/sink/http/web/favicon.svg

@@ -0,0 +1,12 @@
+<svg viewBox="1 1 98 98" xmlns="http://www.w3.org/2000/svg">
+  <!-- The wisps of logwisp-logo.svg, broadened to read at 16 px -->
+  <defs>
+    <path id="wisp" d="M50,50 C50,37 59.2,27 71,25.8 C83.5,24.6 92,33.5 97.2,48
+      A3.6,3.6 0 0 1 90.6,51.8 C86.6,46.2 80.6,43.6 74,43.9 C65.5,44.3 57,46.6 50,50 Z"/>
+  </defs>
+  <use href="#wisp" fill="#E0409A"/>
+  <use href="#wisp" fill="#D4640A" transform="rotate(72 50 50)"/>
+  <use href="#wisp" fill="#169B58" transform="rotate(144 50 50)"/>
+  <use href="#wisp" fill="#0891B2" transform="rotate(216 50 50)"/>
+  <use href="#wisp" fill="#8E5CF0" transform="rotate(288 50 50)"/>
+</svg>

internal/sink/http/web/login.html

@@ -5,6 +5,7 @@
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>LogWisp sign-in</title>
 <link rel="stylesheet" href="style.css">
+<link rel="icon" href="favicon.svg" type="image/svg+xml">
 <script type="module" src="login.js"></script>
 </head>
 <body>

internal/sink/http/web/view.html

@@ -7,6 +7,7 @@
 <meta name="logwisp-status" content="status">
 <meta name="logwisp-login" content="scram">
 <link rel="stylesheet" href="style.css">
+<link rel="icon" href="favicon.svg" type="image/svg+xml">
 <script type="module" src="view.js"></script>
 </head>
 <body class="viewer">

internal/sink/http/web/view.js

@@ -36,8 +36,9 @@ function append(line) {
   if (atBottom) log.scrollTop = log.scrollHeight;
 }
 
-// The status answer names the stream path; its 401 is the only way to tell an
+// The status answer names the stream path and whether the stream has room; its
-// ended session apart, since EventSource hides the status of a failure.
+// 401 is the only way to tell an ended session apart, since EventSource hides
+// the status of a failure.
 async function connect() {
   if (tokenMode && !token) return showLogin("");
   let res;
@@ -52,10 +53,14 @@ async function connect() {
   // An open page has no session to end: a 401 there is a proxy's, so it retries
   if (res.status === 401 && !open) return sessionEnded();
   fresh = false;
-  const path = res.ok ? (await res.json().catch(() => ({})))?.endpoints?.stream : null;
+  const body = res.ok ? await res.json().catch(() => ({})) : {};
+  const path = body?.endpoints?.stream;
   if (typeof path !== "string" || !path.startsWith("/")) {
     return retry(`status unavailable (${res.status})`);
   }
+  // The stream would answer 503, which EventSource hides
+  const { active_clients: active, max_connections: max } = body?.server ?? {};
+  if (max > 0 && active >= max) return retry(`server full (${active} of ${max} streams)`);
   // Stream paths are absolute in logwisp; the proxy may mount it under a prefix
   const url = new URL("./" + path.replace(/^\/+/, ""), base);
   if (tokenMode) streamWithToken(url);