diff --git a/config/logwisp.toml b/config/logwisp.toml index 6f9e58b..ad67033 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -335,6 +335,7 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # client_buffer_size = 256 # Per-client send queue # write_timeout_ms = 0 # Per-event deadline, 0 = none # max_connections = 0 # 0 = unlimited +# replay_lines = 0 # Last entries a new stream gets first, 0 = none # login_page = false # GET /auth/login; scram behind auth.trusted_proxies # viewer_page = false # GET /auth/view, needs login_page; scram + trusted_proxies only # (without auth or under mtls the viewer is always on) diff --git a/doc/cli.md b/doc/cli.md index d02f8b2..8a6951e 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -177,7 +177,8 @@ unknown key fails and lists the valid ones. A `path` is a file, a directory - `tail`: follow files to stdout, like `tail -F` - `path` (required), `from`, `format` (`raw`) - `serve`: a live stream, an [http sink](sinks.md#http): a browser at - `listen` gets the viewer, `/stream` is SSE + `listen` gets the viewer, `/stream` is SSE, each stream starting with the + last 1000 entries - `path`, `from`, `format` (`json`), `listen` (`127.0.0.1:8080`) - `tls`: `off` (default), `self` (a self-signed certificate made at startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or diff --git a/doc/lw.1 b/doc/lw.1 index 5d99d1d..a910583 100644 --- a/doc/lw.1 +++ b/doc/lw.1 @@ -304,7 +304,8 @@ Follow files to standard output, like .It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer , allow , deny Oc Serve a live HTTP stream on .Cm listen -.Pq default Li 127.0.0.1:8080 . +.Pq default Li 127.0.0.1:8080 , +each stream starting with the last 1000 entries. .Cm tls is .Cm off , diff --git a/doc/sinks.md b/doc/sinks.md index e47fd0a..05f9e5b 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -170,6 +170,9 @@ Options, each as type and default: - `client_buffer_size` (int, `256`): per-client send queue depth. - `write_timeout_ms` (int, `0`): per-event write deadline; `0` = none. - `max_connections` (int, `0`): concurrent stream cap; `0` = unlimited. +- `replay_lines` (int, `0`; `1000` from the `serve` preset): the last entries + a new stream gets after its `connected` event, before the live ones, so a + browser opened late sees a backlog; `0` = none. - `login_page` (bool, `false`): `scram` in proxy mode only (an error elsewhere): the browser login page at `/auth/login`. - `viewer_page` (bool, `false`): `scram` in proxy mode only (an error @@ -212,8 +215,8 @@ Options, each as type and default: - never under `scram` on the sink's own TLS, which a browser cannot log in to: `/` is refused like any other path; - an endpoint at `/` keeps the root, and the viewer stays at `/auth/view`. - The viewer shows entries from when it connects (the sink keeps no backlog) - and holds one stream against `max_connections`. + The viewer starts with the `replay_lines` backlog, holds one stream against + `max_connections`, and says so when all are taken (`server full`). - Refusals are logged at WARN and counted in `auth_rejected`. The authorized identity is recorded in the client's session as `auth_method` / `auth_identity`. diff --git a/doc/todo.md b/doc/todo.md index d25063a..004fecd 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -65,16 +65,11 @@ identity, and both skeletons download the tagged source. Smaller items found along the way, each independent of the others. -1. A late browser sees no backlog: the `http` sink keeps none, so the viewer - shows entries from when it connects. A bounded replay (a `replay_lines` - option, sent after `event: connected`) would fill it. The viewer also hides - a `503` from `max_connections` behind "retrying"; the status JSON it already - fetches could name it. -2. Level detection. `source.ExtractLogLevel` needs a delimiter after the name +1. Level detection. `source.ExtractLogLevel` needs a delimiter after the name (`ERROR:`, `[WARN]`, ` INFO `), so `warning disk full` or `DBG x` get no level, hence no color. Match the names as words, as the console sink's painter does, with one table for both. -3. Command line: +2. Command line: - a positional word at the top level (`lw tail`, `lw -t FILE`) still reaches config with an unclear message; name it ("unexpected argument X; the file is -c FILE"); @@ -84,11 +79,9 @@ Smaller items found along the way, each independent of the others. - `lw tls cert --host` could become `--hosts` (keeping `--host`), to match `tls.hosts` and the preset key; - top-level usage errors exit 1, subcommand ones 2. -4. `GET /favicon.ico` answers 404; the logo could serve as the icon once the - pages' CSP allows `img-src 'self'`. -5. TLS: the generated-certificate key is one per process, so a pin taken from a +3. TLS: the generated-certificate key is one per process, so a pin taken from a `self_signed` listener also matches an `issuer` listener of the same process; document it or key per listener. Pins are checked in `VerifyPeerCertificate`, which Go skips on resumption: guard it if a dialer ever keeps a `ClientSessionCache`. -6. `core.ShutdownTimeout` is unused. +4. `core.ShutdownTimeout` is unused. diff --git a/internal/config/config.go b/internal/config/config.go index 4d23964..c685794 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -293,6 +293,7 @@ type HTTPSinkOptions struct { ClientBufferSize int64 `toml:"client_buffer_size"` // per-client send queue WriteTimeoutMS int64 `toml:"write_timeout_ms"` // per-SSE-write deadline, 0 = none MaxConnections int64 `toml:"max_connections"` // 0 = unlimited + ReplayLines int64 `toml:"replay_lines"` // entries a new stream gets first, 0 = none LoginPage bool `toml:"login_page"` // GET /auth/login, needs auth.trusted_proxies ViewerPage bool `toml:"viewer_page"` // GET /auth/view, needs login_page Auth *AuthOptions `toml:"auth"` diff --git a/internal/config/preset.go b/internal/config/preset.go index 4d58ee2..92123ad 100644 --- a/internal/config/preset.go +++ b/internal/config/preset.go @@ -97,6 +97,7 @@ var presets = []Preset{ default: return fmt.Errorf("viewer %q (valid: true, false)", v["viewer"]) } + sink["replay_lines"] = int64(1000) // a browser opened late sees the recent entries p.Flow.Format = &FormatConfig{Type: v["format"]} p.PluginSinks = append(p.PluginSinks, PluginSinkConfig{ID: "http", Type: "http", Config: sink}) return nil diff --git a/internal/sink/http/http.go b/internal/sink/http/http.go index dd9a806..f1e5d2d 100644 --- a/internal/sink/http/http.go +++ b/internal/sink/http/http.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "path" + "slices" "strconv" "strings" "sync" @@ -67,8 +68,9 @@ type HTTPSink struct { input chan core.TransportEvent logger *log.Logger - // Client registry + // Client registry, and the last replay_lines entries, under clientsMu clients map[uint64]*sseClient + replay [][]byte clientsMu sync.Mutex nextClientID atomic.Uint64 writeTimeout time.Duration @@ -399,6 +401,9 @@ func (h *HTTPSink) brokerLoop() { var stale []uint64 h.clientsMu.Lock() + if n := int(h.config.ReplayLines); n > 0 { + h.replay = append(h.replay[max(len(h.replay)+1-n, 0):], event.Payload) + } for id, c := range h.clients { if _, exists := h.proxy.GetSession(c.sessionID); !exists { stale = append(stale, id) @@ -506,18 +511,6 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { return } - // Registered only now: a client the broker can queue into before its reader - // reaches the loop below loses a burst to a buffer nobody is draining. - h.clientsMu.Lock() - h.clients[id] = c - h.clientsMu.Unlock() - - // A stream with nothing to carry still has to prove the peer is there. The - // comment refreshes the session the broker evicts on, and fails on a peer - // that stopped reading. - idle := time.NewTicker(h.keepalive) - defer idle.Stop() - send := func(payload []byte) bool { if writeSSE(w, payload) != nil || rc.Flush() != nil { return false @@ -525,6 +518,27 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { h.proxy.UpdateActivity(sess.ID) return true } + + // Registered only now: a client the broker can queue into before its reader + // reaches the loop below loses a burst to a buffer nobody is draining. The + // backlog is taken under the same lock, so it ends where the queue begins. + h.clientsMu.Lock() + h.clients[id] = c + backlog := slices.Clone(h.replay) + h.clientsMu.Unlock() + for _, payload := range backlog { + h.armWrite(rc) + if !send(payload) { + return + } + } + + // A stream with nothing to carry still has to prove the peer is there. The + // comment refreshes the session the broker evicts on, and fails on a peer + // that stopped reading. + idle := time.NewTicker(h.keepalive) + defer idle.Stop() + clientGone := r.Context().Done() for { select { diff --git a/internal/sink/http/http_test.go b/internal/sink/http/http_test.go index 454af37..6722512 100644 --- a/internal/sink/http/http_test.go +++ b/internal/sink/http/http_test.go @@ -229,7 +229,7 @@ func TestProxyModeServesBrowserFiles(t *testing.T) { client, baseURL := serveTestHTTPSink(t, h) for file, ctype := range map[string]string{ "scram.js": "text/javascript", "login": "text/html", "login.js": "text/javascript", - "style.css": "text/css", "view": "text/html", "view.js": "text/javascript", + "style.css": "text/css", "view": "text/html", "view.js": "text/javascript", "favicon.svg": "image/svg+xml", } { req, _ := http.NewRequest(http.MethodGet, baseURL+"/auth/"+file, nil) resp, err := client.Do(proxied(req)) @@ -239,7 +239,7 @@ func TestProxyModeServesBrowserFiles(t *testing.T) { body, _ := io.ReadAll(resp.Body) resp.Body.Close() if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), ctype) || - !strings.Contains(resp.Header.Get("Content-Security-Policy"), "script-src 'self'") { + !strings.Contains(resp.Header.Get("Content-Security-Policy"), "script-src 'self'; connect-src 'self'; style-src 'self'; img-src 'self';") { t.Errorf("/auth/%s: %d %q %q", file, resp.StatusCode, resp.Header.Get("Content-Type"), resp.Header.Get("Content-Security-Policy")) } if file == "view" && !bytes.Contains(body, []byte(`content="api/status"`)) { @@ -474,3 +474,45 @@ func TestStopWritesQueuedEventsToReadingStreams(t *testing.T) { t.Fatalf("reading stream received %d of %d lines, disconnect %v", r.lines, n, r.disconnect) } } + +// A stream that connects late gets the last replay_lines entries after its +// connected event, then the live ones, none twice +func TestLateStreamGetsTheBacklog(t *testing.T) { + httpSink, _ := newTestHTTPSink(t, map[string]any{"replay_lines": int64(2)}) + client, baseURL := serveTestHTTPSink(t, httpSink) + until := func(cond func() bool) { + for deadline := time.Now().Add(2 * time.Second); !cond(); time.Sleep(10 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("timed out") + } + } + } + for i := range 3 { + httpSink.Input() <- core.TransportEvent{Payload: fmt.Appendf(nil, "line %d\n", i)} + } + until(func() bool { return httpSink.totalProcessed.Load() == 3 }) + resp, err := client.Get(baseURL + "/stream") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() && scanner.Text() != "event: connected" { + } + scanner.Scan() // its data + until(func() bool { + httpSink.clientsMu.Lock() + defer httpSink.clientsMu.Unlock() + return len(httpSink.clients) == 1 + }) + httpSink.Input() <- core.TransportEvent{Payload: []byte("line 3\n")} + var got []string + for len(got) < 3 && scanner.Scan() { + if line, ok := strings.CutPrefix(scanner.Text(), "data: "); ok { + got = append(got, line) + } + } + if !slices.Equal(got, []string{"line 1", "line 2", "line 3"}) { + t.Fatalf("stream got %q", got) + } +} diff --git a/internal/sink/http/web.go b/internal/sink/http/web.go index f7f795c..7584f49 100644 --- a/internal/sink/http/web.go +++ b/internal/sink/http/web.go @@ -16,18 +16,19 @@ import ( "github.com/lixenwraith/logwisp/internal/netacl" ) -//go:embed web/*.js web/*.html web/*.css +//go:embed web/*.js web/*.html web/*.css web/*.svg var webFiles embed.FS -// pageCSP confines the pages to their own scripts and styles: no inline code, -// no other origin, no framing +// pageCSP confines the pages to their own scripts, styles and icon: no inline +// code, no other origin, no framing const pageCSP = "default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; " + - "form-action 'self'; frame-ancestors 'none'; base-uri 'none'" + "img-src 'self'; form-action 'self'; frame-ancestors 'none'; base-uri 'none'" var webTypes = map[string]string{ ".js": "text/javascript; charset=utf-8", ".css": "text/css; charset=utf-8", ".html": "text/html; charset=utf-8", + ".svg": "image/svg+xml", } // webRoutes are the browser files under /auth/, each with the page that needs @@ -38,6 +39,7 @@ var webRoutes = []struct { }{ {"scram.js", "scram.js", ""}, {"style.css", "style.css", ""}, + {"favicon.svg", "favicon.svg", ""}, {"login", "login.html", "login"}, {"login.js", "login.js", "login"}, {"view", "view.html", "view"}, diff --git a/internal/sink/http/web/favicon.svg b/internal/sink/http/web/favicon.svg new file mode 100644 index 0000000..a2b52f1 --- /dev/null +++ b/internal/sink/http/web/favicon.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + diff --git a/internal/sink/http/web/login.html b/internal/sink/http/web/login.html index 83db93b..ae372a6 100644 --- a/internal/sink/http/web/login.html +++ b/internal/sink/http/web/login.html @@ -5,6 +5,7 @@ LogWisp sign-in + diff --git a/internal/sink/http/web/view.html b/internal/sink/http/web/view.html index 1ea9891..ac3599a 100644 --- a/internal/sink/http/web/view.html +++ b/internal/sink/http/web/view.html @@ -7,6 +7,7 @@ + diff --git a/internal/sink/http/web/view.js b/internal/sink/http/web/view.js index 3f9813e..2f7fb89 100644 --- a/internal/sink/http/web/view.js +++ b/internal/sink/http/web/view.js @@ -36,8 +36,9 @@ function append(line) { if (atBottom) log.scrollTop = log.scrollHeight; } -// The status answer names the stream path; its 401 is the only way to tell an -// ended session apart, since EventSource hides the status of a failure. +// The status answer names the stream path and whether the stream has room; its +// 401 is the only way to tell an ended session apart, since EventSource hides +// the status of a failure. async function connect() { if (tokenMode && !token) return showLogin(""); let res; @@ -52,10 +53,14 @@ async function connect() { // An open page has no session to end: a 401 there is a proxy's, so it retries if (res.status === 401 && !open) return sessionEnded(); fresh = false; - const path = res.ok ? (await res.json().catch(() => ({})))?.endpoints?.stream : null; + const body = res.ok ? await res.json().catch(() => ({})) : {}; + const path = body?.endpoints?.stream; if (typeof path !== "string" || !path.startsWith("/")) { return retry(`status unavailable (${res.status})`); } + // The stream would answer 503, which EventSource hides + const { active_clients: active, max_connections: max } = body?.server ?? {}; + if (max > 0 && active >= max) return retry(`server full (${active} of ${max} streams)`); // Stream paths are absolute in logwisp; the proxy may mount it under a prefix const url = new URL("./" + path.replace(/^\/+/, ""), base); if (tokenMode) streamWithToken(url);