v0.18.1 file watcher retirement diagnostics update
This commit is contained in:
@@ -33,7 +33,7 @@ install: build
|
|||||||
|
|
||||||
# Uninstall the binary
|
# Uninstall the binary
|
||||||
uninstall:
|
uninstall:
|
||||||
rm -f $(BINDIR)/$(BINARY_PATH)
|
rm -f $(BINDIR)/$(BINARY_NAME)
|
||||||
|
|
||||||
# Clean build artifacts
|
# Clean build artifacts
|
||||||
clean:
|
clean:
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
<td>
|
<td>
|
||||||
<h1>LogWisp</h1>
|
<h1>LogWisp</h1>
|
||||||
<p>
|
<p>
|
||||||
<a href="https://golang.org"><img src="https://img.shields.io/badge/Go-1.26-00ADD8?style=flat&logo=go" alt="Go"></a>
|
<a href="https://golang.org"><img src="https://img.shields.io/badge/Go-1.27.1-00ADD8?style=flat&logo=go" alt="Go"></a>
|
||||||
<a href="https://opensource.org/licenses/BSD-3-Clause"><img src="https://img.shields.io/badge/License-BSD_3--Clause-blue.svg" alt="License"></a>
|
<a href="https://opensource.org/licenses/BSD-3-Clause"><img src="https://img.shields.io/badge/License-BSD_3--Clause-blue.svg" alt="License"></a>
|
||||||
<a href="doc/"><img src="https://img.shields.io/badge/Docs-Available-green.svg" alt="Documentation"></a>
|
<a href="doc/"><img src="https://img.shields.io/badge/Docs-Available-green.svg" alt="Documentation"></a>
|
||||||
</p>
|
</p>
|
||||||
@@ -137,7 +137,7 @@ synthetic generator writing JSON to stdout.
|
|||||||
|
|
||||||
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
|
- **Operating systems**: Linux (kernel 6.10+), FreeBSD (14.0+)
|
||||||
- **Architecture**: amd64
|
- **Architecture**: amd64
|
||||||
- **Go**: 1.26+ to build from source
|
- **Go**: 1.27.1+ to build from source
|
||||||
|
|
||||||
Network sources and sinks bind and dial over IPv4 only.
|
Network sources and sinks bind and dial over IPv4 only.
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -22,7 +22,7 @@ The Makefile works with both GNU make and BSD make. Targets:
|
|||||||
| `make` / `make build` | Build `bin/logwisp` with version metadata |
|
| `make` / `make build` | Build `bin/logwisp` with version metadata |
|
||||||
| `make dev` | Build with the race detector enabled |
|
| `make dev` | Build with the race detector enabled |
|
||||||
| `make install` | Install the binary to `$(PREFIX)/bin` (default `/usr/local`) |
|
| `make install` | Install the binary to `$(PREFIX)/bin` (default `/usr/local`) |
|
||||||
| `make uninstall` | Intended to remove the installed binary — currently broken: it expands to `$(BINDIR)/bin/logwisp` instead of `$(BINDIR)/logwisp`, so it removes nothing. Delete the binary by hand |
|
| `make uninstall` | Remove `$(BINDIR)/logwisp` |
|
||||||
| `make clean` | Remove the built binary |
|
| `make clean` | Remove the built binary |
|
||||||
| `make version` | Print the version, commit, and build time that would be embedded |
|
| `make version` | Print the version, commit, and build time that would be embedded |
|
||||||
|
|
||||||
|
|||||||
+7
-2
@@ -124,6 +124,9 @@ curl -s http://127.0.0.1:8080/status | jq .
|
|||||||
"tls": false,
|
"tls": false,
|
||||||
"active_clients": 3,
|
"active_clients": 3,
|
||||||
"buffer_size": 1000,
|
"buffer_size": 1000,
|
||||||
|
"client_buffer_size": 256,
|
||||||
|
"max_connections": 32,
|
||||||
|
"write_timeout_ms": 5000,
|
||||||
"uptime_seconds": 8130
|
"uptime_seconds": 8130
|
||||||
},
|
},
|
||||||
"endpoints": { "stream": "/stream", "status": "/status" },
|
"endpoints": { "stream": "/stream", "status": "/status" },
|
||||||
@@ -183,8 +186,10 @@ the filter stage logs several lines per entry evaluated.
|
|||||||
|
|
||||||
Raise `buffer_size` when `total_dropped_by_sink` is climbing but the sink itself
|
Raise `buffer_size` when `total_dropped_by_sink` is climbing but the sink itself
|
||||||
is healthy — that is a burst-absorption problem. Raise `client_buffer_size` when
|
is healthy — that is a burst-absorption problem. Raise `client_buffer_size` when
|
||||||
`dropped_writes` is climbing for network sinks; that is a slow-consumer problem,
|
`dropped_writes` is climbing for network sinks; that is a slow-consumer or
|
||||||
and a bigger buffer only buys time.
|
startup-replay burst problem, and a bigger buffer only buys time. The HTTP
|
||||||
|
status endpoint reports both queue bounds alongside the counters so an operator
|
||||||
|
can distinguish configuration from demand.
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
[pipelines.plugin_sinks.config]
|
[pipelines.plugin_sinks.config]
|
||||||
|
|||||||
+4
-3
@@ -160,9 +160,10 @@ allow = ["viewer-01"]
|
|||||||
- HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1.
|
- HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1.
|
||||||
|
|
||||||
**Status endpoint** returns service and version identity, host, port, TLS flag,
|
**Status endpoint** returns service and version identity, host, port, TLS flag,
|
||||||
the compiled auth policy, active client count, buffer size, uptime, endpoint
|
the compiled auth policy, active client count, sink and per-client buffer sizes,
|
||||||
paths, and the `total_processed` / `dropped_writes` / `rejected_clients` /
|
connection limit, write timeout, uptime, endpoint paths, and the
|
||||||
`auth_rejected` counters.
|
`total_processed` / `dropped_writes` / `rejected_clients` / `auth_rejected`
|
||||||
|
counters.
|
||||||
|
|
||||||
> Without an `auth` block both endpoints are unauthenticated, and the stream
|
> Without an `auth` block both endpoints are unauthenticated, and the stream
|
||||||
> response carries `Access-Control-Allow-Origin: *`, so any web origin can read
|
> response carries `Access-Control-Allow-Origin: *`, so any web origin can read
|
||||||
|
|||||||
@@ -459,6 +459,9 @@ func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
|
|||||||
"auth": h.auth.Describe(),
|
"auth": h.auth.Describe(),
|
||||||
"active_clients": h.activeClients.Load(),
|
"active_clients": h.activeClients.Load(),
|
||||||
"buffer_size": h.config.BufferSize,
|
"buffer_size": h.config.BufferSize,
|
||||||
|
"client_buffer_size": h.config.ClientBufferSize,
|
||||||
|
"max_connections": h.config.MaxConnections,
|
||||||
|
"write_timeout_ms": h.config.WriteTimeoutMS,
|
||||||
"uptime_seconds": int(time.Since(h.startTime).Seconds()),
|
"uptime_seconds": int(time.Since(h.startTime).Seconds()),
|
||||||
},
|
},
|
||||||
"endpoints": map[string]string{
|
"endpoints": map[string]string{
|
||||||
@@ -484,6 +487,9 @@ func (h *HTTPSink) GetStats() sink.SinkStats {
|
|||||||
"host": h.config.Host,
|
"host": h.config.Host,
|
||||||
"port": h.config.Port,
|
"port": h.config.Port,
|
||||||
"buffer_size": h.config.BufferSize,
|
"buffer_size": h.config.BufferSize,
|
||||||
|
"client_buffer_size": h.config.ClientBufferSize,
|
||||||
|
"max_connections": h.config.MaxConnections,
|
||||||
|
"write_timeout_ms": h.config.WriteTimeoutMS,
|
||||||
"tls": h.tlsConfig != nil,
|
"tls": h.tlsConfig != nil,
|
||||||
"dropped_writes": h.droppedWrites.Load(),
|
"dropped_writes": h.droppedWrites.Load(),
|
||||||
"rejected_clients": h.rejectedClients.Load(),
|
"rejected_clients": h.rejectedClients.Load(),
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package http
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"logwisp/internal/session"
|
||||||
|
"logwisp/internal/sink"
|
||||||
|
|
||||||
|
"github.com/lixenwraith/log"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStatusReportsQueueAndConnectionBounds(t *testing.T) {
|
||||||
|
manager := session.NewManager(time.Hour)
|
||||||
|
defer manager.Stop()
|
||||||
|
created, err := NewHTTPSinkPlugin(
|
||||||
|
"stream",
|
||||||
|
map[string]any{
|
||||||
|
"host": "127.0.0.1",
|
||||||
|
"port": int64(8081),
|
||||||
|
"buffer_size": int64(4096),
|
||||||
|
"client_buffer_size": int64(512),
|
||||||
|
"max_connections": int64(32),
|
||||||
|
"write_timeout_ms": int64(5000),
|
||||||
|
},
|
||||||
|
log.NewLogger(),
|
||||||
|
session.NewProxy(manager, "stream"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
httpSink, ok := created.(*HTTPSink)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("sink type = %T", created)
|
||||||
|
}
|
||||||
|
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
httpSink.handleStatus(recorder, httptest.NewRequest("GET", "/status", nil))
|
||||||
|
if recorder.Code != 200 {
|
||||||
|
t.Fatalf("status code = %d", recorder.Code)
|
||||||
|
}
|
||||||
|
var response struct {
|
||||||
|
Server map[string]any `json:"server"`
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(recorder.Body).Decode(&response); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for key, want := range map[string]float64{
|
||||||
|
"buffer_size": 4096,
|
||||||
|
"client_buffer_size": 512,
|
||||||
|
"max_connections": 32,
|
||||||
|
"write_timeout_ms": 5000,
|
||||||
|
} {
|
||||||
|
if got := response.Server[key]; got != want {
|
||||||
|
t.Errorf("server.%s = %v, want %v", key, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stats := httpSink.GetStats()
|
||||||
|
details := stats.Details
|
||||||
|
for key, want := range map[string]int64{
|
||||||
|
"buffer_size": 4096,
|
||||||
|
"client_buffer_size": 512,
|
||||||
|
"max_connections": 32,
|
||||||
|
"write_timeout_ms": 5000,
|
||||||
|
} {
|
||||||
|
if got := details[key]; got != want {
|
||||||
|
t.Errorf("details[%q] = %v, want %v", key, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ sink.Sink = httpSink
|
||||||
|
}
|
||||||
@@ -292,12 +292,21 @@ func (fs *FileSource) ensureWatcher(path string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fs.mu.Lock()
|
fs.removeWatcher(path, w)
|
||||||
delete(fs.watchers, path)
|
|
||||||
fs.mu.Unlock()
|
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// removeWatcher removes only the watcher that finished. A deleted file can be
|
||||||
|
// recreated before its old watcher observes stop; in that case ensureWatcher
|
||||||
|
// has already installed a replacement under the same path, which must survive.
|
||||||
|
func (fs *FileSource) removeWatcher(path string, watcher *fileWatcher) {
|
||||||
|
fs.mu.Lock()
|
||||||
|
if fs.watchers[path] == watcher {
|
||||||
|
delete(fs.watchers, path)
|
||||||
|
}
|
||||||
|
fs.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
// cleanupWatchers stops and removes watchers for files that no longer exist.
|
// cleanupWatchers stops and removes watchers for files that no longer exist.
|
||||||
func (fs *FileSource) cleanupWatchers() {
|
func (fs *FileSource) cleanupWatchers() {
|
||||||
fs.mu.Lock()
|
fs.mu.Lock()
|
||||||
@@ -369,4 +378,3 @@ func globToRegex(glob string) string {
|
|||||||
regex = strings.ReplaceAll(regex, `\?`, `.`)
|
regex = strings.ReplaceAll(regex, `\?`, `.`)
|
||||||
return "^" + regex + "$"
|
return "^" + regex + "$"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package file
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"logwisp/internal/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStoppedWatcherReturnsNormally(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "session.jsonl")
|
||||||
|
if err := os.WriteFile(path, nil, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
watcher := newFileWatcher(path, true, true, func(_ core.LogEntry) {}, nil)
|
||||||
|
watcher.stop()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := watcher.watch(ctx); err != nil {
|
||||||
|
t.Fatalf("stopped watcher returned error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRemoveWatcherPreservesReplacement(t *testing.T) {
|
||||||
|
oldWatcher := &fileWatcher{}
|
||||||
|
replacement := &fileWatcher{}
|
||||||
|
source := &FileSource{
|
||||||
|
watchers: map[string]*fileWatcher{
|
||||||
|
"session.jsonl": replacement,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
source.removeWatcher("session.jsonl", oldWatcher)
|
||||||
|
if got := source.watchers["session.jsonl"]; got != replacement {
|
||||||
|
t.Fatalf("replacement watcher = %p, want %p", got, replacement)
|
||||||
|
}
|
||||||
|
|
||||||
|
source.removeWatcher("session.jsonl", replacement)
|
||||||
|
if _, exists := source.watchers["session.jsonl"]; exists {
|
||||||
|
t.Fatal("finished watcher was not removed")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -80,7 +80,7 @@ func (w *fileWatcher) watch(ctx context.Context) error {
|
|||||||
return ctx.Err()
|
return ctx.Err()
|
||||||
case <-ticker.C:
|
case <-ticker.C:
|
||||||
if w.isStopped() {
|
if w.isStopped() {
|
||||||
return fmt.Errorf("watcher stopped")
|
return nil
|
||||||
}
|
}
|
||||||
if err := w.checkFile(); err != nil {
|
if err := w.checkFile(); err != nil {
|
||||||
// Log error but continue watching
|
// Log error but continue watching
|
||||||
|
|||||||
Reference in New Issue
Block a user