v0.21.0 scram password auth, cred and token management, browser tls-terminating proxy, viewer client

internal/sink/httpchain/httpchain.go

@@ -55,8 +55,9 @@ type HTTPChainSink struct {
 	session *session.Session
 	config  *config.HTTPChainSinkOptions
 
-	node string
-	url  string
+	node    string
+	baseURL string // scheme://host:port, where /auth lives
+	url     string
 
 	tlsEnabled bool
 	mtls       bool
@@ -106,6 +107,8 @@ func NewHTTPChainSinkPlugin(
 		opts.IngestPath = DefaultHTTPChainSinkIngestPath
 	} else if !strings.HasPrefix(opts.IngestPath, "/") {
 		return nil, fmt.Errorf("ingest_path: must start with '/'")
+	} else if opts.IngestPath == chain.AuthPath {
+		return nil, fmt.Errorf("ingest_path: %s is reserved for authentication", chain.AuthPath)
 	}
 	if opts.BufferSize <= 0 {
 		opts.BufferSize = DefaultHTTPChainSinkBufferSize
@@ -142,7 +145,7 @@ func NewHTTPChainSinkPlugin(
 	if err != nil {
 		return nil, err
 	}
-	authPolicy, err := authz.New(opts.Auth, opts.TLS, authz.RoleDialer)
+	authPolicy, err := authz.New(opts.Auth, tlsCfg, authz.RoleDialer, authz.HTTP)
 	if err != nil {
 		return nil, err
 	}
@@ -182,6 +185,7 @@ func NewHTTPChainSinkPlugin(
 		tlsEnabled: tlsCfg != nil,
 		mtls:       tlsCfg != nil && len(tlsCfg.Certificates) > 0,
 		auth:       authPolicy,
+		baseURL:    scheme + "://" + addr,
 		url:        scheme + "://" + addr + opts.IngestPath,
 		// A redirect would resend the batch, even from https to plaintext http
 		client: &http.Client{
@@ -214,6 +218,7 @@ func NewHTTPChainSinkPlugin(
 		"mtls", t.mtls,
 		"auth", authPolicy.Describe())
 	tlsx.LogWarnings(logger, "http_chain_sink", id, opts.TLS, false)
+	authPolicy.LogStartup(logger, "http_chain_sink", id, false)
 	return t, nil
 }
 
@@ -435,10 +440,15 @@ func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool,
 	req.Header.Set("Content-Type", chain.ContentTypeNDJSON)
 	req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion))
 	req.Header.Set(chain.HeaderNode, t.node)
-	// Future: Authorization header for auth
+	// Under scram: log in when no token is held; every failure is transient,
+	// so the batch waits under backoff rather than being dropped
+	if err := t.auth.Prepare(reqCtx, t.client, t.baseURL, req); err != nil {
+		return true, err
+	}
 
 	resp, err := t.client.Do(req)
 	if err != nil {
+		t.auth.Invalidate(0, err) // another server certificate: log in anew
 		return true, err
 	}
 	defer resp.Body.Close()
@@ -448,6 +458,9 @@ func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool,
 	switch {
 	case resp.StatusCode >= 200 && resp.StatusCode < 300:
 		return false, nil
+	case t.auth.Invalidate(resp.StatusCode, nil):
+		// Token expired or the source reloaded; the retry logs in again
+		return true, fmt.Errorf("status %s", resp.Status)
 	case resp.StatusCode == http.StatusRequestTimeout,
 		resp.StatusCode == http.StatusTooManyRequests,
 		resp.StatusCode >= 500:

internal/sink/httpchain/httpchain_test.go

@@ -1,16 +1,24 @@
 package httpchain
 
 import (
+	"crypto/tls"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"path/filepath"
 	"strconv"
 	"sync/atomic"
 	"testing"
 	"time"
 
+	"logwisp/internal/authz"
+	"logwisp/internal/chain"
+	"logwisp/internal/config"
+	"logwisp/internal/core"
 	"logwisp/internal/session"
+	"logwisp/internal/testutil"
 
+	"github.com/lixenwraith/auth"
 	"github.com/lixenwraith/log"
 )
 
@@ -45,3 +53,76 @@ func TestRedirectIsNotFollowed(t *testing.T) {
 		t.Fatal("the batch followed the redirect")
 	}
 }
+
+// Under scram a 401 on ingest (an expired token, a reloaded source) logs in
+// again, and the held batch is delivered exactly once.
+func TestRefusedTokenLogsInAgainAndDeliversOnce(t *testing.T) {
+	pki := testutil.NewPKI(t, "edge-01")
+	dir := t.TempDir()
+	cred, err := auth.NewCredential("edge-01", "edge-01-secret")
+	if err != nil {
+		t.Fatal(err)
+	}
+	data, err := (&authz.Credentials{DecoyKey: make([]byte, 32), Users: []*auth.Credential{cred}}).Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+	creds, password := filepath.Join(dir, "users.toml"), filepath.Join(dir, "edge-01.pass")
+	testutil.WriteFile(t, creds, string(data))
+	testutil.WriteFile(t, password, "edge-01-secret\n")
+
+	serverCert, err := tls.LoadX509KeyPair(pki.ServerCert, pki.ServerKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+	serverTLS := &tls.Config{Certificates: []tls.Certificate{serverCert}}
+	listener, err := authz.New(&config.AuthOptions{Type: authz.MethodSCRAM, CredentialsFile: creds}, serverTLS, authz.RoleChainListener, authz.HTTP)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := listener.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(listener.Close)
+
+	var refused, delivered atomic.Int64
+	mux := http.NewServeMux()
+	mux.HandleFunc("POST "+chain.AuthPath, func(w http.ResponseWriter, r *http.Request) { listener.ServeAuth(w, r) })
+	mux.HandleFunc("POST /ingest", func(w http.ResponseWriter, r *http.Request) {
+		if _, status, err := listener.AuthorizeRequest(r); err != nil {
+			authz.Refuse(w, status)
+			return
+		}
+		if refused.Add(1) == 1 {
+			authz.Refuse(w, http.StatusUnauthorized) // as after a reload
+			return
+		}
+		delivered.Add(1)
+		w.WriteHeader(http.StatusNoContent)
+	})
+	srv := httptest.NewUnstartedServer(mux)
+	srv.TLS = serverTLS
+	srv.StartTLS()
+	t.Cleanup(srv.Close)
+
+	_, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
+	p, _ := strconv.ParseInt(port, 10, 64)
+	manager := session.NewManager(time.Hour)
+	t.Cleanup(manager.Stop)
+	created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{
+		"host": "127.0.0.1", "port": p, "backoff_min_ms": int64(10),
+		"tls":  map[string]any{"enabled": true, "ca_file": pki.CA},
+		"auth": map[string]any{"type": "scram", "username": "edge-01", "password_file": password},
+	}, log.NewLogger(), session.NewProxy(manager, "fwd"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	sink := created.(*HTTPChainSink)
+	sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("entry")})
+	if !sink.flush(t.Context()) || delivered.Load() != 1 || sink.droppedBatches.Load() != 0 {
+		t.Fatalf("delivered %d, dropped %d", delivered.Load(), sink.droppedBatches.Load())
+	}
+	if logins := listener.Stats()["auth_allowed"]; logins != uint64(2) {
+		t.Fatalf("logins = %v, want 2", logins)
+	}
+}