v0.25.0 netacl and fixes
This commit is contained in:
1 parent
9e86d7999d
commit
4af072f4e5
30 files changed
+918
-228
No files matched your search
@@ -216,6 +216,7 @@ type TCPChainSourceOptions struct {
|
||||
HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
|
||||
@@ -230,6 +231,7 @@ type HTTPChainSourceOptions struct {
|
||||
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// --- Sink Options ---
|
||||
@@ -277,6 +279,7 @@ type TCPSinkOptions struct {
|
||||
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
|
||||
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// HTTPSinkOptions defines settings for an HTTP SSE server sink
|
||||
@@ -293,6 +296,7 @@ type HTTPSinkOptions struct {
|
||||
LoginPage bool `toml:"login_page"` // GET /auth/login, needs auth.trusted_proxies
|
||||
ViewerPage bool `toml:"viewer_page"` // GET /auth/view, needs login_page
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
|
||||
@@ -369,6 +373,14 @@ type AuthOptions struct {
|
||||
PasswordFile string `toml:"password_file"`
|
||||
}
|
||||
|
||||
// ACLOptions is a listener's address rules, entries being addresses or CIDRs:
|
||||
// deny wins, then a set allow list admits only its entries. Validation is in
|
||||
// internal/netacl.
|
||||
type ACLOptions struct {
|
||||
Allow []string `toml:"allow"`
|
||||
Deny []string `toml:"deny"`
|
||||
}
|
||||
|
||||
// --- TLS Options ---
|
||||
|
||||
// TLSOptions is one shape for both roles. Listeners present cert_file and
|
||||
|
||||
@@ -40,6 +40,10 @@ var (
|
||||
{Name: "issuer_key", Help: "CA key file, for tls=issuer"},
|
||||
{Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"},
|
||||
}
|
||||
aclParams = []PresetParam{
|
||||
{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family; default: all; ',' between them"},
|
||||
{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"},
|
||||
}
|
||||
transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
|
||||
)
|
||||
|
||||
@@ -60,13 +64,13 @@ var presets = []Preset{
|
||||
return nil
|
||||
}},
|
||||
{"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream",
|
||||
append([]PresetParam{pathParam, fromParam, formatParam("json"),
|
||||
slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"),
|
||||
{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
|
||||
{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
|
||||
{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
|
||||
{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
|
||||
{Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"},
|
||||
}, tlsParams...),
|
||||
}, aclParams, tlsParams),
|
||||
func(p *PipelineConfig, v map[string]string) error {
|
||||
p.PluginSources = pathOrStdin(v)
|
||||
sink, err := listener(v, "off")
|
||||
@@ -145,7 +149,7 @@ var presets = []Preset{
|
||||
return nil
|
||||
}},
|
||||
{"aggregator", "Receive from edges over TLS, authenticated, to files or standard output",
|
||||
append([]PresetParam{
|
||||
slices.Concat([]PresetParam{
|
||||
{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"},
|
||||
transportParam,
|
||||
{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
|
||||
@@ -153,7 +157,7 @@ var presets = []Preset{
|
||||
{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
|
||||
{Name: "out", Help: "directory for the received entries; default: standard output"},
|
||||
formatParam("json"),
|
||||
}, tlsParams...),
|
||||
}, aclParams, tlsParams),
|
||||
func(p *PipelineConfig, v map[string]string) error {
|
||||
typ, err := chainType(v["transport"])
|
||||
if err != nil {
|
||||
@@ -268,14 +272,22 @@ var tlsModes = map[string]struct {
|
||||
"files": {needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
|
||||
}
|
||||
|
||||
// listener starts a listening plugin's options: host, port and tls, whose
|
||||
// mode decides which certificate keys apply.
|
||||
// listener starts a listening plugin's options: host, port, acl and tls,
|
||||
// whose mode decides which certificate keys apply.
|
||||
func listener(v map[string]string, def string) (map[string]any, error) {
|
||||
host, port, err := splitAddr("listen", v["listen"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts := map[string]any{"host": host, "port": port}
|
||||
opts, acl := map[string]any{"host": host, "port": port}, map[string]any{}
|
||||
for _, k := range []string{"allow", "deny"} {
|
||||
if v[k] != "" {
|
||||
acl[k] = list(v[k])
|
||||
}
|
||||
}
|
||||
if len(acl) > 0 {
|
||||
opts["acl"] = acl
|
||||
}
|
||||
mode := cmp.Or(v["tls"], def)
|
||||
m, ok := tlsModes[mode]
|
||||
if !ok {
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
// Package netacl is the address seam of the network listeners, beside tlsx
|
||||
// and authz: a Policy refuses a peer by its address as the socket is
|
||||
// accepted, before TLS or a byte of the protocol is read. New returns nil
|
||||
// when no rule is set, and every method tolerates a nil receiver.
|
||||
package netacl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// warnEvery spaces the WARNs reporting refused peers; acl_denied counts all
|
||||
const warnEvery = time.Minute
|
||||
|
||||
// Policy is the compiled form of config.ACLOptions for one listener
|
||||
type Policy struct {
|
||||
allow, deny []netip.Prefix
|
||||
logger *log.Logger
|
||||
component, id string
|
||||
|
||||
denied atomic.Uint64
|
||||
epoch time.Time // a monotonic base: a stepped clock neither mutes nor floods the WARN
|
||||
nextWarn atomic.Int64 // nanoseconds since epoch
|
||||
}
|
||||
|
||||
// New compiles the rules of a listener on host and warns of what they leave
|
||||
// open. Each entry must be of the family core.Network gives host: one the
|
||||
// listener's sockets can never carry is a mistake, not a rule.
|
||||
func New(o *config.ACLOptions, host string, l *log.Logger, component, id string) (*Policy, error) {
|
||||
if o == nil || len(o.Allow)+len(o.Deny) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
network, err := core.Network(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var widened []string // entry, network pairs
|
||||
compile := func(key string, entries []string) ([]netip.Prefix, error) {
|
||||
var rules []netip.Prefix
|
||||
for _, e := range entries {
|
||||
prefix, err := parse(e, network)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acl: %s entry %q: %w", key, e, err)
|
||||
}
|
||||
if m := prefix.Masked(); m != prefix {
|
||||
widened = append(widened, e, m.String())
|
||||
}
|
||||
rules = append(rules, prefix.Masked())
|
||||
}
|
||||
return rules, nil
|
||||
}
|
||||
p := &Policy{logger: l, component: component, id: id, epoch: time.Now()}
|
||||
if p.allow, err = compile("allow", o.Allow); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.deny, err = compile("deny", o.Deny); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.logStartup(host, widened)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func parse(entry, network string) (netip.Prefix, error) {
|
||||
entry = strings.TrimSpace(entry)
|
||||
prefix, err := netip.ParsePrefix(entry)
|
||||
if err != nil {
|
||||
addr, aerr := netip.ParseAddr(entry)
|
||||
if aerr != nil || addr.Zone() != "" {
|
||||
return netip.Prefix{}, fmt.Errorf("neither an address nor a CIDR (and no zone)")
|
||||
}
|
||||
prefix = netip.PrefixFrom(addr, addr.BitLen())
|
||||
}
|
||||
switch a := prefix.Addr(); {
|
||||
case a.Is4In6():
|
||||
return netip.Prefix{}, fmt.Errorf("IPv4-mapped; write it as IPv4")
|
||||
case network == "tcp4" && !a.Is4(), network == "tcp6" && a.Is4():
|
||||
return netip.Prefix{}, fmt.Errorf("not of the listener's family (%s)", network)
|
||||
}
|
||||
return prefix, nil
|
||||
}
|
||||
|
||||
func (p *Policy) logStartup(host string, widened []string) {
|
||||
for i := 0; i < len(widened); i += 2 {
|
||||
p.logger.Warn("msg", "acl entry has host bits set and matches its whole network",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"entry", widened[i],
|
||||
"network", widened[i+1])
|
||||
}
|
||||
for _, q := range p.allow {
|
||||
if q.Bits() == 0 {
|
||||
p.logger.Warn("msg", "acl.allow admits every address of its family",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"entry", q.String(),
|
||||
"hint", "list only the networks that may connect")
|
||||
}
|
||||
}
|
||||
if addr, err := netip.ParseAddr(host); len(p.allow) == 0 && (host == "" || err == nil && addr.IsUnspecified()) {
|
||||
p.logger.Warn("msg", "Listener on every address admits all that acl.deny does not list",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"hint", "set acl.allow to the networks that may connect")
|
||||
}
|
||||
}
|
||||
|
||||
// admits applies the rules: deny wins, then a set allow list admits only its
|
||||
// entries. A peer whose address is unknown is refused.
|
||||
func (p *Policy) admits(addr netip.Addr) bool {
|
||||
addr = addr.Unmap().WithZone("") // a zoned address is in no prefix
|
||||
in := func(q netip.Prefix) bool { return q.Contains(addr) }
|
||||
return addr.IsValid() && !slices.ContainsFunc(p.deny, in) &&
|
||||
(len(p.allow) == 0 || slices.ContainsFunc(p.allow, in))
|
||||
}
|
||||
|
||||
// Listener returns ln refusing the peers the rules exclude as it accepts them,
|
||||
// so the plugin never sees one. Close stays ln's: a sink's shutdown flush
|
||||
// reaches it unchanged. A nil Policy returns ln.
|
||||
func (p *Policy) Listener(ln net.Listener) net.Listener {
|
||||
if p == nil {
|
||||
return ln
|
||||
}
|
||||
return &listener{Listener: ln, p: p}
|
||||
}
|
||||
|
||||
type listener struct {
|
||||
net.Listener
|
||||
p *Policy
|
||||
}
|
||||
|
||||
func (l *listener) Accept() (net.Conn, error) {
|
||||
for {
|
||||
conn, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var addr netip.Addr
|
||||
remote := conn.RemoteAddr()
|
||||
if t, ok := remote.(*net.TCPAddr); ok {
|
||||
addr = t.AddrPort().Addr()
|
||||
}
|
||||
if l.p.admits(addr) {
|
||||
return conn, nil
|
||||
}
|
||||
conn.Close()
|
||||
l.p.refused(fmt.Sprint(remote))
|
||||
}
|
||||
}
|
||||
|
||||
// refused counts a refusal and WARNs at most once per warnEvery
|
||||
func (p *Policy) refused(remote string) {
|
||||
n := p.denied.Add(1)
|
||||
now, next := int64(time.Since(p.epoch)), p.nextWarn.Load()
|
||||
if now < next || !p.nextWarn.CompareAndSwap(next, now+int64(warnEvery)) {
|
||||
return
|
||||
}
|
||||
p.logger.Warn("msg", "Connection refused by acl; more within a minute are only counted",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"remote_addr", remote,
|
||||
"acl_denied", n)
|
||||
}
|
||||
|
||||
// Describe renders the policy for log lines, stats and the status endpoint
|
||||
func (p *Policy) Describe() string {
|
||||
if p == nil {
|
||||
return "none"
|
||||
}
|
||||
allow := "any"
|
||||
if len(p.allow) > 0 {
|
||||
allow = fmt.Sprint(len(p.allow))
|
||||
}
|
||||
return fmt.Sprintf("allow=%s deny=%d", allow, len(p.deny))
|
||||
}
|
||||
|
||||
func (p *Policy) Denied() uint64 {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
return p.denied.Load()
|
||||
}
|
||||
|
||||
// Stats reports the policy for a plugin's stats details, merged in with
|
||||
// maps.Copy beside authz's. A nil Policy reports nothing.
|
||||
func (p *Policy) Stats() map[string]any {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{"acl": p.Describe(), "acl_denied": p.denied.Load()}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package netacl
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func mustNew(t *testing.T, host string, allow, deny []string) *Policy {
|
||||
t.Helper()
|
||||
p, err := New(&config.ACLOptions{Allow: allow, Deny: deny}, host, log.NewLogger(), "test", "acl")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Deny wins over allow, a set allow list admits only its entries, an empty
|
||||
// one admits all not denied, an IPv4-mapped peer matches as IPv4, and a peer
|
||||
// of unknown address is refused.
|
||||
func TestDenyWinsThenAllowListAdmitsOnlyItsEntries(t *testing.T) {
|
||||
listed := mustNew(t, "0.0.0.0", []string{"127.0.0.0/8"}, []string{"127.0.0.2"})
|
||||
denyOnly := mustNew(t, "0.0.0.0", nil, []string{"10.0.0.0/8"})
|
||||
for _, c := range []struct {
|
||||
p *Policy
|
||||
addr string
|
||||
want bool
|
||||
}{
|
||||
{listed, "127.0.0.1", true},
|
||||
{listed, "127.0.0.2", false},
|
||||
{listed, "192.0.2.1", false},
|
||||
{denyOnly, "10.1.2.3", false},
|
||||
{denyOnly, "192.0.2.1", true},
|
||||
{denyOnly, "::ffff:10.1.2.3", false},
|
||||
{denyOnly, "", false},
|
||||
} {
|
||||
addr, _ := netip.ParseAddr(c.addr)
|
||||
if got := c.p.admits(addr); got != c.want {
|
||||
t.Errorf("%s under %s: admitted %v", c.addr, c.p.Describe(), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Entries are of the family core.Network gives the listener's host, as its
|
||||
// sockets carry no other: IPv4 on an IPv4 listener, IPv6 on an IPv6 one,
|
||||
// either behind a hostname; never IPv4-mapped, never with a zone.
|
||||
func TestEntriesFollowTheListenerFamily(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
host, entry string
|
||||
ok bool
|
||||
}{
|
||||
{"0.0.0.0", "10.0.0.0/8", true},
|
||||
{"0.0.0.0", "2001:db8::/32", false},
|
||||
{"::", "2001:db8::1", true},
|
||||
{"::", "10.0.0.1", false},
|
||||
{"::", "::ffff:10.0.0.1", false},
|
||||
{"::", "fe80::1%eth0", false},
|
||||
{"logs.example", "10.0.0.0/8", true},
|
||||
{"logs.example", "2001:db8::/32", true},
|
||||
} {
|
||||
_, err := New(&config.ACLOptions{Deny: []string{c.entry}}, c.host, log.NewLogger(), "test", "acl")
|
||||
if (err == nil) != c.ok {
|
||||
t.Errorf("%s on %q: %v", c.entry, c.host, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A link-local peer carries its interface zone, which no prefix contains:
|
||||
// it is matched without it, or a deny of fe80::/10 would admit it.
|
||||
func TestZonedPeerMatchesItsPrefix(t *testing.T) {
|
||||
p := mustNew(t, "::", nil, []string{"fe80::/10"})
|
||||
if p.admits(netip.MustParseAddr("fe80::1%eth0")) {
|
||||
t.Fatal("a zoned link-local peer passed the deny of its prefix")
|
||||
}
|
||||
}
|
||||
|
||||
// A refused peer is closed as it is accepted and counted: the plugin's
|
||||
// Accept returns the next admitted one.
|
||||
func TestRefusedPeerNeverReachesAccept(t *testing.T) {
|
||||
inner := &fakeListener{conns: make(chan net.Conn, 2)}
|
||||
refused, refusedPeer := net.Pipe()
|
||||
admitted, _ := net.Pipe()
|
||||
inner.conns <- remoteAt(refused, "192.0.2.1")
|
||||
inner.conns <- remoteAt(admitted, "127.0.0.1")
|
||||
ln := mustNew(t, "127.0.0.1", nil, []string{"192.0.2.0/24"}).Listener(inner)
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := conn.RemoteAddr().String(); !strings.HasPrefix(got, "127.0.0.1:") {
|
||||
t.Fatalf("Accept returned %s", got)
|
||||
}
|
||||
refusedPeer.SetReadDeadline(time.Now().Add(time.Second))
|
||||
if _, err := refusedPeer.Read(make([]byte, 1)); err != io.EOF {
|
||||
t.Fatalf("the refused peer reads %v, want EOF", err)
|
||||
}
|
||||
if n := ln.(*listener).p.Stats()["acl_denied"]; n != uint64(1) {
|
||||
t.Fatalf("acl_denied = %v", n)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeListener struct {
|
||||
net.Listener
|
||||
conns chan net.Conn
|
||||
}
|
||||
|
||||
func (f *fakeListener) Accept() (net.Conn, error) { return <-f.conns, nil }
|
||||
|
||||
type remoteConn struct {
|
||||
net.Conn
|
||||
remote net.Addr
|
||||
}
|
||||
|
||||
func (c remoteConn) RemoteAddr() net.Addr { return c.remote }
|
||||
|
||||
func remoteAt(c net.Conn, addr string) net.Conn {
|
||||
return remoteConn{c, net.TCPAddrFromAddrPort(netip.AddrPortFrom(netip.MustParseAddr(addr), 40000))}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/chain"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/sink"
|
||||
@@ -78,6 +79,7 @@ type HTTPSink struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
web map[string]http.Handler // GET paths of the browser files and the root
|
||||
|
||||
// Runtime. Stop closes flush; the broker then moves what remains of its
|
||||
@@ -158,6 +160,10 @@ func NewHTTPSinkPlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "http_sink", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case (opts.LoginPage || opts.ViewerPage) && !authPolicy.BehindProxy():
|
||||
return nil, errors.New("login_page and viewer_page apply to scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls the viewer is always served")
|
||||
@@ -185,6 +191,7 @@ func NewHTTPSinkPlugin(
|
||||
keepalive: core.StreamKeepaliveInterval,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
web: web,
|
||||
}
|
||||
h.lastProcessed.Store(time.Time{})
|
||||
@@ -199,6 +206,7 @@ func NewHTTPSinkPlugin(
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe(),
|
||||
"login_page", opts.LoginPage,
|
||||
"viewer_page", web[chain.AuthPath+"/view"] != nil)
|
||||
tlsx.LogWarnings(logger, "http_sink", id, opts.TLS, true)
|
||||
@@ -244,6 +252,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error {
|
||||
ln.Close()
|
||||
return err
|
||||
}
|
||||
ln = h.acl.Listener(ln)
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped patterns: mux answers 405 with Allow header on non-GET
|
||||
mux.HandleFunc(http.MethodGet+" "+exact(h.config.StreamPath), h.handleStream)
|
||||
@@ -569,6 +578,7 @@ func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
"port": h.config.Port,
|
||||
"tls": h.tlsConfig != nil,
|
||||
"auth": h.auth.Describe(),
|
||||
"acl": h.acl.Describe(),
|
||||
"active_clients": h.activeClients.Load(),
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"client_buffer_size": h.config.ClientBufferSize,
|
||||
@@ -585,6 +595,7 @@ func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
"dropped_writes": h.droppedWrites.Load(),
|
||||
"rejected_clients": h.rejectedClients.Load(),
|
||||
"auth_rejected": h.auth.Rejected(),
|
||||
"acl_denied": h.acl.Denied(),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -611,6 +622,7 @@ func (h *HTTPSink) GetStats() sink.SinkStats {
|
||||
},
|
||||
}
|
||||
maps.Copy(details, h.auth.Stats())
|
||||
maps.Copy(details, h.acl.Stats())
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: h.id,
|
||||
|
||||
@@ -256,8 +256,8 @@ func (t *HTTPChainSink) Start(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the loop. Worst case: one in-flight request timeout plus
|
||||
// one final-flush request timeout.
|
||||
// Stop delivers what is queued and batched within sink.FlushBound, then ends
|
||||
// the loop
|
||||
func (t *HTTPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping HTTP chain sink",
|
||||
"component", "http_chain_sink",
|
||||
@@ -305,49 +305,41 @@ func (t *HTTPChainSink) GetStats() sink.SinkStats {
|
||||
func (t *HTTPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
|
||||
// Fold done channel into a context for request/backoff interruption
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
// Stop leaves sink.FlushBound to deliver the queue and the batch, so a
|
||||
// finite input arrives whole; a request or backoff in flight then ends,
|
||||
// and every later flush drops its batch at once
|
||||
ctx, cancel := sink.FlushContext(ctx, t.done, t.reqTimeout)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-runCtx.Done():
|
||||
t.finalFlush()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if t.batchCount > 0 && !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
if t.batchCount > 0 {
|
||||
t.flush(ctx)
|
||||
}
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
t.finalFlush()
|
||||
return
|
||||
case event := <-t.input:
|
||||
if t.append(event) {
|
||||
t.flush(ctx)
|
||||
}
|
||||
t.append(event)
|
||||
if t.batchCount >= t.config.MaxBatchCount ||
|
||||
int64(t.batch.Len()) >= t.config.MaxBatchBytes {
|
||||
if !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
case <-t.done:
|
||||
for len(t.input) > 0 {
|
||||
if t.append(<-t.input) {
|
||||
t.flush(ctx)
|
||||
}
|
||||
}
|
||||
if t.batchCount > 0 {
|
||||
t.flush(ctx)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// append serializes one event into the pending batch
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
// append serializes one event into the pending batch; true once it is full
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) bool {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
@@ -358,17 +350,17 @@ func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "http_chain_sink",
|
||||
"error", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
t.batch.Write(line)
|
||||
t.batch.WriteByte('\n')
|
||||
t.batchCount++
|
||||
return t.batchCount >= t.config.MaxBatchCount || int64(t.batch.Len()) >= t.config.MaxBatchBytes
|
||||
}
|
||||
|
||||
// flush delivers the pending batch, retrying transient failures with backoff.
|
||||
// Returns false when shutdown interrupts delivery; undelivered batch is dropped
|
||||
// by finalFlush semantics (batch already consumed here).
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
// flush delivers the pending batch, retrying transient failures with backoff
|
||||
// until shutdown drops it
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) {
|
||||
body := bytes.Clone(t.batch.Bytes())
|
||||
count := t.batchCount
|
||||
t.batch.Reset()
|
||||
@@ -377,8 +369,7 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
failures := 0
|
||||
for {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
break
|
||||
}
|
||||
transient, err := t.post(ctx, body)
|
||||
if err == nil {
|
||||
@@ -386,7 +377,7 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
return true
|
||||
return
|
||||
}
|
||||
t.requestErrors.Add(1)
|
||||
if !transient {
|
||||
@@ -396,11 +387,10 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
"target", t.url,
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return true
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
break
|
||||
}
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain batch delivery failed",
|
||||
@@ -409,27 +399,11 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
|
||||
func (t *HTTPChainSink) finalFlush() {
|
||||
if t.batchCount == 0 {
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
count := t.batchCount
|
||||
if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Warn("msg", "Final chain batch dropped on shutdown",
|
||||
"component", "http_chain_sink",
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Warn("msg", "Chain batch dropped on shutdown",
|
||||
"component", "http_chain_sink",
|
||||
"target", t.url,
|
||||
"entries", count)
|
||||
}
|
||||
|
||||
// post sends one NDJSON batch; transient=true marks retryable failures
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -121,7 +123,8 @@ func TestRefusedTokenLogsInAgainAndDeliversOnce(t *testing.T) {
|
||||
}
|
||||
sink := created.(*HTTPChainSink)
|
||||
sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("entry")})
|
||||
if !sink.flush(t.Context()) || delivered.Load() != 1 || sink.droppedBatches.Load() != 0 {
|
||||
sink.flush(t.Context())
|
||||
if delivered.Load() != 1 || sink.droppedBatches.Load() != 0 {
|
||||
t.Fatalf("delivered %d, dropped %d", delivered.Load(), sink.droppedBatches.Load())
|
||||
}
|
||||
if logins := listener.Stats()["auth_allowed"]; logins != uint64(2) {
|
||||
@@ -188,7 +191,8 @@ func TestLinkOverIPv6LoopbackWithTLS(t *testing.T) {
|
||||
sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("over IPv6")})
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
if !sink.flush(ctx) || sink.droppedBatches.Load() != 0 {
|
||||
sink.flush(ctx)
|
||||
if sink.droppedBatches.Load() != 0 {
|
||||
t.Fatalf("batch not delivered: %d request errors", sink.requestErrors.Load())
|
||||
}
|
||||
select {
|
||||
@@ -200,3 +204,50 @@ func TestLinkOverIPv6LoopbackWithTLS(t *testing.T) {
|
||||
t.Fatal("the source published nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// Stop delivers what is queued and batched, so the end of a finite input
|
||||
// reaches the source whole, and returns at the flush bound when the source
|
||||
// stops answering.
|
||||
func TestStopDeliversTheQueueWithinTheBound(t *testing.T) {
|
||||
const n = 250 // three batches of the default 100
|
||||
manager := session.NewManager(time.Hour)
|
||||
t.Cleanup(manager.Stop)
|
||||
// stop queues n entries, starts and stops a sink, and returns how long
|
||||
// Stop took and how many entries an answering source received
|
||||
stop := func(answering bool) (time.Duration, int64) {
|
||||
var received atomic.Int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if !answering {
|
||||
<-r.Context().Done() // the sink gave up
|
||||
return
|
||||
}
|
||||
received.Add(int64(bytes.Count(body, []byte("\n"))))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
_, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
|
||||
p, _ := strconv.ParseInt(port, 10, 64)
|
||||
created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{"host": "127.0.0.1", "port": p},
|
||||
log.NewLogger(), session.NewProxy(manager, "fwd"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := created.(*HTTPChainSink)
|
||||
for range n {
|
||||
sink.Input() <- core.TransportEvent{Time: time.Now(), Payload: []byte("entry")}
|
||||
}
|
||||
if err := sink.Start(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
sink.Stop()
|
||||
return time.Since(start), received.Load()
|
||||
}
|
||||
if _, got := stop(true); got != n {
|
||||
t.Fatalf("the source received %d of %d entries", got, n)
|
||||
}
|
||||
if d, _ := stop(false); d > 3500*time.Millisecond {
|
||||
t.Fatalf("Stop took %v past a 2 s flush bound", d)
|
||||
}
|
||||
}
|
||||
+21
-3
@@ -36,9 +36,9 @@ type SinkStats struct {
|
||||
Details map[string]any
|
||||
}
|
||||
|
||||
// FlushBound is how long a network sink's Stop gives its clients to take
|
||||
// what is queued: write_timeout_ms, at most core.SinkFlushTimeout, which a
|
||||
// reload also waits out for a client that stopped reading.
|
||||
// FlushBound is how long a network sink's Stop gives what is queued to reach
|
||||
// its clients or its downstream: its write or request timeout, at most
|
||||
// core.SinkFlushTimeout, which a reload also waits out for a stalled peer.
|
||||
func FlushBound(writeTimeout time.Duration) time.Duration {
|
||||
if writeTimeout > 0 && writeTimeout < core.SinkFlushTimeout {
|
||||
return writeTimeout
|
||||
@@ -46,6 +46,24 @@ func FlushBound(writeTimeout time.Duration) time.Duration {
|
||||
return core.SinkFlushTimeout
|
||||
}
|
||||
|
||||
// FlushContext is ctx ending FlushBound(timeout) after done closes: a sink
|
||||
// that dials runs its loop under it, so Stop leaves its queue that long.
|
||||
func FlushContext(ctx context.Context, done <-chan struct{}, timeout time.Duration) (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
go func() {
|
||||
select {
|
||||
case <-done:
|
||||
select {
|
||||
case <-time.After(FlushBound(timeout)):
|
||||
case <-ctx.Done():
|
||||
}
|
||||
case <-ctx.Done():
|
||||
}
|
||||
cancel()
|
||||
}()
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
// Drain takes the payloads queued in input without waiting: a network sink's
|
||||
// last events, which each client then writes on its own.
|
||||
func Drain(input <-chan core.TransportEvent) [][]byte {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/authz"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/sink"
|
||||
@@ -74,6 +75,7 @@ type TCPSink struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
|
||||
// Runtime. Stop closes flush; the broadcast loop then moves what remains
|
||||
// of its input to tail and closes flushed; each writer writes its queue
|
||||
@@ -149,6 +151,10 @@ func NewTCPSinkPlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "tcp_sink", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
t := &TCPSink{
|
||||
id: id,
|
||||
@@ -166,6 +172,7 @@ func NewTCPSinkPlugin(
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
@@ -176,7 +183,8 @@ func NewTCPSinkPlugin(
|
||||
"port", opts.Port,
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe())
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe())
|
||||
tlsx.LogWarnings(logger, "tcp_sink", id, opts.TLS, true)
|
||||
authPolicy.LogStartup(logger, "tcp_sink", id, false)
|
||||
return t, nil
|
||||
@@ -214,6 +222,7 @@ func (t *TCPSink) listen() (net.Listener, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ln = t.acl.Listener(ln)
|
||||
if t.tlsConfig != nil {
|
||||
ln = tls.NewListener(ln, t.tlsConfig)
|
||||
}
|
||||
@@ -556,6 +565,7 @@ func (t *TCPSink) GetStats() sink.SinkStats {
|
||||
"tls_handshake_errors": t.tlsHandshakeErrors.Load(),
|
||||
}
|
||||
maps.Copy(details, t.auth.Stats())
|
||||
maps.Copy(details, t.acl.Stats())
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
|
||||
@@ -219,8 +219,7 @@ func (t *TCPChainSink) Start(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the forwarding loop. Worst-case latency: one write timeout
|
||||
// plus one backoff wait (both interruptible or bounded).
|
||||
// Stop delivers what is queued within sink.FlushBound, then ends the loop
|
||||
func (t *TCPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP chain sink",
|
||||
"component", "tcp_chain_sink",
|
||||
@@ -273,49 +272,53 @@ func (t *TCPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
defer t.closeConn()
|
||||
|
||||
// Fold done into the context, so a connect or exchange in flight ends on Stop
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
// Stop leaves sink.FlushBound to deliver the queue, so a finite input
|
||||
// arrives whole; a connect, exchange or link still open then ends
|
||||
ctx, cancel := sink.FlushContext(ctx, t.done, t.writeTimeout)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
for len(t.input) > 0 {
|
||||
if !t.forward(ctx, <-t.input) {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
case event := <-t.input:
|
||||
if !t.forward(ctx, event) {
|
||||
return
|
||||
}
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "tcp_chain_sink",
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
if !t.deliver(ctx, append(line, '\n')) {
|
||||
return // shutdown during retry
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// forward delivers one event; false once shutdown ends the delivery
|
||||
func (t *TCPChainSink) forward(ctx context.Context, event core.TransportEvent) bool {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
// Non-transient: drop
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "tcp_chain_sink",
|
||||
"error", err)
|
||||
return true
|
||||
}
|
||||
if !t.deliver(ctx, append(line, '\n')) {
|
||||
return false
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
return true
|
||||
}
|
||||
|
||||
// deliver writes one line, holding it across reconnects until sent or shutdown.
|
||||
// Backpressure during outage propagates to the pipeline dispatch drop counter.
|
||||
func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool {
|
||||
@@ -399,11 +402,14 @@ func (t *TCPChainSink) connect(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
// A source never writes once a link is up: a line is a refusal this sink
|
||||
// could not otherwise see, EOF a close. Stop writing into either.
|
||||
// could not otherwise see, EOF a close. Stop writing into either. The end
|
||||
// of ctx closes the link too, cutting short a write in flight.
|
||||
stop := context.AfterFunc(ctx, func() { conn.Close() })
|
||||
t.wg.Add(1)
|
||||
go func() {
|
||||
defer t.wg.Done()
|
||||
err := authz.AwaitClose(r)
|
||||
stop()
|
||||
conn.Close()
|
||||
if errors.Is(err, authz.ErrRefused) {
|
||||
t.logger.Warn("msg", "Chain link refused",
|
||||
@@ -450,7 +456,5 @@ func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.done:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,10 @@ package tcpchain
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -66,3 +68,72 @@ func TestRefusingSourceIsRetriedWithBackoff(t *testing.T) {
|
||||
t.Fatalf("%d connections in 1.5 s against a refusing source", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop delivers what is queued, so the end of a finite input reaches the
|
||||
// source whole, and returns at the flush bound when the source stops reading.
|
||||
func TestStopDeliversTheQueueWithinTheBound(t *testing.T) {
|
||||
const n = 2000 // of 8 KiB: more than a stalled source's socket buffers hold
|
||||
padding := strings.Repeat("x", 8192)
|
||||
manager := session.NewManager(time.Hour)
|
||||
t.Cleanup(manager.Stop)
|
||||
// stop queues n entries, starts and stops a sink, and returns how long
|
||||
// Stop took and how many entries a reading source received
|
||||
stop := func(reading bool) (time.Duration, int) {
|
||||
ln, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
lines := make(chan int, 1)
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
if !reading {
|
||||
<-t.Context().Done()
|
||||
return
|
||||
}
|
||||
count, scanner := -1, bufio.NewScanner(conn) // -1: the hello
|
||||
scanner.Buffer(nil, 64*1024)
|
||||
for scanner.Scan() {
|
||||
count++
|
||||
}
|
||||
lines <- count
|
||||
}()
|
||||
_, port, _ := net.SplitHostPort(ln.Addr().String())
|
||||
p, _ := strconv.ParseInt(port, 10, 64)
|
||||
created, err := NewTCPChainSinkPlugin("fwd", map[string]any{
|
||||
"host": "127.0.0.1", "port": p, "buffer_size": int64(n), // writes time out at 5 s
|
||||
}, log.NewLogger(), session.NewProxy(manager, "fwd"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := created.(*TCPChainSink)
|
||||
for i := range n {
|
||||
sink.Input() <- core.TransportEvent{Time: time.Now(), Payload: fmt.Appendf(nil, "entry %d %s", i, padding)}
|
||||
}
|
||||
if err := sink.Start(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
sink.Stop()
|
||||
d := time.Since(start)
|
||||
if !reading {
|
||||
return d, 0
|
||||
}
|
||||
select {
|
||||
case got := <-lines:
|
||||
return d, got
|
||||
case <-time.After(5 * time.Second):
|
||||
return d, 0
|
||||
}
|
||||
}
|
||||
if _, got := stop(true); got != n {
|
||||
t.Fatalf("the source received %d of %d entries", got, n)
|
||||
}
|
||||
if d, _ := stop(false); d > 3500*time.Millisecond {
|
||||
t.Fatalf("Stop took %v past a 2 s flush bound", d)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/chain"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/source"
|
||||
@@ -59,6 +60,7 @@ type HTTPChainSource struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
|
||||
// Session cache: one session per remote host + node + authenticated identity
|
||||
sessions map[string]string // key -> sessionID
|
||||
@@ -120,6 +122,10 @@ func NewHTTPChainSourcePlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "http_chain_source", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &HTTPChainSource{
|
||||
id: id,
|
||||
@@ -131,6 +137,7 @@ func NewHTTPChainSourcePlugin(
|
||||
logger: logger,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
@@ -142,7 +149,8 @@ func NewHTTPChainSourcePlugin(
|
||||
"ingest_path", opts.IngestPath,
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe())
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe())
|
||||
tlsx.LogWarnings(logger, "http_chain_source", id, opts.TLS, true)
|
||||
authPolicy.LogStartup(logger, "http_chain_source", id, opts.TrustNode)
|
||||
return s, nil
|
||||
@@ -180,6 +188,7 @@ func (s *HTTPChainSource) Start() error {
|
||||
s.auth.Close()
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
ln = s.acl.Listener(ln)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped pattern: mux answers 405 with Allow header on non-POST
|
||||
@@ -265,6 +274,7 @@ func (s *HTTPChainSource) GetStats() source.SourceStats {
|
||||
"trust_node": s.config.TrustNode,
|
||||
}
|
||||
maps.Copy(details, s.auth.Stats())
|
||||
maps.Copy(details, s.acl.Stats())
|
||||
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/chain"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/source"
|
||||
@@ -55,6 +56,7 @@ type TCPChainSource struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
@@ -105,6 +107,10 @@ func NewTCPChainSourcePlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "tcp_chain_source", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &TCPChainSource{
|
||||
id: id,
|
||||
@@ -116,6 +122,7 @@ func NewTCPChainSourcePlugin(
|
||||
logger: logger,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
@@ -126,7 +133,8 @@ func NewTCPChainSourcePlugin(
|
||||
"port", opts.Port,
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe())
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe())
|
||||
tlsx.LogWarnings(logger, "tcp_chain_source", id, opts.TLS, true)
|
||||
authPolicy.LogStartup(logger, "tcp_chain_source", id, opts.TrustNode)
|
||||
return s, nil
|
||||
@@ -166,6 +174,7 @@ func (s *TCPChainSource) Start() error {
|
||||
s.auth.Close()
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
ln = s.acl.Listener(ln)
|
||||
if s.tlsConfig != nil {
|
||||
ln = tls.NewListener(ln, s.tlsConfig)
|
||||
}
|
||||
@@ -226,6 +235,7 @@ func (s *TCPChainSource) GetStats() source.SourceStats {
|
||||
"trust_node": s.config.TrustNode,
|
||||
}
|
||||
maps.Copy(details, s.auth.Stats())
|
||||
maps.Copy(details, s.acl.Stats())
|
||||
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
|
||||
Reference in new issue
Block a user