From ee01cb00fe51f9d02dddab82a62d67d177976f535cd27ddbd8792ca125e3b94b Mon Sep 17 00:00:00 2001 From: Lixen Wraith Date: Mon, 5 Oct 2026 01:24:02 -0400 Subject: [PATCH] v0.26.0 proxy headers for acl --- config/logwisp.toml | 22 +- deploy/lw-deploy.sh | 2 +- doc/README.md | 4 +- doc/architecture.md | 5 +- doc/deployment.md | 2 +- doc/installation.md | 5 +- doc/networking.md | 51 +-- doc/scram-auth-plan.md | 4 +- doc/security.md | 150 +++++---- doc/sinks.md | 13 +- doc/sources.md | 18 +- doc/todo.md | 94 +----- internal/config/config.go | 11 +- internal/netacl/netacl.go | 433 ++++++++++++++++++++++--- internal/netacl/netacl_test.go | 375 ++++++++++++++++++--- internal/sink/http/http.go | 4 + internal/sink/http/http_test.go | 21 ++ internal/sink/tcp/tcp.go | 3 + internal/source/httpchain/httpchain.go | 8 +- internal/source/tcpchain/tcpchain.go | 3 + test/acl-test.sh | 185 ++++++++++- 21 files changed, 1115 insertions(+), 298 deletions(-) diff --git a/config/logwisp.toml b/config/logwisp.toml index b57f949..9d19454 100644 --- a/config/logwisp.toml +++ b/config/logwisp.toml @@ -10,8 +10,9 @@ ### available in this version. TLS is available on every network source and ### sink (the [...tls] blocks below), the [...auth] blocks authenticate peers by ### certificate (mtls) or password (scram; credentials via `lw auth`), and the -### listeners' [...acl] blocks admit peers by address. Unknown keys anywhere in -### this file or in any plugin config are rejected at startup. +### listeners' [...acl] blocks admit peers by address, or by the client an L4 +### proxy's PROXY header names. Unknown keys anywhere in this file or in any +### plugin config are rejected at startup. ### ### Environment overrides use the LOGWISP_ prefix ### (LOGWISP_QUIET, LOGWISP_LOGGING_LEVEL, ...). Bare names are ignored. @@ -163,8 +164,15 @@ name = "default" ### ### Applied as a connection is accepted, before TLS: a refused peer is closed ### at once and counted in acl_denied, with a WARN at most once a minute. -### allow = [] addresses or CIDRs of the listener's family; empty = all +### allow = [] addresses or CIDRs of the listener's family (either behind +### proxy_from); empty = all ### deny = [] refused, listed in allow or not +### proxy_protocol = "off" off | optional | required: PROXY v1/v2 headers, +### read before TLS; the rules then see the client. +### optional lets a headerless stream pass as the +### proxy: use it only while rolling headers out +### proxy_from = [] the proxies that may send one; a header from any +### other peer is refused ###============================================================================ ###============================================================================ @@ -236,6 +244,8 @@ from = "end" # "end" or "start" of a newly discovered # [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS # allow = [] # empty = every address deny does not list # deny = [] +# proxy_protocol = "off" # off | optional | required +# proxy_from = [] # proxies that may send a PROXY header ## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks) # [[pipelines.plugin_sources]] @@ -266,6 +276,8 @@ from = "end" # "end" or "start" of a newly discovered # [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS # allow = [] # empty = every address deny does not list # deny = [] +# proxy_protocol = "off" # off | optional | required +# proxy_from = [] # proxies that may send a PROXY header ###============================================================================ ### Sinks (1+ required, fan-out) @@ -334,6 +346,8 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS # allow = [] # empty = every address deny does not list # deny = [] +# proxy_protocol = "off" # off | optional | required +# proxy_from = [] # proxies that may send a PROXY header ## TCP sink (streaming server) # [[pipelines.plugin_sinks]] @@ -363,6 +377,8 @@ target = "stdout" # stdout|stderr ("split" NOT supported) # [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS # allow = [] # empty = every address deny does not list # deny = [] +# proxy_protocol = "off" # off | optional | required +# proxy_from = [] # proxies that may send a PROXY header ## TCP chain sink (stdlib client; forwards to downstream tcp_chain source) ## Do NOT point a chain sink at a chain source in the SAME pipeline: entries diff --git a/deploy/lw-deploy.sh b/deploy/lw-deploy.sh index e36d499..490d434 100755 --- a/deploy/lw-deploy.sh +++ b/deploy/lw-deploy.sh @@ -120,7 +120,7 @@ Not covered by this script; edit the generated configuration instead - filters, rate limits, heartbeat, formatter flags and timestamp layouts - more than one pipeline; both chain transports, or a further hop, on one node - proxy mode: browser logins through a TLS-terminating proxy (trusted_proxies) -- acl address rules on the listeners +- acl on the listeners: address rules, PROXY headers - mtls identities other than the certificate CN; node_binding other than force - creating certificates: openssl or your PKI (doc/security.md#enabling-mtls) - several native instances on one host: use one per host or jail diff --git a/doc/README.md b/doc/README.md index 87d7bd8..3e7f40f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -30,8 +30,8 @@ streams, or downstream LogWisp nodes. - [CLI](cli.md): flags, signals, exit codes, `lw auth`; also the `lw(1)` manual, [`lw.1`](lw.1) - [Operations](operations.md): running, monitoring, tuning, troubleshooting -- [To Do](todo.md): planned work in priority order: network access control - and the PROXY protocol, packaging, smaller follow-ups +- [To Do](todo.md): planned work in priority order: per-client limits, + packaging, smaller follow-ups A fully annotated configuration covering every option lives at [`config/logwisp.toml`](../config/logwisp.toml). diff --git a/doc/architecture.md b/doc/architecture.md index c9255d9..0dd7aa9 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -167,8 +167,9 @@ TLS is built in exactly one place, `internal/tlsx`, which exposes Each pipeline owns a `session.Manager`. Plugins receive a `session.Proxy` scoped to their instance id, so one plugin cannot see or remove another's sessions. A session records the remote address, creation and last-activity -timestamps, and metadata — including `tls` and `tls_peer_cn` for TLS peers, and -`auth_method` / `auth_identity` for authorized ones. +timestamps, and metadata — including `tls` and `tls_peer_cn` for TLS peers, +`auth_method` / `auth_identity` for authorized ones, and `peer_addr`, the proxy, +for a client a PROXY header names. Idle sessions are reaped every 5 minutes against a 30-minute idle limit. The HTTP sink's broker treats a vanished session as an eviction signal and closes diff --git a/doc/deployment.md b/doc/deployment.md index 02b35f8..669eac4 100644 --- a/doc/deployment.md +++ b/doc/deployment.md @@ -184,7 +184,7 @@ every key. - filters, rate limits, heartbeat, formatter flags and timestamp layouts - more than one pipeline; both chain transports, or a further hop, on one node - proxy mode: browser logins through a TLS-terminating proxy -- `acl` address rules on the listeners +- `acl` on the listeners: address rules, PROXY headers - mtls identities other than the certificate CN; `node_binding` other than `force` - creating certificates diff --git a/doc/installation.md b/doc/installation.md index fae045b..8d17820 100644 --- a/doc/installation.md +++ b/doc/installation.md @@ -312,4 +312,7 @@ or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target. - `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed and checked; ports 15871-15879 - `acl-test.sh`: each listener admitting or refusing peers by address before - TLS, with clients from 127.0.0.1 and 127.0.0.2; ports 15881-15889 + TLS, with clients from 127.0.0.1 and 127.0.0.2, then by the client a PROXY + header names behind a stub proxy on 127.0.0.11-13 (and nginx `stream` on + 127.0.0.14 when installed), per-client login throttling included; ports + 15881-15889; skips (exit 77) where 127.0.0.2 cannot be bound diff --git a/doc/networking.md b/doc/networking.md index 44506f1..e77c74f 100644 --- a/doc/networking.md +++ b/doc/networking.md @@ -208,13 +208,22 @@ headers, and entry encoding. - Check the port is not already bound by another pipeline in the same process. **Closed as soon as it connects, before TLS** -- The listener's `acl` refused the address: `acl_denied` rises, and its log has - `Connection refused by acl` at WARN, once a minute at most. The dialer sees - `EOF` or `connection reset by peer` (curl: exit 52 or 56). -- Behind a proxy (an L4 passthrough, or the `http` sink's - `auth.trusted_proxies`) every peer has the proxy's address: allow the proxy. +- The listener's `acl` refused it: `acl_denied` rises, and its log has + `Connection refused by acl` at WARN with a `reason`, once a minute at most. + The dialer sees `EOF` or `connection reset by peer` (curl: exit 52 or 56). + - `address rules` — `allow` or `deny` excludes the client: the one a PROXY + header names (`remote_addr`, the proxy as `peer_addr`), else the socket + peer. Behind the `http` sink's `auth.trusted_proxies` that is the proxy: + allow it. + - `no PROXY header` — a peer in `proxy_from` sent none under + `proxy_protocol = "required"`: the proxy lacks `proxy_protocol on` (nginx) + or `send-proxy` (HAProxy), or the proxy's host dialed directly. + - `PROXY header: …` — malformed, too long, or incomplete after 10 s. + - `PROXY header from a peer outside proxy_from` — at the first read, after + the connection opened: list the proxy in `proxy_from`. - `acl: allow entry "...": not of the listener's family` at startup — an IPv4 entry on an IPv6 `host`, or the reverse: each family's listener takes its own. + Behind `proxy_from`, `allow` and `deny` take both; `proxy_from` its own. **TLS handshake failure** - `client didn't provide a certificate` — the listener has `client_auth = true` @@ -271,9 +280,9 @@ headers, and entry encoding. `X-Forwarded-For`. - `too many attempts` — the address failed or abandoned logins faster than one per second beyond a burst of 10, or has 4 unfinished; it clears within seconds - once the failing peer stops. Peers behind one NAT or one passthrough proxy - share the budget. `busy` — 4,096 logins in flight, or the listener is - stopping. + once the failing peer stops. Peers behind one NAT, or one passthrough proxy + that sends no PROXY header, share the budget. `busy` — 4,096 logins in + flight, or the listener is stopping. - `authentication not enabled` — the dialer has `scram`; the listener's `auth.type` is `none` or `mtls`. - `no challenge within 10s (older logwisp, or not a scram listener)` — the @@ -301,20 +310,18 @@ headers, and entry encoding. proxy mode, the proxy gate. **Behind an L4 proxy** -- Every connection through the proxy fails: it sends a PROXY header (nginx - `stream` with `proxy_protocol on`, HAProxy `send-proxy`) to a LogWisp TLS - listener, which cannot read one yet. The header lands in front of the TLS - handshake, which fails; the listener logs the proxy's address and - `first record does not look like a TLS handshake` (the `tcp` sink at DEBUG). - Route LogWisp without it: see - [Behind nginx or another proxy](security.md#behind-nginx-or-another-proxy). -- All peers share one throttling budget behind a passthrough proxy: without - PROXY every `scram` peer arrives from the proxy's address. One client failing - logins exhausts everyone's, and a fifth login while 4 are unfinished (edges - reconnecting together after a restart) is refused; both answer - `too many attempts` ([Throttling](security.md#throttling)). - Expose the LogWisp ports directly, or keep the proxy hop on a trusted - network so only trusted clients share the budget. +- Every connection through the proxy fails, the listener logging the proxy's + address and `first record does not look like a TLS handshake` (the `tcp` + sink at DEBUG): the proxy sends a PROXY header (nginx `stream` with + `proxy_protocol on`, HAProxy `send-proxy`) to a listener without + `acl.proxy_protocol`. Set it, with the proxy in `proxy_from`: see + [PROXY protocol](security.md#proxy-protocol). +- All peers share one throttling budget and logs name the proxy: it sends no + PROXY header, so every `scram` peer arrives from its address. One client + failing logins exhausts everyone's, and a fifth login while 4 are unfinished + (edges reconnecting together after a restart) is refused; both answer + `too many attempts` ([Throttling](security.md#throttling)). Have the proxy + send the header. **Entries not arriving over a chain link** - Check the sink's `connected` statistic and its `reconnects` count. diff --git a/doc/scram-auth-plan.md b/doc/scram-auth-plan.md index 06a86b3..d727460 100644 --- a/doc/scram-auth-plan.md +++ b/doc/scram-auth-plan.md @@ -374,8 +374,8 @@ and plaintext-forwarded requests `403`. The existing scripts keep passing. ## Not Implemented -1. **PROXY protocol behind TLS passthrough**: planned in - [To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd). +None. The PROXY protocol behind TLS passthrough, deferred here, shipped with +the ACL work: [Security](security.md#proxy-protocol). ## mTLS Hardening diff --git a/doc/security.md b/doc/security.md index ead4f24..e921607 100644 --- a/doc/security.md +++ b/doc/security.md @@ -26,7 +26,8 @@ configure it, and — equally important — what it does not yet do. [Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy). - Server pinning by dialers: certificate identity (`mtls`), bound certificate (`scram`). -- Address rules on every listener, applied before TLS: see +- Address rules on every listener, applied before TLS, to the client a PROXY + header (v1 or v2) from a listed L4 proxy names: see [The ACL Block](#the-acl-block). - Startup warnings for expiring certificates and risky settings: see [Startup Warnings](#startup-warnings). @@ -53,11 +54,8 @@ configure it, and — equally important — what it does not yet do. or credentials file. - Per-address connection or request limits: only SCRAM logins are throttled per address. -- Address rules for clients behind a proxy: `acl` sees the proxy, never the - client a PROXY header or `X-Forwarded-For` names. -- PROXY protocol: behind a proxy that passes TLS through, every peer shares the - proxy's address; see - [Behind nginx or another proxy](#behind-nginx-or-another-proxy). +- Address rules for `X-Forwarded-For` clients: in the `http` sink's proxy mode + `acl` sees the L7 proxy, never the client it forwards. Two credentials are supported. Certificates (`mtls`) are the one the transport already carries: the `tls` block establishes that a peer chains to your CA, and @@ -334,25 +332,63 @@ peer costs no handshake and never reaches `auth`. [pipelines.plugin_sources.config.acl] allow = ["192.0.2.0/24", "198.51.100.7"] # addresses or CIDRs; empty: all deny = ["192.0.2.66"] # refused, listed in allow or not +proxy_protocol = "required" # off (default), optional, required +proxy_from = ["10.0.0.5"] # L4 proxies sending PROXY headers ``` - `deny` wins; then a non-empty `allow` admits only its entries, an empty one every address `deny` does not list. - Entries are of the listener's [family](networking.md#address-family): IPv4 on an IPv4 listener, IPv6 on an IPv6 one, either behind a hostname (only the - family it binds ever matches). An entry of the other family, an IPv4-mapped - address, one with a zone, or anything but an address or CIDR fails - construction. A link-local peer matches without its zone. + family it binds ever matches). Behind `proxy_from` they take either, as a + header may name a client of either. An entry of the other family, an + IPv4-mapped address, one with a zone, or anything but an address or CIDR + fails construction. A link-local peer matches without its zone. - A refused connection is closed at once and counted in `acl_denied` (plugin - stats, the `http` sink's status). A WARN names the peer and the count, at - most once a minute per listener; the refusals in between are only counted. -- The rules see the socket peer: behind a proxy (an L4 passthrough, or the - `http` sink's `trusted_proxies`), the proxy (see - [Behind nginx or another proxy](#behind-nginx-or-another-proxy)). + stats, the `http` sink's status). A WARN names the peer, the reason and the + count, at most once a minute per listener; the refusals in between are only + counted. +- The rules see the client: the socket peer, or the client a PROXY header + names. Behind the `http` sink's `trusted_proxies` they see the L7 proxy. - The `serve` and `aggregator` presets take `allow` and `deny` list keys. - Dialers have no `acl`: on a `tcp_chain` or `http_chain` sink it is an unknown key. +### PROXY protocol + +Behind an L4 proxy that passes TLS through (nginx `stream`, HAProxy +`mode tcp`) every peer comes from the proxy's address. With `proxy_protocol` +the proxy names the client in a PROXY header, v1 (text) or v2 (binary), sent +before the TLS ClientHello; LogWisp still ends TLS, so channel binding and +client certificates are unchanged. + +- `proxy_protocol` and `proxy_from` come together; `proxy_from` holds the + proxies' addresses or CIDRs, of the listener's family. A peer with a zone + (link-local IPv6) is never a proxy. +- A peer in `proxy_from`: + - sends its header within `tlsx.HandshakeTimeout` (10 s); a malformed, + oversized (v1 over 107 bytes, a v2 block over 2048) or late one is + refused; + - under `required` is refused without a header; under `optional` it keeps + its socket address, after up to 10 s if it sends nothing at first. A + client the proxy forwards without a header then passes as the proxy, so + `optional` (which warns at startup) suits only a rollout; set `required` + once every route from `proxy_from` sends the header; + - keeps its socket address for `LOCAL` (v2) and `UNKNOWN` (v1), the proxies' + health checks, and for v2 families other than TCP over IPv4 and IPv6. TLVs + are skipped. +- Headers are read off the accept loop: a slow proxy holds only its own + connection. +- A peer not in `proxy_from` connects as itself; one that sends a header is + refused at its first read, as any peer can write one. +- The connection then reports the client (an IPv4-mapped one as IPv4) to the + rules, SCRAM [throttling](#throttling), sessions and logs. Session metadata + keeps the proxy as `peer_addr`, and a refusal's WARN names it. +- `acl_proxy_headers` counts the headers read. +- The `http` sink's proxy mode is the L7 counterpart: `trusted_proxies` trusts + `X-Forwarded-For` from a proxy that ends TLS, `proxy_from` a PROXY header + from one that passes it through. + ## Startup Warnings Some settings work but are usually mistakes. Each plugin reports them at WARN @@ -370,7 +406,9 @@ when it is constructed, so startup and every reload repeat them: (once per path per process) - an `acl` entry with host bits set (`10.0.0.1/8`), naming the network it matches; an `acl.allow` entry of a whole family (`0.0.0.0/0`, `::/0`); a - wildcard listener whose `acl` has `deny` but no `allow` + wildcard listener whose `acl` has `deny` but no `allow`; an `acl.proxy_from` + entry reaching beyond private, loopback and link-local addresses; + `acl.proxy_protocol = "optional"` ## Certificates made at startup @@ -584,9 +622,10 @@ exchange. ### Throttling -Logins are throttled per remote socket address or, on an `http` sink in proxy -mode, per forwarded client (an IPv6 client per /64); forwarded headers are read -only from `trusted_proxies`. Each exchange takes a token from a bucket of 10 +Logins are throttled per client (an IPv6 client per /64): the socket peer, the +client a [PROXY header](#proxy-protocol) names, or on an `http` sink in proxy +mode the forwarded client; forwarded headers are read only from +`trusted_proxies`. Each exchange takes a token from a bucket of 10 that refills at one per second, and a successful login gives it back, so only failed or abandoned attempts drain it. At most 4 exchanges per address may be unfinished: an HTTP challenge never answered holds its slot for up to 30 s, a @@ -595,8 +634,8 @@ TCP connection that ends mid-exchange frees it at once. A refused start answers 65,536 addresses and refuses new ones when full. Separately, at most 4,096 exchanges may be in flight per listener; beyond that, or while the plugin stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`. Peers behind -one NAT or one passthrough proxy share a bucket: there, one client can exhaust -every other client's logins. +one NAT, or one passthrough proxy that sends no PROXY header, share a bucket: +there, one client can exhaust every other client's logins. ### Credentials file @@ -646,9 +685,9 @@ different users. - TLS must terminate at LogWisp, except on an `http` sink in proxy mode. Behind any other terminating proxy or load balancer every login fails by design; - pass TLS through instead (TCP or SNI routing). All clients then share the - proxy's address, and so one throttling budget: see - [Behind nginx or another proxy](#behind-nginx-or-another-proxy). + pass TLS through instead (TCP or SNI routing), with a + [PROXY header](#proxy-protocol) so each client keeps its own throttling + budget. - One process per HTTP address. Handshake state and the token key live in one instance, so behind a balancer the proof or the token can reach an instance that never saw the login. @@ -756,7 +795,8 @@ A proxy in front of LogWisp works at one of two layers: overwrite `X-Forwarded-For` with the real client: the rightmost hop that is not a proxy is the client either way. - **L4, passing TLS through to LogWisp's own TLS** (nginx `stream` with - `ssl_preread`, HAProxy `mode tcp`): whatever needs TLS to end at LogWisp. + `ssl_preread`, HAProxy `mode tcp`), with a [PROXY header](#proxy-protocol): + whatever needs TLS to end at LogWisp. - Chain links (`tcp_chain`, `http_chain`). - `lw auth stream` viewers of a `tcp` sink. - An `http` sink that keeps its own TLS, and with it SCRAM channel binding @@ -767,7 +807,7 @@ A common shape serves both from port 443: ``` :443 nginx stream: ssl_preread routes by SNI, proxy_protocol on logs.example.org -> 127.0.0.1:8443 nginx http - relay.example.org -> a LogWisp TLS listener, without PROXY (below) + relay.example.org -> a LogWisp TLS listener, which reads the header (below) 127.0.0.1:8443 nginx http: listens with proxy_protocol, ends TLS /logs/ -> 127.0.0.1:8081 LogWisp http sink, proxy mode ``` @@ -796,52 +836,40 @@ client or, when it is trusted (both on `127.0.0.1`), the hop to its left: one the client wrote, so the client picks its own throttling address. LogWisp ignores `X-Real-IP`. -**LogWisp's own TLS listeners take no PROXY header.** LogWisp cannot read the -PROXY protocol yet; it is planned first in the ACL work -([To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd)). -A stream route that sends `proxy_protocol` straight to a LogWisp TLS listener -breaks every connection: the header lands in front of the TLS handshake. nginx -sets `proxy_protocol` per stream `server`, so route LogWisp SNIs without it: - -- Route 1: LogWisp gets its own stream `server` and port, without - `proxy_protocol`. -- Route 2: port 443 sends LogWisp SNIs to an internal stream `server` that - `listen`s with `proxy_protocol` and proxy_passes to LogWisp without it; that - hop consumes the header. -- Either way dialers must send the name: a `host` that is one, or - `tls.server_name` (`--server-name` for `lw auth`); an IP literal sends no SNI. -- With HAProxy, leave `send-proxy` off the LogWisp backends. +**LogWisp's own TLS listeners read the header** when their `acl` sets +`proxy_protocol` and lists the stream server in `proxy_from`; one stream +`server` with `proxy_protocol on` then routes every name: ```nginx stream { map $ssl_preread_server_name $route { - relay.example.org 127.0.0.1:10443; # route 2: the internal hop - tail.example.org 127.0.0.1:10443; - default 127.0.0.1:8443; # the http block, logs.example.org - } - map $ssl_preread_server_name $logwisp { relay.example.org 127.0.0.1:9001; # tcp_chain source tail.example.org 127.0.0.1:9002; # tcp sink, for lw auth stream + default 127.0.0.1:8443; # the http block, logs.example.org } server { listen 443; ssl_preread on; - proxy_protocol on; # on every route of this server + proxy_protocol on; proxy_pass $route; } - server { # route 1: its own port, no header - listen 9443; - ssl_preread on; - proxy_pass $logwisp; - } - server { # route 2: consumes the header - listen 127.0.0.1:10443 proxy_protocol; - ssl_preread on; - proxy_pass $logwisp; - } } ``` +```toml +[pipelines.plugin_sources.config.acl] # tcp_chain source; tcp sink alike +proxy_protocol = "required" +proxy_from = ["127.0.0.1"] # the stream server +``` + +- HAProxy: `send-proxy` or `send-proxy-v2` on the LogWisp backends. +- A listener without `proxy_protocol` behind such a route fails every + connection: the header lands in front of its TLS handshake. +- Dialers must send the name: a `host` that is one, or `tls.server_name` + (`--server-name` for `lw auth`); an IP literal sends no SNI. +- `test/acl-test.sh --auto` runs each listener kind behind a stub proxy + sending a v1 or v2 header, and the `tcp` sink behind nginx when installed. + **Passthrough without PROXY** brings every peer from the proxy's address: - All peers share one SCRAM [throttling](#throttling) budget: 10 failed or @@ -853,14 +881,8 @@ stream { (`too many attempts` or `busy`) keeps sending on its token to the server it pinned, retrying every few seconds, and holds its batches only once that token has expired; any other refusal ends the token at once. -- Logs and sessions name the proxy, not the peer, and `acl` rules see only the - proxy: allow it, not the clients. -- Proxy mode is unaffected: it throttles on the forwarded client. -- Until LogWisp reads PROXY, mitigate: - - expose the chain ports directly, without the proxy, where you can; - - keep the proxy hop on a trusted network, so only trusted clients share the - budget; - - or accept the shared budget. +- Logs and sessions name the proxy, and `acl` rules see only the proxy: allow + it, not the clients. ## What Each Layer Enforces diff --git a/doc/sinks.md b/doc/sinks.md index 5754ae1..5d49322 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -179,7 +179,8 @@ Options, each as type and default: - `auth` (table): client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block). - `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused - before TLS; see [Security](security.md#the-acl-block). + before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY + header names; see [Security](security.md#the-acl-block). **Behaviour** @@ -251,8 +252,8 @@ buffer sizes, connection limit, write timeout, uptime, endpoint paths, and the **Statistics**: `dropped_writes`, `rejected_clients`, `auth`, `auth_allowed`, `auth_rejected`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`, `auth_binding_mismatch` and `auth_token_lifetime_ms`, and in -proxy mode `auth_trusted_proxies`; with `acl` rules also `acl` and -`acl_denied`. +proxy mode `auth_trusted_proxies`; with `acl` rules or `proxy_from` also +`acl`, `acl_denied` and `acl_proxy_headers`. > Without an `auth` block both endpoints are unauthenticated, and the stream > response carries `Access-Control-Allow-Origin: *`, so any web origin can read @@ -306,7 +307,8 @@ Options, each as type and default: - `auth` (table): client authentication (`mtls` or `scram`); see [Security](security.md#the-auth-block). - `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused - before TLS; see [Security](security.md#the-acl-block). + before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY + header names; see [Security](security.md#the-acl-block). **Behaviour** @@ -333,7 +335,8 @@ Options, each as type and default: **Statistics**: `write_errors`, `dropped_writes`, `rejected_conns`, `tls_handshake_errors`, `auth`, `auth_allowed`, `auth_rejected`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy` and `auth_binding_mismatch`; -with `acl` rules also `acl` and `acl_denied`. +with `acl` rules or `proxy_from` also `acl`, `acl_denied` and +`acl_proxy_headers`. --- diff --git a/doc/sources.md b/doc/sources.md index 211d41e..6ce5010 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -215,7 +215,8 @@ Options, each as type and default: - `auth` (table): peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block). - `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused - before TLS; see [Security](security.md#the-acl-block). + before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY + header names; see [Security](security.md#the-acl-block). For passwords instead of certificates, the `auth` block takes `type = "scram"` and a `credentials_file`, and `client_auth` becomes optional; see @@ -240,8 +241,9 @@ and a `credentials_file`, and `client_auth` becomes optional; see leaves per-entry labels to `trust_node`, so a relay can forward other nodes' entries while proving its own identity. - Each accepted connection gets a session recording the remote address, node - label, — under TLS — `tls` and `tls_peer_cn`, and — under auth — - `auth_method` and `auth_identity`. + label, — under TLS — `tls` and `tls_peer_cn`, — under auth — + `auth_method` and `auth_identity`, and behind a PROXY header the proxy as + `peer_addr`. - A malformed entry line increments `parse_errors` and is skipped; the connection survives. A line over 1 MiB is a protocol violation and terminates the connection. @@ -249,7 +251,8 @@ and a `credentials_file`, and `client_auth` becomes optional; see **Statistics**: `active_connections`, `rejected_conns`, `parse_errors`, `tls_handshake_errors`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`, `node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy` -and `auth_binding_mismatch`; with `acl` rules also `acl` and `acl_denied`. +and `auth_binding_mismatch`; with `acl` rules or `proxy_from` also `acl`, +`acl_denied` and `acl_proxy_headers`. --- @@ -301,7 +304,8 @@ Options, each as type and default: - `auth` (table): peer authentication (`mtls` or `scram`) and node binding; see [Security](security.md#the-auth-block). - `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused - before TLS; see [Security](security.md#the-acl-block). + before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY + header names; see [Security](security.md#the-acl-block). **Behaviour** @@ -328,8 +332,8 @@ Options, each as type and default: **Statistics**: `total_requests`, `rejected_requests`, `parse_errors`, `cached_sessions`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`, `node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`, -`auth_binding_mismatch` and `auth_token_lifetime_ms`; with `acl` rules also -`acl` and `acl_denied`. +`auth_binding_mismatch` and `auth_token_lifetime_ms`; with `acl` rules or +`proxy_from` also `acl`, `acl_denied` and `acl_proxy_headers`. --- diff --git a/doc/todo.md b/doc/todo.md index 0081e80..9d97cd5 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -7,98 +7,40 @@ they belong to. ## 1. Network access control (ACL) The seam exists: `internal/netacl` wraps each listener's socket before TLS, -and the `acl` block's `allow` and `deny` refuse peers by address +reads the PROXY header of the L4 proxies `proxy_from` lists, and the `acl` +block's `allow` and `deny` refuse the client by address ([Security](security.md#the-acl-block)). Every limit except SCRAM throttling -is still global, and behind an L4 proxy every peer is the proxy. +is still global. -### 1.1 The filtering listener - -- `Accept` is to read an optional PROXY header (1.2) before the address rules - and apply per-address connection caps (1.4) after them, returning a - `net.Conn` whose `RemoteAddr` is the real client. -- Everything downstream then sees the real client with no further change: - `authz.remoteIP`, SCRAM throttling, sessions, logs and `http.Request.RemoteAddr`. -- Keep the proxy's own address as `peer_addr` in session metadata for audit. - -### 1.2 PROXY protocol (deferred gap of SCRAM, see scram-auth-plan.md) - -- Why: behind an L4 proxy that passes TLS through (nginx `stream` with - `ssl_preread`, HAProxy `mode tcp`), every peer arrives from the proxy's - address. - - All peers share one SCRAM throttling budget, so one client can lock out - every login. - - Logs, sessions and the ACL itself see only the proxy. -- What: accept PROXY v1 (text) and v2 (binary) headers, but only from - `proxy_from` addresses or CIDRs. Two config keys in the ACL block: - - `proxy_protocol = "off" | "optional" | "required"`, default off. - - `proxy_from = [...]`. -- Rules: - - A listed peer must send the header when `required`. - - An unlisted peer that sends a header is refused, since the header is - spoofable. - - The header must arrive within a short deadline (reuse - `tlsx.HandshakeTimeout`), with a size cap: v1 at most 107 bytes, v2 with a - bounded TLV length. - - `LOCAL`/`UNKNOWN` commands (proxy health checks) keep the socket address. - - TLVs are ignored. -- The header precedes the TLS ClientHello, so SCRAM channel binding is - unaffected: LogWisp still terminates TLS. -- nginx side: - - In the `stream` server, `proxy_protocol on` on the route to LogWisp (it - sends v1). - - HAProxy: `send-proxy` or `send-proxy-v2`. -- Relation to `auth.trusted_proxies` (HTTP proxy mode): - - That list trusts `X-Forwarded-For` from a TLS-terminating L7 proxy; - `proxy_from` trusts a PROXY header from an L4 proxy. - - Keep both, with distinct names and docs. They answer different layers, and - one deployment can use both: nginx `stream` with `proxy_protocol` into an - nginx `http` block that sets `X-Forwarded-For $proxy_protocol_addr`. - -### 1.3 Address rules for forwarded clients - -- Rules match the real client (after PROXY), never the proxy. -- In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after - the connection-level decision, so rules for forwarded HTTP clients belong in - a second, request-level check there, reusing the compiled `netacl` rules. - `authz.parseProxies` then takes netacl's address-or-CIDR parsing too. -- Startup warning: `proxy_from` covering public ranges. - -### 1.4 Per-client limits +### 1.1 Per-client limits - Generalize the SCRAM limiter in `internal/authz/scram.go` (token bucket per key, bounded table that fails closed, idle sweep, IPv6 per /64 via `throttleKey`) into the shared per-key limiter. One mechanism (AGENTS.md): SCRAM throttling then uses it too. -- Keys: +- Keys in the `acl` block: - `max_connections_per_client` on every listener (concurrent connections per - address). + client), applied by the netacl listener after the rules. - `requests_per_second_per_client` on HTTP listeners (ingest, stream and status requests). +- A full table refuses new clients and counts them in `acl_limited`, beside + `acl_denied` and `acl_proxy_headers`. - Later: per-identity limits once a peer is authenticated (the mtls plan's deferred item 2), keyed by identity instead of address. -### 1.5 Config, validation, docs +### 1.2 Address rules for forwarded clients -- `proxy_protocol`, `proxy_from` and the 1.4 keys join `allow` and `deny` in - the `acl` block. -- Intent rule as in `authz`: `proxy_from` without `proxy_protocol` is an error. -- Stats: `acl_proxy_headers` and `acl_limited` beside `acl_denied`. -- Docs: the security.md section, plus networking.md troubleshooting lines for - each refusal. +- In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after + the connection-level decision, so rules for forwarded HTTP clients belong in + a second, request-level check there, reusing the compiled `netacl` rules. + `authz.parseProxies` then takes netacl's address-or-CIDR parsing too. -### 1.6 Verification +### 1.3 Verification -- Go tests, one per rule: - - PROXY v1 and v2 parse, including malformed, oversized and slow headers. - - A header from an unlisted peer refused; `required` without a header - refused; `LOCAL` kept. - - Per-client connection cap; the limiter table failing closed. - - The real client reaching SCRAM throttling and session metadata. -- `test/acl-test.sh`, which covers the address rules, adds: - - A Go stub proxy that sends v1 and v2 headers in front of each listener - type. - - nginx `stream` with `proxy_protocol on` when nginx is installed. - - Per-client SCRAM budgets behind one proxy address. +- Go tests, one per rule: the per-client connection cap; the request rate; the + limiter table failing closed; a forwarded client refused by the rules. +- `test/acl-test.sh` adds per-client connection caps, behind its stub proxy + too. ## 2. Packaging: AUR, FreeBSD ports, Debian diff --git a/internal/config/config.go b/internal/config/config.go index 478767c..732bcd2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -374,11 +374,14 @@ type AuthOptions struct { } // ACLOptions is a listener's address rules, entries being addresses or CIDRs: -// deny wins, then a set allow list admits only its entries. Validation is in -// internal/netacl. +// deny wins, then a set allow list admits only its entries. proxy_from lists +// the L4 proxies whose PROXY header names the client the rules then see. +// Validation is in internal/netacl. type ACLOptions struct { - Allow []string `toml:"allow"` - Deny []string `toml:"deny"` + Allow []string `toml:"allow"` + Deny []string `toml:"deny"` + ProxyProtocol string `toml:"proxy_protocol"` // off, optional or required + ProxyFrom []string `toml:"proxy_from"` } // --- TLS Options --- diff --git a/internal/netacl/netacl.go b/internal/netacl/netacl.go index 7dd63ea..a8ca64e 100644 --- a/internal/netacl/netacl.go +++ b/internal/netacl/netacl.go @@ -1,43 +1,87 @@ // Package netacl is the address seam of the network listeners, beside tlsx -// and authz: a Policy refuses a peer by its address as the socket is -// accepted, before TLS or a byte of the protocol is read. New returns nil -// when no rule is set, and every method tolerates a nil receiver. +// and authz: a Policy reads the PROXY header of the proxies it trusts, then +// refuses a peer by its client's address, before TLS or a byte of the +// protocol is read. New returns nil when nothing is set, and every method +// tolerates a nil receiver. package netacl import ( + "bufio" + "bytes" + "context" + "encoding/binary" + "errors" "fmt" "net" "net/netip" "slices" + "strconv" "strings" + "sync" "sync/atomic" "time" "github.com/lixenwraith/logwisp/internal/config" "github.com/lixenwraith/logwisp/internal/core" + "github.com/lixenwraith/logwisp/internal/tlsx" "github.com/lixenwraith/log" ) -// warnEvery spaces the WARNs reporting refused peers; acl_denied counts all -const warnEvery = time.Minute +const ( + // warnEvery spaces the WARNs reporting refused peers; acl_denied counts all + warnEvery = time.Minute + v1Max = 107 // the longest PROXY v1 line, CRLF included + v2Max = 2048 // the longest PROXY v2 address block, TLVs included +) + +var ( + sigV1 = []byte("PROXY ") + sigV2 = []byte("\r\n\r\n\x00\r\nQUIT\n") + // local is where a proxy_from entry lies without a startup warning + local = []netip.Prefix{ + netip.MustParsePrefix("10.0.0.0/8"), netip.MustParsePrefix("172.16.0.0/12"), + netip.MustParsePrefix("192.168.0.0/16"), netip.MustParsePrefix("127.0.0.0/8"), + netip.MustParsePrefix("169.254.0.0/16"), netip.MustParsePrefix("fc00::/7"), + netip.MustParsePrefix("fe80::/10"), netip.MustParsePrefix("::1/128"), + } + errScreened = errors.New("PROXY header from a peer outside proxy_from") + // v2Families holds the family bytes v2 defines, each with the address + // bytes taken under PROXY: TCP over IPv4 and IPv6 name the client, the + // rest keep the socket address + v2Families = map[byte]int{0x00: 0, 0x11: 12, 0x12: 0, 0x21: 36, 0x22: 0, 0x31: 0, 0x32: 0} +) // Policy is the compiled form of config.ACLOptions for one listener type Policy struct { - allow, deny []netip.Prefix - logger *log.Logger - component, id string + allow, deny, proxyFrom []netip.Prefix + mode string // proxy_protocol, when proxy_from is set + headerWait time.Duration // for a proxy's header: tlsx.HandshakeTimeout + logger *log.Logger + component, id string - denied atomic.Uint64 - epoch time.Time // a monotonic base: a stepped clock neither mutes nor floods the WARN - nextWarn atomic.Int64 // nanoseconds since epoch + denied, headers atomic.Uint64 + epoch time.Time // a monotonic base: a stepped clock neither mutes nor floods the WARN + nextWarn atomic.Int64 // nanoseconds since epoch } // New compiles the rules of a listener on host and warns of what they leave -// open. Each entry must be of the family core.Network gives host: one the -// listener's sockets can never carry is a mistake, not a rule. +// open. Entries are of the family core.Network gives host, as one the sockets +// never carry is a mistake; behind proxy_from, allow and deny take both, as +// the proxy names clients of either. func New(o *config.ACLOptions, host string, l *log.Logger, component, id string) (*Policy, error) { - if o == nil || len(o.Allow)+len(o.Deny) == 0 { + if o == nil { + return nil, nil + } + proxied := o.ProxyProtocol == "optional" || o.ProxyProtocol == "required" + switch { + case !proxied && o.ProxyProtocol != "" && o.ProxyProtocol != "off": + return nil, fmt.Errorf("acl: proxy_protocol %q is none of off, optional and required", o.ProxyProtocol) + case proxied && len(o.ProxyFrom) == 0: + return nil, fmt.Errorf("acl: proxy_protocol %s needs proxy_from, the proxies that may send the header", o.ProxyProtocol) + case !proxied && len(o.ProxyFrom) > 0: + return nil, errors.New("acl: proxy_from needs proxy_protocol optional or required") + case len(o.Allow)+len(o.Deny)+len(o.ProxyFrom) == 0: return nil, nil } network, err := core.Network(host) @@ -45,7 +89,7 @@ func New(o *config.ACLOptions, host string, l *log.Logger, component, id string) return nil, err } var widened []string // entry, network pairs - compile := func(key string, entries []string) ([]netip.Prefix, error) { + compile := func(key, network string, entries []string) ([]netip.Prefix, error) { var rules []netip.Prefix for _, e := range entries { prefix, err := parse(e, network) @@ -59,11 +103,17 @@ func New(o *config.ACLOptions, host string, l *log.Logger, component, id string) } return rules, nil } - p := &Policy{logger: l, component: component, id: id, epoch: time.Now()} - if p.allow, err = compile("allow", o.Allow); err != nil { + p := &Policy{mode: o.ProxyProtocol, headerWait: tlsx.HandshakeTimeout, logger: l, component: component, id: id, epoch: time.Now()} + if p.proxyFrom, err = compile("proxy_from", network, o.ProxyFrom); err != nil { return nil, err } - if p.deny, err = compile("deny", o.Deny); err != nil { + if proxied { + network = "tcp" + } + if p.allow, err = compile("allow", network, o.Allow); err != nil { + return nil, err + } + if p.deny, err = compile("deny", network, o.Deny); err != nil { return nil, err } p.logStartup(host, widened) @@ -106,7 +156,22 @@ func (p *Policy) logStartup(host string, widened []string) { "hint", "list only the networks that may connect") } } - if addr, err := netip.ParseAddr(host); len(p.allow) == 0 && (host == "" || err == nil && addr.IsUnspecified()) { + if p.mode == "optional" { + p.logger.Warn("msg", "acl.proxy_protocol optional: a client a proxy_from peer forwards without a header passes as the proxy", + "component", p.component, + "instance_id", p.id, + "hint", "set required once every route from proxy_from sends the header") + } + for _, q := range p.proxyFrom { + if !slices.ContainsFunc(local, func(l netip.Prefix) bool { return l.Bits() <= q.Bits() && l.Contains(q.Addr()) }) { + p.logger.Warn("msg", "acl.proxy_from reaches public addresses, any of which may name any client", + "component", p.component, + "instance_id", p.id, + "entry", q.String(), + "hint", "list only the proxies' own addresses") + } + } + if addr, err := netip.ParseAddr(host); len(p.allow) == 0 && len(p.deny) > 0 && (host == "" || err == nil && addr.IsUnspecified()) { p.logger.Warn("msg", "Listener on every address admits all that acl.deny does not list", "component", p.component, "instance_id", p.id, @@ -114,61 +179,323 @@ func (p *Policy) logStartup(host string, widened []string) { } } +// in reports whether addr lies in a rule, matched unmapped and without its +// zone: a zoned address is in no prefix +func in(rules []netip.Prefix, addr netip.Addr) bool { + addr = addr.Unmap().WithZone("") + return slices.ContainsFunc(rules, func(q netip.Prefix) bool { return q.Contains(addr) }) +} + // admits applies the rules: deny wins, then a set allow list admits only its // entries. A peer whose address is unknown is refused. func (p *Policy) admits(addr netip.Addr) bool { - addr = addr.Unmap().WithZone("") // a zoned address is in no prefix - in := func(q netip.Prefix) bool { return q.Contains(addr) } - return addr.IsValid() && !slices.ContainsFunc(p.deny, in) && - (len(p.allow) == 0 || slices.ContainsFunc(p.allow, in)) + return addr.IsValid() && !in(p.deny, addr) && (len(p.allow) == 0 || in(p.allow, addr)) } -// Listener returns ln refusing the peers the rules exclude as it accepts them, -// so the plugin never sees one. Close stays ln's: a sink's shutdown flush -// reaches it unchanged. A nil Policy returns ln. +// proxies reports a peer in proxy_from. A zoned one never is: a link-local +// address is any link's to claim. +func (p *Policy) proxies(addr netip.Addr) bool { + addr = addr.Unmap() + return slices.ContainsFunc(p.proxyFrom, func(q netip.Prefix) bool { return q.Contains(addr) }) +} + +func addrOf(c net.Conn) netip.Addr { + if t, ok := c.RemoteAddr().(*net.TCPAddr); ok { + return t.AddrPort().Addr() + } + return netip.Addr{} +} + +// Listener returns ln deciding each peer before Accept returns it. Headers are +// read off the accept loop, so a slow proxy holds only its own connection; the +// loop starts at once, so call this in Start. A nil Policy returns ln. func (p *Policy) Listener(ln net.Listener) net.Listener { if p == nil { return ln } - return &listener{Listener: ln, p: p} + ctx, cancel := context.WithCancel(context.Background()) + l := &listener{Listener: ln, p: p, accepted: make(chan accepted), ctx: ctx, cancel: cancel} + go l.run() + return l } type listener struct { net.Listener - p *Policy + p *Policy + accepted chan accepted + ctx context.Context // done at Close + cancel context.CancelFunc +} + +type accepted struct { + conn net.Conn + err error } func (l *listener) Accept() (net.Conn, error) { - for { - conn, err := l.Listener.Accept() - if err != nil { - return nil, err - } - var addr netip.Addr - remote := conn.RemoteAddr() - if t, ok := remote.(*net.TCPAddr); ok { - addr = t.AddrPort().Addr() - } - if l.p.admits(addr) { - return conn, nil - } - conn.Close() - l.p.refused(fmt.Sprint(remote)) + select { + case a := <-l.accepted: + return a.conn, a.err + case <-l.ctx.Done(): + return nil, net.ErrClosed } } -// refused counts a refusal and WARNs at most once per warnEvery -func (p *Policy) refused(remote string) { - n := p.denied.Add(1) +// Close also closes the connections still sending a header +func (l *listener) Close() error { + l.cancel() + return l.Listener.Close() +} + +func (l *listener) run() { + for { + c, err := l.Listener.Accept() + if err != nil { + if !l.deliver(nil, err) || errors.Is(err, net.ErrClosed) { + return + } + continue + } + switch { + case l.p.proxies(addrOf(c)): + go l.proxied(c) + case len(l.p.proxyFrom) > 0: + l.admit(&conn{Conn: c, r: bufio.NewReaderSize(c, v1Max), screen: l.p}) + default: + l.admit(c) + } + } +} + +// proxied reads the header of a peer in proxy_from, then admits the +// connection by the client it names +func (l *listener) proxied(c net.Conn) { + stop := context.AfterFunc(l.ctx, func() { c.Close() }) + c.SetReadDeadline(time.Now().Add(l.p.headerWait)) + br := bufio.NewReaderSize(c, v1Max) + client, found, err := readHeader(br) + c.SetReadDeadline(time.Time{}) + if !stop() { + return // closed with the listener + } + pc := &conn{Conn: c, r: br} + switch { + case err != nil: + l.p.refuse(c, "PROXY header: "+err.Error()) + return + case !found && l.p.mode == "required": + l.p.refuse(c, "no PROXY header") + return + case found: + l.p.headers.Add(1) + if client.IsValid() { + pc.client = net.TCPAddrFromAddrPort(client) + } + } + l.admit(pc) +} + +// admit hands c to Accept if the rules admit the address it ends with +func (l *listener) admit(c net.Conn) { + if !l.p.admits(addrOf(c)) { + l.p.refuse(c, "address rules") + return + } + l.deliver(c, nil) +} + +func (l *listener) deliver(c net.Conn, err error) bool { + select { + case l.accepted <- accepted{c, err}: + return true + case <-l.ctx.Done(): + if c != nil { + c.Close() + } + return false + } +} + +// refuse closes c, counts it and WARNs at most once per warnEvery +func (p *Policy) refuse(c net.Conn, reason string) { + n := p.denied.Add(1) // first: a peer that sees the close sees the count + c.Close() now, next := int64(time.Since(p.epoch)), p.nextWarn.Load() if now < next || !p.nextWarn.CompareAndSwap(next, now+int64(warnEvery)) { return } - p.logger.Warn("msg", "Connection refused by acl; more within a minute are only counted", + fields := []any{"msg", "Connection refused by acl; more within a minute are only counted", "component", p.component, "instance_id", p.id, - "remote_addr", remote, - "acl_denied", n) + "remote_addr", fmt.Sprint(c.RemoteAddr()), + "reason", reason, + "acl_denied", n} + if peer := PeerAddr(c); peer != "" { + fields = append(fields, "peer_addr", peer) + } + p.logger.Warn(fields...) +} + +// conn is read through the buffer its first bytes went to: past a PROXY +// header, whose client it reports, or under a screen refusing one from a peer +// outside proxy_from at the first Read +type conn struct { + net.Conn + r *bufio.Reader + client net.Addr // nil keeps the socket's + screen *Policy + once sync.Once + err error +} + +func (c *conn) Read(b []byte) (int, error) { + c.once.Do(func() { + if c.screen == nil { + return + } + if v, _ := signature(c.r); v != 0 { + c.err = errScreened + c.screen.refuse(c, errScreened.Error()) + } + }) + if c.err != nil { + return 0, c.err + } + return c.r.Read(b) +} + +func (c *conn) RemoteAddr() net.Addr { + if c.client != nil { + return c.client + } + return c.Conn.RemoteAddr() +} + +// PeerAddr is the proxy whose PROXY header named c's client, "" when c came +// direct; c may be the *tls.Conn over the listener's connection +func PeerAddr(c net.Conn) string { + if t, ok := c.(interface{ NetConn() net.Conn }); ok { + c = t.NetConn() + } + if pc, ok := c.(*conn); ok && pc.client != nil { + return pc.Conn.RemoteAddr().String() + } + return "" +} + +type peerKey struct{} + +// ConnContext, as an http.Server's, carries PeerAddr to the requests of a +// connection for ContextPeerAddr +func ConnContext(ctx context.Context, c net.Conn) context.Context { + if peer := PeerAddr(c); peer != "" { + return context.WithValue(ctx, peerKey{}, peer) + } + return ctx +} + +func ContextPeerAddr(ctx context.Context) string { + peer, _ := ctx.Value(peerKey{}).(string) + return peer +} + +// signature reads as far as tells whether a PROXY header starts the stream: +// its version, or 0 when other bytes or none come first +func signature(br *bufio.Reader) (int, error) { + for n := 1; ; n++ { + b, err := br.Peek(n) + switch { + case len(b) == 0 || !bytes.HasPrefix(sigV1, b) && !bytes.HasPrefix(sigV2, b): + return 0, nil + case bytes.Equal(b, sigV1): + return 1, nil + case bytes.Equal(b, sigV2): + return 2, nil + case err != nil: + return 0, fmt.Errorf("signature cut short: %w", err) + } + } +} + +// readHeader reads the PROXY header starting br, if one does: found is false, +// with nothing consumed, when none does. The client is invalid for LOCAL, +// UNKNOWN and the families v2 leaves to the socket address. +func readHeader(br *bufio.Reader) (client netip.AddrPort, found bool, err error) { + v, err := signature(br) + switch v { + case 1: + client, err = readV1(br) + case 2: + client, err = readV2(br) + } + return client, v != 0 && err == nil, err +} + +// readV1 reads "PROXY TCP4|TCP6 src dst sport dport\r\n" or "PROXY UNKNOWN...\r\n" +func readV1(br *bufio.Reader) (netip.AddrPort, error) { + line, err := br.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + return netip.AddrPort{}, fmt.Errorf("v1 line over %d bytes", v1Max) + } else if err != nil { + return netip.AddrPort{}, fmt.Errorf("v1 line cut short: %w", err) + } + s, crlf := strings.CutSuffix(string(line), "\r\n") + f := strings.Split(s, " ") + switch { + case !crlf: + return netip.AddrPort{}, errors.New("v1 line not ended by CRLF") + case f[1] == "UNKNOWN": + return netip.AddrPort{}, nil + case len(f) != 6 || f[1] != "TCP4" && f[1] != "TCP6": + return netip.AddrPort{}, fmt.Errorf("malformed v1 line %q", s) + } + src, err1 := netip.ParseAddr(f[2]) + dst, err2 := netip.ParseAddr(f[3]) + port, err3 := strconv.ParseUint(f[4], 10, 16) + _, err4 := strconv.ParseUint(f[5], 10, 16) + four := f[1] == "TCP4" + if errors.Join(err1, err2, err3, err4) != nil || src.Zone()+dst.Zone() != "" || src.Is4() != four || dst.Is4() != four { + return netip.AddrPort{}, fmt.Errorf("malformed v1 line %q", s) + } + return netip.AddrPortFrom(src.Unmap(), uint16(port)), nil +} + +// readV2 reads the 16-byte prefix and the block after it, taking the address +// of TCP over IPv4 or IPv6 and skipping the rest, TLVs included +func readV2(br *bufio.Reader) (netip.AddrPort, error) { + h, err := br.Peek(16) + if err != nil { + return netip.AddrPort{}, fmt.Errorf("v2 prefix cut short: %w", err) + } + cmd, fam, n := h[12], h[13], int(binary.BigEndian.Uint16(h[14:])) + need, known := v2Families[fam] + if cmd&0xf != 1 { + need, known = 0, true // LOCAL ignores the block + } + switch { + case cmd>>4 != 2 || cmd&0xf > 1: + return netip.AddrPort{}, fmt.Errorf("v2 version and command %#x", cmd) + case !known: + return netip.AddrPort{}, fmt.Errorf("v2 family %#x", fam) + case n > v2Max: + return netip.AddrPort{}, fmt.Errorf("v2 block of %d bytes over %d", n, v2Max) + case n < need: + return netip.AddrPort{}, fmt.Errorf("v2 block of %d bytes short for family %#x", n, fam) + } + br.Discard(16) + var client netip.AddrPort + if need > 0 { + b, err := br.Peek(need) + if err != nil { + return netip.AddrPort{}, fmt.Errorf("v2 block cut short: %w", err) + } + addr, _ := netip.AddrFromSlice(b[:(need-4)/2]) // source, destination, two ports + client = netip.AddrPortFrom(addr.Unmap(), binary.BigEndian.Uint16(b[need-4:])) + } + if _, err := br.Discard(n); err != nil { + return netip.AddrPort{}, fmt.Errorf("v2 block cut short: %w", err) + } + return client, nil } // Describe renders the policy for log lines, stats and the status endpoint @@ -180,7 +507,11 @@ func (p *Policy) Describe() string { if len(p.allow) > 0 { allow = fmt.Sprint(len(p.allow)) } - return fmt.Sprintf("allow=%s deny=%d", allow, len(p.deny)) + s := fmt.Sprintf("allow=%s deny=%d", allow, len(p.deny)) + if len(p.proxyFrom) > 0 { + s += fmt.Sprintf(" proxy_protocol=%s proxy_from=%d", p.mode, len(p.proxyFrom)) + } + return s } func (p *Policy) Denied() uint64 { @@ -196,5 +527,5 @@ func (p *Policy) Stats() map[string]any { if p == nil { return nil } - return map[string]any{"acl": p.Describe(), "acl_denied": p.denied.Load()} + return map[string]any{"acl": p.Describe(), "acl_denied": p.denied.Load(), "acl_proxy_headers": p.headers.Load()} } diff --git a/internal/netacl/netacl_test.go b/internal/netacl/netacl_test.go index 85800a1..b1e6bf0 100644 --- a/internal/netacl/netacl_test.go +++ b/internal/netacl/netacl_test.go @@ -1,10 +1,16 @@ package netacl import ( + "bufio" + "bytes" + "crypto/tls" + "errors" "io" "net" "net/netip" + "slices" "strings" + "sync" "testing" "time" @@ -13,9 +19,9 @@ import ( "github.com/lixenwraith/log" ) -func mustNew(t *testing.T, host string, allow, deny []string) *Policy { +func mustNew(t *testing.T, host string, o config.ACLOptions) *Policy { t.Helper() - p, err := New(&config.ACLOptions{Allow: allow, Deny: deny}, host, log.NewLogger(), "test", "acl") + p, err := New(&o, host, log.NewLogger(), "test", "acl") if err != nil { t.Fatal(err) } @@ -26,8 +32,8 @@ func mustNew(t *testing.T, host string, allow, deny []string) *Policy { // one admits all not denied, an IPv4-mapped peer matches as IPv4, and a peer // of unknown address is refused. func TestDenyWinsThenAllowListAdmitsOnlyItsEntries(t *testing.T) { - listed := mustNew(t, "0.0.0.0", []string{"127.0.0.0/8"}, []string{"127.0.0.2"}) - denyOnly := mustNew(t, "0.0.0.0", nil, []string{"10.0.0.0/8"}) + listed := mustNew(t, "0.0.0.0", config.ACLOptions{Allow: []string{"127.0.0.0/8"}, Deny: []string{"127.0.0.2"}}) + denyOnly := mustNew(t, "0.0.0.0", config.ACLOptions{Deny: []string{"10.0.0.0/8"}}) for _, c := range []struct { p *Policy addr string @@ -50,68 +56,351 @@ func TestDenyWinsThenAllowListAdmitsOnlyItsEntries(t *testing.T) { // Entries are of the family core.Network gives the listener's host, as its // sockets carry no other: IPv4 on an IPv4 listener, IPv6 on an IPv6 one, -// either behind a hostname; never IPv4-mapped, never with a zone. +// either behind a hostname; never IPv4-mapped, never with a zone. Behind +// proxy_from, which keeps that family, deny takes both: the proxy names either. func TestEntriesFollowTheListenerFamily(t *testing.T) { for _, c := range []struct { - host, entry string - ok bool + host, from, entry string + ok bool }{ - {"0.0.0.0", "10.0.0.0/8", true}, - {"0.0.0.0", "2001:db8::/32", false}, - {"::", "2001:db8::1", true}, - {"::", "10.0.0.1", false}, - {"::", "::ffff:10.0.0.1", false}, - {"::", "fe80::1%eth0", false}, - {"logs.example", "10.0.0.0/8", true}, - {"logs.example", "2001:db8::/32", true}, + {"0.0.0.0", "", "10.0.0.0/8", true}, + {"0.0.0.0", "", "2001:db8::/32", false}, + {"::", "", "2001:db8::1", true}, + {"::", "", "10.0.0.1", false}, + {"::", "", "::ffff:10.0.0.1", false}, + {"::", "", "fe80::1%eth0", false}, + {"logs.example", "", "10.0.0.0/8", true}, + {"logs.example", "", "2001:db8::/32", true}, + {"0.0.0.0", "10.0.0.5", "2001:db8::/32", true}, + {"0.0.0.0", "2001:db8::5", "10.0.0.0/8", false}, } { - _, err := New(&config.ACLOptions{Deny: []string{c.entry}}, c.host, log.NewLogger(), "test", "acl") - if (err == nil) != c.ok { - t.Errorf("%s on %q: %v", c.entry, c.host, err) + o := &config.ACLOptions{Deny: []string{c.entry}} + if c.from != "" { + o.ProxyProtocol, o.ProxyFrom = "required", []string{c.from} + } + if _, err := New(o, c.host, log.NewLogger(), "test", "acl"); (err == nil) != c.ok { + t.Errorf("%s behind %q on %q: %v", c.entry, c.from, c.host, err) } } } -// A link-local peer carries its interface zone, which no prefix contains: -// it is matched without it, or a deny of fe80::/10 would admit it. -func TestZonedPeerMatchesItsPrefix(t *testing.T) { - p := mustNew(t, "::", nil, []string{"fe80::/10"}) - if p.admits(netip.MustParseAddr("fe80::1%eth0")) { - t.Fatal("a zoned link-local peer passed the deny of its prefix") +// A link-local peer carries its interface zone, which no prefix contains: the +// rules match it without, or a deny of fe80::/10 would admit it. It is never +// a proxy, as any link can claim its address. +func TestZonedPeerMatchesRulesButIsNoProxy(t *testing.T) { + p := mustNew(t, "::", config.ACLOptions{Deny: []string{"fe80::/10"}, ProxyProtocol: "required", ProxyFrom: []string{"fe80::/10"}}) + if zoned := netip.MustParseAddr("fe80::1%eth0"); p.admits(zoned) || p.proxies(zoned) { + t.Fatal("a zoned link-local peer passed the deny of its prefix, or as a proxy") } } // A refused peer is closed as it is accepted and counted: the plugin's // Accept returns the next admitted one. func TestRefusedPeerNeverReachesAccept(t *testing.T) { - inner := &fakeListener{conns: make(chan net.Conn, 2)} - refused, refusedPeer := net.Pipe() - admitted, _ := net.Pipe() - inner.conns <- remoteAt(refused, "192.0.2.1") - inner.conns <- remoteAt(admitted, "127.0.0.1") - ln := mustNew(t, "127.0.0.1", nil, []string{"192.0.2.0/24"}).Listener(inner) - conn, err := ln.Accept() - if err != nil { - t.Fatal(err) - } - if got := conn.RemoteAddr().String(); !strings.HasPrefix(got, "127.0.0.1:") { + p := mustNew(t, "127.0.0.1", config.ACLOptions{Deny: []string{"192.0.2.0/24"}}) + ln, dial := listening(t, p) + refused := dial("192.0.2.1") + dial("127.0.0.1") + if got := accept(t, ln).RemoteAddr().String(); !strings.HasPrefix(got, "127.0.0.1:") { t.Fatalf("Accept returned %s", got) } - refusedPeer.SetReadDeadline(time.Now().Add(time.Second)) - if _, err := refusedPeer.Read(make([]byte, 1)); err != io.EOF { - t.Fatalf("the refused peer reads %v, want EOF", err) - } - if n := ln.(*listener).p.Stats()["acl_denied"]; n != uint64(1) { + closed(t, refused) + if n := p.Stats()["acl_denied"]; n != uint64(1) { t.Fatalf("acl_denied = %v", n) } } +// proxy_protocol and proxy_from come together: either alone is a mistake, as +// is a mode that is none of off, optional and required +func TestProxyProtocolAndProxyFromComeTogether(t *testing.T) { + for _, c := range []struct { + mode string + from []string + ok bool + }{ + {"required", nil, false}, + {"", []string{"127.0.0.1"}, false}, + {"off", []string{"127.0.0.1"}, false}, + {"always", nil, false}, + {"optional", []string{"127.0.0.1"}, true}, + } { + _, err := New(&config.ACLOptions{ProxyProtocol: c.mode, ProxyFrom: c.from}, "127.0.0.1", log.NewLogger(), "test", "acl") + if (err == nil) != c.ok { + t.Errorf("proxy_protocol %q with proxy_from %v: %v", c.mode, c.from, err) + } + } +} + +// A v1 or v2 header names the client, IPv4-mapped as IPv4 (as throttling +// keys it), and leaves the bytes after it, TLVs skipped. +func TestHeaderNamesItsClient(t *testing.T) { + for _, c := range []struct{ header, client string }{ + {"PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\n", "198.51.100.1:40000"}, + {"PROXY TCP6 2001:db8::1 2001:db8::2 40000 443\r\n", "[2001:db8::1]:40000"}, + {"PROXY TCP6 ::ffff:198.51.100.1 ::ffff:192.0.2.1 40000 443\r\n", "198.51.100.1:40000"}, + {v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")), "198.51.100.1:40000"}, + {v2(0x21, 0x21, block("2001:db8::1", "2001:db8::2", 0x04, 0x00, 0x01, 0xff)), "[2001:db8::1]:40000"}, + {v2(0x21, 0x21, block("::ffff:198.51.100.1", "::ffff:192.0.2.1")), "198.51.100.1:40000"}, + } { + client, found, rest, err := parseHeader(c.header + "rest") + if err != nil || !found || client.String() != c.client || rest != "rest" { + t.Errorf("%q: client %v found %v rest %q: %v", c.header, client, found, rest, err) + } + } +} + +// LOCAL, UNKNOWN and the families v2 leaves unspecified keep the socket +// address, whatever address block they carry +func TestLocalAndUnknownKeepTheSocketAddress(t *testing.T) { + for _, header := range []string{ + "PROXY UNKNOWN\r\n", + "PROXY UNKNOWN " + strings.Repeat("x", 91) + "\r\n", // 107 bytes + v2(0x20, 0x11, block("198.51.100.1", "192.0.2.1")), + v2(0x20, 0x00, nil), + v2(0x21, 0x00, nil), + v2(0x21, 0x12, block("198.51.100.1", "192.0.2.1")), + v2(0x21, 0x31, make([]byte, 216)), + } { + client, found, rest, err := parseHeader(header + "rest") + if err != nil || !found || client.IsValid() || rest != "rest" { + t.Errorf("%q: client %v found %v rest %q: %v", header, client, found, rest, err) + } + } +} + +// A malformed, oversized or truncated header is an error, never a client +func TestMalformedHeaderIsRefused(t *testing.T) { + inet := v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")) + for _, header := range []string{ + "PROXY UNKNOWN 198.51.100.1 192.0.2.1 40000 443\n", + "PROXY UNKNOWN " + strings.Repeat("x", 92) + "\r\n", + "PROXY TCP4 2001:db8::1 192.0.2.1 40000 443\r\n", + "PROXY TCP4 198.51.100.1 2001:db8::2 40000 443\r\n", + "PROXY TCP6 198.51.100.1 192.0.2.1 40000 443\r\n", + "PROXY TCP4 198.51.100.1 192.0.2.1 70000 443\r\n", + "PROXY TCP4 198.51.100.1 192.0.2.1 40000\r\n", + "PROXY TCP6 fe80::1%eth0 fe80::2 40000 443\r\n", + "PROXY UDP4 198.51.100.1 192.0.2.1 40000 443\r\n", + "PROXY TCP4 198.51.100.1 192.0.2.1 40000 443", + "PROX", + v2(0x11, 0x11, block("198.51.100.1", "192.0.2.1")), + v2(0x22, 0x11, block("198.51.100.1", "192.0.2.1")), + v2(0x21, 0x41, block("198.51.100.1", "192.0.2.1")), + v2(0x21, 0x11, make([]byte, v2Max+1)), + v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")[:8]) + "rest", + inet[:14], + inet[:20], + } { + if client, found, _, err := parseHeader(header); err == nil || found { + t.Errorf("%q: client %v found %v, no error", header, client, found) + } + } +} + +// A peer outside proxy_from that sends a header is refused at its first +// read, as the header is spoofable; its other bytes pass untouched +func TestHeaderFromUnlistedPeerIsRefused(t *testing.T) { + p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "optional", ProxyFrom: []string{"10.0.0.1"}}) + ln, dial := listening(t, p) + for _, c := range []struct { + send string + refused bool + }{ + {"PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\n", true}, + {v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")), true}, + {"POST / HTTP/1.1\r\n", false}, + } { + go dial("192.0.2.9").Write([]byte(c.send)) + b := make([]byte, len(c.send)) + _, err := io.ReadFull(accept(t, ln), b) + if (err != nil) != c.refused || !c.refused && string(b) != c.send { + t.Errorf("%q from an unlisted peer: read %q, %v", c.send, b, err) + } + } + if n := p.Denied(); n != 2 { + t.Fatalf("acl_denied = %d", n) + } +} + +// Under required a peer in proxy_from must send a header; under optional +// one that sends none keeps its socket address and its bytes +func TestRequiredRefusesAListedPeerWithoutHeader(t *testing.T) { + for mode, refused := range map[string]bool{"required": true, "optional": false} { + p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: mode, ProxyFrom: []string{"127.0.0.1"}}) + ln, dial := listening(t, p) + peer := dial("127.0.0.1") + peer.Write([]byte("GET /")) + if refused { + closed(t, peer) + continue + } + conn := accept(t, ln) + b := make([]byte, 5) + if _, err := io.ReadFull(conn, b); err != nil || string(b) != "GET /" || conn.RemoteAddr().String() != "127.0.0.1:40000" { + t.Errorf("optional: %s read %q: %v", conn.RemoteAddr(), b, err) + } + } +} + +// The client a header names is what the rules, RemoteAddr and the stats see; +// the proxy stays as PeerAddr, under TLS too +func TestRulesSeeTheClientAHeaderNames(t *testing.T) { + p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "required", ProxyFrom: []string{"127.0.0.1"}, Deny: []string{"198.51.100.2"}}) + ln, dial := listening(t, p) + denied := dial("127.0.0.1") + denied.Write([]byte("PROXY TCP4 198.51.100.2 192.0.2.1 40000 443\r\n")) + go dial("127.0.0.1").Write([]byte("PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\nhello")) + conn := accept(t, ln) + b := make([]byte, 5) + if _, err := io.ReadFull(conn, b); err != nil || string(b) != "hello" { + t.Fatalf("read %q: %v", b, err) + } + if got, peer := conn.RemoteAddr().String(), PeerAddr(tls.Server(conn, nil)); got != "198.51.100.1:40000" || peer != "127.0.0.1:40000" { + t.Fatalf("RemoteAddr %s, PeerAddr %q", got, peer) + } + closed(t, denied) + if s := p.Stats(); s["acl_proxy_headers"] != uint64(2) || s["acl_denied"] != uint64(1) { + t.Fatalf("stats %v", s) + } +} + +// A header still on its way holds only its own connection, and is refused +// at the deadline +func TestSlowHeaderHoldsOnlyItsConnection(t *testing.T) { + p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "required", ProxyFrom: []string{"127.0.0.1"}}) + p.headerWait = time.Second + ln, dial := listening(t, p) + slow := dial("127.0.0.1") + slow.Write([]byte("PROXY TCP4 198")) + dial("192.0.2.9") + if got := accept(t, ln).RemoteAddr().String(); !strings.HasPrefix(got, "192.0.2.9:") || p.Denied() != 0 { + t.Fatalf("Accept returned %s after %d refusals", got, p.Denied()) + } + closed(t, slow) + if n := p.Denied(); n != 1 { + t.Fatalf("acl_denied = %d", n) + } +} + +// Close also closes the connections whose header has not arrived +func TestCloseReleasesHeadersInFlight(t *testing.T) { + p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "required", ProxyFrom: []string{"127.0.0.1"}}) + ln, dial := listening(t, p) + slow := dial("127.0.0.1") + slow.Write([]byte("PROXY ")) + ln.Close() + closed(t, slow) + if _, err := ln.Accept(); !errors.Is(err, net.ErrClosed) { + t.Fatalf("Accept after Close: %v", err) + } +} + +// Whatever the bytes, the parser neither panics nor reads past a header's +// bound, consumes nothing when no header starts the stream, and names a +// client only unmapped and without a zone. +func FuzzReadHeader(f *testing.F) { + for _, seed := range []string{ + "PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\nrest", + "PROXY TCP6 2001:db8::1 2001:db8::2 40000 443\r\n", + "PROXY UNKNOWN\r\n", + v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")), + v2(0x21, 0x21, block("2001:db8::1", "2001:db8::2", 0x04, 0x00, 0x01, 0xff)), + v2(0x20, 0x00, nil), + "GET / HTTP/1.1\r\n", + } { + f.Add([]byte(seed)) + } + f.Fuzz(func(t *testing.T, data []byte) { + r := bytes.NewReader(data) + br := bufio.NewReaderSize(r, v1Max) + client, found, err := readHeader(br) + consumed := len(data) - br.Buffered() - r.Len() + a := client.Addr() + switch { + case consumed > 16+v2Max: + t.Fatalf("consumed %d bytes", consumed) + case err == nil && !found && consumed != 0: + t.Fatalf("consumed %d bytes with no header", consumed) + case client.IsValid() && (!found || a.Is4In6() || a.Zone() != ""): + t.Fatalf("client %v, found %v", client, found) + } + }) +} + +func parseHeader(s string) (client netip.AddrPort, found bool, rest string, err error) { + br := bufio.NewReaderSize(strings.NewReader(s), v1Max) + client, found, err = readHeader(br) + b, _ := io.ReadAll(br) + return client, found, string(b), err +} + +// v2 is a PROXY v2 header: version and command, family, then the block +func v2(cmd, fam byte, block []byte) string { + h := append(slices.Clone(sigV2), cmd, fam, byte(len(block)>>8), byte(len(block))) + return string(append(h, block...)) +} + +// block is a v2 address block from src:40000 to dst:443, then tlv +func block(src, dst string, tlv ...byte) []byte { + b := append(netip.MustParseAddr(src).AsSlice(), netip.MustParseAddr(dst).AsSlice()...) + return append(append(b, 0x9c, 0x40, 0x01, 0xbb), tlv...) +} + +// listening runs p's listener over a fake one; dial connects a peer from +// addr:40000 and returns its end +func listening(t *testing.T, p *Policy) (net.Listener, func(addr string) net.Conn) { + inner := &fakeListener{conns: make(chan net.Conn, 4)} + ln := p.Listener(inner) + t.Cleanup(func() { ln.Close() }) + return ln, func(addr string) net.Conn { + server, client := net.Pipe() + t.Cleanup(func() { client.Close() }) + inner.conns <- remoteConn{server, net.TCPAddrFromAddrPort(netip.AddrPortFrom(netip.MustParseAddr(addr), 40000))} + return client + } +} + +func accept(t *testing.T, ln net.Listener) net.Conn { + t.Helper() + got := make(chan net.Conn, 1) + go func() { c, _ := ln.Accept(); got <- c }() + select { + case c := <-got: + if c == nil { + t.Fatal("Accept failed") + } + t.Cleanup(func() { c.Close() }) + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + return c + case <-time.After(5 * time.Second): + t.Fatal("Accept returned nothing") + } + return nil +} + +// closed waits for the listener to close peer's connection +func closed(t *testing.T, peer net.Conn) { + t.Helper() + peer.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := peer.Read(make([]byte, 1)); err != io.EOF { + t.Fatalf("the peer reads %v, want EOF", err) + } +} + type fakeListener struct { net.Listener conns chan net.Conn + once sync.Once } -func (f *fakeListener) Accept() (net.Conn, error) { return <-f.conns, nil } +func (f *fakeListener) Accept() (net.Conn, error) { + if c, ok := <-f.conns; ok { + return c, nil + } + return nil, net.ErrClosed +} + +func (f *fakeListener) Close() error { f.once.Do(func() { close(f.conns) }); return nil } type remoteConn struct { net.Conn @@ -119,7 +408,3 @@ type remoteConn struct { } func (c remoteConn) RemoteAddr() net.Addr { return c.remote } - -func remoteAt(c net.Conn, addr string) net.Conn { - return remoteConn{c, net.TCPAddrFromAddrPort(netip.AddrPortFrom(netip.MustParseAddr(addr), 40000))} -} diff --git a/internal/sink/http/http.go b/internal/sink/http/http.go index f41979c..259b7ca 100644 --- a/internal/sink/http/http.go +++ b/internal/sink/http/http.go @@ -280,6 +280,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error { h.server = &http.Server{ Handler: handler, ReadHeaderTimeout: HTTPReadHeaderTimeout, + ConnContext: netacl.ConnContext, // WriteTimeout unset by design: SSE responses are long-lived. // net/http bounds the TLS handshake by min(ReadHeaderTimeout, // ReadTimeout, WriteTimeout), so ReadHeaderTimeout covers it here. @@ -434,6 +435,9 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) { meta := map[string]any{ "type": "http_client", } + if peer := netacl.ContextPeerAddr(r.Context()); peer != "" { + meta["peer_addr"] = peer + } if r.TLS != nil { meta["tls"] = true if cn := tlsx.PeerCN(*r.TLS); cn != "" { diff --git a/internal/sink/http/http_test.go b/internal/sink/http/http_test.go index a5abbd7..454af37 100644 --- a/internal/sink/http/http_test.go +++ b/internal/sink/http/http_test.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "maps" + "net" "net/http" "net/http/httptest" "slices" @@ -347,6 +348,26 @@ func TestProxyModeRefusesDirectPeers(t *testing.T) { } } +// A PROXY header names the client a stream's session records; the proxy +// stays as its peer_addr +func TestProxiedStreamSessionKeepsTheProxy(t *testing.T) { + h, manager := newTestHTTPSink(t, map[string]any{"acl": map[string]any{"proxy_protocol": "required", "proxy_from": []any{"127.0.0.1"}}}) + _, baseURL := serveTestHTTPSink(t, h) + conn, err := net.Dial("tcp4", strings.TrimPrefix(baseURL, "http://")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { conn.Close() }) + fmt.Fprint(conn, "PROXY TCP4 198.51.100.7 127.0.0.1 40000 8081\r\nGET /stream HTTP/1.1\r\nHost: logwisp\r\n\r\n") + if _, err := http.ReadResponse(bufio.NewReader(conn), nil); err != nil { // its body is the stream + t.Fatal(err) + } + sessions := manager.GetActiveSessions() + if len(sessions) != 1 || sessions[0].RemoteAddr != "198.51.100.7:40000" || sessions[0].Metadata["peer_addr"] != conn.LocalAddr().String() { + t.Fatalf("sessions %+v, proxy %s", sessions, conn.LocalAddr()) + } +} + // The constructor refuses what it cannot serve: the login pages outside // scram's proxy mode, a viewer without the login page, a path under /auth, and // a path ServeMux or a URL would not take literally, which panicked at Start. diff --git a/internal/sink/tcp/tcp.go b/internal/sink/tcp/tcp.go index 06a2308..4024bed 100644 --- a/internal/sink/tcp/tcp.go +++ b/internal/sink/tcp/tcp.go @@ -371,6 +371,9 @@ func (t *TCPSink) handleConn(conn net.Conn) { "type": "tcp_client", "remote_addr": remote, } + if peer := netacl.PeerAddr(conn); peer != "" { + meta["peer_addr"] = peer + } var tlsState *tls.ConnectionState if tc, ok := conn.(*tls.Conn); ok { hctx, cancel := context.WithTimeout(context.Background(), tlsx.HandshakeTimeout) diff --git a/internal/source/httpchain/httpchain.go b/internal/source/httpchain/httpchain.go index 29e87fb..ca1866a 100644 --- a/internal/source/httpchain/httpchain.go +++ b/internal/source/httpchain/httpchain.go @@ -200,6 +200,7 @@ func (s *HTTPChainSource) Start() error { Handler: mux, ReadTimeout: time.Duration(s.config.ReadTimeoutMS) * time.Millisecond, ReadHeaderTimeout: HTTPChainReadHeaderTimeout, + ConnContext: netacl.ConnContext, // TLS handshake bounded by min(ReadTimeout, ReadHeaderTimeout) ErrorLog: tlsx.HTTPErrorLog(s.logger, "http_chain_source"), } @@ -371,7 +372,7 @@ func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) { for _, entry := range entries { s.publish(entry) } - s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode, r.TLS, ident)) + s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode, netacl.ContextPeerAddr(r.Context()), r.TLS, ident)) w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries))) w.WriteHeader(http.StatusNoContent) @@ -399,7 +400,7 @@ func (s *HTTPChainSource) handleAuth(w http.ResponseWriter, r *http.Request) { // sessionFor returns the cached session for a remote+node+identity, // recreating after idle expiry. Identity is part of the key so two peers // sharing a remote address never share a session. -func (s *HTTPChainSource) sessionFor(remoteHost, node string, cs *tls.ConnectionState, ident authz.Identity) string { +func (s *HTTPChainSource) sessionFor(remoteHost, node, peer string, cs *tls.ConnectionState, ident authz.Identity) string { key := remoteHost + "|" + node + "|" + ident.Name s.sessionsMu.Lock() defer s.sessionsMu.Unlock() @@ -413,6 +414,9 @@ func (s *HTTPChainSource) sessionFor(remoteHost, node string, cs *tls.Connection "type": "http_chain", "node": node, } + if peer != "" { + meta["peer_addr"] = peer + } if cs != nil { meta["tls"] = true if cn := tlsx.PeerCN(*cs); cn != "" { diff --git a/internal/source/tcpchain/tcpchain.go b/internal/source/tcpchain/tcpchain.go index 4e6cf1a..71c787a 100644 --- a/internal/source/tcpchain/tcpchain.go +++ b/internal/source/tcpchain/tcpchain.go @@ -366,6 +366,9 @@ func (s *TCPChainSource) handleConn(conn net.Conn) { "type": "tcp_chain", "node": connNode, } + if peer := netacl.PeerAddr(conn); peer != "" { + meta["peer_addr"] = peer + } if tlsState != nil { meta["tls"] = true if cn := tlsx.PeerCN(*tlsState); cn != "" { diff --git a/test/acl-test.sh b/test/acl-test.sh index 97410a7..631005d 100755 --- a/test/acl-test.sh +++ b/test/acl-test.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # logwisp acl test: each listener kind admits or refuses peers by address -# before TLS, counts and warns of refusals, rejects rules it can never apply, -# and still hands a finite input whole to the clients it admits. One-shot. -# Ports: 15881-15889 on 127.0.0.1. Requires: bash 5+, curl, nc; Linux. +# before TLS, or by the client a PROXY header names (stub proxy, nginx if +# installed), counts and warns of refusals, and throttles logins per client. +# One-shot. Ports: 15881-15889 on 127.0.0.1, the proxies' on 127.0.0.11-14. +# Requires: bash 5+, curl, nc, go; Linux. set -u . "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh" @@ -12,11 +13,72 @@ AUTH=$RUN e2e_init "$@" section "Setup" -need curl nc -ports_free 15881 15882 15884 15885 15889 +need curl nc go +ports_free 15881 15882 15883 15884 15885 15886 15887 15888 15889 rm -rf "$RUN" -mkdir -p "$RUN/out" "$LOG" +mkdir -p "$RUN/out" "$RUN/proxy" "$LOG" cd "$RUN" || abort "no run directory" +# Each ROUTE is LISTEN,BACKEND,VERSION,CLIENT: a connection to LISTEN reaches +# BACKEND behind a PROXY header of VERSION naming CLIENT, with its own port +cat >"$RUN/proxy/main.go" <<'EOF' +package main + +import ( + "encoding/binary" + "fmt" + "io" + "log" + "net" + "net/netip" + "os" + "strings" +) + +func main() { + for _, route := range os.Args[1:] { + f := strings.Split(route, ",") + ln, err := net.Listen("tcp4", f[0]) + if err != nil { + log.Fatal(err) + } + go func() { + for { + c, err := ln.Accept() + if err != nil { + log.Fatal(err) + } + go relay(c, f[1], f[2], netip.MustParseAddr(f[3])) + } + }() + } + fmt.Println("ready") + select {} +} + +func relay(c net.Conn, backend, version string, client netip.Addr) { + defer c.Close() + b, err := net.Dial("tcp4", backend) + if err != nil { + return + } + defer b.Close() + src := netip.AddrPortFrom(client, c.RemoteAddr().(*net.TCPAddr).AddrPort().Port()) + dst := b.RemoteAddr().(*net.TCPAddr).AddrPort() + if version == "1" { + fmt.Fprintf(b, "PROXY TCP4 %s %s %d %d\r\n", src.Addr(), dst.Addr(), src.Port(), dst.Port()) + } else { + h := append([]byte("\r\n\r\n\x00\r\nQUIT\n\x21\x11\x00\x0c"), src.Addr().AsSlice()...) + h = binary.BigEndian.AppendUint16(append(h, dst.Addr().AsSlice()...), src.Port()) + b.Write(binary.BigEndian.AppendUint16(h, dst.Port())) + } + go func() { io.Copy(b, c); b.(*net.TCPConn).CloseWrite() }() + io.Copy(c, b) +} +EOF +if ! (cd "$RUN/proxy" && go build -o proxy main.go) >"$LOG/proxy-build.out" 2>&1; then + tail -n 5 "$LOG/proxy-build.out" | sed 's/^/ /' + abort "the stub proxy did not build" +fi for v in $(compgen -e LOGWISP_); do unset "$v"; done export HOME=$RUN lw() { "$BIN" "$@"; } @@ -25,7 +87,8 @@ get() { curl -s --noproxy '*' --max-time 3 "$@"; } # curl ARGS...: never through get --interface 127.0.0.2 http://127.0.0.1:15889/ (($? == 45)) && skip_all "this host cannot send from 127.0.0.2" seq 1 300 | sed 's/^/edge line /' >"$RUN/edge.log" -add_users "$RUN/users.toml" edge-01 +add_users "$RUN/users.toml" edge-01 viewer-01 +echo wrong >"$RUN/wrong.pass" section "Rules" # rejects SINK PATTERN: lw --check fails on SINK with PATTERN in its error @@ -37,6 +100,10 @@ check "an acl on a dialer is refused" "$(rejects 'tcp_chain,host=127.0.0.1,port= check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp,host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')" check "a deny list alone on a wildcard listener warns" "$(warns 'tcp,port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')" check "an allow entry of a whole family warns" "$(warns 'tcp,port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')" +check "a proxy_from entry reaching public addresses warns" \ + "$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=203.0.113.0/24' 'proxy_from reaches public addresses')" +check "optional proxy_protocol warns that a headerless client passes as the proxy" \ + "$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=127.0.0.1' 'passes as the proxy')" section "HTTP sink" spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2" @@ -71,12 +138,12 @@ section "Chain sources" # pin DAEMON: the pin_sha256 a self-signed listener logged pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; } received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; } -edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; } +edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; } spawn agg-allow "$BIN" --preset "aggregator,listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow" spawn agg-deny "$BIN" --preset "aggregator,listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny" wait_port 15884 && wait_port 15885 || abort "an aggregator did not listen" agg-allow -edge edge-allow 15884 tcp "$(pin agg-allow)" -edge edge-deny 15885 http "$(pin agg-deny)" +edge edge-allow 127.0.0.1:15884 tcp "$(pin agg-allow)" +edge edge-deny 127.0.0.1:15885 http "$(pin agg-deny)" timeout 3 nc -s 127.0.0.2 127.0.0.1 15884 = 300 ))' check "a tcp_chain source admits an edge its allow list names ($(received allow)), and refuses 127.0.0.2" \ @@ -85,4 +152,102 @@ wait_until 10 logged agg-deny 'refused by acl' check "an http_chain source refuses a denied edge, which delivers nothing ($(received deny))" \ "$(logged agg-deny 'refused by acl' && (($(received deny) == 0)) && echo 1 || echo 0)" +section "Behind a PROXY header" +# Every listener here takes headers from 127.0.0.1 only, where the proxies +# connect from; each is up before a proxy shares its port on another address +behind=acl.proxy_protocol=required,acl.proxy_from=127.0.0.1 +scram="tls.enabled=true,tls.self_signed=true,auth.type=scram,auth.credentials_file=$RUN/users.toml" +spawn sink-behind "$BIN" --logging.level=info --source null --sink "http,host=127.0.0.1,port=15886,$scram,$behind,acl.deny=198.51.100.2" +for t in tcp http; do + spawn "agg-$t-behind" "$BIN" --logging.level=info \ + --source "${t}_chain,host=127.0.0.1,port=$([[ $t == tcp ]] && echo 15888 || echo 15883),$scram,$behind" \ + --sink "file,directory=$RUN/out/$t-behind,name=aggregate" +done +# the tcp sink sends its input once the clients connected +{ wait_until 30 test -e "$RUN/feed"; seq 1 100 | sed 's/^/behind /'; } | + lw --sink "tcp,host=127.0.0.1,port=15887,$behind,acl.deny=198.51.100.2,acl.deny=127.0.0.6" 2>"$LOG/tcp-behind.out" & +fpid=$! +for p in 15883 15886 15887 15888; do wait_port $p || abort "a listener behind the proxy did not listen on $p" sink-behind; done +spawn proxy "$RUN/proxy/proxy" 127.0.0.11:15886,127.0.0.1:15886,1,198.51.100.1 \ + 127.0.0.12:15886,127.0.0.1:15886,2,198.51.100.2 127.0.0.13:15886,127.0.0.1:15886,2,198.51.100.3 \ + 127.0.0.11:15887,127.0.0.1:15887,2,198.51.100.1 127.0.0.12:15887,127.0.0.1:15887,1,198.51.100.2 \ + 127.0.0.11:15888,127.0.0.1:15888,1,198.51.100.1 127.0.0.13:15883,127.0.0.1:15883,2,198.51.100.3 +wait_until 10 logged proxy ready || abort "the stub proxy did not start" proxy +NGINX=0 +if command -v nginx >/dev/null; then + mkdir -p "$RUN/nginx/logs" + # a dynamic stream module loads by its absolute path; a built-in has none + so=$(nginx -V 2>&1 | grep -o -- '--modules-path=[^ ]*' | cut -d= -f2)/ngx_stream_module.so + modules=$([[ -f $so ]] && echo "load_module $so;") + cat >"$RUN/nginx/nginx.conf" <"$RUN/$1.out"; } +listen_via admitted 127.0.0.11 & +pids=($!) +listen_via denied 127.0.0.12 & +pids+=($!) +if ((NGINX)); then + listen_via nginx -s 127.0.0.5 127.0.0.14 & + pids+=($!) + listen_via nginx-denied -s 127.0.0.6 127.0.0.14 & + pids+=($!) +fi +sleep 1 +touch "$RUN/feed" +wait "$fpid" "${pids[@]}" +lines() { grep -c '^behind ' "$RUN/$1.out"; } +check "a tcp sink streams to the client a v2 header names ($(lines admitted) lines)" "$(($(lines admitted) == 100))" +check "and refuses one a v1 header names, logging both addresses ($(lines denied) lines)" \ + "$( (($(lines denied) == 0)) && grep -q 'refused by acl.* remote_addr 198.51.100.2:.*peer_addr 127.0.0.1:' "$LOG/tcp-behind.out" && echo 1 || echo 0)" +if ((NGINX)); then + check "nginx stream with proxy_protocol on names its clients ($(lines nginx) lines, $(lines nginx-denied) to a denied one)" \ + "$( (($(lines nginx) == 100 && $(lines nginx-denied) == 0)) && echo 1 || echo 0)" +else + skip "nginx stream with proxy_protocol on (nginx with its stream module is not available)" +fi + +# http sink: each client its own login budget behind the one proxy address +PIN=$(pin sink-behind) +token() { lw auth token --url "https://$1:15886" -u viewer-01 --password-file "$2" --pin-sha256 "$PIN" 2>&1; } +token 127.0.0.12 "$RUN/viewer-01.pass" >/dev/null +check "an http sink refuses the client a v2 header names, logging both addresses" \ + "$(logged sink-behind 'refused by acl.* remote_addr 198.51.100.2:.*peer_addr 127.0.0.1:' && echo 1 || echo 0)" +for ((i = 1; i <= 30; i++)); do + token 127.0.0.11 "$RUN/wrong.pass" >/dev/null + logged sink-behind ' throttled' && break +done +check "wrong passwords throttle a client by the address its v1 header names ($i attempts)" \ + "$(logged sink-behind 'Login rejected.* remote_addr 198.51.100.1:.* 198.51.100.1 throttled' && echo 1 || echo 0)" +check "while another behind the same proxy still logs in" \ + "$(token 127.0.0.13 "$RUN/viewer-01.pass" >/dev/null && logged sink-behind 'Login accepted.* remote_addr 198.51.100.3:' && echo 1 || echo 0)" + +# chain sources: an edge through the proxy is the client its header names +edge edge-tcp-behind 127.0.0.11:15888 tcp "$(pin agg-tcp-behind)" +edge edge-http-behind 127.0.0.13:15883 http "$(pin agg-http-behind)" +printf 'PROXY TCP4 198.51.100.9 127.0.0.1 40000 15888\r\n' | timeout 3 nc -s 127.0.0.2 127.0.0.1 15888 +wait_until 10 eval '(( $(received tcp-behind) >= 300 && $(received http-behind) >= 300 ))' +check "a tcp_chain source takes an edge as the client a v1 header names ($(received tcp-behind))" \ + "$( (($(received tcp-behind) == 300)) && logged agg-tcp-behind 'Chain connection established.* remote_addr 198.51.100.1:' && echo 1 || echo 0)" +check "an http_chain source takes one as the client a v2 header names ($(received http-behind))" \ + "$( (($(received http-behind) == 300)) && logged agg-http-behind 'Login accepted.* remote_addr 198.51.100.3:' && echo 1 || echo 0)" +check "a header from a peer outside proxy_from is refused" \ + "$(logged agg-tcp-behind 'refused by acl.* remote_addr 127.0.0.2:.* reason "PROXY header from a peer outside proxy_from"' && echo 1 || echo 0)" + summary