v0.26.0 proxy headers for acl
Test / test (push) Canceled after 0s
Test / e2e (push) Canceled after 0s

This commit is contained in:
lixen committed 2026-10-05 01:24:02 -04:00
1 parent 4af072f4e5
commit ee01cb00fe
21 files changed
+1107 -290

No files matched your search

+19 -3
View File
@@ -10,8 +10,9 @@
### available in this version. TLS is available on every network source and
### sink (the [...tls] blocks below), the [...auth] blocks authenticate peers by
### certificate (mtls) or password (scram; credentials via `lw auth`), and the
### listeners' [...acl] blocks admit peers by address. Unknown keys anywhere in
### this file or in any plugin config are rejected at startup.
### listeners' [...acl] blocks admit peers by address, or by the client an L4
### proxy's PROXY header names. Unknown keys anywhere in this file or in any
### plugin config are rejected at startup.
###
### Environment overrides use the LOGWISP_ prefix
### (LOGWISP_QUIET, LOGWISP_LOGGING_LEVEL, ...). Bare names are ignored.
@@ -163,8 +164,15 @@ name = "default"
###
### Applied as a connection is accepted, before TLS: a refused peer is closed
### at once and counted in acl_denied, with a WARN at most once a minute.
### allow = [] addresses or CIDRs of the listener's family; empty = all
### allow = [] addresses or CIDRs of the listener's family (either behind
### proxy_from); empty = all
### deny = [] refused, listed in allow or not
### proxy_protocol = "off" off | optional | required: PROXY v1/v2 headers,
### read before TLS; the rules then see the client.
### optional lets a headerless stream pass as the
### proxy: use it only while rolling headers out
### proxy_from = [] the proxies that may send one; a header from any
### other peer is refused
###============================================================================
###============================================================================
@@ -236,6 +244,8 @@ from = "end" # "end" or "start" of a newly discovered
# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks)
# [[pipelines.plugin_sources]]
@@ -266,6 +276,8 @@ from = "end" # "end" or "start" of a newly discovered
# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
###============================================================================
### Sinks (1+ required, fan-out)
@@ -334,6 +346,8 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
## TCP sink (streaming server)
# [[pipelines.plugin_sinks]]
@@ -363,6 +377,8 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS
# allow = [] # empty = every address deny does not list
# deny = []
# proxy_protocol = "off" # off | optional | required
# proxy_from = [] # proxies that may send a PROXY header
## TCP chain sink (stdlib client; forwards to downstream tcp_chain source)
## Do NOT point a chain sink at a chain source in the SAME pipeline: entries
+1 -1
View File
@@ -120,7 +120,7 @@ Not covered by this script; edit the generated configuration instead
- filters, rate limits, heartbeat, formatter flags and timestamp layouts
- more than one pipeline; both chain transports, or a further hop, on one node
- proxy mode: browser logins through a TLS-terminating proxy (trusted_proxies)
- acl address rules on the listeners
- acl on the listeners: address rules, PROXY headers
- mtls identities other than the certificate CN; node_binding other than force
- creating certificates: openssl or your PKI (doc/security.md#enabling-mtls)
- several native instances on one host: use one per host or jail
+2 -2
View File
@@ -30,8 +30,8 @@ streams, or downstream LogWisp nodes.
- [CLI](cli.md): flags, signals, exit codes, `lw auth`; also the `lw(1)`
manual, [`lw.1`](lw.1)
- [Operations](operations.md): running, monitoring, tuning, troubleshooting
- [To Do](todo.md): planned work in priority order: network access control
and the PROXY protocol, packaging, smaller follow-ups
- [To Do](todo.md): planned work in priority order: per-client limits,
packaging, smaller follow-ups
A fully annotated configuration covering every option lives at
[`config/logwisp.toml`](../config/logwisp.toml).
+3 -2
View File
@@ -167,8 +167,9 @@ TLS is built in exactly one place, `internal/tlsx`, which exposes
Each pipeline owns a `session.Manager`. Plugins receive a `session.Proxy`
scoped to their instance id, so one plugin cannot see or remove another's
sessions. A session records the remote address, creation and last-activity
timestamps, and metadata — including `tls` and `tls_peer_cn` for TLS peers, and
`auth_method` / `auth_identity` for authorized ones.
timestamps, and metadata — including `tls` and `tls_peer_cn` for TLS peers,
`auth_method` / `auth_identity` for authorized ones, and `peer_addr`, the proxy,
for a client a PROXY header names.
Idle sessions are reaped every 5 minutes against a 30-minute idle limit. The
HTTP sink's broker treats a vanished session as an eviction signal and closes
+1 -1
View File
@@ -184,7 +184,7 @@ every key.
- filters, rate limits, heartbeat, formatter flags and timestamp layouts
- more than one pipeline; both chain transports, or a further hop, on one node
- proxy mode: browser logins through a TLS-terminating proxy
- `acl` address rules on the listeners
- `acl` on the listeners: address rules, PROXY headers
- mtls identities other than the certificate CN; `node_binding` other than
`force`
- creating certificates
+4 -1
View File
@@ -312,4 +312,7 @@ or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target.
- `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed
and checked; ports 15871-15879
- `acl-test.sh`: each listener admitting or refusing peers by address before
TLS, with clients from 127.0.0.1 and 127.0.0.2; ports 15881-15889
TLS, with clients from 127.0.0.1 and 127.0.0.2, then by the client a PROXY
header names behind a stub proxy on 127.0.0.11-13 (and nginx `stream` on
127.0.0.14 when installed), per-client login throttling included; ports
15881-15889; skips (exit 77) where 127.0.0.2 cannot be bound
+29 -22
View File
@@ -208,13 +208,22 @@ headers, and entry encoding.
- Check the port is not already bound by another pipeline in the same process.
**Closed as soon as it connects, before TLS**
- The listener's `acl` refused the address: `acl_denied` rises, and its log has
`Connection refused by acl` at WARN, once a minute at most. The dialer sees
`EOF` or `connection reset by peer` (curl: exit 52 or 56).
- Behind a proxy (an L4 passthrough, or the `http` sink's
`auth.trusted_proxies`) every peer has the proxy's address: allow the proxy.
- The listener's `acl` refused it: `acl_denied` rises, and its log has
`Connection refused by acl` at WARN with a `reason`, once a minute at most.
The dialer sees `EOF` or `connection reset by peer` (curl: exit 52 or 56).
- `address rules` — `allow` or `deny` excludes the client: the one a PROXY
header names (`remote_addr`, the proxy as `peer_addr`), else the socket
peer. Behind the `http` sink's `auth.trusted_proxies` that is the proxy:
allow it.
- `no PROXY header` — a peer in `proxy_from` sent none under
`proxy_protocol = "required"`: the proxy lacks `proxy_protocol on` (nginx)
or `send-proxy` (HAProxy), or the proxy's host dialed directly.
- `PROXY header: …` — malformed, too long, or incomplete after 10 s.
- `PROXY header from a peer outside proxy_from` — at the first read, after
the connection opened: list the proxy in `proxy_from`.
- `acl: allow entry "...": not of the listener's family` at startup — an IPv4
entry on an IPv6 `host`, or the reverse: each family's listener takes its own.
Behind `proxy_from`, `allow` and `deny` take both; `proxy_from` its own.
**TLS handshake failure**
- `client didn't provide a certificate` — the listener has `client_auth = true`
@@ -271,9 +280,9 @@ headers, and entry encoding.
`X-Forwarded-For`.
- `too many attempts` — the address failed or abandoned logins faster than one
per second beyond a burst of 10, or has 4 unfinished; it clears within seconds
once the failing peer stops. Peers behind one NAT or one passthrough proxy
share the budget. `busy` — 4,096 logins in flight, or the listener is
stopping.
once the failing peer stops. Peers behind one NAT, or one passthrough proxy
that sends no PROXY header, share the budget. `busy` — 4,096 logins in
flight, or the listener is stopping.
- `authentication not enabled` — the dialer has `scram`; the listener's
`auth.type` is `none` or `mtls`.
- `no challenge within 10s (older logwisp, or not a scram listener)` — the
@@ -301,20 +310,18 @@ headers, and entry encoding.
proxy mode, the proxy gate.
**Behind an L4 proxy**
- Every connection through the proxy fails: it sends a PROXY header (nginx
`stream` with `proxy_protocol on`, HAProxy `send-proxy`) to a LogWisp TLS
listener, which cannot read one yet. The header lands in front of the TLS
handshake, which fails; the listener logs the proxy's address and
`first record does not look like a TLS handshake` (the `tcp` sink at DEBUG).
Route LogWisp without it: see
[Behind nginx or another proxy](security.md#behind-nginx-or-another-proxy).
- All peers share one throttling budget behind a passthrough proxy: without
PROXY every `scram` peer arrives from the proxy's address. One client failing
logins exhausts everyone's, and a fifth login while 4 are unfinished (edges
reconnecting together after a restart) is refused; both answer
`too many attempts` ([Throttling](security.md#throttling)).
Expose the LogWisp ports directly, or keep the proxy hop on a trusted
network so only trusted clients share the budget.
- Every connection through the proxy fails, the listener logging the proxy's
address and `first record does not look like a TLS handshake` (the `tcp`
sink at DEBUG): the proxy sends a PROXY header (nginx `stream` with
`proxy_protocol on`, HAProxy `send-proxy`) to a listener without
`acl.proxy_protocol`. Set it, with the proxy in `proxy_from`: see
[PROXY protocol](security.md#proxy-protocol).
- All peers share one throttling budget and logs name the proxy: it sends no
PROXY header, so every `scram` peer arrives from its address. One client
failing logins exhausts everyone's, and a fifth login while 4 are unfinished
(edges reconnecting together after a restart) is refused; both answer
`too many attempts` ([Throttling](security.md#throttling)). Have the proxy
send the header.
**Entries not arriving over a chain link**
- Check the sink's `connected` statistic and its `reconnects` count.
+2 -2
View File
@@ -374,8 +374,8 @@ and plaintext-forwarded requests `403`. The existing scripts keep passing.
## Not Implemented
1. **PROXY protocol behind TLS passthrough**: planned in
[To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd).
None. The PROXY protocol behind TLS passthrough, deferred here, shipped with
the ACL work: [Security](security.md#proxy-protocol).
## mTLS Hardening
+86 -64
View File
@@ -26,7 +26,8 @@ configure it, and — equally important — what it does not yet do.
[Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy).
- Server pinning by dialers: certificate identity (`mtls`), bound certificate
(`scram`).
- Address rules on every listener, applied before TLS: see
- Address rules on every listener, applied before TLS, to the client a PROXY
header (v1 or v2) from a listed L4 proxy names: see
[The ACL Block](#the-acl-block).
- Startup warnings for expiring certificates and risky settings: see
[Startup Warnings](#startup-warnings).
@@ -53,11 +54,8 @@ configure it, and — equally important — what it does not yet do.
or credentials file.
- Per-address connection or request limits: only SCRAM logins are throttled
per address.
- Address rules for clients behind a proxy: `acl` sees the proxy, never the
client a PROXY header or `X-Forwarded-For` names.
- PROXY protocol: behind a proxy that passes TLS through, every peer shares the
proxy's address; see
[Behind nginx or another proxy](#behind-nginx-or-another-proxy).
- Address rules for `X-Forwarded-For` clients: in the `http` sink's proxy mode
`acl` sees the L7 proxy, never the client it forwards.
Two credentials are supported. Certificates (`mtls`) are the one the transport
already carries: the `tls` block establishes that a peer chains to your CA, and
@@ -334,25 +332,63 @@ peer costs no handshake and never reaches `auth`.
[pipelines.plugin_sources.config.acl]
allow = ["192.0.2.0/24", "198.51.100.7"] # addresses or CIDRs; empty: all
deny = ["192.0.2.66"] # refused, listed in allow or not
proxy_protocol = "required" # off (default), optional, required
proxy_from = ["10.0.0.5"] # L4 proxies sending PROXY headers
```
- `deny` wins; then a non-empty `allow` admits only its entries, an empty one
every address `deny` does not list.
- Entries are of the listener's [family](networking.md#address-family): IPv4
on an IPv4 listener, IPv6 on an IPv6 one, either behind a hostname (only the
family it binds ever matches). An entry of the other family, an IPv4-mapped
address, one with a zone, or anything but an address or CIDR fails
construction. A link-local peer matches without its zone.
family it binds ever matches). Behind `proxy_from` they take either, as a
header may name a client of either. An entry of the other family, an
IPv4-mapped address, one with a zone, or anything but an address or CIDR
fails construction. A link-local peer matches without its zone.
- A refused connection is closed at once and counted in `acl_denied` (plugin
stats, the `http` sink's status). A WARN names the peer and the count, at
most once a minute per listener; the refusals in between are only counted.
- The rules see the socket peer: behind a proxy (an L4 passthrough, or the
`http` sink's `trusted_proxies`), the proxy (see
[Behind nginx or another proxy](#behind-nginx-or-another-proxy)).
stats, the `http` sink's status). A WARN names the peer, the reason and the
count, at most once a minute per listener; the refusals in between are only
counted.
- The rules see the client: the socket peer, or the client a PROXY header
names. Behind the `http` sink's `trusted_proxies` they see the L7 proxy.
- The `serve` and `aggregator` presets take `allow` and `deny` list keys.
- Dialers have no `acl`: on a `tcp_chain` or `http_chain` sink it is an
unknown key.
### PROXY protocol
Behind an L4 proxy that passes TLS through (nginx `stream`, HAProxy
`mode tcp`) every peer comes from the proxy's address. With `proxy_protocol`
the proxy names the client in a PROXY header, v1 (text) or v2 (binary), sent
before the TLS ClientHello; LogWisp still ends TLS, so channel binding and
client certificates are unchanged.
- `proxy_protocol` and `proxy_from` come together; `proxy_from` holds the
proxies' addresses or CIDRs, of the listener's family. A peer with a zone
(link-local IPv6) is never a proxy.
- A peer in `proxy_from`:
- sends its header within `tlsx.HandshakeTimeout` (10 s); a malformed,
oversized (v1 over 107 bytes, a v2 block over 2048) or late one is
refused;
- under `required` is refused without a header; under `optional` it keeps
its socket address, after up to 10 s if it sends nothing at first. A
client the proxy forwards without a header then passes as the proxy, so
`optional` (which warns at startup) suits only a rollout; set `required`
once every route from `proxy_from` sends the header;
- keeps its socket address for `LOCAL` (v2) and `UNKNOWN` (v1), the proxies'
health checks, and for v2 families other than TCP over IPv4 and IPv6. TLVs
are skipped.
- Headers are read off the accept loop: a slow proxy holds only its own
connection.
- A peer not in `proxy_from` connects as itself; one that sends a header is
refused at its first read, as any peer can write one.
- The connection then reports the client (an IPv4-mapped one as IPv4) to the
rules, SCRAM [throttling](#throttling), sessions and logs. Session metadata
keeps the proxy as `peer_addr`, and a refusal's WARN names it.
- `acl_proxy_headers` counts the headers read.
- The `http` sink's proxy mode is the L7 counterpart: `trusted_proxies` trusts
`X-Forwarded-For` from a proxy that ends TLS, `proxy_from` a PROXY header
from one that passes it through.
## Startup Warnings
Some settings work but are usually mistakes. Each plugin reports them at WARN
@@ -370,7 +406,9 @@ when it is constructed, so startup and every reload repeat them:
(once per path per process)
- an `acl` entry with host bits set (`10.0.0.1/8`), naming the network it
matches; an `acl.allow` entry of a whole family (`0.0.0.0/0`, `::/0`); a
wildcard listener whose `acl` has `deny` but no `allow`
wildcard listener whose `acl` has `deny` but no `allow`; an `acl.proxy_from`
entry reaching beyond private, loopback and link-local addresses;
`acl.proxy_protocol = "optional"`
## Certificates made at startup
@@ -584,9 +622,10 @@ exchange.
### Throttling
Logins are throttled per remote socket address or, on an `http` sink in proxy
mode, per forwarded client (an IPv6 client per /64); forwarded headers are read
only from `trusted_proxies`. Each exchange takes a token from a bucket of 10
Logins are throttled per client (an IPv6 client per /64): the socket peer, the
client a [PROXY header](#proxy-protocol) names, or on an `http` sink in proxy
mode the forwarded client; forwarded headers are read only from
`trusted_proxies`. Each exchange takes a token from a bucket of 10
that refills at one per second, and a successful login gives it back, so only
failed or abandoned attempts drain it. At most 4 exchanges per address may be
unfinished: an HTTP challenge never answered holds its slot for up to 30 s, a
@@ -595,8 +634,8 @@ TCP connection that ends mid-exchange frees it at once. A refused start answers
65,536 addresses and refuses new ones when full. Separately, at most 4,096
exchanges may be in flight per listener; beyond that, or while the plugin
stops, a login gets `busy` (HTTP `503`) and counts `auth_busy`. Peers behind
one NAT or one passthrough proxy share a bucket: there, one client can exhaust
every other client's logins.
one NAT, or one passthrough proxy that sends no PROXY header, share a bucket:
there, one client can exhaust every other client's logins.
### Credentials file
@@ -646,9 +685,9 @@ different users.
- TLS must terminate at LogWisp, except on an `http` sink in proxy mode. Behind
any other terminating proxy or load balancer every login fails by design;
pass TLS through instead (TCP or SNI routing). All clients then share the
proxy's address, and so one throttling budget: see
[Behind nginx or another proxy](#behind-nginx-or-another-proxy).
pass TLS through instead (TCP or SNI routing), with a
[PROXY header](#proxy-protocol) so each client keeps its own throttling
budget.
- One process per HTTP address. Handshake state and the token key live in one
instance, so behind a balancer the proof or the token can reach an instance
that never saw the login.
@@ -756,7 +795,8 @@ A proxy in front of LogWisp works at one of two layers:
overwrite `X-Forwarded-For` with the real client: the rightmost hop that is
not a proxy is the client either way.
- **L4, passing TLS through to LogWisp's own TLS** (nginx `stream` with
`ssl_preread`, HAProxy `mode tcp`): whatever needs TLS to end at LogWisp.
`ssl_preread`, HAProxy `mode tcp`), with a [PROXY header](#proxy-protocol):
whatever needs TLS to end at LogWisp.
- Chain links (`tcp_chain`, `http_chain`).
- `lw auth stream` viewers of a `tcp` sink.
- An `http` sink that keeps its own TLS, and with it SCRAM channel binding
@@ -767,7 +807,7 @@ A common shape serves both from port 443:
```
:443 nginx stream: ssl_preread routes by SNI, proxy_protocol on
logs.example.org -> 127.0.0.1:8443 nginx http
relay.example.org -> a LogWisp TLS listener, without PROXY (below)
relay.example.org -> a LogWisp TLS listener, which reads the header (below)
127.0.0.1:8443 nginx http: listens with proxy_protocol, ends TLS
/logs/ -> 127.0.0.1:8081 LogWisp http sink, proxy mode
```
@@ -796,52 +836,40 @@ client or, when it is trusted (both on `127.0.0.1`), the hop to its left: one
the client wrote, so the client picks its own throttling address. LogWisp
ignores `X-Real-IP`.
**LogWisp's own TLS listeners take no PROXY header.** LogWisp cannot read the
PROXY protocol yet; it is planned first in the ACL work
([To Do, 1.2](todo.md#12-proxy-protocol-deferred-gap-of-scram-see-scram-auth-planmd)).
A stream route that sends `proxy_protocol` straight to a LogWisp TLS listener
breaks every connection: the header lands in front of the TLS handshake. nginx
sets `proxy_protocol` per stream `server`, so route LogWisp SNIs without it:
- Route 1: LogWisp gets its own stream `server` and port, without
`proxy_protocol`.
- Route 2: port 443 sends LogWisp SNIs to an internal stream `server` that
`listen`s with `proxy_protocol` and proxy_passes to LogWisp without it; that
hop consumes the header.
- Either way dialers must send the name: a `host` that is one, or
`tls.server_name` (`--server-name` for `lw auth`); an IP literal sends no SNI.
- With HAProxy, leave `send-proxy` off the LogWisp backends.
**LogWisp's own TLS listeners read the header** when their `acl` sets
`proxy_protocol` and lists the stream server in `proxy_from`; one stream
`server` with `proxy_protocol on` then routes every name:
```nginx
stream {
map $ssl_preread_server_name $route {
relay.example.org 127.0.0.1:10443; # route 2: the internal hop
tail.example.org 127.0.0.1:10443;
default 127.0.0.1:8443; # the http block, logs.example.org
}
map $ssl_preread_server_name $logwisp {
relay.example.org 127.0.0.1:9001; # tcp_chain source
tail.example.org 127.0.0.1:9002; # tcp sink, for lw auth stream
default 127.0.0.1:8443; # the http block, logs.example.org
}
server {
listen 443;
ssl_preread on;
proxy_protocol on; # on every route of this server
proxy_protocol on;
proxy_pass $route;
}
server { # route 1: its own port, no header
listen 9443;
ssl_preread on;
proxy_pass $logwisp;
}
server { # route 2: consumes the header
listen 127.0.0.1:10443 proxy_protocol;
ssl_preread on;
proxy_pass $logwisp;
}
}
```
```toml
[pipelines.plugin_sources.config.acl] # tcp_chain source; tcp sink alike
proxy_protocol = "required"
proxy_from = ["127.0.0.1"] # the stream server
```
- HAProxy: `send-proxy` or `send-proxy-v2` on the LogWisp backends.
- A listener without `proxy_protocol` behind such a route fails every
connection: the header lands in front of its TLS handshake.
- Dialers must send the name: a `host` that is one, or `tls.server_name`
(`--server-name` for `lw auth`); an IP literal sends no SNI.
- `test/acl-test.sh --auto` runs each listener kind behind a stub proxy
sending a v1 or v2 header, and the `tcp` sink behind nginx when installed.
**Passthrough without PROXY** brings every peer from the proxy's address:
- All peers share one SCRAM [throttling](#throttling) budget: 10 failed or
@@ -853,14 +881,8 @@ stream {
(`too many attempts` or `busy`) keeps sending on its token to the server it
pinned, retrying every few seconds, and holds its batches only once that
token has expired; any other refusal ends the token at once.
- Logs and sessions name the proxy, not the peer, and `acl` rules see only the
proxy: allow it, not the clients.
- Proxy mode is unaffected: it throttles on the forwarded client.
- Until LogWisp reads PROXY, mitigate:
- expose the chain ports directly, without the proxy, where you can;
- keep the proxy hop on a trusted network, so only trusted clients share the
budget;
- or accept the shared budget.
- Logs and sessions name the proxy, and `acl` rules see only the proxy: allow
it, not the clients.
## What Each Layer Enforces
+8 -5
View File
@@ -179,7 +179,8 @@ Options, each as type and default:
- `auth` (table): client authentication (`mtls` or `scram`); see
[Security](security.md#the-auth-block).
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
before TLS; see [Security](security.md#the-acl-block).
before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY
header names; see [Security](security.md#the-acl-block).
**Behaviour**
@@ -251,8 +252,8 @@ buffer sizes, connection limit, write timeout, uptime, endpoint paths, and the
**Statistics**: `dropped_writes`, `rejected_clients`, `auth`, `auth_allowed`,
`auth_rejected`; under `scram` also `auth_users`, `auth_throttled`,
`auth_busy`, `auth_binding_mismatch` and `auth_token_lifetime_ms`, and in
proxy mode `auth_trusted_proxies`; with `acl` rules also `acl` and
`acl_denied`.
proxy mode `auth_trusted_proxies`; with `acl` rules or `proxy_from` also
`acl`, `acl_denied` and `acl_proxy_headers`.
> Without an `auth` block both endpoints are unauthenticated, and the stream
> response carries `Access-Control-Allow-Origin: *`, so any web origin can read
@@ -306,7 +307,8 @@ Options, each as type and default:
- `auth` (table): client authentication (`mtls` or `scram`); see
[Security](security.md#the-auth-block).
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
before TLS; see [Security](security.md#the-acl-block).
before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY
header names; see [Security](security.md#the-acl-block).
**Behaviour**
@@ -333,7 +335,8 @@ Options, each as type and default:
**Statistics**: `write_errors`, `dropped_writes`, `rejected_conns`,
`tls_handshake_errors`, `auth`, `auth_allowed`, `auth_rejected`; under `scram`
also `auth_users`, `auth_throttled`, `auth_busy` and `auth_binding_mismatch`;
with `acl` rules also `acl` and `acl_denied`.
with `acl` rules or `proxy_from` also `acl`, `acl_denied` and
`acl_proxy_headers`.
---
+11 -7
View File
@@ -215,7 +215,8 @@ Options, each as type and default:
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
see [Security](security.md#the-auth-block).
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
before TLS; see [Security](security.md#the-acl-block).
before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY
header names; see [Security](security.md#the-acl-block).
For passwords instead of certificates, the `auth` block takes `type = "scram"`
and a `credentials_file`, and `client_auth` becomes optional; see
@@ -240,8 +241,9 @@ and a `credentials_file`, and `client_auth` becomes optional; see
leaves per-entry labels to `trust_node`, so a relay can forward other nodes'
entries while proving its own identity.
- Each accepted connection gets a session recording the remote address, node
label, — under TLS — `tls` and `tls_peer_cn`, and — under auth —
`auth_method` and `auth_identity`.
label, — under TLS — `tls` and `tls_peer_cn`, — under auth —
`auth_method` and `auth_identity`, and behind a PROXY header the proxy as
`peer_addr`.
- A malformed entry line increments `parse_errors` and is skipped; the
connection survives. A line over 1 MiB is a protocol violation and terminates
the connection.
@@ -249,7 +251,8 @@ and a `credentials_file`, and `client_auth` becomes optional; see
**Statistics**: `active_connections`, `rejected_conns`, `parse_errors`,
`tls_handshake_errors`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`
and `auth_binding_mismatch`; with `acl` rules also `acl` and `acl_denied`.
and `auth_binding_mismatch`; with `acl` rules or `proxy_from` also `acl`,
`acl_denied` and `acl_proxy_headers`.
---
@@ -301,7 +304,8 @@ Options, each as type and default:
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
see [Security](security.md#the-auth-block).
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
before TLS; see [Security](security.md#the-acl-block).
before TLS, and `proxy_protocol` with `proxy_from` for the client a PROXY
header names; see [Security](security.md#the-acl-block).
**Behaviour**
@@ -328,8 +332,8 @@ Options, each as type and default:
**Statistics**: `total_requests`, `rejected_requests`, `parse_errors`,
`cached_sessions`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`,
`auth_binding_mismatch` and `auth_token_lifetime_ms`; with `acl` rules also
`acl` and `acl_denied`.
`auth_binding_mismatch` and `auth_token_lifetime_ms`; with `acl` rules or
`proxy_from` also `acl`, `acl_denied` and `acl_proxy_headers`.
---
+18 -76
View File
@@ -7,98 +7,40 @@ they belong to.
## 1. Network access control (ACL)
The seam exists: `internal/netacl` wraps each listener's socket before TLS,
and the `acl` block's `allow` and `deny` refuse peers by address
reads the PROXY header of the L4 proxies `proxy_from` lists, and the `acl`
block's `allow` and `deny` refuse the client by address
([Security](security.md#the-acl-block)). Every limit except SCRAM throttling
is still global, and behind an L4 proxy every peer is the proxy.
is still global.
### 1.1 The filtering listener
- `Accept` is to read an optional PROXY header (1.2) before the address rules
and apply per-address connection caps (1.4) after them, returning a
`net.Conn` whose `RemoteAddr` is the real client.
- Everything downstream then sees the real client with no further change:
`authz.remoteIP`, SCRAM throttling, sessions, logs and `http.Request.RemoteAddr`.
- Keep the proxy's own address as `peer_addr` in session metadata for audit.
### 1.2 PROXY protocol (deferred gap of SCRAM, see scram-auth-plan.md)
- Why: behind an L4 proxy that passes TLS through (nginx `stream` with
`ssl_preread`, HAProxy `mode tcp`), every peer arrives from the proxy's
address.
- All peers share one SCRAM throttling budget, so one client can lock out
every login.
- Logs, sessions and the ACL itself see only the proxy.
- What: accept PROXY v1 (text) and v2 (binary) headers, but only from
`proxy_from` addresses or CIDRs. Two config keys in the ACL block:
- `proxy_protocol = "off" | "optional" | "required"`, default off.
- `proxy_from = [...]`.
- Rules:
- A listed peer must send the header when `required`.
- An unlisted peer that sends a header is refused, since the header is
spoofable.
- The header must arrive within a short deadline (reuse
`tlsx.HandshakeTimeout`), with a size cap: v1 at most 107 bytes, v2 with a
bounded TLV length.
- `LOCAL`/`UNKNOWN` commands (proxy health checks) keep the socket address.
- TLVs are ignored.
- The header precedes the TLS ClientHello, so SCRAM channel binding is
unaffected: LogWisp still terminates TLS.
- nginx side:
- In the `stream` server, `proxy_protocol on` on the route to LogWisp (it
sends v1).
- HAProxy: `send-proxy` or `send-proxy-v2`.
- Relation to `auth.trusted_proxies` (HTTP proxy mode):
- That list trusts `X-Forwarded-For` from a TLS-terminating L7 proxy;
`proxy_from` trusts a PROXY header from an L4 proxy.
- Keep both, with distinct names and docs. They answer different layers, and
one deployment can use both: nginx `stream` with `proxy_protocol` into an
nginx `http` block that sets `X-Forwarded-For $proxy_protocol_addr`.
### 1.3 Address rules for forwarded clients
- Rules match the real client (after PROXY), never the proxy.
- In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after
the connection-level decision, so rules for forwarded HTTP clients belong in
a second, request-level check there, reusing the compiled `netacl` rules.
`authz.parseProxies` then takes netacl's address-or-CIDR parsing too.
- Startup warning: `proxy_from` covering public ranges.
### 1.4 Per-client limits
### 1.1 Per-client limits
- Generalize the SCRAM limiter in `internal/authz/scram.go` (token bucket per
key, bounded table that fails closed, idle sweep, IPv6 per /64 via
`throttleKey`) into the shared per-key limiter. One mechanism (AGENTS.md):
SCRAM throttling then uses it too.
- Keys:
- Keys in the `acl` block:
- `max_connections_per_client` on every listener (concurrent connections per
address).
client), applied by the netacl listener after the rules.
- `requests_per_second_per_client` on HTTP listeners (ingest, stream and
status requests).
- A full table refuses new clients and counts them in `acl_limited`, beside
`acl_denied` and `acl_proxy_headers`.
- Later: per-identity limits once a peer is authenticated (the mtls plan's
deferred item 2), keyed by identity instead of address.
### 1.5 Config, validation, docs
### 1.2 Address rules for forwarded clients
- `proxy_protocol`, `proxy_from` and the 1.4 keys join `allow` and `deny` in
the `acl` block.
- Intent rule as in `authz`: `proxy_from` without `proxy_protocol` is an error.
- Stats: `acl_proxy_headers` and `acl_limited` beside `acl_denied`.
- Docs: the security.md section, plus networking.md troubleshooting lines for
each refusal.
- In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after
the connection-level decision, so rules for forwarded HTTP clients belong in
a second, request-level check there, reusing the compiled `netacl` rules.
`authz.parseProxies` then takes netacl's address-or-CIDR parsing too.
### 1.6 Verification
### 1.3 Verification
- Go tests, one per rule:
- PROXY v1 and v2 parse, including malformed, oversized and slow headers.
- A header from an unlisted peer refused; `required` without a header
refused; `LOCAL` kept.
- Per-client connection cap; the limiter table failing closed.
- The real client reaching SCRAM throttling and session metadata.
- `test/acl-test.sh`, which covers the address rules, adds:
- A Go stub proxy that sends v1 and v2 headers in front of each listener
type.
- nginx `stream` with `proxy_protocol on` when nginx is installed.
- Per-client SCRAM budgets behind one proxy address.
- Go tests, one per rule: the per-client connection cap; the request rate; the
limiter table failing closed; a forwarded client refused by the rules.
- `test/acl-test.sh` adds per-client connection caps, behind its stub proxy
too.
## 2. Packaging: AUR, FreeBSD ports, Debian
+5 -2
View File
@@ -374,11 +374,14 @@ type AuthOptions struct {
}
// ACLOptions is a listener's address rules, entries being addresses or CIDRs:
// deny wins, then a set allow list admits only its entries. Validation is in
// internal/netacl.
// deny wins, then a set allow list admits only its entries. proxy_from lists
// the L4 proxies whose PROXY header names the client the rules then see.
// Validation is in internal/netacl.
type ACLOptions struct {
Allow []string `toml:"allow"`
Deny []string `toml:"deny"`
ProxyProtocol string `toml:"proxy_protocol"` // off, optional or required
ProxyFrom []string `toml:"proxy_from"`
}
// --- TLS Options ---
+377 -46
View File
@@ -1,43 +1,87 @@
// Package netacl is the address seam of the network listeners, beside tlsx
// and authz: a Policy refuses a peer by its address as the socket is
// accepted, before TLS or a byte of the protocol is read. New returns nil
// when no rule is set, and every method tolerates a nil receiver.
// and authz: a Policy reads the PROXY header of the proxies it trusts, then
// refuses a peer by its client's address, before TLS or a byte of the
// protocol is read. New returns nil when nothing is set, and every method
// tolerates a nil receiver.
package netacl
import (
"bufio"
"bytes"
"context"
"encoding/binary"
"errors"
"fmt"
"net"
"net/netip"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/lixenwraith/logwisp/internal/config"
"github.com/lixenwraith/logwisp/internal/core"
"github.com/lixenwraith/logwisp/internal/tlsx"
"github.com/lixenwraith/log"
)
// warnEvery spaces the WARNs reporting refused peers; acl_denied counts all
const warnEvery = time.Minute
const (
// warnEvery spaces the WARNs reporting refused peers; acl_denied counts all
warnEvery = time.Minute
v1Max = 107 // the longest PROXY v1 line, CRLF included
v2Max = 2048 // the longest PROXY v2 address block, TLVs included
)
var (
sigV1 = []byte("PROXY ")
sigV2 = []byte("\r\n\r\n\x00\r\nQUIT\n")
// local is where a proxy_from entry lies without a startup warning
local = []netip.Prefix{
netip.MustParsePrefix("10.0.0.0/8"), netip.MustParsePrefix("172.16.0.0/12"),
netip.MustParsePrefix("192.168.0.0/16"), netip.MustParsePrefix("127.0.0.0/8"),
netip.MustParsePrefix("169.254.0.0/16"), netip.MustParsePrefix("fc00::/7"),
netip.MustParsePrefix("fe80::/10"), netip.MustParsePrefix("::1/128"),
}
errScreened = errors.New("PROXY header from a peer outside proxy_from")
// v2Families holds the family bytes v2 defines, each with the address
// bytes taken under PROXY: TCP over IPv4 and IPv6 name the client, the
// rest keep the socket address
v2Families = map[byte]int{0x00: 0, 0x11: 12, 0x12: 0, 0x21: 36, 0x22: 0, 0x31: 0, 0x32: 0}
)
// Policy is the compiled form of config.ACLOptions for one listener
type Policy struct {
allow, deny []netip.Prefix
allow, deny, proxyFrom []netip.Prefix
mode string // proxy_protocol, when proxy_from is set
headerWait time.Duration // for a proxy's header: tlsx.HandshakeTimeout
logger *log.Logger
component, id string
denied atomic.Uint64
denied, headers atomic.Uint64
epoch time.Time // a monotonic base: a stepped clock neither mutes nor floods the WARN
nextWarn atomic.Int64 // nanoseconds since epoch
}
// New compiles the rules of a listener on host and warns of what they leave
// open. Each entry must be of the family core.Network gives host: one the
// listener's sockets can never carry is a mistake, not a rule.
// open. Entries are of the family core.Network gives host, as one the sockets
// never carry is a mistake; behind proxy_from, allow and deny take both, as
// the proxy names clients of either.
func New(o *config.ACLOptions, host string, l *log.Logger, component, id string) (*Policy, error) {
if o == nil || len(o.Allow)+len(o.Deny) == 0 {
if o == nil {
return nil, nil
}
proxied := o.ProxyProtocol == "optional" || o.ProxyProtocol == "required"
switch {
case !proxied && o.ProxyProtocol != "" && o.ProxyProtocol != "off":
return nil, fmt.Errorf("acl: proxy_protocol %q is none of off, optional and required", o.ProxyProtocol)
case proxied && len(o.ProxyFrom) == 0:
return nil, fmt.Errorf("acl: proxy_protocol %s needs proxy_from, the proxies that may send the header", o.ProxyProtocol)
case !proxied && len(o.ProxyFrom) > 0:
return nil, errors.New("acl: proxy_from needs proxy_protocol optional or required")
case len(o.Allow)+len(o.Deny)+len(o.ProxyFrom) == 0:
return nil, nil
}
network, err := core.Network(host)
@@ -45,7 +89,7 @@ func New(o *config.ACLOptions, host string, l *log.Logger, component, id string)
return nil, err
}
var widened []string // entry, network pairs
compile := func(key string, entries []string) ([]netip.Prefix, error) {
compile := func(key, network string, entries []string) ([]netip.Prefix, error) {
var rules []netip.Prefix
for _, e := range entries {
prefix, err := parse(e, network)
@@ -59,11 +103,17 @@ func New(o *config.ACLOptions, host string, l *log.Logger, component, id string)
}
return rules, nil
}
p := &Policy{logger: l, component: component, id: id, epoch: time.Now()}
if p.allow, err = compile("allow", o.Allow); err != nil {
p := &Policy{mode: o.ProxyProtocol, headerWait: tlsx.HandshakeTimeout, logger: l, component: component, id: id, epoch: time.Now()}
if p.proxyFrom, err = compile("proxy_from", network, o.ProxyFrom); err != nil {
return nil, err
}
if p.deny, err = compile("deny", o.Deny); err != nil {
if proxied {
network = "tcp"
}
if p.allow, err = compile("allow", network, o.Allow); err != nil {
return nil, err
}
if p.deny, err = compile("deny", network, o.Deny); err != nil {
return nil, err
}
p.logStartup(host, widened)
@@ -106,7 +156,22 @@ func (p *Policy) logStartup(host string, widened []string) {
"hint", "list only the networks that may connect")
}
}
if addr, err := netip.ParseAddr(host); len(p.allow) == 0 && (host == "" || err == nil && addr.IsUnspecified()) {
if p.mode == "optional" {
p.logger.Warn("msg", "acl.proxy_protocol optional: a client a proxy_from peer forwards without a header passes as the proxy",
"component", p.component,
"instance_id", p.id,
"hint", "set required once every route from proxy_from sends the header")
}
for _, q := range p.proxyFrom {
if !slices.ContainsFunc(local, func(l netip.Prefix) bool { return l.Bits() <= q.Bits() && l.Contains(q.Addr()) }) {
p.logger.Warn("msg", "acl.proxy_from reaches public addresses, any of which may name any client",
"component", p.component,
"instance_id", p.id,
"entry", q.String(),
"hint", "list only the proxies' own addresses")
}
}
if addr, err := netip.ParseAddr(host); len(p.allow) == 0 && len(p.deny) > 0 && (host == "" || err == nil && addr.IsUnspecified()) {
p.logger.Warn("msg", "Listener on every address admits all that acl.deny does not list",
"component", p.component,
"instance_id", p.id,
@@ -114,61 +179,323 @@ func (p *Policy) logStartup(host string, widened []string) {
}
}
// in reports whether addr lies in a rule, matched unmapped and without its
// zone: a zoned address is in no prefix
func in(rules []netip.Prefix, addr netip.Addr) bool {
addr = addr.Unmap().WithZone("")
return slices.ContainsFunc(rules, func(q netip.Prefix) bool { return q.Contains(addr) })
}
// admits applies the rules: deny wins, then a set allow list admits only its
// entries. A peer whose address is unknown is refused.
func (p *Policy) admits(addr netip.Addr) bool {
addr = addr.Unmap().WithZone("") // a zoned address is in no prefix
in := func(q netip.Prefix) bool { return q.Contains(addr) }
return addr.IsValid() && !slices.ContainsFunc(p.deny, in) &&
(len(p.allow) == 0 || slices.ContainsFunc(p.allow, in))
return addr.IsValid() && !in(p.deny, addr) && (len(p.allow) == 0 || in(p.allow, addr))
}
// Listener returns ln refusing the peers the rules exclude as it accepts them,
// so the plugin never sees one. Close stays ln's: a sink's shutdown flush
// reaches it unchanged. A nil Policy returns ln.
// proxies reports a peer in proxy_from. A zoned one never is: a link-local
// address is any link's to claim.
func (p *Policy) proxies(addr netip.Addr) bool {
addr = addr.Unmap()
return slices.ContainsFunc(p.proxyFrom, func(q netip.Prefix) bool { return q.Contains(addr) })
}
func addrOf(c net.Conn) netip.Addr {
if t, ok := c.RemoteAddr().(*net.TCPAddr); ok {
return t.AddrPort().Addr()
}
return netip.Addr{}
}
// Listener returns ln deciding each peer before Accept returns it. Headers are
// read off the accept loop, so a slow proxy holds only its own connection; the
// loop starts at once, so call this in Start. A nil Policy returns ln.
func (p *Policy) Listener(ln net.Listener) net.Listener {
if p == nil {
return ln
}
return &listener{Listener: ln, p: p}
ctx, cancel := context.WithCancel(context.Background())
l := &listener{Listener: ln, p: p, accepted: make(chan accepted), ctx: ctx, cancel: cancel}
go l.run()
return l
}
type listener struct {
net.Listener
p *Policy
accepted chan accepted
ctx context.Context // done at Close
cancel context.CancelFunc
}
type accepted struct {
conn net.Conn
err error
}
func (l *listener) Accept() (net.Conn, error) {
for {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
var addr netip.Addr
remote := conn.RemoteAddr()
if t, ok := remote.(*net.TCPAddr); ok {
addr = t.AddrPort().Addr()
}
if l.p.admits(addr) {
return conn, nil
}
conn.Close()
l.p.refused(fmt.Sprint(remote))
select {
case a := <-l.accepted:
return a.conn, a.err
case <-l.ctx.Done():
return nil, net.ErrClosed
}
}
// refused counts a refusal and WARNs at most once per warnEvery
func (p *Policy) refused(remote string) {
n := p.denied.Add(1)
// Close also closes the connections still sending a header
func (l *listener) Close() error {
l.cancel()
return l.Listener.Close()
}
func (l *listener) run() {
for {
c, err := l.Listener.Accept()
if err != nil {
if !l.deliver(nil, err) || errors.Is(err, net.ErrClosed) {
return
}
continue
}
switch {
case l.p.proxies(addrOf(c)):
go l.proxied(c)
case len(l.p.proxyFrom) > 0:
l.admit(&conn{Conn: c, r: bufio.NewReaderSize(c, v1Max), screen: l.p})
default:
l.admit(c)
}
}
}
// proxied reads the header of a peer in proxy_from, then admits the
// connection by the client it names
func (l *listener) proxied(c net.Conn) {
stop := context.AfterFunc(l.ctx, func() { c.Close() })
c.SetReadDeadline(time.Now().Add(l.p.headerWait))
br := bufio.NewReaderSize(c, v1Max)
client, found, err := readHeader(br)
c.SetReadDeadline(time.Time{})
if !stop() {
return // closed with the listener
}
pc := &conn{Conn: c, r: br}
switch {
case err != nil:
l.p.refuse(c, "PROXY header: "+err.Error())
return
case !found && l.p.mode == "required":
l.p.refuse(c, "no PROXY header")
return
case found:
l.p.headers.Add(1)
if client.IsValid() {
pc.client = net.TCPAddrFromAddrPort(client)
}
}
l.admit(pc)
}
// admit hands c to Accept if the rules admit the address it ends with
func (l *listener) admit(c net.Conn) {
if !l.p.admits(addrOf(c)) {
l.p.refuse(c, "address rules")
return
}
l.deliver(c, nil)
}
func (l *listener) deliver(c net.Conn, err error) bool {
select {
case l.accepted <- accepted{c, err}:
return true
case <-l.ctx.Done():
if c != nil {
c.Close()
}
return false
}
}
// refuse closes c, counts it and WARNs at most once per warnEvery
func (p *Policy) refuse(c net.Conn, reason string) {
n := p.denied.Add(1) // first: a peer that sees the close sees the count
c.Close()
now, next := int64(time.Since(p.epoch)), p.nextWarn.Load()
if now < next || !p.nextWarn.CompareAndSwap(next, now+int64(warnEvery)) {
return
}
p.logger.Warn("msg", "Connection refused by acl; more within a minute are only counted",
fields := []any{"msg", "Connection refused by acl; more within a minute are only counted",
"component", p.component,
"instance_id", p.id,
"remote_addr", remote,
"acl_denied", n)
"remote_addr", fmt.Sprint(c.RemoteAddr()),
"reason", reason,
"acl_denied", n}
if peer := PeerAddr(c); peer != "" {
fields = append(fields, "peer_addr", peer)
}
p.logger.Warn(fields...)
}
// conn is read through the buffer its first bytes went to: past a PROXY
// header, whose client it reports, or under a screen refusing one from a peer
// outside proxy_from at the first Read
type conn struct {
net.Conn
r *bufio.Reader
client net.Addr // nil keeps the socket's
screen *Policy
once sync.Once
err error
}
func (c *conn) Read(b []byte) (int, error) {
c.once.Do(func() {
if c.screen == nil {
return
}
if v, _ := signature(c.r); v != 0 {
c.err = errScreened
c.screen.refuse(c, errScreened.Error())
}
})
if c.err != nil {
return 0, c.err
}
return c.r.Read(b)
}
func (c *conn) RemoteAddr() net.Addr {
if c.client != nil {
return c.client
}
return c.Conn.RemoteAddr()
}
// PeerAddr is the proxy whose PROXY header named c's client, "" when c came
// direct; c may be the *tls.Conn over the listener's connection
func PeerAddr(c net.Conn) string {
if t, ok := c.(interface{ NetConn() net.Conn }); ok {
c = t.NetConn()
}
if pc, ok := c.(*conn); ok && pc.client != nil {
return pc.Conn.RemoteAddr().String()
}
return ""
}
type peerKey struct{}
// ConnContext, as an http.Server's, carries PeerAddr to the requests of a
// connection for ContextPeerAddr
func ConnContext(ctx context.Context, c net.Conn) context.Context {
if peer := PeerAddr(c); peer != "" {
return context.WithValue(ctx, peerKey{}, peer)
}
return ctx
}
func ContextPeerAddr(ctx context.Context) string {
peer, _ := ctx.Value(peerKey{}).(string)
return peer
}
// signature reads as far as tells whether a PROXY header starts the stream:
// its version, or 0 when other bytes or none come first
func signature(br *bufio.Reader) (int, error) {
for n := 1; ; n++ {
b, err := br.Peek(n)
switch {
case len(b) == 0 || !bytes.HasPrefix(sigV1, b) && !bytes.HasPrefix(sigV2, b):
return 0, nil
case bytes.Equal(b, sigV1):
return 1, nil
case bytes.Equal(b, sigV2):
return 2, nil
case err != nil:
return 0, fmt.Errorf("signature cut short: %w", err)
}
}
}
// readHeader reads the PROXY header starting br, if one does: found is false,
// with nothing consumed, when none does. The client is invalid for LOCAL,
// UNKNOWN and the families v2 leaves to the socket address.
func readHeader(br *bufio.Reader) (client netip.AddrPort, found bool, err error) {
v, err := signature(br)
switch v {
case 1:
client, err = readV1(br)
case 2:
client, err = readV2(br)
}
return client, v != 0 && err == nil, err
}
// readV1 reads "PROXY TCP4|TCP6 src dst sport dport\r\n" or "PROXY UNKNOWN...\r\n"
func readV1(br *bufio.Reader) (netip.AddrPort, error) {
line, err := br.ReadSlice('\n')
if errors.Is(err, bufio.ErrBufferFull) {
return netip.AddrPort{}, fmt.Errorf("v1 line over %d bytes", v1Max)
} else if err != nil {
return netip.AddrPort{}, fmt.Errorf("v1 line cut short: %w", err)
}
s, crlf := strings.CutSuffix(string(line), "\r\n")
f := strings.Split(s, " ")
switch {
case !crlf:
return netip.AddrPort{}, errors.New("v1 line not ended by CRLF")
case f[1] == "UNKNOWN":
return netip.AddrPort{}, nil
case len(f) != 6 || f[1] != "TCP4" && f[1] != "TCP6":
return netip.AddrPort{}, fmt.Errorf("malformed v1 line %q", s)
}
src, err1 := netip.ParseAddr(f[2])
dst, err2 := netip.ParseAddr(f[3])
port, err3 := strconv.ParseUint(f[4], 10, 16)
_, err4 := strconv.ParseUint(f[5], 10, 16)
four := f[1] == "TCP4"
if errors.Join(err1, err2, err3, err4) != nil || src.Zone()+dst.Zone() != "" || src.Is4() != four || dst.Is4() != four {
return netip.AddrPort{}, fmt.Errorf("malformed v1 line %q", s)
}
return netip.AddrPortFrom(src.Unmap(), uint16(port)), nil
}
// readV2 reads the 16-byte prefix and the block after it, taking the address
// of TCP over IPv4 or IPv6 and skipping the rest, TLVs included
func readV2(br *bufio.Reader) (netip.AddrPort, error) {
h, err := br.Peek(16)
if err != nil {
return netip.AddrPort{}, fmt.Errorf("v2 prefix cut short: %w", err)
}
cmd, fam, n := h[12], h[13], int(binary.BigEndian.Uint16(h[14:]))
need, known := v2Families[fam]
if cmd&0xf != 1 {
need, known = 0, true // LOCAL ignores the block
}
switch {
case cmd>>4 != 2 || cmd&0xf > 1:
return netip.AddrPort{}, fmt.Errorf("v2 version and command %#x", cmd)
case !known:
return netip.AddrPort{}, fmt.Errorf("v2 family %#x", fam)
case n > v2Max:
return netip.AddrPort{}, fmt.Errorf("v2 block of %d bytes over %d", n, v2Max)
case n < need:
return netip.AddrPort{}, fmt.Errorf("v2 block of %d bytes short for family %#x", n, fam)
}
br.Discard(16)
var client netip.AddrPort
if need > 0 {
b, err := br.Peek(need)
if err != nil {
return netip.AddrPort{}, fmt.Errorf("v2 block cut short: %w", err)
}
addr, _ := netip.AddrFromSlice(b[:(need-4)/2]) // source, destination, two ports
client = netip.AddrPortFrom(addr.Unmap(), binary.BigEndian.Uint16(b[need-4:]))
}
if _, err := br.Discard(n); err != nil {
return netip.AddrPort{}, fmt.Errorf("v2 block cut short: %w", err)
}
return client, nil
}
// Describe renders the policy for log lines, stats and the status endpoint
@@ -180,7 +507,11 @@ func (p *Policy) Describe() string {
if len(p.allow) > 0 {
allow = fmt.Sprint(len(p.allow))
}
return fmt.Sprintf("allow=%s deny=%d", allow, len(p.deny))
s := fmt.Sprintf("allow=%s deny=%d", allow, len(p.deny))
if len(p.proxyFrom) > 0 {
s += fmt.Sprintf(" proxy_protocol=%s proxy_from=%d", p.mode, len(p.proxyFrom))
}
return s
}
func (p *Policy) Denied() uint64 {
@@ -196,5 +527,5 @@ func (p *Policy) Stats() map[string]any {
if p == nil {
return nil
}
return map[string]any{"acl": p.Describe(), "acl_denied": p.denied.Load()}
return map[string]any{"acl": p.Describe(), "acl_denied": p.denied.Load(), "acl_proxy_headers": p.headers.Load()}
}
+329 -44
View File
@@ -1,10 +1,16 @@
package netacl
import (
"bufio"
"bytes"
"crypto/tls"
"errors"
"io"
"net"
"net/netip"
"slices"
"strings"
"sync"
"testing"
"time"
@@ -13,9 +19,9 @@ import (
"github.com/lixenwraith/log"
)
func mustNew(t *testing.T, host string, allow, deny []string) *Policy {
func mustNew(t *testing.T, host string, o config.ACLOptions) *Policy {
t.Helper()
p, err := New(&config.ACLOptions{Allow: allow, Deny: deny}, host, log.NewLogger(), "test", "acl")
p, err := New(&o, host, log.NewLogger(), "test", "acl")
if err != nil {
t.Fatal(err)
}
@@ -26,8 +32,8 @@ func mustNew(t *testing.T, host string, allow, deny []string) *Policy {
// one admits all not denied, an IPv4-mapped peer matches as IPv4, and a peer
// of unknown address is refused.
func TestDenyWinsThenAllowListAdmitsOnlyItsEntries(t *testing.T) {
listed := mustNew(t, "0.0.0.0", []string{"127.0.0.0/8"}, []string{"127.0.0.2"})
denyOnly := mustNew(t, "0.0.0.0", nil, []string{"10.0.0.0/8"})
listed := mustNew(t, "0.0.0.0", config.ACLOptions{Allow: []string{"127.0.0.0/8"}, Deny: []string{"127.0.0.2"}})
denyOnly := mustNew(t, "0.0.0.0", config.ACLOptions{Deny: []string{"10.0.0.0/8"}})
for _, c := range []struct {
p *Policy
addr string
@@ -50,68 +56,351 @@ func TestDenyWinsThenAllowListAdmitsOnlyItsEntries(t *testing.T) {
// Entries are of the family core.Network gives the listener's host, as its
// sockets carry no other: IPv4 on an IPv4 listener, IPv6 on an IPv6 one,
// either behind a hostname; never IPv4-mapped, never with a zone.
// either behind a hostname; never IPv4-mapped, never with a zone. Behind
// proxy_from, which keeps that family, deny takes both: the proxy names either.
func TestEntriesFollowTheListenerFamily(t *testing.T) {
for _, c := range []struct {
host, entry string
host, from, entry string
ok bool
}{
{"0.0.0.0", "10.0.0.0/8", true},
{"0.0.0.0", "2001:db8::/32", false},
{"::", "2001:db8::1", true},
{"::", "10.0.0.1", false},
{"::", "::ffff:10.0.0.1", false},
{"::", "fe80::1%eth0", false},
{"logs.example", "10.0.0.0/8", true},
{"logs.example", "2001:db8::/32", true},
{"0.0.0.0", "", "10.0.0.0/8", true},
{"0.0.0.0", "", "2001:db8::/32", false},
{"::", "", "2001:db8::1", true},
{"::", "", "10.0.0.1", false},
{"::", "", "::ffff:10.0.0.1", false},
{"::", "", "fe80::1%eth0", false},
{"logs.example", "", "10.0.0.0/8", true},
{"logs.example", "", "2001:db8::/32", true},
{"0.0.0.0", "10.0.0.5", "2001:db8::/32", true},
{"0.0.0.0", "2001:db8::5", "10.0.0.0/8", false},
} {
_, err := New(&config.ACLOptions{Deny: []string{c.entry}}, c.host, log.NewLogger(), "test", "acl")
if (err == nil) != c.ok {
t.Errorf("%s on %q: %v", c.entry, c.host, err)
o := &config.ACLOptions{Deny: []string{c.entry}}
if c.from != "" {
o.ProxyProtocol, o.ProxyFrom = "required", []string{c.from}
}
if _, err := New(o, c.host, log.NewLogger(), "test", "acl"); (err == nil) != c.ok {
t.Errorf("%s behind %q on %q: %v", c.entry, c.from, c.host, err)
}
}
}
// A link-local peer carries its interface zone, which no prefix contains:
// it is matched without it, or a deny of fe80::/10 would admit it.
func TestZonedPeerMatchesItsPrefix(t *testing.T) {
p := mustNew(t, "::", nil, []string{"fe80::/10"})
if p.admits(netip.MustParseAddr("fe80::1%eth0")) {
t.Fatal("a zoned link-local peer passed the deny of its prefix")
// A link-local peer carries its interface zone, which no prefix contains: the
// rules match it without, or a deny of fe80::/10 would admit it. It is never
// a proxy, as any link can claim its address.
func TestZonedPeerMatchesRulesButIsNoProxy(t *testing.T) {
p := mustNew(t, "::", config.ACLOptions{Deny: []string{"fe80::/10"}, ProxyProtocol: "required", ProxyFrom: []string{"fe80::/10"}})
if zoned := netip.MustParseAddr("fe80::1%eth0"); p.admits(zoned) || p.proxies(zoned) {
t.Fatal("a zoned link-local peer passed the deny of its prefix, or as a proxy")
}
}
// A refused peer is closed as it is accepted and counted: the plugin's
// Accept returns the next admitted one.
func TestRefusedPeerNeverReachesAccept(t *testing.T) {
inner := &fakeListener{conns: make(chan net.Conn, 2)}
refused, refusedPeer := net.Pipe()
admitted, _ := net.Pipe()
inner.conns <- remoteAt(refused, "192.0.2.1")
inner.conns <- remoteAt(admitted, "127.0.0.1")
ln := mustNew(t, "127.0.0.1", nil, []string{"192.0.2.0/24"}).Listener(inner)
conn, err := ln.Accept()
if err != nil {
t.Fatal(err)
}
if got := conn.RemoteAddr().String(); !strings.HasPrefix(got, "127.0.0.1:") {
p := mustNew(t, "127.0.0.1", config.ACLOptions{Deny: []string{"192.0.2.0/24"}})
ln, dial := listening(t, p)
refused := dial("192.0.2.1")
dial("127.0.0.1")
if got := accept(t, ln).RemoteAddr().String(); !strings.HasPrefix(got, "127.0.0.1:") {
t.Fatalf("Accept returned %s", got)
}
refusedPeer.SetReadDeadline(time.Now().Add(time.Second))
if _, err := refusedPeer.Read(make([]byte, 1)); err != io.EOF {
t.Fatalf("the refused peer reads %v, want EOF", err)
}
if n := ln.(*listener).p.Stats()["acl_denied"]; n != uint64(1) {
closed(t, refused)
if n := p.Stats()["acl_denied"]; n != uint64(1) {
t.Fatalf("acl_denied = %v", n)
}
}
// proxy_protocol and proxy_from come together: either alone is a mistake, as
// is a mode that is none of off, optional and required
func TestProxyProtocolAndProxyFromComeTogether(t *testing.T) {
for _, c := range []struct {
mode string
from []string
ok bool
}{
{"required", nil, false},
{"", []string{"127.0.0.1"}, false},
{"off", []string{"127.0.0.1"}, false},
{"always", nil, false},
{"optional", []string{"127.0.0.1"}, true},
} {
_, err := New(&config.ACLOptions{ProxyProtocol: c.mode, ProxyFrom: c.from}, "127.0.0.1", log.NewLogger(), "test", "acl")
if (err == nil) != c.ok {
t.Errorf("proxy_protocol %q with proxy_from %v: %v", c.mode, c.from, err)
}
}
}
// A v1 or v2 header names the client, IPv4-mapped as IPv4 (as throttling
// keys it), and leaves the bytes after it, TLVs skipped.
func TestHeaderNamesItsClient(t *testing.T) {
for _, c := range []struct{ header, client string }{
{"PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\n", "198.51.100.1:40000"},
{"PROXY TCP6 2001:db8::1 2001:db8::2 40000 443\r\n", "[2001:db8::1]:40000"},
{"PROXY TCP6 ::ffff:198.51.100.1 ::ffff:192.0.2.1 40000 443\r\n", "198.51.100.1:40000"},
{v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")), "198.51.100.1:40000"},
{v2(0x21, 0x21, block("2001:db8::1", "2001:db8::2", 0x04, 0x00, 0x01, 0xff)), "[2001:db8::1]:40000"},
{v2(0x21, 0x21, block("::ffff:198.51.100.1", "::ffff:192.0.2.1")), "198.51.100.1:40000"},
} {
client, found, rest, err := parseHeader(c.header + "rest")
if err != nil || !found || client.String() != c.client || rest != "rest" {
t.Errorf("%q: client %v found %v rest %q: %v", c.header, client, found, rest, err)
}
}
}
// LOCAL, UNKNOWN and the families v2 leaves unspecified keep the socket
// address, whatever address block they carry
func TestLocalAndUnknownKeepTheSocketAddress(t *testing.T) {
for _, header := range []string{
"PROXY UNKNOWN\r\n",
"PROXY UNKNOWN " + strings.Repeat("x", 91) + "\r\n", // 107 bytes
v2(0x20, 0x11, block("198.51.100.1", "192.0.2.1")),
v2(0x20, 0x00, nil),
v2(0x21, 0x00, nil),
v2(0x21, 0x12, block("198.51.100.1", "192.0.2.1")),
v2(0x21, 0x31, make([]byte, 216)),
} {
client, found, rest, err := parseHeader(header + "rest")
if err != nil || !found || client.IsValid() || rest != "rest" {
t.Errorf("%q: client %v found %v rest %q: %v", header, client, found, rest, err)
}
}
}
// A malformed, oversized or truncated header is an error, never a client
func TestMalformedHeaderIsRefused(t *testing.T) {
inet := v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1"))
for _, header := range []string{
"PROXY UNKNOWN 198.51.100.1 192.0.2.1 40000 443\n",
"PROXY UNKNOWN " + strings.Repeat("x", 92) + "\r\n",
"PROXY TCP4 2001:db8::1 192.0.2.1 40000 443\r\n",
"PROXY TCP4 198.51.100.1 2001:db8::2 40000 443\r\n",
"PROXY TCP6 198.51.100.1 192.0.2.1 40000 443\r\n",
"PROXY TCP4 198.51.100.1 192.0.2.1 70000 443\r\n",
"PROXY TCP4 198.51.100.1 192.0.2.1 40000\r\n",
"PROXY TCP6 fe80::1%eth0 fe80::2 40000 443\r\n",
"PROXY UDP4 198.51.100.1 192.0.2.1 40000 443\r\n",
"PROXY TCP4 198.51.100.1 192.0.2.1 40000 443",
"PROX",
v2(0x11, 0x11, block("198.51.100.1", "192.0.2.1")),
v2(0x22, 0x11, block("198.51.100.1", "192.0.2.1")),
v2(0x21, 0x41, block("198.51.100.1", "192.0.2.1")),
v2(0x21, 0x11, make([]byte, v2Max+1)),
v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")[:8]) + "rest",
inet[:14],
inet[:20],
} {
if client, found, _, err := parseHeader(header); err == nil || found {
t.Errorf("%q: client %v found %v, no error", header, client, found)
}
}
}
// A peer outside proxy_from that sends a header is refused at its first
// read, as the header is spoofable; its other bytes pass untouched
func TestHeaderFromUnlistedPeerIsRefused(t *testing.T) {
p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "optional", ProxyFrom: []string{"10.0.0.1"}})
ln, dial := listening(t, p)
for _, c := range []struct {
send string
refused bool
}{
{"PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\n", true},
{v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")), true},
{"POST / HTTP/1.1\r\n", false},
} {
go dial("192.0.2.9").Write([]byte(c.send))
b := make([]byte, len(c.send))
_, err := io.ReadFull(accept(t, ln), b)
if (err != nil) != c.refused || !c.refused && string(b) != c.send {
t.Errorf("%q from an unlisted peer: read %q, %v", c.send, b, err)
}
}
if n := p.Denied(); n != 2 {
t.Fatalf("acl_denied = %d", n)
}
}
// Under required a peer in proxy_from must send a header; under optional
// one that sends none keeps its socket address and its bytes
func TestRequiredRefusesAListedPeerWithoutHeader(t *testing.T) {
for mode, refused := range map[string]bool{"required": true, "optional": false} {
p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: mode, ProxyFrom: []string{"127.0.0.1"}})
ln, dial := listening(t, p)
peer := dial("127.0.0.1")
peer.Write([]byte("GET /"))
if refused {
closed(t, peer)
continue
}
conn := accept(t, ln)
b := make([]byte, 5)
if _, err := io.ReadFull(conn, b); err != nil || string(b) != "GET /" || conn.RemoteAddr().String() != "127.0.0.1:40000" {
t.Errorf("optional: %s read %q: %v", conn.RemoteAddr(), b, err)
}
}
}
// The client a header names is what the rules, RemoteAddr and the stats see;
// the proxy stays as PeerAddr, under TLS too
func TestRulesSeeTheClientAHeaderNames(t *testing.T) {
p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "required", ProxyFrom: []string{"127.0.0.1"}, Deny: []string{"198.51.100.2"}})
ln, dial := listening(t, p)
denied := dial("127.0.0.1")
denied.Write([]byte("PROXY TCP4 198.51.100.2 192.0.2.1 40000 443\r\n"))
go dial("127.0.0.1").Write([]byte("PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\nhello"))
conn := accept(t, ln)
b := make([]byte, 5)
if _, err := io.ReadFull(conn, b); err != nil || string(b) != "hello" {
t.Fatalf("read %q: %v", b, err)
}
if got, peer := conn.RemoteAddr().String(), PeerAddr(tls.Server(conn, nil)); got != "198.51.100.1:40000" || peer != "127.0.0.1:40000" {
t.Fatalf("RemoteAddr %s, PeerAddr %q", got, peer)
}
closed(t, denied)
if s := p.Stats(); s["acl_proxy_headers"] != uint64(2) || s["acl_denied"] != uint64(1) {
t.Fatalf("stats %v", s)
}
}
// A header still on its way holds only its own connection, and is refused
// at the deadline
func TestSlowHeaderHoldsOnlyItsConnection(t *testing.T) {
p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "required", ProxyFrom: []string{"127.0.0.1"}})
p.headerWait = time.Second
ln, dial := listening(t, p)
slow := dial("127.0.0.1")
slow.Write([]byte("PROXY TCP4 198"))
dial("192.0.2.9")
if got := accept(t, ln).RemoteAddr().String(); !strings.HasPrefix(got, "192.0.2.9:") || p.Denied() != 0 {
t.Fatalf("Accept returned %s after %d refusals", got, p.Denied())
}
closed(t, slow)
if n := p.Denied(); n != 1 {
t.Fatalf("acl_denied = %d", n)
}
}
// Close also closes the connections whose header has not arrived
func TestCloseReleasesHeadersInFlight(t *testing.T) {
p := mustNew(t, "127.0.0.1", config.ACLOptions{ProxyProtocol: "required", ProxyFrom: []string{"127.0.0.1"}})
ln, dial := listening(t, p)
slow := dial("127.0.0.1")
slow.Write([]byte("PROXY "))
ln.Close()
closed(t, slow)
if _, err := ln.Accept(); !errors.Is(err, net.ErrClosed) {
t.Fatalf("Accept after Close: %v", err)
}
}
// Whatever the bytes, the parser neither panics nor reads past a header's
// bound, consumes nothing when no header starts the stream, and names a
// client only unmapped and without a zone.
func FuzzReadHeader(f *testing.F) {
for _, seed := range []string{
"PROXY TCP4 198.51.100.1 192.0.2.1 40000 443\r\nrest",
"PROXY TCP6 2001:db8::1 2001:db8::2 40000 443\r\n",
"PROXY UNKNOWN\r\n",
v2(0x21, 0x11, block("198.51.100.1", "192.0.2.1")),
v2(0x21, 0x21, block("2001:db8::1", "2001:db8::2", 0x04, 0x00, 0x01, 0xff)),
v2(0x20, 0x00, nil),
"GET / HTTP/1.1\r\n",
} {
f.Add([]byte(seed))
}
f.Fuzz(func(t *testing.T, data []byte) {
r := bytes.NewReader(data)
br := bufio.NewReaderSize(r, v1Max)
client, found, err := readHeader(br)
consumed := len(data) - br.Buffered() - r.Len()
a := client.Addr()
switch {
case consumed > 16+v2Max:
t.Fatalf("consumed %d bytes", consumed)
case err == nil && !found && consumed != 0:
t.Fatalf("consumed %d bytes with no header", consumed)
case client.IsValid() && (!found || a.Is4In6() || a.Zone() != ""):
t.Fatalf("client %v, found %v", client, found)
}
})
}
func parseHeader(s string) (client netip.AddrPort, found bool, rest string, err error) {
br := bufio.NewReaderSize(strings.NewReader(s), v1Max)
client, found, err = readHeader(br)
b, _ := io.ReadAll(br)
return client, found, string(b), err
}
// v2 is a PROXY v2 header: version and command, family, then the block
func v2(cmd, fam byte, block []byte) string {
h := append(slices.Clone(sigV2), cmd, fam, byte(len(block)>>8), byte(len(block)))
return string(append(h, block...))
}
// block is a v2 address block from src:40000 to dst:443, then tlv
func block(src, dst string, tlv ...byte) []byte {
b := append(netip.MustParseAddr(src).AsSlice(), netip.MustParseAddr(dst).AsSlice()...)
return append(append(b, 0x9c, 0x40, 0x01, 0xbb), tlv...)
}
// listening runs p's listener over a fake one; dial connects a peer from
// addr:40000 and returns its end
func listening(t *testing.T, p *Policy) (net.Listener, func(addr string) net.Conn) {
inner := &fakeListener{conns: make(chan net.Conn, 4)}
ln := p.Listener(inner)
t.Cleanup(func() { ln.Close() })
return ln, func(addr string) net.Conn {
server, client := net.Pipe()
t.Cleanup(func() { client.Close() })
inner.conns <- remoteConn{server, net.TCPAddrFromAddrPort(netip.AddrPortFrom(netip.MustParseAddr(addr), 40000))}
return client
}
}
func accept(t *testing.T, ln net.Listener) net.Conn {
t.Helper()
got := make(chan net.Conn, 1)
go func() { c, _ := ln.Accept(); got <- c }()
select {
case c := <-got:
if c == nil {
t.Fatal("Accept failed")
}
t.Cleanup(func() { c.Close() })
c.SetReadDeadline(time.Now().Add(5 * time.Second))
return c
case <-time.After(5 * time.Second):
t.Fatal("Accept returned nothing")
}
return nil
}
// closed waits for the listener to close peer's connection
func closed(t *testing.T, peer net.Conn) {
t.Helper()
peer.SetReadDeadline(time.Now().Add(5 * time.Second))
if _, err := peer.Read(make([]byte, 1)); err != io.EOF {
t.Fatalf("the peer reads %v, want EOF", err)
}
}
type fakeListener struct {
net.Listener
conns chan net.Conn
once sync.Once
}
func (f *fakeListener) Accept() (net.Conn, error) { return <-f.conns, nil }
func (f *fakeListener) Accept() (net.Conn, error) {
if c, ok := <-f.conns; ok {
return c, nil
}
return nil, net.ErrClosed
}
func (f *fakeListener) Close() error { f.once.Do(func() { close(f.conns) }); return nil }
type remoteConn struct {
net.Conn
@@ -119,7 +408,3 @@ type remoteConn struct {
}
func (c remoteConn) RemoteAddr() net.Addr { return c.remote }
func remoteAt(c net.Conn, addr string) net.Conn {
return remoteConn{c, net.TCPAddrFromAddrPort(netip.AddrPortFrom(netip.MustParseAddr(addr), 40000))}
}
+4
View File
@@ -280,6 +280,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error {
h.server = &http.Server{
Handler: handler,
ReadHeaderTimeout: HTTPReadHeaderTimeout,
ConnContext: netacl.ConnContext,
// WriteTimeout unset by design: SSE responses are long-lived.
// net/http bounds the TLS handshake by min(ReadHeaderTimeout,
// ReadTimeout, WriteTimeout), so ReadHeaderTimeout covers it here.
@@ -434,6 +435,9 @@ func (h *HTTPSink) handleStream(w http.ResponseWriter, r *http.Request) {
meta := map[string]any{
"type": "http_client",
}
if peer := netacl.ContextPeerAddr(r.Context()); peer != "" {
meta["peer_addr"] = peer
}
if r.TLS != nil {
meta["tls"] = true
if cn := tlsx.PeerCN(*r.TLS); cn != "" {
+21
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"io"
"maps"
"net"
"net/http"
"net/http/httptest"
"slices"
@@ -347,6 +348,26 @@ func TestProxyModeRefusesDirectPeers(t *testing.T) {
}
}
// A PROXY header names the client a stream's session records; the proxy
// stays as its peer_addr
func TestProxiedStreamSessionKeepsTheProxy(t *testing.T) {
h, manager := newTestHTTPSink(t, map[string]any{"acl": map[string]any{"proxy_protocol": "required", "proxy_from": []any{"127.0.0.1"}}})
_, baseURL := serveTestHTTPSink(t, h)
conn, err := net.Dial("tcp4", strings.TrimPrefix(baseURL, "http://"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { conn.Close() })
fmt.Fprint(conn, "PROXY TCP4 198.51.100.7 127.0.0.1 40000 8081\r\nGET /stream HTTP/1.1\r\nHost: logwisp\r\n\r\n")
if _, err := http.ReadResponse(bufio.NewReader(conn), nil); err != nil { // its body is the stream
t.Fatal(err)
}
sessions := manager.GetActiveSessions()
if len(sessions) != 1 || sessions[0].RemoteAddr != "198.51.100.7:40000" || sessions[0].Metadata["peer_addr"] != conn.LocalAddr().String() {
t.Fatalf("sessions %+v, proxy %s", sessions, conn.LocalAddr())
}
}
// The constructor refuses what it cannot serve: the login pages outside
// scram's proxy mode, a viewer without the login page, a path under /auth, and
// a path ServeMux or a URL would not take literally, which panicked at Start.
+3
View File
@@ -371,6 +371,9 @@ func (t *TCPSink) handleConn(conn net.Conn) {
"type": "tcp_client",
"remote_addr": remote,
}
if peer := netacl.PeerAddr(conn); peer != "" {
meta["peer_addr"] = peer
}
var tlsState *tls.ConnectionState
if tc, ok := conn.(*tls.Conn); ok {
hctx, cancel := context.WithTimeout(context.Background(), tlsx.HandshakeTimeout)
+6 -2
View File
@@ -200,6 +200,7 @@ func (s *HTTPChainSource) Start() error {
Handler: mux,
ReadTimeout: time.Duration(s.config.ReadTimeoutMS) * time.Millisecond,
ReadHeaderTimeout: HTTPChainReadHeaderTimeout,
ConnContext: netacl.ConnContext,
// TLS handshake bounded by min(ReadTimeout, ReadHeaderTimeout)
ErrorLog: tlsx.HTTPErrorLog(s.logger, "http_chain_source"),
}
@@ -371,7 +372,7 @@ func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) {
for _, entry := range entries {
s.publish(entry)
}
s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode, r.TLS, ident))
s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode, netacl.ContextPeerAddr(r.Context()), r.TLS, ident))
w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries)))
w.WriteHeader(http.StatusNoContent)
@@ -399,7 +400,7 @@ func (s *HTTPChainSource) handleAuth(w http.ResponseWriter, r *http.Request) {
// sessionFor returns the cached session for a remote+node+identity,
// recreating after idle expiry. Identity is part of the key so two peers
// sharing a remote address never share a session.
func (s *HTTPChainSource) sessionFor(remoteHost, node string, cs *tls.ConnectionState, ident authz.Identity) string {
func (s *HTTPChainSource) sessionFor(remoteHost, node, peer string, cs *tls.ConnectionState, ident authz.Identity) string {
key := remoteHost + "|" + node + "|" + ident.Name
s.sessionsMu.Lock()
defer s.sessionsMu.Unlock()
@@ -413,6 +414,9 @@ func (s *HTTPChainSource) sessionFor(remoteHost, node string, cs *tls.Connection
"type": "http_chain",
"node": node,
}
if peer != "" {
meta["peer_addr"] = peer
}
if cs != nil {
meta["tls"] = true
if cn := tlsx.PeerCN(*cs); cn != "" {
+3
View File
@@ -366,6 +366,9 @@ func (s *TCPChainSource) handleConn(conn net.Conn) {
"type": "tcp_chain",
"node": connNode,
}
if peer := netacl.PeerAddr(conn); peer != "" {
meta["peer_addr"] = peer
}
if tlsState != nil {
meta["tls"] = true
if cn := tlsx.PeerCN(*tlsState); cn != "" {
+175 -10
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env bash
# logwisp acl test: each listener kind admits or refuses peers by address
# before TLS, counts and warns of refusals, rejects rules it can never apply,
# and still hands a finite input whole to the clients it admits. One-shot.
# Ports: 15881-15889 on 127.0.0.1. Requires: bash 5+, curl, nc; Linux.
# before TLS, or by the client a PROXY header names (stub proxy, nginx if
# installed), counts and warns of refusals, and throttles logins per client.
# One-shot. Ports: 15881-15889 on 127.0.0.1, the proxies' on 127.0.0.11-14.
# Requires: bash 5+, curl, nc, go; Linux.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
@@ -12,11 +13,72 @@ AUTH=$RUN
e2e_init "$@"
section "Setup"
need curl nc
ports_free 15881 15882 15884 15885 15889
need curl nc go
ports_free 15881 15882 15883 15884 15885 15886 15887 15888 15889
rm -rf "$RUN"
mkdir -p "$RUN/out" "$LOG"
mkdir -p "$RUN/out" "$RUN/proxy" "$LOG"
cd "$RUN" || abort "no run directory"
# Each ROUTE is LISTEN,BACKEND,VERSION,CLIENT: a connection to LISTEN reaches
# BACKEND behind a PROXY header of VERSION naming CLIENT, with its own port
cat >"$RUN/proxy/main.go" <<'EOF'
package main
import (
"encoding/binary"
"fmt"
"io"
"log"
"net"
"net/netip"
"os"
"strings"
)
func main() {
for _, route := range os.Args[1:] {
f := strings.Split(route, ",")
ln, err := net.Listen("tcp4", f[0])
if err != nil {
log.Fatal(err)
}
go func() {
for {
c, err := ln.Accept()
if err != nil {
log.Fatal(err)
}
go relay(c, f[1], f[2], netip.MustParseAddr(f[3]))
}
}()
}
fmt.Println("ready")
select {}
}
func relay(c net.Conn, backend, version string, client netip.Addr) {
defer c.Close()
b, err := net.Dial("tcp4", backend)
if err != nil {
return
}
defer b.Close()
src := netip.AddrPortFrom(client, c.RemoteAddr().(*net.TCPAddr).AddrPort().Port())
dst := b.RemoteAddr().(*net.TCPAddr).AddrPort()
if version == "1" {
fmt.Fprintf(b, "PROXY TCP4 %s %s %d %d\r\n", src.Addr(), dst.Addr(), src.Port(), dst.Port())
} else {
h := append([]byte("\r\n\r\n\x00\r\nQUIT\n\x21\x11\x00\x0c"), src.Addr().AsSlice()...)
h = binary.BigEndian.AppendUint16(append(h, dst.Addr().AsSlice()...), src.Port())
b.Write(binary.BigEndian.AppendUint16(h, dst.Port()))
}
go func() { io.Copy(b, c); b.(*net.TCPConn).CloseWrite() }()
io.Copy(c, b)
}
EOF
if ! (cd "$RUN/proxy" && go build -o proxy main.go) >"$LOG/proxy-build.out" 2>&1; then
tail -n 5 "$LOG/proxy-build.out" | sed 's/^/ /'
abort "the stub proxy did not build"
fi
for v in $(compgen -e LOGWISP_); do unset "$v"; done
export HOME=$RUN
lw() { "$BIN" "$@"; }
@@ -25,7 +87,8 @@ get() { curl -s --noproxy '*' --max-time 3 "$@"; } # curl ARGS...: never through
get --interface 127.0.0.2 http://127.0.0.1:15889/
(($? == 45)) && skip_all "this host cannot send from 127.0.0.2"
seq 1 300 | sed 's/^/edge line /' >"$RUN/edge.log"
add_users "$RUN/users.toml" edge-01
add_users "$RUN/users.toml" edge-01 viewer-01
echo wrong >"$RUN/wrong.pass"
section "Rules"
# rejects SINK PATTERN: lw --check fails on SINK with PATTERN in its error
@@ -37,6 +100,10 @@ check "an acl on a dialer is refused" "$(rejects 'tcp_chain,host=127.0.0.1,port=
check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp,host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')"
check "a deny list alone on a wildcard listener warns" "$(warns 'tcp,port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')"
check "an allow entry of a whole family warns" "$(warns 'tcp,port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')"
check "a proxy_from entry reaching public addresses warns" \
"$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=203.0.113.0/24' 'proxy_from reaches public addresses')"
check "optional proxy_protocol warns that a headerless client passes as the proxy" \
"$(warns 'tcp,host=127.0.0.1,port=15881,acl.proxy_protocol=optional,acl.proxy_from=127.0.0.1' 'passes as the proxy')"
section "HTTP sink"
spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2"
@@ -71,12 +138,12 @@ section "Chain sources"
# pin DAEMON: the pin_sha256 a self-signed listener logged
pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; }
received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; }
edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; }
edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; }
spawn agg-allow "$BIN" --preset "aggregator,listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow"
spawn agg-deny "$BIN" --preset "aggregator,listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny"
wait_port 15884 && wait_port 15885 || abort "an aggregator did not listen" agg-allow
edge edge-allow 15884 tcp "$(pin agg-allow)"
edge edge-deny 15885 http "$(pin agg-deny)"
edge edge-allow 127.0.0.1:15884 tcp "$(pin agg-allow)"
edge edge-deny 127.0.0.1:15885 http "$(pin agg-deny)"
timeout 3 nc -s 127.0.0.2 127.0.0.1 15884 </dev/null
wait_until 10 eval '(( $(received allow) >= 300 ))'
check "a tcp_chain source admits an edge its allow list names ($(received allow)), and refuses 127.0.0.2" \
@@ -85,4 +152,102 @@ wait_until 10 logged agg-deny 'refused by acl'
check "an http_chain source refuses a denied edge, which delivers nothing ($(received deny))" \
"$(logged agg-deny 'refused by acl' && (($(received deny) == 0)) && echo 1 || echo 0)"
section "Behind a PROXY header"
# Every listener here takes headers from 127.0.0.1 only, where the proxies
# connect from; each is up before a proxy shares its port on another address
behind=acl.proxy_protocol=required,acl.proxy_from=127.0.0.1
scram="tls.enabled=true,tls.self_signed=true,auth.type=scram,auth.credentials_file=$RUN/users.toml"
spawn sink-behind "$BIN" --logging.level=info --source null --sink "http,host=127.0.0.1,port=15886,$scram,$behind,acl.deny=198.51.100.2"
for t in tcp http; do
spawn "agg-$t-behind" "$BIN" --logging.level=info \
--source "${t}_chain,host=127.0.0.1,port=$([[ $t == tcp ]] && echo 15888 || echo 15883),$scram,$behind" \
--sink "file,directory=$RUN/out/$t-behind,name=aggregate"
done
# the tcp sink sends its input once the clients connected
{ wait_until 30 test -e "$RUN/feed"; seq 1 100 | sed 's/^/behind /'; } |
lw --sink "tcp,host=127.0.0.1,port=15887,$behind,acl.deny=198.51.100.2,acl.deny=127.0.0.6" 2>"$LOG/tcp-behind.out" &
fpid=$!
for p in 15883 15886 15887 15888; do wait_port $p || abort "a listener behind the proxy did not listen on $p" sink-behind; done
spawn proxy "$RUN/proxy/proxy" 127.0.0.11:15886,127.0.0.1:15886,1,198.51.100.1 \
127.0.0.12:15886,127.0.0.1:15886,2,198.51.100.2 127.0.0.13:15886,127.0.0.1:15886,2,198.51.100.3 \
127.0.0.11:15887,127.0.0.1:15887,2,198.51.100.1 127.0.0.12:15887,127.0.0.1:15887,1,198.51.100.2 \
127.0.0.11:15888,127.0.0.1:15888,1,198.51.100.1 127.0.0.13:15883,127.0.0.1:15883,2,198.51.100.3
wait_until 10 logged proxy ready || abort "the stub proxy did not start" proxy
NGINX=0
if command -v nginx >/dev/null; then
mkdir -p "$RUN/nginx/logs"
# a dynamic stream module loads by its absolute path; a built-in has none
so=$(nginx -V 2>&1 | grep -o -- '--modules-path=[^ ]*' | cut -d= -f2)/ngx_stream_module.so
modules=$([[ -f $so ]] && echo "load_module $so;")
cat >"$RUN/nginx/nginx.conf" <<EOF
daemon off;
pid $RUN/nginx/nginx.pid;
error_log $RUN/nginx/logs/error.log;
$modules
events {}
stream {
server {
listen 127.0.0.14:15887;
proxy_pass 127.0.0.1:15887;
proxy_protocol on;
}
}
EOF
spawn nginx nginx -p "$RUN/nginx" -c "$RUN/nginx/nginx.conf"
wait_until 5 test -s "$RUN/nginx/nginx.pid" && NGINX=1
fi
# tcp sink: one client per route, then the input
listen_via() { timeout 10 nc "${@:2}" 15887 < <(sleep 12) >"$RUN/$1.out"; }
listen_via admitted 127.0.0.11 &
pids=($!)
listen_via denied 127.0.0.12 &
pids+=($!)
if ((NGINX)); then
listen_via nginx -s 127.0.0.5 127.0.0.14 &
pids+=($!)
listen_via nginx-denied -s 127.0.0.6 127.0.0.14 &
pids+=($!)
fi
sleep 1
touch "$RUN/feed"
wait "$fpid" "${pids[@]}"
lines() { grep -c '^behind ' "$RUN/$1.out"; }
check "a tcp sink streams to the client a v2 header names ($(lines admitted) lines)" "$(($(lines admitted) == 100))"
check "and refuses one a v1 header names, logging both addresses ($(lines denied) lines)" \
"$( (($(lines denied) == 0)) && grep -q 'refused by acl.* remote_addr 198.51.100.2:.*peer_addr 127.0.0.1:' "$LOG/tcp-behind.out" && echo 1 || echo 0)"
if ((NGINX)); then
check "nginx stream with proxy_protocol on names its clients ($(lines nginx) lines, $(lines nginx-denied) to a denied one)" \
"$( (($(lines nginx) == 100 && $(lines nginx-denied) == 0)) && echo 1 || echo 0)"
else
skip "nginx stream with proxy_protocol on (nginx with its stream module is not available)"
fi
# http sink: each client its own login budget behind the one proxy address
PIN=$(pin sink-behind)
token() { lw auth token --url "https://$1:15886" -u viewer-01 --password-file "$2" --pin-sha256 "$PIN" 2>&1; }
token 127.0.0.12 "$RUN/viewer-01.pass" >/dev/null
check "an http sink refuses the client a v2 header names, logging both addresses" \
"$(logged sink-behind 'refused by acl.* remote_addr 198.51.100.2:.*peer_addr 127.0.0.1:' && echo 1 || echo 0)"
for ((i = 1; i <= 30; i++)); do
token 127.0.0.11 "$RUN/wrong.pass" >/dev/null
logged sink-behind ' throttled' && break
done
check "wrong passwords throttle a client by the address its v1 header names ($i attempts)" \
"$(logged sink-behind 'Login rejected.* remote_addr 198.51.100.1:.* 198.51.100.1 throttled' && echo 1 || echo 0)"
check "while another behind the same proxy still logs in" \
"$(token 127.0.0.13 "$RUN/viewer-01.pass" >/dev/null && logged sink-behind 'Login accepted.* remote_addr 198.51.100.3:' && echo 1 || echo 0)"
# chain sources: an edge through the proxy is the client its header names
edge edge-tcp-behind 127.0.0.11:15888 tcp "$(pin agg-tcp-behind)"
edge edge-http-behind 127.0.0.13:15883 http "$(pin agg-http-behind)"
printf 'PROXY TCP4 198.51.100.9 127.0.0.1 40000 15888\r\n' | timeout 3 nc -s 127.0.0.2 127.0.0.1 15888
wait_until 10 eval '(( $(received tcp-behind) >= 300 && $(received http-behind) >= 300 ))'
check "a tcp_chain source takes an edge as the client a v1 header names ($(received tcp-behind))" \
"$( (($(received tcp-behind) == 300)) && logged agg-tcp-behind 'Chain connection established.* remote_addr 198.51.100.1:' && echo 1 || echo 0)"
check "an http_chain source takes one as the client a v2 header names ($(received http-behind))" \
"$( (($(received http-behind) == 300)) && logged agg-http-behind 'Login accepted.* remote_addr 198.51.100.3:' && echo 1 || echo 0)"
check "a header from a peer outside proxy_from is refused" \
"$(logged agg-tcp-behind 'refused by acl.* remote_addr 127.0.0.2:.* reason "PROXY header from a peer outside proxy_from"' && echo 1 || echo 0)"
summary