v0.25.0 netacl and fixes
This commit is contained in:
lixen committed
1 parent
9e86d7999d
commit
4af072f4e5
30 files changed
+894
-204
No files matched your search
@@ -80,11 +80,13 @@ change touches.
|
||||
table whose groups (`auth`, `tls`, `preset`) share one flag mechanism;
|
||||
`config.Load` takes the parsed `config.Args`. Presets are rows of one table
|
||||
in `internal/config/preset.go`.
|
||||
- Network security has two seams: `tlsx` (TLS configs, certificates made at
|
||||
startup or by `lw tls`, pins, startup warnings) and
|
||||
`authz`. Listeners admit only through `Admit`/`AuthorizeRequest`, dialers
|
||||
through `Greet`/`Prepare`; `Authorize` refuses under scram, so a plugin gated
|
||||
on it alone fails closed. Nil policies are valid: keep call sites branch-free.
|
||||
- Network security has three seams: `tlsx` (TLS configs, certificates made at
|
||||
startup or by `lw tls`, pins, startup warnings), `authz`, and `netacl`
|
||||
(address rules: a listener wraps the socket `core.Listen` returns in
|
||||
`Policy.Listener`, before TLS). Listeners admit only through
|
||||
`Admit`/`AuthorizeRequest`, dialers through `Greet`/`Prepare`; `Authorize`
|
||||
refuses under scram, so a plugin gated on it alone fails closed. Nil policies
|
||||
are valid: keep call sites branch-free.
|
||||
- A plugin starts goroutines in `Start`, never in its constructor (the SCRAM
|
||||
server too: `Policy.Start`/`Close`): a rejected reload discards constructed
|
||||
plugins without calling `Stop`. Reload is the only rotation path for
|
||||
|
||||
+2
-2
@@ -123,9 +123,9 @@ func TestEveryPresetBuilds(t *testing.T) {
|
||||
samples := map[string]string{
|
||||
"pipe": "pipe,format=txt",
|
||||
"tail": "tail,path=" + dir,
|
||||
"serve": "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users,
|
||||
"serve": "serve,path=" + filepath.Join(dir, "*.log") + ",listen=127.0.0.1:15841,tls=self,users=" + users + ",allow=127.0.0.0/8,deny=127.0.0.2",
|
||||
"edge": "edge,to=127.0.0.1:15842,transport=http,pin=" + pin + ",user=edge-01,password_file=" + pass,
|
||||
"aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out"),
|
||||
"aggregator": "aggregator,listen=127.0.0.1:15843,users=" + users + ",out=" + filepath.Join(dir, "out") + ",allow=10.0.0.0/8,allow=127.0.0.1",
|
||||
}
|
||||
for _, p := range config.Presets() {
|
||||
sample, ok := samples[p.Name]
|
||||
|
||||
+10
-4
@@ -87,16 +87,21 @@ func TestInvalidReloadLeavesStatusReporterRunning(t *testing.T) {
|
||||
}
|
||||
|
||||
// Every registered plugin decodes through the checked decoder, so a typo in
|
||||
// any plugin's config, security tables included, refuses to start.
|
||||
// any plugin's config, security tables included, refuses to start: a typo in
|
||||
// a listener's acl table, and an acl table on a plugin that listens on nothing.
|
||||
func TestEveryPluginRejectsUnknownKeys(t *testing.T) {
|
||||
testLogger(t)
|
||||
manager := session.NewManager(time.Hour)
|
||||
t.Cleanup(manager.Stop)
|
||||
proxy := session.NewProxy(manager, "typo")
|
||||
bad := map[string]any{"no_such_key": true}
|
||||
// the error names the key: acl.no_such_key on a listener, acl elsewhere
|
||||
for key, bad := range map[string]map[string]any{
|
||||
`"no_such_key"`: {"no_such_key": true},
|
||||
`"acl`: {"acl": map[string]any{"no_such_key": true}},
|
||||
} {
|
||||
check := func(kind, name string, err error) {
|
||||
if err == nil || !strings.Contains(err.Error(), "no_such_key") {
|
||||
t.Errorf("%s %q accepted an unknown key: %v", kind, name, err)
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown key "+key) {
|
||||
t.Errorf("%s %q accepted %v: %v", kind, name, bad, err)
|
||||
}
|
||||
}
|
||||
for _, name := range plugin.ListSources() {
|
||||
@@ -110,6 +115,7 @@ func TestEveryPluginRejectsUnknownKeys(t *testing.T) {
|
||||
check("sink", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A misspelled spec key reaches the plugin as the nested key it names, so the
|
||||
// sink refuses to build instead of starting without the TLS it was asked for.
|
||||
|
||||
+27
-6
@@ -6,12 +6,12 @@
|
||||
### Commented values are the built-in defaults unless marked "example".
|
||||
### Uncommenting a default is a no-op.
|
||||
###
|
||||
### NOTE: network access control (ACL), http/tcp ingest sources, and
|
||||
### http_client/tcp_client sinks are not available in this version. TLS is
|
||||
### available on every network source and sink (the [...tls] blocks below), and
|
||||
### the [...auth] blocks authenticate peers by certificate (mtls) or password
|
||||
### (scram; credentials via `lw auth`). Unknown keys anywhere in this file
|
||||
### or in any plugin config are rejected at startup.
|
||||
### NOTE: http/tcp ingest sources and http_client/tcp_client sinks are not
|
||||
### available in this version. TLS is available on every network source and
|
||||
### sink (the [...tls] blocks below), the [...auth] blocks authenticate peers by
|
||||
### certificate (mtls) or password (scram; credentials via `lw auth`), and the
|
||||
### listeners' [...acl] blocks admit peers by address. Unknown keys anywhere in
|
||||
### this file or in any plugin config are rejected at startup.
|
||||
###
|
||||
### Environment overrides use the LOGWISP_ prefix
|
||||
### (LOGWISP_QUIET, LOGWISP_LOGGING_LEVEL, ...). Bare names are ignored.
|
||||
@@ -158,6 +158,15 @@ name = "default"
|
||||
### (addresses or CIDRs); browsers log in through them
|
||||
###============================================================================
|
||||
|
||||
###============================================================================
|
||||
### ACL (beside [...tls] and [...auth] on the listeners; dialers have none)
|
||||
###
|
||||
### Applied as a connection is accepted, before TLS: a refused peer is closed
|
||||
### at once and counted in acl_denied, with a WARN at most once a minute.
|
||||
### allow = [] addresses or CIDRs of the listener's family; empty = all
|
||||
### deny = [] refused, listed in allow or not
|
||||
###============================================================================
|
||||
|
||||
###============================================================================
|
||||
### Sources (1+ required)
|
||||
###============================================================================
|
||||
@@ -224,6 +233,9 @@ from = "end" # "end" or "start" of a newly discovered
|
||||
# allow_patterns = [] # mtls: RE2, anchor them yourself
|
||||
# node_binding = "force" # none | assert | force; overrides trust_node
|
||||
# credentials_file = "" # scram: written by `lw auth add-user`
|
||||
# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS
|
||||
# allow = [] # empty = every address deny does not list
|
||||
# deny = []
|
||||
|
||||
## HTTP chain source (stdlib listener; receives NDJSON batches from upstream http_chain sinks)
|
||||
# [[pipelines.plugin_sources]]
|
||||
@@ -251,6 +263,9 @@ from = "end" # "end" or "start" of a newly discovered
|
||||
# node_binding = "force" # none | assert | force; overrides trust_node
|
||||
# credentials_file = "" # scram: written by `lw auth add-user`
|
||||
# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes
|
||||
# [pipelines.plugin_sources.config.acl] # refuse peers by address, before TLS
|
||||
# allow = [] # empty = every address deny does not list
|
||||
# deny = []
|
||||
|
||||
###============================================================================
|
||||
### Sinks (1+ required, fan-out)
|
||||
@@ -316,6 +331,9 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
|
||||
# credentials_file = "" # scram: log in with `lw auth token`
|
||||
# token_lifetime_ms = 0 # scram bearer tokens, 10 s to 24 h; 0 = 15 minutes
|
||||
# trusted_proxies = [] # scram: proxies ending the browsers' TLS; tls may then be off
|
||||
# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS
|
||||
# allow = [] # empty = every address deny does not list
|
||||
# deny = []
|
||||
|
||||
## TCP sink (streaming server)
|
||||
# [[pipelines.plugin_sinks]]
|
||||
@@ -342,6 +360,9 @@ target = "stdout" # stdout|stderr ("split" NOT supported)
|
||||
# allow = [] # mtls: exact identities; empty = any the CA issued
|
||||
# allow_patterns = [] # mtls: RE2, anchor them yourself
|
||||
# credentials_file = "" # scram: readers use `lw auth stream`
|
||||
# [pipelines.plugin_sinks.config.acl] # refuse peers by address, before TLS
|
||||
# allow = [] # empty = every address deny does not list
|
||||
# deny = []
|
||||
|
||||
## TCP chain sink (stdlib client; forwards to downstream tcp_chain source)
|
||||
## Do NOT point a chain sink at a chain source in the SAME pipeline: entries
|
||||
|
||||
+1
-1
@@ -120,7 +120,7 @@ Not covered by this script; edit the generated configuration instead
|
||||
- filters, rate limits, heartbeat, formatter flags and timestamp layouts
|
||||
- more than one pipeline; both chain transports, or a further hop, on one node
|
||||
- proxy mode: browser logins through a TLS-terminating proxy (trusted_proxies)
|
||||
- network ACLs: not implemented in LogWisp yet
|
||||
- acl address rules on the listeners
|
||||
- mtls identities other than the certificate CN; node_binding other than force
|
||||
- creating certificates: openssl or your PKI (doc/security.md#enabling-mtls)
|
||||
- several native instances on one host: use one per host or jail
|
||||
|
||||
+11
-8
@@ -162,11 +162,11 @@ Pipelines over stdin and stdout are under [Usage Patterns](#usage-patterns).
|
||||
## Presets
|
||||
|
||||
`--preset NAME,key=value,...` takes the [SPEC](#pipelines) syntax. A list
|
||||
key (`hosts`, `proxy`) takes several values, repeated (`hosts=a,hosts=b`) or
|
||||
`,`-separated; any other key takes one. `lw preset NAME [--key value ...]`
|
||||
prints the pipeline a preset expands to, ready for a configuration file; its
|
||||
flags are the keys, spelled as in the SPEC (`--password_file`), and `-u` is
|
||||
`--user`;
|
||||
key (`hosts`, `proxy`, `allow`, `deny`) takes several values, repeated
|
||||
(`hosts=a,hosts=b`) or `,`-separated; any other key takes one.
|
||||
`lw preset NAME [--key value ...]` prints the pipeline a preset expands to,
|
||||
ready for a configuration file; its flags are the keys, spelled as in the SPEC
|
||||
(`--password_file`), and `-u` is `--user`;
|
||||
`lw preset NAME -h` lists its keys and defaults, `lw preset` the presets. An
|
||||
unknown key fails and lists the valid ones. A `path` is a file, a directory
|
||||
(its files) or a glob; a preset that takes one reads stdin without it, and
|
||||
@@ -183,6 +183,8 @@ unknown key fails and lists the valid ones. A `path` is a file, a directory
|
||||
startup), `issuer` (one signed by `issuer_cert` and `issuer_key`), or
|
||||
`files` (`cert` and `key`); `hosts` adds names to a made certificate
|
||||
- without `users` the viewer needs no login, over plain http too
|
||||
- `allow` and `deny`: addresses or CIDRs, the sink's
|
||||
[acl](security.md#the-acl-block)
|
||||
- `users`: a credentials file; readers then log in with SCRAM, and browsers
|
||||
need `proxy` and `viewer=true`: the login page and viewer behind the
|
||||
TLS-terminating proxies `proxy` names, since a browser cannot bind its
|
||||
@@ -198,6 +200,7 @@ unknown key fails and lists the valid ones. A `path` is a file, a directory
|
||||
- `listen` (`0.0.0.0:9000`), `transport` (`tcp`), `format` (`json`)
|
||||
- `tls`: `self` (default), `issuer` or `files`, as for `serve`; never `off`
|
||||
- authenticate by `users` (SCRAM) or `client_ca` (mTLS): one is required
|
||||
- `allow` and `deny`, as for `serve`
|
||||
- `out`: a directory for `aggregate*.log` files; default stdout
|
||||
|
||||
```bash
|
||||
@@ -349,9 +352,9 @@ values from the file, the environment or flags win.
|
||||
finishes when all its sources have ended, and once every pipeline has
|
||||
finished lw shuts down gracefully and exits 0. Only a console source ends on
|
||||
its own: file and network sources run until a signal, and so does a pipeline
|
||||
holding one. Every sink writes what is queued before the exit; the `http`
|
||||
and `tcp` sinks give each connected client `write_timeout_ms`, at most 2 s,
|
||||
to take it.
|
||||
holding one. Every sink writes what is queued before the exit; a network
|
||||
sink gives its clients or its downstream its `write_timeout_ms`
|
||||
(`request_timeout_ms` for `http_chain`), at most 2 s, to take it.
|
||||
|
||||
## Usage Patterns
|
||||
|
||||
|
||||
+1
-1
@@ -184,7 +184,7 @@ every key.
|
||||
- filters, rate limits, heartbeat, formatter flags and timestamp layouts
|
||||
- more than one pipeline; both chain transports, or a further hop, on one node
|
||||
- proxy mode: browser logins through a TLS-terminating proxy
|
||||
- network ACLs: not implemented in LogWisp yet
|
||||
- `acl` address rules on the listeners
|
||||
- mtls identities other than the certificate CN; `node_binding` other than
|
||||
`force`
|
||||
- creating certificates
|
||||
|
||||
@@ -289,6 +289,7 @@ or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target.
|
||||
15831-15832 scram-proxy-test.sh test/run-proxy/
|
||||
15851-15855 ipv6-test.sh test/run-ipv6/
|
||||
15871-15879 deploy-test.sh test/run-deploy/
|
||||
15881-15889 acl-test.sh test/run/acl/
|
||||
```
|
||||
|
||||
- `chain-test.sh`: two edges into a relay, one pipeline per chain transport;
|
||||
@@ -310,3 +311,5 @@ or SKIP (exit 77, e.g. no IPv6 loopback); only a failure fails the target.
|
||||
loopback
|
||||
- `deploy-test.sh`: the configurations `deploy/lw-deploy.sh` writes, deployed
|
||||
and checked; ports 15871-15879
|
||||
- `acl-test.sh`: each listener admitting or refusing peers by address before
|
||||
TLS, with clients from 127.0.0.1 and 127.0.0.2; ports 15881-15889
|
||||
@@ -301,7 +301,7 @@ runs without a configuration.
|
||||
.It Cm tail path Op Cm from , format
|
||||
Follow files to standard output, like
|
||||
.Ql tail \-F .
|
||||
.It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer Oc
|
||||
.It Cm serve Oo Cm path , listen , format , tls , users , proxy , viewer , allow , deny Oc
|
||||
Serve a live HTTP stream on
|
||||
.Cm listen
|
||||
.Pq default Li 127.0.0.1:8080 .
|
||||
@@ -324,6 +324,10 @@ only behind a TLS-terminating proxy named by
|
||||
with
|
||||
.Cm viewer=true ,
|
||||
since a browser cannot bind its login to the TLS channel.
|
||||
.Cm allow
|
||||
and
|
||||
.Cm deny
|
||||
list the addresses or CIDRs the listener admits and refuses.
|
||||
.It Cm edge to Oo Cm path , transport , ca | pin , user , password_file , cert , key , node Oc
|
||||
Forward to an aggregator at
|
||||
.Cm to
|
||||
@@ -340,7 +344,7 @@ or a client
|
||||
.Cm cert
|
||||
and
|
||||
.Cm key .
|
||||
.It Cm aggregator Oo Cm listen , transport , tls , users , client_ca , out , format Oc
|
||||
.It Cm aggregator Oo Cm listen , transport , tls , users , client_ca , out , format , allow , deny Oc
|
||||
Receive from edges on
|
||||
.Cm listen
|
||||
.Pq default Li 0.0.0.0:9000
|
||||
@@ -351,7 +355,12 @@ authenticated by the
|
||||
credentials file or the client certificates
|
||||
.Cm client_ca
|
||||
vouches for, and write to standard output or to files in
|
||||
.Cm out .
|
||||
.Cm out ;
|
||||
.Cm allow
|
||||
and
|
||||
.Cm deny
|
||||
as for
|
||||
.Cm serve .
|
||||
.El
|
||||
.Ss Certificate commands
|
||||
.Nm
|
||||
|
||||
+13
-3
@@ -117,9 +117,10 @@ The `http_chain` source has no connection cap; it bounds work with
|
||||
`max_body_bytes` and `read_timeout_ms` instead.
|
||||
|
||||
Apart from SCRAM logins, which are throttled per address (see
|
||||
[Security](security.md#throttling)), there is **no** per-IP limiting and no IP
|
||||
allow/deny list. `flow.rate_limit` is a pipeline-wide entry rate limit, not a
|
||||
network-level one — it cannot distinguish or throttle an individual peer.
|
||||
[Security](security.md#throttling)), there is **no** per-IP limiting; `acl`
|
||||
admits or refuses peers by address ([Security](security.md#the-acl-block)).
|
||||
`flow.rate_limit` is a pipeline-wide entry rate limit, not a network-level
|
||||
one — it cannot distinguish or throttle an individual peer.
|
||||
|
||||
## Keep-Alive
|
||||
|
||||
@@ -206,6 +207,15 @@ headers, and entry encoding.
|
||||
may resolve to either.
|
||||
- Check the port is not already bound by another pipeline in the same process.
|
||||
|
||||
**Closed as soon as it connects, before TLS**
|
||||
- The listener's `acl` refused the address: `acl_denied` rises, and its log has
|
||||
`Connection refused by acl` at WARN, once a minute at most. The dialer sees
|
||||
`EOF` or `connection reset by peer` (curl: exit 52 or 56).
|
||||
- Behind a proxy (an L4 passthrough, or the `http` sink's
|
||||
`auth.trusted_proxies`) every peer has the proxy's address: allow the proxy.
|
||||
- `acl: allow entry "...": not of the listener's family` at startup — an IPv4
|
||||
entry on an IPv6 `host`, or the reverse: each family's listener takes its own.
|
||||
|
||||
**TLS handshake failure**
|
||||
- `client didn't provide a certificate` — the listener has `client_auth = true`
|
||||
and the dialer has no `cert_file`/`key_file`.
|
||||
|
||||
+4
-4
@@ -17,10 +17,10 @@ lw exits on its own only at the end of input: once every pipeline has
|
||||
finished, a pipeline finishing when all its sources have ended. Only a console
|
||||
source ends, when stdin does; file and network sources run until a signal, and
|
||||
lw shuts down gracefully, exit 0, either way.
|
||||
Every sink writes what is queued before the exit; the `http` and `tcp` sinks
|
||||
give each client `write_timeout_ms`, at most 2 s, to take it. A reload stops
|
||||
the old sinks the same way, so a client that stopped reading delays it by
|
||||
that much. See
|
||||
Every sink writes what is queued before the exit; a network sink gives its
|
||||
clients or its downstream its `write_timeout_ms` (`request_timeout_ms` for
|
||||
`http_chain`), at most 2 s, to take it. A reload stops the old sinks the same
|
||||
way, so a stalled client or downstream delays it by that much. See
|
||||
[CLI](cli.md#built-in-defaults).
|
||||
|
||||
There is no built-in daemon mode. Run LogWisp in the foreground under a
|
||||
|
||||
+48
-6
@@ -26,10 +26,12 @@ configure it, and — equally important — what it does not yet do.
|
||||
[Browsers behind a TLS-terminating proxy](#browsers-behind-a-tls-terminating-proxy).
|
||||
- Server pinning by dialers: certificate identity (`mtls`), bound certificate
|
||||
(`scram`).
|
||||
- Address rules on every listener, applied before TLS: see
|
||||
[The ACL Block](#the-acl-block).
|
||||
- Startup warnings for expiring certificates and risky settings: see
|
||||
[Startup Warnings](#startup-warnings).
|
||||
- Unknown configuration keys rejected: a typo in `tls`, `auth` or a table path
|
||||
fails startup.
|
||||
- Unknown configuration keys rejected: a typo in `tls`, `auth`, `acl` or a
|
||||
table path fails startup.
|
||||
- Configuration input hardened (lixenwraith/config v0.2.2 and
|
||||
lixenwraith/toml v0.1.3, reviewed with adversarial tests and fuzzing like
|
||||
`auth`):
|
||||
@@ -49,8 +51,10 @@ configure it, and — equally important — what it does not yet do.
|
||||
|
||||
- Certificate revocation lists (CRL) or OCSP: revoke by editing the allow-list
|
||||
or credentials file.
|
||||
- IP allow/deny lists, per-IP connection or request limits: only SCRAM logins
|
||||
are throttled per address.
|
||||
- Per-address connection or request limits: only SCRAM logins are throttled
|
||||
per address.
|
||||
- Address rules for clients behind a proxy: `acl` sees the proxy, never the
|
||||
client a PROXY header or `X-Forwarded-For` names.
|
||||
- PROXY protocol: behind a proxy that passes TLS through, every peer shares the
|
||||
proxy's address; see
|
||||
[Behind nginx or another proxy](#behind-nginx-or-another-proxy).
|
||||
@@ -319,6 +323,36 @@ Misconfiguration fails at plugin construction, before the pipeline starts:
|
||||
|
||||
Errors read like `auth: type "mtls" requires tls.client_auth`.
|
||||
|
||||
## The ACL Block
|
||||
|
||||
`acl`, beside `tls` and `auth` on the four listeners (`tcp` and `http` sinks,
|
||||
`tcp_chain` and `http_chain` sources), refuses peers by address as the socket
|
||||
is accepted: before the TLS handshake or a byte of the protocol, so a refused
|
||||
peer costs no handshake and never reaches `auth`.
|
||||
|
||||
```toml
|
||||
[pipelines.plugin_sources.config.acl]
|
||||
allow = ["192.0.2.0/24", "198.51.100.7"] # addresses or CIDRs; empty: all
|
||||
deny = ["192.0.2.66"] # refused, listed in allow or not
|
||||
```
|
||||
|
||||
- `deny` wins; then a non-empty `allow` admits only its entries, an empty one
|
||||
every address `deny` does not list.
|
||||
- Entries are of the listener's [family](networking.md#address-family): IPv4
|
||||
on an IPv4 listener, IPv6 on an IPv6 one, either behind a hostname (only the
|
||||
family it binds ever matches). An entry of the other family, an IPv4-mapped
|
||||
address, one with a zone, or anything but an address or CIDR fails
|
||||
construction. A link-local peer matches without its zone.
|
||||
- A refused connection is closed at once and counted in `acl_denied` (plugin
|
||||
stats, the `http` sink's status). A WARN names the peer and the count, at
|
||||
most once a minute per listener; the refusals in between are only counted.
|
||||
- The rules see the socket peer: behind a proxy (an L4 passthrough, or the
|
||||
`http` sink's `trusted_proxies`), the proxy (see
|
||||
[Behind nginx or another proxy](#behind-nginx-or-another-proxy)).
|
||||
- The `serve` and `aggregator` presets take `allow` and `deny` list keys.
|
||||
- Dialers have no `acl`: on a `tcp_chain` or `http_chain` sink it is an
|
||||
unknown key.
|
||||
|
||||
## Startup Warnings
|
||||
|
||||
Some settings work but are usually mistakes. Each plugin reports them at WARN
|
||||
@@ -334,6 +368,9 @@ when it is constructed, so startup and every reload repeat them:
|
||||
- a `key_file`, `issuer_key_file`, `credentials_file` or `password_file` every
|
||||
local user can read
|
||||
(once per path per process)
|
||||
- an `acl` entry with host bits set (`10.0.0.1/8`), naming the network it
|
||||
matches; an `acl.allow` entry of a whole family (`0.0.0.0/0`, `::/0`); a
|
||||
wildcard listener whose `acl` has `deny` but no `allow`
|
||||
|
||||
## Certificates made at startup
|
||||
|
||||
@@ -816,7 +853,8 @@ stream {
|
||||
(`too many attempts` or `busy`) keeps sending on its token to the server it
|
||||
pinned, retrying every few seconds, and holds its batches only once that
|
||||
token has expired; any other refusal ends the token at once.
|
||||
- Logs and sessions name the proxy, not the peer.
|
||||
- Logs and sessions name the proxy, not the peer, and `acl` rules see only the
|
||||
proxy: allow it, not the clients.
|
||||
- Proxy mode is unaffected: it throttles on the forwarded client.
|
||||
- Until LogWisp reads PROXY, mitigate:
|
||||
- expose the chain ports directly, without the proxy, where you can;
|
||||
@@ -845,6 +883,9 @@ compromised edge cannot attribute its entries to another host, and the `http`
|
||||
sink's stream and status endpoints stop being open to anyone who can reach the
|
||||
port.
|
||||
|
||||
**`acl`** — an address check, per listener, before TLS: it narrows who can
|
||||
reach the other layers, never proves who a peer is.
|
||||
|
||||
**Revocation** is the allow-list or the credentials file, not a CRL. Remove the
|
||||
identity or user and send `SIGHUP`: the reload rebuilds every pipeline, so the
|
||||
change takes effect on the next connection and existing ones are dropped by the
|
||||
@@ -856,7 +897,8 @@ would add.
|
||||
## Surfaces Without Access Control
|
||||
|
||||
An `auth` block (`mtls` or `scram`) closes each of these. Without one, bind them
|
||||
to a trusted interface or front them with an authenticating proxy.
|
||||
to a trusted interface, front them with an authenticating proxy, or narrow them
|
||||
to known networks with `acl.allow`.
|
||||
|
||||
What each exposes when `auth.type = "none"`:
|
||||
|
||||
|
||||
+18
-7
@@ -178,6 +178,8 @@ Options, each as type and default:
|
||||
- `tls` (table): listener TLS; see [Security](security.md).
|
||||
- `auth` (table): client authentication (`mtls` or `scram`); see
|
||||
[Security](security.md#the-auth-block).
|
||||
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
|
||||
before TLS; see [Security](security.md#the-acl-block).
|
||||
|
||||
**Behaviour**
|
||||
|
||||
@@ -241,15 +243,16 @@ Options, each as type and default:
|
||||
- HTTP/2 is negotiated via ALPN when TLS is enabled; plaintext is HTTP/1.1.
|
||||
|
||||
**Status endpoint** returns service and version identity, host, port, TLS flag,
|
||||
the compiled auth policy, active client count, sink and per-client buffer sizes,
|
||||
connection limit, write timeout, uptime, endpoint paths, and the
|
||||
`total_processed` / `dropped_writes` / `rejected_clients` / `auth_rejected`
|
||||
counters.
|
||||
the compiled auth policy and acl, active client count, sink and per-client
|
||||
buffer sizes, connection limit, write timeout, uptime, endpoint paths, and the
|
||||
`total_processed` / `dropped_writes` / `rejected_clients` / `auth_rejected` /
|
||||
`acl_denied` counters.
|
||||
|
||||
**Statistics**: `dropped_writes`, `rejected_clients`, `auth`, `auth_allowed`,
|
||||
`auth_rejected`; under `scram` also `auth_users`, `auth_throttled`,
|
||||
`auth_busy`, `auth_binding_mismatch` and `auth_token_lifetime_ms`, and in
|
||||
proxy mode `auth_trusted_proxies`.
|
||||
proxy mode `auth_trusted_proxies`; with `acl` rules also `acl` and
|
||||
`acl_denied`.
|
||||
|
||||
> Without an `auth` block both endpoints are unauthenticated, and the stream
|
||||
> response carries `Access-Control-Allow-Origin: *`, so any web origin can read
|
||||
@@ -302,6 +305,8 @@ Options, each as type and default:
|
||||
- `tls` (table): listener TLS.
|
||||
- `auth` (table): client authentication (`mtls` or `scram`); see
|
||||
[Security](security.md#the-auth-block).
|
||||
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
|
||||
before TLS; see [Security](security.md#the-acl-block).
|
||||
|
||||
**Behaviour**
|
||||
|
||||
@@ -327,7 +332,8 @@ Options, each as type and default:
|
||||
|
||||
**Statistics**: `write_errors`, `dropped_writes`, `rejected_conns`,
|
||||
`tls_handshake_errors`, `auth`, `auth_allowed`, `auth_rejected`; under `scram`
|
||||
also `auth_users`, `auth_throttled`, `auth_busy` and `auth_binding_mismatch`.
|
||||
also `auth_users`, `auth_throttled`, `auth_busy` and `auth_binding_mismatch`;
|
||||
with `acl` rules also `acl` and `acl_denied`.
|
||||
|
||||
---
|
||||
|
||||
@@ -391,6 +397,9 @@ Options, each as type and default:
|
||||
exponential backoff plus ±20 % jitter between attempts. The delay keeps
|
||||
growing until a link outlives `backoff_min_ms`, so a source that drops every
|
||||
link right after accepting it is not retried in a tight loop.
|
||||
- On shutdown (a reload too) the sink delivers what is queued, connecting
|
||||
first if it must, within `write_timeout_ms`, at most 2 s; then the link
|
||||
closes and what is left is lost.
|
||||
- A source never writes once a link is up, so the sink watches each link: a
|
||||
line from the source is a refusal (a sink without `scram` facing a `scram`
|
||||
source), logged at WARN as `Chain link refused`; EOF ends the link at once
|
||||
@@ -477,7 +486,9 @@ Options, each as type and default:
|
||||
- Redirects are never followed; a `3xx` is permanent too. Following one would
|
||||
resend the batch wherever the response points, plaintext `http` included.
|
||||
- HTTP/2 is off by design; batched NDJSON POSTs gain nothing from it.
|
||||
- On shutdown a single best-effort flush of the pending batch is attempted.
|
||||
- On shutdown (a reload too) the sink delivers what is queued and batched
|
||||
within `request_timeout_ms`, at most 2 s; each batch still undelivered then
|
||||
is dropped and logged at WARN.
|
||||
|
||||
**Statistics**: `target`, `node`, `tls`, `auth`, `batches_sent`,
|
||||
`request_errors`, `dropped_batches`, `synthesized`; under `scram` also
|
||||
|
||||
+7
-2
@@ -214,6 +214,8 @@ Options, each as type and default:
|
||||
- `tls` (table): listener TLS; see [Security](security.md).
|
||||
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
|
||||
see [Security](security.md#the-auth-block).
|
||||
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
|
||||
before TLS; see [Security](security.md#the-acl-block).
|
||||
|
||||
For passwords instead of certificates, the `auth` block takes `type = "scram"`
|
||||
and a `credentials_file`, and `client_auth` becomes optional; see
|
||||
@@ -247,7 +249,7 @@ and a `credentials_file`, and `client_auth` becomes optional; see
|
||||
**Statistics**: `active_connections`, `rejected_conns`, `parse_errors`,
|
||||
`tls_handshake_errors`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
|
||||
`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`
|
||||
and `auth_binding_mismatch`.
|
||||
and `auth_binding_mismatch`; with `acl` rules also `acl` and `acl_denied`.
|
||||
|
||||
---
|
||||
|
||||
@@ -298,6 +300,8 @@ Options, each as type and default:
|
||||
- `tls` (table): listener TLS; see [Security](security.md).
|
||||
- `auth` (table): peer authentication (`mtls` or `scram`) and node binding;
|
||||
see [Security](security.md#the-auth-block).
|
||||
- `acl` (table): `allow` and `deny`, addresses or CIDRs admitted and refused
|
||||
before TLS; see [Security](security.md#the-acl-block).
|
||||
|
||||
**Behaviour**
|
||||
|
||||
@@ -324,7 +328,8 @@ Options, each as type and default:
|
||||
**Statistics**: `total_requests`, `rejected_requests`, `parse_errors`,
|
||||
`cached_sessions`, `trust_node`, `auth`, `auth_allowed`, `auth_rejected`,
|
||||
`node_binding`; under `scram` also `auth_users`, `auth_throttled`, `auth_busy`,
|
||||
`auth_binding_mismatch` and `auth_token_lifetime_ms`.
|
||||
`auth_binding_mismatch` and `auth_token_lifetime_ms`; with `acl` rules also
|
||||
`acl` and `acl_denied`.
|
||||
|
||||
---
|
||||
|
||||
|
||||
+26
-53
@@ -6,29 +6,19 @@ they belong to.
|
||||
|
||||
## 1. Network access control (ACL)
|
||||
|
||||
The next feature push. Today a listener decides *who* a peer is (`tls`, `auth`);
|
||||
nothing decides *where* a peer may connect from, and every limit except SCRAM
|
||||
throttling is global. The ACL adds address rules and per-client limits at the
|
||||
connection level, below TLS, so a refused peer costs no handshake.
|
||||
The seam exists: `internal/netacl` wraps each listener's socket before TLS,
|
||||
and the `acl` block's `allow` and `deny` refuse peers by address
|
||||
([Security](security.md#the-acl-block)). Every limit except SCRAM throttling
|
||||
is still global, and behind an L4 proxy every peer is the proxy.
|
||||
|
||||
### 1.1 One seam: a filtering listener
|
||||
### 1.1 The filtering listener
|
||||
|
||||
- New package `internal/netacl`, the third network seam beside `tlsx` and
|
||||
`authz` (AGENTS.md lists the seams; add it there).
|
||||
- `netacl.New(opts *config.ACLOptions, role) (*Policy, error)` compiles the
|
||||
block; a nil policy is valid and transparent, like `authz`.
|
||||
- `(*Policy).Listener(ln net.Listener) net.Listener` wraps the raw listener
|
||||
before TLS:
|
||||
- TCP plugins wrap it before `tls.NewListener`.
|
||||
- HTTP plugins wrap it before `Serve`/`ServeTLS`.
|
||||
- `Accept` reads an optional PROXY header (1.2), decides allow or deny (1.3),
|
||||
applies per-address connection caps (1.4), and returns a `net.Conn` whose
|
||||
`RemoteAddr` is the real client.
|
||||
- `Accept` is to read an optional PROXY header (1.2) before the address rules
|
||||
and apply per-address connection caps (1.4) after them, returning a
|
||||
`net.Conn` whose `RemoteAddr` is the real client.
|
||||
- Everything downstream then sees the real client with no further change:
|
||||
`authz.remoteIP`, SCRAM throttling, sessions, logs and `http.Request.RemoteAddr`.
|
||||
- Keep the proxy's own address as `peer_addr` in session metadata for audit.
|
||||
- A denied connection is closed before any byte is read or written, with a
|
||||
counter (`acl_denied`) and a rate-limited WARN.
|
||||
|
||||
### 1.2 PROXY protocol (deferred gap of SCRAM, see scram-auth-plan.md)
|
||||
|
||||
@@ -64,20 +54,14 @@ connection level, below TLS, so a refused peer costs no handshake.
|
||||
one deployment can use both: nginx `stream` with `proxy_protocol` into an
|
||||
nginx `http` block that sets `X-Forwarded-For $proxy_protocol_addr`.
|
||||
|
||||
### 1.3 Address rules
|
||||
### 1.3 Address rules for forwarded clients
|
||||
|
||||
- `allow = [CIDR...]`, `deny = [CIDR...]`, IPv4 and IPv6 (strict per family,
|
||||
as the listeners are).
|
||||
- Order: deny wins, then allow; an empty allow list admits everyone not denied.
|
||||
- Rules match the real client (after PROXY), never the proxy.
|
||||
- In HTTP proxy mode, `X-Forwarded-For` is applied by `authz.ClientAddr` after
|
||||
the connection-level decision.
|
||||
- Address rules for forwarded HTTP clients therefore belong in a second,
|
||||
request-level check inside `authz.ClientAddr`: reuse the compiled `netacl`
|
||||
rules there rather than duplicating them.
|
||||
- Startup warnings (in `LogStartup` style): an allow list containing
|
||||
`0.0.0.0/0` or `::/0`, `proxy_from` covering public ranges, a listener on a
|
||||
wildcard host with no `allow`.
|
||||
the connection-level decision, so rules for forwarded HTTP clients belong in
|
||||
a second, request-level check there, reusing the compiled `netacl` rules.
|
||||
`authz.parseProxies` then takes netacl's address-or-CIDR parsing too.
|
||||
- Startup warning: `proxy_from` covering public ranges.
|
||||
|
||||
### 1.4 Per-client limits
|
||||
|
||||
@@ -95,15 +79,11 @@ connection level, below TLS, so a refused peer costs no handshake.
|
||||
|
||||
### 1.5 Config, validation, docs
|
||||
|
||||
- Config shape: `[...config.acl]` beside `tls` and `auth` on the four listener
|
||||
plugins.
|
||||
- Decode through `config.Scan`, so unknown keys fail.
|
||||
- Dialers have no ACL block.
|
||||
- Intent rule as in `authz`: an ACL key on a dialer, or `proxy_from` without
|
||||
`proxy_protocol`, is an error.
|
||||
- Stats: `acl_denied`, `acl_proxy_headers`, `acl_limited`, merged into the
|
||||
plugin stats like `authz.Stats`.
|
||||
- Docs: a security.md section, plus networking.md troubleshooting lines for
|
||||
- `proxy_protocol`, `proxy_from` and the 1.4 keys join `allow` and `deny` in
|
||||
the `acl` block.
|
||||
- Intent rule as in `authz`: `proxy_from` without `proxy_protocol` is an error.
|
||||
- Stats: `acl_proxy_headers` and `acl_limited` beside `acl_denied`.
|
||||
- Docs: the security.md section, plus networking.md troubleshooting lines for
|
||||
each refusal.
|
||||
|
||||
### 1.6 Verification
|
||||
@@ -112,15 +92,13 @@ connection level, below TLS, so a refused peer costs no handshake.
|
||||
- PROXY v1 and v2 parse, including malformed, oversized and slow headers.
|
||||
- A header from an unlisted peer refused; `required` without a header
|
||||
refused; `LOCAL` kept.
|
||||
- Deny over allow; IPv6 rules.
|
||||
- Per-client connection cap; the limiter table failing closed.
|
||||
- The real client reaching SCRAM throttling and session metadata.
|
||||
- `test/acl-test.sh`:
|
||||
- `test/acl-test.sh`, which covers the address rules, adds:
|
||||
- A Go stub proxy that sends v1 and v2 headers in front of each listener
|
||||
type.
|
||||
- nginx `stream` with `proxy_protocol on` when nginx is installed.
|
||||
- Denied and allowed clients; per-client SCRAM budgets behind one proxy
|
||||
address.
|
||||
- Per-client SCRAM budgets behind one proxy address.
|
||||
|
||||
## 2. Packaging: AUR, FreeBSD ports, Debian
|
||||
|
||||
@@ -183,21 +161,16 @@ identity, and both skeletons download the tagged source.
|
||||
|
||||
Smaller items found along the way; each is independent of the ACL.
|
||||
|
||||
1. Chain sinks at the end of a finite input. The `tcp_chain` sink's `Stop`
|
||||
closes its loop without draining its queue, and the `http_chain` sink makes
|
||||
one best-effort flush, so `seq 1 N | lw --preset edge,...` may lose the
|
||||
tail. Verify with a finite stdin into each and an aggregator counting lines;
|
||||
give both a bounded drain like `sink.FlushBound`.
|
||||
2. A late browser sees no backlog: the `http` sink keeps none, so the viewer
|
||||
1. A late browser sees no backlog: the `http` sink keeps none, so the viewer
|
||||
shows entries from when it connects. A bounded replay (a `replay_lines`
|
||||
option, sent after `event: connected`) would fill it. The viewer also hides
|
||||
a `503` from `max_connections` behind "retrying"; the status JSON it already
|
||||
fetches could name it.
|
||||
3. Level detection. `source.ExtractLogLevel` needs a delimiter after the name
|
||||
2. Level detection. `source.ExtractLogLevel` needs a delimiter after the name
|
||||
(`ERROR:`, `[WARN]`, ` INFO `), so `warning disk full` or `DBG x` get no
|
||||
level, hence no color. Match the names as words, as the console sink's
|
||||
painter does, with one table for both.
|
||||
4. Command line:
|
||||
3. Command line:
|
||||
- a positional word at the top level (`lw tail`, `lw -t FILE`) still reaches
|
||||
config with an unclear message; name it ("unexpected argument X; the file
|
||||
is -c FILE");
|
||||
@@ -207,11 +180,11 @@ Smaller items found along the way; each is independent of the ACL.
|
||||
- `lw tls cert --host` could become `--hosts` (keeping `--host`), to match
|
||||
`tls.hosts` and the preset key;
|
||||
- top-level usage errors exit 1, subcommand ones 2.
|
||||
5. `GET /favicon.ico` answers 404; the logo could serve as the icon once the
|
||||
4. `GET /favicon.ico` answers 404; the logo could serve as the icon once the
|
||||
pages' CSP allows `img-src 'self'`.
|
||||
6. TLS: the generated-certificate key is one per process, so a pin taken from a
|
||||
5. TLS: the generated-certificate key is one per process, so a pin taken from a
|
||||
`self_signed` listener also matches an `issuer` listener of the same
|
||||
process; document it or key per listener. Pins are checked in
|
||||
`VerifyPeerCertificate`, which Go skips on resumption: guard it if a dialer
|
||||
ever keeps a `ClientSessionCache`.
|
||||
7. `core.ShutdownTimeout` is unused.
|
||||
6. `core.ShutdownTimeout` is unused.
|
||||
@@ -216,6 +216,7 @@ type TCPChainSourceOptions struct {
|
||||
HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting
|
||||
@@ -230,6 +231,7 @@ type HTTPChainSourceOptions struct {
|
||||
ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline
|
||||
TrustNode bool `toml:"trust_node"` // false: force node label from remote address
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// --- Sink Options ---
|
||||
@@ -277,6 +279,7 @@ type TCPSinkOptions struct {
|
||||
KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"`
|
||||
MaxConnections int64 `toml:"max_connections"` // 0 = unlimited
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// HTTPSinkOptions defines settings for an HTTP SSE server sink
|
||||
@@ -293,6 +296,7 @@ type HTTPSinkOptions struct {
|
||||
LoginPage bool `toml:"login_page"` // GET /auth/login, needs auth.trusted_proxies
|
||||
ViewerPage bool `toml:"viewer_page"` // GET /auth/view, needs login_page
|
||||
Auth *AuthOptions `toml:"auth"`
|
||||
ACL *ACLOptions `toml:"acl"`
|
||||
}
|
||||
|
||||
// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding
|
||||
@@ -369,6 +373,14 @@ type AuthOptions struct {
|
||||
PasswordFile string `toml:"password_file"`
|
||||
}
|
||||
|
||||
// ACLOptions is a listener's address rules, entries being addresses or CIDRs:
|
||||
// deny wins, then a set allow list admits only its entries. Validation is in
|
||||
// internal/netacl.
|
||||
type ACLOptions struct {
|
||||
Allow []string `toml:"allow"`
|
||||
Deny []string `toml:"deny"`
|
||||
}
|
||||
|
||||
// --- TLS Options ---
|
||||
|
||||
// TLSOptions is one shape for both roles. Listeners present cert_file and
|
||||
|
||||
@@ -40,6 +40,10 @@ var (
|
||||
{Name: "issuer_key", Help: "CA key file, for tls=issuer"},
|
||||
{Name: "hosts", List: true, Help: "names and addresses, beyond this host's, of a self or issuer certificate; ',' between them"},
|
||||
}
|
||||
aclParams = []PresetParam{
|
||||
{Name: "allow", List: true, Help: "addresses or CIDRs that may connect, of listen's family; default: all; ',' between them"},
|
||||
{Name: "deny", List: true, Help: "addresses or CIDRs refused, allow or not; ',' between them"},
|
||||
}
|
||||
transportParam = PresetParam{Name: "transport", Default: "tcp", Help: "tcp|http: tcp_chain or http_chain"}
|
||||
)
|
||||
|
||||
@@ -60,13 +64,13 @@ var presets = []Preset{
|
||||
return nil
|
||||
}},
|
||||
{"serve", "Serve files or standard input live: a browser viewer at /, SSE at /stream",
|
||||
append([]PresetParam{pathParam, fromParam, formatParam("json"),
|
||||
slices.Concat([]PresetParam{pathParam, fromParam, formatParam("json"),
|
||||
{Name: "listen", Default: "127.0.0.1:8080", Help: "HOST:PORT; [IPV6]:PORT"},
|
||||
{Name: "tls", Default: "off", Help: "off|self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
|
||||
{Name: "users", Help: "credentials file (lw auth add-user): readers log in with SCRAM"},
|
||||
{Name: "proxy", List: true, Help: "addresses or CIDRs of the TLS-terminating proxies browsers come through; ',' between them"},
|
||||
{Name: "viewer", Default: "false", Help: "true: the login page and viewer for users, behind proxy; without users the viewer is always on"},
|
||||
}, tlsParams...),
|
||||
}, aclParams, tlsParams),
|
||||
func(p *PipelineConfig, v map[string]string) error {
|
||||
p.PluginSources = pathOrStdin(v)
|
||||
sink, err := listener(v, "off")
|
||||
@@ -145,7 +149,7 @@ var presets = []Preset{
|
||||
return nil
|
||||
}},
|
||||
{"aggregator", "Receive from edges over TLS, authenticated, to files or standard output",
|
||||
append([]PresetParam{
|
||||
slices.Concat([]PresetParam{
|
||||
{Name: "listen", Default: "0.0.0.0:9000", Help: "HOST:PORT; [IPV6]:PORT; [::] for IPv6 only"},
|
||||
transportParam,
|
||||
{Name: "tls", Default: "self", Help: "self|issuer|files: a certificate made at startup (self-signed or from issuer_*), or files"},
|
||||
@@ -153,7 +157,7 @@ var presets = []Preset{
|
||||
{Name: "client_ca", Help: "CA file that verifies edge certificates (mTLS)"},
|
||||
{Name: "out", Help: "directory for the received entries; default: standard output"},
|
||||
formatParam("json"),
|
||||
}, tlsParams...),
|
||||
}, aclParams, tlsParams),
|
||||
func(p *PipelineConfig, v map[string]string) error {
|
||||
typ, err := chainType(v["transport"])
|
||||
if err != nil {
|
||||
@@ -268,14 +272,22 @@ var tlsModes = map[string]struct {
|
||||
"files": {needs: map[string]string{"cert": "cert_file", "key": "key_file"}},
|
||||
}
|
||||
|
||||
// listener starts a listening plugin's options: host, port and tls, whose
|
||||
// mode decides which certificate keys apply.
|
||||
// listener starts a listening plugin's options: host, port, acl and tls,
|
||||
// whose mode decides which certificate keys apply.
|
||||
func listener(v map[string]string, def string) (map[string]any, error) {
|
||||
host, port, err := splitAddr("listen", v["listen"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opts := map[string]any{"host": host, "port": port}
|
||||
opts, acl := map[string]any{"host": host, "port": port}, map[string]any{}
|
||||
for _, k := range []string{"allow", "deny"} {
|
||||
if v[k] != "" {
|
||||
acl[k] = list(v[k])
|
||||
}
|
||||
}
|
||||
if len(acl) > 0 {
|
||||
opts["acl"] = acl
|
||||
}
|
||||
mode := cmp.Or(v["tls"], def)
|
||||
m, ok := tlsModes[mode]
|
||||
if !ok {
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
// Package netacl is the address seam of the network listeners, beside tlsx
|
||||
// and authz: a Policy refuses a peer by its address as the socket is
|
||||
// accepted, before TLS or a byte of the protocol is read. New returns nil
|
||||
// when no rule is set, and every method tolerates a nil receiver.
|
||||
package netacl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
// warnEvery spaces the WARNs reporting refused peers; acl_denied counts all
|
||||
const warnEvery = time.Minute
|
||||
|
||||
// Policy is the compiled form of config.ACLOptions for one listener
|
||||
type Policy struct {
|
||||
allow, deny []netip.Prefix
|
||||
logger *log.Logger
|
||||
component, id string
|
||||
|
||||
denied atomic.Uint64
|
||||
epoch time.Time // a monotonic base: a stepped clock neither mutes nor floods the WARN
|
||||
nextWarn atomic.Int64 // nanoseconds since epoch
|
||||
}
|
||||
|
||||
// New compiles the rules of a listener on host and warns of what they leave
|
||||
// open. Each entry must be of the family core.Network gives host: one the
|
||||
// listener's sockets can never carry is a mistake, not a rule.
|
||||
func New(o *config.ACLOptions, host string, l *log.Logger, component, id string) (*Policy, error) {
|
||||
if o == nil || len(o.Allow)+len(o.Deny) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
network, err := core.Network(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var widened []string // entry, network pairs
|
||||
compile := func(key string, entries []string) ([]netip.Prefix, error) {
|
||||
var rules []netip.Prefix
|
||||
for _, e := range entries {
|
||||
prefix, err := parse(e, network)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("acl: %s entry %q: %w", key, e, err)
|
||||
}
|
||||
if m := prefix.Masked(); m != prefix {
|
||||
widened = append(widened, e, m.String())
|
||||
}
|
||||
rules = append(rules, prefix.Masked())
|
||||
}
|
||||
return rules, nil
|
||||
}
|
||||
p := &Policy{logger: l, component: component, id: id, epoch: time.Now()}
|
||||
if p.allow, err = compile("allow", o.Allow); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.deny, err = compile("deny", o.Deny); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.logStartup(host, widened)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func parse(entry, network string) (netip.Prefix, error) {
|
||||
entry = strings.TrimSpace(entry)
|
||||
prefix, err := netip.ParsePrefix(entry)
|
||||
if err != nil {
|
||||
addr, aerr := netip.ParseAddr(entry)
|
||||
if aerr != nil || addr.Zone() != "" {
|
||||
return netip.Prefix{}, fmt.Errorf("neither an address nor a CIDR (and no zone)")
|
||||
}
|
||||
prefix = netip.PrefixFrom(addr, addr.BitLen())
|
||||
}
|
||||
switch a := prefix.Addr(); {
|
||||
case a.Is4In6():
|
||||
return netip.Prefix{}, fmt.Errorf("IPv4-mapped; write it as IPv4")
|
||||
case network == "tcp4" && !a.Is4(), network == "tcp6" && a.Is4():
|
||||
return netip.Prefix{}, fmt.Errorf("not of the listener's family (%s)", network)
|
||||
}
|
||||
return prefix, nil
|
||||
}
|
||||
|
||||
func (p *Policy) logStartup(host string, widened []string) {
|
||||
for i := 0; i < len(widened); i += 2 {
|
||||
p.logger.Warn("msg", "acl entry has host bits set and matches its whole network",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"entry", widened[i],
|
||||
"network", widened[i+1])
|
||||
}
|
||||
for _, q := range p.allow {
|
||||
if q.Bits() == 0 {
|
||||
p.logger.Warn("msg", "acl.allow admits every address of its family",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"entry", q.String(),
|
||||
"hint", "list only the networks that may connect")
|
||||
}
|
||||
}
|
||||
if addr, err := netip.ParseAddr(host); len(p.allow) == 0 && (host == "" || err == nil && addr.IsUnspecified()) {
|
||||
p.logger.Warn("msg", "Listener on every address admits all that acl.deny does not list",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"hint", "set acl.allow to the networks that may connect")
|
||||
}
|
||||
}
|
||||
|
||||
// admits applies the rules: deny wins, then a set allow list admits only its
|
||||
// entries. A peer whose address is unknown is refused.
|
||||
func (p *Policy) admits(addr netip.Addr) bool {
|
||||
addr = addr.Unmap().WithZone("") // a zoned address is in no prefix
|
||||
in := func(q netip.Prefix) bool { return q.Contains(addr) }
|
||||
return addr.IsValid() && !slices.ContainsFunc(p.deny, in) &&
|
||||
(len(p.allow) == 0 || slices.ContainsFunc(p.allow, in))
|
||||
}
|
||||
|
||||
// Listener returns ln refusing the peers the rules exclude as it accepts them,
|
||||
// so the plugin never sees one. Close stays ln's: a sink's shutdown flush
|
||||
// reaches it unchanged. A nil Policy returns ln.
|
||||
func (p *Policy) Listener(ln net.Listener) net.Listener {
|
||||
if p == nil {
|
||||
return ln
|
||||
}
|
||||
return &listener{Listener: ln, p: p}
|
||||
}
|
||||
|
||||
type listener struct {
|
||||
net.Listener
|
||||
p *Policy
|
||||
}
|
||||
|
||||
func (l *listener) Accept() (net.Conn, error) {
|
||||
for {
|
||||
conn, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var addr netip.Addr
|
||||
remote := conn.RemoteAddr()
|
||||
if t, ok := remote.(*net.TCPAddr); ok {
|
||||
addr = t.AddrPort().Addr()
|
||||
}
|
||||
if l.p.admits(addr) {
|
||||
return conn, nil
|
||||
}
|
||||
conn.Close()
|
||||
l.p.refused(fmt.Sprint(remote))
|
||||
}
|
||||
}
|
||||
|
||||
// refused counts a refusal and WARNs at most once per warnEvery
|
||||
func (p *Policy) refused(remote string) {
|
||||
n := p.denied.Add(1)
|
||||
now, next := int64(time.Since(p.epoch)), p.nextWarn.Load()
|
||||
if now < next || !p.nextWarn.CompareAndSwap(next, now+int64(warnEvery)) {
|
||||
return
|
||||
}
|
||||
p.logger.Warn("msg", "Connection refused by acl; more within a minute are only counted",
|
||||
"component", p.component,
|
||||
"instance_id", p.id,
|
||||
"remote_addr", remote,
|
||||
"acl_denied", n)
|
||||
}
|
||||
|
||||
// Describe renders the policy for log lines, stats and the status endpoint
|
||||
func (p *Policy) Describe() string {
|
||||
if p == nil {
|
||||
return "none"
|
||||
}
|
||||
allow := "any"
|
||||
if len(p.allow) > 0 {
|
||||
allow = fmt.Sprint(len(p.allow))
|
||||
}
|
||||
return fmt.Sprintf("allow=%s deny=%d", allow, len(p.deny))
|
||||
}
|
||||
|
||||
func (p *Policy) Denied() uint64 {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
return p.denied.Load()
|
||||
}
|
||||
|
||||
// Stats reports the policy for a plugin's stats details, merged in with
|
||||
// maps.Copy beside authz's. A nil Policy reports nothing.
|
||||
func (p *Policy) Stats() map[string]any {
|
||||
if p == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{"acl": p.Describe(), "acl_denied": p.denied.Load()}
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package netacl
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
)
|
||||
|
||||
func mustNew(t *testing.T, host string, allow, deny []string) *Policy {
|
||||
t.Helper()
|
||||
p, err := New(&config.ACLOptions{Allow: allow, Deny: deny}, host, log.NewLogger(), "test", "acl")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Deny wins over allow, a set allow list admits only its entries, an empty
|
||||
// one admits all not denied, an IPv4-mapped peer matches as IPv4, and a peer
|
||||
// of unknown address is refused.
|
||||
func TestDenyWinsThenAllowListAdmitsOnlyItsEntries(t *testing.T) {
|
||||
listed := mustNew(t, "0.0.0.0", []string{"127.0.0.0/8"}, []string{"127.0.0.2"})
|
||||
denyOnly := mustNew(t, "0.0.0.0", nil, []string{"10.0.0.0/8"})
|
||||
for _, c := range []struct {
|
||||
p *Policy
|
||||
addr string
|
||||
want bool
|
||||
}{
|
||||
{listed, "127.0.0.1", true},
|
||||
{listed, "127.0.0.2", false},
|
||||
{listed, "192.0.2.1", false},
|
||||
{denyOnly, "10.1.2.3", false},
|
||||
{denyOnly, "192.0.2.1", true},
|
||||
{denyOnly, "::ffff:10.1.2.3", false},
|
||||
{denyOnly, "", false},
|
||||
} {
|
||||
addr, _ := netip.ParseAddr(c.addr)
|
||||
if got := c.p.admits(addr); got != c.want {
|
||||
t.Errorf("%s under %s: admitted %v", c.addr, c.p.Describe(), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Entries are of the family core.Network gives the listener's host, as its
|
||||
// sockets carry no other: IPv4 on an IPv4 listener, IPv6 on an IPv6 one,
|
||||
// either behind a hostname; never IPv4-mapped, never with a zone.
|
||||
func TestEntriesFollowTheListenerFamily(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
host, entry string
|
||||
ok bool
|
||||
}{
|
||||
{"0.0.0.0", "10.0.0.0/8", true},
|
||||
{"0.0.0.0", "2001:db8::/32", false},
|
||||
{"::", "2001:db8::1", true},
|
||||
{"::", "10.0.0.1", false},
|
||||
{"::", "::ffff:10.0.0.1", false},
|
||||
{"::", "fe80::1%eth0", false},
|
||||
{"logs.example", "10.0.0.0/8", true},
|
||||
{"logs.example", "2001:db8::/32", true},
|
||||
} {
|
||||
_, err := New(&config.ACLOptions{Deny: []string{c.entry}}, c.host, log.NewLogger(), "test", "acl")
|
||||
if (err == nil) != c.ok {
|
||||
t.Errorf("%s on %q: %v", c.entry, c.host, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A link-local peer carries its interface zone, which no prefix contains:
|
||||
// it is matched without it, or a deny of fe80::/10 would admit it.
|
||||
func TestZonedPeerMatchesItsPrefix(t *testing.T) {
|
||||
p := mustNew(t, "::", nil, []string{"fe80::/10"})
|
||||
if p.admits(netip.MustParseAddr("fe80::1%eth0")) {
|
||||
t.Fatal("a zoned link-local peer passed the deny of its prefix")
|
||||
}
|
||||
}
|
||||
|
||||
// A refused peer is closed as it is accepted and counted: the plugin's
|
||||
// Accept returns the next admitted one.
|
||||
func TestRefusedPeerNeverReachesAccept(t *testing.T) {
|
||||
inner := &fakeListener{conns: make(chan net.Conn, 2)}
|
||||
refused, refusedPeer := net.Pipe()
|
||||
admitted, _ := net.Pipe()
|
||||
inner.conns <- remoteAt(refused, "192.0.2.1")
|
||||
inner.conns <- remoteAt(admitted, "127.0.0.1")
|
||||
ln := mustNew(t, "127.0.0.1", nil, []string{"192.0.2.0/24"}).Listener(inner)
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := conn.RemoteAddr().String(); !strings.HasPrefix(got, "127.0.0.1:") {
|
||||
t.Fatalf("Accept returned %s", got)
|
||||
}
|
||||
refusedPeer.SetReadDeadline(time.Now().Add(time.Second))
|
||||
if _, err := refusedPeer.Read(make([]byte, 1)); err != io.EOF {
|
||||
t.Fatalf("the refused peer reads %v, want EOF", err)
|
||||
}
|
||||
if n := ln.(*listener).p.Stats()["acl_denied"]; n != uint64(1) {
|
||||
t.Fatalf("acl_denied = %v", n)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeListener struct {
|
||||
net.Listener
|
||||
conns chan net.Conn
|
||||
}
|
||||
|
||||
func (f *fakeListener) Accept() (net.Conn, error) { return <-f.conns, nil }
|
||||
|
||||
type remoteConn struct {
|
||||
net.Conn
|
||||
remote net.Addr
|
||||
}
|
||||
|
||||
func (c remoteConn) RemoteAddr() net.Addr { return c.remote }
|
||||
|
||||
func remoteAt(c net.Conn, addr string) net.Conn {
|
||||
return remoteConn{c, net.TCPAddrFromAddrPort(netip.AddrPortFrom(netip.MustParseAddr(addr), 40000))}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/chain"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/sink"
|
||||
@@ -78,6 +79,7 @@ type HTTPSink struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
web map[string]http.Handler // GET paths of the browser files and the root
|
||||
|
||||
// Runtime. Stop closes flush; the broker then moves what remains of its
|
||||
@@ -158,6 +160,10 @@ func NewHTTPSinkPlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "http_sink", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case (opts.LoginPage || opts.ViewerPage) && !authPolicy.BehindProxy():
|
||||
return nil, errors.New("login_page and viewer_page apply to scram behind auth.trusted_proxies, where browsers log in; without auth or under mtls the viewer is always served")
|
||||
@@ -185,6 +191,7 @@ func NewHTTPSinkPlugin(
|
||||
keepalive: core.StreamKeepaliveInterval,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
web: web,
|
||||
}
|
||||
h.lastProcessed.Store(time.Time{})
|
||||
@@ -199,6 +206,7 @@ func NewHTTPSinkPlugin(
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe(),
|
||||
"login_page", opts.LoginPage,
|
||||
"viewer_page", web[chain.AuthPath+"/view"] != nil)
|
||||
tlsx.LogWarnings(logger, "http_sink", id, opts.TLS, true)
|
||||
@@ -244,6 +252,7 @@ func (h *HTTPSink) serve(ctx context.Context, ln net.Listener) error {
|
||||
ln.Close()
|
||||
return err
|
||||
}
|
||||
ln = h.acl.Listener(ln)
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped patterns: mux answers 405 with Allow header on non-GET
|
||||
mux.HandleFunc(http.MethodGet+" "+exact(h.config.StreamPath), h.handleStream)
|
||||
@@ -569,6 +578,7 @@ func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
"port": h.config.Port,
|
||||
"tls": h.tlsConfig != nil,
|
||||
"auth": h.auth.Describe(),
|
||||
"acl": h.acl.Describe(),
|
||||
"active_clients": h.activeClients.Load(),
|
||||
"buffer_size": h.config.BufferSize,
|
||||
"client_buffer_size": h.config.ClientBufferSize,
|
||||
@@ -585,6 +595,7 @@ func (h *HTTPSink) handleStatus(w http.ResponseWriter, r *http.Request) {
|
||||
"dropped_writes": h.droppedWrites.Load(),
|
||||
"rejected_clients": h.rejectedClients.Load(),
|
||||
"auth_rejected": h.auth.Rejected(),
|
||||
"acl_denied": h.acl.Denied(),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -611,6 +622,7 @@ func (h *HTTPSink) GetStats() sink.SinkStats {
|
||||
},
|
||||
}
|
||||
maps.Copy(details, h.auth.Stats())
|
||||
maps.Copy(details, h.acl.Stats())
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: h.id,
|
||||
|
||||
@@ -256,8 +256,8 @@ func (t *HTTPChainSink) Start(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the loop. Worst case: one in-flight request timeout plus
|
||||
// one final-flush request timeout.
|
||||
// Stop delivers what is queued and batched within sink.FlushBound, then ends
|
||||
// the loop
|
||||
func (t *HTTPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping HTTP chain sink",
|
||||
"component", "http_chain_sink",
|
||||
@@ -305,49 +305,41 @@ func (t *HTTPChainSink) GetStats() sink.SinkStats {
|
||||
func (t *HTTPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
|
||||
// Fold done channel into a context for request/backoff interruption
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
// Stop leaves sink.FlushBound to deliver the queue and the batch, so a
|
||||
// finite input arrives whole; a request or backoff in flight then ends,
|
||||
// and every later flush drops its batch at once
|
||||
ctx, cancel := sink.FlushContext(ctx, t.done, t.reqTimeout)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-runCtx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-runCtx.Done():
|
||||
t.finalFlush()
|
||||
return
|
||||
case <-ticker.C:
|
||||
if t.batchCount > 0 && !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
if t.batchCount > 0 {
|
||||
t.flush(ctx)
|
||||
}
|
||||
case event := <-t.input:
|
||||
if t.append(event) {
|
||||
t.flush(ctx)
|
||||
}
|
||||
case <-t.done:
|
||||
for len(t.input) > 0 {
|
||||
if t.append(<-t.input) {
|
||||
t.flush(ctx)
|
||||
}
|
||||
}
|
||||
if t.batchCount > 0 {
|
||||
t.flush(ctx)
|
||||
}
|
||||
return
|
||||
}
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
t.append(event)
|
||||
if t.batchCount >= t.config.MaxBatchCount ||
|
||||
int64(t.batch.Len()) >= t.config.MaxBatchBytes {
|
||||
if !t.flush(runCtx) {
|
||||
t.finalFlush()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// append serializes one event into the pending batch
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
// append serializes one event into the pending batch; true once it is full
|
||||
func (t *HTTPChainSink) append(event core.TransportEvent) bool {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
@@ -358,17 +350,17 @@ func (t *HTTPChainSink) append(event core.TransportEvent) {
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "http_chain_sink",
|
||||
"error", err)
|
||||
return
|
||||
return false
|
||||
}
|
||||
t.batch.Write(line)
|
||||
t.batch.WriteByte('\n')
|
||||
t.batchCount++
|
||||
return t.batchCount >= t.config.MaxBatchCount || int64(t.batch.Len()) >= t.config.MaxBatchBytes
|
||||
}
|
||||
|
||||
// flush delivers the pending batch, retrying transient failures with backoff.
|
||||
// Returns false when shutdown interrupts delivery; undelivered batch is dropped
|
||||
// by finalFlush semantics (batch already consumed here).
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
// flush delivers the pending batch, retrying transient failures with backoff
|
||||
// until shutdown drops it
|
||||
func (t *HTTPChainSink) flush(ctx context.Context) {
|
||||
body := bytes.Clone(t.batch.Bytes())
|
||||
count := t.batchCount
|
||||
t.batch.Reset()
|
||||
@@ -377,8 +369,7 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
failures := 0
|
||||
for {
|
||||
if failures > 0 && !t.waitBackoff(ctx, failures) {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
break
|
||||
}
|
||||
transient, err := t.post(ctx, body)
|
||||
if err == nil {
|
||||
@@ -386,7 +377,7 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
return true
|
||||
return
|
||||
}
|
||||
t.requestErrors.Add(1)
|
||||
if !transient {
|
||||
@@ -396,11 +387,10 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
"target", t.url,
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return true
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
return false
|
||||
break
|
||||
}
|
||||
failures++
|
||||
t.logger.Warn("msg", "Chain batch delivery failed",
|
||||
@@ -409,27 +399,11 @@ func (t *HTTPChainSink) flush(ctx context.Context) bool {
|
||||
"attempt", failures,
|
||||
"error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// finalFlush best-effort delivers the pending batch during shutdown (single attempt)
|
||||
func (t *HTTPChainSink) finalFlush() {
|
||||
if t.batchCount == 0 {
|
||||
return
|
||||
}
|
||||
fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout)
|
||||
defer cancel()
|
||||
|
||||
count := t.batchCount
|
||||
if _, err := t.post(fctx, t.batch.Bytes()); err != nil {
|
||||
t.droppedBatches.Add(1)
|
||||
t.logger.Warn("msg", "Final chain batch dropped on shutdown",
|
||||
t.logger.Warn("msg", "Chain batch dropped on shutdown",
|
||||
"component", "http_chain_sink",
|
||||
"entries", count,
|
||||
"error", err)
|
||||
return
|
||||
}
|
||||
t.batchesSent.Add(1)
|
||||
t.totalProcessed.Add(uint64(count))
|
||||
"target", t.url,
|
||||
"entries", count)
|
||||
}
|
||||
|
||||
// post sends one NDJSON batch; transient=true marks retryable failures
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package httpchain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -121,7 +123,8 @@ func TestRefusedTokenLogsInAgainAndDeliversOnce(t *testing.T) {
|
||||
}
|
||||
sink := created.(*HTTPChainSink)
|
||||
sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("entry")})
|
||||
if !sink.flush(t.Context()) || delivered.Load() != 1 || sink.droppedBatches.Load() != 0 {
|
||||
sink.flush(t.Context())
|
||||
if delivered.Load() != 1 || sink.droppedBatches.Load() != 0 {
|
||||
t.Fatalf("delivered %d, dropped %d", delivered.Load(), sink.droppedBatches.Load())
|
||||
}
|
||||
if logins := listener.Stats()["auth_allowed"]; logins != uint64(2) {
|
||||
@@ -188,7 +191,8 @@ func TestLinkOverIPv6LoopbackWithTLS(t *testing.T) {
|
||||
sink.append(core.TransportEvent{Time: time.Now(), Payload: []byte("over IPv6")})
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
if !sink.flush(ctx) || sink.droppedBatches.Load() != 0 {
|
||||
sink.flush(ctx)
|
||||
if sink.droppedBatches.Load() != 0 {
|
||||
t.Fatalf("batch not delivered: %d request errors", sink.requestErrors.Load())
|
||||
}
|
||||
select {
|
||||
@@ -200,3 +204,50 @@ func TestLinkOverIPv6LoopbackWithTLS(t *testing.T) {
|
||||
t.Fatal("the source published nothing")
|
||||
}
|
||||
}
|
||||
|
||||
// Stop delivers what is queued and batched, so the end of a finite input
|
||||
// reaches the source whole, and returns at the flush bound when the source
|
||||
// stops answering.
|
||||
func TestStopDeliversTheQueueWithinTheBound(t *testing.T) {
|
||||
const n = 250 // three batches of the default 100
|
||||
manager := session.NewManager(time.Hour)
|
||||
t.Cleanup(manager.Stop)
|
||||
// stop queues n entries, starts and stops a sink, and returns how long
|
||||
// Stop took and how many entries an answering source received
|
||||
stop := func(answering bool) (time.Duration, int64) {
|
||||
var received atomic.Int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if !answering {
|
||||
<-r.Context().Done() // the sink gave up
|
||||
return
|
||||
}
|
||||
received.Add(int64(bytes.Count(body, []byte("\n"))))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
_, port, _ := net.SplitHostPort(srv.Listener.Addr().String())
|
||||
p, _ := strconv.ParseInt(port, 10, 64)
|
||||
created, err := NewHTTPChainSinkPlugin("fwd", map[string]any{"host": "127.0.0.1", "port": p},
|
||||
log.NewLogger(), session.NewProxy(manager, "fwd"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := created.(*HTTPChainSink)
|
||||
for range n {
|
||||
sink.Input() <- core.TransportEvent{Time: time.Now(), Payload: []byte("entry")}
|
||||
}
|
||||
if err := sink.Start(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
sink.Stop()
|
||||
return time.Since(start), received.Load()
|
||||
}
|
||||
if _, got := stop(true); got != n {
|
||||
t.Fatalf("the source received %d of %d entries", got, n)
|
||||
}
|
||||
if d, _ := stop(false); d > 3500*time.Millisecond {
|
||||
t.Fatalf("Stop took %v past a 2 s flush bound", d)
|
||||
}
|
||||
}
|
||||
+21
-3
@@ -36,9 +36,9 @@ type SinkStats struct {
|
||||
Details map[string]any
|
||||
}
|
||||
|
||||
// FlushBound is how long a network sink's Stop gives its clients to take
|
||||
// what is queued: write_timeout_ms, at most core.SinkFlushTimeout, which a
|
||||
// reload also waits out for a client that stopped reading.
|
||||
// FlushBound is how long a network sink's Stop gives what is queued to reach
|
||||
// its clients or its downstream: its write or request timeout, at most
|
||||
// core.SinkFlushTimeout, which a reload also waits out for a stalled peer.
|
||||
func FlushBound(writeTimeout time.Duration) time.Duration {
|
||||
if writeTimeout > 0 && writeTimeout < core.SinkFlushTimeout {
|
||||
return writeTimeout
|
||||
@@ -46,6 +46,24 @@ func FlushBound(writeTimeout time.Duration) time.Duration {
|
||||
return core.SinkFlushTimeout
|
||||
}
|
||||
|
||||
// FlushContext is ctx ending FlushBound(timeout) after done closes: a sink
|
||||
// that dials runs its loop under it, so Stop leaves its queue that long.
|
||||
func FlushContext(ctx context.Context, done <-chan struct{}, timeout time.Duration) (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
go func() {
|
||||
select {
|
||||
case <-done:
|
||||
select {
|
||||
case <-time.After(FlushBound(timeout)):
|
||||
case <-ctx.Done():
|
||||
}
|
||||
case <-ctx.Done():
|
||||
}
|
||||
cancel()
|
||||
}()
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
// Drain takes the payloads queued in input without waiting: a network sink's
|
||||
// last events, which each client then writes on its own.
|
||||
func Drain(input <-chan core.TransportEvent) [][]byte {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/authz"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/sink"
|
||||
@@ -74,6 +75,7 @@ type TCPSink struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
|
||||
// Runtime. Stop closes flush; the broadcast loop then moves what remains
|
||||
// of its input to tail and closes flushed; each writer writes its queue
|
||||
@@ -149,6 +151,10 @@ func NewTCPSinkPlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "tcp_sink", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
t := &TCPSink{
|
||||
id: id,
|
||||
@@ -166,6 +172,7 @@ func NewTCPSinkPlugin(
|
||||
writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
}
|
||||
t.lastProcessed.Store(time.Time{})
|
||||
|
||||
@@ -176,7 +183,8 @@ func NewTCPSinkPlugin(
|
||||
"port", opts.Port,
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe())
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe())
|
||||
tlsx.LogWarnings(logger, "tcp_sink", id, opts.TLS, true)
|
||||
authPolicy.LogStartup(logger, "tcp_sink", id, false)
|
||||
return t, nil
|
||||
@@ -214,6 +222,7 @@ func (t *TCPSink) listen() (net.Listener, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ln = t.acl.Listener(ln)
|
||||
if t.tlsConfig != nil {
|
||||
ln = tls.NewListener(ln, t.tlsConfig)
|
||||
}
|
||||
@@ -556,6 +565,7 @@ func (t *TCPSink) GetStats() sink.SinkStats {
|
||||
"tls_handshake_errors": t.tlsHandshakeErrors.Load(),
|
||||
}
|
||||
maps.Copy(details, t.auth.Stats())
|
||||
maps.Copy(details, t.acl.Stats())
|
||||
|
||||
return sink.SinkStats{
|
||||
ID: t.id,
|
||||
|
||||
@@ -219,8 +219,7 @@ func (t *TCPChainSink) Start(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop terminates the forwarding loop. Worst-case latency: one write timeout
|
||||
// plus one backoff wait (both interruptible or bounded).
|
||||
// Stop delivers what is queued within sink.FlushBound, then ends the loop
|
||||
func (t *TCPChainSink) Stop() {
|
||||
t.logger.Info("msg", "Stopping TCP chain sink",
|
||||
"component", "tcp_chain_sink",
|
||||
@@ -273,27 +272,32 @@ func (t *TCPChainSink) runLoop(ctx context.Context) {
|
||||
defer t.wg.Done()
|
||||
defer t.closeConn()
|
||||
|
||||
// Fold done into the context, so a connect or exchange in flight ends on Stop
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
// Stop leaves sink.FlushBound to deliver the queue, so a finite input
|
||||
// arrives whole; a connect, exchange or link still open then ends
|
||||
ctx, cancel := sink.FlushContext(ctx, t.done, t.writeTimeout)
|
||||
defer cancel()
|
||||
go func() {
|
||||
select {
|
||||
case <-t.done:
|
||||
cancel()
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.done:
|
||||
return
|
||||
case event, ok := <-t.input:
|
||||
if !ok {
|
||||
for len(t.input) > 0 {
|
||||
if !t.forward(ctx, <-t.input) {
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
case event := <-t.input:
|
||||
if !t.forward(ctx, event) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// forward delivers one event; false once shutdown ends the delivery
|
||||
func (t *TCPChainSink) forward(ctx context.Context, event core.TransportEvent) bool {
|
||||
entry, synthesized := chain.EntryFromEvent(event, t.node, t.id)
|
||||
if synthesized {
|
||||
t.synthesized.Add(1)
|
||||
@@ -304,16 +308,15 @@ func (t *TCPChainSink) runLoop(ctx context.Context) {
|
||||
t.logger.Error("msg", "Failed to marshal chain entry",
|
||||
"component", "tcp_chain_sink",
|
||||
"error", err)
|
||||
continue
|
||||
return true
|
||||
}
|
||||
if !t.deliver(ctx, append(line, '\n')) {
|
||||
return // shutdown during retry
|
||||
return false
|
||||
}
|
||||
t.totalProcessed.Add(1)
|
||||
t.lastProcessed.Store(time.Now())
|
||||
t.proxy.UpdateActivity(t.session.ID)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// deliver writes one line, holding it across reconnects until sent or shutdown.
|
||||
@@ -399,11 +402,14 @@ func (t *TCPChainSink) connect(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
// A source never writes once a link is up: a line is a refusal this sink
|
||||
// could not otherwise see, EOF a close. Stop writing into either.
|
||||
// could not otherwise see, EOF a close. Stop writing into either. The end
|
||||
// of ctx closes the link too, cutting short a write in flight.
|
||||
stop := context.AfterFunc(ctx, func() { conn.Close() })
|
||||
t.wg.Add(1)
|
||||
go func() {
|
||||
defer t.wg.Done()
|
||||
err := authz.AwaitClose(r)
|
||||
stop()
|
||||
conn.Close()
|
||||
if errors.Is(err, authz.ErrRefused) {
|
||||
t.logger.Warn("msg", "Chain link refused",
|
||||
@@ -450,7 +456,5 @@ func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool {
|
||||
return true
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-t.done:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,10 @@ package tcpchain
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -66,3 +68,72 @@ func TestRefusingSourceIsRetriedWithBackoff(t *testing.T) {
|
||||
t.Fatalf("%d connections in 1.5 s against a refusing source", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Stop delivers what is queued, so the end of a finite input reaches the
|
||||
// source whole, and returns at the flush bound when the source stops reading.
|
||||
func TestStopDeliversTheQueueWithinTheBound(t *testing.T) {
|
||||
const n = 2000 // of 8 KiB: more than a stalled source's socket buffers hold
|
||||
padding := strings.Repeat("x", 8192)
|
||||
manager := session.NewManager(time.Hour)
|
||||
t.Cleanup(manager.Stop)
|
||||
// stop queues n entries, starts and stops a sink, and returns how long
|
||||
// Stop took and how many entries a reading source received
|
||||
stop := func(reading bool) (time.Duration, int) {
|
||||
ln, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { ln.Close() })
|
||||
lines := make(chan int, 1)
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
if !reading {
|
||||
<-t.Context().Done()
|
||||
return
|
||||
}
|
||||
count, scanner := -1, bufio.NewScanner(conn) // -1: the hello
|
||||
scanner.Buffer(nil, 64*1024)
|
||||
for scanner.Scan() {
|
||||
count++
|
||||
}
|
||||
lines <- count
|
||||
}()
|
||||
_, port, _ := net.SplitHostPort(ln.Addr().String())
|
||||
p, _ := strconv.ParseInt(port, 10, 64)
|
||||
created, err := NewTCPChainSinkPlugin("fwd", map[string]any{
|
||||
"host": "127.0.0.1", "port": p, "buffer_size": int64(n), // writes time out at 5 s
|
||||
}, log.NewLogger(), session.NewProxy(manager, "fwd"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sink := created.(*TCPChainSink)
|
||||
for i := range n {
|
||||
sink.Input() <- core.TransportEvent{Time: time.Now(), Payload: fmt.Appendf(nil, "entry %d %s", i, padding)}
|
||||
}
|
||||
if err := sink.Start(t.Context()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
sink.Stop()
|
||||
d := time.Since(start)
|
||||
if !reading {
|
||||
return d, 0
|
||||
}
|
||||
select {
|
||||
case got := <-lines:
|
||||
return d, got
|
||||
case <-time.After(5 * time.Second):
|
||||
return d, 0
|
||||
}
|
||||
}
|
||||
if _, got := stop(true); got != n {
|
||||
t.Fatalf("the source received %d of %d entries", got, n)
|
||||
}
|
||||
if d, _ := stop(false); d > 3500*time.Millisecond {
|
||||
t.Fatalf("Stop took %v past a 2 s flush bound", d)
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/chain"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/source"
|
||||
@@ -59,6 +60,7 @@ type HTTPChainSource struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
|
||||
// Session cache: one session per remote host + node + authenticated identity
|
||||
sessions map[string]string // key -> sessionID
|
||||
@@ -120,6 +122,10 @@ func NewHTTPChainSourcePlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "http_chain_source", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &HTTPChainSource{
|
||||
id: id,
|
||||
@@ -131,6 +137,7 @@ func NewHTTPChainSourcePlugin(
|
||||
logger: logger,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
@@ -142,7 +149,8 @@ func NewHTTPChainSourcePlugin(
|
||||
"ingest_path", opts.IngestPath,
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe())
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe())
|
||||
tlsx.LogWarnings(logger, "http_chain_source", id, opts.TLS, true)
|
||||
authPolicy.LogStartup(logger, "http_chain_source", id, opts.TrustNode)
|
||||
return s, nil
|
||||
@@ -180,6 +188,7 @@ func (s *HTTPChainSource) Start() error {
|
||||
s.auth.Close()
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
ln = s.acl.Listener(ln)
|
||||
|
||||
mux := http.NewServeMux()
|
||||
// Method-scoped pattern: mux answers 405 with Allow header on non-POST
|
||||
@@ -265,6 +274,7 @@ func (s *HTTPChainSource) GetStats() source.SourceStats {
|
||||
"trust_node": s.config.TrustNode,
|
||||
}
|
||||
maps.Copy(details, s.auth.Stats())
|
||||
maps.Copy(details, s.acl.Stats())
|
||||
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/lixenwraith/logwisp/internal/chain"
|
||||
"github.com/lixenwraith/logwisp/internal/config"
|
||||
"github.com/lixenwraith/logwisp/internal/core"
|
||||
"github.com/lixenwraith/logwisp/internal/netacl"
|
||||
"github.com/lixenwraith/logwisp/internal/plugin"
|
||||
"github.com/lixenwraith/logwisp/internal/session"
|
||||
"github.com/lixenwraith/logwisp/internal/source"
|
||||
@@ -55,6 +56,7 @@ type TCPChainSource struct {
|
||||
|
||||
// Authorization
|
||||
auth *authz.Policy
|
||||
acl *netacl.Policy
|
||||
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
@@ -105,6 +107,10 @@ func NewTCPChainSourcePlugin(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aclPolicy, err := netacl.New(opts.ACL, opts.Host, logger, "tcp_chain_source", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s := &TCPChainSource{
|
||||
id: id,
|
||||
@@ -116,6 +122,7 @@ func NewTCPChainSourcePlugin(
|
||||
logger: logger,
|
||||
tlsConfig: tlsCfg,
|
||||
auth: authPolicy,
|
||||
acl: aclPolicy,
|
||||
}
|
||||
s.lastEntryTime.Store(time.Time{})
|
||||
|
||||
@@ -126,7 +133,8 @@ func NewTCPChainSourcePlugin(
|
||||
"port", opts.Port,
|
||||
"tls", tlsCfg != nil,
|
||||
"mtls", tlsCfg != nil && tlsCfg.ClientAuth == tls.RequireAndVerifyClientCert,
|
||||
"auth", authPolicy.Describe())
|
||||
"auth", authPolicy.Describe(),
|
||||
"acl", aclPolicy.Describe())
|
||||
tlsx.LogWarnings(logger, "tcp_chain_source", id, opts.TLS, true)
|
||||
authPolicy.LogStartup(logger, "tcp_chain_source", id, opts.TrustNode)
|
||||
return s, nil
|
||||
@@ -166,6 +174,7 @@ func (s *TCPChainSource) Start() error {
|
||||
s.auth.Close()
|
||||
return fmt.Errorf("listen %s: %w", addr, err)
|
||||
}
|
||||
ln = s.acl.Listener(ln)
|
||||
if s.tlsConfig != nil {
|
||||
ln = tls.NewListener(ln, s.tlsConfig)
|
||||
}
|
||||
@@ -226,6 +235,7 @@ func (s *TCPChainSource) GetStats() source.SourceStats {
|
||||
"trust_node": s.config.TrustNode,
|
||||
}
|
||||
maps.Copy(details, s.auth.Stats())
|
||||
maps.Copy(details, s.acl.Stats())
|
||||
|
||||
return source.SourceStats{
|
||||
ID: s.id,
|
||||
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
# logwisp acl test: each listener kind admits or refuses peers by address
|
||||
# before TLS, counts and warns of refusals, rejects rules it can never apply,
|
||||
# and still hands a finite input whole to the clients it admits. One-shot.
|
||||
# Ports: 15881-15889 on 127.0.0.1. Requires: bash 5+, curl, nc; Linux.
|
||||
|
||||
set -u
|
||||
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
|
||||
|
||||
RUN=$E2E_DIR/run/acl
|
||||
AUTH=$RUN
|
||||
e2e_init "$@"
|
||||
|
||||
section "Setup"
|
||||
need curl nc
|
||||
ports_free 15881 15882 15884 15885 15889
|
||||
rm -rf "$RUN"
|
||||
mkdir -p "$RUN/out" "$LOG"
|
||||
cd "$RUN" || abort "no run directory"
|
||||
for v in $(compgen -e LOGWISP_); do unset "$v"; done
|
||||
export HOME=$RUN
|
||||
lw() { "$BIN" "$@"; }
|
||||
get() { curl -s --noproxy '*' --max-time 3 "$@"; } # curl ARGS...: never through a proxy
|
||||
# exit 45: curl could not bind the address; 7, it bound and found 15889 closed
|
||||
get --interface 127.0.0.2 http://127.0.0.1:15889/
|
||||
(($? == 45)) && skip_all "this host cannot send from 127.0.0.2"
|
||||
seq 1 300 | sed 's/^/edge line /' >"$RUN/edge.log"
|
||||
add_users "$RUN/users.toml" edge-01
|
||||
|
||||
section "Rules"
|
||||
# rejects SINK PATTERN: lw --check fails on SINK with PATTERN in its error
|
||||
rejects() { ! lw --check --source null --sink "$1" >/dev/null 2>"$LOG/check.err" && grep -q -- "$2" "$LOG/check.err" && echo 1 || echo 0; }
|
||||
# warns SINK PATTERN: lw --check builds SINK and WARNs PATTERN
|
||||
warns() { lw --check --source null --sink "$1" 2>&1 >/dev/null | grep ' WARN ' | grep -q -- "$2" && echo 1 || echo 0; }
|
||||
check "an IPv4 entry on an IPv6 listener is refused" "$(rejects 'tcp,host=::1,port=15881,acl.allow=10.0.0.0/8' "not of the listener's family")"
|
||||
check "an acl on a dialer is refused" "$(rejects 'tcp_chain,host=127.0.0.1,port=15881,acl.deny=10.0.0.1' 'unknown key "acl"')"
|
||||
check "a CIDR with host bits warns, naming the network it matches" "$(warns 'tcp,host=127.0.0.1,port=15881,acl.allow=10.0.0.1/8' 'network 10.0.0.0/8')"
|
||||
check "a deny list alone on a wildcard listener warns" "$(warns 'tcp,port=15881,acl.deny=10.0.0.1' 'admits all that acl.deny does not list')"
|
||||
check "an allow entry of a whole family warns" "$(warns 'tcp,port=15881,acl.allow=0.0.0.0/0' 'admits every address of its family')"
|
||||
|
||||
section "HTTP sink"
|
||||
spawn serve "$BIN" --preset "serve,path=$RUN/edge.log,listen=127.0.0.1:15881,allow=127.0.0.0/8,deny=127.0.0.2"
|
||||
wait_port 15881 || abort "the serve preset did not listen" serve
|
||||
check "a client the allow list names reads the status" "$(get http://127.0.0.1:15881/status | grep -q '"acl":"allow=1 deny=1"' && echo 1 || echo 0)"
|
||||
get --interface 127.0.0.2 -o /dev/null http://127.0.0.1:15881/status
|
||||
rc=$?
|
||||
check "a denied client is closed before a byte of HTTP (curl exit $rc)" "$( ((rc == 52 || rc == 56)) && echo 1 || echo 0)"
|
||||
get --interface 127.0.0.2 http://127.0.0.1:15881/status
|
||||
get --interface 127.0.0.2 http://127.0.0.1:15881/status
|
||||
check "the status counts every refusal" "$(get http://127.0.0.1:15881/status | grep -q '"acl_denied":3' && echo 1 || echo 0)"
|
||||
check "and the log warns of them once ($(count serve 'refused by acl'))" \
|
||||
"$( [[ $(count serve 'refused by acl') == 1 ]] && logged serve 'remote_addr 127.0.0.2:' && echo 1 || echo 0)"
|
||||
|
||||
section "TCP sink"
|
||||
# the lines of a finite input, once the clients had time to connect
|
||||
seq 1 20000 | sed 's/^/flush /' >"$RUN/flush.in"
|
||||
{ wait_port 15882; sleep 1; cat "$RUN/flush.in"; } |
|
||||
lw --sink "tcp,host=127.0.0.1,port=15882,acl.allow=127.0.0.1,buffer_size=20000,client_buffer_size=20000" 2>"$LOG/tcp.err" &
|
||||
fpid=$!
|
||||
wait_port 15882
|
||||
tcp_read 15882 15 >"$RUN/flush.out" &
|
||||
rpid=$!
|
||||
denied=$(timeout 3 nc -s 127.0.0.2 127.0.0.1 15882 </dev/null | wc -c)
|
||||
wait "$rpid" "$fpid"
|
||||
n=$(grep -c '^flush ' "$RUN/flush.out")
|
||||
check "a client the acl admits gets all 20000 lines of a finite input ($n)" "$((n == 20000))"
|
||||
check "one it denies gets nothing ($denied bytes) and is logged" \
|
||||
"$( ((denied == 0)) && grep -q 'refused by acl.* remote_addr 127.0.0.2:' "$LOG/tcp.err" && echo 1 || echo 0)"
|
||||
|
||||
section "Chain sources"
|
||||
# pin DAEMON: the pin_sha256 a self-signed listener logged
|
||||
pin() { wait_until 5 logged "$1" pin_sha256 && grep -o 'pin_sha256 "sha256//[^"]*"' "$LOG/$1.out" | head -1 | cut -d'"' -f2; }
|
||||
received() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c 'edge line'; }
|
||||
edge() { spawn "$1" "$BIN" --preset "edge,path=$RUN/edge.log,from=start,to=127.0.0.1:$2,transport=$3,pin=$4,user=edge-01,password_file=$RUN/edge-01.pass"; }
|
||||
spawn agg-allow "$BIN" --preset "aggregator,listen=127.0.0.1:15884,allow=127.0.0.1,users=$RUN/users.toml,out=$RUN/out/allow"
|
||||
spawn agg-deny "$BIN" --preset "aggregator,listen=127.0.0.1:15885,transport=http,deny=127.0.0.0/8,users=$RUN/users.toml,out=$RUN/out/deny"
|
||||
wait_port 15884 && wait_port 15885 || abort "an aggregator did not listen" agg-allow
|
||||
edge edge-allow 15884 tcp "$(pin agg-allow)"
|
||||
edge edge-deny 15885 http "$(pin agg-deny)"
|
||||
timeout 3 nc -s 127.0.0.2 127.0.0.1 15884 </dev/null
|
||||
wait_until 10 eval '(( $(received allow) >= 300 ))'
|
||||
check "a tcp_chain source admits an edge its allow list names ($(received allow)), and refuses 127.0.0.2" \
|
||||
"$( (($(received allow) == 300)) && logged agg-allow 'refused by acl.* remote_addr 127.0.0.2:' && echo 1 || echo 0)"
|
||||
wait_until 10 logged agg-deny 'refused by acl'
|
||||
check "an http_chain source refuses a denied edge, which delivers nothing ($(received deny))" \
|
||||
"$(logged agg-deny 'refused by acl' && (($(received deny) == 0)) && echo 1 || echo 0)"
|
||||
|
||||
summary
|
||||
+9
-1
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# logwisp preset test: colors, presets, --dump, lw tls, listeners that make
|
||||
# their certificate (self-signed and pinned, or issued), the serve preset's
|
||||
# browser viewer, and the http and tcp sinks writing a finite input whole.
|
||||
# browser viewer, and the network sinks delivering a finite input whole.
|
||||
# One-shot: both modes run the checks.
|
||||
# Ports: 15890-15896 on 127.0.0.1.
|
||||
# Requires: bash 5+, coreutils, curl, openssl. Linux dev host only.
|
||||
@@ -114,5 +114,13 @@ wait "$fpid"
|
||||
n=$(grep -c '^data: flush ' "$RUN/flush-http.out")
|
||||
check "an http stream gets all 20000 lines, then the disconnect event ($n)" \
|
||||
"$( ((n == 20000)) && tail -3 "$RUN/flush-http.out" | grep -q '^event: disconnect' && echo 1 || echo 0)"
|
||||
# 1000 lines: the chain sink's queue, which holds them until the link is up
|
||||
flushed() { cat "$RUN/out/$1"/aggregate*.log 2>/dev/null | grep -c '"flush '; }
|
||||
seq 1 1000 | sed 's/^/flush /' | lw --preset "edge,to=127.0.0.1:15893,pin=$PIN,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-tcp-chain.err"
|
||||
wait_until 5 eval '(( $(flushed self) >= 1000 ))'
|
||||
check "a tcp_chain edge delivers all 1000 lines before it exits ($(flushed self))" "$(( $(flushed self) == 1000 ))"
|
||||
seq 1 1000 | sed 's/^/flush /' | lw --preset "edge,to=127.0.0.1:15894,transport=http,ca=$RUN/pki/ca.crt,user=edge-01,password_file=$RUN/edge-01.pass" 2>"$LOG/flush-http-chain.err"
|
||||
wait_until 5 eval '(( $(flushed issuer) >= 1000 ))'
|
||||
check "an http_chain edge delivers all 1000 lines before it exits ($(flushed issuer))" "$(( $(flushed issuer) == 1000 ))"
|
||||
|
||||
summary
|
||||
Reference in new issue
Block a user