lixen/logwisp
SHA256
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.22.0 several enhancements and fixes for security, network, doc, test, packaging
Test / test (push) Canceled after 0s
Details
Test / e2e (push) Canceled after 0s
Details

Browse Source
This commit is contained in:
lixen committed 2026-10-03 14:52:16 -04:00
1 parent 9ab8c356d2
commit 71d1b82f83
102 files changed
+7102 -2178
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/chain-test.sh
+53 -163
View File
@@ -1,98 +1,33 @@
#!/usr/bin/env bash
# logwisp chain topology test
#
# random --> tcp_chain sink --> :15801 tcp_chain src --> :15803 tcp sink
# random --> http_chain sink --> :15802 http_chain src --> :15804 http sink (SSE)
#
# Usage:
# ./chain_test.sh manual mode: 2 edge daemons + relay foreground
# ./chain_test.sh --auto all daemonized, automated curl//dev/tcp checks, teardown
# ./chain_test.sh --keep (with --auto) skip teardown on success
#
# relay runs TWO independent pipelines; no cross-path aggregation:
# relay_tcp: tcp_chain src :15801 -> tcp sink :15803 (edge-tcp only)
# relay_http: http_chain src :15802 -> http sink :15804 (edge-http only)
# For fan-in aggregation (both edges -> both sinks) see chain_agg_test.sh.
#
# logwisp chain test: edge-tcp and edge-http feed a relay that runs one
# pipeline per transport, without fan-in (chain-aggregate-test.sh fans in).
# Requires: bash 5+, coreutils (timeout), curl. Linux dev host only.
set -u
. "$(dirname -- "${BASH_SOURCE[0]}")/lib.sh"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${LOGWISP_BIN:-$SCRIPT_DIR/../bin/logwisp}"
RUN="$SCRIPT_DIR/run"
CONF="$RUN/conf"
LOG="$RUN/log"
RUN=$E2E_DIR/run
PORT_TCP_CHAIN=15801
PORT_HTTP_CHAIN=15802
PORT_TCP_SINK=15803
PORT_HTTP_SINK=15804
PORTS="$PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK"
e2e_init "$@"
AUTO=0; KEEP=0
for a in "$@"; do case "$a" in
--auto) AUTO=1 ;;
--keep) KEEP=1 ;;
*) echo "unknown arg: $a" >&2; exit 1 ;;
esac; done
PIDS=()
cleanup() {
local rc=$?
trap - EXIT INT TERM
if (( ${#PIDS[@]} )); then
echo "--- teardown: stopping ${#PIDS[@]} daemon(s)"
kill -TERM "${PIDS[@]}" 2>/dev/null
local deadline=$(( SECONDS + 10 ))
for pid in "${PIDS[@]}"; do
while kill -0 "$pid" 2>/dev/null && (( SECONDS < deadline )); do sleep 0.2; done
kill -KILL "$pid" 2>/dev/null
done
fi
exit "$rc"
}
trap cleanup EXIT INT TERM
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null && exec 3>&-; }
wait_port() { # port timeout_s
local i; for (( i=0; i < $2 * 10; i++ )); do
port_open "$1" && return 0
sleep 0.1
done
return 1
}
# Reads TCP sink stream for N seconds; connection counts as a sink client
tcp_read() { # port secs
timeout "$2" bash -c "exec 3<>/dev/tcp/127.0.0.1/$1; cat <&3" 2>/dev/null || true
}
start_daemon() { # name conf
"$BIN" -c "$CONF/$2" > "$LOG/$1.out" 2>&1 &
PIDS+=($!)
echo "started $1 (pid $!)"
}
# --- Preflight ---
[[ -x "$BIN" ]] || { echo "binary not found: $BIN (build: go build -o bin/logwisp ./cmd/logwisp)" >&2; exit 1; }
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
port_open "$p" && { echo "port $p already in use" >&2; exit 1; }
done
section "Setup"
need curl
ports_free $PORTS
mkdir -p "$CONF" "$LOG"
# --- Config generation ---
cat > "$CONF/edge_tcp.toml" <<EOF
edge_conf() { # name sink_type port [sink options]
cat >"$CONF/$1.toml" <<EOF
status_reporter = false
[logging]
output = "file"
output = "stdout"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_tcp"
[[pipelines]]
name = "edge_tcp"
name = "$1"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
@@ -102,42 +37,18 @@ format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "tcp_chain"
type = "$2"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_TCP_CHAIN
node = "edge-tcp"
port = $3
node = "${1/_/-}"
${4:-}
EOF
}
edge_conf edge_tcp tcp_chain $PORT_TCP_CHAIN
edge_conf edge_http http_chain $PORT_HTTP_CHAIN "flush_interval_ms = 500"
cat > "$CONF/edge_http.toml" <<EOF
status_reporter = false
[logging]
output = "file"
level = "info"
[logging.file]
directory = "$LOG"
name = "edge_http"
[[pipelines]]
name = "edge_http"
[[pipelines.plugin_sources]]
id = "rand"
type = "random"
[pipelines.plugin_sources.config]
interval_ms = 200
format = "txt"
length = 24
[[pipelines.plugin_sinks]]
id = "to_relay"
type = "http_chain"
[pipelines.plugin_sinks.config]
host = "127.0.0.1"
port = $PORT_HTTP_CHAIN
node = "edge-http"
flush_interval_ms = 500
EOF
cat > "$CONF/relay.toml" <<EOF
cat >"$CONF/relay.toml" <<EOF
status_reporter = false
[logging]
output = "stdout"
@@ -179,70 +90,49 @@ type = "http"
host = "127.0.0.1"
port = $PORT_HTTP_SINK
EOF
info "configurations in $(short "$CONF")/"
# --- Guide ---
cat <<EOF
================================================================
logwisp chain test — port map
$PORT_TCP_CHAIN relay ingest (tcp_chain, from edge-tcp)
$PORT_HTTP_CHAIN relay ingest (http_chain, from edge-http, POST /ingest)
$PORT_TCP_SINK TCP sink -> live check: nc 127.0.0.1 $PORT_TCP_SINK
$PORT_HTTP_SINK HTTP sink -> live check: browser/curl:
http://127.0.0.1:$PORT_HTTP_SINK/stream (SSE)
http://127.0.0.1:$PORT_HTTP_SINK/status (JSON stats)
Use 127.0.0.1, not localhost — sinks reject IPv6.
Expected: json entries, node "edge-tcp" on :$PORT_TCP_SINK, "edge-http" on :$PORT_HTTP_SINK.
Logs: $LOG/
================================================================
EOF
if (( AUTO == 0 )); then
# Manual mode: edges daemonized first (chain sinks backoff-retry), relay foreground
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
echo "--- relay starting in FOREGROUND; Ctrl-C stops relay and both edges"
"$BIN" -c "$CONF/relay.toml"
exit 0
fi
# --- Auto mode ---
section "Startup"
start_daemon relay relay.toml
for p in $PORT_TCP_CHAIN $PORT_HTTP_CHAIN $PORT_TCP_SINK $PORT_HTTP_SINK; do
wait_port "$p" 10 || { echo "FAIL: relay port $p not listening (see $LOG/relay.out)"; exit 1; }
for p in $PORTS; do
wait_port "$p" || abort "relay port $p is not listening" relay
done
start_daemon edge_tcp edge_tcp.toml
# Chain sinks retry with backoff, so the edges could start first too
start_daemon edge_tcp edge_tcp.toml
start_daemon edge_http edge_http.toml
daemons_up
echo "--- settling 3s (connect + first http_chain flush)"
guide "logwisp chain test" <<EOF
Ports (IPv4 only: 127.0.0.1, not localhost):
$PORT_TCP_CHAIN relay tcp_chain ingest, from edge-tcp
$PORT_HTTP_CHAIN relay http_chain ingest (POST /ingest), from edge-http
$PORT_TCP_SINK tcp sink: edge-tcp's entries as JSON
$PORT_HTTP_SINK http sink: edge-http's entries, SSE /stream, JSON /status
Watch:
> nc 127.0.0.1 $PORT_TCP_SINK
> curl -N http://127.0.0.1:$PORT_HTTP_SINK/stream
> curl http://127.0.0.1:$PORT_HTTP_SINK/status
Logs: $(short "$LOG")/
EOF
manual_hold
section "Checks"
info "settling 3 s (connect, first http_chain flush)"
sleep 3
fail=0
check() { # label condition_result
if (( $2 )); then echo "PASS: $1"; else echo "FAIL: $1"; fail=1; fi
}
# 1. TCP chain: edge-tcp -> relay -> tcp sink
tcp_out="$(tcp_read "$PORT_TCP_SINK" 4)"
#n=$(grep -c '"node":"edge-tcp"' <<< "$tcp_out")
n=$(grep -c 'edge-tcp/' <<< "$tcp_out")
check "tcp path: entries on :$PORT_TCP_SINK with node=edge-tcp ($n lines)" $(( n >= 1 ))
n=$(grep -c 'edge-tcp/' <<<"$tcp_out")
check "tcp path: entries on :$PORT_TCP_SINK with node=edge-tcp ($n lines)" $((n >= 1))
# 2. HTTP chain: edge-http -> relay -> SSE sink
sse_out="$(curl -sN --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
n=$(grep -c '^data:.*edge-http/' <<< "$sse_out")
check "http path: SSE events on :$PORT_HTTP_SINK with node=edge-http ($n events)" $(( n >= 1 ))
sse_out="$(curl -sN --noproxy '*' --max-time 4 "http://127.0.0.1:$PORT_HTTP_SINK/stream" || true)"
n=$(grep -c '^data:.*edge-http/' <<<"$sse_out")
check "http path: SSE events on :$PORT_HTTP_SINK with node=edge-http ($n events)" $((n >= 1))
# 3. HTTP sink status endpoint
status="$(curl -s --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<< "$status" | grep -o '[0-9]*' || echo 0)
check "status endpoint: total_processed=$proc > 0" $(( proc > 0 ))
echo "================================================================"
if (( fail == 0 )); then
echo "RESULT: ALL PASS"
(( KEEP )) && { echo "--keep: daemons left running (pids: ${PIDS[*]})"; PIDS=(); }
else
echo "RESULT: FAILURES — inspect $LOG/*.out and $LOG/*.log"
fi
exit "$fail"
status="$(curl -s --noproxy '*' --max-time 3 "http://127.0.0.1:$PORT_HTTP_SINK/status" || true)"
proc=$(grep -o '"total_processed"[ :] *[0-9]*' <<<"$status" | grep -o '[0-9]*' || echo 0)
check "status endpoint: total_processed=$proc > 0" $((proc > 0))
summary