From e7583c4177851b5cedb6745277a36c818d6934cb Mon Sep 17 00:00:00 2001 From: Levalicious Date: Tue, 13 Oct 2026 23:56:45 -0400 Subject: [PATCH] reweave/test/gateway: end-to-end test of the agent's Gateways On test ports of a held address, against a live cluster: the class and the Gateway accepted and programmed with their routes attached, route statuses (including an unresolved backend), a TCP listener forwarding in the kernel to a pod on another node that sees the client's own address, HTTPS with the listener's certificate by SNI over HTTP/2 routed by exact path, prefix with a rewrite and default, X-Forwarded-For equal to that same client address, the HTTP-to-HTTPS redirect, 500 for the unresolved backend and no certificate for an unknown name, a pod inside the cluster reaching it, a renewed certificate served without a restart, failing closed on an address nobody holds and recovering, and the holder's rules gone with the Gateway. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ --- reweave/test/gateway/gateway-test.yaml | 112 ++++++++++++++++++++ reweave/test/gateway/run.sh | 141 +++++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 reweave/test/gateway/gateway-test.yaml create mode 100755 reweave/test/gateway/run.sh diff --git a/reweave/test/gateway/gateway-test.yaml b/reweave/test/gateway/gateway-test.yaml new file mode 100644 index 00000000..f39e4cb2 --- /dev/null +++ b/reweave/test/gateway/gateway-test.yaml @@ -0,0 +1,112 @@ +# Objects for the Gateway e2e (run.sh): two echo backends (agnhost netexec: /hostname, /clientip, /header), one on +# the holder and one elsewhere, and a Gateway of class reweave on test ports of ADDR (HTTPS with the test +# certificate, plain HTTP that redirects, and TCP straight to a backend), with its routes. +apiVersion: v1 +kind: Namespace +metadata: + name: gwtest +--- +apiVersion: v1 +kind: Pod +metadata: {name: echo-a, namespace: gwtest, labels: {app: echo-a}} +spec: + nodeName: HOLDER + containers: + - {name: c, image: "registry.k8s.io/e2e-test-images/agnhost:2.59", args: ["netexec", "--http-port=8080"]} +--- +apiVersion: v1 +kind: Pod +metadata: {name: echo-b, namespace: gwtest, labels: {app: echo-b}} +spec: + nodeName: OTHER + containers: + - {name: c, image: "registry.k8s.io/e2e-test-images/agnhost:2.59", args: ["netexec", "--http-port=8080"]} +--- +apiVersion: v1 +kind: Pod +metadata: {name: client, namespace: gwtest} +spec: + nodeName: OTHER + containers: + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]} +--- +apiVersion: v1 +kind: Service +metadata: {name: echo-a, namespace: gwtest} +spec: + selector: {app: echo-a} + ports: [{name: http, port: 80, targetPort: 8080}] +--- +apiVersion: v1 +kind: Service +metadata: {name: echo-b, namespace: gwtest} +spec: + selector: {app: echo-b} + ports: [{name: http, port: 80, targetPort: 8080}] +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: {name: gwtest, namespace: gateway-system} +spec: + gatewayClassName: reweave + addresses: [{type: IPAddress, value: ADDR}] + listeners: + - name: https + port: 18443 + protocol: HTTPS + hostname: gw.test + tls: {certificateRefs: [{name: gwtest-cert}]} + allowedRoutes: {namespaces: {from: Selector, selector: {matchLabels: {kubernetes.io/metadata.name: gwtest}}}} + - name: http + port: 18080 + protocol: HTTP + allowedRoutes: {namespaces: {from: All}} + - name: tcp + port: 18022 + protocol: TCP + allowedRoutes: {namespaces: {from: All}} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: {name: main, namespace: gwtest} +spec: + parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: https}] + hostnames: [gw.test] + rules: + # exact beats the default prefix "/"; /a/... reaches echo-a with /a cut off; the rest goes to echo-a as is + - matches: [{path: {type: Exact, value: /hostname}}] + backendRefs: [{name: echo-b, port: 80}] + - matches: [{path: {type: PathPrefix, value: /a}}] + backendRefs: [{name: echo-a, port: 80}] + filters: + - type: URLRewrite + urlRewrite: {path: {type: ReplacePrefixMatch, replacePrefixMatch: /}} + - backendRefs: [{name: echo-a, port: 80}] +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: {name: broken, namespace: gwtest} +spec: + parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: https}] + hostnames: [gw.test] + rules: + - matches: [{path: {type: PathPrefix, value: /broken}}] + backendRefs: [{name: missing, port: 80}] +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: {name: redirect, namespace: gwtest} +spec: + parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: http}] + rules: + - filters: + - type: RequestRedirect + requestRedirect: {scheme: https, port: 18443, statusCode: 301} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: TCPRoute +metadata: {name: raw, namespace: gwtest} +spec: + parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: tcp}] + rules: + - backendRefs: [{name: echo-b, port: 80}] diff --git a/reweave/test/gateway/run.sh b/reweave/test/gateway/run.sh new file mode 100755 index 00000000..25a1ad84 --- /dev/null +++ b/reweave/test/gateway/run.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# End-to-end test of the agent's Gateways (class reweave) against a live cluster, on test ports of ADDR. +# +# CP="ssh ... root@" HOLDER_SSH="ssh ... root@" ADDR=
HOLDER= \ +# OTHER= reweave/test/gateway/run.sh +# +# The machine running this script is the outside client. Asserts, in order: +# G0 GatewayClass reweave is accepted +# G1 the Gateway is accepted and programmed at ADDR, its listeners with their routes attached +# G2 route statuses: accepted and resolved, the broken one ResolvedRefs False (BackendNotFound) +# G3 TCP listener: forwarded in the kernel to a pod on another node, which sees this client's own address +# G4 HTTPS from outside: the listener's certificate by SNI, HTTP/2, routing by exact path, by prefix with a +# rewrite, and by default; the backend sees this client (the same address as G3) in X-Forwarded-For +# G5 plain HTTP redirects to HTTPS on the listener's port, path and query kept +# G6 an unresolved backend answers 500; a name no listener has gets no certificate +# G7 a pod inside the cluster reaches the Gateway too +# G8 a renewed certificate is served without a restart +# G9 fail closed: an address nobody holds is not programmed and nothing is served; back at ADDR, it is again +# G10 deleting the Gateway removes its accept and forward rules from the holder +# Leaves nothing behind (namespace gwtest, the Gateway and its Secret are deleted, also on failure). +set -euo pipefail + +: "${CP:?}" "${HOLDER_SSH:?}" "${ADDR:?}" "${HOLDER:?}" "${OTHER:?}" +here=$(cd "$(dirname "$0")" && pwd) +k() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $*"; } +kin() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $* -f -"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo "ok: $*"; } +work=$(mktemp -d) +cleanup() { + k -n gateway-system delete gateway gwtest --ignore-not-found >/dev/null 2>&1 || true + k -n gateway-system delete secret gwtest-cert --ignore-not-found >/dev/null 2>&1 || true + k delete namespace gwtest --ignore-not-found --wait=false >/dev/null 2>&1 || true + rm -rf "$work" +} +trap cleanup EXIT + +eventually() { + local n=$1; shift + for _ in $(seq "$n"); do + if "$@" >/dev/null 2>&1; then return 0; fi + sleep 1 + done + return 1 +} + +# a test certificate for gw.test, as a TLS Secret in gateway-system +cert() { + openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 1 -subj "/CN=gw.test" \ + -addext "subjectAltName=DNS:gw.test" -keyout "$work/$1.key" -out "$work/$1.crt" 2>/dev/null + printf 'apiVersion: v1\nkind: Secret\nmetadata: {name: gwtest-cert, namespace: gateway-system}\ntype: kubernetes.io/tls\ndata:\n tls.crt: %s\n tls.key: %s\n' \ + "$(base64 -w0 < "$work/$1.crt")" "$(base64 -w0 < "$work/$1.key")" | kin apply >/dev/null +} +fingerprint() { openssl x509 -noout -fingerprint -sha256 -in "$1" | cut -d= -f2; } +served() { + echo | openssl s_client -connect "$ADDR:18443" -servername gw.test 2>/dev/null | + openssl x509 -noout -fingerprint -sha256 2>/dev/null | cut -d= -f2 +} +https() { curl -sk --max-time 10 --resolve "gw.test:18443:$ADDR" "$@"; } + +cert v1 +sed -e "s/\bADDR\b/$ADDR/g; s/\bHOLDER\b/$HOLDER/g; s/\bOTHER\b/$OTHER/g" "$here/gateway-test.yaml" | kin apply >/dev/null +k -n gwtest wait --for=condition=Ready pod --all --timeout=300s >/dev/null || fail "test pods not ready" + +cond() { k get $1 -o "jsonpath='{.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}'"; } +is() { [ "$(cond "$1" "$2")" = "$3" ]; } +eventually 60 is "gatewayclass reweave" Accepted "True/Accepted" || fail "G0: GatewayClass: $(cond "gatewayclass reweave" Accepted)" +ok "G0. GatewayClass reweave accepted" + +gw="-n gateway-system gateway gwtest" +eventually 90 is "$gw" Programmed "True/Programmed" || fail "G1: programmed: $(cond "$gw" Programmed)" +is "$gw" Accepted "True/Accepted" || fail "G1: accepted: $(cond "$gw" Accepted)" +[ "$(k get $gw -o "jsonpath='{.status.addresses[0].value}'")" = "$ADDR" ] || fail "G1: status address" +attached=$(k get $gw -o "jsonpath='{range .status.listeners[*]}{.name}={.attachedRoutes} {end}'") +[ "$attached" = "https=2 http=1 tcp=1 " ] || fail "G1: attached routes: '$attached'" +ok "G1. Gateway accepted and programmed at $ADDR; https=2 http=1 tcp=1 routes attached" + +route() { k -n gwtest get "$1" "$2" -o "jsonpath='{.status.parents[0].conditions[?(@.type==\"$3\")].status}/{.status.parents[0].conditions[?(@.type==\"$3\")].reason}'"; } +route_is() { [ "$(route "$1" "$2" "$3")" = "$4" ]; } +for r in "httproute main" "httproute redirect" "tcproute raw" "httproute broken"; do + eventually 30 route_is $r Accepted "True/Accepted" || fail "G2: $r accepted: $(route $r Accepted)" +done +for r in "httproute main" "httproute redirect" "tcproute raw"; do + route_is $r ResolvedRefs "True/ResolvedRefs" || fail "G2: $r resolved: $(route $r ResolvedRefs)" +done +route_is httproute broken ResolvedRefs "False/BackendNotFound" || fail "G2: broken: $(route httproute broken ResolvedRefs)" +ok "G2. routes accepted and resolved; the broken one ResolvedRefs False (BackendNotFound)" + +tcp_hostname() { [ "$(curl -s --max-time 5 "http://$ADDR:18022/hostname")" = echo-b ]; } +eventually 30 tcp_hostname || fail "G3: TCP listener did not reach echo-b" +me=$(curl -s --max-time 5 "http://$ADDR:18022/clientip" | sed 's/:[0-9]*$//') +case "$me" in + 10.*|"") fail "G3: the backend saw '$me', not this client's own address" ;; +esac +ok "G3. TCP listener forwards to echo-b on $OTHER, which sees this client as $me" + +[ "$(served)" = "$(fingerprint "$work/v1.crt")" ] || fail "G4: wrong certificate for gw.test" +[ "$(https -o /dev/null -w '%{http_version}' https://gw.test:18443/hostname)" = 2 ] || fail "G4: no HTTP/2" +[ "$(https https://gw.test:18443/hostname)" = echo-b ] || fail "G4: exact /hostname did not reach echo-b" +[ "$(https https://gw.test:18443/a/hostname)" = echo-a ] || fail "G4: prefix /a (rewritten to /) did not reach echo-a" +xff=$(https "https://gw.test:18443/header?key=X-Forwarded-For") +[ "$xff" = "$me" ] || fail "G4: X-Forwarded-For '$xff', want $me" +proto=$(https "https://gw.test:18443/header?key=X-Forwarded-Proto") +[ "$proto" = https ] || fail "G4: X-Forwarded-Proto '$proto'" +ok "G4. HTTPS: gw.test's certificate, HTTP/2, exact/prefix+rewrite/default routing; X-Forwarded-For $me" + +loc=$(curl -s -o /dev/null --max-time 5 -w '%{http_code} %{redirect_url}' -H 'Host: gw.test' "http://$ADDR:18080/a/x?y=1") +[ "$loc" = "301 https://gw.test:18443/a/x?y=1" ] || fail "G5: redirect: '$loc'" +ok "G5. HTTP redirects to https://gw.test:18443, path and query kept" + +[ "$(https -o /dev/null -w '%{http_code}' https://gw.test:18443/broken)" = 500 ] || fail "G6: broken backend did not answer 500" +if curl -sk --max-time 5 --resolve "other.test:18443:$ADDR" https://other.test:18443/ >/dev/null 2>&1; then + fail "G6: a name no listener has got a certificate" +fi +ok "G6. unresolved backend answers 500; a name no listener has gets no certificate" + +inside=$(k -n gwtest exec client -- curl -sk --max-time 10 --resolve "gw.test:18443:$ADDR" https://gw.test:18443/hostname) +[ "$inside" = echo-b ] || fail "G7: from a pod: '$inside'" +ok "G7. a pod on $OTHER reaches the Gateway" + +cert v2 +renewed() { [ "$(served)" = "$(fingerprint "$work/v2.crt")" ]; } +eventually 60 renewed || fail "G8: the renewed certificate is not served" +ok "G8. renewed certificate served without a restart" + +k patch $gw --type=merge -p "'{\"spec\":{\"addresses\":[{\"type\":\"IPAddress\",\"value\":\"198.51.100.7\"}]}}'" >/dev/null +eventually 60 is "$gw" Programmed "False/AddressNotUsable" || fail "G9: unheld address: $(cond "$gw" Programmed)" +gone() { ! https -o /dev/null https://gw.test:18443/hostname; } +eventually 30 gone || fail "G9: still served from $ADDR after the Gateway moved off it" +k patch $gw --type=merge -p "'{\"spec\":{\"addresses\":[{\"type\":\"IPAddress\",\"value\":\"$ADDR\"}]}}'" >/dev/null +back() { [ "$(https https://gw.test:18443/hostname)" = echo-b ]; } +eventually 60 back || fail "G9: not served again at $ADDR" +ok "G9. an address nobody holds: Programmed False (AddressNotUsable), nothing served; back at $ADDR: served" + +k -n gateway-system delete gateway gwtest >/dev/null +rules_gone() { + ! $HOLDER_SSH "iptables -S REWEAVE-LB-IN; iptables -t nat -S REWEAVE-LB" | grep -q 'gateway-system/gwtest\|gateway gateway-system/gwtest' +} +eventually 60 rules_gone || fail "G10: the holder still has the Gateway's rules" +ok "G10. Gateway deleted: its accept and forward rules are gone from $HOLDER" +echo PASS