From 9261a48983fc444f76a592129c74edc8cff45e6b Mon Sep 17 00:00:00 2001 From: Levalicious Date: Tue, 13 Oct 2026 19:57:22 -0400 Subject: [PATCH] lb: members' ping identifiers in their node's block too The live run found ping from a member on one node failing right after a member on another node pinged the same host: busybox ping uses its PID as the ICMP echo identifier, containers' PIDs are small and alike, so both nodes' echoes reached the holder as the same connection, and the holder dropped the second as a collision (fail closed, as intended). NAT treats the echo identifier as ICMP's port, and iptables accepts a range for it: members' ICMP is now SNATed into the node's block like TCP, UDP and SCTP, so every node's pings are distinct on the holder. Replies follow the holder's tag as before. The end-to-end test's ping and traceroute pods run as root (the curl image's user cannot use raw sockets even with NET_RAW). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ --- lb/egress_datapath.go | 4 ++++ lb/egress_test.go | 6 ++++-- reweave/test/egress/egress-test.yaml | 12 ++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lb/egress_datapath.go b/lb/egress_datapath.go index d349271a..db0cf160 100644 --- a/lb/egress_datapath.go +++ b/lb/egress_datapath.go @@ -106,6 +106,10 @@ func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n", PostChain, m.IP, out, EgressMark, EgressMark, p.name, m.Address, m.From, m.To) } + // ICMP queries (ping) too: NAT treats the echo identifier as a port, and without a block of their own two + // nodes' pods (whose small, alike PIDs make alike identifiers) collide on the holder + fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p icmp -j SNAT --to-source %s:%d-%d\n", + PostChain, m.IP, out, EgressMark, EgressMark, m.Address, m.From, m.To) fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j SNAT --to-source %s\n", PostChain, m.IP, out, EgressMark, EgressMark, m.Address) fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark) diff --git a/lb/egress_test.go b/lb/egress_test.go index d43e30f7..4717b06d 100644 --- a/lb/egress_test.go +++ b/lb/egress_test.go @@ -211,7 +211,9 @@ func TestRenderedEgressRules(t *testing.T) { "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n", "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n", "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p sctp -j SNAT --to-source 86.107.168.15:62024-62535\n", - // everything else (ping, GRE, ...) from the address alone + // ping within the block too (the echo identifier is NAT's port for ICMP) + "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p icmp -j SNAT --to-source 86.107.168.15:62024-62535\n", + // everything else (GRE, ...) from the address alone "-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j SNAT --to-source 86.107.168.15\n", } { if !strings.Contains(r.Nat, want) { @@ -219,7 +221,7 @@ func TestRenderedEgressRules(t *testing.T) { } } // the catch-all SNAT comes after the port-ranged ones (or ports would not stay in the block) - if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p sctp -j SNAT") { + if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p icmp -j SNAT") { t.Fatalf("SNAT order:\n%s", r.Nat) } for _, want := range []string{ diff --git a/reweave/test/egress/egress-test.yaml b/reweave/test/egress/egress-test.yaml index 36d99534..355d7481 100644 --- a/reweave/test/egress/egress-test.yaml +++ b/reweave/test/egress/egress-test.yaml @@ -33,24 +33,24 @@ items: spec: nodeName: REMOTE containers: - # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out) - - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}} + # root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user) + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}} - apiVersion: v1 kind: Pod metadata: {name: member-other, namespace: egtest, labels: {egress: member}} spec: nodeName: OTHER containers: - # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out) - - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}} + # root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user) + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}} - apiVersion: v1 kind: Pod metadata: {name: member-holder, namespace: egtest, labels: {egress: member}} spec: nodeName: HOLDER containers: - # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out) - - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}} + # root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user) + - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}} - apiVersion: v1 kind: Pod metadata: {name: member-dns, namespace: egtest, labels: {egress: member}}