The live run found ping from a member on one node failing right after a member on another node pinged the same host: busybox ping uses its PID as the ICMP echo identifier, containers' PIDs are small and alike, so both nodes' echoes reached the holder as the same connection, and the holder dropped the second as a collision (fail closed, as intended). NAT treats the echo identifier as ICMP's port, and iptables accepts a range for it: members' ICMP is now SNATed into the node's block like TCP, UDP and SCTP, so every node's pings are distinct on the holder. Replies follow the holder's tag as before. The end-to-end test's ping and traceroute pods run as root (the curl image's user cannot use raw sockets even with NET_RAW). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ
This commit is contained in:
1 parent
b040f97149
commit
9261a48983
3 files changed
+14
-8
No files matched your search
@@ -106,6 +106,10 @@ func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle,
|
||||
fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n",
|
||||
PostChain, m.IP, out, EgressMark, EgressMark, p.name, m.Address, m.From, m.To)
|
||||
}
|
||||
// ICMP queries (ping) too: NAT treats the echo identifier as a port, and without a block of their own two
|
||||
// nodes' pods (whose small, alike PIDs make alike identifiers) collide on the holder
|
||||
fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p icmp -j SNAT --to-source %s:%d-%d\n",
|
||||
PostChain, m.IP, out, EgressMark, EgressMark, m.Address, m.From, m.To)
|
||||
fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j SNAT --to-source %s\n",
|
||||
PostChain, m.IP, out, EgressMark, EgressMark, m.Address)
|
||||
fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark)
|
||||
|
||||
+4
-2
@@ -211,7 +211,9 @@ func TestRenderedEgressRules(t *testing.T) {
|
||||
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n",
|
||||
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n",
|
||||
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p sctp -j SNAT --to-source 86.107.168.15:62024-62535\n",
|
||||
// everything else (ping, GRE, ...) from the address alone
|
||||
// ping within the block too (the echo identifier is NAT's port for ICMP)
|
||||
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p icmp -j SNAT --to-source 86.107.168.15:62024-62535\n",
|
||||
// everything else (GRE, ...) from the address alone
|
||||
"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j SNAT --to-source 86.107.168.15\n",
|
||||
} {
|
||||
if !strings.Contains(r.Nat, want) {
|
||||
@@ -219,7 +221,7 @@ func TestRenderedEgressRules(t *testing.T) {
|
||||
}
|
||||
}
|
||||
// the catch-all SNAT comes after the port-ranged ones (or ports would not stay in the block)
|
||||
if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p sctp -j SNAT") {
|
||||
if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p icmp -j SNAT") {
|
||||
t.Fatalf("SNAT order:\n%s", r.Nat)
|
||||
}
|
||||
for _, want := range []string{
|
||||
|
||||
@@ -33,24 +33,24 @@ items:
|
||||
spec:
|
||||
nodeName: REMOTE
|
||||
containers:
|
||||
# NET_RAW: ping and traceroute (the runtime's default capabilities leave it out)
|
||||
- {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}}
|
||||
# root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user)
|
||||
- {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}}
|
||||
- apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: {name: member-other, namespace: egtest, labels: {egress: member}}
|
||||
spec:
|
||||
nodeName: OTHER
|
||||
containers:
|
||||
# NET_RAW: ping and traceroute (the runtime's default capabilities leave it out)
|
||||
- {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}}
|
||||
# root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user)
|
||||
- {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}}
|
||||
- apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: {name: member-holder, namespace: egtest, labels: {egress: member}}
|
||||
spec:
|
||||
nodeName: HOLDER
|
||||
containers:
|
||||
# NET_RAW: ping and traceroute (the runtime's default capabilities leave it out)
|
||||
- {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}}
|
||||
# root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user)
|
||||
- {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}}
|
||||
- apiVersion: v1
|
||||
kind: Pod
|
||||
metadata: {name: member-dns, namespace: egtest, labels: {egress: member}}
|
||||
|
||||
Reference in new issue
Block a user