lb: keep the client's address past weave's masquerade

Weave's expose rule (nat WEAVE: ! -s <alloc> -d <alloc> -j MASQUERADE)
masquerades everything entering the pod network from outside unless it is
for a local pod (weaver-no-masq-local). Live on the SG cluster a pod on
another node than the holder saw the holder's bridge address instead of
the client's. A new chain, REWEAVE-LB-POST, hooked first in nat
POSTROUTING, lets a connection we DNATed to a held address leave the nat
table with its source untouched; one that comes from a pod (hairpin) is
masqueraded to the holder instead, so its reply returns through the
holder's conntrack.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ

lb/datapath.go

@@ -17,6 +17,8 @@ import (
 const (
 	// ForwardChain (nat, from PREROUTING) DNATs held addresses' ports to endpoints.
 	ForwardChain = "REWEAVE-LB"
+	// PostChain (nat, from POSTROUTING, ahead of weave's WEAVE) keeps the client's address on what ForwardChain DNATed.
+	PostChain = "REWEAVE-LB-POST"
 	// MarkChain (mangle, from PREROUTING) marks holders' connections and steers their replies.
 	MarkChain = "REWEAVE-LB-MARK"
 	// servicePrefix names the per-port chains ForwardChain jumps to.
@@ -52,7 +54,21 @@ func Render(d Desired, podNet *net.IPNet, stale []string) Rules {
 	var nat bytes.Buffer
 	nat.WriteString("*nat\n")
 	fmt.Fprintf(&nat, ":%s - [0:0]\n", ForwardChain)
+	fmt.Fprintf(&nat, ":%s - [0:0]\n", PostChain)
 	var body bytes.Buffer
+	seen := map[string]bool{}
+	for _, f := range d.Forwards {
+		a := f.Address.String()
+		if seen[a] {
+			continue
+		}
+		seen[a] = true
+		// weave masquerades everything entering the pod network from outside unless it is for a local pod
+		// (its expose rule); a connection we DNATed must keep the client's address, so it leaves the nat
+		// table here. One from a pod (hairpin) is masqueraded to this node, so its reply comes back here.
+		fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -s %s -j MASQUERADE\n", PostChain, a, podNet)
+		fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -j ACCEPT\n", PostChain, a)
+	}
 	for _, f := range d.Forwards {
 		for _, p := range f.Ports {
 			c := portChain(f, p)
@@ -153,9 +169,14 @@ func (dp *Datapath) Apply(d Desired, podNet *net.IPNet) error {
 	return dp.routes(d.Holders)
 }
 
-// hooks makes sure PREROUTING jumps to our chains first (kube-proxy's KUBE-SERVICES must not see held addresses first).
+// hooks makes sure PREROUTING and POSTROUTING jump to our chains first (kube-proxy's and weave's must not see
+// held addresses' connections first).
 func (dp *Datapath) hooks() error {
-	for _, h := range []struct{ table, chain string }{{"nat", ForwardChain}, {"mangle", MarkChain}} {
+	for _, h := range []struct{ table, from, chain string }{
+		{"nat", "PREROUTING", ForwardChain},
+		{"nat", "POSTROUTING", PostChain},
+		{"mangle", "PREROUTING", MarkChain},
+	} {
 		ok, err := dp.ipt.ChainExists(h.table, h.chain)
 		if err != nil {
 			return err
@@ -166,7 +187,7 @@ func (dp *Datapath) hooks() error {
 			}
 		}
 		rule := []string{"-m", "comment", "--comment", "reweave lb", "-j", h.chain}
-		rules, err := dp.ipt.List(h.table, "PREROUTING")
+		rules, err := dp.ipt.List(h.table, h.from)
 		if err != nil {
 			return err
 		}
@@ -174,10 +195,10 @@ func (dp *Datapath) hooks() error {
 		if len(rules) > 1 && strings.HasSuffix(rules[1], "-j "+h.chain) {
 			continue
 		}
-		if err := dp.ipt.DeleteIfExists(h.table, "PREROUTING", rule...); err != nil {
+		if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
 			return err
 		}
-		if err := dp.ipt.Insert(h.table, "PREROUTING", 1, rule...); err != nil {
+		if err := dp.ipt.Insert(h.table, h.from, 1, rule...); err != nil {
 			return err
 		}
 	}

lb/model_test.go

@@ -206,7 +206,10 @@ func TestRenderedRules(t *testing.T) {
 	d := Compute("w1", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
 	r := Render(d, podNet, []string{"REWEAVE-LBS-DEADBEEF0000"})
 	for _, want := range []string{
-		"*nat\n:REWEAVE-LB - [0:0]\n",
+		"*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\n",
+		// the client's address survives weave's masquerade; a pod's own connection (hairpin) is masqueraded here
+		"-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -s 10.32.0.0/12 -j MASQUERADE\n",
+		"-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -j ACCEPT\n",
 		":REWEAVE-LBS-DEADBEEF0000 - [0:0]\n", // stale chain emptied
 		"-A REWEAVE-LB -d 86.107.168.15/32 -p tcp -m tcp --dport 22 -m comment --comment \"gitea/gitea\" -j REWEAVE-LBS-",
 		"-p tcp -m statistic --mode random --probability 0.3333333333 -j DNAT --to-destination 10.40.0.7:2222\n",
@@ -224,8 +227,8 @@ func TestRenderedRules(t *testing.T) {
 			t.Fatalf("DNAT without its protocol: %q", line)
 		}
 	}
-	if strings.Contains(r.Nat, "SNAT") || strings.Contains(r.Nat, "MASQUERADE") {
-		t.Fatalf("the client's address must survive:\n%s", r.Nat)
+	if strings.Count(r.Nat, "MASQUERADE") != 1 || strings.Contains(r.Nat, "SNAT") {
+		t.Fatalf("only a pod's hairpin connection may be masqueraded:\n%s", r.Nat)
 	}
 	if r.Mangle != "*mangle\n:REWEAVE-LB-MARK - [0:0]\nCOMMIT\n" {
 		t.Fatalf("a holder marks nothing:\n%s", r.Mangle)
@@ -241,7 +244,7 @@ func TestRenderedRules(t *testing.T) {
 			t.Fatalf("mangle lacks %q:\n%s", want, r.Mangle)
 		}
 	}
-	if r.Nat != "*nat\n:REWEAVE-LB - [0:0]\nCOMMIT\n" {
+	if r.Nat != "*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\nCOMMIT\n" {
 		t.Fatalf("a non-holder forwards nothing:\n%s", r.Nat)
 	}
 }