lb: keep the client's address past weave's masquerade
images / images (push) Successful in 19m13s

Weave's expose rule (nat WEAVE: ! -s <alloc> -d <alloc> -j MASQUERADE)
masquerades everything entering the pod network from outside unless it is
for a local pod (weaver-no-masq-local). Live on the SG cluster a pod on
another node than the holder saw the holder's bridge address instead of
the client's. A new chain, REWEAVE-LB-POST, hooked first in nat
POSTROUTING, lets a connection we DNATed to a held address leave the nat
table with its source untouched; one that comes from a pod (hairpin) is
masqueraded to the holder instead, so its reply returns through the
holder's conntrack.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ
This commit is contained in:
levaliciousandClaude Opus 5.5 committed 2026-10-13 13:30:08 -04:00
1 parent ef50dc921f
commit 8018ea4bf9
2 files changed
+33 -9

No files matched your search

+26 -5
View File
@@ -17,6 +17,8 @@ import (
const (
// ForwardChain (nat, from PREROUTING) DNATs held addresses' ports to endpoints.
ForwardChain = "REWEAVE-LB"
// PostChain (nat, from POSTROUTING, ahead of weave's WEAVE) keeps the client's address on what ForwardChain DNATed.
PostChain = "REWEAVE-LB-POST"
// MarkChain (mangle, from PREROUTING) marks holders' connections and steers their replies.
MarkChain = "REWEAVE-LB-MARK"
// servicePrefix names the per-port chains ForwardChain jumps to.
@@ -52,7 +54,21 @@ func Render(d Desired, podNet *net.IPNet, stale []string) Rules {
var nat bytes.Buffer
nat.WriteString("*nat\n")
fmt.Fprintf(&nat, ":%s - [0:0]\n", ForwardChain)
fmt.Fprintf(&nat, ":%s - [0:0]\n", PostChain)
var body bytes.Buffer
seen := map[string]bool{}
for _, f := range d.Forwards {
a := f.Address.String()
if seen[a] {
continue
}
seen[a] = true
// weave masquerades everything entering the pod network from outside unless it is for a local pod
// (its expose rule); a connection we DNATed must keep the client's address, so it leaves the nat
// table here. One from a pod (hairpin) is masqueraded to this node, so its reply comes back here.
fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -s %s -j MASQUERADE\n", PostChain, a, podNet)
fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -j ACCEPT\n", PostChain, a)
}
for _, f := range d.Forwards {
for _, p := range f.Ports {
c := portChain(f, p)
@@ -153,9 +169,14 @@ func (dp *Datapath) Apply(d Desired, podNet *net.IPNet) error {
return dp.routes(d.Holders)
}
// hooks makes sure PREROUTING jumps to our chains first (kube-proxy's KUBE-SERVICES must not see held addresses first).
// hooks makes sure PREROUTING and POSTROUTING jump to our chains first (kube-proxy's and weave's must not see
// held addresses' connections first).
func (dp *Datapath) hooks() error {
for _, h := range []struct{ table, chain string }{{"nat", ForwardChain}, {"mangle", MarkChain}} {
for _, h := range []struct{ table, from, chain string }{
{"nat", "PREROUTING", ForwardChain},
{"nat", "POSTROUTING", PostChain},
{"mangle", "PREROUTING", MarkChain},
} {
ok, err := dp.ipt.ChainExists(h.table, h.chain)
if err != nil {
return err
@@ -166,7 +187,7 @@ func (dp *Datapath) hooks() error {
}
}
rule := []string{"-m", "comment", "--comment", "reweave lb", "-j", h.chain}
rules, err := dp.ipt.List(h.table, "PREROUTING")
rules, err := dp.ipt.List(h.table, h.from)
if err != nil {
return err
}
@@ -174,10 +195,10 @@ func (dp *Datapath) hooks() error {
if len(rules) > 1 && strings.HasSuffix(rules[1], "-j "+h.chain) {
continue
}
if err := dp.ipt.DeleteIfExists(h.table, "PREROUTING", rule...); err != nil {
if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
return err
}
if err := dp.ipt.Insert(h.table, "PREROUTING", 1, rule...); err != nil {
if err := dp.ipt.Insert(h.table, h.from, 1, rule...); err != nil {
return err
}
}