diff --git a/lb/datapath.go b/lb/datapath.go index 35a93b2a..d76ef2ea 100644 --- a/lb/datapath.go +++ b/lb/datapath.go @@ -17,6 +17,8 @@ import ( const ( // ForwardChain (nat, from PREROUTING) DNATs held addresses' ports to endpoints. ForwardChain = "REWEAVE-LB" + // PostChain (nat, from POSTROUTING, ahead of weave's WEAVE) keeps the client's address on what ForwardChain DNATed. + PostChain = "REWEAVE-LB-POST" // MarkChain (mangle, from PREROUTING) marks holders' connections and steers their replies. MarkChain = "REWEAVE-LB-MARK" // servicePrefix names the per-port chains ForwardChain jumps to. @@ -52,7 +54,21 @@ func Render(d Desired, podNet *net.IPNet, stale []string) Rules { var nat bytes.Buffer nat.WriteString("*nat\n") fmt.Fprintf(&nat, ":%s - [0:0]\n", ForwardChain) + fmt.Fprintf(&nat, ":%s - [0:0]\n", PostChain) var body bytes.Buffer + seen := map[string]bool{} + for _, f := range d.Forwards { + a := f.Address.String() + if seen[a] { + continue + } + seen[a] = true + // weave masquerades everything entering the pod network from outside unless it is for a local pod + // (its expose rule); a connection we DNATed must keep the client's address, so it leaves the nat + // table here. One from a pod (hairpin) is masqueraded to this node, so its reply comes back here. + fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -s %s -j MASQUERADE\n", PostChain, a, podNet) + fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -j ACCEPT\n", PostChain, a) + } for _, f := range d.Forwards { for _, p := range f.Ports { c := portChain(f, p) @@ -153,9 +169,14 @@ func (dp *Datapath) Apply(d Desired, podNet *net.IPNet) error { return dp.routes(d.Holders) } -// hooks makes sure PREROUTING jumps to our chains first (kube-proxy's KUBE-SERVICES must not see held addresses first). +// hooks makes sure PREROUTING and POSTROUTING jump to our chains first (kube-proxy's and weave's must not see +// held addresses' connections first). func (dp *Datapath) hooks() error { - for _, h := range []struct{ table, chain string }{{"nat", ForwardChain}, {"mangle", MarkChain}} { + for _, h := range []struct{ table, from, chain string }{ + {"nat", "PREROUTING", ForwardChain}, + {"nat", "POSTROUTING", PostChain}, + {"mangle", "PREROUTING", MarkChain}, + } { ok, err := dp.ipt.ChainExists(h.table, h.chain) if err != nil { return err @@ -166,7 +187,7 @@ func (dp *Datapath) hooks() error { } } rule := []string{"-m", "comment", "--comment", "reweave lb", "-j", h.chain} - rules, err := dp.ipt.List(h.table, "PREROUTING") + rules, err := dp.ipt.List(h.table, h.from) if err != nil { return err } @@ -174,10 +195,10 @@ func (dp *Datapath) hooks() error { if len(rules) > 1 && strings.HasSuffix(rules[1], "-j "+h.chain) { continue } - if err := dp.ipt.DeleteIfExists(h.table, "PREROUTING", rule...); err != nil { + if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil { return err } - if err := dp.ipt.Insert(h.table, "PREROUTING", 1, rule...); err != nil { + if err := dp.ipt.Insert(h.table, h.from, 1, rule...); err != nil { return err } } diff --git a/lb/model_test.go b/lb/model_test.go index 56d37779..bfb17276 100644 --- a/lb/model_test.go +++ b/lb/model_test.go @@ -206,7 +206,10 @@ func TestRenderedRules(t *testing.T) { d := Compute("w1", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl}) r := Render(d, podNet, []string{"REWEAVE-LBS-DEADBEEF0000"}) for _, want := range []string{ - "*nat\n:REWEAVE-LB - [0:0]\n", + "*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\n", + // the client's address survives weave's masquerade; a pod's own connection (hairpin) is masqueraded here + "-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -s 10.32.0.0/12 -j MASQUERADE\n", + "-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -j ACCEPT\n", ":REWEAVE-LBS-DEADBEEF0000 - [0:0]\n", // stale chain emptied "-A REWEAVE-LB -d 86.107.168.15/32 -p tcp -m tcp --dport 22 -m comment --comment \"gitea/gitea\" -j REWEAVE-LBS-", "-p tcp -m statistic --mode random --probability 0.3333333333 -j DNAT --to-destination 10.40.0.7:2222\n", @@ -224,8 +227,8 @@ func TestRenderedRules(t *testing.T) { t.Fatalf("DNAT without its protocol: %q", line) } } - if strings.Contains(r.Nat, "SNAT") || strings.Contains(r.Nat, "MASQUERADE") { - t.Fatalf("the client's address must survive:\n%s", r.Nat) + if strings.Count(r.Nat, "MASQUERADE") != 1 || strings.Contains(r.Nat, "SNAT") { + t.Fatalf("only a pod's hairpin connection may be masqueraded:\n%s", r.Nat) } if r.Mangle != "*mangle\n:REWEAVE-LB-MARK - [0:0]\nCOMMIT\n" { t.Fatalf("a holder marks nothing:\n%s", r.Mangle) @@ -241,7 +244,7 @@ func TestRenderedRules(t *testing.T) { t.Fatalf("mangle lacks %q:\n%s", want, r.Mangle) } } - if r.Nat != "*nat\n:REWEAVE-LB - [0:0]\nCOMMIT\n" { + if r.Nat != "*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\nCOMMIT\n" { t.Fatalf("a non-holder forwards nothing:\n%s", r.Nat) } }