lb: EgressService - a LoadBalancer Service's pods leave from its address
images / images (push) Successful in 18m34s

EgressService (reweave.levk.dev/v1alpha1, named after a LoadBalancer
Service of class reweave.levk.dev/node-address) makes every pod the
Service selects (by pod IP, never readiness) leave the cluster from the
Service's address, or not at all. spec.emit is explicit: Holder (the only
mode so far) tunnels members' packets to the node holding the address,
which emits them.

The holder (S1's: the one node with the address on an interface) is the
status' sole writer: host, address, and a 512-port block per node in
61000-65535 (above the default ephemeral range), stable across churn.
A member's node SNATs it to address:its-block before it leaves, so the
holder forwards it unchanged and routes replies back by port alone: ip
rules (to address, ipproto tcp/udp, dport block) ahead of the local
table, which moves from 0 to 100, since the address is local on the
holder. Members' packets travel both ways in reweave-eg, an ipip tunnel
between weave bridges (NPC drops new routed packets into weave), with
accept_local and loose rp_filter on that device only.

Members' new connections are classed at the tail of nat PREROUTING,
after kube-proxy's DNAT has ended the nat walk for Services: the pod
network, multicast and the nodes' addresses are not outside. Marked
packets follow a per-member rule to the holder's table (or to an
unreachable one), are SNATed into the tunnel, and a FORWARD guard drops
anything else a member sends outside (other protocols, other ways out).
The tunnel is accepted only from the nodes' bridges, a holder accepts
from it only the blocks' ports, and TCP MSS is clamped into it.

Fail closed throughout: until the holder has published the status,
without a block, with no single holder, or for a pod two EgressServices
select, a member's outside traffic is dropped. The CNI plugin asks
weave-npc to admit every pod (unix socket /run/weave-npc/egress.sock,
conflist "egressAdmission": true, now rewritten when it differs) after
IPAM and before the pod has an interface, so a member's first packet
already leaves from the address; no answer, no pod. weave-npc applies
nothing about egress from caches that are not synced (unless the API is
absent): a restart never strips members' rules.

reweave/test/egress/run.sh checks the design's assertions against a
live cluster; both end-to-end scripts now re-evaluate each check on
every retry (it was expanded once, before the loop).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ
This commit is contained in:
levaliciousandClaude Opus 5.5 committed 2026-10-13 14:39:44 -04:00
1 parent 8018ea4bf9
commit 305ac4f3d3
15 files changed
+1866 -64

No files matched your search

+75 -25
View File
@@ -44,17 +44,18 @@ func portChain(f Forward, p Port) string {
// Rules is the full content of the chains this package owns, in iptables-restore form.
type Rules struct {
Nat, Mangle string
Nat, Mangle, Filter string
}
// Render turns the desired state into iptables-restore input for the nat and mangle tables.
// podNet is the weave allocation range (pods and bridges); bridge is the weave bridge name.
// Render turns the desired state (load-balancer forwards and egress) into iptables-restore input for the nat,
// mangle and filter tables. podNet is the weave allocation range (pods and bridges).
// Each payload declares every chain it owns, so a restore with --noflush replaces exactly those.
func Render(d Desired, podNet *net.IPNet, stale []string) Rules {
func Render(d Desired, e EgressDesired, self string, podNet *net.IPNet, stale []string) Rules {
var nat bytes.Buffer
nat.WriteString("*nat\n")
fmt.Fprintf(&nat, ":%s - [0:0]\n", ForwardChain)
fmt.Fprintf(&nat, ":%s - [0:0]\n", PostChain)
for _, c := range []string{ForwardChain, PostChain, ClassChain} {
fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
}
var body bytes.Buffer
seen := map[string]bool{}
for _, f := range d.Forwards {
@@ -93,12 +94,12 @@ func Render(d Desired, podNet *net.IPNet, stale []string) Rules {
// chains of ports no longer forwarded: emptied here, deleted after the restore
fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
}
nat.Write(body.Bytes())
nat.WriteString("COMMIT\n")
var mangle bytes.Buffer
mangle.WriteString("*mangle\n")
fmt.Fprintf(&mangle, ":%s - [0:0]\n", MarkChain)
for _, c := range []string{MarkChain, MSSChain} {
fmt.Fprintf(&mangle, ":%s - [0:0]\n", c)
}
if len(d.Holders) > 0 {
// replies of a holder's connection carry its mark: the policy rule routes them back to it
fmt.Fprintf(&mangle, "-A %s -m conntrack --ctdir REPLY -m connmark ! --mark 0x0/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n",
@@ -110,8 +111,19 @@ func Render(d Desired, podNet *net.IPNet, stale []string) Rules {
MarkChain, bridge, h.Node.WeaveMAC, podNet, podNet, "holder "+h.Node.Name, markValue(h.Mark), markMask)
}
}
var filter bytes.Buffer
filter.WriteString("*filter\n")
for _, c := range []string{FwdChain, InChain} {
fmt.Fprintf(&filter, ":%s - [0:0]\n", c)
}
renderEgress(e, self, podNet, &body, &mangle, &filter)
nat.Write(body.Bytes())
nat.WriteString("COMMIT\n")
mangle.WriteString("COMMIT\n")
return Rules{Nat: nat.String(), Mangle: mangle.String()}
filter.WriteString("COMMIT\n")
return Rules{Nat: nat.String(), Mangle: mangle.String(), Filter: filter.String()}
}
// Datapath applies Desired to this node.
@@ -133,11 +145,17 @@ func NewDatapath(ipt *iptables.IPTables) *Datapath {
}}
}
// Apply makes the node's iptables chains, policy rules and routes exactly what d says.
func (dp *Datapath) Apply(d Desired, podNet *net.IPNet) error {
// Apply makes the node's iptables chains, tunnel, policy rules and routes exactly what d and e say.
func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local) error {
podNet := local.PodNet
if err := dp.hooks(); err != nil {
return err
}
// the tunnel device must exist before rules naming it are restored (-o/-i on a missing device is fine for
// iptables, but the routes and tables need it)
if err := ensureTunnel(e.Tunnelled(self), local); err != nil {
return err
}
existing, err := dp.ipt.ListChains("nat")
if err != nil {
return err
@@ -154,28 +172,39 @@ func (dp *Datapath) Apply(d Desired, podNet *net.IPNet) error {
stale = append(stale, c)
}
}
r := Render(d, podNet, stale)
if err := dp.restore(r.Nat); err != nil {
return err
}
if err := dp.restore(r.Mangle); err != nil {
return err
r := Render(d, e, self, podNet, stale)
// filter first: the guards are in place before the marks and SNATs that they guard
for _, payload := range []string{r.Filter, r.Nat, r.Mangle} {
if err := dp.restore(payload); err != nil {
return err
}
}
for _, c := range stale {
if err := dp.ipt.DeleteChain("nat", c); err != nil {
return fmt.Errorf("deleting stale chain %s: %v", c, err)
}
}
return dp.routes(d.Holders)
if err := dp.routes(d.Holders); err != nil {
return err
}
return egressNet(e, self, local)
}
// hooks makes sure PREROUTING and POSTROUTING jump to our chains first (kube-proxy's and weave's must not see
// held addresses' connections first).
// hooks makes sure the built-in chains jump to ours: first (kube-proxy's and weave's rules must not see these
// connections first), except ClassChain, which must come after kube-proxy's KUBE-SERVICES (whose DNAT ends the
// nat walk, so a connection to a Service is never classed as going outside).
func (dp *Datapath) hooks() error {
for _, h := range []struct{ table, from, chain string }{
{"nat", "PREROUTING", ForwardChain},
{"nat", "POSTROUTING", PostChain},
{"mangle", "PREROUTING", MarkChain},
for _, h := range []struct {
table, from, chain string
last bool
}{
{"nat", "PREROUTING", ForwardChain, false},
{"nat", "POSTROUTING", PostChain, false},
{"nat", "PREROUTING", ClassChain, true},
{"mangle", "PREROUTING", MarkChain, false},
{"mangle", "FORWARD", MSSChain, false},
{"filter", "FORWARD", FwdChain, false},
{"filter", "INPUT", InChain, false},
} {
ok, err := dp.ipt.ChainExists(h.table, h.chain)
if err != nil {
@@ -191,6 +220,27 @@ func (dp *Datapath) hooks() error {
if err != nil {
return err
}
if h.last {
ours, kube := -1, -1
for i, r := range rules {
if strings.HasSuffix(r, "-j "+h.chain) {
ours = i
}
if strings.HasSuffix(r, "-j KUBE-SERVICES") {
kube = i
}
}
if ours > kube {
continue
}
if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
return err
}
if err := dp.ipt.Append(h.table, h.from, rule...); err != nil {
return err
}
continue
}
// rules[0] is the policy line; our jump must be rules[1]
if len(rules) > 1 && strings.HasSuffix(rules[1], "-j "+h.chain) {
continue