From ee1c2f905de1fdb07d1e66235fcb5196194e0e5b Mon Sep 17 00:00:00 2001 From: Levalicious Date: Tue, 13 Oct 2026 15:31:29 -0400 Subject: [PATCH] lb, weave: three faults the first live EgressService run found - The tunnel needs an address. Reverse-path filtering (loose, as on our nodes) rejects what arrives on a device without an IPv4 address unless the reverse route leaves by that very device: the holder dropped every member packet after decapsulation (ip route get: EXDEV), and a member's node would drop the replies. reweave-eg now carries the node's bridge address as a /32 (it never shows on the wire: the tunnel sets the outer source itself, and what crosses it is forwarded). - The kernel leaves out the priority attribute of a rule at 0, so the default local rule lists as unset (-1) and moveLocalRule never found it: it stayed at 0 beside the new one at 100, and the holder delivered replies to its own address to itself (answering with RSTs). Unset now counts as 0. - install_cni_plugin copied the plugin only when weave-plugin-$VERSION was missing, and every build of ours is "latest": nodes kept a plugin that ignores egressAdmission, so a new member's first connection left from its node. It is now replaced whenever its content differs (sha256), atomically. Also: the end-to-end manifest's non-member label is a string ("none", not YAML's boolean no). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ --- lb/egress_datapath.go | 32 +++++++++++++++++++++++++--- reweave/test/egress/egress-test.yaml | 2 +- weave | 7 ++++-- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/lb/egress_datapath.go b/lb/egress_datapath.go index 49fd8e88..baf6f92d 100644 --- a/lb/egress_datapath.go +++ b/lb/egress_datapath.go @@ -288,10 +288,11 @@ func moveLocalRule() error { var at0, atLocal bool for _, r := range rules { if r.Table == unix.RT_TABLE_LOCAL && r.Src == nil && r.Dst == nil && r.Mark <= 0 { - switch r.Priority { - case 0: + switch { + case r.Priority <= 0: + // the kernel leaves out the priority attribute when it is 0: listed as unset (-1) at0 = true - case LocalRulePriority: + case r.Priority == LocalRulePriority: atLocal = true } } @@ -357,5 +358,30 @@ func ensureTunnel(want bool, local Local) error { return fmt.Errorf("sysctl %s on %s: %v", k, Tunnel, err) } } + // reverse-path filtering rejects what arrives on a device with no IPv4 address unless the reverse route + // leaves by that very device (replies on a member's node, members' packets on the holder do not): the + // tunnel carries this node's bridge address (it never appears on the wire from here: the tunnel sets the + // outer source itself, and what crosses it is forwarded) + addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK} + addrs, err := netlink.AddrList(link, unix.AF_INET) + if err != nil { + return err + } + has := false + for _, a := range addrs { + if a.IP.Equal(local.WeaveIP) { + has = true + continue + } + a := a + if err := netlink.AddrDel(link, &a); err != nil { + return fmt.Errorf("removing stale address %s from %s: %v", a.IPNet, Tunnel, err) + } + } + if !has { + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("address on %s: %v", Tunnel, err) + } + } return netlink.LinkSetUp(link) } diff --git a/reweave/test/egress/egress-test.yaml b/reweave/test/egress/egress-test.yaml index cbaa63a3..096c83aa 100644 --- a/reweave/test/egress/egress-test.yaml +++ b/reweave/test/egress/egress-test.yaml @@ -53,7 +53,7 @@ items: containers: [{name: c, image: "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7", command: ["sleep", "3600"]}] - apiVersion: v1 kind: Pod - metadata: {name: plain, namespace: egtest, labels: {egress: no}} + metadata: {name: plain, namespace: egtest, labels: {egress: "none"}} spec: nodeName: REMOTE containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}] diff --git a/weave b/weave index ee134620..eb73dcef 100755 --- a/weave +++ b/weave @@ -725,8 +725,11 @@ peer_args() { install_cni_plugin() { mkdir -p $1 || return 1 - if [ ! -f "$1/$CNI_PLUGIN_NAME" ]; then - cp /usr/bin/weaveutil "$1/$CNI_PLUGIN_NAME" + # (re)install whenever the image's plugin differs from the installed one: the name + # (weave-plugin-$IMAGE_VERSION) stays the same across builds of one version ("latest") + if [ ! -f "$1/$CNI_PLUGIN_NAME" ] || \ + [ "$(sha256sum < /usr/bin/weaveutil)" != "$(sha256sum < "$1/$CNI_PLUGIN_NAME")" ]; then + cp /usr/bin/weaveutil "$1/$CNI_PLUGIN_NAME.tmp" && mv -f "$1/$CNI_PLUGIN_NAME.tmp" "$1/$CNI_PLUGIN_NAME" fi }