lb: one egress tunnel per peer; replies and ICMP errors follow a tag

The holder could tell a member's node only by source port block, so
replies without ports never got back: ICMP errors (path-MTU "too big",
unreachable, time exceeded), ping, and port-less protocols. Now every
node pair has its own ipip tunnel (rweg-<hash of the peer>, between the
two weave bridges), so the holder knows the sending node by the device a
packet arrives on:

- it tags each connection arriving from a peer's tunnel with that peer
  (connmark: peer index in the top byte, plus 0x200000), once;
- replies and related ICMP errors restore the tag, and a policy rule per
  peer (ahead of the local table, like the port rules) routes them into
  that peer's tunnel, where the member node's conntrack translates them
  back to the member;
- an arrival on one peer's tunnel whose connection another peer already
  holds (a port-less protocol to the same remote from two nodes) is
  dropped, never delivered across;
- the stateless port-block rules stay, now for TCP, UDP and SCTP.

Members' SNAT keeps TCP, UDP and SCTP within the node's block and puts
everything else (ping, GRE, ...) on the address alone; the guard lets any
protocol into the holder's tunnel and nothing anywhere else. The holder
only accepts, per tunnel, the address with that node's ports.

Routes in the reply tables are now read with one dump per pass instead of
one per candidate table.

reweave/test/egress/run.sh also checks ping from every node, that ICMP
errors from routers beyond the holder reach a member (traceroute), and
that ping is dropped while nobody holds the address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ

lb/datapath.go

@@ -151,9 +151,12 @@ func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local)
 	if err := dp.hooks(); err != nil {
 		return err
 	}
-	// the tunnel device must exist before rules naming it are restored (-o/-i on a missing device is fine for
-	// iptables, but the routes and tables need it)
-	if err := ensureTunnel(e.Tunnelled(self), local); err != nil {
+	// the tunnels must exist before the routes into them (iptables rules may name a device that does not exist yet)
+	var peers []Node
+	if e.Tunnelled(self) {
+		peers = e.Peers(self)
+	}
+	if err := ensureTunnels(peers, local); err != nil {
 		return err
 	}
 	existing, err := dp.ipt.ListChains("nat")
@@ -187,7 +190,7 @@ func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local)
 	if err := dp.routes(d.Holders); err != nil {
 		return err
 	}
-	return egressNet(e, self, local)
+	return egressNet(e, self)
 }
 
 // hooks makes sure the built-in chains jump to ours: first (kube-proxy's and weave's rules must not see these
@@ -287,14 +290,14 @@ func (dp *Datapath) routes(holders []Holder) error {
 			return fmt.Errorf("removing stale rule to table %d: %v", r.Table, err)
 		}
 	}
-	for table := TableBase + 1; table <= TableBase+MaxMarks; table++ {
-		h, ok := want[table]
-		if !ok {
-			routes, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: table}, netlink.RT_FILTER_TABLE)
+	existing, err := routesByTable(TableBase+1, TableBase+MaxMarks)
 	if err != nil {
 		return err
 	}
-			for _, rt := range routes {
+	for table := TableBase + 1; table <= TableBase+MaxMarks; table++ {
+		h, ok := want[table]
+		if !ok {
+			for _, rt := range existing[table] {
 				if err := delRoute(rt); err != nil {
 					return fmt.Errorf("removing stale route in table %d: %v", table, err)
 				}

lb/egress_datapath.go

@@ -3,9 +3,11 @@ package lb
 import (
 	"bytes"
 	"fmt"
+	"hash/fnv"
 	"net"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
@@ -17,15 +19,25 @@ const (
 	ClassChain = "REWEAVE-LB-CLASS"
 	// FwdChain (filter, first in FORWARD) drops members' outside packets that would not leave as their address.
 	FwdChain = "REWEAVE-LB-FWD"
-	// InChain (filter, first in INPUT) accepts the egress tunnel only from the cluster's nodes.
+	// InChain (filter, first in INPUT) accepts the egress tunnels only from the cluster's nodes.
 	InChain = "REWEAVE-LB-IN"
-	// MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnel.
+	// MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnels.
 	MSSChain = "REWEAVE-LB-MSS"
 
-	// Tunnel is the egress tunnel: ipip over weave between nodes' weave bridges, no fixed remote (each route names it).
-	Tunnel = "reweave-eg"
+	// TunnelPrefix names the egress tunnels: one ipip device per peer node, between the two nodes' weave bridges,
+	// so a holder knows which node a member's packet came from by the device it arrives on.
+	TunnelPrefix = "rweg-"
+	// tunnelGlob matches every egress tunnel in an iptables rule.
+	tunnelGlob = "rweg+"
+	// legacyTunnel is the single shared tunnel of earlier builds (removed when found).
+	legacyTunnel = "reweave-eg"
+
 	// EgressMark marks a member's packets to the outside (connmark on its connections, fwmark on each packet).
 	EgressMark = uint32(0x100000)
+	// peerFlag, with a peer's index in the top byte, tags (on the holder) a connection that arrived from that peer's
+	// tunnel: replies and related ICMP errors carry it back there.
+	peerFlag = uint32(0x200000)
+	peerMask = markMask | peerFlag
 
 	// HolderRulePriority: replies to a block go to its node; ahead of the local table, which moves to LocalRulePriority,
 	// since the address is usually local on the holder.
@@ -33,12 +45,28 @@ const (
 	LocalRulePriority  = 100
 	// MemberRulePriority: a member's marked packets go to its holder (or nowhere).
 	MemberRulePriority = 9100
-	// EgressTableBase + peer index routes into the tunnel towards that peer; EgressTableBase itself is unreachable.
+	// EgressTableBase + peer index routes into that peer's tunnel; EgressTableBase itself is unreachable.
 	EgressTableBase = 53000
 
 	ipipOverhead = 20
 )
 
+// portProtocols carry ports: their replies are routed by port block alone (stateless), and members' SNAT keeps
+// within the node's block. Everything else is routed back by the tag the holder put on the connection.
+var portProtocols = []struct {
+	name string
+	num  int
+}{{"tcp", unix.IPPROTO_TCP}, {"udp", unix.IPPROTO_UDP}, {"sctp", unix.IPPROTO_SCTP}}
+
+// TunnelName is the egress tunnel towards a peer node (stable, within the 15 characters a device name may have).
+func TunnelName(peer string) string {
+	h := fnv.New32a()
+	h.Write([]byte(peer))
+	return fmt.Sprintf("%s%08x", TunnelPrefix, h.Sum32())
+}
+
+func peerTag(idx uint32) uint32 { return markValue(idx) | peerFlag }
+
 // renderEgress appends the egress rules of each table to the bodies of S1's payloads.
 func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, filter *bytes.Buffer) {
 	members := e.Members
@@ -66,53 +94,110 @@ func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle,
 			fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
 			continue
 		}
-		out := "-o " + Tunnel
+		out := "-o " + TunnelName(m.Holder.Name)
 		if m.Holder.Name == self {
 			// emitted here, from the address: never into the pod network unSNATed
 			out = "! -o " + bridge
 			fmt.Fprintf(filter, "-A %s -s %s/32 -o %s -m mark --mark 0x%x/0x%x -j DROP\n", FwdChain, m.IP, bridge, EgressMark, EgressMark)
 		}
-		for _, proto := range []string{"tcp", "udp"} {
+		// within this node's block for what has ports; the address alone for the rest (the holder tells those
+		// apart by the tunnel they arrive on)
+		for _, p := range portProtocols {
 			fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n",
-				PostChain, m.IP, out, EgressMark, EgressMark, proto, m.Address, m.From, m.To)
-			fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark, proto)
+				PostChain, m.IP, out, EgressMark, EgressMark, p.name, m.Address, m.From, m.To)
 		}
-		// anything else a member sends outside (other protocols, other ways out) does not leave
+		fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j SNAT --to-source %s\n",
+			PostChain, m.IP, out, EgressMark, EgressMark, m.Address)
+		fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark)
+		// a member's outside traffic leaves no other way
 		fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
 	}
 	if !e.Tunnelled(self) {
 		return
 	}
-	// tunnel arrivals: replies to members here (established), members' packets from their blocks (as holder)
+	tags := peerIndex(e, self)
+
+	// as holder: tag each connection arriving from a peer's tunnel with that peer, once; replies and related ICMP
+	// errors restore the tag, and the policy rule for it sends them back into that tunnel
+	byPeer := map[string][]BlockRoute{}
+	var peerNames []string
+	for _, r := range e.Routes {
+		if _, ok := byPeer[r.Peer.Name]; !ok {
+			peerNames = append(peerNames, r.Peer.Name)
+		}
+		byPeer[r.Peer.Name] = append(byPeer[r.Peer.Name], r)
+	}
+	sort.Strings(peerNames)
+	if len(peerNames) > 0 {
+		fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir REPLY -m connmark --mark 0x%x/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n",
+			MarkChain, peerFlag, peerFlag, peerMask, peerMask)
+	}
+	for _, name := range peerNames {
+		dev, tag := TunnelName(name), peerTag(tags[name])
+		fmt.Fprintf(mangle, "-A %s -i %s -m conntrack --ctstate NEW -m connmark --mark 0x0/0x%x -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n",
+			MarkChain, dev, peerMask, "from "+name, tag, peerMask)
+		// a connection another node already holds (a port-less protocol to the same remote): never cross-delivered
+		fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -m connmark ! --mark 0x%x/0x%x -j DROP\n", FwdChain, dev, tag, peerMask)
+		addrs := map[string]bool{}
+		for _, r := range byPeer[name] {
+			for _, p := range portProtocols {
+				fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n",
+					FwdChain, dev, r.Address, p.name, p.name, r.From, r.To, "block of "+name)
+			}
+			addrs[r.Address.String()] = true
+		}
+		for _, a := range sortedSet(addrs) {
+			// outside its own block a node may not use the ports of others
+			for _, p := range portProtocols {
+				fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -j DROP\n", FwdChain, dev, a, p.name)
+			}
+			fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -j RETURN\n", FwdChain, dev, a)
+		}
+	}
+	// as a member's node: replies (and related errors) from the holder; nothing else comes back
 	addrs := map[string]bool{}
 	for _, m := range members {
-		if !m.Unreachable && m.Holder.Name != self && !addrs[m.Address.String()] {
+		if !m.Unreachable && m.Holder.Name != self {
 			addrs[m.Address.String()] = true
+		}
+	}
+	for _, a := range sortedSet(addrs) {
 		// still addressed to the address after nat: no member connection matched it
-			fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, Tunnel, m.Address)
+		fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, tunnelGlob, a)
 	}
-	}
-	fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, Tunnel)
-	for _, r := range e.Routes {
-		for _, proto := range []string{"tcp", "udp"} {
-			fmt.Fprintf(filter, "-A %s -i %s -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n",
-				FwdChain, Tunnel, r.Address, proto, proto, r.From, r.To, "block of "+r.Peer.Name)
-		}
-	}
-	fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, Tunnel)
-	// the tunnel itself only from the cluster's nodes' bridges
+	fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, tunnelGlob)
+	fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, tunnelGlob)
+	// the tunnels themselves only from the cluster's nodes' bridges
 	for _, p := range e.Peers(self) {
 		fmt.Fprintf(filter, "-A %s -p 4 -s %s/32 -m mac --mac-source %s -j RETURN\n", InChain, p.WeaveIP, p.WeaveMAC)
 	}
 	fmt.Fprintf(filter, "-A %s -p 4 -j DROP\n", InChain)
-	fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, Tunnel)
+	fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, tunnelGlob)
 }
 
-// peerTables gives each tunnel peer its routing table (EgressTableBase + a stable index).
+func sortedSet(m map[string]bool) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// peerIndex gives each tunnel peer a stable index (1..MaxMarks): its tag and its routing table.
+func peerIndex(e EgressDesired, self string) map[string]uint32 {
+	out := map[string]uint32{}
+	for _, h := range assignMarks(e.Peers(self)) {
+		out[h.Node.Name] = h.Mark
+	}
+	return out
+}
+
+// peerTables gives each tunnel peer its routing table (EgressTableBase + its index).
 func peerTables(e EgressDesired, self string) map[string]int {
 	out := map[string]int{}
-	for _, h := range assignMarks(e.Peers(self)) {
-		out[h.Node.Name] = EgressTableBase + int(h.Mark)
+	for name, idx := range peerIndex(e, self) {
+		out[name] = EgressTableBase + int(idx)
 	}
 	return out
 }
@@ -123,23 +208,30 @@ type egressRule struct {
 	src, dst   string
 	ipproto    int
 	from, to   uint16
-	mark     uint32
+	mark, mask uint32
 	table      int
 }
 
 func (r egressRule) key() string {
-	return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.table)
+	return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x/%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.mask, r.table)
 }
 
 // wantRules is every egress policy rule this node needs.
 func wantRules(e EgressDesired, self string) []egressRule {
 	tables := peerTables(e, self)
+	tags := peerIndex(e, self)
 	var out []egressRule
+	tagged := map[string]bool{}
 	for _, r := range e.Routes {
-		for _, proto := range []int{unix.IPPROTO_TCP, unix.IPPROTO_UDP} {
-			out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: proto,
+		for _, p := range portProtocols {
+			out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: p.num,
 				from: uint16(r.From), to: uint16(r.To), table: tables[r.Peer.Name]})
 		}
+		if !tagged[r.Peer.Name] {
+			// whatever the tag says came from that peer (replies without ports, ICMP errors) goes back to it
+			tagged[r.Peer.Name] = true
+			out = append(out, egressRule{priority: HolderRulePriority, mark: peerTag(tags[r.Peer.Name]), mask: peerMask, table: tables[r.Peer.Name]})
+		}
 	}
 	for _, m := range e.Members {
 		if !m.Unreachable && m.Holder.Name == self {
@@ -149,7 +241,7 @@ func wantRules(e EgressDesired, self string) []egressRule {
 		if !m.Unreachable {
 			table = tables[m.Holder.Name]
 		}
-		out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, table: table})
+		out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, mask: EgressMark, table: table})
 	}
 	sort.Slice(out, func(i, j int) bool { return out[i].key() < out[j].key() })
 	return out
@@ -168,6 +260,7 @@ func ruleOf(r netlink.Rule) egressRule {
 	}
 	if r.Mark > 0 {
 		er.mark = uint32(r.Mark)
+		er.mask = uint32(r.Mask)
 	}
 	if er.ipproto < 0 {
 		er.ipproto = 0
@@ -175,53 +268,66 @@ func ruleOf(r netlink.Rule) egressRule {
 	return er
 }
 
-// egressNet makes the tunnel, the local rule's place, the policy rules and the tunnel tables exactly what e needs.
-func egressNet(e EgressDesired, self string, local Local) error {
-	tunnelled := e.Tunnelled(self) // the device itself is made first, by Apply (ensureTunnel)
+// routesByTable lists, in one dump, the IPv4 routes of tables lo..hi.
+func routesByTable(lo, hi int) (map[int][]netlink.Route, error) {
+	all, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, netlink.RT_FILTER_TABLE)
+	if err != nil {
+		return nil, err
+	}
+	out := map[int][]netlink.Route{}
+	for _, rt := range all {
+		if rt.Table >= lo && rt.Table <= hi {
+			out[rt.Table] = append(out[rt.Table], rt)
+		}
+	}
+	return out, nil
+}
+
+// egressNet makes the local rule's place, the policy rules and the tunnel tables exactly what e needs (the
+// tunnels themselves are made first, by Apply).
+func egressNet(e EgressDesired, self string) error {
 	if len(e.Routes) > 0 {
 		if err := moveLocalRule(); err != nil {
 			return err
 		}
 	}
 	tables := peerTables(e, self)
-	peers := map[int]Node{}
+	peerOf := map[int]Node{}
 	for _, p := range e.Peers(self) {
-		peers[tables[p.Name]] = p
+		peerOf[tables[p.Name]] = p
 	}
 	unreachable := false
 	for _, m := range e.Members {
 		unreachable = unreachable || m.Unreachable
 	}
-	var link netlink.Link
-	if tunnelled {
-		l, err := netlink.LinkByName(Tunnel)
-		if err != nil {
-			return fmt.Errorf("egress tunnel: %v", err)
-		}
-		link = l
-	}
-	for table := EgressTableBase; table <= EgressTableBase+MaxMarks; table++ {
-		var want *netlink.Route
-		if p, ok := peers[table]; ok && link != nil {
-			want = &netlink.Route{LinkIndex: link.Attrs().Index, Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
-				Gw: p.WeaveIP, Table: table, Flags: int(netlink.FLAG_ONLINK)}
-		} else if table == EgressTableBase && unreachable {
-			want = &netlink.Route{Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, Table: table, Type: unix.RTN_UNREACHABLE}
-		}
-		routes, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: table}, netlink.RT_FILTER_TABLE)
+	existing, err := routesByTable(EgressTableBase, EgressTableBase+MaxMarks)
 	if err != nil {
 		return err
 	}
-		for _, rt := range routes {
+	for table := EgressTableBase; table <= EgressTableBase+MaxMarks; table++ {
+		var want *netlink.Route
+		if p, ok := peerOf[table]; ok {
+			link, err := netlink.LinkByName(TunnelName(p.Name))
+			if err != nil {
+				return fmt.Errorf("egress tunnel to %s: %v", p.Name, err)
+			}
+			want = &netlink.Route{LinkIndex: link.Attrs().Index, Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
+				Scope: netlink.SCOPE_LINK, Table: table}
+		} else if table == EgressTableBase && unreachable {
+			want = &netlink.Route{Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, Table: table, Type: unix.RTN_UNREACHABLE}
+		}
+		keep := false
+		for _, rt := range existing[table] {
 			if want != nil && rt.Type == want.Type &&
-				(want.Type == unix.RTN_UNREACHABLE || rt.LinkIndex == want.LinkIndex && rt.Gw.Equal(want.Gw)) {
+				(want.Type == unix.RTN_UNREACHABLE || rt.LinkIndex == want.LinkIndex && rt.Gw == nil) {
+				keep = true
 				continue
 			}
 			if err := delRoute(rt); err != nil {
 				return fmt.Errorf("removing stale route in table %d: %v", table, err)
 			}
 		}
-		if want != nil {
+		if want != nil && !keep {
 			if err := netlink.RouteReplace(want); err != nil {
 				return fmt.Errorf("route in table %d: %v", table, err)
 			}
@@ -269,7 +375,7 @@ func egressNet(e EgressDesired, self string, local Local) error {
 		}
 		if r.mark != 0 {
 			rule.Mark = int(r.mark)
-			rule.Mask = int(r.mark)
+			rule.Mask = int(r.mask)
 		}
 		if err := netlink.RuleAdd(rule); err != nil {
 			return fmt.Errorf("adding egress rule %s: %v", k, err)
@@ -316,53 +422,85 @@ func moveLocalRule() error {
 	return nil
 }
 
-// ensureTunnel creates (or removes) the egress tunnel: ipip with this node's bridge address as its local end.
-func ensureTunnel(want bool, local Local) error {
-	link, err := netlink.LinkByName(Tunnel)
-	exists := err == nil
-	if !want {
-		if exists {
-			return netlink.LinkDel(link)
+// ensureTunnels makes one ipip tunnel per peer (this node's bridge address to the peer's), and removes every
+// other egress tunnel (including the shared one of earlier builds).
+func ensureTunnels(peers []Node, local Local) error {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return err
 	}
+	want := map[string]Node{}
+	for _, p := range peers {
+		want[TunnelName(p.Name)] = p
+	}
+	have := map[string]netlink.Link{}
+	for _, l := range links {
+		name := l.Attrs().Name
+		if name == legacyTunnel || strings.HasPrefix(name, TunnelPrefix) {
+			if _, ok := want[name]; ok {
+				have[name] = l
+				continue
+			}
+			if err := netlink.LinkDel(l); err != nil {
+				return fmt.Errorf("removing egress tunnel %s: %v", name, err)
+			}
+		}
+	}
+	if len(want) == 0 {
 		return nil
 	}
 	if local.WeaveIP == nil {
-		return fmt.Errorf("egress tunnel needs the weave bridge's address")
+		return fmt.Errorf("egress tunnels need the weave bridge's address")
 	}
 	br, err := netlink.LinkByName(bridge)
 	if err != nil {
 		return fmt.Errorf("weave bridge: %v", err)
 	}
 	mtu := br.Attrs().MTU - ipipOverhead
-	if exists {
+	names := make([]string, 0, len(want))
+	for n := range want {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		peer := want[name]
+		link := have[name]
+		if link != nil {
 			t, ok := link.(*netlink.Iptun)
-		if !ok || !t.Local.Equal(local.WeaveIP) || t.Remote != nil && !t.Remote.IsUnspecified() || t.Attrs().MTU != mtu {
+			if !ok || !t.Local.Equal(local.WeaveIP) || !t.Remote.Equal(peer.WeaveIP) || t.Attrs().MTU != mtu {
 				if err := netlink.LinkDel(link); err != nil {
 					return err
 				}
-			exists = false
+				link = nil
 			}
 		}
-	if !exists {
-		t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: Tunnel, MTU: mtu}, Local: local.WeaveIP, PMtuDisc: 1}
+		if link == nil {
+			t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: name, MTU: mtu}, Local: local.WeaveIP, Remote: peer.WeaveIP, PMtuDisc: 1}
 			if err := netlink.LinkAdd(t); err != nil {
-			return fmt.Errorf("creating egress tunnel: %v", err)
+				return fmt.Errorf("creating egress tunnel %s to %s: %v", name, peer.Name, err)
 			}
-		if link, err = netlink.LinkByName(Tunnel); err != nil {
+			if link, err = netlink.LinkByName(name); err != nil {
 				return err
 			}
 		}
-	// the holder receives members' packets from its own address; only on this device
+		if err := tunnelSetup(link, local); err != nil {
+			return fmt.Errorf("%s: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// tunnelSetup: the holder receives members' packets from its own address (accept_local); reverse-path filtering
+// rejects what arrives on a device with no IPv4 address unless the reverse route leaves by that very device, so
+// the tunnel carries this node's bridge address (never on the wire from here: the tunnel sets the outer source,
+// and what crosses it is forwarded).
+func tunnelSetup(link netlink.Link, local Local) error {
+	name := link.Attrs().Name
 	for k, v := range map[string]string{"accept_local": "1", "rp_filter": "2"} {
-		if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+Tunnel+"/"+k, []byte(v), 0o644); err != nil {
-			return fmt.Errorf("sysctl %s on %s: %v", k, Tunnel, err)
+		if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+name+"/"+k, []byte(v), 0o644); err != nil {
+			return fmt.Errorf("sysctl %s: %v", k, err)
 		}
 	}
-	// reverse-path filtering rejects what arrives on a device with no IPv4 address unless the reverse route
-	// leaves by that very device (replies on a member's node, members' packets on the holder do not): the
-	// tunnel carries this node's bridge address (it never appears on the wire from here: the tunnel sets the
-	// outer source itself, and what crosses it is forwarded)
-	addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK}
 	addrs, err := netlink.AddrList(link, unix.AF_INET)
 	if err != nil {
 		return err
@@ -375,12 +513,13 @@ func ensureTunnel(want bool, local Local) error {
 		}
 		a := a
 		if err := netlink.AddrDel(link, &a); err != nil {
-			return fmt.Errorf("removing stale address %s from %s: %v", a.IPNet, Tunnel, err)
+			return fmt.Errorf("removing stale address %s: %v", a.IPNet, err)
 		}
 	}
 	if !has {
+		addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK}
 		if err := netlink.AddrAdd(link, addr); err != nil {
-			return fmt.Errorf("address on %s: %v", Tunnel, err)
+			return fmt.Errorf("address: %v", err)
 		}
 	}
 	return netlink.LinkSetUp(link)

lb/egress_test.go

@@ -1,12 +1,12 @@
 package lb
 
 import (
+	"fmt"
 	"net"
 	"strings"
 	"testing"
 	"time"
 
-	"golang.org/x/sys/unix"
 	coreapi "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -199,6 +199,7 @@ func TestRenderedEgressRules(t *testing.T) {
 	_, podNet, _ := net.ParseCIDR("10.32.0.0/12")
 	e := egressFor("mail", written())
 	svcs := []*coreapi.Service{mailSvc()}
+	toW1, toW2, toCP := TunnelName("w1"), TunnelName("w2"), TunnelName("cp")
 	member := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
 	r := Render(Desired{}, member, "w2", podNet, nil)
 	for _, want := range []string{
@@ -206,19 +207,27 @@ func TestRenderedEgressRules(t *testing.T) {
 		"-A REWEAVE-LB-CLASS -d 86.107.168.24/32 -j RETURN\n", // a node is not outside
 		"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -m comment --comment \"gitea/mail-0\" -j CONNMARK --set-xmark 0x100000/0x100000\n",
 		"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -j MARK --set-xmark 0x100000/0x100000\n",
-		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n",
-		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		// port protocols within this node's block, into the holder's own tunnel
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p sctp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		// everything else (ping, GRE, ...) from the address alone
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j SNAT --to-source 86.107.168.15\n",
 	} {
 		if !strings.Contains(r.Nat, want) {
 			t.Fatalf("member nat lacks %q:\n%s", want, r.Nat)
 		}
 	}
+	// the catch-all SNAT comes after the port-ranged ones (or ports would not stay in the block)
+	if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p sctp -j SNAT") {
+		t.Fatalf("SNAT order:\n%s", r.Nat)
+	}
 	for _, want := range []string{
-		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o reweave-eg -m mark --mark 0x100000/0x100000 -p tcp -j RETURN\n",
+		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j RETURN\n",
 		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -m mark --mark 0x100000/0x100000 -m comment --comment \"gitea/mail-0\" -j DROP\n",
-		"-A REWEAVE-LB-FWD -i reweave-eg -d 86.107.168.15/32 -j DROP\n",
-		"-A REWEAVE-LB-FWD -i reweave-eg -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n",
-		"-A REWEAVE-LB-FWD -i reweave-eg -j DROP\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -d 86.107.168.15/32 -j DROP\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -j DROP\n",
 		"-A REWEAVE-LB-IN -p 4 -s 10.40.0.0/32 -m mac --mac-source aa:aa:aa:aa:aa:02 -j RETURN\n",
 		"-A REWEAVE-LB-IN -p 4 -j DROP\n",
 	} {
@@ -228,28 +237,52 @@ func TestRenderedEgressRules(t *testing.T) {
 	}
 	for _, want := range []string{
 		"-A REWEAVE-LB-MARK -m conntrack --ctdir ORIGINAL -m connmark --mark 0x100000/0x100000 -j MARK --set-xmark 0x100000/0x100000\n",
-		"-A REWEAVE-LB-MSS -o reweave-eg -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n",
+		"-A REWEAVE-LB-MSS -o rweg+ -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n",
 	} {
 		if !strings.Contains(r.Mangle, want) {
 			t.Fatalf("member mangle lacks %q:\n%s", want, r.Mangle)
 		}
 	}
-	// the guard's DROP comes after the member's own RETURNs: nothing else of its leaves
-	if strings.Index(r.Filter, "-p tcp -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") {
+	if strings.Contains(r.Mangle, "--set-xmark 0x") && strings.Contains(r.Mangle, "-i "+toW1) {
+		t.Fatalf("a member's node does not tag what its holder sends back:\n%s", r.Mangle)
+	}
+	// the guard's DROP comes after the member's own RETURN: nothing else of its leaves
+	if strings.Index(r.Filter, toW1+" -m mark --mark 0x100000/0x100000 -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") {
 		t.Fatalf("guard order:\n%s", r.Filter)
 	}
-	// the holder: accepts each node's block from the tunnel, drops the rest; a member on it SNATs to its own block
+
+	// the holder: tags each tunnel's connections with its node, accepts each node's block, nothing foreign
 	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
 	r = Render(Desired{}, h, "w1", podNet, nil)
+	idx := peerIndex(h, "w1")
+	tagW2 := fmt.Sprintf("0x%x", peerTag(idx["w2"]))
 	for _, want := range []string{
-		"-A REWEAVE-LB-FWD -i reweave-eg -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n",
-		"-A REWEAVE-LB-FWD -i reweave-eg -s 86.107.168.15/32 -p udp -m udp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n",
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir REPLY -m connmark --mark 0x200000/0x200000 -j CONNMARK --restore-mark --nfmask 0xff200000 --ctmask 0xff200000\n",
+		"-A REWEAVE-LB-MARK -i " + toW2 + " -m conntrack --ctstate NEW -m connmark --mark 0x0/0xff200000 -m comment --comment \"from w2\" -j CONNMARK --set-xmark " + tagW2 + "/0xff200000\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("holder mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -m connmark ! --mark " + tagW2 + "/0xff200000 -j DROP\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i " + toCP + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p sctp -m sctp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN\n",
 		"-A REWEAVE-LB-FWD -s 10.40.0.4/32 -o weave -m mark --mark 0x100000/0x100000 -j DROP\n",
 	} {
 		if !strings.Contains(r.Filter, want) {
 			t.Fatalf("holder filter lacks %q:\n%s", want, r.Filter)
 		}
 	}
+	// foreign tags are refused before anything of that tunnel is accepted; a port outside the block before the
+	// catch-all accept
+	if strings.Index(r.Filter, "! --mark "+tagW2) > strings.Index(r.Filter, "\"block of w2\"") ||
+		strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP") >
+			strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN") {
+		t.Fatalf("holder filter order:\n%s", r.Filter)
+	}
 	if !strings.Contains(r.Nat, "-A REWEAVE-LB-POST -s 10.40.0.4/32 ! -o weave -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:61512-62023\n") {
 		t.Fatalf("holder's own member:\n%s", r.Nat)
 	}
@@ -260,32 +293,56 @@ func TestRenderedEgressRules(t *testing.T) {
 	}
 }
 
+func TestTunnelNames(t *testing.T) {
+	a, b := TunnelName("reweave-worker-fe19b040"), TunnelName("reweave-cp-94a8db43")
+	if a == b || len(a) > 15 || !strings.HasPrefix(a, TunnelPrefix) || a != TunnelName("reweave-worker-fe19b040") {
+		t.Fatalf("tunnel names must be distinct, stable, short: %q %q", a, b)
+	}
+}
+
 func TestPolicyRules(t *testing.T) {
 	e := egressFor("mail", written())
 	svcs := []*coreapi.Service{mailSvc()}
 	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
 	rules := wantRules(h, "w1")
 	tables := peerTables(h, "w1")
-	if len(rules) != 4 {
-		t.Fatalf("2 peers x tcp,udp: %+v", rules)
-	}
+	idx := peerIndex(h, "w1")
+	ports, tags := 0, 0
 	for _, r := range rules {
-		if r.priority != HolderRulePriority || r.dst != "86.107.168.15/32" || r.src != "" || r.mark != 0 {
+		if r.priority != HolderRulePriority || r.src != "" {
 			t.Fatalf("holder rule: %+v", r)
 		}
-		if r.ipproto != unix.IPPROTO_TCP && r.ipproto != unix.IPPROTO_UDP {
-			t.Fatalf("protocol: %+v", r)
+		switch {
+		case r.ipproto != 0:
+			ports++
+			if r.dst != "86.107.168.15/32" || r.mark != 0 {
+				t.Fatalf("port rule: %+v", r)
 			}
 			if (r.from == 62024) != (r.table == tables["w2"]) || (r.from == 61000) != (r.table == tables["cp"]) {
 				t.Fatalf("block routed to the wrong node: %+v %v", r, tables)
 			}
+		default:
+			tags++
+			if r.mask != peerMask || r.dst != "" {
+				t.Fatalf("tag rule: %+v", r)
+			}
+			for name, i := range idx {
+				if r.mark == peerTag(i) && r.table != tables[name] {
+					t.Fatalf("tag of %s routed to table %d, want %d", name, r.table, tables[name])
+				}
+			}
+		}
+	}
+	if ports != 2*len(portProtocols) || tags != 2 {
+		t.Fatalf("2 peers: %d port rules (tcp, udp, sctp each), %d tag rules: %+v", ports, tags, rules)
 	}
 	if HolderRulePriority >= LocalRulePriority {
 		t.Fatalf("holder rules must precede the local table")
 	}
 	m := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
 	rules = wantRules(m, "w2")
-	if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].table != peerTables(m, "w2")["w1"] {
+	if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].mask != EgressMark ||
+		rules[0].table != peerTables(m, "w2")["w1"] {
 		t.Fatalf("member rule: %+v", rules)
 	}
 	u := ComputeEgress("w2", nodes, svcs, []*EgressService{egressFor("mail", EgressServiceStatus{})}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)

reweave/test/egress/egress-test.yaml

@@ -32,19 +32,25 @@ items:
     metadata: {name: member-remote, namespace: egtest, labels: {egress: member}}
     spec:
       nodeName: REMOTE
-      containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}]
+      containers:
+        # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out)
+        - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}}
   - apiVersion: v1
     kind: Pod
     metadata: {name: member-other, namespace: egtest, labels: {egress: member}}
     spec:
       nodeName: OTHER
-      containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}]
+      containers:
+        # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out)
+        - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}}
   - apiVersion: v1
     kind: Pod
     metadata: {name: member-holder, namespace: egtest, labels: {egress: member}}
     spec:
       nodeName: HOLDER
-      containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}]
+      containers:
+        # NET_RAW: ping and traceroute (the runtime's default capabilities leave it out)
+        - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {capabilities: {add: [NET_RAW]}}}
   - apiVersion: v1
     kind: Pod
     metadata: {name: member-dns, namespace: egtest, labels: {egress: member}}

reweave/test/egress/run.sh

@@ -10,10 +10,11 @@
 # Asserts, in order:
 #   0. the holder writes the status: host, address, a port block for every node
 #   1. the first packet of a member started after the EgressService leaves as ADDR (CNI admission)
-#   A1/A2. members on every node leave as ADDR (TCP and UDP) and get their replies
+#   A1/A2. members on every node leave as ADDR (TCP and UDP) and get their replies; ping works from every node
+#   ICMP errors from routers on the way (traceroute's time-exceeded) reach a member on another node
 #   A4. a non-member pod on the same node as a member leaves as its node (untouched), and no rule names it
 #   A3. a member talking to a pod sees its own pod address there (east-west untouched)
-#   A5. naming an address nobody holds clears the status, and members' outside traffic is dropped, not leaked
+#   A5. naming an address nobody holds clears the status, and members' outside traffic (TCP, ping) is dropped
 #   A6. deleting the EgressService removes its state: members leave as their node again, rules are gone
 # Leaves nothing behind (namespace egtest is deleted, also on failure).
 set -euo pipefail
@@ -72,6 +73,19 @@ got=$(whoami_udp member-dns || true)
 [ "$got" = "$ADDR" ] || fail "A1: member-dns (UDP) left as '$got', want $ADDR"
 ok "A1/A2. members on $REMOTE, $OTHER and $HOLDER leave as $ADDR over TCP and UDP, replies come back"
 
+for p in member-remote member-other member-holder; do
+    k -n egtest exec $p -- ping -c 2 -W 3 1.1.1.1 >/dev/null || fail "ping from $p got no reply"
+done
+ok "ping from members on all three nodes gets its replies"
+
+# hop 1 is the member's own node, hop 2 the holder (it answers its own address, the member cannot see it); every
+# later hop is a router on the internet whose time-exceeded comes back to ADDR and must find the member
+hops=$(k -n egtest exec member-remote -- traceroute -n -m 8 -q 1 -w 2 1.1.1.1 2>&1 || true)
+echo "$hops" | awk '$1 >= 3 && $2 ~ /^[0-9]+\./ {found=1} END {exit !found}' \
+    || fail "no router beyond the holder answered the member's traceroute:
+$hops"
+ok "ICMP errors from routers beyond the holder reach a member on $REMOTE (traceroute)"
+
 got=$(whoami_tcp plain || true)
 [ "$got" = "$remote_ip" ] || fail "A4: non-member left as '$got', want its node $remote_ip"
 plain_ip=$(k -n egtest get pod plain -o "jsonpath='{.status.podIP}'")
@@ -90,6 +104,9 @@ k -n egtest patch svc mail --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"192
 eventually 60 status_is "  " || fail "A5: status not cleared: '$(status)'"
 got=$(whoami_tcp member-remote || true)
 [ -z "$got" ] || fail "A5: with no holder a member still got out, as '$got'"
+if k -n egtest exec member-remote -- ping -c 1 -W 3 1.1.1.1 >/dev/null 2>&1; then
+    fail "A5: with no holder a member's ping still got out"
+fi
 k -n egtest patch svc mail --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"$ADDR\"}}'" >/dev/null
 eventually 60 status_is "$want" || fail "status not restored: '$(status)'"
 eventually 30 tcp_is member-remote "$ADDR" || fail "member did not get back out as $ADDR"