lb: route filters may not change the forwarding headers

The proxy sets Forwarded, X-Forwarded-For/-Host/-Proto and X-Real-IP from
the connection, but applied a route's request header filters after that,
so whoever could write a route to a backend could tell it any client
address (a backend that trusts the gateway's X-Forwarded-For believes it).

An HTTPRoute whose request header filter sets, adds or removes one of
them (names compared case-insensitively) is now not accepted
(UnsupportedValue), like the other filters we do not serve. The proxy
also sets them after the filters, clearing them first: SetXForwarded
appends to an X-Forwarded-For already there.

The proxy test's backend now prints every X-Forwarded-For value; it
printed only the first, which hid an appended one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ

.gitea/workflows/images.yml

@@ -54,3 +54,14 @@ jobs:
               .
             echo "pushed $image"
           done
+
+      # The images live in the registry now; the runner keeps only a bounded build cache. Without this the
+      # runner's 10Gi Docker volume fills up after a handful of builds (no space left on device mid-build).
+      - name: Free the runner's disk
+        if: always()
+        run: |
+          set -u
+          docker image ls --format '{{.Repository}}:{{.Tag}}' | grep "^$IMAGE_PREFIX/" | xargs -r docker image rm -f
+          docker image prune -f
+          docker builder prune -f --keep-storage 4GB
+          docker system df

lb/addresses.go

@@ -0,0 +1,87 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+// Local is what this node publishes about itself, read from its interfaces.
+type Local struct {
+	Addresses []net.IP
+	WeaveIP   net.IP
+	WeaveMAC  net.HardwareAddr
+	PodNet    *net.IPNet // the weave allocation range: the bridge address's network
+}
+
+// overlay- and container-side interfaces: their addresses are never held ones
+var overlayPrefixes = []string{"weave", "datapath", "vxlan-", "veth", "docker", "cni"}
+
+func overlay(name string) bool {
+	for _, p := range overlayPrefixes {
+		if strings.HasPrefix(name, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// ReadLocal lists the global IPv4 addresses on host interfaces, and the weave bridge's address and MAC.
+func ReadLocal() (Local, error) {
+	var l Local
+	links, err := netlink.LinkList()
+	if err != nil {
+		return l, err
+	}
+	for _, link := range links {
+		name := link.Attrs().Name
+		addrs, err := netlink.AddrList(link, unix.AF_INET)
+		if err != nil {
+			return l, fmt.Errorf("%s: %v", name, err)
+		}
+		if name == bridge {
+			for _, a := range addrs {
+				if a.IPNet != nil && a.IP.To4() != nil {
+					l.WeaveIP = a.IP.To4()
+					l.WeaveMAC = link.Attrs().HardwareAddr
+					l.PodNet = &net.IPNet{IP: a.IP.Mask(a.Mask), Mask: a.Mask}
+					break
+				}
+			}
+			continue
+		}
+		if overlay(name) || link.Attrs().Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		for _, a := range addrs {
+			if a.Scope == unix.RT_SCOPE_UNIVERSE && a.IP.To4() != nil && a.IP.IsGlobalUnicast() {
+				l.Addresses = append(l.Addresses, a.IP.To4())
+			}
+		}
+	}
+	// an address inside the pod network is never a host one (whatever interface it is on)
+	if l.PodNet != nil {
+		kept := l.Addresses[:0]
+		for _, ip := range l.Addresses {
+			if !l.PodNet.Contains(ip) {
+				kept = append(kept, ip)
+			}
+		}
+		l.Addresses = kept
+	}
+	sort.Slice(l.Addresses, func(i, j int) bool { return l.Addresses[i].String() < l.Addresses[j].String() })
+	return l, nil
+}
+
+// Annotations is what this node's Node object should carry for l.
+func (l Local) Annotations() map[string]string {
+	a := map[string]string{AnnAddresses: FormatAddresses(l.Addresses)}
+	if l.WeaveIP != nil && l.WeaveMAC != nil {
+		a[AnnWeave] = l.WeaveIP.String() + "," + l.WeaveMAC.String()
+	}
+	return a
+}

lb/admit/admit.go

@@ -0,0 +1,88 @@
+// Package admit is the protocol between the weave CNI plugin and weave-npc's egress agent: at CNI ADD the
+// plugin tells the agent which pod got which IP, and the agent answers once the pod's egress rules are in
+// place, so an EgressService member's first packet already leaves from the Service's address (or not at all).
+//
+// It depends on the standard library only: the plugin binary must not grow a Kubernetes client.
+package admit
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// SocketPath is where the agent listens (a hostPath directory shared with the host, where the plugin runs).
+const SocketPath = "/run/weave-npc/egress.sock"
+
+// Request is one pod's admission: the IP the plugin just allocated to it.
+type Request struct {
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+	IP        string `json:"ip"`
+}
+
+// Response says whether the pod is an EgressService member (its rules are installed either way when 200).
+type Response struct {
+	Member bool `json:"member"`
+}
+
+// Timeout bounds one admission (the agent may have to fetch the pod from the API server).
+const Timeout = 20 * time.Second
+
+// PodFromArgs reads the pod's namespace and name from CNI_ARGS ("K8S_POD_NAMESPACE=..;K8S_POD_NAME=..").
+func PodFromArgs(args string) (namespace, name string) {
+	for _, kv := range strings.Split(args, ";") {
+		k, v, ok := strings.Cut(kv, "=")
+		if !ok {
+			continue
+		}
+		switch k {
+		case "K8S_POD_NAMESPACE":
+			namespace = v
+		case "K8S_POD_NAME":
+			name = v
+		}
+	}
+	return namespace, name
+}
+
+// Admit asks the agent at socket to admit req, and returns only once the agent has installed the pod's rules.
+func Admit(ctx context.Context, socket string, req Request) (Response, error) {
+	var resp Response
+	body, err := json.Marshal(req)
+	if err != nil {
+		return resp, err
+	}
+	client := &http.Client{Transport: &http.Transport{
+		DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", socket)
+		},
+	}}
+	ctx, cancel := context.WithTimeout(ctx, Timeout)
+	defer cancel()
+	hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://weave-npc/admit", bytes.NewReader(body))
+	if err != nil {
+		return resp, err
+	}
+	hreq.Header.Set("Content-Type", "application/json")
+	hresp, err := client.Do(hreq)
+	if err != nil {
+		return resp, fmt.Errorf("egress admission (weave-npc at %s): %v", socket, err)
+	}
+	defer hresp.Body.Close()
+	out, _ := io.ReadAll(io.LimitReader(hresp.Body, 1<<16))
+	if hresp.StatusCode != http.StatusOK {
+		return resp, fmt.Errorf("egress admission refused (%s): %s", hresp.Status, strings.TrimSpace(string(out)))
+	}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		return resp, fmt.Errorf("egress admission: bad answer: %v", err)
+	}
+	return resp, nil
+}

lb/controller.go

@@ -0,0 +1,629 @@
+package lb
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+
+	"github.com/rajch/weave/common"
+	"github.com/rajch/weave/lb/admit"
+)
+
+// Controller watches Services, EndpointSlices, Nodes, EgressServices and this node's Pods, and keeps this
+// node's part of the picture true. It also answers the CNI plugin's egress admissions.
+type Controller struct {
+	self   string
+	client kubernetes.Interface
+	egress rest.Interface
+	dp     *Datapath
+	// readLocal is the node's own interfaces (swappable in tests)
+	readLocal func() (Local, error)
+
+	services, slices, nodes, pods, egresses cache.Store
+	synced                                  []cache.InformerSynced
+	egressSynced                            cache.InformerSynced // nil while the EgressService API is absent
+	kick                                    chan struct{}
+	ready                                   chan struct{} // closed once the first pass ran
+
+	mu           sync.Mutex // one pass at a time (the loop, or an admission)
+	lastProblems map[string]bool
+	// Services whose status this node wrote since it started (so a restart rewrites ipMode once)
+	written map[types.UID]string
+	// admitted: IP -> pod the CNI plugin gave it; fetched: pods admitted before the pod informer had them
+	admitted map[string]Admission
+	fetched  map[string]*coreapi.Pod
+	at       map[string]time.Time
+	// lastEgress is what was last applied, reused while the EgressService caches are not synced (never wipe
+	// members' rules on a partial view)
+	lastEgress EgressDesired
+
+	// Gateways (nil gateway client: not served). Secrets are read only in gwNamespaces.
+	gateway                                                      rest.Interface
+	gwNamespaces                                                 []string
+	proxy                                                        *Proxy
+	classes, gateways, httpRoutes, tcpRoutes, grants, namespaces cache.Store
+	secrets                                                      []cache.Store
+	gatewaySynced                                                []cache.InformerSynced // empty while the Gateway API is absent
+	unbound                                                      map[string]string      // listeners the proxy could not bind, last pass
+}
+
+// Resync is how often the node re-reads its interfaces and re-applies everything even without API events.
+const Resync = 30 * time.Second
+
+// NewController runs nothing yet; Run does. egress is the EgressService API client.
+func NewController(self string, client kubernetes.Interface, egress rest.Interface, dp *Datapath) *Controller {
+	return &Controller{self: self, client: client, egress: egress, dp: dp, readLocal: ReadLocal,
+		kick: make(chan struct{}, 1), ready: make(chan struct{}), lastProblems: map[string]bool{},
+		written: map[types.UID]string{}, admitted: map[string]Admission{}, fetched: map[string]*coreapi.Pod{},
+		at: map[string]time.Time{}}
+}
+
+func (c *Controller) poke() {
+	select {
+	case c.kick <- struct{}{}:
+	default:
+	}
+}
+
+func newInformer(c *Controller, lw cache.ListerWatcher, obj runtime.Object) (cache.Store, cache.Controller) {
+	return cache.NewInformer(lw, obj, 0, cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(interface{}) { c.poke() },
+		UpdateFunc: func(interface{}, interface{}) { c.poke() },
+		DeleteFunc: func(interface{}) { c.poke() },
+	})
+}
+
+// egressAPI reports whether the EgressService API is served (the CRD is installed).
+func (c *Controller) egressAPI() (bool, error) {
+	_, err := c.client.Discovery().ServerResourcesForGroupVersion(GroupVersion.String())
+	if err == nil {
+		return true, nil
+	}
+	if apierrors.IsNotFound(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+// WithGateway has the controller serve Gateways of the classes whose controllerName is GatewayControllerName,
+// reading their certificates from Secrets in these namespaces only.
+func (c *Controller) WithGateway(client rest.Interface, namespaces []string) *Controller {
+	c.gateway, c.gwNamespaces, c.proxy, c.unbound = client, namespaces, NewProxy(), map[string]string{}
+	return c
+}
+
+// gatewayAPI is which Gateway API resources are served (nil: none, the CRDs are not installed).
+func (c *Controller) gatewayAPI() (map[string]bool, error) {
+	l, err := c.client.Discovery().ServerResourcesForGroupVersion(GatewayGroupVersion.String())
+	if apierrors.IsNotFound(err) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	served := map[string]bool{}
+	for _, r := range l.APIResources {
+		served[r.Name] = true
+	}
+	return served, nil
+}
+
+func (c *Controller) startGateway(ctx context.Context, served map[string]bool) {
+	run := func(store *cache.Store, resource string, obj runtime.Object) {
+		if !served[resource] {
+			return // an older bundle without this kind: nothing of it to serve
+		}
+		var ctl cache.Controller
+		*store, ctl = newInformer(c, cache.NewListWatchFromClient(c.gateway, resource, "", fields.Everything()), obj)
+		go ctl.Run(ctx.Done())
+		c.gatewaySynced = append(c.gatewaySynced, ctl.HasSynced)
+	}
+	run(&c.classes, GatewayClasses, &GatewayClass{})
+	run(&c.gateways, Gateways, &Gateway{})
+	run(&c.httpRoutes, HTTPRoutes, &HTTPRoute{})
+	run(&c.tcpRoutes, TCPRoutes, &TCPRoute{})
+	run(&c.grants, ReferenceGrants, &ReferenceGrant{})
+	var ctl cache.Controller
+	c.namespaces, ctl = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "namespaces", "", fields.Everything()), &coreapi.Namespace{})
+	go ctl.Run(ctx.Done())
+	c.gatewaySynced = append(c.gatewaySynced, ctl.HasSynced)
+	for _, ns := range c.gwNamespaces {
+		var store cache.Store
+		store, ctl = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "secrets", ns, fields.Everything()), &coreapi.Secret{})
+		go ctl.Run(ctx.Done())
+		c.secrets = append(c.secrets, store)
+		c.gatewaySynced = append(c.gatewaySynced, ctl.HasSynced)
+	}
+	common.Log.Infof("lb: serving Gateways of controller %s (certificates from %v)", GatewayControllerName, c.gwNamespaces)
+}
+
+// gatewayReady: the Gateway view can be trusted (the API is served and every informer has synced).
+func (c *Controller) gatewayReady() bool {
+	if c.gateway == nil || len(c.gatewaySynced) == 0 {
+		return false
+	}
+	for _, s := range c.gatewaySynced {
+		if !s() {
+			return false
+		}
+	}
+	return true
+}
+
+// tryStartGateway starts the Gateway informers once the API is served.
+func (c *Controller) tryStartGateway(ctx context.Context) {
+	if c.gateway == nil || len(c.gatewaySynced) != 0 {
+		return
+	}
+	served, err := c.gatewayAPI()
+	if err != nil {
+		common.Log.Warnf("lb: cannot tell whether the Gateway API is served: %v", err)
+		return
+	}
+	if served[Gateways] {
+		c.startGateway(ctx, served)
+	}
+}
+
+func (c *Controller) startEgress(ctx context.Context) {
+	var ctl cache.Controller
+	c.egresses, ctl = newInformer(c, cache.NewListWatchFromClient(c.egress, EgressServices, "", fields.Everything()), &EgressService{})
+	go ctl.Run(ctx.Done())
+	c.egressSynced = ctl.HasSynced
+}
+
+// Run blocks until ctx ends.
+func (c *Controller) Run(ctx context.Context) {
+	var sc, ec, nc, pc cache.Controller
+	c.services, sc = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "services", "", fields.Everything()), &coreapi.Service{})
+	c.slices, ec = newInformer(c, cache.NewListWatchFromClient(c.client.DiscoveryV1().RESTClient(), "endpointslices", "", fields.Everything()), &discovery.EndpointSlice{})
+	c.nodes, nc = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "nodes", "", fields.Everything()), &coreapi.Node{})
+	c.pods, pc = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "pods", "",
+		fields.OneTermEqualSelector("spec.nodeName", c.self)), &coreapi.Pod{})
+	for _, ctl := range []cache.Controller{sc, ec, nc, pc} {
+		go ctl.Run(ctx.Done())
+		c.synced = append(c.synced, ctl.HasSynced)
+	}
+	// with the API present, nothing is applied before EgressServices are known: an empty view would wipe
+	// members' rules (their traffic would leave from the node); without it, there can be no members
+	for {
+		present, err := c.egressAPI()
+		if err == nil {
+			if present {
+				c.startEgress(ctx)
+				c.synced = append(c.synced, c.egressSynced)
+			}
+			break
+		}
+		common.Log.Warnf("lb: cannot tell whether the EgressService API is served (retrying): %v", err)
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(5 * time.Second):
+		}
+	}
+	c.tryStartGateway(ctx)
+	if c.proxy != nil {
+		defer c.proxy.Close()
+	}
+	go c.serveAdmission(ctx)
+	if !cache.WaitForCacheSync(ctx.Done(), c.synced...) {
+		return
+	}
+	tick := time.NewTicker(Resync)
+	defer tick.Stop()
+	c.poke()
+	first := true
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+			c.mu.Lock()
+			c.tryStartGateway(ctx)
+			c.mu.Unlock()
+			if c.egressSynced == nil {
+				// the API may have been installed since
+				if present, err := c.egressAPI(); err == nil && present {
+					c.mu.Lock()
+					c.startEgress(ctx)
+					c.mu.Unlock()
+				}
+			}
+		case <-c.kick:
+			// let a burst of events settle into one pass
+			time.Sleep(200 * time.Millisecond)
+			select {
+			case <-c.kick:
+			default:
+			}
+		}
+		c.mu.Lock()
+		err := c.sync(ctx)
+		c.mu.Unlock()
+		if err != nil {
+			common.Log.Errorf("lb: %v", err)
+		}
+		if first {
+			first = false
+			close(c.ready)
+		}
+	}
+}
+
+// egressViewSynced: the EgressService view can be trusted (absent API: trivially empty; else synced).
+func (c *Controller) egressViewSynced() bool {
+	return c.egressSynced == nil || c.egressSynced()
+}
+
+// sync is one pass: publish this node, compute, apply, write statuses. Callers hold c.mu.
+func (c *Controller) sync(ctx context.Context) error {
+	local, err := c.readLocal()
+	if err != nil {
+		return fmt.Errorf("reading interfaces: %v", err)
+	}
+	if err := c.publish(ctx, local); err != nil {
+		return err
+	}
+	if local.PodNet == nil {
+		return fmt.Errorf("weave bridge has no address yet; not forwarding")
+	}
+
+	var nodes []Node
+	var problems []string
+	for _, o := range c.nodes.List() {
+		n, err := ParseNode(o.(*coreapi.Node))
+		if err != nil {
+			problems = append(problems, err.Error())
+		}
+		nodes = append(nodes, n)
+	}
+	var svcs []*coreapi.Service
+	for _, o := range c.services.List() {
+		svcs = append(svcs, o.(*coreapi.Service))
+	}
+	var slices []*discovery.EndpointSlice
+	for _, o := range c.slices.List() {
+		slices = append(slices, o.(*discovery.EndpointSlice))
+	}
+	d := Compute(c.self, nodes, svcs, slices)
+	var g *GatewayDesired
+	if c.gatewayReady() {
+		gd := ComputeGateways(c.self, nodes, c.gatewayInput(svcs, slices))
+		d = MergeGateways(d, gd, c.self, nodes)
+		g = &gd
+	} else if c.gateway != nil && len(c.gatewaySynced) != 0 {
+		problems = append(problems, "Gateway view not synced yet: Gateways not served")
+	}
+	// rewrite once after start what an earlier run may have written without ipMode
+	for _, f := range d.Forwards {
+		if s := c.service(f.Namespace, f.Name); s != nil && c.written[s.UID] != f.Address.String() && !hasStatusFor(d, f) {
+			d.Status = append(d.Status, Status{Namespace: f.Namespace, Name: f.Name, Address: f.Address})
+		}
+	}
+
+	e := c.lastEgress
+	e.Status = nil // reusing the datapath, never the writes of an earlier view
+	if c.egressViewSynced() {
+		e = c.computeEgress(nodes, svcs)
+	} else {
+		problems = append(problems, "EgressServices not synced yet: keeping the egress rules last applied")
+	}
+
+	current := map[string]bool{}
+	for _, p := range append(append(problems, d.Problems...), e.Problems...) {
+		current[p] = true
+		if !c.lastProblems[p] {
+			common.Log.Warnf("lb: %s", p)
+		}
+	}
+	c.lastProblems = current
+
+	if err := c.dp.Apply(d, e, c.self, local); err != nil {
+		return fmt.Errorf("applying datapath: %v", err)
+	}
+	c.lastEgress = e
+	if g != nil {
+		// the ports are let in by now; bind what this node serves, close what it no longer does
+		unbound := map[string]string{}
+		for k, err := range c.proxy.Apply(g.Serve) {
+			unbound[k] = err.Error()
+		}
+		if !sameJSON(unbound, c.unbound) {
+			c.unbound = unbound
+			c.poke() // the statuses say so on the next pass
+		}
+		c.writeGatewayStatuses(ctx, *g)
+	}
+	for _, st := range d.Status {
+		if err := c.writeStatus(ctx, st); err != nil {
+			return err
+		}
+	}
+	for _, st := range e.Status {
+		if err := PatchEgressStatus(ctx, c.egress, st.Namespace, st.Name, st.Status); err != nil {
+			return fmt.Errorf("status of egress %s/%s: %v", st.Namespace, st.Name, err)
+		}
+		common.Log.Infof("lb: egress %s/%s: host %q, address %q, %d port blocks", st.Namespace, st.Name, st.Status.Host, st.Status.Address, len(st.Status.Blocks))
+	}
+	return nil
+}
+
+// computeEgress gathers this node's pods (informer, plus pods admitted before the informer had them) and computes.
+func (c *Controller) computeEgress(nodes []Node, svcs []*coreapi.Service) EgressDesired {
+	var egresses []*EgressService
+	if c.egresses != nil {
+		for _, o := range c.egresses.List() {
+			egresses = append(egresses, o.(*EgressService))
+		}
+	}
+	seen := map[string]bool{}
+	var pods []*coreapi.Pod
+	for _, o := range c.pods.List() {
+		p := o.(*coreapi.Pod)
+		seen[p.Namespace+"/"+p.Name] = true
+		pods = append(pods, p)
+	}
+	// forget admissions the API has caught up with or that are long gone
+	for ip, a := range c.admitted {
+		key := a.Namespace + "/" + a.Name
+		o, ok, _ := c.pods.GetByKey(key)
+		caughtUp := ok && o.(*coreapi.Pod).Status.PodIP == ip
+		if caughtUp || (!ok && time.Since(c.at[ip]) > 2*time.Minute) {
+			delete(c.admitted, ip)
+			delete(c.at, ip)
+		}
+	}
+	for key, p := range c.fetched {
+		stillAdmitted := false
+		for _, a := range c.admitted {
+			if a.Namespace+"/"+a.Name == key {
+				stillAdmitted = true
+			}
+		}
+		if !stillAdmitted {
+			delete(c.fetched, key)
+			continue
+		}
+		if !seen[key] {
+			pods = append(pods, p)
+		}
+	}
+	return ComputeEgress(c.self, nodes, svcs, egresses, pods, c.admitted)
+}
+
+func hasStatusFor(d Desired, f Forward) bool {
+	for _, s := range d.Status {
+		if s.Namespace == f.Namespace && s.Name == f.Name {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *Controller) service(ns, name string) *coreapi.Service {
+	o, ok, err := c.services.GetByKey(ns + "/" + name)
+	if err != nil || !ok {
+		return nil
+	}
+	return o.(*coreapi.Service)
+}
+
+// serveAdmission answers the CNI plugin on admit.SocketPath: record the pod's IP, run a pass, answer once the
+// pod's rules are in place. A pod is refused (the plugin fails the ADD, kubelet retries) while this node cannot
+// yet tell whether it is a member.
+func (c *Controller) serveAdmission(ctx context.Context) {
+	if err := os.MkdirAll(filepath.Dir(admit.SocketPath), 0o700); err != nil {
+		common.Log.Errorf("lb: admission socket: %v", err)
+		return
+	}
+	_ = os.Remove(admit.SocketPath)
+	l, err := net.Listen("unix", admit.SocketPath)
+	if err != nil {
+		common.Log.Errorf("lb: admission socket: %v", err)
+		return
+	}
+	_ = os.Chmod(admit.SocketPath, 0o600)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/admit", func(w http.ResponseWriter, r *http.Request) {
+		var req admit.Request
+		if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil {
+			http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest)
+			return
+		}
+		member, code, err := c.admit(r.Context(), req)
+		if err != nil {
+			common.Log.Warnf("lb: admission of %s/%s (%s): %v", req.Namespace, req.Name, req.IP, err)
+			http.Error(w, err.Error(), code)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(admit.Response{Member: member})
+	})
+	srv := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+	go func() {
+		<-ctx.Done()
+		_ = srv.Close()
+	}()
+	if err := srv.Serve(l); err != nil && err != http.ErrServerClosed {
+		common.Log.Errorf("lb: admission server: %v", err)
+	}
+}
+
+func (c *Controller) admit(ctx context.Context, req admit.Request) (bool, int, error) {
+	ip := net.ParseIP(req.IP).To4()
+	if req.Namespace == "" || req.Name == "" || ip == nil {
+		return false, http.StatusBadRequest, fmt.Errorf("need namespace, name and an IPv4 address")
+	}
+	select {
+	case <-c.ready:
+	case <-ctx.Done():
+		return false, http.StatusServiceUnavailable, ctx.Err()
+	case <-time.After(admit.Timeout - 5*time.Second):
+		return false, http.StatusServiceUnavailable, fmt.Errorf("not ready yet")
+	}
+	key := req.Namespace + "/" + req.Name
+	var pod *coreapi.Pod
+	if o, ok, _ := c.pods.GetByKey(key); ok {
+		pod = o.(*coreapi.Pod)
+	} else {
+		gctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		p, err := c.client.CoreV1().Pods(req.Namespace).Get(gctx, req.Name, metav1.GetOptions{})
+		cancel()
+		if err != nil {
+			return false, http.StatusServiceUnavailable, fmt.Errorf("fetching pod: %v", err)
+		}
+		pod = p
+	}
+	if pod.Spec.NodeName != c.self {
+		return false, http.StatusConflict, fmt.Errorf("pod is bound to %q, not this node", pod.Spec.NodeName)
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if !c.egressViewSynced() {
+		return false, http.StatusServiceUnavailable, fmt.Errorf("EgressServices not synced yet")
+	}
+	c.admitted[ip.String()] = Admission{Namespace: req.Namespace, Name: req.Name}
+	c.at[ip.String()] = time.Now()
+	if _, ok, _ := c.pods.GetByKey(key); !ok {
+		c.fetched[key] = pod
+	}
+	if err := c.sync(ctx); err != nil {
+		// with no EgressService and no member rules in place there is nothing to protect or to clean up for
+		// this IP: a broken pass must not stop every pod on the node from starting
+		if len(c.lastEgress.Members) == 0 && (c.egresses == nil || len(c.egresses.List()) == 0) {
+			common.Log.Warnf("lb: admitting %s/%s despite a failed pass (no EgressServices): %v", req.Namespace, req.Name, err)
+			return false, http.StatusOK, nil
+		}
+		return false, http.StatusServiceUnavailable, err
+	}
+	for _, m := range c.lastEgress.Members {
+		if m.IP.Equal(ip) {
+			return true, http.StatusOK, nil
+		}
+	}
+	return false, http.StatusOK, nil
+}
+
+// publish keeps this node's annotations equal to what its interfaces say.
+func (c *Controller) publish(ctx context.Context, local Local) error {
+	want := local.Annotations()
+	o, ok, err := c.nodes.GetByKey(c.self)
+	if err != nil {
+		return err
+	}
+	if ok {
+		have := o.(*coreapi.Node).Annotations
+		same := true
+		for k, v := range want {
+			if have[k] != v {
+				same = false
+			}
+		}
+		if same {
+			return nil
+		}
+	}
+	patch, err := json.Marshal(map[string]interface{}{"metadata": map[string]interface{}{"annotations": want}})
+	if err != nil {
+		return err
+	}
+	if _, err := c.client.CoreV1().Nodes().Patch(ctx, c.self, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
+		return fmt.Errorf("publishing node annotations: %v", err)
+	}
+	common.Log.Infof("lb: node %s holds [%s]", c.self, want[AnnAddresses])
+	return nil
+}
+
+// writeStatus sets (ipMode Proxy: kube-proxy leaves the address to us) or clears a Service's load-balancer status.
+func (c *Controller) writeStatus(ctx context.Context, st Status) error {
+	ingress := []map[string]string{}
+	if st.Address != nil {
+		ingress = append(ingress, map[string]string{"ip": st.Address.String(), "ipMode": "Proxy"})
+	}
+	patch, err := json.Marshal(map[string]interface{}{"status": map[string]interface{}{"loadBalancer": map[string]interface{}{"ingress": ingress}}})
+	if err != nil {
+		return err
+	}
+	s, err := c.client.CoreV1().Services(st.Namespace).Patch(ctx, st.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status")
+	if err != nil {
+		return fmt.Errorf("status of %s/%s: %v", st.Namespace, st.Name, err)
+	}
+	if st.Address != nil {
+		c.written[s.UID] = st.Address.String()
+		common.Log.Infof("lb: %s/%s at %s", st.Namespace, st.Name, st.Address)
+	} else {
+		delete(c.written, s.UID)
+		common.Log.Infof("lb: %s/%s has no address", st.Namespace, st.Name)
+	}
+	return nil
+}
+
+// gatewayInput is the Gateway view the policy reads.
+func (c *Controller) gatewayInput(svcs []*coreapi.Service, slices []*discovery.EndpointSlice) GatewayInput {
+	in := GatewayInput{Services: svcs, Slices: slices, SecretNamespaces: map[string]bool{}, Unbound: c.unbound, Now: metav1.Now()}
+	list := func(s cache.Store, each func(interface{})) {
+		if s != nil {
+			for _, o := range s.List() {
+				each(o)
+			}
+		}
+	}
+	list(c.classes, func(o interface{}) { in.Classes = append(in.Classes, o.(*GatewayClass)) })
+	list(c.gateways, func(o interface{}) { in.Gateways = append(in.Gateways, o.(*Gateway)) })
+	list(c.httpRoutes, func(o interface{}) { in.HTTPRoutes = append(in.HTTPRoutes, o.(*HTTPRoute)) })
+	list(c.tcpRoutes, func(o interface{}) { in.TCPRoutes = append(in.TCPRoutes, o.(*TCPRoute)) })
+	list(c.grants, func(o interface{}) { in.Grants = append(in.Grants, o.(*ReferenceGrant)) })
+	list(c.namespaces, func(o interface{}) { in.Namespaces = append(in.Namespaces, o.(*coreapi.Namespace)) })
+	for _, s := range c.secrets {
+		list(s, func(o interface{}) { in.Secrets = append(in.Secrets, o.(*coreapi.Secret)) })
+	}
+	for _, ns := range c.gwNamespaces {
+		in.SecretNamespaces[ns] = true
+	}
+	return in
+}
+
+// writeGatewayStatuses writes what the policy says this node writes. Each write is conditional on the version it
+// was computed from: one built from a stale view fails, and the next pass recomputes it.
+func (c *Controller) writeGatewayStatuses(ctx context.Context, g GatewayDesired) {
+	for _, w := range g.Classes {
+		if err := patchStatus(ctx, c.gateway, GatewayClasses, "", w.Name, w.ResourceVersion, w.Status); err != nil {
+			common.Log.Warnf("lb: status of GatewayClass %s: %v", w.Name, err)
+			continue
+		}
+		common.Log.Infof("lb: GatewayClass %s accepted", w.Name)
+	}
+	for _, w := range g.Gateways {
+		if err := patchStatus(ctx, c.gateway, Gateways, w.Namespace, w.Name, w.ResourceVersion, w.Status); err != nil {
+			common.Log.Warnf("lb: status of Gateway %s/%s: %v", w.Namespace, w.Name, err)
+			continue
+		}
+		var conds []string
+		for _, cd := range w.Status.Conditions {
+			conds = append(conds, fmt.Sprintf("%s=%s (%s)", cd.Type, cd.Status, cd.Reason))
+		}
+		common.Log.Infof("lb: Gateway %s/%s: %v", w.Namespace, w.Name, conds)
+	}
+	for _, w := range g.Routes {
+		if err := patchStatus(ctx, c.gateway, w.Resource, w.Namespace, w.Name, w.ResourceVersion, w.Status); err != nil {
+			common.Log.Warnf("lb: status of %s %s/%s: %v", w.Resource, w.Namespace, w.Name, err)
+		}
+	}
+}

lb/datapath.go

@@ -0,0 +1,340 @@
+package lb
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// ForwardChain (nat, from PREROUTING) DNATs held addresses' ports to endpoints.
+	ForwardChain = "REWEAVE-LB"
+	// PostChain (nat, from POSTROUTING, ahead of weave's WEAVE) keeps the client's address on what ForwardChain DNATed.
+	PostChain = "REWEAVE-LB-POST"
+	// MarkChain (mangle, from PREROUTING) marks holders' connections and steers their replies.
+	MarkChain = "REWEAVE-LB-MARK"
+	// servicePrefix names the per-port chains ForwardChain jumps to.
+	servicePrefix = "REWEAVE-LBS-"
+
+	markShift = 24
+	markMask  = uint32(0xff) << markShift
+	// RulePriority is where the reply-steering rules sit: before the main table (32766).
+	RulePriority = 9000
+	// TableBase + mark is the routing table that sends replies to that mark's holder.
+	TableBase = 52000
+
+	bridge = "weave"
+)
+
+func markValue(m uint32) uint32 { return m << markShift }
+
+// portChain is the per-(service, protocol, port) chain name: stable, short enough for iptables (28 chars).
+func portChain(f Forward, p Port) string {
+	h := sha256.Sum256([]byte(fmt.Sprintf("%s/%s/%s/%d", f.Namespace, f.Name, p.Protocol, p.Port)))
+	return servicePrefix + strings.ToUpper(hex.EncodeToString(h[:])[:12])
+}
+
+// Rules is the full content of the chains this package owns, in iptables-restore form.
+type Rules struct {
+	Nat, Mangle, Filter string
+}
+
+// Render turns the desired state (load-balancer forwards and egress) into iptables-restore input for the nat,
+// mangle and filter tables. podNet is the weave allocation range (pods and bridges).
+// Each payload declares every chain it owns, so a restore with --noflush replaces exactly those.
+func Render(d Desired, e EgressDesired, self string, podNet *net.IPNet, stale []string) Rules {
+	var nat bytes.Buffer
+	nat.WriteString("*nat\n")
+	for _, c := range []string{ForwardChain, PostChain, ClassChain} {
+		fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
+	}
+	var body bytes.Buffer
+	seen := map[string]bool{}
+	for _, f := range d.Forwards {
+		a := f.Address.String()
+		if seen[a] {
+			continue
+		}
+		seen[a] = true
+		// weave masquerades everything entering the pod network from outside unless it is for a local pod
+		// (its expose rule); a connection we DNATed must keep the client's address, so it leaves the nat
+		// table here. One from a pod (hairpin) is masqueraded to this node, so its reply comes back here.
+		fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -s %s -j MASQUERADE\n", PostChain, a, podNet)
+		fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -j ACCEPT\n", PostChain, a)
+	}
+	for _, f := range d.Forwards {
+		for _, p := range f.Ports {
+			c := portChain(f, p)
+			fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
+			proto := strings.ToLower(string(p.Protocol))
+			fmt.Fprintf(&body, "-A %s -d %s/32 -p %s -m %s --dport %d -m comment --comment %q -j %s\n",
+				ForwardChain, f.Address, proto, proto, p.Port, f.Namespace+"/"+f.Name, c)
+			n := len(p.Endpoints)
+			for i, e := range p.Endpoints {
+				// DNAT to an ip:port needs the protocol in the same rule
+				if i < n-1 {
+					// the i-th endpoint takes 1/(n-i) of what the earlier ones left: uniform overall
+					fmt.Fprintf(&body, "-A %s -p %s -m statistic --mode random --probability %.10f -j DNAT --to-destination %s\n",
+						c, proto, 1/float64(n-i), e)
+				} else {
+					fmt.Fprintf(&body, "-A %s -p %s -j DNAT --to-destination %s\n", c, proto, e)
+				}
+			}
+		}
+	}
+	for _, c := range stale {
+		// chains of ports no longer forwarded: emptied here, deleted after the restore
+		fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
+	}
+
+	var mangle bytes.Buffer
+	mangle.WriteString("*mangle\n")
+	for _, c := range []string{MarkChain, MSSChain} {
+		fmt.Fprintf(&mangle, ":%s - [0:0]\n", c)
+	}
+	if len(d.Holders) > 0 {
+		// replies of a holder's connection carry its mark: the policy rule routes them back to it
+		fmt.Fprintf(&mangle, "-A %s -m conntrack --ctdir REPLY -m connmark ! --mark 0x0/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n",
+			MarkChain, markMask, markMask, markMask)
+		for _, h := range d.Holders {
+			// a new connection to a pod here, arriving from the holder's bridge, from outside the pod network:
+			// the holder DNATed it and must see its replies
+			fmt.Fprintf(&mangle, "-A %s -i %s -m mac --mac-source %s ! -s %s -d %s -m conntrack --ctstate NEW -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n",
+				MarkChain, bridge, h.Node.WeaveMAC, podNet, podNet, "holder "+h.Node.Name, markValue(h.Mark), markMask)
+		}
+	}
+
+	var filter bytes.Buffer
+	filter.WriteString("*filter\n")
+	for _, c := range []string{FwdChain, InChain} {
+		fmt.Fprintf(&filter, ":%s - [0:0]\n", c)
+	}
+	for _, a := range d.Accept {
+		// served by the agent itself (a Gateway's HTTP or HTTPS listener): in, whatever the host firewall says
+		fmt.Fprintf(&filter, "-A %s -d %s/32 -p tcp -m tcp --dport %d -m comment --comment %q -j ACCEPT\n", InChain, a.Address, a.Port, a.Comment)
+	}
+	renderEgress(e, self, podNet, &body, &mangle, &filter)
+
+	nat.Write(body.Bytes())
+	nat.WriteString("COMMIT\n")
+	mangle.WriteString("COMMIT\n")
+	filter.WriteString("COMMIT\n")
+	return Rules{Nat: nat.String(), Mangle: mangle.String(), Filter: filter.String()}
+}
+
+// Datapath applies Desired to this node.
+type Datapath struct {
+	ipt *iptables.IPTables
+	// restore runs iptables-restore --noflush with the given input (swappable in tests)
+	restore func(input string) error
+}
+
+// NewDatapath uses the host's iptables (whichever backend the image's launch script selected).
+func NewDatapath(ipt *iptables.IPTables) *Datapath {
+	return &Datapath{ipt: ipt, restore: func(input string) error {
+		cmd := exec.Command("iptables-restore", "--noflush", "-w", "5")
+		cmd.Stdin = strings.NewReader(input)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return fmt.Errorf("iptables-restore: %v: %s", err, out)
+		}
+		return nil
+	}}
+}
+
+// Apply makes the node's iptables chains, tunnel, policy rules and routes exactly what d and e say.
+func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local) error {
+	podNet := local.PodNet
+	if err := dp.hooks(); err != nil {
+		return err
+	}
+	// the tunnels must exist before the routes into them (iptables rules may name a device that does not exist yet)
+	var peers []Node
+	if e.Tunnelled(self) {
+		peers = e.Peers(self)
+	}
+	if err := ensureTunnels(peers, local); err != nil {
+		return err
+	}
+	existing, err := dp.ipt.ListChains("nat")
+	if err != nil {
+		return err
+	}
+	want := map[string]bool{}
+	for _, f := range d.Forwards {
+		for _, p := range f.Ports {
+			want[portChain(f, p)] = true
+		}
+	}
+	var stale []string
+	for _, c := range existing {
+		if strings.HasPrefix(c, servicePrefix) && !want[c] {
+			stale = append(stale, c)
+		}
+	}
+	r := Render(d, e, self, podNet, stale)
+	// filter first: the guards are in place before the marks and SNATs that they guard
+	for _, payload := range []string{r.Filter, r.Nat, r.Mangle} {
+		if err := dp.restore(payload); err != nil {
+			return err
+		}
+	}
+	for _, c := range stale {
+		if err := dp.ipt.DeleteChain("nat", c); err != nil {
+			return fmt.Errorf("deleting stale chain %s: %v", c, err)
+		}
+	}
+	if err := dp.routes(d.Holders); err != nil {
+		return err
+	}
+	return egressNet(e, self)
+}
+
+// hooks makes sure the built-in chains jump to ours: first (kube-proxy's and weave's rules must not see these
+// connections first), except ClassChain, which must come after kube-proxy's KUBE-SERVICES (whose DNAT ends the
+// nat walk, so a connection to a Service is never classed as going outside).
+func (dp *Datapath) hooks() error {
+	for _, h := range []struct {
+		table, from, chain string
+		last               bool
+	}{
+		{"nat", "PREROUTING", ForwardChain, false},
+		{"nat", "POSTROUTING", PostChain, false},
+		{"nat", "PREROUTING", ClassChain, true},
+		{"mangle", "PREROUTING", MarkChain, false},
+		{"mangle", "FORWARD", MSSChain, false},
+		{"filter", "FORWARD", FwdChain, false},
+		{"filter", "INPUT", InChain, false},
+	} {
+		ok, err := dp.ipt.ChainExists(h.table, h.chain)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			if err := dp.ipt.NewChain(h.table, h.chain); err != nil {
+				return err
+			}
+		}
+		rule := []string{"-m", "comment", "--comment", "reweave lb", "-j", h.chain}
+		rules, err := dp.ipt.List(h.table, h.from)
+		if err != nil {
+			return err
+		}
+		if h.last {
+			ours, kube := -1, -1
+			for i, r := range rules {
+				if strings.HasSuffix(r, "-j "+h.chain) {
+					ours = i
+				}
+				if strings.HasSuffix(r, "-j KUBE-SERVICES") {
+					kube = i
+				}
+			}
+			if ours > kube {
+				continue
+			}
+			if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
+				return err
+			}
+			if err := dp.ipt.Append(h.table, h.from, rule...); err != nil {
+				return err
+			}
+			continue
+		}
+		// rules[0] is the policy line; our jump must be rules[1]
+		if len(rules) > 1 && strings.HasSuffix(rules[1], "-j "+h.chain) {
+			continue
+		}
+		if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
+			return err
+		}
+		if err := dp.ipt.Insert(h.table, h.from, 1, rule...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// routes keeps one policy rule and one table per holder: marked replies go to the holder's bridge.
+func (dp *Datapath) routes(holders []Holder) error {
+	link, err := netlink.LinkByName(bridge)
+	if err != nil {
+		if len(holders) == 0 {
+			return nil
+		}
+		return fmt.Errorf("weave bridge: %v", err)
+	}
+	want := map[int]Holder{}
+	for _, h := range holders {
+		want[TableBase+int(h.Mark)] = h
+	}
+	rules, err := netlink.RuleList(unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	have := map[int]bool{}
+	for _, r := range rules {
+		if r.Priority != RulePriority || r.Table < TableBase || r.Table > TableBase+MaxMarks {
+			continue
+		}
+		h, ok := want[r.Table]
+		if ok && uint32(r.Mark) == markValue(h.Mark) && uint32(r.Mask) == markMask {
+			have[r.Table] = true
+			continue
+		}
+		r := r
+		if err := netlink.RuleDel(&r); err != nil {
+			return fmt.Errorf("removing stale rule to table %d: %v", r.Table, err)
+		}
+	}
+	existing, err := routesByTable(TableBase+1, TableBase+MaxMarks)
+	if err != nil {
+		return err
+	}
+	for table := TableBase + 1; table <= TableBase+MaxMarks; table++ {
+		h, ok := want[table]
+		if !ok {
+			for _, rt := range existing[table] {
+				if err := delRoute(rt); err != nil {
+					return fmt.Errorf("removing stale route in table %d: %v", table, err)
+				}
+			}
+			continue
+		}
+		if err := netlink.RouteReplace(&netlink.Route{
+			LinkIndex: link.Attrs().Index,
+			Dst:       &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
+			Gw:        h.Node.WeaveIP,
+			Table:     table,
+			Flags:     int(netlink.FLAG_ONLINK),
+		}); err != nil {
+			return fmt.Errorf("route to holder %s: %v", h.Node.Name, err)
+		}
+		if !have[table] {
+			rule := netlink.NewRule()
+			rule.Priority = RulePriority
+			rule.Table = table
+			rule.Mark = int(markValue(h.Mark))
+			rule.Mask = int(markMask)
+			if err := netlink.RuleAdd(rule); err != nil {
+				return fmt.Errorf("rule for holder %s: %v", h.Node.Name, err)
+			}
+		}
+	}
+	return nil
+}
+
+// delRoute deletes a route as RouteList returned it: a default route lists without Dst, which RouteDel refuses.
+func delRoute(rt netlink.Route) error {
+	if rt.Dst == nil {
+		rt.Dst = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}
+	}
+	return netlink.RouteDel(&rt)
+}

lb/egress.go

@@ -0,0 +1,340 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+
+	coreapi "k8s.io/api/core/v1"
+)
+
+// Port blocks: every node gets its own source ports on an EgressService's address, so the holder can route a
+// reply back to the member's node by port alone (no state on the holder). The range lies above the default
+// ephemeral range (32768-60999), so it never meets the holder's own connections from the address.
+const (
+	BlockBase  = 61000
+	BlockSize  = 512
+	BlockCount = 8 // 61000-65095
+)
+
+// Admission is a pod the CNI plugin announced at ADD, before the API shows its IP.
+type Admission struct {
+	Namespace, Name string
+}
+
+// Member is one pod on this node that an EgressService selects, and where its outside traffic goes.
+type Member struct {
+	IP      net.IP
+	Pod     string // namespace/name
+	Service string // namespace/name of its EgressService (empty if it is in several)
+	// Unreachable: its outside traffic is dropped (no holder, no block, several EgressServices, ...)
+	Unreachable bool
+	Address     net.IP // it leaves from Address:[From, To]
+	From, To    int32
+	Holder      Node // the node that emits it; Holder.Name == self: emitted here
+}
+
+// BlockRoute is, on the holder, the replies to Address:[From, To] going back to Peer.
+type BlockRoute struct {
+	Address  net.IP
+	From, To int32
+	Peer     Node
+}
+
+// EgressStatusWrite is a status this node writes (the holder, or the first node by name to clear it).
+type EgressStatusWrite struct {
+	Namespace, Name string
+	Status          EgressServiceStatus
+}
+
+// EgressDesired is this node's part of every EgressService.
+type EgressDesired struct {
+	Members   []Member
+	Routes    []BlockRoute
+	Status    []EgressStatusWrite
+	NodeAddrs []net.IP // the cluster's node addresses: members' traffic to them is not "outside"
+	Problems  []string
+}
+
+// Tunnelled reports whether this node sends or receives anything over the egress tunnel.
+func (e EgressDesired) Tunnelled(self string) bool {
+	if len(e.Routes) > 0 {
+		return true
+	}
+	for _, m := range e.Members {
+		if !m.Unreachable && m.Holder.Name != self {
+			return true
+		}
+	}
+	return false
+}
+
+// Peers are the nodes this node exchanges tunnelled packets with (sorted by name, deduplicated).
+func (e EgressDesired) Peers(self string) []Node {
+	seen := map[string]Node{}
+	for _, r := range e.Routes {
+		seen[r.Peer.Name] = r.Peer
+	}
+	for _, m := range e.Members {
+		if !m.Unreachable && m.Holder.Name != self {
+			seen[m.Holder.Name] = m.Holder
+		}
+	}
+	out := make([]Node, 0, len(seen))
+	for _, n := range seen {
+		out = append(out, n)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// AllocateBlocks keeps every valid block a node already has and gives each node without one the lowest free
+// block, in name order. A node beyond BlockCount gets none (its members are dropped, and that is reported).
+func AllocateBlocks(current []PortBlock, nodeNames []string) []PortBlock {
+	present := map[string]bool{}
+	for _, n := range nodeNames {
+		present[n] = true
+	}
+	used := map[int]bool{}
+	have := map[string]PortBlock{}
+	for _, b := range current {
+		k := int(b.From-BlockBase) / BlockSize
+		valid := present[b.Node] && b.From >= BlockBase && int(b.From-BlockBase)%BlockSize == 0 &&
+			k < BlockCount && b.To == b.From+BlockSize-1
+		if !valid || used[k] {
+			continue
+		}
+		if _, dup := have[b.Node]; dup {
+			continue
+		}
+		used[k] = true
+		have[b.Node] = b
+	}
+	names := append([]string(nil), nodeNames...)
+	sort.Strings(names)
+	for _, n := range names {
+		if _, ok := have[n]; ok {
+			continue
+		}
+		for k := 0; k < BlockCount; k++ {
+			if !used[k] {
+				used[k] = true
+				from := int32(BlockBase + k*BlockSize)
+				have[n] = PortBlock{Node: n, From: from, To: from + BlockSize - 1}
+				break
+			}
+		}
+	}
+	out := make([]PortBlock, 0, len(have))
+	for _, b := range have {
+		out = append(out, b)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
+	return out
+}
+
+func sameStatus(a, b EgressServiceStatus) bool {
+	if a.Host != b.Host || a.Address != b.Address || len(a.Blocks) != len(b.Blocks) {
+		return false
+	}
+	for i := range a.Blocks {
+		if a.Blocks[i] != b.Blocks[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// holderOf is S1's rule for a Service's address: the one node that holds spec.loadBalancerIP.
+func holderOf(s *coreapi.Service, nodes []Node) (*Node, net.IP) {
+	want := net.ParseIP(s.Spec.LoadBalancerIP).To4()
+	if want == nil {
+		return nil, nil
+	}
+	var hs []Node
+	for _, n := range nodes {
+		if n.Holds(want) {
+			hs = append(hs, n)
+		}
+	}
+	if len(hs) != 1 {
+		return nil, want
+	}
+	return &hs[0], want
+}
+
+// selects reports whether a Service's selector (non-empty) matches the labels.
+func selects(s *coreapi.Service, labels map[string]string) bool {
+	if len(s.Spec.Selector) == 0 {
+		return false
+	}
+	for k, v := range s.Spec.Selector {
+		if labels[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+// ComputeEgress decides this node's part of every EgressService: the statuses it writes, the members it hosts and
+// where their traffic goes, and (as a holder) where replies to each node's port block go.
+// pods are this node's pods; admitted maps an IP to the pod the CNI plugin gave it (newer than the API may show).
+func ComputeEgress(self string, nodes []Node, services []*coreapi.Service, egresses []*EgressService,
+	pods []*coreapi.Pod, admitted map[string]Admission) EgressDesired {
+	var d EgressDesired
+	sorted := append([]Node(nil), nodes...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
+	byName := map[string]Node{}
+	var names []string
+	for _, n := range sorted {
+		byName[n.Name] = n
+		names = append(names, n.Name)
+		d.NodeAddrs = append(d.NodeAddrs, n.Addresses...)
+	}
+	svcs := map[string]*coreapi.Service{}
+	for _, s := range services {
+		svcs[s.Namespace+"/"+s.Name] = s
+	}
+	es := append([]*EgressService(nil), egresses...)
+	sort.Slice(es, func(i, j int) bool { return es[i].Namespace+"/"+es[i].Name < es[j].Namespace+"/"+es[j].Name })
+
+	// valid[key]: the EgressService is in force: its Service is served from an address one node holds, and the
+	// holder has published that (status) - what members and holder both act on
+	valid := map[string]bool{}
+	for _, e := range es {
+		key := e.Namespace + "/" + e.Name
+		var want EgressServiceStatus
+		var holder *Node
+		var addr net.IP
+		s := svcs[key]
+		switch {
+		case e.Spec.Emit != EmitHolder:
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: spec.emit must be %q (the only mode so far), got %q", key, EmitHolder, e.Spec.Emit))
+		case e.Spec.SourceIPBy != "" && e.Spec.SourceIPBy != SourceIPByLoadBalancerIP:
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: spec.sourceIPBy must be %q, got %q", key, SourceIPByLoadBalancerIP, e.Spec.SourceIPBy))
+		case s == nil:
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: no Service %s", key, key))
+		case !eligible(s):
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: Service %s is not a LoadBalancer of class %s", key, key, Class))
+		default:
+			holder, addr = holderOf(s, sorted)
+			if holder == nil {
+				d.Problems = append(d.Problems, fmt.Sprintf("egress %s: no single node holds %s; members' outside traffic is dropped", key, s.Spec.LoadBalancerIP))
+			}
+		}
+		writer := ""
+		if holder != nil {
+			want = EgressServiceStatus{Host: holder.Name, Address: addr.String(), Blocks: AllocateBlocks(e.Status.Blocks, names)}
+			writer = holder.Name
+			if len(want.Blocks) < len(names) {
+				d.Problems = append(d.Problems, fmt.Sprintf("egress %s: only %d port blocks for %d nodes; members on nodes without one are dropped", key, BlockCount, len(names)))
+			}
+		} else if len(names) > 0 {
+			writer = names[0]
+		}
+		if writer == self && !sameStatus(e.Status, want) {
+			d.Status = append(d.Status, EgressStatusWrite{Namespace: e.Namespace, Name: e.Name, Status: want})
+		}
+		// in force only once the status in the API says what holds now (members and holder read the same thing)
+		if holder != nil && e.Status.Host == holder.Name && e.Status.Address == addr.String() {
+			valid[key] = true
+			if holder.Name == self {
+				for _, b := range e.Status.Blocks {
+					if b.Node == self {
+						continue
+					}
+					peer, ok := byName[b.Node]
+					if !ok || peer.WeaveIP == nil {
+						d.Problems = append(d.Problems, fmt.Sprintf("egress %s: node %s has no weave bridge published; its members' replies cannot reach it", key, b.Node))
+						continue
+					}
+					d.Routes = append(d.Routes, BlockRoute{Address: addr, From: b.From, To: b.To, Peer: peer})
+				}
+			}
+		}
+	}
+
+	// this node's pods, one per IP: the CNI plugin's latest word wins, then live pods over terminating ones
+	type candidate struct {
+		pod *coreapi.Pod
+		ip  string
+	}
+	byIP := map[string]*coreapi.Pod{}
+	podByKey := map[string]*coreapi.Pod{}
+	for _, p := range pods {
+		podByKey[p.Namespace+"/"+p.Name] = p
+	}
+	var cands []candidate
+	for _, p := range pods {
+		if p.Spec.NodeName != self || p.Spec.HostNetwork || p.Status.Phase == coreapi.PodSucceeded || p.Status.Phase == coreapi.PodFailed {
+			continue
+		}
+		if ip := net.ParseIP(p.Status.PodIP).To4(); ip != nil {
+			cands = append(cands, candidate{p, ip.String()})
+		}
+	}
+	for ip, a := range admitted {
+		if p := podByKey[a.Namespace+"/"+a.Name]; p != nil && p.Spec.NodeName == self && !p.Spec.HostNetwork {
+			cands = append(cands, candidate{p, ip})
+		}
+	}
+	for _, c := range cands {
+		if a, ok := admitted[c.ip]; ok && (a.Namespace != c.pod.Namespace || a.Name != c.pod.Name) {
+			continue // the IP was given to another pod since
+		}
+		cur := byIP[c.ip]
+		if cur == nil || (cur.DeletionTimestamp != nil && c.pod.DeletionTimestamp == nil) ||
+			(cur.DeletionTimestamp == nil) == (c.pod.DeletionTimestamp == nil) && c.pod.CreationTimestamp.After(cur.CreationTimestamp.Time) {
+			byIP[c.ip] = c.pod
+		}
+	}
+	ips := make([]string, 0, len(byIP))
+	for ip := range byIP {
+		ips = append(ips, ip)
+	}
+	sort.Strings(ips)
+	for _, ip := range ips {
+		p := byIP[ip]
+		var in []*EgressService
+		for _, e := range es {
+			if e.Namespace != p.Namespace {
+				continue
+			}
+			if s := svcs[e.Namespace+"/"+e.Name]; s != nil && selects(s, p.Labels) {
+				in = append(in, e)
+			}
+		}
+		if len(in) == 0 {
+			continue
+		}
+		m := Member{IP: net.ParseIP(ip).To4(), Pod: p.Namespace + "/" + p.Name, Unreachable: true}
+		if len(in) > 1 {
+			keys := make([]string, len(in))
+			for i, e := range in {
+				keys[i] = e.Namespace + "/" + e.Name
+			}
+			d.Problems = append(d.Problems, fmt.Sprintf("pod %s is selected by several EgressServices (%s); its outside traffic is dropped", m.Pod, strings.Join(keys, ", ")))
+			d.Members = append(d.Members, m)
+			continue
+		}
+		e := in[0]
+		m.Service = e.Namespace + "/" + e.Name
+		if valid[m.Service] {
+			holder := byName[e.Status.Host]
+			for _, b := range e.Status.Blocks {
+				if b.Node == self && (holder.Name == self || holder.WeaveIP != nil) {
+					m.Unreachable = false
+					m.Address = net.ParseIP(e.Status.Address).To4()
+					m.From, m.To = b.From, b.To
+					m.Holder = holder
+				}
+			}
+			if m.Unreachable {
+				d.Problems = append(d.Problems, fmt.Sprintf("pod %s: egress %s has no port block for %s (or its holder's bridge is unknown); its outside traffic is dropped", m.Pod, m.Service, self))
+			}
+		}
+		d.Members = append(d.Members, m)
+	}
+	return d
+}

lb/egress_api.go

@@ -0,0 +1,144 @@
+package lb
+
+import (
+	"context"
+	"encoding/json"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+)
+
+// EgressService (reweave.levk.dev/v1alpha1) makes the pods a LoadBalancer Service selects leave the cluster
+// from the Service's address. It is named after the Service (same namespace, same name). Its semantics follow
+// OVN-Kubernetes' EgressService (k8s.ovn.org/v1, sourceIPBy: LoadBalancerIP) where they agree; they differ in
+// that membership is every pod the Service's selector matches (by pod IP, never by readiness), the holder is
+// the node that holds the address (no election), and a member's traffic to the outside leaves as the address
+// or not at all.
+type EgressService struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              EgressServiceSpec   `json:"spec"`
+	Status            EgressServiceStatus `json:"status,omitempty"`
+}
+
+// EgressServiceSpec is what the operator asks for.
+type EgressServiceSpec struct {
+	// SourceIPBy is LoadBalancerIP: members leave from the Service's load-balancer address (the only mode in v1).
+	SourceIPBy string `json:"sourceIPBy,omitempty"`
+	// Emit is where members' packets leave the cluster. Explicit, no default: Holder (tunnelled to the node
+	// holding the address, which emits them) is the only mode in v1.
+	Emit string `json:"emit"`
+}
+
+// EgressServiceStatus is written by the holder alone.
+type EgressServiceStatus struct {
+	// Host is the node that holds the address and emits members' packets.
+	Host string `json:"host,omitempty"`
+	// Address is the address members leave from.
+	Address string `json:"address,omitempty"`
+	// Blocks gives every node its own source ports on Address, so the holder routes replies back by port.
+	Blocks []PortBlock `json:"blocks,omitempty"`
+}
+
+// PortBlock is the source-port range [From, To] on the address that one node's members use.
+type PortBlock struct {
+	Node string `json:"node"`
+	From int32  `json:"from"`
+	To   int32  `json:"to"`
+}
+
+// EgressServiceList is a list of EgressServices.
+type EgressServiceList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []EgressService `json:"items"`
+}
+
+const (
+	// Group and Version of the EgressService API.
+	Group   = "reweave.levk.dev"
+	Version = "v1alpha1"
+	// EgressServices is the resource name.
+	EgressServices = "egressservices"
+
+	SourceIPByLoadBalancerIP = "LoadBalancerIP"
+	EmitHolder               = "Holder"
+)
+
+// GroupVersion of the EgressService API.
+var GroupVersion = schema.GroupVersion{Group: Group, Version: Version}
+
+// DeepCopyObject implements runtime.Object.
+func (e *EgressService) DeepCopyObject() runtime.Object { return e.DeepCopy() }
+
+// DeepCopy copies the whole object.
+func (e *EgressService) DeepCopy() *EgressService {
+	if e == nil {
+		return nil
+	}
+	out := *e
+	e.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Status.Blocks = append([]PortBlock(nil), e.Status.Blocks...)
+	return &out
+}
+
+// DeepCopyObject implements runtime.Object.
+func (l *EgressServiceList) DeepCopyObject() runtime.Object {
+	if l == nil {
+		return nil
+	}
+	out := *l
+	l.ListMeta.DeepCopyInto(&out.ListMeta)
+	out.Items = make([]EgressService, len(l.Items))
+	for i := range l.Items {
+		out.Items[i] = *l.Items[i].DeepCopy()
+	}
+	return &out
+}
+
+func addKnownTypes(s *runtime.Scheme) error {
+	s.AddKnownTypes(GroupVersion, &EgressService{}, &EgressServiceList{})
+	metav1.AddToGroupVersion(s, GroupVersion)
+	return nil
+}
+
+// NewEgressClient is a REST client for the EgressService API (no generated clientset: two plain types).
+func NewEgressClient(base *rest.Config) (*rest.RESTClient, error) {
+	scheme := runtime.NewScheme()
+	if err := addKnownTypes(scheme); err != nil {
+		return nil, err
+	}
+	cfg := *base
+	cfg.GroupVersion = &GroupVersion
+	cfg.APIPath = "/apis"
+	cfg.ContentType = runtime.ContentTypeJSON
+	cfg.NegotiatedSerializer = serializer.NewCodecFactory(scheme).WithoutConversion()
+	if cfg.UserAgent == "" {
+		cfg.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+	return rest.RESTClientFor(&cfg)
+}
+
+// PatchEgressStatus merge-patches an EgressService's status (nil fields are removed).
+func PatchEgressStatus(ctx context.Context, c rest.Interface, namespace, name string, st EgressServiceStatus) error {
+	body := map[string]interface{}{"host": nil, "address": nil, "blocks": nil}
+	if st.Host != "" {
+		body["host"] = st.Host
+	}
+	if st.Address != "" {
+		body["address"] = st.Address
+	}
+	if len(st.Blocks) > 0 {
+		body["blocks"] = st.Blocks
+	}
+	patch, err := json.Marshal(map[string]interface{}{"status": body})
+	if err != nil {
+		return err
+	}
+	return c.Patch(types.MergePatchType).Namespace(namespace).Resource(EgressServices).Name(name).
+		SubResource("status").Body(patch).Do(ctx).Error()
+}

lb/egress_datapath.go

@@ -0,0 +1,530 @@
+package lb
+
+import (
+	"bytes"
+	"fmt"
+	"hash/fnv"
+	"net"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// ClassChain (nat, appended to PREROUTING after kube-proxy's KUBE-SERVICES) marks members' new connections to
+	// the outside. kube-proxy's DNAT ends the nat walk, so a connection to a Service never gets here.
+	ClassChain = "REWEAVE-LB-CLASS"
+	// FwdChain (filter, first in FORWARD) drops members' outside packets that would not leave as their address.
+	FwdChain = "REWEAVE-LB-FWD"
+	// InChain (filter, first in INPUT) accepts the egress tunnels only from the cluster's nodes.
+	InChain = "REWEAVE-LB-IN"
+	// MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnels.
+	MSSChain = "REWEAVE-LB-MSS"
+
+	// TunnelPrefix names the egress tunnels: one ipip device per peer node, between the two nodes' weave bridges,
+	// so a holder knows which node a member's packet came from by the device it arrives on.
+	TunnelPrefix = "rweg-"
+	// tunnelGlob matches every egress tunnel in an iptables rule.
+	tunnelGlob = "rweg+"
+	// legacyTunnel is the single shared tunnel of earlier builds (removed when found).
+	legacyTunnel = "reweave-eg"
+
+	// EgressMark marks a member's packets to the outside (connmark on its connections, fwmark on each packet).
+	EgressMark = uint32(0x100000)
+	// peerFlag, with a peer's index in the top byte, tags (on the holder) a connection that arrived from that peer's
+	// tunnel: replies and related ICMP errors carry it back there.
+	peerFlag = uint32(0x200000)
+	peerMask = markMask | peerFlag
+
+	// HolderRulePriority: replies to a block go to its node; ahead of the local table, which moves to LocalRulePriority,
+	// since the address is usually local on the holder.
+	HolderRulePriority = 90
+	LocalRulePriority  = 100
+	// MemberRulePriority: a member's marked packets go to its holder (or nowhere).
+	MemberRulePriority = 9100
+	// EgressTableBase + peer index routes into that peer's tunnel; EgressTableBase itself is unreachable.
+	EgressTableBase = 53000
+
+	ipipOverhead = 20
+)
+
+// portProtocols carry ports: their replies are routed by port block alone (stateless), and members' SNAT keeps
+// within the node's block. Everything else is routed back by the tag the holder put on the connection.
+var portProtocols = []struct {
+	name string
+	num  int
+}{{"tcp", unix.IPPROTO_TCP}, {"udp", unix.IPPROTO_UDP}, {"sctp", unix.IPPROTO_SCTP}}
+
+// TunnelName is the egress tunnel towards a peer node (stable, within the 15 characters a device name may have).
+func TunnelName(peer string) string {
+	h := fnv.New32a()
+	h.Write([]byte(peer))
+	return fmt.Sprintf("%s%08x", TunnelPrefix, h.Sum32())
+}
+
+func peerTag(idx uint32) uint32 { return markValue(idx) | peerFlag }
+
+// renderEgress appends the egress rules of each table to the bodies of S1's payloads.
+func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, filter *bytes.Buffer) {
+	members := e.Members
+	if len(members) > 0 {
+		// what is not "outside": the pod network, multicast, the cluster's nodes
+		fmt.Fprintf(nat, "-A %s -d %s -j RETURN\n", ClassChain, podNet)
+		fmt.Fprintf(nat, "-A %s -d 224.0.0.0/4 -j RETURN\n", ClassChain)
+		seen := map[string]bool{}
+		for _, a := range e.NodeAddrs {
+			if !seen[a.String()] {
+				seen[a.String()] = true
+				fmt.Fprintf(nat, "-A %s -d %s/32 -j RETURN\n", ClassChain, a)
+			}
+		}
+		for _, m := range members {
+			fmt.Fprintf(nat, "-A %s -s %s/32 -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n", ClassChain, m.IP, m.Pod, EgressMark, EgressMark)
+			fmt.Fprintf(nat, "-A %s -s %s/32 -j MARK --set-xmark 0x%x/0x%x\n", ClassChain, m.IP, EgressMark, EgressMark)
+		}
+		// later packets of a member's outside connection carry the mark too (the nat table saw only the first)
+		fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir ORIGINAL -m connmark --mark 0x%x/0x%x -j MARK --set-xmark 0x%x/0x%x\n",
+			MarkChain, EgressMark, EgressMark, EgressMark, EgressMark)
+	}
+	for _, m := range members {
+		if m.Unreachable {
+			fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
+			continue
+		}
+		out := "-o " + TunnelName(m.Holder.Name)
+		if m.Holder.Name == self {
+			// emitted here, from the address: never into the pod network unSNATed
+			out = "! -o " + bridge
+			fmt.Fprintf(filter, "-A %s -s %s/32 -o %s -m mark --mark 0x%x/0x%x -j DROP\n", FwdChain, m.IP, bridge, EgressMark, EgressMark)
+		}
+		// within this node's block for what has ports; the address alone for the rest (the holder tells those
+		// apart by the tunnel they arrive on)
+		for _, p := range portProtocols {
+			fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n",
+				PostChain, m.IP, out, EgressMark, EgressMark, p.name, m.Address, m.From, m.To)
+		}
+		// ICMP queries (ping) too: NAT treats the echo identifier as a port, and without a block of their own two
+		// nodes' pods (whose small, alike PIDs make alike identifiers) collide on the holder
+		fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p icmp -j SNAT --to-source %s:%d-%d\n",
+			PostChain, m.IP, out, EgressMark, EgressMark, m.Address, m.From, m.To)
+		fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j SNAT --to-source %s\n",
+			PostChain, m.IP, out, EgressMark, EgressMark, m.Address)
+		fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark)
+		// a member's outside traffic leaves no other way
+		fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
+	}
+	if !e.Tunnelled(self) {
+		return
+	}
+	tags := peerIndex(e, self)
+
+	// as holder: tag each connection arriving from a peer's tunnel with that peer, once; replies and related ICMP
+	// errors restore the tag, and the policy rule for it sends them back into that tunnel
+	byPeer := map[string][]BlockRoute{}
+	var peerNames []string
+	for _, r := range e.Routes {
+		if _, ok := byPeer[r.Peer.Name]; !ok {
+			peerNames = append(peerNames, r.Peer.Name)
+		}
+		byPeer[r.Peer.Name] = append(byPeer[r.Peer.Name], r)
+	}
+	sort.Strings(peerNames)
+	if len(peerNames) > 0 {
+		fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir REPLY -m connmark --mark 0x%x/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n",
+			MarkChain, peerFlag, peerFlag, peerMask, peerMask)
+	}
+	for _, name := range peerNames {
+		dev, tag := TunnelName(name), peerTag(tags[name])
+		fmt.Fprintf(mangle, "-A %s -i %s -m conntrack --ctstate NEW -m connmark --mark 0x0/0x%x -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n",
+			MarkChain, dev, peerMask, "from "+name, tag, peerMask)
+		// a connection another node already holds (a port-less protocol to the same remote): never cross-delivered
+		fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -m connmark ! --mark 0x%x/0x%x -j DROP\n", FwdChain, dev, tag, peerMask)
+		addrs := map[string]bool{}
+		for _, r := range byPeer[name] {
+			for _, p := range portProtocols {
+				fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n",
+					FwdChain, dev, r.Address, p.name, p.name, r.From, r.To, "block of "+name)
+			}
+			addrs[r.Address.String()] = true
+		}
+		for _, a := range sortedSet(addrs) {
+			// outside its own block a node may not use the ports of others
+			for _, p := range portProtocols {
+				fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -j DROP\n", FwdChain, dev, a, p.name)
+			}
+			fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -j RETURN\n", FwdChain, dev, a)
+		}
+	}
+	// as a member's node: replies (and related errors) from the holder; nothing else comes back
+	addrs := map[string]bool{}
+	for _, m := range members {
+		if !m.Unreachable && m.Holder.Name != self {
+			addrs[m.Address.String()] = true
+		}
+	}
+	for _, a := range sortedSet(addrs) {
+		// still addressed to the address after nat: no member connection matched it
+		fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, tunnelGlob, a)
+	}
+	fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, tunnelGlob)
+	fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, tunnelGlob)
+	// the tunnels themselves only from the cluster's nodes' bridges
+	for _, p := range e.Peers(self) {
+		fmt.Fprintf(filter, "-A %s -p 4 -s %s/32 -m mac --mac-source %s -j RETURN\n", InChain, p.WeaveIP, p.WeaveMAC)
+	}
+	fmt.Fprintf(filter, "-A %s -p 4 -j DROP\n", InChain)
+	fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, tunnelGlob)
+}
+
+func sortedSet(m map[string]bool) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// peerIndex gives each tunnel peer a stable index (1..MaxMarks): its tag and its routing table.
+func peerIndex(e EgressDesired, self string) map[string]uint32 {
+	out := map[string]uint32{}
+	for _, h := range assignMarks(e.Peers(self)) {
+		out[h.Node.Name] = h.Mark
+	}
+	return out
+}
+
+// peerTables gives each tunnel peer its routing table (EgressTableBase + its index).
+func peerTables(e EgressDesired, self string) map[string]int {
+	out := map[string]int{}
+	for name, idx := range peerIndex(e, self) {
+		out[name] = EgressTableBase + int(idx)
+	}
+	return out
+}
+
+// egressRule is a policy rule this package owns, in a comparable form.
+type egressRule struct {
+	priority   int
+	src, dst   string
+	ipproto    int
+	from, to   uint16
+	mark, mask uint32
+	table      int
+}
+
+func (r egressRule) key() string {
+	return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x/%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.mask, r.table)
+}
+
+// wantRules is every egress policy rule this node needs.
+func wantRules(e EgressDesired, self string) []egressRule {
+	tables := peerTables(e, self)
+	tags := peerIndex(e, self)
+	var out []egressRule
+	tagged := map[string]bool{}
+	for _, r := range e.Routes {
+		for _, p := range portProtocols {
+			out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: p.num,
+				from: uint16(r.From), to: uint16(r.To), table: tables[r.Peer.Name]})
+		}
+		if !tagged[r.Peer.Name] {
+			// whatever the tag says came from that peer (replies without ports, ICMP errors) goes back to it
+			tagged[r.Peer.Name] = true
+			out = append(out, egressRule{priority: HolderRulePriority, mark: peerTag(tags[r.Peer.Name]), mask: peerMask, table: tables[r.Peer.Name]})
+		}
+	}
+	for _, m := range e.Members {
+		if !m.Unreachable && m.Holder.Name == self {
+			continue // emitted here: the main table already sends it out
+		}
+		table := EgressTableBase
+		if !m.Unreachable {
+			table = tables[m.Holder.Name]
+		}
+		out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, mask: EgressMark, table: table})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].key() < out[j].key() })
+	return out
+}
+
+func ruleOf(r netlink.Rule) egressRule {
+	er := egressRule{priority: r.Priority, ipproto: r.IPProto, table: r.Table}
+	if r.Src != nil {
+		er.src = r.Src.String()
+	}
+	if r.Dst != nil {
+		er.dst = r.Dst.String()
+	}
+	if r.Dport != nil {
+		er.from, er.to = r.Dport.Start, r.Dport.End
+	}
+	if r.Mark > 0 {
+		er.mark = uint32(r.Mark)
+		er.mask = uint32(r.Mask)
+	}
+	if er.ipproto < 0 {
+		er.ipproto = 0
+	}
+	return er
+}
+
+// routesByTable lists, in one dump, the IPv4 routes of tables lo..hi.
+func routesByTable(lo, hi int) (map[int][]netlink.Route, error) {
+	all, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, netlink.RT_FILTER_TABLE)
+	if err != nil {
+		return nil, err
+	}
+	out := map[int][]netlink.Route{}
+	for _, rt := range all {
+		if rt.Table >= lo && rt.Table <= hi {
+			out[rt.Table] = append(out[rt.Table], rt)
+		}
+	}
+	return out, nil
+}
+
+// egressNet makes the local rule's place, the policy rules and the tunnel tables exactly what e needs (the
+// tunnels themselves are made first, by Apply).
+func egressNet(e EgressDesired, self string) error {
+	if len(e.Routes) > 0 {
+		if err := moveLocalRule(); err != nil {
+			return err
+		}
+	}
+	tables := peerTables(e, self)
+	peerOf := map[int]Node{}
+	for _, p := range e.Peers(self) {
+		peerOf[tables[p.Name]] = p
+	}
+	unreachable := false
+	for _, m := range e.Members {
+		unreachable = unreachable || m.Unreachable
+	}
+	existing, err := routesByTable(EgressTableBase, EgressTableBase+MaxMarks)
+	if err != nil {
+		return err
+	}
+	for table := EgressTableBase; table <= EgressTableBase+MaxMarks; table++ {
+		var want *netlink.Route
+		if p, ok := peerOf[table]; ok {
+			link, err := netlink.LinkByName(TunnelName(p.Name))
+			if err != nil {
+				return fmt.Errorf("egress tunnel to %s: %v", p.Name, err)
+			}
+			want = &netlink.Route{LinkIndex: link.Attrs().Index, Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
+				Scope: netlink.SCOPE_LINK, Table: table}
+		} else if table == EgressTableBase && unreachable {
+			want = &netlink.Route{Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, Table: table, Type: unix.RTN_UNREACHABLE}
+		}
+		keep := false
+		for _, rt := range existing[table] {
+			if want != nil && rt.Type == want.Type &&
+				(want.Type == unix.RTN_UNREACHABLE || rt.LinkIndex == want.LinkIndex && rt.Gw == nil) {
+				keep = true
+				continue
+			}
+			if err := delRoute(rt); err != nil {
+				return fmt.Errorf("removing stale route in table %d: %v", table, err)
+			}
+		}
+		if want != nil && !keep {
+			if err := netlink.RouteReplace(want); err != nil {
+				return fmt.Errorf("route in table %d: %v", table, err)
+			}
+		}
+	}
+	want := map[string]egressRule{}
+	for _, r := range wantRules(e, self) {
+		want[r.key()] = r
+	}
+	rules, err := netlink.RuleList(unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	have := map[string]bool{}
+	for _, r := range rules {
+		if r.Priority != HolderRulePriority && r.Priority != MemberRulePriority {
+			continue
+		}
+		k := ruleOf(r).key()
+		if _, ok := want[k]; ok && !have[k] {
+			have[k] = true
+			continue
+		}
+		r := r
+		if err := netlink.RuleDel(&r); err != nil {
+			return fmt.Errorf("removing stale egress rule: %v", err)
+		}
+	}
+	for k, r := range want {
+		if have[k] {
+			continue
+		}
+		rule := netlink.NewRule()
+		rule.Priority = r.priority
+		rule.Table = r.table
+		if r.src != "" {
+			_, rule.Src, _ = net.ParseCIDR(r.src)
+		}
+		if r.dst != "" {
+			_, rule.Dst, _ = net.ParseCIDR(r.dst)
+		}
+		if r.ipproto != 0 {
+			rule.IPProto = r.ipproto
+			rule.Dport = netlink.NewRulePortRange(r.from, r.to)
+		}
+		if r.mark != 0 {
+			rule.Mark = int(r.mark)
+			rule.Mask = int(r.mask)
+		}
+		if err := netlink.RuleAdd(rule); err != nil {
+			return fmt.Errorf("adding egress rule %s: %v", k, err)
+		}
+	}
+	return nil
+}
+
+// moveLocalRule puts the local table's lookup after HolderRulePriority (it is at 0 by default), so the holder can
+// route replies to its own address onwards to their node instead of delivering them to itself.
+func moveLocalRule() error {
+	rules, err := netlink.RuleList(unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	var at0, atLocal bool
+	for _, r := range rules {
+		if r.Table == unix.RT_TABLE_LOCAL && r.Src == nil && r.Dst == nil && r.Mark <= 0 {
+			switch {
+			case r.Priority <= 0:
+				// the kernel leaves out the priority attribute when it is 0: listed as unset (-1)
+				at0 = true
+			case r.Priority == LocalRulePriority:
+				atLocal = true
+			}
+		}
+	}
+	if !atLocal {
+		rule := netlink.NewRule()
+		rule.Priority = LocalRulePriority
+		rule.Table = unix.RT_TABLE_LOCAL
+		if err := netlink.RuleAdd(rule); err != nil {
+			return fmt.Errorf("adding local table rule at %d: %v", LocalRulePriority, err)
+		}
+	}
+	if at0 {
+		rule := netlink.NewRule()
+		rule.Priority = 0
+		rule.Table = unix.RT_TABLE_LOCAL
+		if err := netlink.RuleDel(rule); err != nil {
+			return fmt.Errorf("removing local table rule at 0: %v", err)
+		}
+	}
+	return nil
+}
+
+// ensureTunnels makes one ipip tunnel per peer (this node's bridge address to the peer's), and removes every
+// other egress tunnel (including the shared one of earlier builds).
+func ensureTunnels(peers []Node, local Local) error {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return err
+	}
+	want := map[string]Node{}
+	for _, p := range peers {
+		want[TunnelName(p.Name)] = p
+	}
+	have := map[string]netlink.Link{}
+	for _, l := range links {
+		name := l.Attrs().Name
+		if name == legacyTunnel || strings.HasPrefix(name, TunnelPrefix) {
+			if _, ok := want[name]; ok {
+				have[name] = l
+				continue
+			}
+			if err := netlink.LinkDel(l); err != nil {
+				return fmt.Errorf("removing egress tunnel %s: %v", name, err)
+			}
+		}
+	}
+	if len(want) == 0 {
+		return nil
+	}
+	if local.WeaveIP == nil {
+		return fmt.Errorf("egress tunnels need the weave bridge's address")
+	}
+	br, err := netlink.LinkByName(bridge)
+	if err != nil {
+		return fmt.Errorf("weave bridge: %v", err)
+	}
+	mtu := br.Attrs().MTU - ipipOverhead
+	names := make([]string, 0, len(want))
+	for n := range want {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		peer := want[name]
+		link := have[name]
+		if link != nil {
+			t, ok := link.(*netlink.Iptun)
+			if !ok || !t.Local.Equal(local.WeaveIP) || !t.Remote.Equal(peer.WeaveIP) || t.Attrs().MTU != mtu {
+				if err := netlink.LinkDel(link); err != nil {
+					return err
+				}
+				link = nil
+			}
+		}
+		if link == nil {
+			t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: name, MTU: mtu}, Local: local.WeaveIP, Remote: peer.WeaveIP, PMtuDisc: 1}
+			if err := netlink.LinkAdd(t); err != nil {
+				return fmt.Errorf("creating egress tunnel %s to %s: %v", name, peer.Name, err)
+			}
+			if link, err = netlink.LinkByName(name); err != nil {
+				return err
+			}
+		}
+		if err := tunnelSetup(link, local); err != nil {
+			return fmt.Errorf("%s: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// tunnelSetup: the holder receives members' packets from its own address (accept_local); reverse-path filtering
+// rejects what arrives on a device with no IPv4 address unless the reverse route leaves by that very device, so
+// the tunnel carries this node's bridge address (never on the wire from here: the tunnel sets the outer source,
+// and what crosses it is forwarded).
+func tunnelSetup(link netlink.Link, local Local) error {
+	name := link.Attrs().Name
+	for k, v := range map[string]string{"accept_local": "1", "rp_filter": "2"} {
+		if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+name+"/"+k, []byte(v), 0o644); err != nil {
+			return fmt.Errorf("sysctl %s: %v", k, err)
+		}
+	}
+	addrs, err := netlink.AddrList(link, unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	has := false
+	for _, a := range addrs {
+		if a.IP.Equal(local.WeaveIP) {
+			has = true
+			continue
+		}
+		a := a
+		if err := netlink.AddrDel(link, &a); err != nil {
+			return fmt.Errorf("removing stale address %s: %v", a.IPNet, err)
+		}
+	}
+	if !has {
+		addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK}
+		if err := netlink.AddrAdd(link, addr); err != nil {
+			return fmt.Errorf("address: %v", err)
+		}
+	}
+	return netlink.LinkSetUp(link)
+}

lb/egress_test.go

@@ -0,0 +1,355 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// the S1 fixtures: cp, w1 (holds 86.107.168.15), w2
+
+func egressFor(name string, status EgressServiceStatus) *EgressService {
+	return &EgressService{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name},
+		Spec: EgressServiceSpec{Emit: EmitHolder}, Status: status}
+}
+
+func mailSvc() *coreapi.Service {
+	return svc("mail", "86.107.168.15", func(s *coreapi.Service) {
+		s.Spec.Selector = map[string]string{"app": "mail"}
+	})
+}
+
+func pod(name, node, ip string, labels map[string]string, mutate ...func(*coreapi.Pod)) *coreapi.Pod {
+	p := &coreapi.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name, Labels: labels,
+		CreationTimestamp: metav1.NewTime(time.Unix(1000, 0))},
+		Spec: coreapi.PodSpec{NodeName: node}, Status: coreapi.PodStatus{PodIP: ip, Phase: coreapi.PodRunning}}
+	for _, m := range mutate {
+		m(p)
+	}
+	return p
+}
+
+var mailLabels = map[string]string{"app": "mail"}
+
+// the status the holder (w1) writes for three nodes, blocks in name order
+func written() EgressServiceStatus {
+	return EgressServiceStatus{Host: "w1", Address: "86.107.168.15", Blocks: []PortBlock{
+		{Node: "cp", From: 61000, To: 61511}, {Node: "w1", From: 61512, To: 62023}, {Node: "w2", From: 62024, To: 62535}}}
+}
+
+func TestBlocksAreStableAndCapped(t *testing.T) {
+	got := AllocateBlocks(nil, []string{"w2", "cp", "w1"})
+	if !sameStatus(EgressServiceStatus{Blocks: got}, EgressServiceStatus{Blocks: written().Blocks}) {
+		t.Fatalf("fresh blocks: %+v", got)
+	}
+	// a node leaves, another joins: survivors keep their blocks, the newcomer takes the freed lowest one
+	got = AllocateBlocks(got, []string{"w1", "w2", "w3"})
+	want := []PortBlock{{"w1", 61512, 62023}, {"w2", 62024, 62535}, {"w3", 61000, 61511}}
+	if !sameStatus(EgressServiceStatus{Blocks: got}, EgressServiceStatus{Blocks: want}) {
+		t.Fatalf("after churn: %+v", got)
+	}
+	// misaligned or duplicate blocks are not kept
+	got = AllocateBlocks([]PortBlock{{"w1", 61001, 61512}, {"w2", 61000, 61511}, {"w3", 61000, 61511}}, []string{"w1", "w2", "w3"})
+	for _, b := range got {
+		if (b.From-BlockBase)%BlockSize != 0 || b.To != b.From+BlockSize-1 {
+			t.Fatalf("invalid block kept: %+v", got)
+		}
+	}
+	if got[1].Node != "w2" || got[1].From != 61000 {
+		t.Fatalf("first valid claim wins: %+v", got)
+	}
+	var many []string
+	for i := 0; i < BlockCount+2; i++ {
+		many = append(many, string(rune('a'+i)))
+	}
+	if n := len(AllocateBlocks(nil, many)); n != BlockCount {
+		t.Fatalf("more blocks than exist: %d", n)
+	}
+	for _, b := range AllocateBlocks(nil, many) {
+		if b.To > 65535 || b.From < 61000 {
+			t.Fatalf("block outside the range: %+v", b)
+		}
+	}
+}
+
+func TestHolderWritesStatusAndMembersWaitForIt(t *testing.T) {
+	e := egressFor("mail", EgressServiceStatus{})
+	pods := []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}
+	d := ComputeEgress("w1", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{e}, nil, nil)
+	if len(d.Status) != 1 || !sameStatus(d.Status[0].Status, written()) {
+		t.Fatalf("holder's status: %+v", d.Status)
+	}
+	// until the status is in the API, a member's outside traffic goes nowhere (fail closed)
+	d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{e}, pods, nil)
+	if len(d.Status) != 0 || len(d.Members) != 1 || !d.Members[0].Unreachable {
+		t.Fatalf("member before status: %+v", d)
+	}
+	if d.Tunnelled("w2") {
+		t.Fatalf("no tunnel for an unreachable member")
+	}
+}
+
+func TestMemberTunnelsToHolderWhichRoutesItsBlockBack(t *testing.T) {
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	pods := []*coreapi.Pod{
+		pod("mail-0", "w2", "10.44.0.9", mailLabels),
+		pod("web", "w2", "10.44.0.10", map[string]string{"app": "web"}), // not a member (A4)
+	}
+	d := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, pods, nil)
+	if len(d.Members) != 1 || d.Members[0].Pod != "gitea/mail-0" {
+		t.Fatalf("members: %+v", d.Members)
+	}
+	m := d.Members[0]
+	if m.Unreachable || !m.Address.Equal(net.ParseIP("86.107.168.15")) || m.From != 62024 || m.To != 62535 || m.Holder.Name != "w1" {
+		t.Fatalf("member path: %+v", m)
+	}
+	if !d.Tunnelled("w2") || len(d.Peers("w2")) != 1 || d.Peers("w2")[0].Name != "w1" {
+		t.Fatalf("w2 tunnels to w1 only: %+v", d.Peers("w2"))
+	}
+	if len(d.Status) != 0 || len(d.Routes) != 0 {
+		t.Fatalf("only the holder writes and routes: %+v", d)
+	}
+	// the holder: blocks of the other nodes route back to them; its own block is plain conntrack
+	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
+	if len(h.Status) != 0 {
+		t.Fatalf("status already right: %+v", h.Status)
+	}
+	if len(h.Routes) != 2 || h.Routes[0].Peer.Name != "cp" || h.Routes[1].Peer.Name != "w2" || h.Routes[1].From != 62024 {
+		t.Fatalf("holder routes: %+v", h.Routes)
+	}
+	// a member on the holder leaves from there
+	hm := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
+	if len(hm.Members) != 1 || hm.Members[0].Holder.Name != "w1" || hm.Members[0].From != 61512 {
+		t.Fatalf("member on the holder: %+v", hm.Members)
+	}
+}
+
+func TestEgressFailsClosed(t *testing.T) {
+	pods := []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}
+	// no node holds the address: status cleared (by the first node by name), member dropped
+	unheld := svc("mail", "86.107.168.99", func(s *coreapi.Service) { s.Spec.Selector = mailLabels })
+	for _, self := range []string{"cp", "w2"} {
+		d := ComputeEgress(self, nodes, []*coreapi.Service{unheld}, []*EgressService{egressFor("mail", written())}, pods, nil)
+		clears := len(d.Status) == 1 && d.Status[0].Status.Host == ""
+		if clears != (self == "cp") {
+			t.Fatalf("%s clears=%v: %+v", self, clears, d.Status)
+		}
+		if self == "w2" && (len(d.Members) != 1 || !d.Members[0].Unreachable) {
+			t.Fatalf("member of an unheld address: %+v", d.Members)
+		}
+	}
+	// two EgressServices select the pod: dropped, reported
+	other := svc("mail2", "86.107.168.15", func(s *coreapi.Service) { s.Spec.Selector = mailLabels })
+	d := ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc(), other},
+		[]*EgressService{egressFor("mail", written()), egressFor("mail2", written())}, pods, nil)
+	if len(d.Members) != 1 || !d.Members[0].Unreachable || !strings.Contains(strings.Join(d.Problems, "\n"), "several EgressServices") {
+		t.Fatalf("double member: %+v", d)
+	}
+	// a spec this agent does not implement is refused, not guessed at
+	bad := egressFor("mail", written())
+	bad.Spec.Emit = "Direct"
+	d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{bad}, pods, nil)
+	if len(d.Members) != 1 || !d.Members[0].Unreachable {
+		t.Fatalf("unknown emit mode: %+v", d)
+	}
+	// hostNetwork pods are never members (their traffic is the node's)
+	hn := pod("mail-h", "w2", "86.107.168.88", mailLabels, func(p *coreapi.Pod) { p.Spec.HostNetwork = true })
+	d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{egressFor("mail", written())}, []*coreapi.Pod{hn}, nil)
+	if len(d.Members) != 0 {
+		t.Fatalf("hostNetwork pod became a member: %+v", d.Members)
+	}
+}
+
+func TestAdmissionDecidesWhoOwnsAnIP(t *testing.T) {
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	old := pod("mail-0", "w2", "10.44.0.9", mailLabels, func(p *coreapi.Pod) {
+		now := metav1.Now()
+		p.DeletionTimestamp = &now
+	})
+	newer := pod("web", "w2", "", map[string]string{"app": "web"}) // API has no IP for it yet
+	// the CNI plugin just gave 10.44.0.9 to web: the terminating member must not keep it (A4)
+	d := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{old, newer},
+		map[string]Admission{"10.44.0.9": {Namespace: "gitea", Name: "web"}})
+	if len(d.Members) != 0 {
+		t.Fatalf("reused IP kept the old member's rules: %+v", d.Members)
+	}
+	// and the other way round: a member admitted before the API shows its IP is protected at once (A1)
+	fresh := pod("mail-1", "w2", "", mailLabels)
+	d = ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{fresh},
+		map[string]Admission{"10.44.0.11": {Namespace: "gitea", Name: "mail-1"}})
+	if len(d.Members) != 1 || d.Members[0].IP.String() != "10.44.0.11" || d.Members[0].Unreachable {
+		t.Fatalf("admitted member: %+v", d.Members)
+	}
+	// without an admission, a live pod wins a shared IP over a terminating one
+	live := pod("web", "w2", "10.44.0.9", map[string]string{"app": "web"})
+	d = ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{old, live}, nil)
+	if len(d.Members) != 0 {
+		t.Fatalf("terminating member won the IP: %+v", d.Members)
+	}
+}
+
+func TestRenderedEgressRules(t *testing.T) {
+	_, podNet, _ := net.ParseCIDR("10.32.0.0/12")
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	toW1, toW2, toCP := TunnelName("w1"), TunnelName("w2"), TunnelName("cp")
+	member := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
+	r := Render(Desired{}, member, "w2", podNet, nil)
+	for _, want := range []string{
+		"-A REWEAVE-LB-CLASS -d 10.32.0.0/12 -j RETURN\n",
+		"-A REWEAVE-LB-CLASS -d 86.107.168.24/32 -j RETURN\n", // a node is not outside
+		"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -m comment --comment \"gitea/mail-0\" -j CONNMARK --set-xmark 0x100000/0x100000\n",
+		"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -j MARK --set-xmark 0x100000/0x100000\n",
+		// port protocols within this node's block, into the holder's own tunnel
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p sctp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		// ping within the block too (the echo identifier is NAT's port for ICMP)
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p icmp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		// everything else (GRE, ...) from the address alone
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j SNAT --to-source 86.107.168.15\n",
+	} {
+		if !strings.Contains(r.Nat, want) {
+			t.Fatalf("member nat lacks %q:\n%s", want, r.Nat)
+		}
+	}
+	// the catch-all SNAT comes after the port-ranged ones (or ports would not stay in the block)
+	if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p icmp -j SNAT") {
+		t.Fatalf("SNAT order:\n%s", r.Nat)
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j RETURN\n",
+		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -m mark --mark 0x100000/0x100000 -m comment --comment \"gitea/mail-0\" -j DROP\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -d 86.107.168.15/32 -j DROP\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -j DROP\n",
+		"-A REWEAVE-LB-IN -p 4 -s 10.40.0.0/32 -m mac --mac-source aa:aa:aa:aa:aa:02 -j RETURN\n",
+		"-A REWEAVE-LB-IN -p 4 -j DROP\n",
+	} {
+		if !strings.Contains(r.Filter, want) {
+			t.Fatalf("member filter lacks %q:\n%s", want, r.Filter)
+		}
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir ORIGINAL -m connmark --mark 0x100000/0x100000 -j MARK --set-xmark 0x100000/0x100000\n",
+		"-A REWEAVE-LB-MSS -o rweg+ -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("member mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	if strings.Contains(r.Mangle, "--set-xmark 0x") && strings.Contains(r.Mangle, "-i "+toW1) {
+		t.Fatalf("a member's node does not tag what its holder sends back:\n%s", r.Mangle)
+	}
+	// the guard's DROP comes after the member's own RETURN: nothing else of its leaves
+	if strings.Index(r.Filter, toW1+" -m mark --mark 0x100000/0x100000 -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") {
+		t.Fatalf("guard order:\n%s", r.Filter)
+	}
+
+	// the holder: tags each tunnel's connections with its node, accepts each node's block, nothing foreign
+	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
+	r = Render(Desired{}, h, "w1", podNet, nil)
+	idx := peerIndex(h, "w1")
+	tagW2 := fmt.Sprintf("0x%x", peerTag(idx["w2"]))
+	for _, want := range []string{
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir REPLY -m connmark --mark 0x200000/0x200000 -j CONNMARK --restore-mark --nfmask 0xff200000 --ctmask 0xff200000\n",
+		"-A REWEAVE-LB-MARK -i " + toW2 + " -m conntrack --ctstate NEW -m connmark --mark 0x0/0xff200000 -m comment --comment \"from w2\" -j CONNMARK --set-xmark " + tagW2 + "/0xff200000\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("holder mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -m connmark ! --mark " + tagW2 + "/0xff200000 -j DROP\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i " + toCP + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p sctp -m sctp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN\n",
+		"-A REWEAVE-LB-FWD -s 10.40.0.4/32 -o weave -m mark --mark 0x100000/0x100000 -j DROP\n",
+	} {
+		if !strings.Contains(r.Filter, want) {
+			t.Fatalf("holder filter lacks %q:\n%s", want, r.Filter)
+		}
+	}
+	// foreign tags are refused before anything of that tunnel is accepted; a port outside the block before the
+	// catch-all accept
+	if strings.Index(r.Filter, "! --mark "+tagW2) > strings.Index(r.Filter, "\"block of w2\"") ||
+		strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP") >
+			strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN") {
+		t.Fatalf("holder filter order:\n%s", r.Filter)
+	}
+	if !strings.Contains(r.Nat, "-A REWEAVE-LB-POST -s 10.40.0.4/32 ! -o weave -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:61512-62023\n") {
+		t.Fatalf("holder's own member:\n%s", r.Nat)
+	}
+	for _, line := range strings.Split(r.Nat+r.Filter, "\n") {
+		if strings.Contains(line, "10.44.0.10") {
+			t.Fatalf("a rule names a non-member: %s", line)
+		}
+	}
+}
+
+func TestTunnelNames(t *testing.T) {
+	a, b := TunnelName("reweave-worker-fe19b040"), TunnelName("reweave-cp-94a8db43")
+	if a == b || len(a) > 15 || !strings.HasPrefix(a, TunnelPrefix) || a != TunnelName("reweave-worker-fe19b040") {
+		t.Fatalf("tunnel names must be distinct, stable, short: %q %q", a, b)
+	}
+}
+
+func TestPolicyRules(t *testing.T) {
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
+	rules := wantRules(h, "w1")
+	tables := peerTables(h, "w1")
+	idx := peerIndex(h, "w1")
+	ports, tags := 0, 0
+	for _, r := range rules {
+		if r.priority != HolderRulePriority || r.src != "" {
+			t.Fatalf("holder rule: %+v", r)
+		}
+		switch {
+		case r.ipproto != 0:
+			ports++
+			if r.dst != "86.107.168.15/32" || r.mark != 0 {
+				t.Fatalf("port rule: %+v", r)
+			}
+			if (r.from == 62024) != (r.table == tables["w2"]) || (r.from == 61000) != (r.table == tables["cp"]) {
+				t.Fatalf("block routed to the wrong node: %+v %v", r, tables)
+			}
+		default:
+			tags++
+			if r.mask != peerMask || r.dst != "" {
+				t.Fatalf("tag rule: %+v", r)
+			}
+			for name, i := range idx {
+				if r.mark == peerTag(i) && r.table != tables[name] {
+					t.Fatalf("tag of %s routed to table %d, want %d", name, r.table, tables[name])
+				}
+			}
+		}
+	}
+	if ports != 2*len(portProtocols) || tags != 2 {
+		t.Fatalf("2 peers: %d port rules (tcp, udp, sctp each), %d tag rules: %+v", ports, tags, rules)
+	}
+	if HolderRulePriority >= LocalRulePriority {
+		t.Fatalf("holder rules must precede the local table")
+	}
+	m := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
+	rules = wantRules(m, "w2")
+	if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].mask != EgressMark ||
+		rules[0].table != peerTables(m, "w2")["w1"] {
+		t.Fatalf("member rule: %+v", rules)
+	}
+	u := ComputeEgress("w2", nodes, svcs, []*EgressService{egressFor("mail", EgressServiceStatus{})}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
+	rules = wantRules(u, "w2")
+	if len(rules) != 1 || rules[0].table != EgressTableBase {
+		t.Fatalf("unreachable member must hit the unreachable table: %+v", rules)
+	}
+}

lb/gateway_api.go

@@ -0,0 +1,415 @@
+package lb
+
+import (
+	"context"
+	"encoding/json"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+)
+
+// The Gateway API (gateway.networking.k8s.io/v1, as the v1.6.3 standard channel serves it): the subset this agent
+// implements, as plain types. The upstream Go module needs a newer client-go than the fork's; the JSON is the
+// contract, and fields this agent does not implement are simply not decoded.
+
+const (
+	GatewayGroup   = "gateway.networking.k8s.io"
+	GatewayVersion = "v1"
+	// GatewayControllerName is the spec.controllerName of the GatewayClasses this agent implements.
+	GatewayControllerName = "reweave.levk.dev/gateway"
+
+	GatewayClasses  = "gatewayclasses"
+	Gateways        = "gateways"
+	HTTPRoutes      = "httproutes"
+	TCPRoutes       = "tcproutes"
+	ReferenceGrants = "referencegrants"
+)
+
+// GatewayGroupVersion of the Gateway API.
+var GatewayGroupVersion = schema.GroupVersion{Group: GatewayGroup, Version: GatewayVersion}
+
+type GatewayClass struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              GatewayClassSpec   `json:"spec"`
+	Status            GatewayClassStatus `json:"status,omitempty"`
+}
+
+type GatewayClassSpec struct {
+	ControllerName string `json:"controllerName"`
+}
+
+type GatewayClassStatus struct {
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+}
+
+type Gateway struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              GatewaySpec   `json:"spec"`
+	Status            GatewayStatus `json:"status,omitempty"`
+}
+
+type GatewaySpec struct {
+	GatewayClassName string           `json:"gatewayClassName"`
+	Listeners        []Listener       `json:"listeners"`
+	Addresses        []GatewayAddress `json:"addresses,omitempty"`
+}
+
+// GatewayAddress is a spec or status address; Type defaults to IPAddress.
+type GatewayAddress struct {
+	Type  *string `json:"type,omitempty"`
+	Value string  `json:"value"`
+}
+
+type Listener struct {
+	Name          string             `json:"name"`
+	Hostname      *string            `json:"hostname,omitempty"`
+	Port          int32              `json:"port"`
+	Protocol      string             `json:"protocol"`
+	TLS           *ListenerTLSConfig `json:"tls,omitempty"`
+	AllowedRoutes *AllowedRoutes     `json:"allowedRoutes,omitempty"`
+}
+
+type ListenerTLSConfig struct {
+	Mode            *string                 `json:"mode,omitempty"`
+	CertificateRefs []SecretObjectReference `json:"certificateRefs,omitempty"`
+}
+
+type SecretObjectReference struct {
+	Group     *string `json:"group,omitempty"`
+	Kind      *string `json:"kind,omitempty"`
+	Name      string  `json:"name"`
+	Namespace *string `json:"namespace,omitempty"`
+}
+
+type AllowedRoutes struct {
+	Namespaces *RouteNamespaces `json:"namespaces,omitempty"`
+	Kinds      []RouteGroupKind `json:"kinds,omitempty"`
+}
+
+type RouteNamespaces struct {
+	From     *string               `json:"from,omitempty"`
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+}
+
+type RouteGroupKind struct {
+	Group *string `json:"group,omitempty"`
+	Kind  string  `json:"kind"`
+}
+
+type GatewayStatus struct {
+	Addresses  []GatewayAddress   `json:"addresses,omitempty"`
+	Conditions []metav1.Condition `json:"conditions,omitempty"`
+	Listeners  []ListenerStatus   `json:"listeners,omitempty"`
+}
+
+type ListenerStatus struct {
+	Name           string             `json:"name"`
+	SupportedKinds []RouteGroupKind   `json:"supportedKinds"`
+	AttachedRoutes int32              `json:"attachedRoutes"`
+	Conditions     []metav1.Condition `json:"conditions"`
+}
+
+type ParentReference struct {
+	Group       *string `json:"group,omitempty"`
+	Kind        *string `json:"kind,omitempty"`
+	Namespace   *string `json:"namespace,omitempty"`
+	Name        string  `json:"name"`
+	SectionName *string `json:"sectionName,omitempty"`
+	Port        *int32  `json:"port,omitempty"`
+}
+
+type BackendRef struct {
+	Group     *string `json:"group,omitempty"`
+	Kind      *string `json:"kind,omitempty"`
+	Name      string  `json:"name"`
+	Namespace *string `json:"namespace,omitempty"`
+	Port      *int32  `json:"port,omitempty"`
+	Weight    *int32  `json:"weight,omitempty"`
+}
+
+type RouteStatus struct {
+	Parents []RouteParentStatus `json:"parents"`
+}
+
+type RouteParentStatus struct {
+	ParentRef      ParentReference    `json:"parentRef"`
+	ControllerName string             `json:"controllerName"`
+	Conditions     []metav1.Condition `json:"conditions,omitempty"`
+}
+
+type HTTPRoute struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              HTTPRouteSpec `json:"spec"`
+	Status            RouteStatus   `json:"status,omitempty"`
+}
+
+type HTTPRouteSpec struct {
+	ParentRefs []ParentReference `json:"parentRefs,omitempty"`
+	Hostnames  []string          `json:"hostnames,omitempty"`
+	Rules      []HTTPRouteRule   `json:"rules,omitempty"`
+}
+
+type HTTPRouteRule struct {
+	Matches     []HTTPRouteMatch  `json:"matches,omitempty"`
+	Filters     []HTTPRouteFilter `json:"filters,omitempty"`
+	BackendRefs []HTTPBackendRef  `json:"backendRefs,omitempty"`
+}
+
+type HTTPRouteMatch struct {
+	Path        *HTTPPathMatch        `json:"path,omitempty"`
+	Headers     []HTTPHeaderMatch     `json:"headers,omitempty"`
+	QueryParams []HTTPQueryParamMatch `json:"queryParams,omitempty"`
+	Method      *string               `json:"method,omitempty"`
+}
+
+type HTTPPathMatch struct {
+	Type  *string `json:"type,omitempty"`
+	Value *string `json:"value,omitempty"`
+}
+
+type HTTPHeaderMatch struct {
+	Type  *string `json:"type,omitempty"`
+	Name  string  `json:"name"`
+	Value string  `json:"value"`
+}
+
+type HTTPQueryParamMatch struct {
+	Type  *string `json:"type,omitempty"`
+	Name  string  `json:"name"`
+	Value string  `json:"value"`
+}
+
+// HTTPRouteFilter: the filters this agent implements are decoded; for the others only their presence is known
+// (through Type), which is enough to refuse them.
+type HTTPRouteFilter struct {
+	Type                   string                     `json:"type"`
+	RequestHeaderModifier  *HTTPHeaderFilter          `json:"requestHeaderModifier,omitempty"`
+	ResponseHeaderModifier *HTTPHeaderFilter          `json:"responseHeaderModifier,omitempty"`
+	RequestRedirect        *HTTPRequestRedirectFilter `json:"requestRedirect,omitempty"`
+	URLRewrite             *HTTPURLRewriteFilter      `json:"urlRewrite,omitempty"`
+}
+
+type HTTPHeaderFilter struct {
+	Set    []HTTPHeader `json:"set,omitempty"`
+	Add    []HTTPHeader `json:"add,omitempty"`
+	Remove []string     `json:"remove,omitempty"`
+}
+
+type HTTPHeader struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+type HTTPRequestRedirectFilter struct {
+	Scheme     *string           `json:"scheme,omitempty"`
+	Hostname   *string           `json:"hostname,omitempty"`
+	Path       *HTTPPathModifier `json:"path,omitempty"`
+	Port       *int32            `json:"port,omitempty"`
+	StatusCode *int              `json:"statusCode,omitempty"`
+}
+
+type HTTPURLRewriteFilter struct {
+	Hostname *string           `json:"hostname,omitempty"`
+	Path     *HTTPPathModifier `json:"path,omitempty"`
+}
+
+type HTTPPathModifier struct {
+	Type               string  `json:"type"`
+	ReplaceFullPath    *string `json:"replaceFullPath,omitempty"`
+	ReplacePrefixMatch *string `json:"replacePrefixMatch,omitempty"`
+}
+
+type HTTPBackendRef struct {
+	BackendRef `json:",inline"`
+	Filters    []HTTPRouteFilter `json:"filters,omitempty"`
+}
+
+type TCPRoute struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              TCPRouteSpec `json:"spec"`
+	Status            RouteStatus  `json:"status,omitempty"`
+}
+
+type TCPRouteSpec struct {
+	ParentRefs []ParentReference `json:"parentRefs,omitempty"`
+	Rules      []TCPRouteRule    `json:"rules,omitempty"`
+}
+
+type TCPRouteRule struct {
+	BackendRefs []BackendRef `json:"backendRefs,omitempty"`
+}
+
+type ReferenceGrant struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              ReferenceGrantSpec `json:"spec"`
+}
+
+type ReferenceGrantSpec struct {
+	From []ReferenceGrantFrom `json:"from"`
+	To   []ReferenceGrantTo   `json:"to"`
+}
+
+type ReferenceGrantFrom struct {
+	Group     string `json:"group"`
+	Kind      string `json:"kind"`
+	Namespace string `json:"namespace"`
+}
+
+type ReferenceGrantTo struct {
+	Group string  `json:"group"`
+	Kind  string  `json:"kind"`
+	Name  *string `json:"name,omitempty"`
+}
+
+type GatewayClassList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []GatewayClass `json:"items"`
+}
+
+type GatewayList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Gateway `json:"items"`
+}
+
+type HTTPRouteList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []HTTPRoute `json:"items"`
+}
+
+type TCPRouteList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []TCPRoute `json:"items"`
+}
+
+type ReferenceGrantList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ReferenceGrant `json:"items"`
+}
+
+// jsonCopy deep-copies through JSON: these types are only what the JSON says, so nothing is lost.
+func jsonCopy(in, out interface{}) {
+	b, err := json.Marshal(in)
+	if err != nil {
+		panic(err)
+	}
+	if err := json.Unmarshal(b, out); err != nil {
+		panic(err)
+	}
+}
+
+func (o *GatewayClass) DeepCopyObject() runtime.Object {
+	out := &GatewayClass{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *Gateway) DeepCopyObject() runtime.Object {
+	out := &Gateway{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *HTTPRoute) DeepCopyObject() runtime.Object {
+	out := &HTTPRoute{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *TCPRoute) DeepCopyObject() runtime.Object {
+	out := &TCPRoute{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *ReferenceGrant) DeepCopyObject() runtime.Object {
+	out := &ReferenceGrant{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *GatewayClassList) DeepCopyObject() runtime.Object {
+	out := &GatewayClassList{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *GatewayList) DeepCopyObject() runtime.Object {
+	out := &GatewayList{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *HTTPRouteList) DeepCopyObject() runtime.Object {
+	out := &HTTPRouteList{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *TCPRouteList) DeepCopyObject() runtime.Object {
+	out := &TCPRouteList{}
+	jsonCopy(o, out)
+	return out
+}
+
+func (o *ReferenceGrantList) DeepCopyObject() runtime.Object {
+	out := &ReferenceGrantList{}
+	jsonCopy(o, out)
+	return out
+}
+
+func addGatewayTypes(s *runtime.Scheme) error {
+	s.AddKnownTypes(GatewayGroupVersion,
+		&GatewayClass{}, &GatewayClassList{}, &Gateway{}, &GatewayList{}, &HTTPRoute{}, &HTTPRouteList{},
+		&TCPRoute{}, &TCPRouteList{}, &ReferenceGrant{}, &ReferenceGrantList{})
+	metav1.AddToGroupVersion(s, GatewayGroupVersion)
+	return nil
+}
+
+// NewGatewayClient is a REST client for the Gateway API kinds this agent reads and writes the status of.
+func NewGatewayClient(base *rest.Config) (*rest.RESTClient, error) {
+	scheme := runtime.NewScheme()
+	if err := addGatewayTypes(scheme); err != nil {
+		return nil, err
+	}
+	cfg := *base
+	cfg.GroupVersion = &GatewayGroupVersion
+	cfg.APIPath = "/apis"
+	cfg.ContentType = runtime.ContentTypeJSON
+	cfg.NegotiatedSerializer = serializer.NewCodecFactory(scheme).WithoutConversion()
+	if cfg.UserAgent == "" {
+		cfg.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+	return rest.RESTClientFor(&cfg)
+}
+
+// patchStatus merge-patches an object's status. resourceVersion, when given, makes it conditional: a status built
+// from a stale read fails instead of overwriting what someone else wrote since.
+func patchStatus(ctx context.Context, c rest.Interface, resource, namespace, name, resourceVersion string, status interface{}) error {
+	body := map[string]interface{}{"status": status}
+	if resourceVersion != "" {
+		body["metadata"] = map[string]interface{}{"resourceVersion": resourceVersion}
+	}
+	patch, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	req := c.Patch(types.MergePatchType).Resource(resource).Name(name).SubResource("status").Body(patch)
+	if namespace != "" {
+		req = req.Namespace(namespace)
+	}
+	return req.Do(ctx).Error()
+}

lb/gateway_proxy.go

@@ -0,0 +1,547 @@
+package lb
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+
+	"github.com/rajch/weave/common"
+)
+
+// Proxy serves the HTTP and HTTPS listeners of the Gateways this node holds. TLS is terminated with the
+// listener's certificate, chosen by SNI and swapped in place when its Secret changes (no restart, no rebind).
+// Requests are routed by the attached HTTPRoutes in the order the spec prescribes and passed to a ready endpoint
+// of the chosen backend, with the client's own address in X-Forwarded-For. Bodies are streamed, never limited.
+type Proxy struct {
+	mu      sync.Mutex
+	servers map[string]*portServer // by "address:port"
+	// listen binds an address (swappable in tests)
+	listen    func(addr string) (net.Listener, error)
+	transport http.RoundTripper
+	certs     map[string]*tls.Certificate // parsed certificates, by Secret uid@resourceVersion
+	rr        sync.Map                    // backend key -> *uint64, for round robin over its endpoints
+}
+
+func NewProxy() *Proxy {
+	return &Proxy{
+		servers: map[string]*portServer{},
+		certs:   map[string]*tls.Certificate{},
+		listen:  func(addr string) (net.Listener, error) { return net.Listen("tcp", addr) },
+		transport: &http.Transport{
+			DialContext:           (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
+			MaxIdleConns:          256,
+			MaxIdleConnsPerHost:   32,
+			IdleConnTimeout:       90 * time.Second,
+			ExpectContinueTimeout: time.Second,
+		},
+	}
+}
+
+// portServer is one bound address:port and what it currently serves.
+type portServer struct {
+	addr  string
+	https bool
+	ln    net.Listener
+	srv   *http.Server
+	cfg   atomic.Pointer[portConfig]
+}
+
+// portConfig is everything one address:port serves: its listeners, most specific hostname first.
+type portConfig struct {
+	listeners []*liveListener
+}
+
+type liveListener struct {
+	spec ServeListener
+	cert *tls.Certificate
+}
+
+// Apply serves exactly these listeners: ports no longer listed are closed, new ones bound, and every served
+// port's routes and certificates replaced at once. It returns, by "namespace/gateway/listener", the listeners
+// it could not bind.
+func (p *Proxy) Apply(listeners []ServeListener) map[string]error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	failed := map[string]error{}
+	byAddr := map[string][]ServeListener{}
+	for _, l := range listeners {
+		a := net.JoinHostPort(l.Address.String(), strconv.Itoa(int(l.Port)))
+		byAddr[a] = append(byAddr[a], l)
+	}
+	for a, s := range p.servers {
+		if group, ok := byAddr[a]; !ok || group[0].HTTPS != s.https {
+			s.close()
+			delete(p.servers, a)
+		}
+	}
+	used := map[string]bool{}
+	for a, group := range byAddr {
+		cfg := p.compile(group, used)
+		s := p.servers[a]
+		if s == nil {
+			var err error
+			if s, err = p.start(a, group[0].HTTPS); err != nil {
+				for _, l := range group {
+					failed[l.Gateway+"/"+l.Name] = err
+				}
+				continue
+			}
+			p.servers[a] = s
+		}
+		s.cfg.Store(cfg)
+	}
+	// forget certificates no listener uses any more
+	for k := range p.certs {
+		if !used[k] {
+			delete(p.certs, k)
+		}
+	}
+	return failed
+}
+
+// Close stops serving everything.
+func (p *Proxy) Close() {
+	p.Apply(nil)
+}
+
+func (p *Proxy) compile(group []ServeListener, used map[string]bool) *portConfig {
+	cfg := &portConfig{}
+	for _, l := range group {
+		ll := &liveListener{spec: l}
+		if l.HTTPS && l.Cert != nil {
+			ll.cert = p.cert(l.Cert)
+			used[certKey(l.Cert)] = true
+		}
+		cfg.listeners = append(cfg.listeners, ll)
+	}
+	// most specific first: exact hostnames, then wildcards (longest first), then the catch-all
+	sort.SliceStable(cfg.listeners, func(i, j int) bool {
+		return hostRank(cfg.listeners[i].spec.Hostname) > hostRank(cfg.listeners[j].spec.Hostname)
+	})
+	return cfg
+}
+
+func certKey(s *coreapi.Secret) string {
+	return string(s.UID) + "@" + s.ResourceVersion + "/" + s.Namespace + "/" + s.Name
+}
+
+// cert parses a Secret's certificate once per version of the Secret.
+func (p *Proxy) cert(s *coreapi.Secret) *tls.Certificate {
+	k := certKey(s)
+	if c, ok := p.certs[k]; ok {
+		return c
+	}
+	c, err := tls.X509KeyPair(s.Data[coreapi.TLSCertKey], s.Data[coreapi.TLSPrivateKeyKey])
+	if err != nil {
+		common.Log.Warnf("lb: gateway certificate %s/%s: %v", s.Namespace, s.Name, err)
+		return nil
+	}
+	p.certs[k] = &c
+	return &c
+}
+
+// hostRank orders listener hostnames: exact (by length) above wildcards (by length) above none.
+func hostRank(h string) int {
+	switch {
+	case h == "":
+		return 0
+	case strings.HasPrefix(h, "*."):
+		return 1000 + len(h)
+	}
+	return 100000 + len(h)
+}
+
+func (p *Proxy) start(addr string, https bool) (*portServer, error) {
+	ln, err := p.listen(addr)
+	if err != nil {
+		return nil, err
+	}
+	s := &portServer{addr: addr, https: https, ln: ln}
+	s.cfg.Store(&portConfig{})
+	s.srv = &http.Server{
+		Handler:           http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { p.serve(s, w, r) }),
+		ReadHeaderTimeout: 30 * time.Second,
+		IdleTimeout:       2 * time.Minute,
+		// handshakes from scanners and the like would flood the log
+		ErrorLog: log.New(debugWriter{}, "lb: gateway "+addr+": ", 0),
+	}
+	if https {
+		s.srv.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12, GetCertificate: s.certificate}
+		go func() {
+			if err := s.srv.ServeTLS(ln, "", ""); err != nil && err != http.ErrServerClosed {
+				common.Log.Errorf("lb: gateway %s: %v", addr, err)
+			}
+		}()
+	} else {
+		go func() {
+			if err := s.srv.Serve(ln); err != nil && err != http.ErrServerClosed {
+				common.Log.Errorf("lb: gateway %s: %v", addr, err)
+			}
+		}()
+	}
+	common.Log.Infof("lb: gateway serving %s (%s)", addr, map[bool]string{true: "HTTPS", false: "HTTP"}[https])
+	return s, nil
+}
+
+func (s *portServer) close() {
+	srv := s.srv
+	go func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = srv.Shutdown(ctx)
+		_ = srv.Close()
+	}()
+	common.Log.Infof("lb: gateway no longer serving %s", s.addr)
+}
+
+type debugWriter struct{}
+
+func (debugWriter) Write(b []byte) (int, error) {
+	common.Log.Debugf("%s", strings.TrimRight(string(b), "\n"))
+	return len(b), nil
+}
+
+// listenerFor picks the listener a name (SNI or Host) reaches on a port, most specific hostname first.
+func (c *portConfig) listenerFor(name string) *liveListener {
+	name = strings.ToLower(name)
+	for _, l := range c.listeners {
+		if l.spec.Hostname == "" || hostMatches(l.spec.Hostname, name) {
+			return l
+		}
+	}
+	return nil
+}
+
+// certificate answers a TLS handshake with the certificate of the listener its SNI names.
+func (s *portServer) certificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) {
+	cfg := s.cfg.Load()
+	l := cfg.listenerFor(hello.ServerName)
+	if l == nil || l.cert == nil {
+		return nil, fmt.Errorf("no listener for %q", hello.ServerName)
+	}
+	return l.cert, nil
+}
+
+// candidate is one match of one rule that a request satisfies, with what decides its precedence.
+type candidate struct {
+	route   *ServeRoute
+	rule    *ServeRule
+	ruleIdx int
+	// the matched hostname: characters if not a wildcard, and characters
+	hostExact, hostLen int
+	exact              bool
+	prefixLen          int
+	method             bool
+	headers, query     int
+	prefix             string // the path prefix matched (for ReplacePrefixMatch)
+}
+
+func (p *Proxy) serve(s *portServer, w http.ResponseWriter, r *http.Request) {
+	cfg := s.cfg.Load()
+	host := hostOnly(r.Host)
+	name := host
+	if r.TLS != nil && r.TLS.ServerName != "" {
+		name = r.TLS.ServerName
+	}
+	l := cfg.listenerFor(name)
+	if l == nil {
+		http.Error(w, "no listener for this host", http.StatusNotFound)
+		return
+	}
+	c := pick(l, host, r)
+	if c == nil {
+		http.Error(w, "no route", http.StatusNotFound)
+		return
+	}
+	for _, f := range c.rule.Filters {
+		if f.Type == filterRequestRedirect && f.RequestRedirect != nil {
+			to, code := redirect(r, f.RequestRedirect, l.spec.Port, c.prefix)
+			http.Redirect(w, r, to, code)
+			return
+		}
+	}
+	b := chooseBackend(c.rule.Backends)
+	if b == nil {
+		// no resolved backend to send it to: the spec's answer for unresolved backends
+		http.Error(w, "no usable backend", http.StatusInternalServerError)
+		return
+	}
+	if len(b.Endpoints) == 0 {
+		http.Error(w, "no ready endpoint", http.StatusServiceUnavailable)
+		return
+	}
+	n, _ := p.rr.LoadOrStore(b.Key, new(uint64))
+	ep := b.Endpoints[int(atomic.AddUint64(n.(*uint64), 1)-1)%len(b.Endpoints)]
+	filters := append(append([]HTTPRouteFilter(nil), c.rule.Filters...), b.Filters...)
+	rp := &httputil.ReverseProxy{
+		Transport:     p.transport,
+		FlushInterval: -1, // stream as it comes (git, server-sent events)
+		Rewrite: func(pr *httputil.ProxyRequest) {
+			pr.SetURL(&url.URL{Scheme: "http", Host: ep.String()})
+			pr.Out.Host = pr.In.Host // forwarded unmodified unless a filter says otherwise
+			for _, f := range filters {
+				switch {
+				case f.Type == filterRequestHeaderModifier && f.RequestHeaderModifier != nil:
+					modifyHeaders(pr.Out.Header, f.RequestHeaderModifier)
+				case f.Type == filterURLRewrite && f.URLRewrite != nil:
+					if f.URLRewrite.Hostname != nil {
+						pr.Out.Host = *f.URLRewrite.Hostname
+					}
+					if f.URLRewrite.Path != nil {
+						pr.Out.URL.Path = rewritePath(pr.In.URL.Path, f.URLRewrite.Path, c.prefix)
+						pr.Out.URL.RawPath = ""
+					}
+				}
+			}
+			// After the filters, so none can change these (the model already refuses routes that try): only this
+			// hop's view of the connection is passed on. Cleared first, as SetXForwarded appends to an
+			// X-Forwarded-For already there.
+			for _, h := range forwardingHeaders {
+				pr.Out.Header.Del(h)
+			}
+			pr.SetXForwarded()
+			if ip, _, err := net.SplitHostPort(pr.In.RemoteAddr); err == nil {
+				pr.Out.Header.Set("X-Real-IP", ip)
+			}
+		},
+		ModifyResponse: func(resp *http.Response) error {
+			for _, f := range filters {
+				if f.Type == filterResponseHeaderModifier && f.ResponseHeaderModifier != nil {
+					modifyHeaders(resp.Header, f.ResponseHeaderModifier)
+				}
+			}
+			return nil
+		},
+		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
+			common.Log.Debugf("lb: gateway %s -> %s: %v", s.addr, ep, err)
+			w.WriteHeader(http.StatusBadGateway)
+		},
+	}
+	rp.ServeHTTP(w, r)
+}
+
+// pick is the match that takes the request, by the spec's precedence: the most specific matching hostname, then
+// an exact path, the longest prefix, a method, the most header matches, the most query matches, then the oldest
+// route, its namespace/name, and the first rule.
+func pick(l *liveListener, host string, r *http.Request) *candidate {
+	var best *candidate
+	for ri := range l.spec.Routes {
+		route := &l.spec.Routes[ri]
+		hostExact, hostLen, ok := routeHost(route, l.spec.Hostname, host)
+		if !ok {
+			continue
+		}
+		for i := range route.Rules {
+			rule := &route.Rules[i]
+			for _, m := range rule.Matches {
+				c, ok := matches(m, r)
+				if !ok {
+					continue
+				}
+				c.route, c.rule, c.ruleIdx, c.hostExact, c.hostLen = route, rule, i, hostExact, hostLen
+				if best == nil || better(c, *best) {
+					cc := c
+					best = &cc
+				}
+			}
+		}
+	}
+	return best
+}
+
+func better(a, b candidate) bool {
+	switch {
+	case a.hostExact != b.hostExact:
+		return a.hostExact > b.hostExact
+	case a.hostLen != b.hostLen:
+		return a.hostLen > b.hostLen
+	case a.exact != b.exact:
+		return a.exact
+	case a.prefixLen != b.prefixLen:
+		return a.prefixLen > b.prefixLen
+	case a.method != b.method:
+		return a.method
+	case a.headers != b.headers:
+		return a.headers > b.headers
+	case a.query != b.query:
+		return a.query > b.query
+	case !a.route.Created.Equal(b.route.Created):
+		return a.route.Created.Before(b.route.Created)
+	case a.route.Key != b.route.Key:
+		return a.route.Key < b.route.Key
+	}
+	return a.ruleIdx < b.ruleIdx
+}
+
+// routeHost reports whether a route answers this host on its listener, and how specifically.
+func routeHost(route *ServeRoute, listenerHost, host string) (int, int, bool) {
+	names := route.Hostnames
+	if len(names) == 0 {
+		if listenerHost == "" {
+			return 0, 0, true
+		}
+		names = []string{listenerHost}
+	}
+	exact, length, ok := 0, 0, false
+	for _, h := range names {
+		if !hostMatches(h, host) {
+			continue
+		}
+		ok = true
+		e := 0
+		if !strings.HasPrefix(h, "*.") {
+			e = len(h)
+		}
+		if e > exact || (e == exact && len(h) > length) {
+			exact, length = e, len(h)
+		}
+	}
+	return exact, length, ok
+}
+
+func matches(m HTTPRouteMatch, r *http.Request) (candidate, bool) {
+	var c candidate
+	path, value := r.URL.Path, str(m.Path.Value)
+	switch str(m.Path.Type) {
+	case pathExact:
+		if path != value {
+			return c, false
+		}
+		c.exact = true
+		c.prefixLen = len(value)
+	default:
+		trimmed := strings.TrimSuffix(value, "/")
+		if trimmed != "" && path != trimmed && !strings.HasPrefix(path, trimmed+"/") {
+			return c, false
+		}
+		c.prefixLen = len(value)
+		c.prefix = value
+	}
+	if m.Method != nil {
+		if r.Method != *m.Method {
+			return c, false
+		}
+		c.method = true
+	}
+	for _, h := range m.Headers {
+		if r.Header.Get(h.Name) != h.Value {
+			return c, false
+		}
+	}
+	c.headers = len(m.Headers)
+	q := r.URL.Query()
+	for _, qp := range m.QueryParams {
+		if q.Get(qp.Name) != qp.Value {
+			return c, false
+		}
+	}
+	c.query = len(m.QueryParams)
+	return c, true
+}
+
+// chooseBackend picks a backend by weight (nil when there is none to pick).
+func chooseBackend(bs []ServeBackend) *ServeBackend {
+	total := int32(0)
+	for _, b := range bs {
+		if b.Weight > 0 {
+			total += b.Weight
+		}
+	}
+	if total == 0 {
+		return nil
+	}
+	n := rand.Int31n(total)
+	for i := range bs {
+		if bs[i].Weight <= 0 {
+			continue
+		}
+		if n < bs[i].Weight {
+			return &bs[i]
+		}
+		n -= bs[i].Weight
+	}
+	return nil
+}
+
+// redirect is where a RequestRedirect filter sends a request, and with which status.
+func redirect(r *http.Request, f *HTTPRequestRedirectFilter, listenerPort int32, prefix string) (string, int) {
+	scheme := "http"
+	if r.TLS != nil {
+		scheme = "https"
+	}
+	port := int(listenerPort)
+	if f.Scheme != nil {
+		// a new scheme brings its well-known port, unless a port is given
+		scheme = *f.Scheme
+		port = map[string]int{"http": 80, "https": 443}[scheme]
+	}
+	if f.Port != nil {
+		port = int(*f.Port)
+	}
+	host := hostOnly(r.Host)
+	if f.Hostname != nil {
+		host = *f.Hostname
+	}
+	if !(scheme == "http" && port == 80) && !(scheme == "https" && port == 443) && port != 0 {
+		host = net.JoinHostPort(host, strconv.Itoa(port))
+	}
+	path := r.URL.Path
+	if f.Path != nil {
+		path = rewritePath(path, f.Path, prefix)
+	}
+	code := http.StatusFound
+	if f.StatusCode != nil {
+		code = *f.StatusCode
+	}
+	return (&url.URL{Scheme: scheme, Host: host, Path: path, RawQuery: r.URL.RawQuery}).String(), code
+}
+
+// rewritePath applies a path modifier: the whole path, or the matched prefix replaced.
+func rewritePath(path string, m *HTTPPathModifier, prefix string) string {
+	switch m.Type {
+	case "ReplaceFullPath":
+		return str(m.ReplaceFullPath)
+	case "ReplacePrefixMatch":
+		rest := strings.TrimPrefix(path, strings.TrimSuffix(prefix, "/"))
+		out := strings.TrimSuffix(str(m.ReplacePrefixMatch), "/") + rest
+		if !strings.HasPrefix(out, "/") {
+			out = "/" + out
+		}
+		return out
+	}
+	return path
+}
+
+func modifyHeaders(h http.Header, f *HTTPHeaderFilter) {
+	for _, s := range f.Set {
+		h.Set(s.Name, s.Value)
+	}
+	for _, a := range f.Add {
+		h.Add(a.Name, a.Value)
+	}
+	for _, name := range f.Remove {
+		h.Del(name)
+	}
+}
+
+// hostOnly is a Host header without its port, lowercased (the spec ignores the port when matching).
+func hostOnly(h string) string {
+	if host, _, err := net.SplitHostPort(h); err == nil {
+		h = host
+	}
+	return strings.ToLower(strings.TrimSuffix(h, "."))
+}

lb/gateway_proxy_test.go

@@ -0,0 +1,333 @@
+package lb
+
+import (
+	"bytes"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// testProxy binds every requested address on loopback and remembers where.
+type testProxy struct {
+	*Proxy
+	mu     sync.Mutex
+	actual map[string]string
+}
+
+func newTestProxy(t *testing.T) *testProxy {
+	tp := &testProxy{Proxy: NewProxy(), actual: map[string]string{}}
+	tp.listen = func(addr string) (net.Listener, error) {
+		ln, err := net.Listen("tcp", "127.0.0.1:0")
+		if err == nil {
+			tp.mu.Lock()
+			tp.actual[addr] = ln.Addr().String()
+			tp.mu.Unlock()
+		}
+		return ln, err
+	}
+	t.Cleanup(tp.Close)
+	return tp
+}
+
+func (tp *testProxy) at(port int32) string {
+	tp.mu.Lock()
+	defer tp.mu.Unlock()
+	return tp.actual[net.JoinHostPort("192.0.2.1", strconv.Itoa(int(port)))]
+}
+
+// named is a backend that answers with its name and the request it saw.
+func named(t *testing.T, name string) (*httptest.Server, Endpoint) {
+	s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-Backend", name)
+		n, _ := io.Copy(io.Discard, r.Body)
+		fmt.Fprintf(w, "%s path=%s host=%s xff=%s proto=%s xhost=%s realip=%s added=%s removed=%s body=%d",
+			name, r.URL.Path, r.Host, strings.Join(r.Header.Values("X-Forwarded-For"), ","), r.Header.Get("X-Forwarded-Proto"),
+			r.Header.Get("X-Forwarded-Host"), r.Header.Get("X-Real-IP"), strings.Join(r.Header.Values("X-Added"), ","),
+			r.Header.Get("X-Remove-Me"), n)
+	}))
+	t.Cleanup(s.Close)
+	_, port, _ := net.SplitHostPort(s.Listener.Addr().String())
+	p, _ := strconv.Atoi(port)
+	return s, Endpoint{IP: net.ParseIP("127.0.0.1").To4(), Port: int32(p)}
+}
+
+func backend(name string, ep Endpoint) []ServeBackend {
+	return []ServeBackend{{Key: name, Weight: 1, Endpoints: []Endpoint{ep}}}
+}
+
+func prefix(v string) HTTPRouteMatch {
+	return HTTPRouteMatch{Path: &HTTPPathMatch{Type: ptr(pathPrefix), Value: ptr(v)}}
+}
+
+func exact(v string) HTTPRouteMatch {
+	return HTTPRouteMatch{Path: &HTTPPathMatch{Type: ptr(pathExact), Value: ptr(v)}}
+}
+
+func get(t *testing.T, url, host string, hdr map[string]string) (int, string, http.Header) {
+	t.Helper()
+	req, _ := http.NewRequest("GET", url, nil)
+	if host != "" {
+		req.Host = host
+	}
+	for k, v := range hdr {
+		req.Header.Set(k, v)
+	}
+	c := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+	resp, err := c.Do(req)
+	if err != nil {
+		t.Fatalf("GET %s: %v", url, err)
+	}
+	defer resp.Body.Close()
+	b, _ := io.ReadAll(resp.Body)
+	return resp.StatusCode, string(b), resp.Header
+}
+
+func TestProxyRoutesByPrecedence(t *testing.T) {
+	tp := newTestProxy(t)
+	_, a := named(t, "A")
+	_, b := named(t, "B")
+	_, c := named(t, "C")
+	_, d := named(t, "D")
+	_, e := named(t, "E")
+	t0 := time.Date(2026, 10, 14, 1, 0, 0, 0, time.UTC)
+	routes := []ServeRoute{
+		{Key: "ns/first", Created: t0, Rules: []ServeRule{
+			{Matches: []HTTPRouteMatch{exact("/exact")}, Backends: backend("A", a)},
+			{Matches: []HTTPRouteMatch{prefix("/api")}, Backends: backend("B", b)},
+			{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("C", c)},
+		}},
+		{Key: "ns/second", Created: t0.Add(time.Minute), Rules: []ServeRule{
+			{Matches: []HTTPRouteMatch{prefix("/api/v2")}, Backends: backend("D", d)},
+			{Matches: []HTTPRouteMatch{{Path: &HTTPPathMatch{Type: ptr(pathPrefix), Value: ptr("/api")},
+				Headers: []HTTPHeaderMatch{{Name: "X-Ver", Value: "2"}}}}, Backends: backend("E", e)},
+		}},
+	}
+	if f := tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8080, Routes: routes}}); len(f) != 0 {
+		t.Fatal(f)
+	}
+	base := "http://" + tp.at(8080)
+	for path, want := range map[string]string{
+		"/exact": "A", "/exactly": "C", "/api/x": "B", "/api": "B", "/api/v2/y": "D", "/apix": "C", "/": "C",
+	} {
+		if _, body, _ := get(t, base+path, "example.com", nil); !strings.HasPrefix(body, want+" ") {
+			t.Errorf("%s went to %q, want %s", path, body, want)
+		}
+	}
+	// same prefix length: the match with more headers takes it, though its route is younger
+	if _, body, _ := get(t, base+"/api/x", "example.com", map[string]string{"X-Ver": "2"}); !strings.HasPrefix(body, "E ") {
+		t.Errorf("header match lost: %q", body)
+	}
+}
+
+func TestProxyHostPrecedence(t *testing.T) {
+	tp := newTestProxy(t)
+	_, w := named(t, "W")
+	_, x := named(t, "X")
+	routes := []ServeRoute{
+		{Key: "ns/wild", Hostnames: []string{"*.example.com"}, Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("W", w)}}},
+		{Key: "ns/exact", Hostnames: []string{"a.example.com"}, Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("X", x)}}},
+	}
+	tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8081, Routes: routes}})
+	base := "http://" + tp.at(8081)
+	for host, want := range map[string]string{"a.example.com": "X", "A.Example.com:8081": "X", "b.example.com": "W"} {
+		if _, body, _ := get(t, base+"/", host, nil); !strings.HasPrefix(body, want+" ") {
+			t.Errorf("host %s went to %q, want %s", host, body, want)
+		}
+	}
+	if code, _, _ := get(t, base+"/", "example.org", nil); code != http.StatusNotFound {
+		t.Errorf("a host no route names: %d, want 404", code)
+	}
+}
+
+func certSecret(t *testing.T, name, rv string) *coreapi.Secret {
+	s := tlsSecret(t, "gateway-system", strings.ReplaceAll(name, ".", "-"), name)
+	s.UID = types.UID("uid-" + name)
+	s.ResourceVersion = rv
+	return s
+}
+
+func dialTLS(t *testing.T, addr, sni string) (*x509.Certificate, error) {
+	conn, err := tls.Dial("tcp", addr, &tls.Config{ServerName: sni, InsecureSkipVerify: true})
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	return conn.ConnectionState().PeerCertificates[0], nil
+}
+
+func TestProxyTLSBySNIAndSwapsCertificates(t *testing.T) {
+	tp := newTestProxy(t)
+	_, a := named(t, "A")
+	_, b := named(t, "B")
+	listeners := func(certA *coreapi.Secret) []ServeListener {
+		return []ServeListener{
+			{Gateway: "ns/gw", Name: "a", Address: net.ParseIP("192.0.2.1"), Port: 8443, HTTPS: true, Hostname: "a.test", Cert: certA,
+				Routes: []ServeRoute{{Key: "ns/a", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a)}}}}},
+			{Gateway: "ns/gw", Name: "b", Address: net.ParseIP("192.0.2.1"), Port: 8443, HTTPS: true, Hostname: "b.test", Cert: certSecret(t, "b.test", "1"),
+				Routes: []ServeRoute{{Key: "ns/b", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("B", b)}}}}},
+		}
+	}
+	first := certSecret(t, "a.test", "1")
+	if f := tp.Apply(listeners(first)); len(f) != 0 {
+		t.Fatal(f)
+	}
+	addr := tp.at(8443)
+	ca, err := dialTLS(t, addr, "a.test")
+	if err != nil || ca.Subject.CommonName != "a.test" {
+		t.Fatalf("SNI a.test: %v %v", ca, err)
+	}
+	cb, err := dialTLS(t, addr, "b.test")
+	if err != nil || cb.Subject.CommonName != "b.test" {
+		t.Fatalf("SNI b.test: %v %v", cb, err)
+	}
+	if _, err := dialTLS(t, addr, "c.test"); err == nil {
+		t.Fatal("a name no listener has must not get a certificate")
+	}
+	// requests go by listener, over HTTP/2 when the client offers it
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true, ServerName: "b.test"}, ForceAttemptHTTP2: true}}
+	req, _ := http.NewRequest("GET", "https://"+addr+"/", nil)
+	req.Host = "b.test"
+	resp, err := client.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	if !strings.HasPrefix(string(body), "B ") || !strings.Contains(string(body), "proto=https") || resp.ProtoMajor != 2 {
+		t.Fatalf("over HTTPS to b.test: %s %q", resp.Proto, body)
+	}
+	// a renewed Secret: new handshakes get the new certificate on the same, still-bound port
+	renewed := certSecret(t, "a.test", "2")
+	tp.Apply(listeners(renewed))
+	if tp.at(8443) != addr {
+		t.Fatal("the port was rebound for a certificate change")
+	}
+	cn, err := dialTLS(t, addr, "a.test")
+	if err != nil || cn.SerialNumber.Cmp(ca.SerialNumber) == 0 {
+		t.Fatalf("still the old certificate after the swap: %v", err)
+	}
+}
+
+func TestProxyRedirectsAndForwards(t *testing.T) {
+	tp := newTestProxy(t)
+	_, a := named(t, "A")
+	routes := []ServeRoute{{Key: "ns/r", Rules: []ServeRule{
+		{Matches: []HTTPRouteMatch{prefix("/old")}, Filters: []HTTPRouteFilter{{Type: filterRequestRedirect,
+			RequestRedirect: &HTTPRequestRedirectFilter{Scheme: ptr("https"), StatusCode: ptr(301)}}}},
+		{Matches: []HTTPRouteMatch{prefix("/moved")}, Filters: []HTTPRouteFilter{{Type: filterRequestRedirect,
+			RequestRedirect: &HTTPRequestRedirectFilter{Path: &HTTPPathModifier{Type: "ReplacePrefixMatch", ReplacePrefixMatch: ptr("/new")}}}}},
+		{Matches: []HTTPRouteMatch{prefix("/v1")}, Backends: backend("A", a), Filters: []HTTPRouteFilter{
+			{Type: filterRequestHeaderModifier, RequestHeaderModifier: &HTTPHeaderFilter{
+				Set: []HTTPHeader{{Name: "X-Set", Value: "s"}}, Add: []HTTPHeader{{Name: "X-Added", Value: "two"}}, Remove: []string{"X-Remove-Me"}}},
+			{Type: filterResponseHeaderModifier, ResponseHeaderModifier: &HTTPHeaderFilter{Set: []HTTPHeader{{Name: "X-Resp", Value: "r"}}}},
+			{Type: filterURLRewrite, URLRewrite: &HTTPURLRewriteFilter{Path: &HTTPPathModifier{Type: "ReplacePrefixMatch", ReplacePrefixMatch: ptr("/v2")}}},
+		}},
+		{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a)},
+	}}}
+	tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 80, Routes: routes}})
+	base := "http://" + tp.at(80)
+	code, _, h := get(t, base+"/old/page?q=1", "git.levk.dev", nil)
+	if code != http.StatusMovedPermanently || h.Get("Location") != "https://git.levk.dev/old/page?q=1" {
+		t.Fatalf("scheme redirect: %d %q", code, h.Get("Location"))
+	}
+	code, _, h = get(t, base+"/moved/x", "git.levk.dev", nil)
+	if code != http.StatusFound || h.Get("Location") != "http://git.levk.dev/new/x" {
+		t.Fatalf("prefix redirect: %d %q", code, h.Get("Location"))
+	}
+	// the client's own claims about who it is are not passed on; the backend sees this hop's view
+	_, body, _ := get(t, base+"/", "git.levk.dev", map[string]string{"X-Forwarded-For": "6.6.6.6", "X-Real-IP": "6.6.6.6"})
+	if !strings.Contains(body, "host=git.levk.dev") || !strings.Contains(body, "xff=127.0.0.1 ") ||
+		!strings.Contains(body, "proto=http ") || !strings.Contains(body, "xhost=git.levk.dev") || !strings.Contains(body, "realip=127.0.0.1") {
+		t.Fatalf("forwarded headers: %q", body)
+	}
+	_, body, h = get(t, base+"/v1/thing", "git.levk.dev", map[string]string{"X-Added": "one", "X-Remove-Me": "x"})
+	if !strings.Contains(body, "path=/v2/thing") || !strings.Contains(body, "added=one,two") || !strings.Contains(body, "removed= ") || h.Get("X-Resp") != "r" {
+		t.Fatalf("filters: %q %v", body, h)
+	}
+}
+
+// The model refuses routes whose filters touch the forwarding headers; the proxy holds the line anyway.
+func TestProxyForwardingHeadersSurviveFilters(t *testing.T) {
+	tp := newTestProxy(t)
+	_, a := named(t, "A")
+	routes := []ServeRoute{{Key: "ns/r", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a),
+		Filters: []HTTPRouteFilter{{Type: filterRequestHeaderModifier, RequestHeaderModifier: &HTTPHeaderFilter{
+			Set:    []HTTPHeader{{Name: "X-Real-IP", Value: "6.6.6.6"}, {Name: "x-forwarded-host", Value: "evil.example"}},
+			Add:    []HTTPHeader{{Name: "X-Forwarded-For", Value: "6.6.6.6"}},
+			Remove: []string{"X-Forwarded-Proto"},
+		}}}}}}}
+	tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 80, Routes: routes}})
+	_, body, _ := get(t, "http://"+tp.at(80)+"/", "git.levk.dev", map[string]string{"X-Forwarded-For": "7.7.7.7"})
+	for _, want := range []string{"xff=127.0.0.1 ", "proto=http ", "xhost=git.levk.dev ", "realip=127.0.0.1 "} {
+		if !strings.Contains(body, want) {
+			t.Fatalf("a filter changed the forwarding headers: %q (want %q)", body, want)
+		}
+	}
+}
+
+func TestProxyAnswersWhatItCannotRoute(t *testing.T) {
+	tp := newTestProxy(t)
+	routes := []ServeRoute{{Key: "ns/r", Rules: []ServeRule{
+		{Matches: []HTTPRouteMatch{prefix("/broken")}, Broken: true},
+		{Matches: []HTTPRouteMatch{prefix("/empty")}, Backends: []ServeBackend{{Key: "ns/svc:80", Weight: 1}}},
+	}}}
+	tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8082, Routes: routes}})
+	base := "http://" + tp.at(8082)
+	for path, want := range map[string]int{"/broken": 500, "/empty": 503, "/nothing": 404} {
+		if code, _, _ := get(t, base+path, "x", nil); code != want {
+			t.Errorf("%s: %d, want %d", path, code, want)
+		}
+	}
+}
+
+func TestProxyStreamsLargeBodies(t *testing.T) {
+	tp := newTestProxy(t)
+	_, a := named(t, "A")
+	tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8083,
+		Routes: []ServeRoute{{Key: "ns/r", Rules: []ServeRule{{Matches: []HTTPRouteMatch{prefix("/")}, Backends: backend("A", a)}}}}}})
+	const size = 32 << 20
+	resp, err := http.Post("http://"+tp.at(8083)+"/upload", "application/octet-stream", bytes.NewReader(make([]byte, size)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	if !strings.Contains(string(body), fmt.Sprintf("body=%d", size)) {
+		t.Fatalf("upload: %q", body)
+	}
+}
+
+func TestProxyBindFailureAndClose(t *testing.T) {
+	tp := newTestProxy(t)
+	tp.listen = func(addr string) (net.Listener, error) { return nil, fmt.Errorf("address in use") }
+	f := tp.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8084}})
+	if f["ns/gw/http"] == nil {
+		t.Fatalf("bind failure not reported: %v", f)
+	}
+	tp2 := newTestProxy(t)
+	tp2.Apply([]ServeListener{{Gateway: "ns/gw", Name: "http", Address: net.ParseIP("192.0.2.1"), Port: 8085}})
+	addr := tp2.at(8085)
+	tp2.Apply(nil)
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		c, err := net.DialTimeout("tcp", addr, 200*time.Millisecond)
+		if err != nil {
+			return
+		}
+		c.Close()
+		time.Sleep(50 * time.Millisecond)
+	}
+	t.Fatal("a port no longer served still accepts connections")
+}

lb/gateway_test.go

@@ -0,0 +1,611 @@
+package lb
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var gwNow = metav1.NewTime(time.Date(2026, 10, 14, 4, 0, 0, 0, time.UTC))
+
+// selfSigned is a certificate and key for the given names, PEM-encoded.
+func selfSigned(t *testing.T, names ...string) ([]byte, []byte) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: names[0]}, DNSNames: names,
+		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(24 * time.Hour),
+		KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	kb, _ := x509.MarshalECPrivateKey(key)
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: kb})
+}
+
+func tlsSecret(t *testing.T, namespace, name string, names ...string) *coreapi.Secret {
+	crt, key := selfSigned(t, names...)
+	return &coreapi.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}, Type: coreapi.SecretTypeTLS,
+		Data: map[string][]byte{coreapi.TLSCertKey: crt, coreapi.TLSPrivateKeyKey: key}}
+}
+
+func gwNodes() []Node {
+	return []Node{
+		{Name: "cp", Addresses: []net.IP{net.ParseIP("86.107.168.24").To4()}, WeaveIP: net.ParseIP("10.32.0.1").To4(), WeaveMAC: mustMAC("02:00:00:00:00:01")},
+		{Name: "w15", Addresses: []net.IP{net.ParseIP("86.107.168.15").To4()}, WeaveIP: net.ParseIP("10.44.0.0").To4(), WeaveMAC: mustMAC("02:00:00:00:00:02")},
+	}
+}
+
+func mustMAC(s string) net.HardwareAddr {
+	m, _ := net.ParseMAC(s)
+	return m
+}
+
+func ourClass() *GatewayClass {
+	return &GatewayClass{ObjectMeta: metav1.ObjectMeta{Name: "reweave", Generation: 1}, Spec: GatewayClassSpec{ControllerName: GatewayControllerName}}
+}
+
+// giteaGateway is the shape Gitea's documentation prescribes: HTTPS with a certificate, plain HTTP, and SSH over TCP.
+func giteaGateway() *Gateway {
+	return &Gateway{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "gateway-system", Name: "shared", Generation: 3},
+		Spec: GatewaySpec{
+			GatewayClassName: "reweave",
+			Addresses:        []GatewayAddress{{Value: "86.107.168.15"}},
+			Listeners: []Listener{
+				{Name: "https-gitea", Port: 443, Protocol: ProtocolHTTPS, Hostname: ptr("git.levk.dev"),
+					TLS:           &ListenerTLSConfig{CertificateRefs: []SecretObjectReference{{Name: "git-levk-dev"}}},
+					AllowedRoutes: &AllowedRoutes{Namespaces: &RouteNamespaces{From: ptr("Selector"), Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitea"}}}}},
+				{Name: "http", Port: 80, Protocol: ProtocolHTTP, Hostname: ptr("git.levk.dev"),
+					AllowedRoutes: &AllowedRoutes{Namespaces: &RouteNamespaces{From: ptr("All")}}},
+				{Name: "ssh", Port: 22, Protocol: ProtocolTCP,
+					AllowedRoutes: &AllowedRoutes{Namespaces: &RouteNamespaces{From: ptr("All")}}},
+			},
+		},
+	}
+}
+
+func giteaService() *coreapi.Service {
+	return &coreapi.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea"}, Spec: coreapi.ServiceSpec{
+		Ports: []coreapi.ServicePort{{Name: "http", Port: 3000}, {Name: "ssh", Port: 22}},
+	}}
+}
+
+func giteaSlice() *discovery.EndpointSlice {
+	return &discovery.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Namespace: "gitea", Name: "gitea-abc", Labels: map[string]string{serviceNameLabel: "gitea"}},
+		AddressType: discovery.AddressTypeIPv4,
+		Ports:       []discovery.EndpointPort{{Name: ptr("http"), Port: ptr(int32(3000))}, {Name: ptr("ssh"), Port: ptr(int32(22))}},
+		Endpoints:   []discovery.Endpoint{{Addresses: []string{"10.44.0.12"}, Conditions: discovery.EndpointConditions{Ready: ptr(true)}}},
+	}
+}
+
+func giteaRoutes() ([]*HTTPRoute, []*TCPRoute) {
+	t0 := metav1.NewTime(time.Date(2026, 10, 14, 1, 0, 0, 0, time.UTC))
+	https := &HTTPRoute{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea", Generation: 1, CreationTimestamp: t0},
+		Spec: HTTPRouteSpec{
+			ParentRefs: []ParentReference{{Namespace: ptr("gateway-system"), Name: "shared", SectionName: ptr("https-gitea")}},
+			Hostnames:  []string{"git.levk.dev"},
+			Rules:      []HTTPRouteRule{{BackendRefs: []HTTPBackendRef{{BackendRef: BackendRef{Name: "gitea", Port: ptr(int32(3000))}}}}},
+		},
+	}
+	redirect := &HTTPRoute{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea-redirect", Generation: 1, CreationTimestamp: t0},
+		Spec: HTTPRouteSpec{
+			ParentRefs: []ParentReference{{Namespace: ptr("gateway-system"), Name: "shared", SectionName: ptr("http")}},
+			Rules: []HTTPRouteRule{{Filters: []HTTPRouteFilter{{Type: filterRequestRedirect,
+				RequestRedirect: &HTTPRequestRedirectFilter{Scheme: ptr("https"), StatusCode: ptr(301)}}}}},
+		},
+	}
+	ssh := &TCPRoute{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gitea-ssh", Generation: 1, CreationTimestamp: t0},
+		Spec: TCPRouteSpec{
+			ParentRefs: []ParentReference{{Namespace: ptr("gateway-system"), Name: "shared", SectionName: ptr("ssh")}},
+			Rules:      []TCPRouteRule{{BackendRefs: []BackendRef{{Name: "gitea", Port: ptr(int32(22))}}}},
+		},
+	}
+	return []*HTTPRoute{https, redirect}, []*TCPRoute{ssh}
+}
+
+func giteaInput(t *testing.T) GatewayInput {
+	hr, tr := giteaRoutes()
+	return GatewayInput{
+		Classes: []*GatewayClass{ourClass()}, Gateways: []*Gateway{giteaGateway()}, HTTPRoutes: hr, TCPRoutes: tr,
+		Namespaces: []*coreapi.Namespace{
+			{ObjectMeta: metav1.ObjectMeta{Name: "gitea", Labels: map[string]string{"kubernetes.io/metadata.name": "gitea"}}},
+			{ObjectMeta: metav1.ObjectMeta{Name: "gateway-system", Labels: map[string]string{"kubernetes.io/metadata.name": "gateway-system"}}},
+		},
+		Services: []*coreapi.Service{giteaService()}, Slices: []*discovery.EndpointSlice{giteaSlice()},
+		Secrets:          []*coreapi.Secret{tlsSecret(t, "gateway-system", "git-levk-dev", "git.levk.dev")},
+		SecretNamespaces: map[string]bool{"gateway-system": true},
+		Now:              gwNow,
+	}
+}
+
+func condOf(cs []metav1.Condition, typ string) metav1.Condition {
+	for _, c := range cs {
+		if c.Type == typ {
+			return c
+		}
+	}
+	return metav1.Condition{}
+}
+
+func TestGatewayServedByItsHolder(t *testing.T) {
+	in := giteaInput(t)
+	d := ComputeGateways("w15", gwNodes(), in)
+	if len(d.Problems) != 0 {
+		t.Fatalf("problems: %v", d.Problems)
+	}
+	// HTTPS and HTTP served by the agent; SSH forwarded in the kernel to Gitea's pod with the client address intact
+	if len(d.Serve) != 2 {
+		t.Fatalf("want the https and http listeners served, got %+v", d.Serve)
+	}
+	https := d.Serve[0]
+	if !https.HTTPS || https.Port != 443 || https.Hostname != "git.levk.dev" || https.Cert == nil || len(https.Routes) != 1 {
+		t.Fatalf("https listener: %+v", https)
+	}
+	be := https.Routes[0].Rules[0].Backends
+	if len(be) != 1 || be[0].Key != "gitea/gitea:3000" || len(be[0].Endpoints) != 1 || be[0].Endpoints[0].String() != "10.44.0.12:3000" {
+		t.Fatalf("https backends: %+v", be)
+	}
+	if len(d.Forwards) != 1 || d.Forwards[0].Ports[0].Port != 22 || d.Forwards[0].Ports[0].Endpoints[0].String() != "10.44.0.12:22" {
+		t.Fatalf("ssh forward: %+v", d.Forwards)
+	}
+	if len(d.Holding) != 1 || d.Holding[0] != "w15" {
+		t.Fatalf("holding: %v", d.Holding)
+	}
+	// the holder writes the Gateway's status: accepted, programmed, its address, each listener with one route
+	if len(d.Gateways) != 1 {
+		t.Fatalf("gateway status writes: %+v", d.Gateways)
+	}
+	st := d.Gateways[0].Status
+	if c := condOf(st.Conditions, condProgrammed); c.Status != metav1.ConditionTrue || c.ObservedGeneration != 3 {
+		t.Fatalf("programmed: %+v", c)
+	}
+	if len(st.Addresses) != 1 || st.Addresses[0].Value != "86.107.168.15" {
+		t.Fatalf("addresses: %+v", st.Addresses)
+	}
+	for _, l := range st.Listeners {
+		if l.AttachedRoutes != 1 || condOf(l.Conditions, condProgrammed).Status != metav1.ConditionTrue {
+			t.Fatalf("listener %s: %+v", l.Name, l)
+		}
+	}
+	// route statuses come from the first node by name, not the holder
+	if len(d.Routes) != 0 || len(d.Classes) != 0 {
+		t.Fatalf("w15 wrote route or class statuses: %+v %+v", d.Routes, d.Classes)
+	}
+}
+
+func TestGatewayElsewhere(t *testing.T) {
+	in := giteaInput(t)
+	d := ComputeGateways("cp", gwNodes(), in)
+	if len(d.Serve) != 0 || len(d.Forwards) != 0 || len(d.Gateways) != 0 {
+		t.Fatalf("cp serves or writes what w15 holds: %+v %+v %+v", d.Serve, d.Forwards, d.Gateways)
+	}
+	// but steers replies of w15's forwards back to it, and writes class and route statuses (first by name)
+	if len(d.Holding) != 1 || d.Holding[0] != "w15" {
+		t.Fatalf("holding: %v", d.Holding)
+	}
+	if len(d.Classes) != 1 || condOf(d.Classes[0].Status.Conditions, condAccepted).Status != metav1.ConditionTrue {
+		t.Fatalf("class status: %+v", d.Classes)
+	}
+	if len(d.Routes) != 3 {
+		t.Fatalf("want 3 route statuses, got %+v", d.Routes)
+	}
+	for _, r := range d.Routes {
+		if len(r.Status.Parents) != 1 || condOf(r.Status.Parents[0].Conditions, condAccepted).Status != metav1.ConditionTrue ||
+			condOf(r.Status.Parents[0].Conditions, condResolvedRefs).Status != metav1.ConditionTrue {
+			t.Fatalf("route %s/%s status: %+v", r.Namespace, r.Name, r.Status)
+		}
+	}
+}
+
+func TestGatewayStatusIsWrittenOnce(t *testing.T) {
+	in := giteaInput(t)
+	d := ComputeGateways("w15", gwNodes(), in)
+	in.Gateways[0].Status = d.Gateways[0].Status
+	in.Now = metav1.NewTime(gwNow.Add(time.Hour)) // later passes see the status they wrote
+	if again := ComputeGateways("w15", gwNodes(), in); len(again.Gateways) != 0 {
+		t.Fatalf("unchanged status rewritten: %+v", again.Gateways)
+	}
+	c := ComputeGateways("cp", gwNodes(), in)
+	for _, r := range c.Routes {
+		for _, hr := range in.HTTPRoutes {
+			if hr.Name == r.Name {
+				hr.Status = r.Status
+			}
+		}
+		for _, tr := range in.TCPRoutes {
+			if tr.Name == r.Name {
+				tr.Status = r.Status
+			}
+		}
+	}
+	in.Classes[0].Status = c.Classes[0].Status
+	if again := ComputeGateways("cp", gwNodes(), in); len(again.Routes) != 0 || len(again.Classes) != 0 {
+		t.Fatalf("unchanged route or class status rewritten: %+v %+v", again.Routes, again.Classes)
+	}
+}
+
+func TestGatewayFailsClosedWithoutHolder(t *testing.T) {
+	in := giteaInput(t)
+	in.Gateways[0].Spec.Addresses[0].Value = "198.51.100.7"
+	for _, self := range []string{"cp", "w15"} {
+		d := ComputeGateways(self, gwNodes(), in)
+		if len(d.Serve) != 0 || len(d.Forwards) != 0 || len(d.Holding) != 0 {
+			t.Fatalf("%s serves an address nobody holds: %+v", self, d)
+		}
+	}
+	d := ComputeGateways("cp", gwNodes(), in)
+	if len(d.Gateways) != 1 {
+		t.Fatalf("the first node by name must write the failure: %+v", d.Gateways)
+	}
+	c := condOf(d.Gateways[0].Status.Conditions, condProgrammed)
+	if c.Status != metav1.ConditionFalse || c.Reason != "AddressNotUsable" || len(d.Gateways[0].Status.Addresses) != 0 {
+		t.Fatalf("status: %+v", d.Gateways[0].Status)
+	}
+}
+
+func TestGatewayAddressMustBeOneIPv4(t *testing.T) {
+	in := giteaInput(t)
+	in.Gateways[0].Spec.Addresses = nil
+	d := ComputeGateways("cp", gwNodes(), in)
+	if c := condOf(d.Gateways[0].Status.Conditions, condAccepted); c.Status != metav1.ConditionFalse || c.Reason != "UnsupportedAddress" {
+		t.Fatalf("accepted: %+v", c)
+	}
+}
+
+func TestListenerCertificateReferences(t *testing.T) {
+	cases := map[string]struct {
+		mutate func(in *GatewayInput)
+		reason string
+	}{
+		"missing Secret": {func(in *GatewayInput) { in.Secrets = nil }, "InvalidCertificateRef"},
+		"other namespace without grant": {func(in *GatewayInput) {
+			in.Gateways[0].Spec.Listeners[0].TLS.CertificateRefs[0].Namespace = ptr("gitea")
+			in.SecretNamespaces["gitea"] = true
+		}, "RefNotPermitted"},
+		"namespace the agent cannot read": {func(in *GatewayInput) {
+			in.SecretNamespaces = map[string]bool{}
+		}, "RefNotPermitted"},
+		"not a certificate": {func(in *GatewayInput) {
+			in.Secrets[0].Data[coreapi.TLSCertKey] = []byte("nope")
+		}, "InvalidCertificateRef"},
+	}
+	for name, c := range cases {
+		in := giteaInput(t)
+		c.mutate(&in)
+		d := ComputeGateways("w15", gwNodes(), in)
+		var https ListenerStatus
+		for _, l := range d.Gateways[0].Status.Listeners {
+			if l.Name == "https-gitea" {
+				https = l
+			}
+		}
+		if r := condOf(https.Conditions, condResolvedRefs); r.Status != metav1.ConditionFalse || r.Reason != c.reason {
+			t.Errorf("%s: resolvedRefs %+v", name, r)
+		}
+		if p := condOf(https.Conditions, condProgrammed); p.Status != metav1.ConditionFalse {
+			t.Errorf("%s: an HTTPS listener without its certificate must not be programmed", name)
+		}
+		for _, s := range d.Serve {
+			if s.HTTPS {
+				t.Errorf("%s: HTTPS listener served without a certificate", name)
+			}
+		}
+		if a := condOf(d.Gateways[0].Status.Conditions, condAccepted); a.Reason != "ListenersNotValid" || a.Status != metav1.ConditionTrue {
+			t.Errorf("%s: gateway accepted %+v", name, a)
+		}
+	}
+}
+
+func TestCrossNamespaceCertificateWithGrant(t *testing.T) {
+	in := giteaInput(t)
+	in.Gateways[0].Spec.Listeners[0].TLS.CertificateRefs[0].Namespace = ptr("gitea")
+	in.Secrets = []*coreapi.Secret{tlsSecret(t, "gitea", "git-levk-dev", "git.levk.dev")}
+	in.SecretNamespaces = map[string]bool{"gitea": true}
+	in.Grants = []*ReferenceGrant{{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: "gw"}, Spec: ReferenceGrantSpec{
+		From: []ReferenceGrantFrom{{Group: GatewayGroup, Kind: kindGateway, Namespace: "gateway-system"}},
+		To:   []ReferenceGrantTo{{Group: "", Kind: kindSecret}},
+	}}}
+	d := ComputeGateways("w15", gwNodes(), in)
+	if len(d.Serve) != 2 || d.Serve[0].Cert == nil {
+		t.Fatalf("a granted certificate in another namespace must be served: %+v", d.Serve)
+	}
+}
+
+func TestListenerConflicts(t *testing.T) {
+	in := giteaInput(t)
+	l := in.Gateways[0].Spec.Listeners
+	in.Gateways[0].Spec.Listeners = append(l,
+		Listener{Name: "ssh2", Port: 22, Protocol: ProtocolTCP},          // two TCP listeners on 22
+		Listener{Name: "http-on-443", Port: 443, Protocol: ProtocolHTTP}, // HTTP and HTTPS on 443
+	)
+	d := ComputeGateways("w15", gwNodes(), in)
+	want := map[string]string{"ssh": "HostnameConflict", "ssh2": "HostnameConflict", "https-gitea": "ProtocolConflict", "http-on-443": "ProtocolConflict", "http": ""}
+	for _, ls := range d.Gateways[0].Status.Listeners {
+		c := condOf(ls.Conditions, condConflicted)
+		if want[ls.Name] == "" {
+			if c.Status != metav1.ConditionFalse {
+				t.Errorf("%s: conflicted %+v", ls.Name, c)
+			}
+			continue
+		}
+		if c.Status != metav1.ConditionTrue || c.Reason != want[ls.Name] {
+			t.Errorf("%s: want conflicted %s, got %+v", ls.Name, want[ls.Name], c)
+		}
+	}
+	if len(d.Forwards) != 0 {
+		t.Fatalf("conflicted TCP listeners forwarded: %+v", d.Forwards)
+	}
+	if len(d.Serve) != 1 || d.Serve[0].Name != "http" {
+		t.Fatalf("only the plain http listener on 80 is unconflicted: %+v", d.Serve)
+	}
+}
+
+func TestListenerPortTakenByLoadBalancerService(t *testing.T) {
+	in := giteaInput(t)
+	class := Class
+	in.Services = append(in.Services, &coreapi.Service{ObjectMeta: metav1.ObjectMeta{Namespace: "mail", Name: "smtp"}, Spec: coreapi.ServiceSpec{
+		Type: coreapi.ServiceTypeLoadBalancer, LoadBalancerClass: &class, LoadBalancerIP: "86.107.168.15",
+		Ports: []coreapi.ServicePort{{Port: 22}},
+	}})
+	d := ComputeGateways("w15", gwNodes(), in)
+	for _, ls := range d.Gateways[0].Status.Listeners {
+		if ls.Name == "ssh" {
+			if c := condOf(ls.Conditions, condAccepted); c.Status != metav1.ConditionFalse || c.Reason != "PortUnavailable" {
+				t.Fatalf("ssh accepted: %+v", c)
+			}
+		}
+	}
+	if len(d.Forwards) != 0 {
+		t.Fatalf("a listener on a port a Service forwards must not be forwarded: %+v", d.Forwards)
+	}
+}
+
+func TestRouteAttachment(t *testing.T) {
+	cases := map[string]struct {
+		mutate func(in *GatewayInput)
+		route  string
+		reason string
+	}{
+		"unknown section":                 {func(in *GatewayInput) { in.HTTPRoutes[0].Spec.ParentRefs[0].SectionName = ptr("nope") }, "gitea", "NoMatchingParent"},
+		"namespace not selected":          {func(in *GatewayInput) { in.Namespaces[0].Labels = nil }, "gitea", "NotAllowedByListeners"},
+		"hostname outside the listener's": {func(in *GatewayInput) { in.HTTPRoutes[0].Spec.Hostnames = []string{"git.example.net"} }, "gitea", "NoMatchingListenerHostname"},
+		"unsupported filter": {func(in *GatewayInput) {
+			in.HTTPRoutes[0].Spec.Rules[0].Filters = []HTTPRouteFilter{{Type: "RequestMirror"}}
+		}, "gitea", "UnsupportedValue"},
+		"regular expression path": {func(in *GatewayInput) {
+			in.HTTPRoutes[0].Spec.Rules[0].Matches = []HTTPRouteMatch{{Path: &HTTPPathMatch{Type: ptr("RegularExpression"), Value: ptr("/.*")}}}
+		}, "gitea", "UnsupportedValue"},
+		"filter sets X-Forwarded-For": {func(in *GatewayInput) {
+			in.HTTPRoutes[0].Spec.Rules[0].Filters = []HTTPRouteFilter{{Type: filterRequestHeaderModifier,
+				RequestHeaderModifier: &HTTPHeaderFilter{Set: []HTTPHeader{{Name: "X-Forwarded-For", Value: "6.6.6.6"}}}}}
+		}, "gitea", "UnsupportedValue"},
+		"backend filter adds x-real-ip": {func(in *GatewayInput) {
+			in.HTTPRoutes[0].Spec.Rules[0].BackendRefs[0].Filters = []HTTPRouteFilter{{Type: filterRequestHeaderModifier,
+				RequestHeaderModifier: &HTTPHeaderFilter{Add: []HTTPHeader{{Name: "x-real-ip", Value: "6.6.6.6"}}}}}
+		}, "gitea", "UnsupportedValue"},
+		"filter removes Forwarded": {func(in *GatewayInput) {
+			in.HTTPRoutes[0].Spec.Rules[0].Filters = []HTTPRouteFilter{{Type: filterRequestHeaderModifier,
+				RequestHeaderModifier: &HTTPHeaderFilter{Remove: []string{"FORWARDED"}}}}
+		}, "gitea", "UnsupportedValue"},
+		"wrong port": {func(in *GatewayInput) { in.TCPRoutes[0].Spec.ParentRefs[0].Port = ptr(int32(2222)) }, "gitea-ssh", "NoMatchingParent"},
+	}
+	for name, c := range cases {
+		in := giteaInput(t)
+		c.mutate(&in)
+		d := ComputeGateways("cp", gwNodes(), in)
+		found := false
+		for _, r := range d.Routes {
+			if r.Name != c.route {
+				continue
+			}
+			found = true
+			a := condOf(r.Status.Parents[0].Conditions, condAccepted)
+			if a.Status != metav1.ConditionFalse || a.Reason != c.reason {
+				t.Errorf("%s: accepted %+v", name, a)
+			}
+		}
+		if !found {
+			t.Errorf("%s: no status for route %s", name, c.route)
+		}
+		h := ComputeGateways("w15", gwNodes(), in)
+		if c.route == "gitea" {
+			for _, s := range h.Serve {
+				if s.Name == "https-gitea" && len(s.Routes) != 0 {
+					t.Errorf("%s: a route that is not accepted was attached", name)
+				}
+			}
+		}
+	}
+}
+
+func TestForwardingHeadersAreTheGateways(t *testing.T) {
+	accepted := func(name string, value string) metav1.Condition {
+		in := giteaInput(t)
+		in.HTTPRoutes[0].Spec.Rules[0].Filters = []HTTPRouteFilter{{Type: filterRequestHeaderModifier,
+			RequestHeaderModifier: &HTTPHeaderFilter{Set: []HTTPHeader{{Name: name, Value: value}}}}}
+		for _, r := range ComputeGateways("cp", gwNodes(), in).Routes {
+			if r.Name == "gitea" {
+				return condOf(r.Status.Parents[0].Conditions, condAccepted)
+			}
+		}
+		t.Fatalf("no status for route gitea")
+		return metav1.Condition{}
+	}
+	a := accepted("x-forwarded-proto", "https")
+	if a.Status != metav1.ConditionFalse || a.Reason != "UnsupportedValue" || !strings.Contains(a.Message, "X-Forwarded-Proto") {
+		t.Fatalf("a filter on X-Forwarded-Proto: %+v", a)
+	}
+	// a header that only looks like one of them is the route's to set
+	if a := accepted("X-Forwarded-Prefix", "/git"); a.Status != metav1.ConditionTrue {
+		t.Fatalf("a filter on X-Forwarded-Prefix: %+v", a)
+	}
+}
+
+func TestBackendNotFoundBreaksTheRule(t *testing.T) {
+	in := giteaInput(t)
+	in.HTTPRoutes[0].Spec.Rules[0].BackendRefs[0].Name = "nope"
+	d := ComputeGateways("cp", gwNodes(), in)
+	for _, r := range d.Routes {
+		if r.Name == "gitea" {
+			if c := condOf(r.Status.Parents[0].Conditions, condResolvedRefs); c.Status != metav1.ConditionFalse || c.Reason != "BackendNotFound" {
+				t.Fatalf("resolvedRefs: %+v", c)
+			}
+			if c := condOf(r.Status.Parents[0].Conditions, condAccepted); c.Status != metav1.ConditionTrue {
+				t.Fatalf("a route with an unresolved backend is still accepted: %+v", c)
+			}
+		}
+	}
+	h := ComputeGateways("w15", gwNodes(), in)
+	for _, s := range h.Serve {
+		if s.Name == "https-gitea" && (len(s.Routes) != 1 || !s.Routes[0].Rules[0].Broken) {
+			t.Fatalf("the rule must be served broken (500): %+v", s.Routes)
+		}
+	}
+}
+
+func TestTCPListenerForwardsTheOldestRoute(t *testing.T) {
+	in := giteaInput(t)
+	later := in.TCPRoutes[0].DeepCopyObject().(*TCPRoute)
+	later.Name = "other-ssh"
+	later.CreationTimestamp = metav1.NewTime(later.CreationTimestamp.Add(time.Minute))
+	later.Spec.Rules[0].BackendRefs[0].Name = "other"
+	in.TCPRoutes = append(in.TCPRoutes, later)
+	d := ComputeGateways("cp", gwNodes(), in)
+	for _, r := range d.Routes {
+		a := condOf(r.Status.Parents[0].Conditions, condAccepted)
+		switch r.Name {
+		case "gitea-ssh":
+			if a.Status != metav1.ConditionTrue {
+				t.Fatalf("oldest route: %+v", a)
+			}
+		case "other-ssh":
+			if a.Status != metav1.ConditionFalse || !strings.Contains(a.Message, "gitea/gitea-ssh") {
+				t.Fatalf("second route: %+v", a)
+			}
+		}
+	}
+}
+
+func TestOtherControllersParentsKept(t *testing.T) {
+	in := giteaInput(t)
+	theirs := RouteParentStatus{ParentRef: ParentReference{Name: "elsewhere"}, ControllerName: "example.com/other",
+		Conditions: []metav1.Condition{{Type: condAccepted, Status: metav1.ConditionTrue, Reason: "Accepted", LastTransitionTime: gwNow}}}
+	in.HTTPRoutes[0].Status.Parents = []RouteParentStatus{theirs}
+	d := ComputeGateways("cp", gwNodes(), in)
+	for _, r := range d.Routes {
+		if r.Name == "gitea" {
+			if len(r.Status.Parents) != 2 || r.Status.Parents[0].ControllerName != "example.com/other" {
+				t.Fatalf("another controller's entry must be kept: %+v", r.Status.Parents)
+			}
+		}
+	}
+}
+
+func TestOtherClassesIgnored(t *testing.T) {
+	in := giteaInput(t)
+	in.Classes[0].Spec.ControllerName = "example.com/other"
+	for _, self := range []string{"cp", "w15"} {
+		d := ComputeGateways(self, gwNodes(), in)
+		if len(d.Serve)+len(d.Forwards)+len(d.Gateways)+len(d.Routes)+len(d.Classes) != 0 {
+			t.Fatalf("%s acted on another controller's class: %+v", self, d)
+		}
+	}
+}
+
+func TestNarrowHostnames(t *testing.T) {
+	cases := []struct {
+		listener string
+		route    []string
+		want     []string
+		ok       bool
+	}{
+		{"", nil, nil, true},
+		{"git.levk.dev", nil, []string{"git.levk.dev"}, true},
+		{"", []string{"a.example.com"}, []string{"a.example.com"}, true},
+		{"test.example.com", []string{"*.example.com"}, []string{"test.example.com"}, true},
+		{"*.example.com", []string{"test.example.com", "test.example.net", "foo.test.example.com"}, []string{"test.example.com", "foo.test.example.com"}, true},
+		{"*.example.com", []string{"example.com"}, nil, false},
+		{"*.example.com", []string{"*.a.example.com"}, []string{"*.a.example.com"}, true},
+		{"*.a.example.com", []string{"*.example.com"}, []string{"*.a.example.com"}, true},
+		{"git.levk.dev", []string{"git.example.net"}, nil, false},
+	}
+	for _, c := range cases {
+		got, ok := narrowHostnames(c.listener, c.route)
+		if ok != c.ok || strings.Join(got, ",") != strings.Join(c.want, ",") {
+			t.Errorf("listener %q route %v: got %v %v, want %v %v", c.listener, c.route, got, ok, c.want, c.ok)
+		}
+	}
+}
+
+func TestMergeGateways(t *testing.T) {
+	in := giteaInput(t)
+	nodes := gwNodes()
+	// on the holder: Gitea's SSH forwarded, its HTTP(S) ports let in
+	d := MergeGateways(Compute("w15", nodes, nil, nil), ComputeGateways("w15", nodes, in), "w15", nodes)
+	if len(d.Forwards) != 1 || len(d.Accept) != 2 || len(d.Holders) != 0 {
+		t.Fatalf("holder: forwards %+v accept %+v holders %+v", d.Forwards, d.Accept, d.Holders)
+	}
+	r := Render(d, EgressDesired{}, "w15", &net.IPNet{IP: net.ParseIP("10.32.0.0"), Mask: net.CIDRMask(12, 32)}, nil)
+	for _, want := range []string{
+		"-A REWEAVE-LB-IN -d 86.107.168.15/32 -p tcp -m tcp --dport 443",
+		"-A REWEAVE-LB-IN -d 86.107.168.15/32 -p tcp -m tcp --dport 80",
+	} {
+		if !strings.Contains(r.Filter, want) {
+			t.Errorf("filter lacks %q:\n%s", want, r.Filter)
+		}
+	}
+	if !strings.Contains(r.Nat, "-A REWEAVE-LB -d 86.107.168.15/32 -p tcp -m tcp --dport 22") || !strings.Contains(r.Nat, "--to-destination 10.44.0.12:22") {
+		t.Errorf("nat lacks the SSH forward:\n%s", r.Nat)
+	}
+	// elsewhere: w15 is a holder whose replies come back to it
+	e := MergeGateways(Compute("cp", nodes, nil, nil), ComputeGateways("cp", nodes, in), "cp", nodes)
+	if len(e.Forwards) != 0 || len(e.Accept) != 0 || len(e.Holders) != 1 || e.Holders[0].Node.Name != "w15" {
+		t.Fatalf("cp: forwards %+v accept %+v holders %+v", e.Forwards, e.Accept, e.Holders)
+	}
+}
+
+func TestUnboundListenerReported(t *testing.T) {
+	in := giteaInput(t)
+	in.Unbound = map[string]string{"gateway-system/shared/https-gitea": "address already in use"}
+	d := ComputeGateways("w15", gwNodes(), in)
+	for _, l := range d.Gateways[0].Status.Listeners {
+		if l.Name != "https-gitea" {
+			continue
+		}
+		if a := condOf(l.Conditions, condAccepted); a.Status != metav1.ConditionFalse || a.Reason != "PortUnavailable" {
+			t.Fatalf("accepted: %+v", a)
+		}
+		if p := condOf(l.Conditions, condProgrammed); p.Status != metav1.ConditionFalse {
+			t.Fatalf("programmed: %+v", p)
+		}
+	}
+	served := false
+	for _, s := range d.Serve {
+		served = served || s.Name == "https-gitea"
+	}
+	if !served {
+		t.Fatal("an unbound listener is still tried on every pass")
+	}
+}

lb/model.go

@@ -0,0 +1,318 @@
+// Package lb gives LoadBalancer Services an address that a node holds, and
+// forwards that address's traffic to the Service's endpoints with the
+// client's address intact.
+//
+// A Service opts in with spec.loadBalancerClass = Class and names its
+// address in spec.loadBalancerIP. The node that holds the address (has it on
+// one of its interfaces, as published in its AnnAddresses annotation) is the
+// address's holder: it alone writes the Service's status (ipMode Proxy, so
+// kube-proxy installs no rules for the address) and DNATs the address's
+// ports to ready endpoints, without SNAT. Every other node marks the
+// connections that the holder forwards to its pods, and routes their replies
+// back to the holder over weave, so that the holder's conntrack can undo the
+// DNAT and the reply leaves from the address the client connected to.
+//
+// Compute is the whole policy as a pure function of what the API shows; the
+// datapath and the controller only carry it out.
+package lb
+
+import (
+	"fmt"
+	"hash/fnv"
+	"net"
+	"sort"
+	"strings"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+)
+
+const (
+	// Class is the spec.loadBalancerClass a Service sets to be served by this agent.
+	Class = "reweave.levk.dev/node-address"
+	// AnnAddresses lists the IPv4 addresses a node holds on its host interfaces (comma-separated).
+	AnnAddresses = "reweave.levk.dev/addresses"
+	// AnnWeave is "<ip>,<mac>" of the node's weave bridge: where replies to its forwards go.
+	AnnWeave = "reweave.levk.dev/weave"
+
+	serviceNameLabel = "kubernetes.io/service-name"
+)
+
+// Node is what one node publishes about itself.
+type Node struct {
+	Name      string
+	Addresses []net.IP
+	WeaveIP   net.IP
+	WeaveMAC  net.HardwareAddr
+}
+
+// Holds reports whether the node has ip on one of its host interfaces.
+func (n Node) Holds(ip net.IP) bool {
+	for _, a := range n.Addresses {
+		if a.Equal(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// ParseNode reads a Node's annotations. A node that has published nothing
+// (yet) holds nothing and cannot take replies.
+func ParseNode(n *coreapi.Node) (Node, error) {
+	out := Node{Name: n.Name}
+	if s := n.Annotations[AnnAddresses]; s != "" {
+		for _, f := range strings.Split(s, ",") {
+			ip := net.ParseIP(strings.TrimSpace(f)).To4()
+			if ip == nil {
+				return out, fmt.Errorf("node %s: %s: %q is not an IPv4 address", n.Name, AnnAddresses, f)
+			}
+			out.Addresses = append(out.Addresses, ip)
+		}
+	}
+	if s := n.Annotations[AnnWeave]; s != "" {
+		f := strings.Split(s, ",")
+		if len(f) != 2 {
+			return out, fmt.Errorf("node %s: %s: want \"<ip>,<mac>\", got %q", n.Name, AnnWeave, s)
+		}
+		ip := net.ParseIP(f[0]).To4()
+		mac, err := net.ParseMAC(f[1])
+		if ip == nil || err != nil {
+			return out, fmt.Errorf("node %s: %s: want \"<ip>,<mac>\", got %q", n.Name, AnnWeave, s)
+		}
+		out.WeaveIP, out.WeaveMAC = ip, mac
+	}
+	return out, nil
+}
+
+// FormatAddresses is the AnnAddresses value for a set of addresses (sorted, so it only changes when the set does).
+func FormatAddresses(ips []net.IP) string {
+	s := make([]string, 0, len(ips))
+	for _, ip := range ips {
+		s = append(s, ip.String())
+	}
+	sort.Strings(s)
+	return strings.Join(s, ",")
+}
+
+// Endpoint is one ready backend of one Service port.
+type Endpoint struct {
+	IP   net.IP
+	Port int32
+}
+
+func (e Endpoint) String() string { return fmt.Sprintf("%s:%d", e.IP, e.Port) }
+
+// Port is one Service port at the held address.
+type Port struct {
+	Protocol  coreapi.Protocol
+	Port      int32
+	Endpoints []Endpoint
+}
+
+// Forward is a Service whose address this node holds.
+type Forward struct {
+	Namespace, Name string
+	Address         net.IP
+	Ports           []Port
+}
+
+// Status is a write to a Service's status.loadBalancer: Address set, or cleared when nil.
+type Status struct {
+	Namespace, Name string
+	Address         net.IP
+}
+
+// Holder is another node that forwards held addresses' connections to pods; Mark identifies its connections here.
+type Holder struct {
+	Node Node
+	Mark uint32 // 1..MaxMarks, in the connmark's top byte
+}
+
+// MaxMarks is how many holders a node can tell apart (the top byte of the mark, 0 meaning none).
+const MaxMarks = 250
+
+// Desired is what one node must do.
+type Desired struct {
+	Forwards []Forward
+	Holders  []Holder
+	Status   []Status
+	// Accept are the address:ports this node serves itself (Gateways' HTTP and HTTPS listeners), let in ahead of
+	// the host firewall
+	Accept   []Accept
+	Problems []string // what an operator should hear about (logged and evented by the controller)
+}
+
+// Accept is a TCP address:port this node serves itself.
+type Accept struct {
+	Address net.IP
+	Port    int32
+	Comment string
+}
+
+// eligible reports whether the Service asks for this agent.
+func eligible(s *coreapi.Service) bool {
+	return s.Spec.Type == coreapi.ServiceTypeLoadBalancer && s.Spec.LoadBalancerClass != nil &&
+		*s.Spec.LoadBalancerClass == Class
+}
+
+// shown is the address status.loadBalancer currently shows (nil if none or not a single IP).
+func shown(s *coreapi.Service) net.IP {
+	if len(s.Status.LoadBalancer.Ingress) != 1 {
+		return nil
+	}
+	return net.ParseIP(s.Status.LoadBalancer.Ingress[0].IP).To4()
+}
+
+func hasStatus(s *coreapi.Service) bool { return len(s.Status.LoadBalancer.Ingress) != 0 }
+
+// Compute decides what node `self` does, given every node, every Service and every EndpointSlice.
+func Compute(self string, nodes []Node, services []*coreapi.Service, slices []*discovery.EndpointSlice) Desired {
+	var d Desired
+	sortedNodes := append([]Node(nil), nodes...)
+	sort.Slice(sortedNodes, func(i, j int) bool { return sortedNodes[i].Name < sortedNodes[j].Name })
+	holdersOf := func(ip net.IP) []Node {
+		var hs []Node
+		for _, n := range sortedNodes {
+			if ip != nil && n.Holds(ip) {
+				hs = append(hs, n)
+			}
+		}
+		return hs
+	}
+	// who may write a status that shows `ip`: its single holder, or (nobody holds it) the first node by name
+	writerFor := func(ip net.IP) string {
+		if hs := holdersOf(ip); len(hs) == 1 {
+			return hs[0].Name
+		}
+		if len(sortedNodes) > 0 {
+			return sortedNodes[0].Name
+		}
+		return ""
+	}
+	holding := map[string]bool{} // nodes that forward some address
+
+	svcs := append([]*coreapi.Service(nil), services...)
+	sort.Slice(svcs, func(i, j int) bool { return key(svcs[i]) < key(svcs[j]) })
+	for _, s := range svcs {
+		if !eligible(s) {
+			// loadBalancerClass is immutable and only allowed on type LoadBalancer, so a
+			// Service never leaves the class while keeping a status we wrote
+			continue
+		}
+		want := net.ParseIP(s.Spec.LoadBalancerIP).To4()
+		var holder *Node
+		switch hs := holdersOf(want); {
+		case want == nil:
+			d.Problems = append(d.Problems, fmt.Sprintf("%s: spec.loadBalancerIP must name an IPv4 address a node holds (got %q)", key(s), s.Spec.LoadBalancerIP))
+		case len(hs) == 0:
+			d.Problems = append(d.Problems, fmt.Sprintf("%s: no node holds %s", key(s), want))
+		case len(hs) > 1:
+			names := make([]string, len(hs))
+			for i, h := range hs {
+				names[i] = h.Name
+			}
+			d.Problems = append(d.Problems, fmt.Sprintf("%s: %s is held by several nodes (%s); serving it from none", key(s), want, strings.Join(names, ", ")))
+		default:
+			holder = &hs[0]
+		}
+		if holder == nil {
+			// fail closed: nobody forwards, and the status must not claim an address
+			if hasStatus(s) && writerFor(shown(s)) == self {
+				d.Status = append(d.Status, Status{Namespace: s.Namespace, Name: s.Name})
+			}
+			continue
+		}
+		holding[holder.Name] = true
+		if holder.Name != self {
+			continue
+		}
+		if cur := shown(s); cur == nil || !cur.Equal(want) {
+			d.Status = append(d.Status, Status{Namespace: s.Namespace, Name: s.Name, Address: want})
+		}
+		d.Forwards = append(d.Forwards, Forward{Namespace: s.Namespace, Name: s.Name, Address: want, Ports: ports(s, slices)})
+	}
+
+	var others []Node
+	for _, n := range sortedNodes {
+		if holding[n.Name] && n.Name != self {
+			if n.WeaveIP == nil {
+				d.Problems = append(d.Problems, fmt.Sprintf("node %s holds an address but has not published its weave bridge; replies to its forwards cannot reach it", n.Name))
+				continue
+			}
+			others = append(others, n)
+		}
+	}
+	d.Holders = assignMarks(others)
+	return d
+}
+
+// assignMarks gives each holder a mark from its name, probing past collisions in name order: stable while the set is.
+func assignMarks(nodes []Node) []Holder {
+	taken := map[uint32]bool{}
+	out := make([]Holder, 0, len(nodes))
+	for _, n := range nodes {
+		if len(out) == MaxMarks {
+			break
+		}
+		h := fnv.New32a()
+		h.Write([]byte(n.Name))
+		m := h.Sum32()%MaxMarks + 1
+		for taken[m] {
+			m = m%MaxMarks + 1
+		}
+		taken[m] = true
+		out = append(out, Holder{Node: n, Mark: m})
+	}
+	return out
+}
+
+func key(s *coreapi.Service) string { return s.Namespace + "/" + s.Name }
+
+// ports pairs each Service port with its ready endpoints.
+func ports(s *coreapi.Service, slices []*discovery.EndpointSlice) []Port {
+	var out []Port
+	for _, sp := range s.Spec.Ports {
+		proto := sp.Protocol
+		if proto == "" {
+			proto = coreapi.ProtocolTCP
+		}
+		seen := map[string]bool{}
+		var eps []Endpoint
+		for _, sl := range slices {
+			if sl.Namespace != s.Namespace || sl.Labels[serviceNameLabel] != s.Name || sl.AddressType != discovery.AddressTypeIPv4 {
+				continue
+			}
+			for _, p := range sl.Ports {
+				pname, pproto := "", coreapi.ProtocolTCP
+				if p.Name != nil {
+					pname = *p.Name
+				}
+				if p.Protocol != nil {
+					pproto = *p.Protocol
+				}
+				if pname != sp.Name || pproto != proto || p.Port == nil {
+					continue
+				}
+				for _, ep := range sl.Endpoints {
+					if ep.Conditions.Ready != nil && !*ep.Conditions.Ready {
+						continue
+					}
+					for _, a := range ep.Addresses {
+						ip := net.ParseIP(a).To4()
+						if ip == nil {
+							continue
+						}
+						e := Endpoint{IP: ip, Port: *p.Port}
+						if !seen[e.String()] {
+							seen[e.String()] = true
+							eps = append(eps, e)
+						}
+					}
+				}
+			}
+		}
+		sort.Slice(eps, func(i, j int) bool { return eps[i].String() < eps[j].String() })
+		out = append(out, Port{Protocol: proto, Port: sp.Port, Endpoints: eps})
+	}
+	return out
+}

lb/model_test.go

@@ -0,0 +1,253 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func mac(s string) net.HardwareAddr {
+	m, err := net.ParseMAC(s)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}
+
+var (
+	cp = Node{Name: "cp", Addresses: []net.IP{net.ParseIP("86.107.168.24").To4()},
+		WeaveIP: net.ParseIP("10.32.0.1").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:01")}
+	w1 = Node{Name: "w1", Addresses: []net.IP{net.ParseIP("86.107.168.15").To4()},
+		WeaveIP: net.ParseIP("10.40.0.0").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:02")}
+	w2 = Node{Name: "w2", Addresses: []net.IP{net.ParseIP("86.107.168.88").To4()},
+		WeaveIP: net.ParseIP("10.44.0.0").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:03")}
+	nodes = []Node{w2, cp, w1}
+)
+
+func svc(name, ip string, mutate ...func(*coreapi.Service)) *coreapi.Service {
+	class := Class
+	s := &coreapi.Service{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name},
+		Spec: coreapi.ServiceSpec{
+			Type:              coreapi.ServiceTypeLoadBalancer,
+			LoadBalancerClass: &class,
+			LoadBalancerIP:    ip,
+			Ports: []coreapi.ServicePort{
+				{Name: "ssh", Protocol: coreapi.ProtocolTCP, Port: 22},
+				{Name: "https", Port: 443}, // protocol defaults to TCP
+			},
+		},
+	}
+	for _, m := range mutate {
+		m(s)
+	}
+	return s
+}
+
+func shows(ip string) func(*coreapi.Service) {
+	return func(s *coreapi.Service) {
+		s.Status.LoadBalancer.Ingress = []coreapi.LoadBalancerIngress{{IP: ip}}
+	}
+}
+
+func ptr[T any](v T) *T { return &v }
+
+func slice(svcName string, eps map[string]bool, ports map[string]int32) *discovery.EndpointSlice {
+	sl := &discovery.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Namespace: "gitea", Name: svcName + "-x", Labels: map[string]string{serviceNameLabel: svcName}},
+		AddressType: discovery.AddressTypeIPv4,
+	}
+	for ip, ready := range eps {
+		sl.Endpoints = append(sl.Endpoints, discovery.Endpoint{Addresses: []string{ip}, Conditions: discovery.EndpointConditions{Ready: ptr(ready)}})
+	}
+	for name, port := range ports {
+		sl.Ports = append(sl.Ports, discovery.EndpointPort{Name: ptr(name), Protocol: ptr(coreapi.ProtocolTCP), Port: ptr(port)})
+	}
+	return sl
+}
+
+func TestHolderForwardsToReadyEndpointsAndWritesStatus(t *testing.T) {
+	s := svc("gitea", "86.107.168.15")
+	sl := slice("gitea", map[string]bool{"10.40.0.7": true, "10.44.0.3": true, "10.44.0.9": false},
+		map[string]int32{"ssh": 2222, "https": 3000})
+	d := Compute("w1", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
+	if len(d.Problems) != 0 {
+		t.Fatalf("problems: %v", d.Problems)
+	}
+	if len(d.Forwards) != 1 || !d.Forwards[0].Address.Equal(net.ParseIP("86.107.168.15")) {
+		t.Fatalf("forwards: %+v", d.Forwards)
+	}
+	f := d.Forwards[0]
+	if len(f.Ports) != 2 || f.Ports[0].Port != 22 || f.Ports[1].Port != 443 || f.Ports[1].Protocol != coreapi.ProtocolTCP {
+		t.Fatalf("ports: %+v", f.Ports)
+	}
+	if got := f.Ports[0].Endpoints; len(got) != 2 || got[0].String() != "10.40.0.7:2222" || got[1].String() != "10.44.0.3:2222" {
+		t.Fatalf("ssh endpoints (not-ready one must be left out): %v", got)
+	}
+	if got := f.Ports[1].Endpoints; len(got) != 2 || got[0].Port != 3000 {
+		t.Fatalf("https endpoints: %v", got)
+	}
+	if len(d.Status) != 1 || !d.Status[0].Address.Equal(net.ParseIP("86.107.168.15")) {
+		t.Fatalf("status: %+v", d.Status)
+	}
+	if len(d.Holders) != 0 {
+		t.Fatalf("a holder does not steer its own replies: %+v", d.Holders)
+	}
+	// already shown: nothing to write
+	d = Compute("w1", nodes, []*coreapi.Service{svc("gitea", "86.107.168.15", shows("86.107.168.15"))}, nil)
+	if len(d.Status) != 0 {
+		t.Fatalf("status rewritten: %+v", d.Status)
+	}
+}
+
+func TestOtherNodesSteerRepliesToTheHolderAndNeverWrite(t *testing.T) {
+	s := svc("gitea", "86.107.168.15")
+	for _, self := range []string{"cp", "w2"} {
+		d := Compute(self, nodes, []*coreapi.Service{s}, nil)
+		if len(d.Forwards) != 0 || len(d.Status) != 0 {
+			t.Fatalf("%s: only the holder forwards and writes: %+v", self, d)
+		}
+		if len(d.Holders) != 1 || d.Holders[0].Node.Name != "w1" || d.Holders[0].Mark < 1 || d.Holders[0].Mark > MaxMarks {
+			t.Fatalf("%s: holders: %+v", self, d.Holders)
+		}
+	}
+}
+
+func TestUnheldOrContestedAddressFailsClosed(t *testing.T) {
+	// nobody holds it: no forward anywhere; the stale status is cleared by exactly one node (first by name)
+	s := svc("gitea", "86.107.168.99", shows("86.107.168.99"))
+	for _, self := range []string{"cp", "w1", "w2"} {
+		d := Compute(self, nodes, []*coreapi.Service{s}, nil)
+		if len(d.Forwards) != 0 || len(d.Holders) != 0 || len(d.Problems) != 1 || !strings.Contains(d.Problems[0], "no node holds") {
+			t.Fatalf("%s: %+v", self, d)
+		}
+		clears := len(d.Status) == 1 && d.Status[0].Address == nil
+		if clears != (self == "cp") {
+			t.Fatalf("%s: clears=%v", self, clears)
+		}
+	}
+	// two nodes claim it: neither serves it, its holder's view clears the status
+	both := w2
+	both.Addresses = append([]net.IP{net.ParseIP("86.107.168.15").To4()}, w2.Addresses...)
+	d := Compute("w1", []Node{cp, w1, both}, []*coreapi.Service{svc("gitea", "86.107.168.15", shows("86.107.168.15"))}, nil)
+	if len(d.Forwards) != 0 || len(d.Problems) != 1 || !strings.Contains(d.Problems[0], "several nodes") {
+		t.Fatalf("contested: %+v", d)
+	}
+	// a bad request is reported, not guessed at
+	d = Compute("w1", nodes, []*coreapi.Service{svc("gitea", "")}, nil)
+	if len(d.Forwards) != 0 || len(d.Problems) != 1 || !strings.Contains(d.Problems[0], "spec.loadBalancerIP") {
+		t.Fatalf("no address: %+v", d)
+	}
+}
+
+func TestOnlyOptedInLoadBalancersAreServed(t *testing.T) {
+	other := svc("a", "86.107.168.15", func(s *coreapi.Service) { s.Spec.LoadBalancerClass = ptr("example.com/other") })
+	noClass := svc("b", "86.107.168.15", func(s *coreapi.Service) { s.Spec.LoadBalancerClass = nil })
+	clusterIP := svc("c", "86.107.168.15", func(s *coreapi.Service) {
+		s.Spec.Type = coreapi.ServiceTypeClusterIP
+	})
+	d := Compute("w1", nodes, []*coreapi.Service{other, noClass, clusterIP}, nil)
+	if len(d.Forwards)+len(d.Status)+len(d.Holders)+len(d.Problems) != 0 {
+		t.Fatalf("touched a Service that did not opt in: %+v", d)
+	}
+}
+
+func TestParseNodeAndAnnotationsRoundTrip(t *testing.T) {
+	l := Local{Addresses: []net.IP{net.ParseIP("86.107.168.88").To4(), net.ParseIP("86.107.168.15").To4()},
+		WeaveIP: net.ParseIP("10.44.0.0").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:03")}
+	a := l.Annotations()
+	if a[AnnAddresses] != "86.107.168.15,86.107.168.88" || a[AnnWeave] != "10.44.0.0,aa:aa:aa:aa:aa:03" {
+		t.Fatalf("annotations: %v", a)
+	}
+	n, err := ParseNode(&coreapi.Node{ObjectMeta: metav1.ObjectMeta{Name: "w2", Annotations: a}})
+	if err != nil || !n.Holds(net.ParseIP("86.107.168.15")) || !n.WeaveIP.Equal(l.WeaveIP) || n.WeaveMAC.String() != "aa:aa:aa:aa:aa:03" {
+		t.Fatalf("parsed %+v, %v", n, err)
+	}
+	for _, bad := range []map[string]string{{AnnAddresses: "1.2.3.4,nope"}, {AnnWeave: "10.44.0.0"}, {AnnWeave: "x,aa:aa:aa:aa:aa:03"}} {
+		if _, err := ParseNode(&coreapi.Node{ObjectMeta: metav1.ObjectMeta{Name: "w2", Annotations: bad}}); err == nil {
+			t.Fatalf("accepted %v", bad)
+		}
+	}
+	if n, err := ParseNode(&coreapi.Node{ObjectMeta: metav1.ObjectMeta{Name: "new"}}); err != nil || len(n.Addresses) != 0 || n.WeaveIP != nil {
+		t.Fatalf("a node that published nothing holds nothing: %+v %v", n, err)
+	}
+}
+
+func TestMarksAreDistinctAndStable(t *testing.T) {
+	var ns []Node
+	for _, name := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
+		ns = append(ns, Node{Name: name})
+	}
+	first := assignMarks(ns)
+	seen := map[uint32]bool{}
+	for _, h := range first {
+		if seen[h.Mark] || h.Mark < 1 || h.Mark > MaxMarks {
+			t.Fatalf("bad or repeated mark: %+v", first)
+		}
+		seen[h.Mark] = true
+	}
+	again := assignMarks(ns)
+	for i := range first {
+		if first[i].Mark != again[i].Mark {
+			t.Fatalf("marks moved between passes")
+		}
+	}
+}
+
+func TestRenderedRules(t *testing.T) {
+	_, podNet, _ := net.ParseCIDR("10.32.0.0/12")
+	s := svc("gitea", "86.107.168.15")
+	sl := slice("gitea", map[string]bool{"10.40.0.7": true, "10.44.0.3": true, "10.44.0.4": true}, map[string]int32{"ssh": 2222, "https": 3000})
+	d := Compute("w1", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
+	r := Render(d, EgressDesired{}, "w1", podNet, []string{"REWEAVE-LBS-DEADBEEF0000"})
+	for _, want := range []string{
+		"*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\n:REWEAVE-LB-CLASS - [0:0]\n",
+		// the client's address survives weave's masquerade; a pod's own connection (hairpin) is masqueraded here
+		"-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -s 10.32.0.0/12 -j MASQUERADE\n",
+		"-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -j ACCEPT\n",
+		":REWEAVE-LBS-DEADBEEF0000 - [0:0]\n", // stale chain emptied
+		"-A REWEAVE-LB -d 86.107.168.15/32 -p tcp -m tcp --dport 22 -m comment --comment \"gitea/gitea\" -j REWEAVE-LBS-",
+		"-p tcp -m statistic --mode random --probability 0.3333333333 -j DNAT --to-destination 10.40.0.7:2222\n",
+		"-p tcp -m statistic --mode random --probability 0.5000000000 -j DNAT --to-destination 10.44.0.3:2222\n",
+		"-p tcp -j DNAT --to-destination 10.44.0.4:2222\n",
+		"COMMIT\n",
+	} {
+		if !strings.Contains(r.Nat, want) {
+			t.Fatalf("nat lacks %q:\n%s", want, r.Nat)
+		}
+	}
+	for _, line := range strings.Split(r.Nat, "\n") {
+		// iptables refuses a DNAT to ip:port without the protocol in the same rule
+		if strings.Contains(line, "-j DNAT") && !strings.Contains(line, " -p tcp ") {
+			t.Fatalf("DNAT without its protocol: %q", line)
+		}
+	}
+	if strings.Count(r.Nat, "MASQUERADE") != 1 || strings.Contains(r.Nat, "SNAT") {
+		t.Fatalf("only a pod's hairpin connection may be masqueraded:\n%s", r.Nat)
+	}
+	if r.Mangle != "*mangle\n:REWEAVE-LB-MARK - [0:0]\n:REWEAVE-LB-MSS - [0:0]\nCOMMIT\n" {
+		t.Fatalf("a holder marks nothing:\n%s", r.Mangle)
+	}
+	d = Compute("w2", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
+	r = Render(d, EgressDesired{}, "w2", podNet, nil)
+	m := d.Holders[0].Mark << 24
+	for _, want := range []string{
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir REPLY -m connmark ! --mark 0x0/0xff000000 -j CONNMARK --restore-mark --nfmask 0xff000000 --ctmask 0xff000000\n",
+		"-A REWEAVE-LB-MARK -i weave -m mac --mac-source aa:aa:aa:aa:aa:02 ! -s 10.32.0.0/12 -d 10.32.0.0/12 -m conntrack --ctstate NEW -m comment --comment \"holder w1\" -j CONNMARK --set-xmark " + fmt.Sprintf("0x%x", m) + "/0xff000000\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	if r.Nat != "*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\n:REWEAVE-LB-CLASS - [0:0]\nCOMMIT\n" {
+		t.Fatalf("a non-holder forwards nothing:\n%s", r.Nat)
+	}
+	if r.Filter != "*filter\n:REWEAVE-LB-FWD - [0:0]\n:REWEAVE-LB-IN - [0:0]\nCOMMIT\n" {
+		t.Fatalf("no EgressService, no filter rules:\n%s", r.Filter)
+	}
+}

net/portlock.go

@@ -0,0 +1,220 @@
+package net
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netlink/nl"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+// A pod's port is locked to the pod's own addresses: what the pod sends in through the host side of its veth has to
+// carry the pod's MAC as its Ethernet source and be either IPv4 from the pod's address or ARP speaking for exactly
+// that MAC and address (or probing, from 0.0.0.0). Everything else is dropped before the bridge sees it, so a pod can
+// claim neither another pod's identity nor a node's, on its own node or, through weave, on any other.
+//
+// The lock is a set of tc filters on the veth's ingress qdisc. They match the way iproute2 lays out "ether src"
+// (the Ethernet source at -8 and -4 from the network header) and "ip src" (at 12), and they go away with the veth.
+
+const (
+	portLockPrioIPv4  = 1
+	portLockPrioARP   = 2
+	portLockPrioProbe = 3
+	portLockPrioDrop  = 9
+)
+
+var ingressParent = netlink.MakeHandle(0xffff, 0)
+
+// PortLockFilters are the filters that lock the port with the given index to a pod's MAC and IPv4 address.
+func PortLockFilters(index int, mac net.HardwareAddr, ip net.IP) ([]netlink.Filter, error) {
+	if len(mac) != 6 {
+		return nil, fmt.Errorf("port lock needs an Ethernet MAC, got %q", mac)
+	}
+	ip4 := ip.To4()
+	if ip4 == nil {
+		return nil, fmt.Errorf("port lock needs an IPv4 address, got %q", ip)
+	}
+	src := etherSource(mac)
+	arp := func(spa net.IP) []netlink.TcU32Key {
+		keys := append([]netlink.TcU32Key{}, src...)
+		return append(keys,
+			// Ethernet/IPv4 ARP: hardware type 1, protocol 0x0800, address lengths 6 and 4 (any operation)
+			netlink.TcU32Key{Mask: 0xffffffff, Val: 0x00010800, Off: 0},
+			netlink.TcU32Key{Mask: 0xffff0000, Val: 0x06040000, Off: 4},
+			// the sender hardware address (8..13) and the sender protocol address (14..17)
+			netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: 8},
+			netlink.TcU32Key{Mask: 0xffffffff, Val: uint32(binary.BigEndian.Uint16(mac[4:6]))<<16 | uint32(binary.BigEndian.Uint16(spa[0:2])), Off: 12},
+			netlink.TcU32Key{Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(spa[2:4])) << 16, Off: 16},
+		)
+	}
+	ipv4 := append(append([]netlink.TcU32Key{}, src...), netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(ip4), Off: 12})
+	return []netlink.Filter{
+		portLockU32(index, portLockPrioIPv4, unix.ETH_P_IP, ipv4),
+		portLockU32(index, portLockPrioARP, unix.ETH_P_ARP, arp(ip4)),
+		portLockU32(index, portLockPrioProbe, unix.ETH_P_ARP, arp(net.IPv4zero.To4())),
+		&netlink.MatchAll{
+			FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: portLockPrioDrop, Protocol: unix.ETH_P_ALL},
+			Actions:     []netlink.Action{gact(netlink.TC_ACT_SHOT)},
+		},
+	}, nil
+}
+
+// etherSource matches the Ethernet source address, which sits 8 bytes before the network header.
+func etherSource(mac net.HardwareAddr) []netlink.TcU32Key {
+	return []netlink.TcU32Key{
+		{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: -8},
+		{Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(mac[4:6])) << 16, Off: -4},
+	}
+}
+
+func portLockU32(index int, prio uint16, proto uint16, keys []netlink.TcU32Key) *netlink.U32 {
+	// clipped: the vendored netlink serialises cap(Keys) keys, and spare capacity would go out as extra all-zero keys
+	keys = keys[:len(keys):len(keys)]
+	return &netlink.U32{
+		FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: prio, Protocol: proto},
+		Sel:         &netlink.TcU32Sel{Flags: nl.TC_U32_TERMINAL, Keys: keys},
+		Actions:     []netlink.Action{gact(netlink.TC_ACT_OK)},
+	}
+}
+
+func gact(action netlink.TcAct) netlink.Action {
+	return &netlink.GenericAction{ActionAttrs: netlink.ActionAttrs{Action: action}}
+}
+
+func ingressQdisc(index int) *netlink.Ingress {
+	return &netlink.Ingress{QdiscAttrs: netlink.QdiscAttrs{LinkIndex: index, Handle: ingressParent, Parent: netlink.HANDLE_INGRESS}}
+}
+
+// LockPort locks a pod's port (the host side of its veth) to the pod's MAC and IPv4 address. A port already locked
+// to them is left alone; anything else on its ingress is replaced, and the pod's own traffic is let through before
+// the rest is dropped, so a running pod never loses its own frames.
+func LockPort(link netlink.Link, mac net.HardwareAddr, ip net.IP) error {
+	index := link.Attrs().Index
+	want, err := PortLockFilters(index, mac, ip)
+	if err != nil {
+		return err
+	}
+	if locked, _ := portHasFilters(link, want); locked {
+		return nil
+	}
+	// removing the ingress qdisc takes its filters with it
+	if err := netlink.QdiscDel(ingressQdisc(index)); err != nil && !errors.Is(err, unix.ENOENT) && !errors.Is(err, unix.EINVAL) {
+		return fmt.Errorf("clearing ingress of %s: %w", link.Attrs().Name, err)
+	}
+	if err := netlink.QdiscAdd(ingressQdisc(index)); err != nil {
+		return fmt.Errorf("adding ingress qdisc to %s: %w", link.Attrs().Name, err)
+	}
+	for _, f := range want {
+		if err := netlink.FilterAdd(f); err != nil {
+			return fmt.Errorf("adding port lock filter (prio %d) to %s: %w", f.Attrs().Priority, link.Attrs().Name, err)
+		}
+	}
+	return nil
+}
+
+// LockContainerPort locks the port of a pod attached by the CNI plugin: the host side of the veth whose container
+// end is ifName in ns, to that end's MAC and the pod's address.
+func LockContainerPort(ns netns.NsHandle, ifName string, ip net.IP) error {
+	var mac net.HardwareAddr
+	peer := 0
+	if err := WithNetNSLink(ns, ifName, func(link netlink.Link) error {
+		// a veth's link attribute is its peer's index, in the peer's namespace: the host's
+		mac, peer = link.Attrs().HardwareAddr, link.Attrs().ParentIndex
+		return nil
+	}); err != nil {
+		return err
+	}
+	if peer == 0 {
+		return fmt.Errorf("%s has no peer: not a veth", ifName)
+	}
+	host, err := netlink.LinkByIndex(peer)
+	if err != nil {
+		return fmt.Errorf("host side of %s: %w", ifName, err)
+	}
+	if host.Type() != "veth" || !strings.HasPrefix(host.Attrs().Name, vethPrefix+"pl") {
+		return fmt.Errorf("host side of %s is %s (%s), not a weave pod veth", ifName, host.Attrs().Name, host.Type())
+	}
+	return LockPort(host, mac, ip)
+}
+
+// PortLocked reports whether a port is locked to exactly this MAC and address.
+func PortLocked(link netlink.Link, mac net.HardwareAddr, ip net.IP) (bool, error) {
+	want, err := PortLockFilters(link.Attrs().Index, mac, ip)
+	if err != nil {
+		return false, err
+	}
+	return portHasFilters(link, want)
+}
+
+func portHasFilters(link netlink.Link, want []netlink.Filter) (bool, error) {
+	have, err := netlink.FilterList(link, ingressParent)
+	if err != nil {
+		return false, err
+	}
+	var got []netlink.Filter
+	for _, f := range have {
+		// u32 lists the hash table that holds a filter as a filter of its own, without a selector
+		if u, ok := f.(*netlink.U32); ok && u.Sel == nil {
+			continue
+		}
+		got = append(got, f)
+	}
+	if len(got) != len(want) {
+		return false, nil
+	}
+	for _, w := range want {
+		found := false
+		for _, g := range got {
+			if sameFilter(g, w) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func sameFilter(a, b netlink.Filter) bool {
+	if a.Attrs().Priority != b.Attrs().Priority || a.Attrs().Protocol != b.Attrs().Protocol || a.Type() != b.Type() {
+		return false
+	}
+	switch fa := a.(type) {
+	case *netlink.U32:
+		fb := b.(*netlink.U32)
+		if fa.Sel == nil || fb.Sel == nil || len(fa.Sel.Keys) != len(fb.Sel.Keys) || fa.Sel.Flags&nl.TC_U32_TERMINAL == 0 {
+			return false
+		}
+		for i := range fa.Sel.Keys {
+			ka, kb := fa.Sel.Keys[i], fb.Sel.Keys[i]
+			if ka.Mask != kb.Mask || ka.Val != kb.Val || ka.Off != kb.Off {
+				return false
+			}
+		}
+		return sameActions(fa.Actions, fb.Actions)
+	case *netlink.MatchAll:
+		return sameActions(fa.Actions, b.(*netlink.MatchAll).Actions)
+	}
+	return false
+}
+
+func sameActions(a, b []netlink.Action) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		ga, ok1 := a[i].(*netlink.GenericAction)
+		gb, ok2 := b[i].(*netlink.GenericAction)
+		if !ok1 || !ok2 || ga.Attrs().Action != gb.Attrs().Action {
+			return false
+		}
+	}
+	return true
+}

net/portlock_kernel_test.go

@@ -0,0 +1,228 @@
+package net
+
+import (
+	"bytes"
+	"encoding/binary"
+	"net"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+// TestPortLockInKernel builds a bridge with two pods in throwaway network namespaces, locks pod A's port and sends
+// frames from A: what A may send reaches B, what claims anything but A's own addresses does not. It needs root.
+func TestPortLockInKernel(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("needs root (creates network namespaces)")
+	}
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	orig, err := netns.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer orig.Close()
+	defer netns.Set(orig)
+
+	hub, err := netns.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer hub.Close()
+	podA, _ := netns.New()
+	defer podA.Close()
+	podB, _ := netns.New()
+	defer podB.Close()
+	must(t, netns.Set(hub))
+
+	br := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: "br0"}}
+	must(t, netlink.LinkAdd(br))
+	must(t, netlink.LinkSetUp(br))
+	ipA, ipB := net.IPv4(10, 99, 0, 1).To4(), net.IPv4(10, 99, 0, 2).To4()
+	hostA, macA := attach(t, hub, podA, br, "vethwepltsta", ipA)
+	_, macB := attach(t, hub, podB, br, "vethwepltstb", ipB)
+
+	// as the CNI plugin does it: from the pod's end of the veth, in the pod's namespace
+	must(t, LockContainerPort(podA, "eth0", ipA))
+	if ok, err := PortLocked(hostA, macA, ipA); err != nil || !ok {
+		dumpIngress(t, hostA)
+		t.Fatalf("port not locked after LockContainerPort: %v %v", ok, err)
+	}
+	// locking again changes nothing (the filters are compared, not rebuilt)
+	must(t, LockPort(hostA, macA, ipA))
+	if ok, _ := PortLocked(hostA, macA, ipA); !ok {
+		t.Fatal("port lost its lock on a repeated LockPort")
+	}
+
+	send := socketIn(t, podA, "eth0")
+	recv := socketIn(t, podB, "eth0")
+	defer unix.Close(send)
+	defer unix.Close(recv)
+	other, _ := net.ParseMAC("02:00:00:00:00:99")
+	spoofIP := net.IPv4(10, 99, 0, 200).To4()
+
+	cases := []struct {
+		name   string
+		frame  []byte
+		passes bool
+	}{
+		{"ipv4 from its own addresses", ipv4Frame(macB, macA, ipA, ipB, "own-ip"), true},
+		{"ipv4 from another MAC", ipv4Frame(macB, other, ipA, ipB, "mac-spoof"), false},
+		{"ipv4 from another address", ipv4Frame(macB, macA, spoofIP, ipB, "ip-spoof"), false},
+		{"arp for itself", arpFrame(macA, macA, ipA, ipB, "own-arp"), true},
+		{"arp probe", arpFrame(macA, macA, net.IPv4zero.To4(), ipB, "probe"), true},
+		{"arp claiming another address", arpFrame(macA, macA, spoofIP, ipB, "arp-ip-spoof"), false},
+		{"arp claiming another MAC", arpFrame(macA, other, ipA, ipB, "arp-sha-spoof"), false},
+		{"arp from another MAC", arpFrame(other, other, ipA, ipB, "arp-eth-spoof"), false},
+		{"ipv6", etherFrame(macB, macA, 0x86dd, []byte("ipv6-frame")), false},
+		{"other ethertype", etherFrame(macB, macA, 0x88b5, []byte("experimental")), false},
+	}
+	for _, c := range cases {
+		drain(recv)
+		if _, err := unix.Write(send, c.frame); err != nil {
+			t.Fatalf("%s: send: %v", c.name, err)
+		}
+		got := sawPayload(recv, c.frame[len(c.frame)-12:], 300*time.Millisecond)
+		if got != c.passes {
+			t.Errorf("%s: reached B = %v, want %v", c.name, got, c.passes)
+		}
+	}
+}
+
+func dumpIngress(t *testing.T, link netlink.Link) {
+	have, _ := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0))
+	for _, f := range have {
+		t.Logf("have %s %+v", f.Type(), f.Attrs())
+		if u, ok := f.(*netlink.U32); ok && u.Sel != nil {
+			t.Logf("  sel flags %x keys %+v actions %+v", u.Sel.Flags, u.Sel.Keys, u.Actions)
+		}
+		if m, ok := f.(*netlink.MatchAll); ok {
+			t.Logf("  actions %+v", m.Actions)
+		}
+	}
+}
+
+func must(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// attach plugs a pod namespace into the bridge through a veth named host on the hub side, eth0 on the pod side.
+func attach(t *testing.T, hub, pod netns.NsHandle, br netlink.Link, host string, ip net.IP) (netlink.Link, net.HardwareAddr) {
+	t.Helper()
+	must(t, netlink.LinkAdd(&netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: host}, PeerName: "tmp" + host[len(host)-2:]}))
+	h, err := netlink.LinkByName(host)
+	must(t, err)
+	must(t, netlink.LinkSetMaster(h, br))
+	must(t, netlink.LinkSetUp(h))
+	p, err := netlink.LinkByName("tmp" + host[len(host)-2:])
+	must(t, err)
+	must(t, netlink.LinkSetNsFd(p, int(pod)))
+	must(t, netns.Set(pod))
+	p, err = netlink.LinkByName("tmp" + host[len(host)-2:])
+	must(t, err)
+	must(t, netlink.LinkSetName(p, "eth0"))
+	must(t, netlink.AddrAdd(p, &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: net.CIDRMask(24, 32)}}))
+	must(t, netlink.LinkSetUp(p))
+	mac := p.Attrs().HardwareAddr
+	must(t, netns.Set(hub))
+	return h, mac
+}
+
+// socketIn opens a raw packet socket on an interface of a namespace (the socket stays in that namespace).
+func socketIn(t *testing.T, ns netns.NsHandle, ifName string) int {
+	t.Helper()
+	cur, _ := netns.Get()
+	defer cur.Close()
+	must(t, netns.Set(ns))
+	defer netns.Set(cur)
+	link, err := netlink.LinkByName(ifName)
+	must(t, err)
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
+	must(t, err)
+	must(t, unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: link.Attrs().Index}))
+	return fd
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }
+
+func drain(fd int) {
+	buf := make([]byte, 2048)
+	for {
+		if _, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT); err != nil {
+			return
+		}
+	}
+}
+
+func sawPayload(fd int, marker []byte, within time.Duration) bool {
+	buf := make([]byte, 2048)
+	deadline := time.Now().Add(within)
+	for time.Now().Before(deadline) {
+		n, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT)
+		if err != nil {
+			time.Sleep(5 * time.Millisecond)
+			continue
+		}
+		if bytes.Contains(buf[:n], marker) {
+			return true
+		}
+	}
+	return false
+}
+
+func etherFrame(dst, src net.HardwareAddr, ethertype uint16, payload []byte) []byte {
+	f := make([]byte, 14, 14+len(payload)+12)
+	copy(f[0:6], dst)
+	copy(f[6:12], src)
+	binary.BigEndian.PutUint16(f[12:14], ethertype)
+	f = append(f, payload...)
+	// a fixed-size tail the receiver looks for, so each case is told apart
+	tail := make([]byte, 12)
+	copy(tail, payload)
+	return append(f, tail...)
+}
+
+func ipv4Frame(dst, src net.HardwareAddr, from, to net.IP, tag string) []byte {
+	body := []byte(tag)
+	h := make([]byte, 20)
+	h[0] = 0x45
+	binary.BigEndian.PutUint16(h[2:4], uint16(20+len(body)+12))
+	h[8] = 64
+	h[9] = 253 // experimental protocol: nothing answers it
+	copy(h[12:16], from)
+	copy(h[16:20], to)
+	binary.BigEndian.PutUint16(h[10:12], checksum(h))
+	return etherFrame(dst, src, unix.ETH_P_IP, append(h, body...))
+}
+
+func arpFrame(ethSrc, sha net.HardwareAddr, spa, tpa net.IP, tag string) []byte {
+	a := make([]byte, 28)
+	binary.BigEndian.PutUint16(a[0:2], 1)
+	binary.BigEndian.PutUint16(a[2:4], unix.ETH_P_IP)
+	a[4], a[5] = 6, 4
+	binary.BigEndian.PutUint16(a[6:8], 1) // request
+	copy(a[8:14], sha)
+	copy(a[14:18], spa)
+	copy(a[24:28], tpa)
+	bcast, _ := net.ParseMAC("ff:ff:ff:ff:ff:ff")
+	return etherFrame(bcast, ethSrc, unix.ETH_P_ARP, append(a, []byte(tag)...))
+}
+
+func checksum(b []byte) uint16 {
+	var s uint32
+	for i := 0; i+1 < len(b); i += 2 {
+		s += uint32(binary.BigEndian.Uint16(b[i:]))
+	}
+	for s>>16 != 0 {
+		s = s&0xffff + s>>16
+	}
+	return ^uint16(s)
+}

net/portlock_test.go

@@ -0,0 +1,115 @@
+package net
+
+import (
+	"net"
+	"testing"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netlink/nl"
+	"golang.org/x/sys/unix"
+)
+
+// The keys iproute2 writes for "u32 match ether src 02:11:22:33:44:55 match ip src 10.32.0.7/32" (tc filter show,
+// iproute2 6.x): the port lock has to match the same bytes.
+func TestPortLockMatchesIproute2Layout(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	fs, err := PortLockFilters(7, mac, net.ParseIP("10.32.0.7"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fs) != 4 {
+		t.Fatalf("want 4 filters, got %d", len(fs))
+	}
+	ipv4 := fs[0].(*netlink.U32)
+	want := []netlink.TcU32Key{
+		{Val: 0x02112233, Mask: 0xffffffff, Off: -8},
+		{Val: 0x44550000, Mask: 0xffff0000, Off: -4},
+		{Val: 0x0a200007, Mask: 0xffffffff, Off: 12},
+	}
+	if ipv4.Protocol != unix.ETH_P_IP || ipv4.Priority != portLockPrioIPv4 || ipv4.Sel.Flags&nl.TC_U32_TERMINAL == 0 {
+		t.Fatalf("ipv4 filter attrs: %+v flags %x", ipv4.FilterAttrs, ipv4.Sel.Flags)
+	}
+	assertKeys(t, "ipv4", ipv4.Sel.Keys, want)
+	for _, f := range fs {
+		if f.Attrs().LinkIndex != 7 || f.Attrs().Parent != netlink.MakeHandle(0xffff, 0) {
+			t.Fatalf("filter not on ingress of link 7: %+v", f.Attrs())
+		}
+	}
+}
+
+func TestPortLockARP(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	fs, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.7"))
+	common := []netlink.TcU32Key{
+		{Val: 0x02112233, Mask: 0xffffffff, Off: -8},
+		{Val: 0x44550000, Mask: 0xffff0000, Off: -4},
+		{Val: 0x00010800, Mask: 0xffffffff, Off: 0},
+		{Val: 0x06040000, Mask: 0xffff0000, Off: 4},
+		{Val: 0x02112233, Mask: 0xffffffff, Off: 8},
+	}
+	own := append(append([]netlink.TcU32Key{}, common...),
+		netlink.TcU32Key{Val: 0x44550a20, Mask: 0xffffffff, Off: 12}, // sha 44:55, spa 10.32
+		netlink.TcU32Key{Val: 0x00070000, Mask: 0xffff0000, Off: 16}, // spa .0.7
+	)
+	probe := append(append([]netlink.TcU32Key{}, common...),
+		netlink.TcU32Key{Val: 0x44550000, Mask: 0xffffffff, Off: 12},
+		netlink.TcU32Key{Val: 0x00000000, Mask: 0xffff0000, Off: 16},
+	)
+	arp, pr := fs[1].(*netlink.U32), fs[2].(*netlink.U32)
+	if arp.Protocol != unix.ETH_P_ARP || pr.Protocol != unix.ETH_P_ARP {
+		t.Fatal("ARP filters must be protocol arp")
+	}
+	assertKeys(t, "arp", arp.Sel.Keys, own)
+	assertKeys(t, "probe", pr.Sel.Keys, probe)
+	drop := fs[3].(*netlink.MatchAll)
+	if drop.Protocol != unix.ETH_P_ALL || drop.Priority != portLockPrioDrop {
+		t.Fatalf("drop filter attrs: %+v", drop.FilterAttrs)
+	}
+	if a := drop.Actions[0].(*netlink.GenericAction); a.Attrs().Action != netlink.TC_ACT_SHOT {
+		t.Fatalf("last filter must drop, has action %v", a.Attrs().Action)
+	}
+	for _, f := range fs[:3] {
+		if a := f.(*netlink.U32).Actions[0].(*netlink.GenericAction); a.Attrs().Action != netlink.TC_ACT_OK {
+			t.Fatalf("prio %d must pass, has action %v", f.Attrs().Priority, a.Attrs().Action)
+		}
+	}
+}
+
+func TestPortLockRefusesWhatItCannotPin(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	if _, err := PortLockFilters(7, mac, net.ParseIP("fd00::1")); err == nil {
+		t.Fatal("an IPv6 address must be refused: the lock pins IPv4 and ARP")
+	}
+	long, _ := net.ParseMAC("02:11:22:33:44:55:66:77")
+	if _, err := PortLockFilters(7, long, net.ParseIP("10.32.0.7")); err == nil {
+		t.Fatal("a non-Ethernet hardware address must be refused")
+	}
+}
+
+func TestSameFilterComparesKeysAndActions(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	a, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.7"))
+	b, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.8"))
+	if !sameFilter(a[0], a[0]) || sameFilter(a[0], b[0]) {
+		t.Fatal("ipv4 filters for different addresses must differ")
+	}
+	if !sameFilter(a[3], b[3]) {
+		t.Fatal("the drop filters are the same for every pod")
+	}
+	if sameFilter(a[1], a[2]) {
+		t.Fatal("the ARP and probe filters differ in priority and keys")
+	}
+}
+
+func assertKeys(t *testing.T, what string, got, want []netlink.TcU32Key) {
+	t.Helper()
+	if len(got) != len(want) {
+		t.Fatalf("%s: want %d keys, got %d: %+v", what, len(want), len(got), got)
+	}
+	for i := range want {
+		if got[i].Val != want[i].Val || got[i].Mask != want[i].Mask || got[i].Off != want[i].Off {
+			t.Fatalf("%s key %d: want %08x/%08x at %d, got %08x/%08x at %d", what, i,
+				want[i].Val, want[i].Mask, want[i].Off, got[i].Val, got[i].Mask, got[i].Off)
+		}
+	}
+}

plugin/net/cni.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/json"
 	"errors"
@@ -14,6 +15,7 @@ import (
 	current "github.com/containernetworking/cni/pkg/types/100"
 	"github.com/containernetworking/plugins/pkg/ipam"
 	weaveapi "github.com/rajch/weave/api"
+	"github.com/rajch/weave/lb/admit"
 	weavenet "github.com/rajch/weave/net"
 	ipamplugin "github.com/rajch/weave/plugin/ipam"
 	"github.com/vishvananda/netlink"
@@ -86,6 +88,18 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 	// Only expecting one address
 	ip := result.IPs[0]
 
+	// An EgressService member's first packet must already leave from its Service's address (or not at all):
+	// weave-npc installs the pod's egress rules before the pod has an interface. No answer, no pod (the
+	// runtime calls DEL, which releases the address, and kubelet retries).
+	if conf.EgressAdmission {
+		if ns, name := admit.PodFromArgs(args.Args); ns != "" && name != "" {
+			if _, err := admit.Admit(context.Background(), admit.SocketPath,
+				admit.Request{Namespace: ns, Name: name, IP: ip.Address.IP.String()}); err != nil {
+				return err
+			}
+		}
+	}
+
 	// If config says nothing about routes or gateway, default one will be via the bridge
 	if len(result.Routes) == 0 && ip.Gateway == nil {
 		bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &ip.Address)
@@ -147,6 +161,11 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 	}); err != nil {
 		return fmt.Errorf("error setting up routes: %s", err)
 	}
+	// the pod may only speak as itself: its port drops whatever claims another MAC or address. The pod has not
+	// started yet; if its port cannot be locked, it does not start.
+	if err := weavenet.LockContainerPort(ns, args.IfName, ip.Address.IP); err != nil {
+		return fmt.Errorf("error locking the pod's port: %s", err)
+	}
 
 	result.DNS = conf.DNS
 	return types.PrintResult(result, conf.CNIVersion)
@@ -241,4 +260,6 @@ type NetConf struct {
 	IPMasq      bool   `json:"ipMasq"`
 	MTU         int    `json:"mtu"`
 	HairpinMode bool   `json:"hairpinMode"`
+	// EgressAdmission: weave-npc must admit every pod before it gets its interface (EgressService members)
+	EgressAdmission bool `json:"egressAdmission"`
 }

prog/weave-kube/init.sh

@@ -63,4 +63,9 @@ if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then
 fi
 mkdir -p $HOST_ROOT/etc/cni/net.d
 export HOST_ROOT
-/home/weave/weave --local setup-cni
+/home/weave/weave --local setup-cni
+
+# Lock the ports of pods attached before the CNI plugin locked ports itself (it locks every pod it attaches from
+# now on), from the runtime's record of each attachment. A port that cannot be locked stays as it was and is
+# reported; weave starts regardless, since holding back the whole node's network would not make that port safer.
+/usr/bin/weaveutil lock-ports "$HOST_ROOT/var/lib/cni/results" weave || echo "Some pods' ports could not be locked (see above)" >&2

prog/weave-kube/weave-daemonset-k8s-1.11.yaml

@@ -1,6 +1,68 @@
 apiVersion: v1
 kind: List
 items:
+  # EgressService (reweave.levk.dev/v1alpha1), served by weave-npc's lb agent: a LoadBalancer Service's pods
+  # leave the cluster from the Service's address (named after the Service, same namespace)
+  - apiVersion: apiextensions.k8s.io/v1
+    kind: CustomResourceDefinition
+    metadata:
+      name: egressservices.reweave.levk.dev
+      labels:
+        name: weave-net
+    spec:
+      group: reweave.levk.dev
+      scope: Namespaced
+      names:
+        plural: egressservices
+        singular: egressservice
+        kind: EgressService
+        listKind: EgressServiceList
+      versions:
+        - name: v1alpha1
+          served: true
+          storage: true
+          subresources:
+            status: {}
+          additionalPrinterColumns:
+            - {name: Emit, type: string, jsonPath: .spec.emit}
+            - {name: Host, type: string, jsonPath: .status.host}
+            - {name: Address, type: string, jsonPath: .status.address}
+            - {name: Age, type: date, jsonPath: .metadata.creationTimestamp}
+          schema:
+            openAPIV3Schema:
+              type: object
+              description: >-
+                Makes every pod the same-named LoadBalancer Service (class reweave.levk.dev/node-address)
+                selects leave the cluster from that Service's address, or not at all.
+              required: [spec]
+              properties:
+                spec:
+                  type: object
+                  required: [emit]
+                  properties:
+                    sourceIPBy:
+                      type: string
+                      description: LoadBalancerIP - members leave from the Service's load-balancer address.
+                      enum: [LoadBalancerIP]
+                      default: LoadBalancerIP
+                    emit:
+                      type: string
+                      description: Holder - members' packets are tunnelled to the node holding the address, which emits them.
+                      enum: [Holder]
+                status:
+                  type: object
+                  properties:
+                    host: {type: string}
+                    address: {type: string}
+                    blocks:
+                      type: array
+                      items:
+                        type: object
+                        required: [node, from, to]
+                        properties:
+                          node: {type: string}
+                          from: {type: integer, format: int32}
+                          to: {type: integer, format: int32}
   - apiVersion: v1
     kind: ServiceAccount
     metadata:
@@ -48,6 +110,72 @@ items:
         verbs:
         - patch
         - update
+      # weave-npc's LoadBalancer agent (lb): Services of class reweave.levk.dev/node-address
+      - apiGroups:
+        - ''
+        resources:
+        - services
+        verbs:
+        - get
+        - list
+        - watch
+      - apiGroups:
+        - ''
+        resources:
+        - services/status
+        verbs:
+        - patch
+      - apiGroups:
+        - discovery.k8s.io
+        resources:
+        - endpointslices
+        verbs:
+        - get
+        - list
+        - watch
+      # each node publishes the addresses it holds and its weave bridge (annotations)
+      - apiGroups:
+        - ''
+        resources:
+        - nodes
+        verbs:
+        - patch
+      - apiGroups:
+        - reweave.levk.dev
+        resources:
+        - egressservices
+        verbs:
+        - get
+        - list
+        - watch
+      - apiGroups:
+        - reweave.levk.dev
+        resources:
+        - egressservices/status
+        verbs:
+        - patch
+      # Gateways of controller reweave.levk.dev/gateway (the Gateway API CRDs are installed separately)
+      - apiGroups:
+        - gateway.networking.k8s.io
+        resources:
+        - gatewayclasses
+        - gateways
+        - httproutes
+        - tcproutes
+        - referencegrants
+        verbs:
+        - get
+        - list
+        - watch
+      - apiGroups:
+        - gateway.networking.k8s.io
+        resources:
+        - gatewayclasses/status
+        - gateways/status
+        - httproutes/status
+        - tcproutes/status
+        verbs:
+        - patch
   - apiVersion: rbac.authorization.k8s.io/v1
     kind: ClusterRoleBinding
     metadata:
@@ -100,6 +228,42 @@ items:
       - kind: ServiceAccount
         name: weave-net
         namespace: kube-system
+  # Gateways' certificates: the agent reads Secrets in gateway-system only (weave-npc --gateway-namespaces)
+  - apiVersion: v1
+    kind: Namespace
+    metadata:
+      name: gateway-system
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: Role
+    metadata:
+      name: weave-net-gateway-certificates
+      namespace: gateway-system
+      labels:
+        name: weave-net
+    rules:
+      - apiGroups:
+        - ''
+        resources:
+        - secrets
+        verbs:
+        - get
+        - list
+        - watch
+  - apiVersion: rbac.authorization.k8s.io/v1
+    kind: RoleBinding
+    metadata:
+      name: weave-net-gateway-certificates
+      namespace: gateway-system
+      labels:
+        name: weave-net
+    roleRef:
+      kind: Role
+      name: weave-net-gateway-certificates
+      apiGroup: rbac.authorization.k8s.io
+    subjects:
+      - kind: ServiceAccount
+        name: weave-net
+        namespace: kube-system
   - apiVersion: apps/v1
     kind: DaemonSet
     metadata:
@@ -139,6 +303,15 @@ items:
                 - name: xtables-lock
                   mountPath: /run/xtables.lock
                   readOnly: false
+                # the runtime's record of each pod attachment, and the pods' network namespaces it names: init
+                # locks the ports of pods attached before the CNI plugin locked ports itself
+                - name: cni-cache
+                  mountPath: /host/var/lib/cni
+                  readOnly: true
+                - name: netns
+                  mountPath: /var/run/netns
+                  mountPropagation: HostToContainer
+                  readOnly: true
           containers:
             - name: weave
               command:
@@ -196,6 +369,8 @@ items:
                 - name: xtables-lock
                   mountPath: /run/xtables.lock
                   readOnly: false
+                - name: weave-npc-run
+                  mountPath: /run/weave-npc
           hostNetwork: true
           dnsPolicy: ClusterFirstWithHostNet
           hostPID: false
@@ -234,6 +409,18 @@ items:
               hostPath:
                 path: /run/xtables.lock
                 type: FileOrCreate
+            - name: weave-npc-run
+              hostPath:
+                path: /run/weave-npc
+                type: DirectoryOrCreate
+            - name: cni-cache
+              hostPath:
+                path: /var/lib/cni
+                type: DirectoryOrCreate
+            - name: netns
+              hostPath:
+                path: /var/run/netns
+                type: DirectoryOrCreate
           priorityClassName: system-node-critical
       updateStrategy:
         type: RollingUpdate

prog/weave-npc/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/signal"
 	goruntime "runtime"
+	"strings"
 	"syscall"
 
 	"github.com/coreos/go-iptables/iptables"
@@ -21,6 +22,7 @@ import (
 
 	"github.com/rajch/weave/common"
 	"github.com/rajch/weave/common/chains"
+	"github.com/rajch/weave/lb"
 	"github.com/rajch/weave/net"
 	"github.com/rajch/weave/net/ipset"
 	"github.com/rajch/weave/npc"
@@ -29,13 +31,15 @@ import (
 )
 
 var (
-	version        = "unreleased"
-	metricsAddr    string
-	logLevel       string
-	allowMcast     bool
-	nodeName       string
-	maxList        int
-	bridgePortName string
+	version           = "unreleased"
+	metricsAddr       string
+	logLevel          string
+	allowMcast        bool
+	nodeName          string
+	maxList           int
+	bridgePortName    string
+	serveLB           bool
+	gatewayNamespaces string
 )
 
 func handleError(err error) {
@@ -337,6 +341,21 @@ func root(cmd *cobra.Command, args []string) {
 	go podController.Run(wait.NeverStop)
 	go npController.Run(wait.NeverStop)
 
+	if serveLB {
+		// LoadBalancer Services of class lb.Class: addresses this node holds, client address kept; and
+		// EgressServices: those Services' pods leave the cluster from that address (or not at all)
+		egress, err := lb.NewEgressClient(config)
+		handleFatal(err)
+		ctl := lb.NewController(nodeName, client, egress, lb.NewDatapath(ipt))
+		if gatewayNamespaces != "" {
+			// and Gateways of class controller lb.GatewayControllerName, from the node holding their address
+			gw, err := lb.NewGatewayClient(config)
+			handleFatal(err)
+			ctl = ctl.WithGateway(gw, strings.Split(gatewayNamespaces, ","))
+		}
+		go ctl.Run(context.Background())
+	}
+
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
 	common.Log.Fatalf("Exiting: %v", <-signals)
@@ -354,6 +373,9 @@ func main() {
 	rootCmd.PersistentFlags().StringVar(&nodeName, "node-name", "", "only generate rules that apply to this node")
 	rootCmd.PersistentFlags().IntVar(&maxList, "max-list-size", 1024, "maximum size of ipset list (for namespaces)")
 	rootCmd.PersistentFlags().StringVar(&bridgePortName, "bridge-port-name", "vethwe-bridge", "name of the brige port on which packets are received and sent")
+	rootCmd.PersistentFlags().BoolVar(&serveLB, "lb", true, "serve LoadBalancer Services of class "+lb.Class+" from addresses nodes hold")
+	rootCmd.PersistentFlags().StringVar(&gatewayNamespaces, "gateway-namespaces", "gateway-system",
+		"serve Gateways of controller "+lb.GatewayControllerName+", with certificates from Secrets in these namespaces (comma-separated; empty: no Gateways)")
 
 	handleFatal(rootCmd.Execute())
 }

prog/weaveutil/main.go

@@ -42,6 +42,7 @@ func init() {
 		"rewrite-etc-hosts":        rewriteEtcHosts,
 		"get-db-flag":              getDBFlag,
 		"set-db-flag":              setDBFlag,
+		"lock-ports":               lockPorts,
 	}
 }
 

prog/weaveutil/portlock.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/vishvananda/netns"
+
+	weavenet "github.com/rajch/weave/net"
+)
+
+// lockPorts locks the ports of pods that were attached before the CNI plugin locked ports itself (it locks every
+// pod it attaches). It works from the runtime's own record of each attachment, libcni's result cache: the pod's
+// network namespace, the interface the plugin set up in it and the address it gave it. Ports already locked are
+// left alone; pods whose namespace is gone are skipped.
+func lockPorts(args []string) error {
+	if len(args) != 2 {
+		cmdUsage("lock-ports", "<cni-result-cache-dir> <network-name>")
+	}
+	dir, network := args[0], args[1]
+	entries, err := os.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return nil // nothing attached yet
+	}
+	if err != nil {
+		return err
+	}
+	var locked, gone, failed int
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasPrefix(e.Name(), network+"-") {
+			continue
+		}
+		data, err := os.ReadFile(filepath.Join(dir, e.Name()))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "lock-ports: %s: %v\n", e.Name(), err)
+			failed++
+			continue
+		}
+		a, err := parseAttachment(data)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "lock-ports: %s: %v\n", e.Name(), err)
+			failed++
+			continue
+		}
+		if a.network != network {
+			continue
+		}
+		ns, err := netns.GetFromPath(a.netns)
+		if os.IsNotExist(err) {
+			gone++ // the pod is gone; the runtime has yet to clear its record
+			continue
+		}
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "lock-ports: %s: namespace %s: %v\n", a.containerID, a.netns, err)
+			failed++
+			continue
+		}
+		err = weavenet.LockContainerPort(ns, a.ifName, a.ip)
+		ns.Close()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "lock-ports: %s (%s): %v\n", a.containerID, a.ip, err)
+			failed++
+			continue
+		}
+		locked++
+	}
+	fmt.Printf("lock-ports: %d locked, %d gone, %d failed\n", locked, gone, failed)
+	if failed > 0 {
+		return fmt.Errorf("lock-ports: %d pods' ports not locked", failed)
+	}
+	return nil
+}
+
+type attachment struct {
+	network, containerID, netns, ifName string
+	ip                                  net.IP
+}
+
+// parseAttachment reads one libcni cache entry ("cniCacheV1"): which network attached which container, in which
+// namespace, through which interface, with which IPv4 address.
+func parseAttachment(data []byte) (attachment, error) {
+	var c struct {
+		Kind        string `json:"kind"`
+		ContainerID string `json:"containerId"`
+		IfName      string `json:"ifName"`
+		NetworkName string `json:"networkName"`
+		NetNS       string `json:"netns"`
+		Result      struct {
+			IPs []struct {
+				Address string `json:"address"`
+			} `json:"ips"`
+		} `json:"result"`
+	}
+	if err := json.Unmarshal(data, &c); err != nil {
+		return attachment{}, err
+	}
+	if c.Kind != "cniCacheV1" {
+		return attachment{}, fmt.Errorf("unknown cache entry kind %q", c.Kind)
+	}
+	if c.NetNS == "" || c.IfName == "" {
+		return attachment{}, fmt.Errorf("cache entry for %s names no namespace or interface", c.ContainerID)
+	}
+	for _, a := range c.Result.IPs {
+		ip, _, err := net.ParseCIDR(a.Address)
+		if err == nil && ip.To4() != nil {
+			return attachment{network: c.NetworkName, containerID: c.ContainerID, netns: c.NetNS, ifName: c.IfName, ip: ip.To4()}, nil
+		}
+	}
+	return attachment{}, fmt.Errorf("cache entry for %s has no IPv4 address", c.ContainerID)
+}

prog/weaveutil/portlock_test.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"net"
+	"testing"
+)
+
+// shaped like CRI-O's libcni cache on the SG nodes (cniCacheV1, CNI 1.0.0 result); the config is elided
+const cacheEntry = `{"kind":"cniCacheV1","containerId":"2b61982132d6039b210e2fa5896555c8a921a07c756bd8753963440e07dc5f98",
+"config":"eyJ9","ifName":"eth0","networkName":"weave","netns":"/var/run/netns/8828bbba-9a68-460c-980b-266dc3c9342c",
+"cniArgs":[["K8S_POD_NAMESPACE","default"]],"capabilityArgs":{},
+"result":{"cniVersion":"1.0.0","ips":[{"address":"10.44.0.12/12","gateway":"10.44.0.0"}]}}`
+
+func TestParseAttachment(t *testing.T) {
+	a, err := parseAttachment([]byte(cacheEntry))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if a.network != "weave" || a.ifName != "eth0" || a.netns != "/var/run/netns/8828bbba-9a68-460c-980b-266dc3c9342c" ||
+		!a.ip.Equal(net.ParseIP("10.44.0.12")) || a.containerID[:7] != "2b61982" {
+		t.Fatalf("parsed %+v", a)
+	}
+}
+
+func TestParseAttachmentRefusesWhatItCannotLock(t *testing.T) {
+	for name, entry := range map[string]string{
+		"other kind":   `{"kind":"cniCacheV2","netns":"/x","ifName":"eth0","result":{"ips":[{"address":"10.0.0.1/8"}]}}`,
+		"no netns":     `{"kind":"cniCacheV1","ifName":"eth0","result":{"ips":[{"address":"10.0.0.1/8"}]}}`,
+		"no ipv4":      `{"kind":"cniCacheV1","netns":"/x","ifName":"eth0","result":{"ips":[{"address":"fd00::1/64"}]}}`,
+		"not json":     `{`,
+		"no addresses": `{"kind":"cniCacheV1","netns":"/x","ifName":"eth0","result":{}}`,
+	} {
+		if _, err := parseAttachment([]byte(entry)); err == nil {
+			t.Errorf("%s: want an error", name)
+		}
+	}
+}

reweave/test/egress/egress-test.yaml

@@ -0,0 +1,76 @@
+# End-to-end test of EgressService (weave-npc's lb agent). Placeholders are filled by run.sh: ADDR (an
+# address HOLDER holds), HOLDER, REMOTE and OTHER (the cluster's other two nodes).
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: Namespace
+    metadata:
+      name: egtest
+  - apiVersion: v1
+    kind: Service
+    metadata:
+      name: mail
+      namespace: egtest
+    spec:
+      type: LoadBalancer
+      loadBalancerClass: reweave.levk.dev/node-address
+      loadBalancerIP: ADDR
+      allocateLoadBalancerNodePorts: false
+      selector: {egress: member}
+      ports: [{name: probe, port: 18090, targetPort: 8080}]
+  - apiVersion: reweave.levk.dev/v1alpha1
+    kind: EgressService
+    metadata:
+      name: mail
+      namespace: egtest
+    spec:
+      emit: Holder
+  # members on every node, and one that is not a member
+  - apiVersion: v1
+    kind: Pod
+    metadata: {name: member-remote, namespace: egtest, labels: {egress: member}}
+    spec:
+      nodeName: REMOTE
+      containers:
+        # root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user)
+        - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}}
+  - apiVersion: v1
+    kind: Pod
+    metadata: {name: member-other, namespace: egtest, labels: {egress: member}}
+    spec:
+      nodeName: OTHER
+      containers:
+        # root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user)
+        - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}}
+  - apiVersion: v1
+    kind: Pod
+    metadata: {name: member-holder, namespace: egtest, labels: {egress: member}}
+    spec:
+      nodeName: HOLDER
+      containers:
+        # root + NET_RAW: ping and traceroute need raw sockets (the image runs as an unprivileged user)
+        - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"], securityContext: {runAsUser: 0, capabilities: {add: [NET_RAW]}}}
+  - apiVersion: v1
+    kind: Pod
+    metadata: {name: member-dns, namespace: egtest, labels: {egress: member}}
+    spec:
+      nodeName: REMOTE
+      containers: [{name: c, image: "registry.k8s.io/e2e-test-images/jessie-dnsutils:1.7", command: ["sleep", "3600"]}]
+  - apiVersion: v1
+    kind: Pod
+    metadata: {name: plain, namespace: egtest, labels: {egress: "none"}}
+    spec:
+      nodeName: REMOTE
+      containers: [{name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}]
+  # east-west: /clientip answers with the address the connection came from
+  - apiVersion: v1
+    kind: Pod
+    metadata: {name: echo, namespace: egtest, labels: {app: echo}}
+    spec:
+      nodeName: OTHER
+      containers:
+        - name: echo
+          image: registry.k8s.io/e2e-test-images/agnhost:2.53
+          args: ["netexec", "--http-port=8080"]
+          readinessProbe: {httpGet: {path: /healthz, port: 8080}}

reweave/test/egress/run.sh

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+# End-to-end test of EgressService against a live cluster (assertions A1-A6 of the design, plus the first
+# packet). Members ask outside services which address they come from: api.ipify.org (TCP) and OpenDNS's
+# myip.opendns.com (UDP).
+#
+#   CP="ssh ... root@<cp>" HOLDER_SSH="ssh ... root@<holder>" REMOTE_SSH="ssh ... root@<remote>" \
+#   ADDR=<address HOLDER holds> HOLDER=<node> REMOTE=<node> OTHER=<node> \
+#   reweave/test/egress/run.sh
+#
+# Asserts, in order:
+#   0. the holder writes the status: host, address, a port block for every node
+#   1. the first packet of a member started after the EgressService leaves as ADDR (CNI admission)
+#   A1/A2. members on every node leave as ADDR (TCP and UDP) and get their replies; ping works from every node
+#   ICMP errors from routers on the way (traceroute's time-exceeded) reach a member on another node
+#   A4. a non-member pod on the same node as a member leaves as its node (untouched), and no rule names it
+#   A3. a member talking to a pod sees its own pod address there (east-west untouched)
+#   A5. naming an address nobody holds clears the status, and members' outside traffic (TCP, ping) is dropped
+#   A6. deleting the EgressService removes its state: members leave as their node again, rules are gone
+# Leaves nothing behind (namespace egtest is deleted, also on failure).
+set -euo pipefail
+
+: "${CP:?}" "${HOLDER_SSH:?}" "${REMOTE_SSH:?}" "${ADDR:?}" "${HOLDER:?}" "${REMOTE:?}" "${OTHER:?}"
+here=$(cd "$(dirname "$0")" && pwd)
+k() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $*"; }
+fail() { echo "FAIL: $*" >&2; exit 1; }
+ok() { echo "ok: $*"; }
+cleanup() { k delete namespace egtest --ignore-not-found --wait=false >/dev/null 2>&1 || true; }
+trap cleanup EXIT
+
+eventually() {
+    local n=$1; shift
+    for _ in $(seq "$n"); do
+        if "$@" >/dev/null 2>&1; then return 0; fi
+        sleep 1
+    done
+    return 1
+}
+whoami_tcp() { k -n egtest exec "$1" -- curl -s --max-time 10 https://api.ipify.org; }
+# each check re-reads on every try of eventually
+tcp_is() { [ "$(whoami_tcp "$1" || true)" = "$2" ]; }
+whoami_udp() { k -n egtest exec "$1" -- dig +short +time=5 +tries=1 myip.opendns.com @208.67.222.222; }
+node_ip() { k get node "$1" -o "jsonpath='{.status.addresses[?(@.type==\"InternalIP\")].address}'"; }
+
+remote_ip=$(node_ip "$REMOTE")
+[ -n "$remote_ip" ] || fail "no address for $REMOTE"
+
+sed -e "s/\bADDR\b/$ADDR/g; s/\bHOLDER\b/$HOLDER/g; s/\bREMOTE\b/$REMOTE/g; s/\bOTHER\b/$OTHER/g" \
+    "$here/egress-test.yaml" | $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -" >/dev/null
+k -n egtest wait --for=condition=Ready pod --all --timeout=300s >/dev/null || fail "test pods not ready"
+
+status() { k -n egtest get egressservice mail -o "jsonpath='{.status.host} {.status.address} {.status.blocks[*].node}'"; }
+status_is() { [ "$(status)" = "$1" ]; }
+first_line() { k -n egtest logs member-first 2>/dev/null | head -1; }
+said_something() { [ -n "$(first_line)" ]; }
+want="$HOLDER $ADDR $(printf '%s\n' "$HOLDER" "$REMOTE" "$OTHER" | sort | tr '\n' ' ' | sed 's/ $//')"
+eventually 60 status_is "$want" || fail "status: '$(status)', want '$want'"
+ok "0. status: host $HOLDER, address $ADDR, a port block for each node"
+
+# a member created now: its very first connection must already leave as ADDR
+$CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n egtest run member-first --labels egress=member --restart=Never \
+    --overrides='{\"spec\":{\"nodeName\":\"$REMOTE\"}}' --image=curlimages/curl:8.10.1 --command -- \
+    sh -c 'curl -s --max-time 10 https://api.ipify.org; echo; sleep 3600'" >/dev/null
+eventually 120 said_something || fail "member-first said nothing"
+first=$(first_line)
+[ "$first" = "$ADDR" ] || fail "member-first's first connection left as '$first', want $ADDR"
+ok "1. a new member's first connection leaves as $ADDR"
+
+for p in member-remote member-other member-holder; do
+    got=$(whoami_tcp $p || true)
+    [ "$got" = "$ADDR" ] || fail "A1: $p left as '$got', want $ADDR"
+done
+got=$(whoami_udp member-dns || true)
+[ "$got" = "$ADDR" ] || fail "A1: member-dns (UDP) left as '$got', want $ADDR"
+ok "A1/A2. members on $REMOTE, $OTHER and $HOLDER leave as $ADDR over TCP and UDP, replies come back"
+
+for p in member-remote member-other member-holder; do
+    k -n egtest exec $p -- ping -c 2 -W 3 1.1.1.1 >/dev/null || fail "ping from $p got no reply"
+done
+ok "ping from members on all three nodes gets its replies"
+
+# hop 1 is the member's own node, hop 2 the holder (it answers its own address, the member cannot see it); every
+# later hop is a router on the internet whose time-exceeded comes back to ADDR and must find the member
+hops=$(k -n egtest exec member-remote -- traceroute -n -m 8 -q 1 -w 2 1.1.1.1 2>&1 || true)
+echo "$hops" | awk '$1 >= 3 && $2 ~ /^[0-9]+\./ {found=1} END {exit !found}' \
+    || fail "no router beyond the holder answered the member's traceroute:
+$hops"
+ok "ICMP errors from routers beyond the holder reach a member on $REMOTE (traceroute)"
+
+got=$(whoami_tcp plain || true)
+[ "$got" = "$remote_ip" ] || fail "A4: non-member left as '$got', want its node $remote_ip"
+plain_ip=$(k -n egtest get pod plain -o "jsonpath='{.status.podIP}'")
+if $REMOTE_SSH "iptables-save | grep -F -- '$plain_ip/32'; ip rule | grep -F -- '$plain_ip'"; then
+    fail "A4: a rule names the non-member $plain_ip"
+fi
+ok "A4. the non-member leaves as its node ($remote_ip); no rule names it"
+
+echo_ip=$(k -n egtest get pod echo -o "jsonpath='{.status.podIP}'")
+member_ip=$(k -n egtest get pod member-remote -o "jsonpath='{.status.podIP}'")
+got=$(k -n egtest exec member-remote -- curl -s --max-time 5 "http://$echo_ip:8080/clientip" || true)
+[ "${got%:*}" = "$member_ip" ] || fail "A3: echo saw '$got', want the member's pod address $member_ip"
+ok "A3. east-west: the echo pod sees the member's own address ($member_ip)"
+
+k -n egtest patch svc mail --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"192.0.2.1\"}}'" >/dev/null
+eventually 60 status_is "  " || fail "A5: status not cleared: '$(status)'"
+got=$(whoami_tcp member-remote || true)
+[ -z "$got" ] || fail "A5: with no holder a member still got out, as '$got'"
+if k -n egtest exec member-remote -- ping -c 1 -W 3 1.1.1.1 >/dev/null 2>&1; then
+    fail "A5: with no holder a member's ping still got out"
+fi
+k -n egtest patch svc mail --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"$ADDR\"}}'" >/dev/null
+eventually 60 status_is "$want" || fail "status not restored: '$(status)'"
+eventually 30 tcp_is member-remote "$ADDR" || fail "member did not get back out as $ADDR"
+ok "A5. no holder: status cleared, members dropped (not leaked); restored afterwards"
+
+k -n egtest delete egressservice mail >/dev/null
+eventually 60 tcp_is member-remote "$remote_ip" || fail "A6: after delete, member left as '$(whoami_tcp member-remote || true)'"
+if $REMOTE_SSH "ip rule | grep -E '^9100:'"; then fail "A6: member rules left on $REMOTE"; fi
+if $HOLDER_SSH "ip rule | grep -E '^90:'"; then fail "A6: holder rules left on $HOLDER"; fi
+ok "A6. EgressService deleted: members leave as their node again, no rules left"
+echo PASS

reweave/test/gateway/gateway-test.yaml

@@ -0,0 +1,126 @@
+# Objects for the Gateway e2e (run.sh): two echo backends (agnhost netexec: /hostname, /clientip, /header), one on
+# the holder and one elsewhere, and a Gateway of class reweave on test ports of ADDR (HTTPS with the test
+# certificate, plain HTTP that redirects, and TCP straight to a backend), with its routes.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gwtest
+---
+apiVersion: v1
+kind: Pod
+metadata: {name: echo-a, namespace: gwtest, labels: {app: echo-a}}
+spec:
+  nodeName: HOLDER
+  containers:
+    - {name: c, image: "registry.k8s.io/e2e-test-images/agnhost:2.59", args: ["netexec", "--http-port=8080"]}
+---
+apiVersion: v1
+kind: Pod
+metadata: {name: echo-b, namespace: gwtest, labels: {app: echo-b}}
+spec:
+  nodeName: OTHER
+  containers:
+    - {name: c, image: "registry.k8s.io/e2e-test-images/agnhost:2.59", args: ["netexec", "--http-port=8080"]}
+---
+apiVersion: v1
+kind: Pod
+metadata: {name: client, namespace: gwtest}
+spec:
+  nodeName: OTHER
+  containers:
+    - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}
+---
+apiVersion: v1
+kind: Service
+metadata: {name: echo-a, namespace: gwtest}
+spec:
+  selector: {app: echo-a}
+  ports: [{name: http, port: 80, targetPort: 8080}]
+---
+apiVersion: v1
+kind: Service
+metadata: {name: echo-b, namespace: gwtest}
+spec:
+  selector: {app: echo-b}
+  ports: [{name: http, port: 80, targetPort: 8080}]
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata: {name: gwtest, namespace: gateway-system}
+spec:
+  gatewayClassName: reweave
+  addresses: [{type: IPAddress, value: ADDR}]
+  listeners:
+    - name: https
+      port: 18443
+      protocol: HTTPS
+      hostname: gw.test
+      tls: {certificateRefs: [{name: gwtest-cert}]}
+      allowedRoutes: {namespaces: {from: Selector, selector: {matchLabels: {kubernetes.io/metadata.name: gwtest}}}}
+    - name: http
+      port: 18080
+      protocol: HTTP
+      allowedRoutes: {namespaces: {from: All}}
+    - name: tcp
+      port: 18022
+      protocol: TCP
+      allowedRoutes: {namespaces: {from: All}}
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata: {name: main, namespace: gwtest}
+spec:
+  parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: https}]
+  hostnames: [gw.test]
+  rules:
+    # exact beats the default prefix "/"; /a/... reaches echo-a with /a cut off; the rest goes to echo-a as is
+    - matches: [{path: {type: Exact, value: /hostname}}]
+      backendRefs: [{name: echo-b, port: 80}]
+    - matches: [{path: {type: PathPrefix, value: /a}}]
+      backendRefs: [{name: echo-a, port: 80}]
+      filters:
+        - type: URLRewrite
+          urlRewrite: {path: {type: ReplacePrefixMatch, replacePrefixMatch: /}}
+    - backendRefs: [{name: echo-a, port: 80}]
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata: {name: broken, namespace: gwtest}
+spec:
+  parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: https}]
+  hostnames: [gw.test]
+  rules:
+    - matches: [{path: {type: PathPrefix, value: /broken}}]
+      backendRefs: [{name: missing, port: 80}]
+---
+# refused: the gateway sets the forwarding headers itself, so no route may change them
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata: {name: forge, namespace: gwtest}
+spec:
+  parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: https}]
+  hostnames: [gw.test]
+  rules:
+    - matches: [{path: {type: PathPrefix, value: /forge}}]
+      filters:
+        - type: RequestHeaderModifier
+          requestHeaderModifier: {set: [{name: X-Forwarded-For, value: 6.6.6.6}]}
+      backendRefs: [{name: echo-a, port: 80}]
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata: {name: redirect, namespace: gwtest}
+spec:
+  parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: http}]
+  rules:
+    - filters:
+        - type: RequestRedirect
+          requestRedirect: {scheme: https, port: 18443, statusCode: 301}
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: TCPRoute
+metadata: {name: raw, namespace: gwtest}
+spec:
+  parentRefs: [{name: gwtest, namespace: gateway-system, sectionName: tcp}]
+  rules:
+    - backendRefs: [{name: echo-b, port: 80}]

reweave/test/gateway/run.sh

@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# End-to-end test of the agent's Gateways (class reweave) against a live cluster, on test ports of ADDR.
+#
+#   CP="ssh ... root@<cp>" HOLDER_SSH="ssh ... root@<holder>" ADDR=<address HOLDER holds> HOLDER=<node> \
+#   OTHER=<node> reweave/test/gateway/run.sh
+#
+# The machine running this script is the outside client. Asserts, in order:
+#   G0  GatewayClass reweave is accepted
+#   G1  the Gateway is accepted and programmed at ADDR, its listeners with their routes attached
+#   G2  route statuses: accepted and resolved, the broken one ResolvedRefs False (BackendNotFound), and the one whose
+#       filter sets X-Forwarded-For refused (Accepted False, UnsupportedValue)
+#   G3  TCP listener: forwarded in the kernel to a pod on another node, which sees this client's own address
+#   G4  HTTPS from outside: the listener's certificate by SNI, HTTP/2, routing by exact path, by prefix with a
+#       rewrite, and by default; the backend sees this client (the same address as G3) in X-Forwarded-For
+#   G5  plain HTTP redirects to HTTPS on the listener's port, path and query kept
+#   G6  an unresolved backend answers 500; a name no listener has gets no certificate
+#   G7  a pod inside the cluster reaches the Gateway too
+#   G8  a renewed certificate is served without a restart
+#   G9  fail closed: an address nobody holds is not programmed and nothing is served; back at ADDR, it is again
+#   G10 deleting the Gateway removes its accept and forward rules from the holder
+# Leaves nothing behind (namespace gwtest, the Gateway and its Secret are deleted, also on failure).
+set -euo pipefail
+
+: "${CP:?}" "${HOLDER_SSH:?}" "${ADDR:?}" "${HOLDER:?}" "${OTHER:?}"
+here=$(cd "$(dirname "$0")" && pwd)
+k() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $*"; }
+kin() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $* -f -"; }
+fail() { echo "FAIL: $*" >&2; exit 1; }
+ok() { echo "ok: $*"; }
+work=$(mktemp -d)
+cleanup() {
+    k -n gateway-system delete gateway gwtest --ignore-not-found >/dev/null 2>&1 || true
+    k -n gateway-system delete secret gwtest-cert --ignore-not-found >/dev/null 2>&1 || true
+    k delete namespace gwtest --ignore-not-found --wait=false >/dev/null 2>&1 || true
+    rm -rf "$work"
+}
+trap cleanup EXIT
+
+eventually() {
+    local n=$1; shift
+    for _ in $(seq "$n"); do
+        if "$@" >/dev/null 2>&1; then return 0; fi
+        sleep 1
+    done
+    return 1
+}
+
+# a test certificate for gw.test, as a TLS Secret in gateway-system
+cert() {
+    openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:P-256 -nodes -days 1 -subj "/CN=gw.test" \
+        -addext "subjectAltName=DNS:gw.test" -keyout "$work/$1.key" -out "$work/$1.crt" 2>/dev/null
+    printf 'apiVersion: v1\nkind: Secret\nmetadata: {name: gwtest-cert, namespace: gateway-system}\ntype: kubernetes.io/tls\ndata:\n  tls.crt: %s\n  tls.key: %s\n' \
+        "$(base64 -w0 < "$work/$1.crt")" "$(base64 -w0 < "$work/$1.key")" | kin apply >/dev/null
+}
+fingerprint() { openssl x509 -noout -fingerprint -sha256 -in "$1" | cut -d= -f2; }
+served() {
+    echo | openssl s_client -connect "$ADDR:18443" -servername gw.test 2>/dev/null |
+        openssl x509 -noout -fingerprint -sha256 2>/dev/null | cut -d= -f2
+}
+https() { curl -sk --max-time 10 --resolve "gw.test:18443:$ADDR" "$@"; }
+
+cert v1
+sed -e "s/\bADDR\b/$ADDR/g; s/\bHOLDER\b/$HOLDER/g; s/\bOTHER\b/$OTHER/g" "$here/gateway-test.yaml" | kin apply >/dev/null
+k -n gwtest wait --for=condition=Ready pod --all --timeout=300s >/dev/null || fail "test pods not ready"
+
+cond() { k get $1 -o "jsonpath='{.status.conditions[?(@.type==\"$2\")].status}/{.status.conditions[?(@.type==\"$2\")].reason}'"; }
+is() { [ "$(cond "$1" "$2")" = "$3" ]; }
+eventually 60 is "gatewayclass reweave" Accepted "True/Accepted" || fail "G0: GatewayClass: $(cond "gatewayclass reweave" Accepted)"
+ok "G0. GatewayClass reweave accepted"
+
+gw="-n gateway-system gateway gwtest"
+eventually 90 is "$gw" Programmed "True/Programmed" || fail "G1: programmed: $(cond "$gw" Programmed)"
+is "$gw" Accepted "True/Accepted" || fail "G1: accepted: $(cond "$gw" Accepted)"
+[ "$(k get $gw -o "jsonpath='{.status.addresses[0].value}'")" = "$ADDR" ] || fail "G1: status address"
+attached=$(k get $gw -o "jsonpath='{range .status.listeners[*]}{.name}={.attachedRoutes} {end}'")
+[ "$attached" = "https=2 http=1 tcp=1 " ] || fail "G1: attached routes: '$attached'"
+ok "G1. Gateway accepted and programmed at $ADDR; https=2 http=1 tcp=1 routes attached"
+
+route() { k -n gwtest get "$1" "$2" -o "jsonpath='{.status.parents[0].conditions[?(@.type==\"$3\")].status}/{.status.parents[0].conditions[?(@.type==\"$3\")].reason}'"; }
+route_is() { [ "$(route "$1" "$2" "$3")" = "$4" ]; }
+for r in "httproute main" "httproute redirect" "tcproute raw" "httproute broken"; do
+    eventually 30 route_is $r Accepted "True/Accepted" || fail "G2: $r accepted: $(route $r Accepted)"
+done
+for r in "httproute main" "httproute redirect" "tcproute raw"; do
+    route_is $r ResolvedRefs "True/ResolvedRefs" || fail "G2: $r resolved: $(route $r ResolvedRefs)"
+done
+route_is httproute broken ResolvedRefs "False/BackendNotFound" || fail "G2: broken: $(route httproute broken ResolvedRefs)"
+eventually 30 route_is httproute forge Accepted "False/UnsupportedValue" || fail "G2: forge: $(route httproute forge Accepted)"
+ok "G2. routes accepted and resolved; the broken one ResolvedRefs False (BackendNotFound); the forging one refused"
+
+tcp_hostname() { [ "$(curl -s --max-time 5 "http://$ADDR:18022/hostname")" = echo-b ]; }
+eventually 30 tcp_hostname || fail "G3: TCP listener did not reach echo-b"
+me=$(curl -s --max-time 5 "http://$ADDR:18022/clientip" | sed 's/:[0-9]*$//')
+case "$me" in
+    10.*|"") fail "G3: the backend saw '$me', not this client's own address" ;;
+esac
+ok "G3. TCP listener forwards to echo-b on $OTHER, which sees this client as $me"
+
+[ "$(served)" = "$(fingerprint "$work/v1.crt")" ] || fail "G4: wrong certificate for gw.test"
+[ "$(https -o /dev/null -w '%{http_version}' https://gw.test:18443/hostname)" = 2 ] || fail "G4: no HTTP/2"
+[ "$(https https://gw.test:18443/hostname)" = echo-b ] || fail "G4: exact /hostname did not reach echo-b"
+[ "$(https https://gw.test:18443/a/hostname)" = echo-a ] || fail "G4: prefix /a (rewritten to /) did not reach echo-a"
+xff=$(https "https://gw.test:18443/header?key=X-Forwarded-For")
+[ "$xff" = "$me" ] || fail "G4: X-Forwarded-For '$xff', want $me"
+proto=$(https "https://gw.test:18443/header?key=X-Forwarded-Proto")
+[ "$proto" = https ] || fail "G4: X-Forwarded-Proto '$proto'"
+ok "G4. HTTPS: gw.test's certificate, HTTP/2, exact/prefix+rewrite/default routing; X-Forwarded-For $me"
+
+loc=$(curl -s -o /dev/null --max-time 5 -w '%{http_code} %{redirect_url}' -H 'Host: gw.test' "http://$ADDR:18080/a/x?y=1")
+[ "$loc" = "301 https://gw.test:18443/a/x?y=1" ] || fail "G5: redirect: '$loc'"
+ok "G5. HTTP redirects to https://gw.test:18443, path and query kept"
+
+[ "$(https -o /dev/null -w '%{http_code}' https://gw.test:18443/broken)" = 500 ] || fail "G6: broken backend did not answer 500"
+if curl -sk --max-time 5 --resolve "other.test:18443:$ADDR" https://other.test:18443/ >/dev/null 2>&1; then
+    fail "G6: a name no listener has got a certificate"
+fi
+ok "G6. unresolved backend answers 500; a name no listener has gets no certificate"
+
+inside=$(k -n gwtest exec client -- curl -sk --max-time 10 --resolve "gw.test:18443:$ADDR" https://gw.test:18443/hostname)
+[ "$inside" = echo-b ] || fail "G7: from a pod: '$inside'"
+ok "G7. a pod on $OTHER reaches the Gateway"
+
+cert v2
+renewed() { [ "$(served)" = "$(fingerprint "$work/v2.crt")" ]; }
+eventually 60 renewed || fail "G8: the renewed certificate is not served"
+ok "G8. renewed certificate served without a restart"
+
+k patch $gw --type=merge -p "'{\"spec\":{\"addresses\":[{\"type\":\"IPAddress\",\"value\":\"198.51.100.7\"}]}}'" >/dev/null
+eventually 60 is "$gw" Programmed "False/AddressNotUsable" || fail "G9: unheld address: $(cond "$gw" Programmed)"
+gone() { ! https -o /dev/null https://gw.test:18443/hostname; }
+eventually 30 gone || fail "G9: still served from $ADDR after the Gateway moved off it"
+k patch $gw --type=merge -p "'{\"spec\":{\"addresses\":[{\"type\":\"IPAddress\",\"value\":\"$ADDR\"}]}}'" >/dev/null
+back() { [ "$(https https://gw.test:18443/hostname)" = echo-b ]; }
+eventually 60 back || fail "G9: not served again at $ADDR"
+ok "G9. an address nobody holds: Programmed False (AddressNotUsable), nothing served; back at $ADDR: served"
+
+k -n gateway-system delete gateway gwtest >/dev/null
+rules_gone() {
+    ! $HOLDER_SSH "iptables -S REWEAVE-LB-IN; iptables -t nat -S REWEAVE-LB" | grep -q 'gateway-system/gwtest\|gateway gateway-system/gwtest'
+}
+eventually 60 rules_gone || fail "G10: the holder still has the Gateway's rules"
+ok "G10. Gateway deleted: its accept and forward rules are gone from $HOLDER"
+echo PASS

reweave/test/lb/lb-test.yaml

@@ -0,0 +1,75 @@
+# End-to-end test of weave-npc's LoadBalancer agent (lb). Placeholders are
+# filled by run.sh: ADDR (an address HOLDER holds), HOLDER (that node),
+# REMOTE (another node: its pod's replies must be steered back to HOLDER),
+# OTHER (a third node: a pod there connects to ADDR from inside the cluster).
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: Namespace
+    metadata:
+      name: lbtest
+  # /clientip answers with the address the connection came from
+  - apiVersion: v1
+    kind: Pod
+    metadata:
+      name: echo-remote
+      namespace: lbtest
+      labels: {app: echo-remote}
+    spec:
+      nodeName: REMOTE
+      containers:
+        - name: echo
+          image: registry.k8s.io/e2e-test-images/agnhost:2.53
+          args: ["netexec", "--http-port=8080"]
+          ports: [{name: http, containerPort: 8080}]
+          readinessProbe: {httpGet: {path: /healthz, port: 8080}}
+  - apiVersion: v1
+    kind: Pod
+    metadata:
+      name: echo-local
+      namespace: lbtest
+      labels: {app: echo-local}
+    spec:
+      nodeName: HOLDER
+      containers:
+        - name: echo
+          image: registry.k8s.io/e2e-test-images/agnhost:2.53
+          args: ["netexec", "--http-port=8080"]
+          ports: [{name: http, containerPort: 8080}]
+          readinessProbe: {httpGet: {path: /healthz, port: 8080}}
+  - apiVersion: v1
+    kind: Pod
+    metadata:
+      name: client
+      namespace: lbtest
+    spec:
+      nodeName: OTHER
+      containers:
+        - name: client
+          image: registry.k8s.io/e2e-test-images/agnhost:2.53
+          args: ["pause"]
+  - apiVersion: v1
+    kind: Service
+    metadata:
+      name: echo-remote
+      namespace: lbtest
+    spec:
+      type: LoadBalancer
+      loadBalancerClass: reweave.levk.dev/node-address
+      loadBalancerIP: ADDR
+      allocateLoadBalancerNodePorts: false
+      selector: {app: echo-remote}
+      ports: [{name: http, port: 18080, targetPort: http}]
+  - apiVersion: v1
+    kind: Service
+    metadata:
+      name: echo-local
+      namespace: lbtest
+    spec:
+      type: LoadBalancer
+      loadBalancerClass: reweave.levk.dev/node-address
+      loadBalancerIP: ADDR
+      allocateLoadBalancerNodePorts: false
+      selector: {app: echo-local}
+      ports: [{name: http, port: 18081, targetPort: http}]

reweave/test/lb/run.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# End-to-end test of weave-npc's LoadBalancer agent against a live cluster.
+#
+#   CP="ssh ... root@<control plane>"  HOLDER_SSH="ssh ... root@<holder>" \
+#   ADDR=<address HOLDER holds> HOLDER=<node> REMOTE=<node> OTHER=<node> \
+#   reweave/test/lb/run.sh
+#
+# CP runs kubectl (admin.conf); HOLDER_SSH reads the holder's iptables. The
+# machine running this script is the outside client. Asserts, in order:
+#   1. each Service's status is exactly [{ip: ADDR, ipMode: Proxy}]
+#   2. kube-proxy installed nothing for ADDR (ipMode Proxy leaves it to us)
+#   3. from outside, both ports answer, and the pod sees the client's own
+#      address: from the holder's own pod, and from a pod on another node
+#      (whose replies must be steered back through the holder)
+#   4. a pod on a third node reaches ADDR too
+#   5. fail closed: naming an address nobody holds clears the status and
+#      stops the forward
+#   6. deleting the Services removes the holder's chains
+# Leaves nothing behind (namespace lbtest is deleted, also on failure).
+set -euo pipefail
+
+: "${CP:?}" "${HOLDER_SSH:?}" "${ADDR:?}" "${HOLDER:?}" "${REMOTE:?}" "${OTHER:?}"
+here=$(cd "$(dirname "$0")" && pwd)
+k() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $*"; }
+fail() { echo "FAIL: $*" >&2; exit 1; }
+ok() { echo "ok: $*"; }
+cleanup() { k delete namespace lbtest --ignore-not-found --wait=false >/dev/null 2>&1 || true; }
+trap cleanup EXIT
+
+# eventually <tries> <command...>: retry once a second
+eventually() {
+    local n=$1; shift
+    for _ in $(seq "$n"); do
+        if "$@" >/dev/null 2>&1; then return 0; fi
+        sleep 1
+    done
+    return 1
+}
+
+sed -e "s/\bADDR\b/$ADDR/g; s/\bHOLDER\b/$HOLDER/g; s/\bREMOTE\b/$REMOTE/g; s/\bOTHER\b/$OTHER/g" \
+    "$here/lb-test.yaml" | $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -" >/dev/null
+k -n lbtest wait --for=condition=Ready pod --all --timeout=180s >/dev/null || fail "test pods not ready"
+
+status() { k -n lbtest get svc "$1" -o "jsonpath='{.status.loadBalancer.ingress}'"; }
+# each check re-reads on every try of eventually
+status_is() { [ "$(status "$1")" = "$2" ]; }
+chains_gone() { [ "$($HOLDER_SSH "iptables-save -t nat | grep -c '^:REWEAVE-LBS-'" || true)" = 0 ]; }
+for s in echo-remote echo-local; do
+    want="[{\"ip\":\"$ADDR\",\"ipMode\":\"Proxy\"}]"
+    eventually 30 status_is $s "$want" || fail "$s status: $(status $s), want $want"
+done
+ok "1. status [{ip: $ADDR, ipMode: Proxy}] on both Services"
+
+if $HOLDER_SSH "iptables-save -t nat | grep -E '^-A KUBE-' | grep -F -- '-d $ADDR/32'"; then
+    fail "kube-proxy has rules for $ADDR"
+fi
+ok "2. kube-proxy installed nothing for $ADDR"
+
+# the client's own address, as the holder sees this machine's SSH session
+me=$($HOLDER_SSH 'echo ${SSH_CLIENT%% *}')
+for port in 18080 18081; do
+    got=""
+    for _ in $(seq 30); do
+        got=$(curl -s --max-time 3 "http://$ADDR:$port/clientip" || true)
+        [ -n "$got" ] && break
+        sleep 1
+    done
+    [ "${got%:*}" = "$me" ] || fail "port $port: pod saw '${got}', want this client ($me)"
+done
+ok "3. from outside, both pods see this client's own address ($me), remote pod's replies steered back"
+
+k -n lbtest exec client -- /agnhost connect "$ADDR:18080" --timeout 5s >/dev/null || fail "pod on $OTHER cannot reach $ADDR:18080"
+ok "4. a pod on $OTHER reaches $ADDR:18080"
+
+k -n lbtest patch svc echo-remote --type merge -p "'{\"spec\":{\"loadBalancerIP\":\"192.0.2.1\"}}'" >/dev/null
+eventually 30 status_is echo-remote "" || fail "status not cleared for an unheld address: $(status echo-remote)"
+if curl -s --max-time 3 "http://$ADDR:18080/clientip" >/dev/null; then
+    fail "$ADDR:18080 still forwards after its Service moved to an unheld address"
+fi
+ok "5. unheld address: status cleared, forward gone (fail closed)"
+
+k -n lbtest delete svc echo-remote echo-local >/dev/null
+eventually 30 chains_gone || fail "holder kept per-port chains after the Services went away"
+ok "6. Services deleted: holder's chains removed"
+echo PASS

reweave/test/portlock/netns.py

@@ -0,0 +1,14 @@
+# netns.py <namespace> <pod>: the network namespace the weave CNI plugin attached a pod in, from the runtime's
+# record of the attachment (libcni's result cache). Runs on the node.
+import glob
+import json
+import sys
+
+namespace, pod = sys.argv[1], sys.argv[2]
+for path in glob.glob("/var/lib/cni/results/weave-*"):
+    with open(path) as f:
+        entry = json.load(f)
+    args = dict(entry.get("cniArgs") or [])
+    if args.get("K8S_POD_NAMESPACE") == namespace and args.get("K8S_POD_NAME") == pod:
+        print(entry["netns"])
+        break

reweave/test/portlock/portlock-test.yaml

@@ -0,0 +1,30 @@
+# Pods for the port lock e2e (run.sh): a forger and a victim on node A, a victim on node B. None has any added
+# capability; the forged frames are sent from the forger's network namespace by root on the node.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: pltest
+---
+apiVersion: v1
+kind: Pod
+metadata: {name: forger, namespace: pltest}
+spec:
+  nodeName: NODE_A
+  containers:
+    - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}
+---
+apiVersion: v1
+kind: Pod
+metadata: {name: victim-local, namespace: pltest}
+spec:
+  nodeName: NODE_A
+  containers:
+    - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}
+---
+apiVersion: v1
+kind: Pod
+metadata: {name: victim-remote, namespace: pltest}
+spec:
+  nodeName: NODE_B
+  containers:
+    - {name: c, image: "curlimages/curl:8.10.1", command: ["sleep", "3600"]}

reweave/test/portlock/probe/main.go

@@ -0,0 +1,200 @@
+// Command probe sends hand-made Ethernet frames out of an interface, or waits for one carrying a tag. The port lock
+// e2e runs it inside pods' network namespaces (nsenter, as root on the node): a pod forging another's addresses
+// must not reach anyone.
+//
+//	probe send <ifname> <dst-mac> <src-mac|own> ipv4|arp <src-ip|own> <dst-ip> <tag>
+//	probe listen <ifname> <tag> <seconds>
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+	}
+	var err error
+	switch os.Args[1] {
+	case "send":
+		if len(os.Args) != 9 {
+			usage()
+		}
+		err = send(os.Args[2], os.Args[3], os.Args[4], os.Args[5], os.Args[6], os.Args[7], os.Args[8])
+	case "listen":
+		if len(os.Args) != 5 {
+			usage()
+		}
+		secs, _ := strconv.Atoi(os.Args[4])
+		var seen bool
+		seen, err = listen(os.Args[2], os.Args[3], time.Duration(secs)*time.Second)
+		if err == nil {
+			if seen {
+				fmt.Println("seen")
+				return
+			}
+			fmt.Println("not seen")
+			os.Exit(1)
+		}
+	default:
+		usage()
+	}
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(2)
+	}
+}
+
+func usage() {
+	fmt.Fprintln(os.Stderr, "usage: probe send <ifname> <dst-mac> <src-mac|own> ipv4|arp <src-ip|own> <dst-ip> <tag>")
+	fmt.Fprintln(os.Stderr, "       probe listen <ifname> <tag> <seconds>")
+	os.Exit(2)
+}
+
+func send(ifName, dstMAC, srcMAC, kind, srcIP, dstIP, tag string) error {
+	iface, err := net.InterfaceByName(ifName)
+	if err != nil {
+		return err
+	}
+	dst, err := net.ParseMAC(dstMAC)
+	if err != nil {
+		return err
+	}
+	src := iface.HardwareAddr
+	if srcMAC != "own" {
+		if src, err = net.ParseMAC(srcMAC); err != nil {
+			return err
+		}
+	}
+	var from net.IP
+	if srcIP == "own" {
+		addrs, err := iface.Addrs()
+		if err != nil {
+			return err
+		}
+		for _, a := range addrs {
+			if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil {
+				from = n.IP.To4()
+			}
+		}
+		if from == nil {
+			return fmt.Errorf("%s has no IPv4 address", ifName)
+		}
+	} else if from = net.ParseIP(srcIP).To4(); from == nil {
+		return fmt.Errorf("bad source address %q", srcIP)
+	}
+	to := net.ParseIP(dstIP).To4()
+	if to == nil {
+		return fmt.Errorf("bad destination address %q", dstIP)
+	}
+	var frame []byte
+	switch kind {
+	case "ipv4":
+		frame = ipv4Frame(dst, src, from, to, []byte(tag))
+	case "arp":
+		frame = arpFrame(dst, src, from, to, []byte(tag))
+	default:
+		return fmt.Errorf("unknown kind %q", kind)
+	}
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
+	if err != nil {
+		return err
+	}
+	defer unix.Close(fd)
+	if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: iface.Index}); err != nil {
+		return err
+	}
+	// a few copies, spaced out: the bridge and weave may still be learning where the destination is
+	for i := 0; i < 3; i++ {
+		if _, err := unix.Write(fd, frame); err != nil {
+			return err
+		}
+		time.Sleep(200 * time.Millisecond)
+	}
+	return nil
+}
+
+func listen(ifName, tag string, within time.Duration) (bool, error) {
+	iface, err := net.InterfaceByName(ifName)
+	if err != nil {
+		return false, err
+	}
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
+	if err != nil {
+		return false, err
+	}
+	defer unix.Close(fd)
+	if err := unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: iface.Index}); err != nil {
+		return false, err
+	}
+	tv := unix.NsecToTimeval((100 * time.Millisecond).Nanoseconds())
+	if err := unix.SetsockoptTimeval(fd, unix.SOL_SOCKET, unix.SO_RCVTIMEO, &tv); err != nil {
+		return false, err
+	}
+	buf := make([]byte, 65536)
+	deadline := time.Now().Add(within)
+	for time.Now().Before(deadline) {
+		n, _, err := unix.Recvfrom(fd, buf, 0)
+		if err != nil {
+			continue
+		}
+		if bytes.Contains(buf[:n], []byte(tag)) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }
+
+func ether(dst, src net.HardwareAddr, ethertype uint16, payload []byte) []byte {
+	f := make([]byte, 14, 14+len(payload))
+	copy(f[0:6], dst)
+	copy(f[6:12], src)
+	binary.BigEndian.PutUint16(f[12:14], ethertype)
+	return append(f, payload...)
+}
+
+func ipv4Frame(dst, src net.HardwareAddr, from, to net.IP, body []byte) []byte {
+	h := make([]byte, 20)
+	h[0] = 0x45
+	binary.BigEndian.PutUint16(h[2:4], uint16(20+len(body)))
+	h[8] = 64
+	h[9] = 253 // experimental protocol: nothing answers it, nothing on the way acts on it
+	copy(h[12:16], from)
+	copy(h[16:20], to)
+	binary.BigEndian.PutUint16(h[10:12], checksum(h))
+	return ether(dst, src, unix.ETH_P_IP, append(h, body...))
+}
+
+// arpFrame is an ARP request speaking for (src, from), asking for to; the tag rides as Ethernet trailer.
+func arpFrame(dst, src net.HardwareAddr, from, to net.IP, tag []byte) []byte {
+	a := make([]byte, 28)
+	binary.BigEndian.PutUint16(a[0:2], 1)
+	binary.BigEndian.PutUint16(a[2:4], unix.ETH_P_IP)
+	a[4], a[5] = 6, 4
+	binary.BigEndian.PutUint16(a[6:8], 1)
+	copy(a[8:14], src)
+	copy(a[14:18], from)
+	copy(a[24:28], to)
+	return ether(dst, src, unix.ETH_P_ARP, append(a, tag...))
+}
+
+func checksum(b []byte) uint16 {
+	var s uint32
+	for i := 0; i+1 < len(b); i += 2 {
+		s += uint32(binary.BigEndian.Uint16(b[i:]))
+	}
+	for s>>16 != 0 {
+		s = s&0xffff + s>>16
+	}
+	return ^uint16(s)
+}

reweave/test/portlock/run.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# End-to-end test of the port lock against a live cluster (assertion A7): a pod speaks only as itself.
+#
+#   CP="ssh ... root@<cp>" A_SSH="ssh ... root@<a>" B_SSH="ssh ... root@<b>" A=<node a> B=<node b> \
+#   reweave/test/portlock/run.sh
+#
+# Asserts, in order:
+#   0. every pod port on the three nodes (the CP's, A's and B's) carries the lock
+#   1. a pod's own frames reach a pod on its node and a pod on another node (the control)
+#   2. frames forging another pod's address reach neither
+#   3. frames forging a node's bridge (its weave MAC and address) reach neither
+#   4. ARP claiming another pod's address reaches no one; ARP for itself does
+# The forged frames are sent from the forging pod's network namespace by root on its node (nsenter), the most a
+# pod could ever do; the pod itself has no added capability.
+# Leaves nothing behind (namespace pltest is deleted, also on failure; the probe is removed from the nodes).
+set -euo pipefail
+
+: "${CP:?}" "${A_SSH:?}" "${B_SSH:?}" "${A:?}" "${B:?}"
+here=$(cd "$(dirname "$0")" && pwd)
+k() { $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl $*"; }
+fail() { echo "FAIL: $*" >&2; exit 1; }
+ok() { echo "ok: $*"; }
+work=$(mktemp -d)
+cleanup() {
+    k delete namespace pltest --ignore-not-found --wait=false >/dev/null 2>&1 || true
+    for s in "$A_SSH" "$B_SSH"; do $s 'rm -f /root/plprobe /root/plnetns.py' >/dev/null 2>&1 || true; done
+    rm -rf "$work"
+}
+trap cleanup EXIT
+
+(cd "$here/../../.." && CGO_ENABLED=0 go build -o "$work/probe" ./reweave/test/portlock/probe/)
+for s in "$A_SSH" "$B_SSH"; do
+    $s 'cat > /root/plprobe && chmod +x /root/plprobe' < "$work/probe"
+    $s 'cat > /root/plnetns.py' < "$here/netns.py"
+done
+
+# a port is locked when its ingress carries the three passes and the drop
+unlocked() {
+    $1 'for p in $(ip -br link show master weave | sed "s/@.*//; s/ .*//" | grep ^vethwepl); do
+            f=$(tc filter show dev $p ingress)
+            for want in "protocol ip pref 1 u32" "protocol arp pref 2 u32" "protocol arp pref 3 u32" \
+                        "protocol all pref 9 matchall" "gact action drop"; do
+                echo "$f" | grep -q "$want" || { echo "$p"; break; }
+            done
+        done'
+}
+all_locked() {
+    for s in "$CP" "$A_SSH" "$B_SSH"; do
+        u=$(unlocked "$s")
+        [ -z "$u" ] || fail "0. unlocked pod ports ($1): $u"
+    done
+}
+all_locked "before the test pods"
+ok "0. every pod port on the three nodes carries the lock"
+
+sed -e "s/\bNODE_A\b/$A/g; s/\bNODE_B\b/$B/g" "$here/portlock-test.yaml" |
+    $CP "KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -" >/dev/null
+k -n pltest wait --for=condition=Ready pod --all --timeout=300s >/dev/null || fail "test pods not ready"
+all_locked "with the test pods"
+
+netns_of() { $1 "python3 /root/plnetns.py pltest $2"; }
+mac_in() { $1 "nsenter --net=$2 ip -br link show eth0" | awk '{print $3}'; }
+pod_ip() { k -n pltest get pod "$1" -o "jsonpath='{.status.podIP}'"; }
+f_ns=$(netns_of "$A_SSH" forger)
+vl_ns=$(netns_of "$A_SSH" victim-local)
+vr_ns=$(netns_of "$B_SSH" victim-remote)
+[ -n "$f_ns" ] && [ -n "$vl_ns" ] && [ -n "$vr_ns" ] || fail "test pods' namespaces not found"
+vl_mac=$(mac_in "$A_SSH" "$vl_ns")
+vr_mac=$(mac_in "$B_SSH" "$vr_ns")
+vl_ip=$(pod_ip victim-local)
+vr_ip=$(pod_ip victim-remote)
+# the nodes' bridges, from the annotation the lb agent writes ("ip,mac")
+bridge_of() { k get node "$1" -o "jsonpath='{.metadata.annotations.reweave\.levk\.dev/weave}'"; }
+b_bridge=$(bridge_of "$B"); b_ip=${b_bridge%,*}; b_mac=${b_bridge#*,}
+a_bridge=$(bridge_of "$A"); a_ip=${a_bridge%,*}; a_mac=${a_bridge#*,}
+[ -n "$b_ip" ] && [ -n "$b_mac" ] && [ -n "$a_ip" ] && [ -n "$a_mac" ] || fail "no weave bridge annotation on $A or $B"
+
+# expect <seen|not seen> <victim: local|remote> <tag> <send args...>: the victim listens while the forger sends
+expect() {
+    local want=$1 where=$2 tag=$3-$RANDOM
+    shift 3
+    local vs vns
+    if [ "$where" = local ]; then vs=$A_SSH vns=$vl_ns; else vs=$B_SSH vns=$vr_ns; fi
+    $vs "nsenter --net=$vns /root/plprobe listen eth0 $tag 6" > "$work/listen" 2>&1 &
+    local listener=$!
+    sleep 1
+    $A_SSH "nsenter --net=$f_ns /root/plprobe send eth0 $* $tag" || fail "probe could not send: $*"
+    wait $listener || true
+    local got
+    got=$(cat "$work/listen")
+    [ "$got" = "$want" ] || fail "$tag ($where): '$got', want '$want'"
+}
+
+expect "seen" local own-local "$vl_mac own ipv4 own $vl_ip"
+expect "seen" remote own-remote "$vr_mac own ipv4 own $vr_ip"
+ok "1. the forger's own frames reach a pod on its node and one on $B"
+
+expect "not seen" remote pod-ip-remote "$vr_mac own ipv4 $vl_ip $vr_ip"
+expect "not seen" local pod-ip-local "$vl_mac own ipv4 $vr_ip $vl_ip"
+expect "not seen" remote pod-mac-remote "$vr_mac $vl_mac ipv4 $vl_ip $vr_ip"
+ok "2. frames forging another pod's address reach no one"
+
+expect "not seen" local node-b-local "$vl_mac $b_mac ipv4 $b_ip $vl_ip"
+expect "not seen" remote node-b-remote "$vr_mac $b_mac ipv4 $b_ip $vr_ip"
+expect "not seen" remote node-a-remote "$vr_mac $a_mac ipv4 $a_ip $vr_ip"
+ok "3. frames forging a node's bridge ($A's, $B's) reach no one"
+
+expect "not seen" local arp-claim-local "ff:ff:ff:ff:ff:ff own arp $vr_ip $vl_ip"
+expect "not seen" remote arp-claim-remote "ff:ff:ff:ff:ff:ff own arp $vl_ip $vr_ip"
+expect "seen" remote arp-own-remote "ff:ff:ff:ff:ff:ff own arp own $vr_ip"
+ok "4. ARP claiming another pod's address reaches no one; the forger's ARP for itself does"
+echo "PASS"

weave

@@ -725,8 +725,11 @@ peer_args() {
 
 install_cni_plugin() {
     mkdir -p $1 || return 1
-    if [ ! -f "$1/$CNI_PLUGIN_NAME" ]; then
-        cp /usr/bin/weaveutil "$1/$CNI_PLUGIN_NAME"
+    # (re)install whenever the image's plugin differs from the installed one: the name
+    # (weave-plugin-$IMAGE_VERSION) stays the same across builds of one version ("latest")
+    if [ ! -f "$1/$CNI_PLUGIN_NAME" ] || \
+       [ "$(sha256sum < /usr/bin/weaveutil)" != "$(sha256sum < "$1/$CNI_PLUGIN_NAME")" ]; then
+        cp /usr/bin/weaveutil "$1/$CNI_PLUGIN_NAME.tmp" && mv -f "$1/$CNI_PLUGIN_NAME.tmp" "$1/$CNI_PLUGIN_NAME"
     fi
 }
 
@@ -763,7 +766,8 @@ create_cni_config() {
         {
             "name": "weave",
             "type": "weave-net",
-            "hairpinMode": $2
+            "hairpinMode": $2,
+            "egressAdmission": $3
         },
         {
             "type": "portmap",
@@ -778,6 +782,9 @@ EOF
 setup_cni() {
     # if env var HAIRPIN_MODE is not set, default it to true
     HAIRPIN_MODE=${HAIRPIN_MODE:-true}
+    # weave-npc admits every pod before it gets its interface (EgressService members leave from their
+    # Service's address from the first packet); set to false only where weave-npc runs with --lb=false
+    EGRESS_ADMISSION=${EGRESS_ADMISSION:-true}
     if install_cni_plugin $CNI_PLUGIN_DIR ; then
         upgrade_cni_plugin $CNI_PLUGIN_DIR
     fi
@@ -786,8 +793,15 @@ setup_cni() {
     # clusters with Weave version <= 2.4.0 remove existing 10-weave.conf
     rm -f $HOST_ROOT/etc/cni/net.d/10-weave.conf
 
-    if [ -d $HOST_ROOT/etc/cni/net.d -a ! -f $HOST_ROOT/etc/cni/net.d/10-weave.conflist ] ; then
-        create_cni_config $HOST_ROOT/etc/cni/net.d/10-weave.conflist $HAIRPIN_MODE
+    # (re)write the conflist whenever what it should say changed, so upgrades reach existing nodes
+    if [ -d $HOST_ROOT/etc/cni/net.d ] ; then
+        CONFLIST=$HOST_ROOT/etc/cni/net.d/10-weave.conflist
+        create_cni_config $CONFLIST.new $HAIRPIN_MODE $EGRESS_ADMISSION
+        if [ -f $CONFLIST ] && [ "$(cat $CONFLIST.new)" = "$(cat $CONFLIST)" ] ; then
+            rm -f $CONFLIST.new
+        else
+            mv -f $CONFLIST.new $CONFLIST
+        fi
     fi
 }