cni: lock each pod's port to the pod's own addresses

A pod could put any source MAC or address on the wire: weave has no layer-2
anti-spoofing, so a pod with NET_RAW could pose as another pod or as a node's
bridge, on its own node (local bridging never reaches the router) and, through
weave, on others. That undercut every rule that trusts a source MAC or address,
including the egress holder's check of which node a tunnel packet came from.

The CNI plugin now locks the host side of each pod's veth with tc ingress
filters: IPv4 from the pod's MAC and address, ARP speaking for exactly those
(or probing from 0.0.0.0) pass; everything else is dropped before the bridge
sees it. The u32 keys follow iproute2's layout for "ether src" (-8, -4) and
"ip src" (12). ADD fails if the port cannot be locked, so an unlocked pod never
starts. The filters go away with the veth.

Pods attached before this plugin get locked by weave-init: "weaveutil
lock-ports" walks libcni's result cache (each attachment's netns, interface and
address) and locks their ports, leaving locked ones alone. weave-init mounts
the cache and /var/run/netns read-only for it.

TestPortLockInKernel (root) builds a bridge and two pods in throwaway
namespaces and checks what reaches the other pod; reweave/test/portlock runs
the same against a live cluster, across nodes, after checking every port is
locked.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UHXzGrsxfkGk8LmwrBbHNZ

.gitea/workflows/images.yml

@@ -0,0 +1,67 @@
+# Build the images the cluster runs (weave-kube, weave-npc) and push them to
+# this Gitea's container registry. CI stops here: rolling them out is the
+# CD side's job, not this workflow's.
+#
+# Tags are the full commit SHA, so a tag never moves: rebuilding a commit
+# pushes the same name again instead of changing what an existing name means.
+# linux/amd64 only for now (every node is amd64).
+name: images
+
+on:
+  push:
+    paths-ignore:
+      - "website/**"
+      - "docs/**"
+      - "**.md"
+  workflow_dispatch:
+
+# The job token can't log in to the registry (no package scope there), so
+# the push uses the reweave-ci bot's token from the org secret REGISTRY_TOKEN.
+permissions:
+  contents: read
+
+jobs:
+  images:
+    runs-on: ubuntu-latest
+    env:
+      REGISTRY: git.levk.dev
+      IMAGE_PREFIX: git.levk.dev/reweave
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0 # tags, for the version string
+          submodules: true # tools/ (weaveworks/build-tools): make all builds tools/runner
+
+      - name: Log in to the registry
+        run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login "$REGISTRY" -u reweave-ci --password-stdin
+
+      - name: Build and push (linux/amd64)
+        run: |
+          set -eu
+          sha=$(git rev-parse HEAD)
+          version=$(git describe --tags --always | sed 's/^v//')
+          for pair in weavekubeimage:weave-kube weavenpcimage:weave-npc; do
+            target=${pair%%:*}
+            image="$IMAGE_PREFIX/${pair#*:}:$sha"
+            docker buildx build --push --progress=plain \
+              --platform=linux/amd64 \
+              --target="$target" \
+              --build-arg=WEAVE_VERSION="$version" \
+              --build-arg=revision="$sha" \
+              --build-arg=imageversion="$version" \
+              -f reweave/build/Dockerfile \
+              -t "$image" \
+              .
+            echo "pushed $image"
+          done
+
+      # The images live in the registry now; the runner keeps only a bounded build cache. Without this the
+      # runner's 10Gi Docker volume fills up after a handful of builds (no space left on device mid-build).
+      - name: Free the runner's disk
+        if: always()
+        run: |
+          set -u
+          docker image ls --format '{{.Repository}}:{{.Tag}}' | grep "^$IMAGE_PREFIX/" | xargs -r docker image rm -f
+          docker image prune -f
+          docker builder prune -f --keep-storage 4GB
+          docker system df

.github/workflows/github-pages-publish.yml

@@ -0,0 +1,53 @@
+# Workflow for building and deploying a Jekyll site to GitHub Pages
+name: Build using Jekyll, deploy to GitHub Pages
+
+on:
+  # Runs on pushes targeting the master branch, if changes found in
+  # the website directory
+  push:
+    branches: ["master"]
+    paths: ["website/**"]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+      - name: Build with Jekyll
+        uses: actions/jekyll-build-pages@v1
+        with:
+          source: ./website/
+          destination: ./_site
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

CHANGELOG.md

@@ -1,3 +1,55 @@
+## Release 2.9.0
+
+This release has upgraded dependencies based on CVE recommendations.
+
+Also, from this release onwards, all weave images will use the `iptables-nft` backend for `iptables` by default. To use the legacy mode, set the environment variable `IPTABLES_BACKEND` to `legacy`. To do this for the Docker plugin, export the variable before using the `weave` script. For the Kubernetes addon, set the variable in the manifest.
+
+## Release 2.8.10
+
+This release bumps the Go version to 1.23.1 and the alpine base image to version 3.20.3 as per CVE recommendations.
+
+## Release 2.8.9
+
+This release bumps the Go version to 1.22.2, the alpine base image to version 3.20.2, and upgrades a dependency as per CVE recommendations.
+
+## Release 2.8.8
+
+The alpine base image used by `weave-kube` and `weave-npc` has been upgraded to version 3.20.0 in this release. This removes several CVEs.
+
+## Release 2.8.7
+
+This release has upgraded dependencies based on CVE recommendations.
+
+Also, this release does not provides images for the `ppc64le` and `s390x` architectures, because a crucial upstream dependency (libpcap-0.8, from the debian buster respository) does not provide binaries for those architectures any more.
+
+## Release 2.8.6
+
+From this release onwards, the Docker V2 Plugin will be published for all supported architectures. It will be named using the pattern `rajchaudhuri/net-plugin:<version>-<arch>`. The latest version will also be published as `rajchaudhuri/net-plugin:latest_release-<arch>`. Supported architectures are `amd64`,`arm`,`arm64`,`ppc64le` and `s390x`. It can be installed on a Swarm-mode-enabled Docker host as follows:
+
+```bash
+ARCH=$(docker version -f '{{.Server.Arch}}')
+docker plugin install rajchaudhuri/net-plugin:latest_release-$ARCH
+```
+
+## Release 2.8.5
+
+This release has upgraded dependencies based on CVE recommendations.
+
+* Build with Go version 1.21.9
+* A dependency has been updated
+
+## Release 2.8.4
+
+The alpine base image used by `weave-kube` and `weave-npc` was upgraded to version 3.19.1 in the previous release. In this version, the default iptables backend is nftables, and the legacy backend is not included. Our scripts and programs assume legacy as the default backend, and change to nft if autodetected, or if we ask for it. So, the build Dockerfile (reweave/build/Dockerfile) was changed to also install the Alpine `iptables-legacy` package , and change the `iptables-{save,restore}` symbolic links to point to the legacy backend by default.
+
+The `weave-kube` and `weave-npc` images can now log traces if the environment WEAVE_DEBUG is set in the manifest.
+
+## Release 2.8.3
+
+The docker API client version, used by the proxy package and the weaveutil command, was bumped from 1.18 to 1.24. As of March 2024, Docker API versions below 1.24 are deprecated. This means that the minimum supported Docker version is now 1.12.0.
+
+Provision was made in weaveutil program and the weave script to override the API version used, via the environment variable `DOCKER_API_VERSION`. The same variable is used by standard docker clients.
+
 ## Release 2.8.2
 
 This is the first release since forking from weaveworks/weave. While no new functionality has been added, there have been many changes.

README.md

@@ -1,30 +1,36 @@
-# ReWeave - an effort to keep Weave Net alive
+# Weave Net
 
 This repository contains a fork of Weave Net, the first product developed by Weaveworks. Since Weaveworks has shut down, this repo aims to continue maintaining Weave Net, and to publish releases regularly.
 
 [![Go Report Card](https://goreportcard.com/badge/github.com/rajch/weave)](https://goreportcard.com/report/github.com/rajch/weave)
-[![Docker Pulls](https://img.shields.io/docker/pulls/rajchaudhuri/weave-kube)](https://hub.docker.com/r/rajchaudhuri/weave-kube)
-![GitHub release (latest by date)](https://img.shields.io/github/v/release/rajch/ttws?include_prereleases)
-[![Unique vulnerability count in all images](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Frajch%2Fweave%2Fmaster%2Freweave%2Fscans%2Fbadge.json&label=Vulnerabilty%20count)](reweave/scans/report.md)
+[![Docker Pulls](https://img.shields.io/docker/pulls/rajchaudhuri/weave-kube "Number of times the weave-kube image was pulled from the Docker Hub")](https://hub.docker.com/r/rajchaudhuri/weave-kube)
+[![GitHub release (latest by date)](https://img.shields.io/github/v/release/rajch/weave?include_prereleases)](https://github.com/rajch/weave/releases)
+[![Unique CVE count in all images](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Frajch%2Fweave%2Fmaster%2Freweave%2Fscans%2Fbadge.json&label=CVE%20count "The number of unique CVEs reported by scanning all images")](reweave/scans/report.md)
 
-The history of the ReWeave effort can be found in [HISTORY.md](HISTORY.md).
+The history of the fork can be found in [HISTORY.md](HISTORY.md).
 
-## Using Weave on Kubernetes
+## Using Weave Net on Kubernetes
 
 On a newly created Kubernetes cluster, the Weave Net CNI pluging can be installed by running the following command:
 
 ```
-kubectl apply -f https://reweave.azurewebsites.net/k8s/v1.28/net.yaml
+kubectl apply -f https://reweave.azurewebsites.net/k8s/v1.29/net.yaml
 ```
 
-Replace `v1.28` with the version on Kubernetes on your cluster.
+Replace `v1.29` with the version on Kubernetes on your cluster.
 
 That endpoint is provided by the companion project [weave-endpoint](https://github.com/rajch/weave-endpoint).
 
-## Building Weave
+## Using Weave Net in other ways
+
+Please refer to the [documentation](https://rajch.github.io/weave).
+
+## Building Weave Net
 
 Details can be found [here](reweave/BUILDING.md). 
 
 ## Documentation status
 
-At this point, any information found in directories other than `reweave`, such as `docs` or `site`, should be considered obsolete. In time, those will be updated.
+The public documentation that used to exist in the `site` directory has been moved to the `original/site` directory. A new `website` directory has been created, and populated with the content of the `original/site` directory, rearranged and reformatted for being built with Jekyll and published to the GitHub pages site [https://rajch.github.io/weave](https://rajch.github.io/weave).
+
+The documentation will now be maintained and published from the `website` directory exclusively.

go.mod

@@ -1,6 +1,6 @@
 module github.com/rajch/weave
 
-go 1.21
+go 1.23.1
 
 require (
 	github.com/andybalholm/go-bit v1.0.1
@@ -10,7 +10,7 @@ require (
 	github.com/containernetworking/cni v1.1.2
 	github.com/containernetworking/plugins v1.2.0
 	github.com/coreos/go-iptables v0.6.0
-	github.com/docker/docker v24.0.7+incompatible
+	github.com/docker/docker v25.0.6+incompatible
 	github.com/docker/libnetwork v0.8.0-dev.2.0.20180212220201-119badbd737b
 	github.com/docker/machine v0.14.0
 	github.com/fsouza/go-dockerclient v1.10.0
@@ -30,9 +30,9 @@ require (
 	github.com/weaveworks/go-checkpoint v0.0.0-20170503165305-ebbb8b0518ab
 	github.com/weaveworks/go-odp v0.0.0-20181017121109-6b0aa22550d9
 	github.com/weaveworks/mesh v0.0.0-20191105120815-58dbcc3e8e63
-	golang.org/x/crypto v0.17.0
-	golang.org/x/sys v0.15.0
-	golang.org/x/tools v0.10.0
+	golang.org/x/crypto v0.31.0
+	golang.org/x/sys v0.28.0
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d
 	k8s.io/api v0.26.2
 	k8s.io/apimachinery v0.26.2
 	k8s.io/client-go v0.26.2
@@ -48,20 +48,23 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containerd/containerd v1.7.11 // indirect
+	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.8.2-beta.1+incompatible // indirect
+	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186 // indirect
@@ -76,6 +79,7 @@ require (
 	github.com/mdlayher/raw v0.0.0-20181016155347-fa5ef3332ca9 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.3.0 // indirect
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -83,25 +87,29 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
-	github.com/opencontainers/runc v1.1.12 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/rootless-containers/rootlesskit v1.1.0 // indirect
 	github.com/safchain/ethtool v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
+	go.opentelemetry.io/otel v1.19.0 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/oauth2 v0.10.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/term v0.27.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gotest.tools/v3 v3.4.0 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
 	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect

go.sum

@@ -32,6 +32,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -39,6 +40,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
+github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -56,6 +58,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -68,6 +72,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw=
 github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
 github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw=
 github.com/containernetworking/plugins v1.2.0 h1:SWgg3dQG1yzUo4d9iD8cwSVh1VqI+bP7mkPDoSfP9VU=
@@ -77,13 +83,14 @@ github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFE
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/docker/distribution v2.8.2-beta.1+incompatible h1:gILO60VLD2v28ozemv4aAwDb8ds5U2O/vD/sBXbd7Rw=
-github.com/docker/distribution v2.8.2-beta.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
-github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg=
+github.com/docker/docker v25.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
@@ -101,9 +108,12 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
+github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/fsouza/go-dockerclient v1.10.0 h1:ppSBsbR60I1DFbV4Ag7LlHlHakHFRNLk9XakATW1yVQ=
 github.com/fsouza/go-dockerclient v1.10.0/go.mod h1:+iNzAW78AzClIBTZ6WFjkaMvOgz68GyCJ236b1opLTs=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -118,8 +128,11 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -176,8 +189,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -200,6 +213,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186 h1:URgjUo+bs1KwatoNbwG0uCO4dHN4r1jsp4a5AGgHRjo=
 github.com/hashicorp/go-cleanhttp v0.0.0-20171218145408-d5fe4b57a186/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -259,6 +274,8 @@ github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkV
 github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
 github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
+github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo=
+github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
 github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA=
 github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -283,18 +300,19 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs=
+github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/onsi/gomega v1.24.2 h1:J/tulyYK6JwBldPViHJReihxxZ+22FHs0piGjQAvoUE=
+github.com/onsi/gomega v1.24.2/go.mod h1:gs3J10IS7Z7r7eXRoNJIrNqU4ToQukCJhFtKrWgHWnk=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8=
 github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
-github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss=
-github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -330,8 +348,6 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rootless-containers/rootlesskit v1.1.0 h1:cRaRIYxY8oce4eE/zeAUZhgKu/4tU1p9YHN4+suwV7M=
-github.com/rootless-containers/rootlesskit v1.1.0/go.mod h1:H+o9ndNe7tS91WqU0/+vpvc+VaCd7TCIWaJjnV0ujUo=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/safchain/ethtool v0.2.0 h1:dILxMBqDnQfX192cCAPjZr9v2IgVXeElHPy435Z/IdE=
 github.com/safchain/ethtool v0.2.0/go.mod h1:WkKB1DnNtvsMlDmQ50sgwowDJV/hGbJSOvJoEXs1AJQ=
@@ -377,6 +393,22 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 h1:x8Z78aZx8cOF0+Kkazoc7lwUNMGy0LrzEMxTm4BbTxg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0/go.mod h1:62CPTSry9QZtOaSsE3tOzhx6LzDhHnXJ6xHeMNNiM6Q=
+go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs=
+go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
+go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE=
+go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8=
+go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o=
+go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A=
+go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg=
+go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo=
+go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
+go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -384,8 +416,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -416,8 +448,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
-golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -453,8 +485,8 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -474,7 +506,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -515,6 +548,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -525,12 +559,12 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -538,8 +572,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -588,8 +622,9 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg=
-golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -648,6 +683,11 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g=
+google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw=
+google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -660,6 +700,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
 google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
 google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ=
+google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -672,8 +714,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -699,6 +741,7 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o=
+gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

lb/addresses.go

@@ -0,0 +1,87 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+// Local is what this node publishes about itself, read from its interfaces.
+type Local struct {
+	Addresses []net.IP
+	WeaveIP   net.IP
+	WeaveMAC  net.HardwareAddr
+	PodNet    *net.IPNet // the weave allocation range: the bridge address's network
+}
+
+// overlay- and container-side interfaces: their addresses are never held ones
+var overlayPrefixes = []string{"weave", "datapath", "vxlan-", "veth", "docker", "cni"}
+
+func overlay(name string) bool {
+	for _, p := range overlayPrefixes {
+		if strings.HasPrefix(name, p) {
+			return true
+		}
+	}
+	return false
+}
+
+// ReadLocal lists the global IPv4 addresses on host interfaces, and the weave bridge's address and MAC.
+func ReadLocal() (Local, error) {
+	var l Local
+	links, err := netlink.LinkList()
+	if err != nil {
+		return l, err
+	}
+	for _, link := range links {
+		name := link.Attrs().Name
+		addrs, err := netlink.AddrList(link, unix.AF_INET)
+		if err != nil {
+			return l, fmt.Errorf("%s: %v", name, err)
+		}
+		if name == bridge {
+			for _, a := range addrs {
+				if a.IPNet != nil && a.IP.To4() != nil {
+					l.WeaveIP = a.IP.To4()
+					l.WeaveMAC = link.Attrs().HardwareAddr
+					l.PodNet = &net.IPNet{IP: a.IP.Mask(a.Mask), Mask: a.Mask}
+					break
+				}
+			}
+			continue
+		}
+		if overlay(name) || link.Attrs().Flags&net.FlagLoopback != 0 {
+			continue
+		}
+		for _, a := range addrs {
+			if a.Scope == unix.RT_SCOPE_UNIVERSE && a.IP.To4() != nil && a.IP.IsGlobalUnicast() {
+				l.Addresses = append(l.Addresses, a.IP.To4())
+			}
+		}
+	}
+	// an address inside the pod network is never a host one (whatever interface it is on)
+	if l.PodNet != nil {
+		kept := l.Addresses[:0]
+		for _, ip := range l.Addresses {
+			if !l.PodNet.Contains(ip) {
+				kept = append(kept, ip)
+			}
+		}
+		l.Addresses = kept
+	}
+	sort.Slice(l.Addresses, func(i, j int) bool { return l.Addresses[i].String() < l.Addresses[j].String() })
+	return l, nil
+}
+
+// Annotations is what this node's Node object should carry for l.
+func (l Local) Annotations() map[string]string {
+	a := map[string]string{AnnAddresses: FormatAddresses(l.Addresses)}
+	if l.WeaveIP != nil && l.WeaveMAC != nil {
+		a[AnnWeave] = l.WeaveIP.String() + "," + l.WeaveMAC.String()
+	}
+	return a
+}

lb/admit/admit.go

@@ -0,0 +1,88 @@
+// Package admit is the protocol between the weave CNI plugin and weave-npc's egress agent: at CNI ADD the
+// plugin tells the agent which pod got which IP, and the agent answers once the pod's egress rules are in
+// place, so an EgressService member's first packet already leaves from the Service's address (or not at all).
+//
+// It depends on the standard library only: the plugin binary must not grow a Kubernetes client.
+package admit
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// SocketPath is where the agent listens (a hostPath directory shared with the host, where the plugin runs).
+const SocketPath = "/run/weave-npc/egress.sock"
+
+// Request is one pod's admission: the IP the plugin just allocated to it.
+type Request struct {
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+	IP        string `json:"ip"`
+}
+
+// Response says whether the pod is an EgressService member (its rules are installed either way when 200).
+type Response struct {
+	Member bool `json:"member"`
+}
+
+// Timeout bounds one admission (the agent may have to fetch the pod from the API server).
+const Timeout = 20 * time.Second
+
+// PodFromArgs reads the pod's namespace and name from CNI_ARGS ("K8S_POD_NAMESPACE=..;K8S_POD_NAME=..").
+func PodFromArgs(args string) (namespace, name string) {
+	for _, kv := range strings.Split(args, ";") {
+		k, v, ok := strings.Cut(kv, "=")
+		if !ok {
+			continue
+		}
+		switch k {
+		case "K8S_POD_NAMESPACE":
+			namespace = v
+		case "K8S_POD_NAME":
+			name = v
+		}
+	}
+	return namespace, name
+}
+
+// Admit asks the agent at socket to admit req, and returns only once the agent has installed the pod's rules.
+func Admit(ctx context.Context, socket string, req Request) (Response, error) {
+	var resp Response
+	body, err := json.Marshal(req)
+	if err != nil {
+		return resp, err
+	}
+	client := &http.Client{Transport: &http.Transport{
+		DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
+			var d net.Dialer
+			return d.DialContext(ctx, "unix", socket)
+		},
+	}}
+	ctx, cancel := context.WithTimeout(ctx, Timeout)
+	defer cancel()
+	hreq, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://weave-npc/admit", bytes.NewReader(body))
+	if err != nil {
+		return resp, err
+	}
+	hreq.Header.Set("Content-Type", "application/json")
+	hresp, err := client.Do(hreq)
+	if err != nil {
+		return resp, fmt.Errorf("egress admission (weave-npc at %s): %v", socket, err)
+	}
+	defer hresp.Body.Close()
+	out, _ := io.ReadAll(io.LimitReader(hresp.Body, 1<<16))
+	if hresp.StatusCode != http.StatusOK {
+		return resp, fmt.Errorf("egress admission refused (%s): %s", hresp.Status, strings.TrimSpace(string(out)))
+	}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		return resp, fmt.Errorf("egress admission: bad answer: %v", err)
+	}
+	return resp, nil
+}

lb/controller.go

@@ -0,0 +1,460 @@
+package lb
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+
+	"github.com/rajch/weave/common"
+	"github.com/rajch/weave/lb/admit"
+)
+
+// Controller watches Services, EndpointSlices, Nodes, EgressServices and this node's Pods, and keeps this
+// node's part of the picture true. It also answers the CNI plugin's egress admissions.
+type Controller struct {
+	self   string
+	client kubernetes.Interface
+	egress rest.Interface
+	dp     *Datapath
+	// readLocal is the node's own interfaces (swappable in tests)
+	readLocal func() (Local, error)
+
+	services, slices, nodes, pods, egresses cache.Store
+	synced                                  []cache.InformerSynced
+	egressSynced                            cache.InformerSynced // nil while the EgressService API is absent
+	kick                                    chan struct{}
+	ready                                   chan struct{} // closed once the first pass ran
+
+	mu           sync.Mutex // one pass at a time (the loop, or an admission)
+	lastProblems map[string]bool
+	// Services whose status this node wrote since it started (so a restart rewrites ipMode once)
+	written map[types.UID]string
+	// admitted: IP -> pod the CNI plugin gave it; fetched: pods admitted before the pod informer had them
+	admitted map[string]Admission
+	fetched  map[string]*coreapi.Pod
+	at       map[string]time.Time
+	// lastEgress is what was last applied, reused while the EgressService caches are not synced (never wipe
+	// members' rules on a partial view)
+	lastEgress EgressDesired
+}
+
+// Resync is how often the node re-reads its interfaces and re-applies everything even without API events.
+const Resync = 30 * time.Second
+
+// NewController runs nothing yet; Run does. egress is the EgressService API client.
+func NewController(self string, client kubernetes.Interface, egress rest.Interface, dp *Datapath) *Controller {
+	return &Controller{self: self, client: client, egress: egress, dp: dp, readLocal: ReadLocal,
+		kick: make(chan struct{}, 1), ready: make(chan struct{}), lastProblems: map[string]bool{},
+		written: map[types.UID]string{}, admitted: map[string]Admission{}, fetched: map[string]*coreapi.Pod{},
+		at: map[string]time.Time{}}
+}
+
+func (c *Controller) poke() {
+	select {
+	case c.kick <- struct{}{}:
+	default:
+	}
+}
+
+func newInformer(c *Controller, lw cache.ListerWatcher, obj runtime.Object) (cache.Store, cache.Controller) {
+	return cache.NewInformer(lw, obj, 0, cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(interface{}) { c.poke() },
+		UpdateFunc: func(interface{}, interface{}) { c.poke() },
+		DeleteFunc: func(interface{}) { c.poke() },
+	})
+}
+
+// egressAPI reports whether the EgressService API is served (the CRD is installed).
+func (c *Controller) egressAPI() (bool, error) {
+	_, err := c.client.Discovery().ServerResourcesForGroupVersion(GroupVersion.String())
+	if err == nil {
+		return true, nil
+	}
+	if apierrors.IsNotFound(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func (c *Controller) startEgress(ctx context.Context) {
+	var ctl cache.Controller
+	c.egresses, ctl = newInformer(c, cache.NewListWatchFromClient(c.egress, EgressServices, "", fields.Everything()), &EgressService{})
+	go ctl.Run(ctx.Done())
+	c.egressSynced = ctl.HasSynced
+}
+
+// Run blocks until ctx ends.
+func (c *Controller) Run(ctx context.Context) {
+	var sc, ec, nc, pc cache.Controller
+	c.services, sc = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "services", "", fields.Everything()), &coreapi.Service{})
+	c.slices, ec = newInformer(c, cache.NewListWatchFromClient(c.client.DiscoveryV1().RESTClient(), "endpointslices", "", fields.Everything()), &discovery.EndpointSlice{})
+	c.nodes, nc = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "nodes", "", fields.Everything()), &coreapi.Node{})
+	c.pods, pc = newInformer(c, cache.NewListWatchFromClient(c.client.CoreV1().RESTClient(), "pods", "",
+		fields.OneTermEqualSelector("spec.nodeName", c.self)), &coreapi.Pod{})
+	for _, ctl := range []cache.Controller{sc, ec, nc, pc} {
+		go ctl.Run(ctx.Done())
+		c.synced = append(c.synced, ctl.HasSynced)
+	}
+	// with the API present, nothing is applied before EgressServices are known: an empty view would wipe
+	// members' rules (their traffic would leave from the node); without it, there can be no members
+	for {
+		present, err := c.egressAPI()
+		if err == nil {
+			if present {
+				c.startEgress(ctx)
+				c.synced = append(c.synced, c.egressSynced)
+			}
+			break
+		}
+		common.Log.Warnf("lb: cannot tell whether the EgressService API is served (retrying): %v", err)
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(5 * time.Second):
+		}
+	}
+	go c.serveAdmission(ctx)
+	if !cache.WaitForCacheSync(ctx.Done(), c.synced...) {
+		return
+	}
+	tick := time.NewTicker(Resync)
+	defer tick.Stop()
+	c.poke()
+	first := true
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-tick.C:
+			if c.egressSynced == nil {
+				// the API may have been installed since
+				if present, err := c.egressAPI(); err == nil && present {
+					c.mu.Lock()
+					c.startEgress(ctx)
+					c.mu.Unlock()
+				}
+			}
+		case <-c.kick:
+			// let a burst of events settle into one pass
+			time.Sleep(200 * time.Millisecond)
+			select {
+			case <-c.kick:
+			default:
+			}
+		}
+		c.mu.Lock()
+		err := c.sync(ctx)
+		c.mu.Unlock()
+		if err != nil {
+			common.Log.Errorf("lb: %v", err)
+		}
+		if first {
+			first = false
+			close(c.ready)
+		}
+	}
+}
+
+// egressViewSynced: the EgressService view can be trusted (absent API: trivially empty; else synced).
+func (c *Controller) egressViewSynced() bool {
+	return c.egressSynced == nil || c.egressSynced()
+}
+
+// sync is one pass: publish this node, compute, apply, write statuses. Callers hold c.mu.
+func (c *Controller) sync(ctx context.Context) error {
+	local, err := c.readLocal()
+	if err != nil {
+		return fmt.Errorf("reading interfaces: %v", err)
+	}
+	if err := c.publish(ctx, local); err != nil {
+		return err
+	}
+	if local.PodNet == nil {
+		return fmt.Errorf("weave bridge has no address yet; not forwarding")
+	}
+
+	var nodes []Node
+	var problems []string
+	for _, o := range c.nodes.List() {
+		n, err := ParseNode(o.(*coreapi.Node))
+		if err != nil {
+			problems = append(problems, err.Error())
+		}
+		nodes = append(nodes, n)
+	}
+	var svcs []*coreapi.Service
+	for _, o := range c.services.List() {
+		svcs = append(svcs, o.(*coreapi.Service))
+	}
+	var slices []*discovery.EndpointSlice
+	for _, o := range c.slices.List() {
+		slices = append(slices, o.(*discovery.EndpointSlice))
+	}
+	d := Compute(c.self, nodes, svcs, slices)
+	// rewrite once after start what an earlier run may have written without ipMode
+	for _, f := range d.Forwards {
+		if s := c.service(f.Namespace, f.Name); s != nil && c.written[s.UID] != f.Address.String() && !hasStatusFor(d, f) {
+			d.Status = append(d.Status, Status{Namespace: f.Namespace, Name: f.Name, Address: f.Address})
+		}
+	}
+
+	e := c.lastEgress
+	e.Status = nil // reusing the datapath, never the writes of an earlier view
+	if c.egressViewSynced() {
+		e = c.computeEgress(nodes, svcs)
+	} else {
+		problems = append(problems, "EgressServices not synced yet: keeping the egress rules last applied")
+	}
+
+	current := map[string]bool{}
+	for _, p := range append(append(problems, d.Problems...), e.Problems...) {
+		current[p] = true
+		if !c.lastProblems[p] {
+			common.Log.Warnf("lb: %s", p)
+		}
+	}
+	c.lastProblems = current
+
+	if err := c.dp.Apply(d, e, c.self, local); err != nil {
+		return fmt.Errorf("applying datapath: %v", err)
+	}
+	c.lastEgress = e
+	for _, st := range d.Status {
+		if err := c.writeStatus(ctx, st); err != nil {
+			return err
+		}
+	}
+	for _, st := range e.Status {
+		if err := PatchEgressStatus(ctx, c.egress, st.Namespace, st.Name, st.Status); err != nil {
+			return fmt.Errorf("status of egress %s/%s: %v", st.Namespace, st.Name, err)
+		}
+		common.Log.Infof("lb: egress %s/%s: host %q, address %q, %d port blocks", st.Namespace, st.Name, st.Status.Host, st.Status.Address, len(st.Status.Blocks))
+	}
+	return nil
+}
+
+// computeEgress gathers this node's pods (informer, plus pods admitted before the informer had them) and computes.
+func (c *Controller) computeEgress(nodes []Node, svcs []*coreapi.Service) EgressDesired {
+	var egresses []*EgressService
+	if c.egresses != nil {
+		for _, o := range c.egresses.List() {
+			egresses = append(egresses, o.(*EgressService))
+		}
+	}
+	seen := map[string]bool{}
+	var pods []*coreapi.Pod
+	for _, o := range c.pods.List() {
+		p := o.(*coreapi.Pod)
+		seen[p.Namespace+"/"+p.Name] = true
+		pods = append(pods, p)
+	}
+	// forget admissions the API has caught up with or that are long gone
+	for ip, a := range c.admitted {
+		key := a.Namespace + "/" + a.Name
+		o, ok, _ := c.pods.GetByKey(key)
+		caughtUp := ok && o.(*coreapi.Pod).Status.PodIP == ip
+		if caughtUp || (!ok && time.Since(c.at[ip]) > 2*time.Minute) {
+			delete(c.admitted, ip)
+			delete(c.at, ip)
+		}
+	}
+	for key, p := range c.fetched {
+		stillAdmitted := false
+		for _, a := range c.admitted {
+			if a.Namespace+"/"+a.Name == key {
+				stillAdmitted = true
+			}
+		}
+		if !stillAdmitted {
+			delete(c.fetched, key)
+			continue
+		}
+		if !seen[key] {
+			pods = append(pods, p)
+		}
+	}
+	return ComputeEgress(c.self, nodes, svcs, egresses, pods, c.admitted)
+}
+
+func hasStatusFor(d Desired, f Forward) bool {
+	for _, s := range d.Status {
+		if s.Namespace == f.Namespace && s.Name == f.Name {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *Controller) service(ns, name string) *coreapi.Service {
+	o, ok, err := c.services.GetByKey(ns + "/" + name)
+	if err != nil || !ok {
+		return nil
+	}
+	return o.(*coreapi.Service)
+}
+
+// serveAdmission answers the CNI plugin on admit.SocketPath: record the pod's IP, run a pass, answer once the
+// pod's rules are in place. A pod is refused (the plugin fails the ADD, kubelet retries) while this node cannot
+// yet tell whether it is a member.
+func (c *Controller) serveAdmission(ctx context.Context) {
+	if err := os.MkdirAll(filepath.Dir(admit.SocketPath), 0o700); err != nil {
+		common.Log.Errorf("lb: admission socket: %v", err)
+		return
+	}
+	_ = os.Remove(admit.SocketPath)
+	l, err := net.Listen("unix", admit.SocketPath)
+	if err != nil {
+		common.Log.Errorf("lb: admission socket: %v", err)
+		return
+	}
+	_ = os.Chmod(admit.SocketPath, 0o600)
+	mux := http.NewServeMux()
+	mux.HandleFunc("/admit", func(w http.ResponseWriter, r *http.Request) {
+		var req admit.Request
+		if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<16)).Decode(&req); err != nil {
+			http.Error(w, "bad request: "+err.Error(), http.StatusBadRequest)
+			return
+		}
+		member, code, err := c.admit(r.Context(), req)
+		if err != nil {
+			common.Log.Warnf("lb: admission of %s/%s (%s): %v", req.Namespace, req.Name, req.IP, err)
+			http.Error(w, err.Error(), code)
+			return
+		}
+		_ = json.NewEncoder(w).Encode(admit.Response{Member: member})
+	})
+	srv := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+	go func() {
+		<-ctx.Done()
+		_ = srv.Close()
+	}()
+	if err := srv.Serve(l); err != nil && err != http.ErrServerClosed {
+		common.Log.Errorf("lb: admission server: %v", err)
+	}
+}
+
+func (c *Controller) admit(ctx context.Context, req admit.Request) (bool, int, error) {
+	ip := net.ParseIP(req.IP).To4()
+	if req.Namespace == "" || req.Name == "" || ip == nil {
+		return false, http.StatusBadRequest, fmt.Errorf("need namespace, name and an IPv4 address")
+	}
+	select {
+	case <-c.ready:
+	case <-ctx.Done():
+		return false, http.StatusServiceUnavailable, ctx.Err()
+	case <-time.After(admit.Timeout - 5*time.Second):
+		return false, http.StatusServiceUnavailable, fmt.Errorf("not ready yet")
+	}
+	key := req.Namespace + "/" + req.Name
+	var pod *coreapi.Pod
+	if o, ok, _ := c.pods.GetByKey(key); ok {
+		pod = o.(*coreapi.Pod)
+	} else {
+		gctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		p, err := c.client.CoreV1().Pods(req.Namespace).Get(gctx, req.Name, metav1.GetOptions{})
+		cancel()
+		if err != nil {
+			return false, http.StatusServiceUnavailable, fmt.Errorf("fetching pod: %v", err)
+		}
+		pod = p
+	}
+	if pod.Spec.NodeName != c.self {
+		return false, http.StatusConflict, fmt.Errorf("pod is bound to %q, not this node", pod.Spec.NodeName)
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if !c.egressViewSynced() {
+		return false, http.StatusServiceUnavailable, fmt.Errorf("EgressServices not synced yet")
+	}
+	c.admitted[ip.String()] = Admission{Namespace: req.Namespace, Name: req.Name}
+	c.at[ip.String()] = time.Now()
+	if _, ok, _ := c.pods.GetByKey(key); !ok {
+		c.fetched[key] = pod
+	}
+	if err := c.sync(ctx); err != nil {
+		// with no EgressService and no member rules in place there is nothing to protect or to clean up for
+		// this IP: a broken pass must not stop every pod on the node from starting
+		if len(c.lastEgress.Members) == 0 && (c.egresses == nil || len(c.egresses.List()) == 0) {
+			common.Log.Warnf("lb: admitting %s/%s despite a failed pass (no EgressServices): %v", req.Namespace, req.Name, err)
+			return false, http.StatusOK, nil
+		}
+		return false, http.StatusServiceUnavailable, err
+	}
+	for _, m := range c.lastEgress.Members {
+		if m.IP.Equal(ip) {
+			return true, http.StatusOK, nil
+		}
+	}
+	return false, http.StatusOK, nil
+}
+
+// publish keeps this node's annotations equal to what its interfaces say.
+func (c *Controller) publish(ctx context.Context, local Local) error {
+	want := local.Annotations()
+	o, ok, err := c.nodes.GetByKey(c.self)
+	if err != nil {
+		return err
+	}
+	if ok {
+		have := o.(*coreapi.Node).Annotations
+		same := true
+		for k, v := range want {
+			if have[k] != v {
+				same = false
+			}
+		}
+		if same {
+			return nil
+		}
+	}
+	patch, err := json.Marshal(map[string]interface{}{"metadata": map[string]interface{}{"annotations": want}})
+	if err != nil {
+		return err
+	}
+	if _, err := c.client.CoreV1().Nodes().Patch(ctx, c.self, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil {
+		return fmt.Errorf("publishing node annotations: %v", err)
+	}
+	common.Log.Infof("lb: node %s holds [%s]", c.self, want[AnnAddresses])
+	return nil
+}
+
+// writeStatus sets (ipMode Proxy: kube-proxy leaves the address to us) or clears a Service's load-balancer status.
+func (c *Controller) writeStatus(ctx context.Context, st Status) error {
+	ingress := []map[string]string{}
+	if st.Address != nil {
+		ingress = append(ingress, map[string]string{"ip": st.Address.String(), "ipMode": "Proxy"})
+	}
+	patch, err := json.Marshal(map[string]interface{}{"status": map[string]interface{}{"loadBalancer": map[string]interface{}{"ingress": ingress}}})
+	if err != nil {
+		return err
+	}
+	s, err := c.client.CoreV1().Services(st.Namespace).Patch(ctx, st.Name, types.MergePatchType, patch, metav1.PatchOptions{}, "status")
+	if err != nil {
+		return fmt.Errorf("status of %s/%s: %v", st.Namespace, st.Name, err)
+	}
+	if st.Address != nil {
+		c.written[s.UID] = st.Address.String()
+		common.Log.Infof("lb: %s/%s at %s", st.Namespace, st.Name, st.Address)
+	} else {
+		delete(c.written, s.UID)
+		common.Log.Infof("lb: %s/%s has no address", st.Namespace, st.Name)
+	}
+	return nil
+}

lb/datapath.go

@@ -0,0 +1,336 @@
+package lb
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"net"
+	"os/exec"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// ForwardChain (nat, from PREROUTING) DNATs held addresses' ports to endpoints.
+	ForwardChain = "REWEAVE-LB"
+	// PostChain (nat, from POSTROUTING, ahead of weave's WEAVE) keeps the client's address on what ForwardChain DNATed.
+	PostChain = "REWEAVE-LB-POST"
+	// MarkChain (mangle, from PREROUTING) marks holders' connections and steers their replies.
+	MarkChain = "REWEAVE-LB-MARK"
+	// servicePrefix names the per-port chains ForwardChain jumps to.
+	servicePrefix = "REWEAVE-LBS-"
+
+	markShift = 24
+	markMask  = uint32(0xff) << markShift
+	// RulePriority is where the reply-steering rules sit: before the main table (32766).
+	RulePriority = 9000
+	// TableBase + mark is the routing table that sends replies to that mark's holder.
+	TableBase = 52000
+
+	bridge = "weave"
+)
+
+func markValue(m uint32) uint32 { return m << markShift }
+
+// portChain is the per-(service, protocol, port) chain name: stable, short enough for iptables (28 chars).
+func portChain(f Forward, p Port) string {
+	h := sha256.Sum256([]byte(fmt.Sprintf("%s/%s/%s/%d", f.Namespace, f.Name, p.Protocol, p.Port)))
+	return servicePrefix + strings.ToUpper(hex.EncodeToString(h[:])[:12])
+}
+
+// Rules is the full content of the chains this package owns, in iptables-restore form.
+type Rules struct {
+	Nat, Mangle, Filter string
+}
+
+// Render turns the desired state (load-balancer forwards and egress) into iptables-restore input for the nat,
+// mangle and filter tables. podNet is the weave allocation range (pods and bridges).
+// Each payload declares every chain it owns, so a restore with --noflush replaces exactly those.
+func Render(d Desired, e EgressDesired, self string, podNet *net.IPNet, stale []string) Rules {
+	var nat bytes.Buffer
+	nat.WriteString("*nat\n")
+	for _, c := range []string{ForwardChain, PostChain, ClassChain} {
+		fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
+	}
+	var body bytes.Buffer
+	seen := map[string]bool{}
+	for _, f := range d.Forwards {
+		a := f.Address.String()
+		if seen[a] {
+			continue
+		}
+		seen[a] = true
+		// weave masquerades everything entering the pod network from outside unless it is for a local pod
+		// (its expose rule); a connection we DNATed must keep the client's address, so it leaves the nat
+		// table here. One from a pod (hairpin) is masqueraded to this node, so its reply comes back here.
+		fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -s %s -j MASQUERADE\n", PostChain, a, podNet)
+		fmt.Fprintf(&body, "-A %s -m conntrack --ctstate DNAT --ctorigdst %s/32 -j ACCEPT\n", PostChain, a)
+	}
+	for _, f := range d.Forwards {
+		for _, p := range f.Ports {
+			c := portChain(f, p)
+			fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
+			proto := strings.ToLower(string(p.Protocol))
+			fmt.Fprintf(&body, "-A %s -d %s/32 -p %s -m %s --dport %d -m comment --comment %q -j %s\n",
+				ForwardChain, f.Address, proto, proto, p.Port, f.Namespace+"/"+f.Name, c)
+			n := len(p.Endpoints)
+			for i, e := range p.Endpoints {
+				// DNAT to an ip:port needs the protocol in the same rule
+				if i < n-1 {
+					// the i-th endpoint takes 1/(n-i) of what the earlier ones left: uniform overall
+					fmt.Fprintf(&body, "-A %s -p %s -m statistic --mode random --probability %.10f -j DNAT --to-destination %s\n",
+						c, proto, 1/float64(n-i), e)
+				} else {
+					fmt.Fprintf(&body, "-A %s -p %s -j DNAT --to-destination %s\n", c, proto, e)
+				}
+			}
+		}
+	}
+	for _, c := range stale {
+		// chains of ports no longer forwarded: emptied here, deleted after the restore
+		fmt.Fprintf(&nat, ":%s - [0:0]\n", c)
+	}
+
+	var mangle bytes.Buffer
+	mangle.WriteString("*mangle\n")
+	for _, c := range []string{MarkChain, MSSChain} {
+		fmt.Fprintf(&mangle, ":%s - [0:0]\n", c)
+	}
+	if len(d.Holders) > 0 {
+		// replies of a holder's connection carry its mark: the policy rule routes them back to it
+		fmt.Fprintf(&mangle, "-A %s -m conntrack --ctdir REPLY -m connmark ! --mark 0x0/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n",
+			MarkChain, markMask, markMask, markMask)
+		for _, h := range d.Holders {
+			// a new connection to a pod here, arriving from the holder's bridge, from outside the pod network:
+			// the holder DNATed it and must see its replies
+			fmt.Fprintf(&mangle, "-A %s -i %s -m mac --mac-source %s ! -s %s -d %s -m conntrack --ctstate NEW -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n",
+				MarkChain, bridge, h.Node.WeaveMAC, podNet, podNet, "holder "+h.Node.Name, markValue(h.Mark), markMask)
+		}
+	}
+
+	var filter bytes.Buffer
+	filter.WriteString("*filter\n")
+	for _, c := range []string{FwdChain, InChain} {
+		fmt.Fprintf(&filter, ":%s - [0:0]\n", c)
+	}
+	renderEgress(e, self, podNet, &body, &mangle, &filter)
+
+	nat.Write(body.Bytes())
+	nat.WriteString("COMMIT\n")
+	mangle.WriteString("COMMIT\n")
+	filter.WriteString("COMMIT\n")
+	return Rules{Nat: nat.String(), Mangle: mangle.String(), Filter: filter.String()}
+}
+
+// Datapath applies Desired to this node.
+type Datapath struct {
+	ipt *iptables.IPTables
+	// restore runs iptables-restore --noflush with the given input (swappable in tests)
+	restore func(input string) error
+}
+
+// NewDatapath uses the host's iptables (whichever backend the image's launch script selected).
+func NewDatapath(ipt *iptables.IPTables) *Datapath {
+	return &Datapath{ipt: ipt, restore: func(input string) error {
+		cmd := exec.Command("iptables-restore", "--noflush", "-w", "5")
+		cmd.Stdin = strings.NewReader(input)
+		if out, err := cmd.CombinedOutput(); err != nil {
+			return fmt.Errorf("iptables-restore: %v: %s", err, out)
+		}
+		return nil
+	}}
+}
+
+// Apply makes the node's iptables chains, tunnel, policy rules and routes exactly what d and e say.
+func (dp *Datapath) Apply(d Desired, e EgressDesired, self string, local Local) error {
+	podNet := local.PodNet
+	if err := dp.hooks(); err != nil {
+		return err
+	}
+	// the tunnels must exist before the routes into them (iptables rules may name a device that does not exist yet)
+	var peers []Node
+	if e.Tunnelled(self) {
+		peers = e.Peers(self)
+	}
+	if err := ensureTunnels(peers, local); err != nil {
+		return err
+	}
+	existing, err := dp.ipt.ListChains("nat")
+	if err != nil {
+		return err
+	}
+	want := map[string]bool{}
+	for _, f := range d.Forwards {
+		for _, p := range f.Ports {
+			want[portChain(f, p)] = true
+		}
+	}
+	var stale []string
+	for _, c := range existing {
+		if strings.HasPrefix(c, servicePrefix) && !want[c] {
+			stale = append(stale, c)
+		}
+	}
+	r := Render(d, e, self, podNet, stale)
+	// filter first: the guards are in place before the marks and SNATs that they guard
+	for _, payload := range []string{r.Filter, r.Nat, r.Mangle} {
+		if err := dp.restore(payload); err != nil {
+			return err
+		}
+	}
+	for _, c := range stale {
+		if err := dp.ipt.DeleteChain("nat", c); err != nil {
+			return fmt.Errorf("deleting stale chain %s: %v", c, err)
+		}
+	}
+	if err := dp.routes(d.Holders); err != nil {
+		return err
+	}
+	return egressNet(e, self)
+}
+
+// hooks makes sure the built-in chains jump to ours: first (kube-proxy's and weave's rules must not see these
+// connections first), except ClassChain, which must come after kube-proxy's KUBE-SERVICES (whose DNAT ends the
+// nat walk, so a connection to a Service is never classed as going outside).
+func (dp *Datapath) hooks() error {
+	for _, h := range []struct {
+		table, from, chain string
+		last               bool
+	}{
+		{"nat", "PREROUTING", ForwardChain, false},
+		{"nat", "POSTROUTING", PostChain, false},
+		{"nat", "PREROUTING", ClassChain, true},
+		{"mangle", "PREROUTING", MarkChain, false},
+		{"mangle", "FORWARD", MSSChain, false},
+		{"filter", "FORWARD", FwdChain, false},
+		{"filter", "INPUT", InChain, false},
+	} {
+		ok, err := dp.ipt.ChainExists(h.table, h.chain)
+		if err != nil {
+			return err
+		}
+		if !ok {
+			if err := dp.ipt.NewChain(h.table, h.chain); err != nil {
+				return err
+			}
+		}
+		rule := []string{"-m", "comment", "--comment", "reweave lb", "-j", h.chain}
+		rules, err := dp.ipt.List(h.table, h.from)
+		if err != nil {
+			return err
+		}
+		if h.last {
+			ours, kube := -1, -1
+			for i, r := range rules {
+				if strings.HasSuffix(r, "-j "+h.chain) {
+					ours = i
+				}
+				if strings.HasSuffix(r, "-j KUBE-SERVICES") {
+					kube = i
+				}
+			}
+			if ours > kube {
+				continue
+			}
+			if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
+				return err
+			}
+			if err := dp.ipt.Append(h.table, h.from, rule...); err != nil {
+				return err
+			}
+			continue
+		}
+		// rules[0] is the policy line; our jump must be rules[1]
+		if len(rules) > 1 && strings.HasSuffix(rules[1], "-j "+h.chain) {
+			continue
+		}
+		if err := dp.ipt.DeleteIfExists(h.table, h.from, rule...); err != nil {
+			return err
+		}
+		if err := dp.ipt.Insert(h.table, h.from, 1, rule...); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// routes keeps one policy rule and one table per holder: marked replies go to the holder's bridge.
+func (dp *Datapath) routes(holders []Holder) error {
+	link, err := netlink.LinkByName(bridge)
+	if err != nil {
+		if len(holders) == 0 {
+			return nil
+		}
+		return fmt.Errorf("weave bridge: %v", err)
+	}
+	want := map[int]Holder{}
+	for _, h := range holders {
+		want[TableBase+int(h.Mark)] = h
+	}
+	rules, err := netlink.RuleList(unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	have := map[int]bool{}
+	for _, r := range rules {
+		if r.Priority != RulePriority || r.Table < TableBase || r.Table > TableBase+MaxMarks {
+			continue
+		}
+		h, ok := want[r.Table]
+		if ok && uint32(r.Mark) == markValue(h.Mark) && uint32(r.Mask) == markMask {
+			have[r.Table] = true
+			continue
+		}
+		r := r
+		if err := netlink.RuleDel(&r); err != nil {
+			return fmt.Errorf("removing stale rule to table %d: %v", r.Table, err)
+		}
+	}
+	existing, err := routesByTable(TableBase+1, TableBase+MaxMarks)
+	if err != nil {
+		return err
+	}
+	for table := TableBase + 1; table <= TableBase+MaxMarks; table++ {
+		h, ok := want[table]
+		if !ok {
+			for _, rt := range existing[table] {
+				if err := delRoute(rt); err != nil {
+					return fmt.Errorf("removing stale route in table %d: %v", table, err)
+				}
+			}
+			continue
+		}
+		if err := netlink.RouteReplace(&netlink.Route{
+			LinkIndex: link.Attrs().Index,
+			Dst:       &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
+			Gw:        h.Node.WeaveIP,
+			Table:     table,
+			Flags:     int(netlink.FLAG_ONLINK),
+		}); err != nil {
+			return fmt.Errorf("route to holder %s: %v", h.Node.Name, err)
+		}
+		if !have[table] {
+			rule := netlink.NewRule()
+			rule.Priority = RulePriority
+			rule.Table = table
+			rule.Mark = int(markValue(h.Mark))
+			rule.Mask = int(markMask)
+			if err := netlink.RuleAdd(rule); err != nil {
+				return fmt.Errorf("rule for holder %s: %v", h.Node.Name, err)
+			}
+		}
+	}
+	return nil
+}
+
+// delRoute deletes a route as RouteList returned it: a default route lists without Dst, which RouteDel refuses.
+func delRoute(rt netlink.Route) error {
+	if rt.Dst == nil {
+		rt.Dst = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}
+	}
+	return netlink.RouteDel(&rt)
+}

lb/egress.go

@@ -0,0 +1,340 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+
+	coreapi "k8s.io/api/core/v1"
+)
+
+// Port blocks: every node gets its own source ports on an EgressService's address, so the holder can route a
+// reply back to the member's node by port alone (no state on the holder). The range lies above the default
+// ephemeral range (32768-60999), so it never meets the holder's own connections from the address.
+const (
+	BlockBase  = 61000
+	BlockSize  = 512
+	BlockCount = 8 // 61000-65095
+)
+
+// Admission is a pod the CNI plugin announced at ADD, before the API shows its IP.
+type Admission struct {
+	Namespace, Name string
+}
+
+// Member is one pod on this node that an EgressService selects, and where its outside traffic goes.
+type Member struct {
+	IP      net.IP
+	Pod     string // namespace/name
+	Service string // namespace/name of its EgressService (empty if it is in several)
+	// Unreachable: its outside traffic is dropped (no holder, no block, several EgressServices, ...)
+	Unreachable bool
+	Address     net.IP // it leaves from Address:[From, To]
+	From, To    int32
+	Holder      Node // the node that emits it; Holder.Name == self: emitted here
+}
+
+// BlockRoute is, on the holder, the replies to Address:[From, To] going back to Peer.
+type BlockRoute struct {
+	Address  net.IP
+	From, To int32
+	Peer     Node
+}
+
+// EgressStatusWrite is a status this node writes (the holder, or the first node by name to clear it).
+type EgressStatusWrite struct {
+	Namespace, Name string
+	Status          EgressServiceStatus
+}
+
+// EgressDesired is this node's part of every EgressService.
+type EgressDesired struct {
+	Members   []Member
+	Routes    []BlockRoute
+	Status    []EgressStatusWrite
+	NodeAddrs []net.IP // the cluster's node addresses: members' traffic to them is not "outside"
+	Problems  []string
+}
+
+// Tunnelled reports whether this node sends or receives anything over the egress tunnel.
+func (e EgressDesired) Tunnelled(self string) bool {
+	if len(e.Routes) > 0 {
+		return true
+	}
+	for _, m := range e.Members {
+		if !m.Unreachable && m.Holder.Name != self {
+			return true
+		}
+	}
+	return false
+}
+
+// Peers are the nodes this node exchanges tunnelled packets with (sorted by name, deduplicated).
+func (e EgressDesired) Peers(self string) []Node {
+	seen := map[string]Node{}
+	for _, r := range e.Routes {
+		seen[r.Peer.Name] = r.Peer
+	}
+	for _, m := range e.Members {
+		if !m.Unreachable && m.Holder.Name != self {
+			seen[m.Holder.Name] = m.Holder
+		}
+	}
+	out := make([]Node, 0, len(seen))
+	for _, n := range seen {
+		out = append(out, n)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// AllocateBlocks keeps every valid block a node already has and gives each node without one the lowest free
+// block, in name order. A node beyond BlockCount gets none (its members are dropped, and that is reported).
+func AllocateBlocks(current []PortBlock, nodeNames []string) []PortBlock {
+	present := map[string]bool{}
+	for _, n := range nodeNames {
+		present[n] = true
+	}
+	used := map[int]bool{}
+	have := map[string]PortBlock{}
+	for _, b := range current {
+		k := int(b.From-BlockBase) / BlockSize
+		valid := present[b.Node] && b.From >= BlockBase && int(b.From-BlockBase)%BlockSize == 0 &&
+			k < BlockCount && b.To == b.From+BlockSize-1
+		if !valid || used[k] {
+			continue
+		}
+		if _, dup := have[b.Node]; dup {
+			continue
+		}
+		used[k] = true
+		have[b.Node] = b
+	}
+	names := append([]string(nil), nodeNames...)
+	sort.Strings(names)
+	for _, n := range names {
+		if _, ok := have[n]; ok {
+			continue
+		}
+		for k := 0; k < BlockCount; k++ {
+			if !used[k] {
+				used[k] = true
+				from := int32(BlockBase + k*BlockSize)
+				have[n] = PortBlock{Node: n, From: from, To: from + BlockSize - 1}
+				break
+			}
+		}
+	}
+	out := make([]PortBlock, 0, len(have))
+	for _, b := range have {
+		out = append(out, b)
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
+	return out
+}
+
+func sameStatus(a, b EgressServiceStatus) bool {
+	if a.Host != b.Host || a.Address != b.Address || len(a.Blocks) != len(b.Blocks) {
+		return false
+	}
+	for i := range a.Blocks {
+		if a.Blocks[i] != b.Blocks[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// holderOf is S1's rule for a Service's address: the one node that holds spec.loadBalancerIP.
+func holderOf(s *coreapi.Service, nodes []Node) (*Node, net.IP) {
+	want := net.ParseIP(s.Spec.LoadBalancerIP).To4()
+	if want == nil {
+		return nil, nil
+	}
+	var hs []Node
+	for _, n := range nodes {
+		if n.Holds(want) {
+			hs = append(hs, n)
+		}
+	}
+	if len(hs) != 1 {
+		return nil, want
+	}
+	return &hs[0], want
+}
+
+// selects reports whether a Service's selector (non-empty) matches the labels.
+func selects(s *coreapi.Service, labels map[string]string) bool {
+	if len(s.Spec.Selector) == 0 {
+		return false
+	}
+	for k, v := range s.Spec.Selector {
+		if labels[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+// ComputeEgress decides this node's part of every EgressService: the statuses it writes, the members it hosts and
+// where their traffic goes, and (as a holder) where replies to each node's port block go.
+// pods are this node's pods; admitted maps an IP to the pod the CNI plugin gave it (newer than the API may show).
+func ComputeEgress(self string, nodes []Node, services []*coreapi.Service, egresses []*EgressService,
+	pods []*coreapi.Pod, admitted map[string]Admission) EgressDesired {
+	var d EgressDesired
+	sorted := append([]Node(nil), nodes...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Name < sorted[j].Name })
+	byName := map[string]Node{}
+	var names []string
+	for _, n := range sorted {
+		byName[n.Name] = n
+		names = append(names, n.Name)
+		d.NodeAddrs = append(d.NodeAddrs, n.Addresses...)
+	}
+	svcs := map[string]*coreapi.Service{}
+	for _, s := range services {
+		svcs[s.Namespace+"/"+s.Name] = s
+	}
+	es := append([]*EgressService(nil), egresses...)
+	sort.Slice(es, func(i, j int) bool { return es[i].Namespace+"/"+es[i].Name < es[j].Namespace+"/"+es[j].Name })
+
+	// valid[key]: the EgressService is in force: its Service is served from an address one node holds, and the
+	// holder has published that (status) - what members and holder both act on
+	valid := map[string]bool{}
+	for _, e := range es {
+		key := e.Namespace + "/" + e.Name
+		var want EgressServiceStatus
+		var holder *Node
+		var addr net.IP
+		s := svcs[key]
+		switch {
+		case e.Spec.Emit != EmitHolder:
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: spec.emit must be %q (the only mode so far), got %q", key, EmitHolder, e.Spec.Emit))
+		case e.Spec.SourceIPBy != "" && e.Spec.SourceIPBy != SourceIPByLoadBalancerIP:
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: spec.sourceIPBy must be %q, got %q", key, SourceIPByLoadBalancerIP, e.Spec.SourceIPBy))
+		case s == nil:
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: no Service %s", key, key))
+		case !eligible(s):
+			d.Problems = append(d.Problems, fmt.Sprintf("egress %s: Service %s is not a LoadBalancer of class %s", key, key, Class))
+		default:
+			holder, addr = holderOf(s, sorted)
+			if holder == nil {
+				d.Problems = append(d.Problems, fmt.Sprintf("egress %s: no single node holds %s; members' outside traffic is dropped", key, s.Spec.LoadBalancerIP))
+			}
+		}
+		writer := ""
+		if holder != nil {
+			want = EgressServiceStatus{Host: holder.Name, Address: addr.String(), Blocks: AllocateBlocks(e.Status.Blocks, names)}
+			writer = holder.Name
+			if len(want.Blocks) < len(names) {
+				d.Problems = append(d.Problems, fmt.Sprintf("egress %s: only %d port blocks for %d nodes; members on nodes without one are dropped", key, BlockCount, len(names)))
+			}
+		} else if len(names) > 0 {
+			writer = names[0]
+		}
+		if writer == self && !sameStatus(e.Status, want) {
+			d.Status = append(d.Status, EgressStatusWrite{Namespace: e.Namespace, Name: e.Name, Status: want})
+		}
+		// in force only once the status in the API says what holds now (members and holder read the same thing)
+		if holder != nil && e.Status.Host == holder.Name && e.Status.Address == addr.String() {
+			valid[key] = true
+			if holder.Name == self {
+				for _, b := range e.Status.Blocks {
+					if b.Node == self {
+						continue
+					}
+					peer, ok := byName[b.Node]
+					if !ok || peer.WeaveIP == nil {
+						d.Problems = append(d.Problems, fmt.Sprintf("egress %s: node %s has no weave bridge published; its members' replies cannot reach it", key, b.Node))
+						continue
+					}
+					d.Routes = append(d.Routes, BlockRoute{Address: addr, From: b.From, To: b.To, Peer: peer})
+				}
+			}
+		}
+	}
+
+	// this node's pods, one per IP: the CNI plugin's latest word wins, then live pods over terminating ones
+	type candidate struct {
+		pod *coreapi.Pod
+		ip  string
+	}
+	byIP := map[string]*coreapi.Pod{}
+	podByKey := map[string]*coreapi.Pod{}
+	for _, p := range pods {
+		podByKey[p.Namespace+"/"+p.Name] = p
+	}
+	var cands []candidate
+	for _, p := range pods {
+		if p.Spec.NodeName != self || p.Spec.HostNetwork || p.Status.Phase == coreapi.PodSucceeded || p.Status.Phase == coreapi.PodFailed {
+			continue
+		}
+		if ip := net.ParseIP(p.Status.PodIP).To4(); ip != nil {
+			cands = append(cands, candidate{p, ip.String()})
+		}
+	}
+	for ip, a := range admitted {
+		if p := podByKey[a.Namespace+"/"+a.Name]; p != nil && p.Spec.NodeName == self && !p.Spec.HostNetwork {
+			cands = append(cands, candidate{p, ip})
+		}
+	}
+	for _, c := range cands {
+		if a, ok := admitted[c.ip]; ok && (a.Namespace != c.pod.Namespace || a.Name != c.pod.Name) {
+			continue // the IP was given to another pod since
+		}
+		cur := byIP[c.ip]
+		if cur == nil || (cur.DeletionTimestamp != nil && c.pod.DeletionTimestamp == nil) ||
+			(cur.DeletionTimestamp == nil) == (c.pod.DeletionTimestamp == nil) && c.pod.CreationTimestamp.After(cur.CreationTimestamp.Time) {
+			byIP[c.ip] = c.pod
+		}
+	}
+	ips := make([]string, 0, len(byIP))
+	for ip := range byIP {
+		ips = append(ips, ip)
+	}
+	sort.Strings(ips)
+	for _, ip := range ips {
+		p := byIP[ip]
+		var in []*EgressService
+		for _, e := range es {
+			if e.Namespace != p.Namespace {
+				continue
+			}
+			if s := svcs[e.Namespace+"/"+e.Name]; s != nil && selects(s, p.Labels) {
+				in = append(in, e)
+			}
+		}
+		if len(in) == 0 {
+			continue
+		}
+		m := Member{IP: net.ParseIP(ip).To4(), Pod: p.Namespace + "/" + p.Name, Unreachable: true}
+		if len(in) > 1 {
+			keys := make([]string, len(in))
+			for i, e := range in {
+				keys[i] = e.Namespace + "/" + e.Name
+			}
+			d.Problems = append(d.Problems, fmt.Sprintf("pod %s is selected by several EgressServices (%s); its outside traffic is dropped", m.Pod, strings.Join(keys, ", ")))
+			d.Members = append(d.Members, m)
+			continue
+		}
+		e := in[0]
+		m.Service = e.Namespace + "/" + e.Name
+		if valid[m.Service] {
+			holder := byName[e.Status.Host]
+			for _, b := range e.Status.Blocks {
+				if b.Node == self && (holder.Name == self || holder.WeaveIP != nil) {
+					m.Unreachable = false
+					m.Address = net.ParseIP(e.Status.Address).To4()
+					m.From, m.To = b.From, b.To
+					m.Holder = holder
+				}
+			}
+			if m.Unreachable {
+				d.Problems = append(d.Problems, fmt.Sprintf("pod %s: egress %s has no port block for %s (or its holder's bridge is unknown); its outside traffic is dropped", m.Pod, m.Service, self))
+			}
+		}
+		d.Members = append(d.Members, m)
+	}
+	return d
+}

lb/egress_api.go

@@ -0,0 +1,144 @@
+package lb
+
+import (
+	"context"
+	"encoding/json"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/rest"
+)
+
+// EgressService (reweave.levk.dev/v1alpha1) makes the pods a LoadBalancer Service selects leave the cluster
+// from the Service's address. It is named after the Service (same namespace, same name). Its semantics follow
+// OVN-Kubernetes' EgressService (k8s.ovn.org/v1, sourceIPBy: LoadBalancerIP) where they agree; they differ in
+// that membership is every pod the Service's selector matches (by pod IP, never by readiness), the holder is
+// the node that holds the address (no election), and a member's traffic to the outside leaves as the address
+// or not at all.
+type EgressService struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	Spec              EgressServiceSpec   `json:"spec"`
+	Status            EgressServiceStatus `json:"status,omitempty"`
+}
+
+// EgressServiceSpec is what the operator asks for.
+type EgressServiceSpec struct {
+	// SourceIPBy is LoadBalancerIP: members leave from the Service's load-balancer address (the only mode in v1).
+	SourceIPBy string `json:"sourceIPBy,omitempty"`
+	// Emit is where members' packets leave the cluster. Explicit, no default: Holder (tunnelled to the node
+	// holding the address, which emits them) is the only mode in v1.
+	Emit string `json:"emit"`
+}
+
+// EgressServiceStatus is written by the holder alone.
+type EgressServiceStatus struct {
+	// Host is the node that holds the address and emits members' packets.
+	Host string `json:"host,omitempty"`
+	// Address is the address members leave from.
+	Address string `json:"address,omitempty"`
+	// Blocks gives every node its own source ports on Address, so the holder routes replies back by port.
+	Blocks []PortBlock `json:"blocks,omitempty"`
+}
+
+// PortBlock is the source-port range [From, To] on the address that one node's members use.
+type PortBlock struct {
+	Node string `json:"node"`
+	From int32  `json:"from"`
+	To   int32  `json:"to"`
+}
+
+// EgressServiceList is a list of EgressServices.
+type EgressServiceList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []EgressService `json:"items"`
+}
+
+const (
+	// Group and Version of the EgressService API.
+	Group   = "reweave.levk.dev"
+	Version = "v1alpha1"
+	// EgressServices is the resource name.
+	EgressServices = "egressservices"
+
+	SourceIPByLoadBalancerIP = "LoadBalancerIP"
+	EmitHolder               = "Holder"
+)
+
+// GroupVersion of the EgressService API.
+var GroupVersion = schema.GroupVersion{Group: Group, Version: Version}
+
+// DeepCopyObject implements runtime.Object.
+func (e *EgressService) DeepCopyObject() runtime.Object { return e.DeepCopy() }
+
+// DeepCopy copies the whole object.
+func (e *EgressService) DeepCopy() *EgressService {
+	if e == nil {
+		return nil
+	}
+	out := *e
+	e.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Status.Blocks = append([]PortBlock(nil), e.Status.Blocks...)
+	return &out
+}
+
+// DeepCopyObject implements runtime.Object.
+func (l *EgressServiceList) DeepCopyObject() runtime.Object {
+	if l == nil {
+		return nil
+	}
+	out := *l
+	l.ListMeta.DeepCopyInto(&out.ListMeta)
+	out.Items = make([]EgressService, len(l.Items))
+	for i := range l.Items {
+		out.Items[i] = *l.Items[i].DeepCopy()
+	}
+	return &out
+}
+
+func addKnownTypes(s *runtime.Scheme) error {
+	s.AddKnownTypes(GroupVersion, &EgressService{}, &EgressServiceList{})
+	metav1.AddToGroupVersion(s, GroupVersion)
+	return nil
+}
+
+// NewEgressClient is a REST client for the EgressService API (no generated clientset: two plain types).
+func NewEgressClient(base *rest.Config) (*rest.RESTClient, error) {
+	scheme := runtime.NewScheme()
+	if err := addKnownTypes(scheme); err != nil {
+		return nil, err
+	}
+	cfg := *base
+	cfg.GroupVersion = &GroupVersion
+	cfg.APIPath = "/apis"
+	cfg.ContentType = runtime.ContentTypeJSON
+	cfg.NegotiatedSerializer = serializer.NewCodecFactory(scheme).WithoutConversion()
+	if cfg.UserAgent == "" {
+		cfg.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+	return rest.RESTClientFor(&cfg)
+}
+
+// PatchEgressStatus merge-patches an EgressService's status (nil fields are removed).
+func PatchEgressStatus(ctx context.Context, c rest.Interface, namespace, name string, st EgressServiceStatus) error {
+	body := map[string]interface{}{"host": nil, "address": nil, "blocks": nil}
+	if st.Host != "" {
+		body["host"] = st.Host
+	}
+	if st.Address != "" {
+		body["address"] = st.Address
+	}
+	if len(st.Blocks) > 0 {
+		body["blocks"] = st.Blocks
+	}
+	patch, err := json.Marshal(map[string]interface{}{"status": body})
+	if err != nil {
+		return err
+	}
+	return c.Patch(types.MergePatchType).Namespace(namespace).Resource(EgressServices).Name(name).
+		SubResource("status").Body(patch).Do(ctx).Error()
+}

lb/egress_datapath.go

@@ -0,0 +1,530 @@
+package lb
+
+import (
+	"bytes"
+	"fmt"
+	"hash/fnv"
+	"net"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// ClassChain (nat, appended to PREROUTING after kube-proxy's KUBE-SERVICES) marks members' new connections to
+	// the outside. kube-proxy's DNAT ends the nat walk, so a connection to a Service never gets here.
+	ClassChain = "REWEAVE-LB-CLASS"
+	// FwdChain (filter, first in FORWARD) drops members' outside packets that would not leave as their address.
+	FwdChain = "REWEAVE-LB-FWD"
+	// InChain (filter, first in INPUT) accepts the egress tunnels only from the cluster's nodes.
+	InChain = "REWEAVE-LB-IN"
+	// MSSChain (mangle, first in FORWARD) clamps TCP MSS into the tunnels.
+	MSSChain = "REWEAVE-LB-MSS"
+
+	// TunnelPrefix names the egress tunnels: one ipip device per peer node, between the two nodes' weave bridges,
+	// so a holder knows which node a member's packet came from by the device it arrives on.
+	TunnelPrefix = "rweg-"
+	// tunnelGlob matches every egress tunnel in an iptables rule.
+	tunnelGlob = "rweg+"
+	// legacyTunnel is the single shared tunnel of earlier builds (removed when found).
+	legacyTunnel = "reweave-eg"
+
+	// EgressMark marks a member's packets to the outside (connmark on its connections, fwmark on each packet).
+	EgressMark = uint32(0x100000)
+	// peerFlag, with a peer's index in the top byte, tags (on the holder) a connection that arrived from that peer's
+	// tunnel: replies and related ICMP errors carry it back there.
+	peerFlag = uint32(0x200000)
+	peerMask = markMask | peerFlag
+
+	// HolderRulePriority: replies to a block go to its node; ahead of the local table, which moves to LocalRulePriority,
+	// since the address is usually local on the holder.
+	HolderRulePriority = 90
+	LocalRulePriority  = 100
+	// MemberRulePriority: a member's marked packets go to its holder (or nowhere).
+	MemberRulePriority = 9100
+	// EgressTableBase + peer index routes into that peer's tunnel; EgressTableBase itself is unreachable.
+	EgressTableBase = 53000
+
+	ipipOverhead = 20
+)
+
+// portProtocols carry ports: their replies are routed by port block alone (stateless), and members' SNAT keeps
+// within the node's block. Everything else is routed back by the tag the holder put on the connection.
+var portProtocols = []struct {
+	name string
+	num  int
+}{{"tcp", unix.IPPROTO_TCP}, {"udp", unix.IPPROTO_UDP}, {"sctp", unix.IPPROTO_SCTP}}
+
+// TunnelName is the egress tunnel towards a peer node (stable, within the 15 characters a device name may have).
+func TunnelName(peer string) string {
+	h := fnv.New32a()
+	h.Write([]byte(peer))
+	return fmt.Sprintf("%s%08x", TunnelPrefix, h.Sum32())
+}
+
+func peerTag(idx uint32) uint32 { return markValue(idx) | peerFlag }
+
+// renderEgress appends the egress rules of each table to the bodies of S1's payloads.
+func renderEgress(e EgressDesired, self string, podNet *net.IPNet, nat, mangle, filter *bytes.Buffer) {
+	members := e.Members
+	if len(members) > 0 {
+		// what is not "outside": the pod network, multicast, the cluster's nodes
+		fmt.Fprintf(nat, "-A %s -d %s -j RETURN\n", ClassChain, podNet)
+		fmt.Fprintf(nat, "-A %s -d 224.0.0.0/4 -j RETURN\n", ClassChain)
+		seen := map[string]bool{}
+		for _, a := range e.NodeAddrs {
+			if !seen[a.String()] {
+				seen[a.String()] = true
+				fmt.Fprintf(nat, "-A %s -d %s/32 -j RETURN\n", ClassChain, a)
+			}
+		}
+		for _, m := range members {
+			fmt.Fprintf(nat, "-A %s -s %s/32 -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n", ClassChain, m.IP, m.Pod, EgressMark, EgressMark)
+			fmt.Fprintf(nat, "-A %s -s %s/32 -j MARK --set-xmark 0x%x/0x%x\n", ClassChain, m.IP, EgressMark, EgressMark)
+		}
+		// later packets of a member's outside connection carry the mark too (the nat table saw only the first)
+		fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir ORIGINAL -m connmark --mark 0x%x/0x%x -j MARK --set-xmark 0x%x/0x%x\n",
+			MarkChain, EgressMark, EgressMark, EgressMark, EgressMark)
+	}
+	for _, m := range members {
+		if m.Unreachable {
+			fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
+			continue
+		}
+		out := "-o " + TunnelName(m.Holder.Name)
+		if m.Holder.Name == self {
+			// emitted here, from the address: never into the pod network unSNATed
+			out = "! -o " + bridge
+			fmt.Fprintf(filter, "-A %s -s %s/32 -o %s -m mark --mark 0x%x/0x%x -j DROP\n", FwdChain, m.IP, bridge, EgressMark, EgressMark)
+		}
+		// within this node's block for what has ports; the address alone for the rest (the holder tells those
+		// apart by the tunnel they arrive on)
+		for _, p := range portProtocols {
+			fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p %s -j SNAT --to-source %s:%d-%d\n",
+				PostChain, m.IP, out, EgressMark, EgressMark, p.name, m.Address, m.From, m.To)
+		}
+		// ICMP queries (ping) too: NAT treats the echo identifier as a port, and without a block of their own two
+		// nodes' pods (whose small, alike PIDs make alike identifiers) collide on the holder
+		fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -p icmp -j SNAT --to-source %s:%d-%d\n",
+			PostChain, m.IP, out, EgressMark, EgressMark, m.Address, m.From, m.To)
+		fmt.Fprintf(nat, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j SNAT --to-source %s\n",
+			PostChain, m.IP, out, EgressMark, EgressMark, m.Address)
+		fmt.Fprintf(filter, "-A %s -s %s/32 %s -m mark --mark 0x%x/0x%x -j RETURN\n", FwdChain, m.IP, out, EgressMark, EgressMark)
+		// a member's outside traffic leaves no other way
+		fmt.Fprintf(filter, "-A %s -s %s/32 -m mark --mark 0x%x/0x%x -m comment --comment %q -j DROP\n", FwdChain, m.IP, EgressMark, EgressMark, m.Pod)
+	}
+	if !e.Tunnelled(self) {
+		return
+	}
+	tags := peerIndex(e, self)
+
+	// as holder: tag each connection arriving from a peer's tunnel with that peer, once; replies and related ICMP
+	// errors restore the tag, and the policy rule for it sends them back into that tunnel
+	byPeer := map[string][]BlockRoute{}
+	var peerNames []string
+	for _, r := range e.Routes {
+		if _, ok := byPeer[r.Peer.Name]; !ok {
+			peerNames = append(peerNames, r.Peer.Name)
+		}
+		byPeer[r.Peer.Name] = append(byPeer[r.Peer.Name], r)
+	}
+	sort.Strings(peerNames)
+	if len(peerNames) > 0 {
+		fmt.Fprintf(mangle, "-A %s -m conntrack --ctdir REPLY -m connmark --mark 0x%x/0x%x -j CONNMARK --restore-mark --nfmask 0x%x --ctmask 0x%x\n",
+			MarkChain, peerFlag, peerFlag, peerMask, peerMask)
+	}
+	for _, name := range peerNames {
+		dev, tag := TunnelName(name), peerTag(tags[name])
+		fmt.Fprintf(mangle, "-A %s -i %s -m conntrack --ctstate NEW -m connmark --mark 0x0/0x%x -m comment --comment %q -j CONNMARK --set-xmark 0x%x/0x%x\n",
+			MarkChain, dev, peerMask, "from "+name, tag, peerMask)
+		// a connection another node already holds (a port-less protocol to the same remote): never cross-delivered
+		fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -m connmark ! --mark 0x%x/0x%x -j DROP\n", FwdChain, dev, tag, peerMask)
+		addrs := map[string]bool{}
+		for _, r := range byPeer[name] {
+			for _, p := range portProtocols {
+				fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -m %s --sport %d:%d -m comment --comment %q -j RETURN\n",
+					FwdChain, dev, r.Address, p.name, p.name, r.From, r.To, "block of "+name)
+			}
+			addrs[r.Address.String()] = true
+		}
+		for _, a := range sortedSet(addrs) {
+			// outside its own block a node may not use the ports of others
+			for _, p := range portProtocols {
+				fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -p %s -j DROP\n", FwdChain, dev, a, p.name)
+			}
+			fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctdir ORIGINAL -s %s/32 -j RETURN\n", FwdChain, dev, a)
+		}
+	}
+	// as a member's node: replies (and related errors) from the holder; nothing else comes back
+	addrs := map[string]bool{}
+	for _, m := range members {
+		if !m.Unreachable && m.Holder.Name != self {
+			addrs[m.Address.String()] = true
+		}
+	}
+	for _, a := range sortedSet(addrs) {
+		// still addressed to the address after nat: no member connection matched it
+		fmt.Fprintf(filter, "-A %s -i %s -d %s/32 -j DROP\n", FwdChain, tunnelGlob, a)
+	}
+	fmt.Fprintf(filter, "-A %s -i %s -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n", FwdChain, tunnelGlob)
+	fmt.Fprintf(filter, "-A %s -i %s -j DROP\n", FwdChain, tunnelGlob)
+	// the tunnels themselves only from the cluster's nodes' bridges
+	for _, p := range e.Peers(self) {
+		fmt.Fprintf(filter, "-A %s -p 4 -s %s/32 -m mac --mac-source %s -j RETURN\n", InChain, p.WeaveIP, p.WeaveMAC)
+	}
+	fmt.Fprintf(filter, "-A %s -p 4 -j DROP\n", InChain)
+	fmt.Fprintf(mangle, "-A %s -o %s -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n", MSSChain, tunnelGlob)
+}
+
+func sortedSet(m map[string]bool) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// peerIndex gives each tunnel peer a stable index (1..MaxMarks): its tag and its routing table.
+func peerIndex(e EgressDesired, self string) map[string]uint32 {
+	out := map[string]uint32{}
+	for _, h := range assignMarks(e.Peers(self)) {
+		out[h.Node.Name] = h.Mark
+	}
+	return out
+}
+
+// peerTables gives each tunnel peer its routing table (EgressTableBase + its index).
+func peerTables(e EgressDesired, self string) map[string]int {
+	out := map[string]int{}
+	for name, idx := range peerIndex(e, self) {
+		out[name] = EgressTableBase + int(idx)
+	}
+	return out
+}
+
+// egressRule is a policy rule this package owns, in a comparable form.
+type egressRule struct {
+	priority   int
+	src, dst   string
+	ipproto    int
+	from, to   uint16
+	mark, mask uint32
+	table      int
+}
+
+func (r egressRule) key() string {
+	return fmt.Sprintf("%d|%s|%s|%d|%d-%d|%x/%x|%d", r.priority, r.src, r.dst, r.ipproto, r.from, r.to, r.mark, r.mask, r.table)
+}
+
+// wantRules is every egress policy rule this node needs.
+func wantRules(e EgressDesired, self string) []egressRule {
+	tables := peerTables(e, self)
+	tags := peerIndex(e, self)
+	var out []egressRule
+	tagged := map[string]bool{}
+	for _, r := range e.Routes {
+		for _, p := range portProtocols {
+			out = append(out, egressRule{priority: HolderRulePriority, dst: r.Address.String() + "/32", ipproto: p.num,
+				from: uint16(r.From), to: uint16(r.To), table: tables[r.Peer.Name]})
+		}
+		if !tagged[r.Peer.Name] {
+			// whatever the tag says came from that peer (replies without ports, ICMP errors) goes back to it
+			tagged[r.Peer.Name] = true
+			out = append(out, egressRule{priority: HolderRulePriority, mark: peerTag(tags[r.Peer.Name]), mask: peerMask, table: tables[r.Peer.Name]})
+		}
+	}
+	for _, m := range e.Members {
+		if !m.Unreachable && m.Holder.Name == self {
+			continue // emitted here: the main table already sends it out
+		}
+		table := EgressTableBase
+		if !m.Unreachable {
+			table = tables[m.Holder.Name]
+		}
+		out = append(out, egressRule{priority: MemberRulePriority, src: m.IP.String() + "/32", mark: EgressMark, mask: EgressMark, table: table})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].key() < out[j].key() })
+	return out
+}
+
+func ruleOf(r netlink.Rule) egressRule {
+	er := egressRule{priority: r.Priority, ipproto: r.IPProto, table: r.Table}
+	if r.Src != nil {
+		er.src = r.Src.String()
+	}
+	if r.Dst != nil {
+		er.dst = r.Dst.String()
+	}
+	if r.Dport != nil {
+		er.from, er.to = r.Dport.Start, r.Dport.End
+	}
+	if r.Mark > 0 {
+		er.mark = uint32(r.Mark)
+		er.mask = uint32(r.Mask)
+	}
+	if er.ipproto < 0 {
+		er.ipproto = 0
+	}
+	return er
+}
+
+// routesByTable lists, in one dump, the IPv4 routes of tables lo..hi.
+func routesByTable(lo, hi int) (map[int][]netlink.Route, error) {
+	all, err := netlink.RouteListFiltered(unix.AF_INET, &netlink.Route{Table: unix.RT_TABLE_UNSPEC}, netlink.RT_FILTER_TABLE)
+	if err != nil {
+		return nil, err
+	}
+	out := map[int][]netlink.Route{}
+	for _, rt := range all {
+		if rt.Table >= lo && rt.Table <= hi {
+			out[rt.Table] = append(out[rt.Table], rt)
+		}
+	}
+	return out, nil
+}
+
+// egressNet makes the local rule's place, the policy rules and the tunnel tables exactly what e needs (the
+// tunnels themselves are made first, by Apply).
+func egressNet(e EgressDesired, self string) error {
+	if len(e.Routes) > 0 {
+		if err := moveLocalRule(); err != nil {
+			return err
+		}
+	}
+	tables := peerTables(e, self)
+	peerOf := map[int]Node{}
+	for _, p := range e.Peers(self) {
+		peerOf[tables[p.Name]] = p
+	}
+	unreachable := false
+	for _, m := range e.Members {
+		unreachable = unreachable || m.Unreachable
+	}
+	existing, err := routesByTable(EgressTableBase, EgressTableBase+MaxMarks)
+	if err != nil {
+		return err
+	}
+	for table := EgressTableBase; table <= EgressTableBase+MaxMarks; table++ {
+		var want *netlink.Route
+		if p, ok := peerOf[table]; ok {
+			link, err := netlink.LinkByName(TunnelName(p.Name))
+			if err != nil {
+				return fmt.Errorf("egress tunnel to %s: %v", p.Name, err)
+			}
+			want = &netlink.Route{LinkIndex: link.Attrs().Index, Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
+				Scope: netlink.SCOPE_LINK, Table: table}
+		} else if table == EgressTableBase && unreachable {
+			want = &netlink.Route{Dst: &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, Table: table, Type: unix.RTN_UNREACHABLE}
+		}
+		keep := false
+		for _, rt := range existing[table] {
+			if want != nil && rt.Type == want.Type &&
+				(want.Type == unix.RTN_UNREACHABLE || rt.LinkIndex == want.LinkIndex && rt.Gw == nil) {
+				keep = true
+				continue
+			}
+			if err := delRoute(rt); err != nil {
+				return fmt.Errorf("removing stale route in table %d: %v", table, err)
+			}
+		}
+		if want != nil && !keep {
+			if err := netlink.RouteReplace(want); err != nil {
+				return fmt.Errorf("route in table %d: %v", table, err)
+			}
+		}
+	}
+	want := map[string]egressRule{}
+	for _, r := range wantRules(e, self) {
+		want[r.key()] = r
+	}
+	rules, err := netlink.RuleList(unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	have := map[string]bool{}
+	for _, r := range rules {
+		if r.Priority != HolderRulePriority && r.Priority != MemberRulePriority {
+			continue
+		}
+		k := ruleOf(r).key()
+		if _, ok := want[k]; ok && !have[k] {
+			have[k] = true
+			continue
+		}
+		r := r
+		if err := netlink.RuleDel(&r); err != nil {
+			return fmt.Errorf("removing stale egress rule: %v", err)
+		}
+	}
+	for k, r := range want {
+		if have[k] {
+			continue
+		}
+		rule := netlink.NewRule()
+		rule.Priority = r.priority
+		rule.Table = r.table
+		if r.src != "" {
+			_, rule.Src, _ = net.ParseCIDR(r.src)
+		}
+		if r.dst != "" {
+			_, rule.Dst, _ = net.ParseCIDR(r.dst)
+		}
+		if r.ipproto != 0 {
+			rule.IPProto = r.ipproto
+			rule.Dport = netlink.NewRulePortRange(r.from, r.to)
+		}
+		if r.mark != 0 {
+			rule.Mark = int(r.mark)
+			rule.Mask = int(r.mask)
+		}
+		if err := netlink.RuleAdd(rule); err != nil {
+			return fmt.Errorf("adding egress rule %s: %v", k, err)
+		}
+	}
+	return nil
+}
+
+// moveLocalRule puts the local table's lookup after HolderRulePriority (it is at 0 by default), so the holder can
+// route replies to its own address onwards to their node instead of delivering them to itself.
+func moveLocalRule() error {
+	rules, err := netlink.RuleList(unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	var at0, atLocal bool
+	for _, r := range rules {
+		if r.Table == unix.RT_TABLE_LOCAL && r.Src == nil && r.Dst == nil && r.Mark <= 0 {
+			switch {
+			case r.Priority <= 0:
+				// the kernel leaves out the priority attribute when it is 0: listed as unset (-1)
+				at0 = true
+			case r.Priority == LocalRulePriority:
+				atLocal = true
+			}
+		}
+	}
+	if !atLocal {
+		rule := netlink.NewRule()
+		rule.Priority = LocalRulePriority
+		rule.Table = unix.RT_TABLE_LOCAL
+		if err := netlink.RuleAdd(rule); err != nil {
+			return fmt.Errorf("adding local table rule at %d: %v", LocalRulePriority, err)
+		}
+	}
+	if at0 {
+		rule := netlink.NewRule()
+		rule.Priority = 0
+		rule.Table = unix.RT_TABLE_LOCAL
+		if err := netlink.RuleDel(rule); err != nil {
+			return fmt.Errorf("removing local table rule at 0: %v", err)
+		}
+	}
+	return nil
+}
+
+// ensureTunnels makes one ipip tunnel per peer (this node's bridge address to the peer's), and removes every
+// other egress tunnel (including the shared one of earlier builds).
+func ensureTunnels(peers []Node, local Local) error {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return err
+	}
+	want := map[string]Node{}
+	for _, p := range peers {
+		want[TunnelName(p.Name)] = p
+	}
+	have := map[string]netlink.Link{}
+	for _, l := range links {
+		name := l.Attrs().Name
+		if name == legacyTunnel || strings.HasPrefix(name, TunnelPrefix) {
+			if _, ok := want[name]; ok {
+				have[name] = l
+				continue
+			}
+			if err := netlink.LinkDel(l); err != nil {
+				return fmt.Errorf("removing egress tunnel %s: %v", name, err)
+			}
+		}
+	}
+	if len(want) == 0 {
+		return nil
+	}
+	if local.WeaveIP == nil {
+		return fmt.Errorf("egress tunnels need the weave bridge's address")
+	}
+	br, err := netlink.LinkByName(bridge)
+	if err != nil {
+		return fmt.Errorf("weave bridge: %v", err)
+	}
+	mtu := br.Attrs().MTU - ipipOverhead
+	names := make([]string, 0, len(want))
+	for n := range want {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, name := range names {
+		peer := want[name]
+		link := have[name]
+		if link != nil {
+			t, ok := link.(*netlink.Iptun)
+			if !ok || !t.Local.Equal(local.WeaveIP) || !t.Remote.Equal(peer.WeaveIP) || t.Attrs().MTU != mtu {
+				if err := netlink.LinkDel(link); err != nil {
+					return err
+				}
+				link = nil
+			}
+		}
+		if link == nil {
+			t := &netlink.Iptun{LinkAttrs: netlink.LinkAttrs{Name: name, MTU: mtu}, Local: local.WeaveIP, Remote: peer.WeaveIP, PMtuDisc: 1}
+			if err := netlink.LinkAdd(t); err != nil {
+				return fmt.Errorf("creating egress tunnel %s to %s: %v", name, peer.Name, err)
+			}
+			if link, err = netlink.LinkByName(name); err != nil {
+				return err
+			}
+		}
+		if err := tunnelSetup(link, local); err != nil {
+			return fmt.Errorf("%s: %v", name, err)
+		}
+	}
+	return nil
+}
+
+// tunnelSetup: the holder receives members' packets from its own address (accept_local); reverse-path filtering
+// rejects what arrives on a device with no IPv4 address unless the reverse route leaves by that very device, so
+// the tunnel carries this node's bridge address (never on the wire from here: the tunnel sets the outer source,
+// and what crosses it is forwarded).
+func tunnelSetup(link netlink.Link, local Local) error {
+	name := link.Attrs().Name
+	for k, v := range map[string]string{"accept_local": "1", "rp_filter": "2"} {
+		if err := os.WriteFile("/proc/sys/net/ipv4/conf/"+name+"/"+k, []byte(v), 0o644); err != nil {
+			return fmt.Errorf("sysctl %s: %v", k, err)
+		}
+	}
+	addrs, err := netlink.AddrList(link, unix.AF_INET)
+	if err != nil {
+		return err
+	}
+	has := false
+	for _, a := range addrs {
+		if a.IP.Equal(local.WeaveIP) {
+			has = true
+			continue
+		}
+		a := a
+		if err := netlink.AddrDel(link, &a); err != nil {
+			return fmt.Errorf("removing stale address %s: %v", a.IPNet, err)
+		}
+	}
+	if !has {
+		addr := &netlink.Addr{IPNet: &net.IPNet{IP: local.WeaveIP, Mask: net.CIDRMask(32, 32)}, Scope: unix.RT_SCOPE_LINK}
+		if err := netlink.AddrAdd(link, addr); err != nil {
+			return fmt.Errorf("address: %v", err)
+		}
+	}
+	return netlink.LinkSetUp(link)
+}

lb/egress_test.go

@@ -0,0 +1,355 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	coreapi "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// the S1 fixtures: cp, w1 (holds 86.107.168.15), w2
+
+func egressFor(name string, status EgressServiceStatus) *EgressService {
+	return &EgressService{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name},
+		Spec: EgressServiceSpec{Emit: EmitHolder}, Status: status}
+}
+
+func mailSvc() *coreapi.Service {
+	return svc("mail", "86.107.168.15", func(s *coreapi.Service) {
+		s.Spec.Selector = map[string]string{"app": "mail"}
+	})
+}
+
+func pod(name, node, ip string, labels map[string]string, mutate ...func(*coreapi.Pod)) *coreapi.Pod {
+	p := &coreapi.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name, Labels: labels,
+		CreationTimestamp: metav1.NewTime(time.Unix(1000, 0))},
+		Spec: coreapi.PodSpec{NodeName: node}, Status: coreapi.PodStatus{PodIP: ip, Phase: coreapi.PodRunning}}
+	for _, m := range mutate {
+		m(p)
+	}
+	return p
+}
+
+var mailLabels = map[string]string{"app": "mail"}
+
+// the status the holder (w1) writes for three nodes, blocks in name order
+func written() EgressServiceStatus {
+	return EgressServiceStatus{Host: "w1", Address: "86.107.168.15", Blocks: []PortBlock{
+		{Node: "cp", From: 61000, To: 61511}, {Node: "w1", From: 61512, To: 62023}, {Node: "w2", From: 62024, To: 62535}}}
+}
+
+func TestBlocksAreStableAndCapped(t *testing.T) {
+	got := AllocateBlocks(nil, []string{"w2", "cp", "w1"})
+	if !sameStatus(EgressServiceStatus{Blocks: got}, EgressServiceStatus{Blocks: written().Blocks}) {
+		t.Fatalf("fresh blocks: %+v", got)
+	}
+	// a node leaves, another joins: survivors keep their blocks, the newcomer takes the freed lowest one
+	got = AllocateBlocks(got, []string{"w1", "w2", "w3"})
+	want := []PortBlock{{"w1", 61512, 62023}, {"w2", 62024, 62535}, {"w3", 61000, 61511}}
+	if !sameStatus(EgressServiceStatus{Blocks: got}, EgressServiceStatus{Blocks: want}) {
+		t.Fatalf("after churn: %+v", got)
+	}
+	// misaligned or duplicate blocks are not kept
+	got = AllocateBlocks([]PortBlock{{"w1", 61001, 61512}, {"w2", 61000, 61511}, {"w3", 61000, 61511}}, []string{"w1", "w2", "w3"})
+	for _, b := range got {
+		if (b.From-BlockBase)%BlockSize != 0 || b.To != b.From+BlockSize-1 {
+			t.Fatalf("invalid block kept: %+v", got)
+		}
+	}
+	if got[1].Node != "w2" || got[1].From != 61000 {
+		t.Fatalf("first valid claim wins: %+v", got)
+	}
+	var many []string
+	for i := 0; i < BlockCount+2; i++ {
+		many = append(many, string(rune('a'+i)))
+	}
+	if n := len(AllocateBlocks(nil, many)); n != BlockCount {
+		t.Fatalf("more blocks than exist: %d", n)
+	}
+	for _, b := range AllocateBlocks(nil, many) {
+		if b.To > 65535 || b.From < 61000 {
+			t.Fatalf("block outside the range: %+v", b)
+		}
+	}
+}
+
+func TestHolderWritesStatusAndMembersWaitForIt(t *testing.T) {
+	e := egressFor("mail", EgressServiceStatus{})
+	pods := []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}
+	d := ComputeEgress("w1", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{e}, nil, nil)
+	if len(d.Status) != 1 || !sameStatus(d.Status[0].Status, written()) {
+		t.Fatalf("holder's status: %+v", d.Status)
+	}
+	// until the status is in the API, a member's outside traffic goes nowhere (fail closed)
+	d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{e}, pods, nil)
+	if len(d.Status) != 0 || len(d.Members) != 1 || !d.Members[0].Unreachable {
+		t.Fatalf("member before status: %+v", d)
+	}
+	if d.Tunnelled("w2") {
+		t.Fatalf("no tunnel for an unreachable member")
+	}
+}
+
+func TestMemberTunnelsToHolderWhichRoutesItsBlockBack(t *testing.T) {
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	pods := []*coreapi.Pod{
+		pod("mail-0", "w2", "10.44.0.9", mailLabels),
+		pod("web", "w2", "10.44.0.10", map[string]string{"app": "web"}), // not a member (A4)
+	}
+	d := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, pods, nil)
+	if len(d.Members) != 1 || d.Members[0].Pod != "gitea/mail-0" {
+		t.Fatalf("members: %+v", d.Members)
+	}
+	m := d.Members[0]
+	if m.Unreachable || !m.Address.Equal(net.ParseIP("86.107.168.15")) || m.From != 62024 || m.To != 62535 || m.Holder.Name != "w1" {
+		t.Fatalf("member path: %+v", m)
+	}
+	if !d.Tunnelled("w2") || len(d.Peers("w2")) != 1 || d.Peers("w2")[0].Name != "w1" {
+		t.Fatalf("w2 tunnels to w1 only: %+v", d.Peers("w2"))
+	}
+	if len(d.Status) != 0 || len(d.Routes) != 0 {
+		t.Fatalf("only the holder writes and routes: %+v", d)
+	}
+	// the holder: blocks of the other nodes route back to them; its own block is plain conntrack
+	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
+	if len(h.Status) != 0 {
+		t.Fatalf("status already right: %+v", h.Status)
+	}
+	if len(h.Routes) != 2 || h.Routes[0].Peer.Name != "cp" || h.Routes[1].Peer.Name != "w2" || h.Routes[1].From != 62024 {
+		t.Fatalf("holder routes: %+v", h.Routes)
+	}
+	// a member on the holder leaves from there
+	hm := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
+	if len(hm.Members) != 1 || hm.Members[0].Holder.Name != "w1" || hm.Members[0].From != 61512 {
+		t.Fatalf("member on the holder: %+v", hm.Members)
+	}
+}
+
+func TestEgressFailsClosed(t *testing.T) {
+	pods := []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}
+	// no node holds the address: status cleared (by the first node by name), member dropped
+	unheld := svc("mail", "86.107.168.99", func(s *coreapi.Service) { s.Spec.Selector = mailLabels })
+	for _, self := range []string{"cp", "w2"} {
+		d := ComputeEgress(self, nodes, []*coreapi.Service{unheld}, []*EgressService{egressFor("mail", written())}, pods, nil)
+		clears := len(d.Status) == 1 && d.Status[0].Status.Host == ""
+		if clears != (self == "cp") {
+			t.Fatalf("%s clears=%v: %+v", self, clears, d.Status)
+		}
+		if self == "w2" && (len(d.Members) != 1 || !d.Members[0].Unreachable) {
+			t.Fatalf("member of an unheld address: %+v", d.Members)
+		}
+	}
+	// two EgressServices select the pod: dropped, reported
+	other := svc("mail2", "86.107.168.15", func(s *coreapi.Service) { s.Spec.Selector = mailLabels })
+	d := ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc(), other},
+		[]*EgressService{egressFor("mail", written()), egressFor("mail2", written())}, pods, nil)
+	if len(d.Members) != 1 || !d.Members[0].Unreachable || !strings.Contains(strings.Join(d.Problems, "\n"), "several EgressServices") {
+		t.Fatalf("double member: %+v", d)
+	}
+	// a spec this agent does not implement is refused, not guessed at
+	bad := egressFor("mail", written())
+	bad.Spec.Emit = "Direct"
+	d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{bad}, pods, nil)
+	if len(d.Members) != 1 || !d.Members[0].Unreachable {
+		t.Fatalf("unknown emit mode: %+v", d)
+	}
+	// hostNetwork pods are never members (their traffic is the node's)
+	hn := pod("mail-h", "w2", "86.107.168.88", mailLabels, func(p *coreapi.Pod) { p.Spec.HostNetwork = true })
+	d = ComputeEgress("w2", nodes, []*coreapi.Service{mailSvc()}, []*EgressService{egressFor("mail", written())}, []*coreapi.Pod{hn}, nil)
+	if len(d.Members) != 0 {
+		t.Fatalf("hostNetwork pod became a member: %+v", d.Members)
+	}
+}
+
+func TestAdmissionDecidesWhoOwnsAnIP(t *testing.T) {
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	old := pod("mail-0", "w2", "10.44.0.9", mailLabels, func(p *coreapi.Pod) {
+		now := metav1.Now()
+		p.DeletionTimestamp = &now
+	})
+	newer := pod("web", "w2", "", map[string]string{"app": "web"}) // API has no IP for it yet
+	// the CNI plugin just gave 10.44.0.9 to web: the terminating member must not keep it (A4)
+	d := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{old, newer},
+		map[string]Admission{"10.44.0.9": {Namespace: "gitea", Name: "web"}})
+	if len(d.Members) != 0 {
+		t.Fatalf("reused IP kept the old member's rules: %+v", d.Members)
+	}
+	// and the other way round: a member admitted before the API shows its IP is protected at once (A1)
+	fresh := pod("mail-1", "w2", "", mailLabels)
+	d = ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{fresh},
+		map[string]Admission{"10.44.0.11": {Namespace: "gitea", Name: "mail-1"}})
+	if len(d.Members) != 1 || d.Members[0].IP.String() != "10.44.0.11" || d.Members[0].Unreachable {
+		t.Fatalf("admitted member: %+v", d.Members)
+	}
+	// without an admission, a live pod wins a shared IP over a terminating one
+	live := pod("web", "w2", "10.44.0.9", map[string]string{"app": "web"})
+	d = ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{old, live}, nil)
+	if len(d.Members) != 0 {
+		t.Fatalf("terminating member won the IP: %+v", d.Members)
+	}
+}
+
+func TestRenderedEgressRules(t *testing.T) {
+	_, podNet, _ := net.ParseCIDR("10.32.0.0/12")
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	toW1, toW2, toCP := TunnelName("w1"), TunnelName("w2"), TunnelName("cp")
+	member := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
+	r := Render(Desired{}, member, "w2", podNet, nil)
+	for _, want := range []string{
+		"-A REWEAVE-LB-CLASS -d 10.32.0.0/12 -j RETURN\n",
+		"-A REWEAVE-LB-CLASS -d 86.107.168.24/32 -j RETURN\n", // a node is not outside
+		"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -m comment --comment \"gitea/mail-0\" -j CONNMARK --set-xmark 0x100000/0x100000\n",
+		"-A REWEAVE-LB-CLASS -s 10.44.0.9/32 -j MARK --set-xmark 0x100000/0x100000\n",
+		// port protocols within this node's block, into the holder's own tunnel
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p udp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p sctp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		// ping within the block too (the echo identifier is NAT's port for ICMP)
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -p icmp -j SNAT --to-source 86.107.168.15:62024-62535\n",
+		// everything else (GRE, ...) from the address alone
+		"-A REWEAVE-LB-POST -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j SNAT --to-source 86.107.168.15\n",
+	} {
+		if !strings.Contains(r.Nat, want) {
+			t.Fatalf("member nat lacks %q:\n%s", want, r.Nat)
+		}
+	}
+	// the catch-all SNAT comes after the port-ranged ones (or ports would not stay in the block)
+	if strings.Index(r.Nat, "-j SNAT --to-source 86.107.168.15\n") < strings.Index(r.Nat, "-p icmp -j SNAT") {
+		t.Fatalf("SNAT order:\n%s", r.Nat)
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -o " + toW1 + " -m mark --mark 0x100000/0x100000 -j RETURN\n",
+		"-A REWEAVE-LB-FWD -s 10.44.0.9/32 -m mark --mark 0x100000/0x100000 -m comment --comment \"gitea/mail-0\" -j DROP\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -d 86.107.168.15/32 -j DROP\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -m conntrack --ctstate ESTABLISHED,RELATED -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i rweg+ -j DROP\n",
+		"-A REWEAVE-LB-IN -p 4 -s 10.40.0.0/32 -m mac --mac-source aa:aa:aa:aa:aa:02 -j RETURN\n",
+		"-A REWEAVE-LB-IN -p 4 -j DROP\n",
+	} {
+		if !strings.Contains(r.Filter, want) {
+			t.Fatalf("member filter lacks %q:\n%s", want, r.Filter)
+		}
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir ORIGINAL -m connmark --mark 0x100000/0x100000 -j MARK --set-xmark 0x100000/0x100000\n",
+		"-A REWEAVE-LB-MSS -o rweg+ -p tcp -m tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("member mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	if strings.Contains(r.Mangle, "--set-xmark 0x") && strings.Contains(r.Mangle, "-i "+toW1) {
+		t.Fatalf("a member's node does not tag what its holder sends back:\n%s", r.Mangle)
+	}
+	// the guard's DROP comes after the member's own RETURN: nothing else of its leaves
+	if strings.Index(r.Filter, toW1+" -m mark --mark 0x100000/0x100000 -j RETURN") > strings.Index(r.Filter, "\"gitea/mail-0\" -j DROP") {
+		t.Fatalf("guard order:\n%s", r.Filter)
+	}
+
+	// the holder: tags each tunnel's connections with its node, accepts each node's block, nothing foreign
+	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-1", "w1", "10.40.0.4", mailLabels)}, nil)
+	r = Render(Desired{}, h, "w1", podNet, nil)
+	idx := peerIndex(h, "w1")
+	tagW2 := fmt.Sprintf("0x%x", peerTag(idx["w2"]))
+	for _, want := range []string{
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir REPLY -m connmark --mark 0x200000/0x200000 -j CONNMARK --restore-mark --nfmask 0xff200000 --ctmask 0xff200000\n",
+		"-A REWEAVE-LB-MARK -i " + toW2 + " -m conntrack --ctstate NEW -m connmark --mark 0x0/0xff200000 -m comment --comment \"from w2\" -j CONNMARK --set-xmark " + tagW2 + "/0xff200000\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("holder mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	for _, want := range []string{
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -m connmark ! --mark " + tagW2 + "/0xff200000 -j DROP\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p tcp -m tcp --sport 62024:62535 -m comment --comment \"block of w2\" -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i " + toCP + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p sctp -m sctp --sport 61000:61511 -m comment --comment \"block of cp\" -j RETURN\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP\n",
+		"-A REWEAVE-LB-FWD -i " + toW2 + " -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN\n",
+		"-A REWEAVE-LB-FWD -s 10.40.0.4/32 -o weave -m mark --mark 0x100000/0x100000 -j DROP\n",
+	} {
+		if !strings.Contains(r.Filter, want) {
+			t.Fatalf("holder filter lacks %q:\n%s", want, r.Filter)
+		}
+	}
+	// foreign tags are refused before anything of that tunnel is accepted; a port outside the block before the
+	// catch-all accept
+	if strings.Index(r.Filter, "! --mark "+tagW2) > strings.Index(r.Filter, "\"block of w2\"") ||
+		strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -p udp -j DROP") >
+			strings.Index(r.Filter, "-i "+toW2+" -m conntrack --ctdir ORIGINAL -s 86.107.168.15/32 -j RETURN") {
+		t.Fatalf("holder filter order:\n%s", r.Filter)
+	}
+	if !strings.Contains(r.Nat, "-A REWEAVE-LB-POST -s 10.40.0.4/32 ! -o weave -m mark --mark 0x100000/0x100000 -p tcp -j SNAT --to-source 86.107.168.15:61512-62023\n") {
+		t.Fatalf("holder's own member:\n%s", r.Nat)
+	}
+	for _, line := range strings.Split(r.Nat+r.Filter, "\n") {
+		if strings.Contains(line, "10.44.0.10") {
+			t.Fatalf("a rule names a non-member: %s", line)
+		}
+	}
+}
+
+func TestTunnelNames(t *testing.T) {
+	a, b := TunnelName("reweave-worker-fe19b040"), TunnelName("reweave-cp-94a8db43")
+	if a == b || len(a) > 15 || !strings.HasPrefix(a, TunnelPrefix) || a != TunnelName("reweave-worker-fe19b040") {
+		t.Fatalf("tunnel names must be distinct, stable, short: %q %q", a, b)
+	}
+}
+
+func TestPolicyRules(t *testing.T) {
+	e := egressFor("mail", written())
+	svcs := []*coreapi.Service{mailSvc()}
+	h := ComputeEgress("w1", nodes, svcs, []*EgressService{e}, nil, nil)
+	rules := wantRules(h, "w1")
+	tables := peerTables(h, "w1")
+	idx := peerIndex(h, "w1")
+	ports, tags := 0, 0
+	for _, r := range rules {
+		if r.priority != HolderRulePriority || r.src != "" {
+			t.Fatalf("holder rule: %+v", r)
+		}
+		switch {
+		case r.ipproto != 0:
+			ports++
+			if r.dst != "86.107.168.15/32" || r.mark != 0 {
+				t.Fatalf("port rule: %+v", r)
+			}
+			if (r.from == 62024) != (r.table == tables["w2"]) || (r.from == 61000) != (r.table == tables["cp"]) {
+				t.Fatalf("block routed to the wrong node: %+v %v", r, tables)
+			}
+		default:
+			tags++
+			if r.mask != peerMask || r.dst != "" {
+				t.Fatalf("tag rule: %+v", r)
+			}
+			for name, i := range idx {
+				if r.mark == peerTag(i) && r.table != tables[name] {
+					t.Fatalf("tag of %s routed to table %d, want %d", name, r.table, tables[name])
+				}
+			}
+		}
+	}
+	if ports != 2*len(portProtocols) || tags != 2 {
+		t.Fatalf("2 peers: %d port rules (tcp, udp, sctp each), %d tag rules: %+v", ports, tags, rules)
+	}
+	if HolderRulePriority >= LocalRulePriority {
+		t.Fatalf("holder rules must precede the local table")
+	}
+	m := ComputeEgress("w2", nodes, svcs, []*EgressService{e}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
+	rules = wantRules(m, "w2")
+	if len(rules) != 1 || rules[0].src != "10.44.0.9/32" || rules[0].mark != EgressMark || rules[0].mask != EgressMark ||
+		rules[0].table != peerTables(m, "w2")["w1"] {
+		t.Fatalf("member rule: %+v", rules)
+	}
+	u := ComputeEgress("w2", nodes, svcs, []*EgressService{egressFor("mail", EgressServiceStatus{})}, []*coreapi.Pod{pod("mail-0", "w2", "10.44.0.9", mailLabels)}, nil)
+	rules = wantRules(u, "w2")
+	if len(rules) != 1 || rules[0].table != EgressTableBase {
+		t.Fatalf("unreachable member must hit the unreachable table: %+v", rules)
+	}
+}

lb/model.go

@@ -0,0 +1,308 @@
+// Package lb gives LoadBalancer Services an address that a node holds, and
+// forwards that address's traffic to the Service's endpoints with the
+// client's address intact.
+//
+// A Service opts in with spec.loadBalancerClass = Class and names its
+// address in spec.loadBalancerIP. The node that holds the address (has it on
+// one of its interfaces, as published in its AnnAddresses annotation) is the
+// address's holder: it alone writes the Service's status (ipMode Proxy, so
+// kube-proxy installs no rules for the address) and DNATs the address's
+// ports to ready endpoints, without SNAT. Every other node marks the
+// connections that the holder forwards to its pods, and routes their replies
+// back to the holder over weave, so that the holder's conntrack can undo the
+// DNAT and the reply leaves from the address the client connected to.
+//
+// Compute is the whole policy as a pure function of what the API shows; the
+// datapath and the controller only carry it out.
+package lb
+
+import (
+	"fmt"
+	"hash/fnv"
+	"net"
+	"sort"
+	"strings"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+)
+
+const (
+	// Class is the spec.loadBalancerClass a Service sets to be served by this agent.
+	Class = "reweave.levk.dev/node-address"
+	// AnnAddresses lists the IPv4 addresses a node holds on its host interfaces (comma-separated).
+	AnnAddresses = "reweave.levk.dev/addresses"
+	// AnnWeave is "<ip>,<mac>" of the node's weave bridge: where replies to its forwards go.
+	AnnWeave = "reweave.levk.dev/weave"
+
+	serviceNameLabel = "kubernetes.io/service-name"
+)
+
+// Node is what one node publishes about itself.
+type Node struct {
+	Name      string
+	Addresses []net.IP
+	WeaveIP   net.IP
+	WeaveMAC  net.HardwareAddr
+}
+
+// Holds reports whether the node has ip on one of its host interfaces.
+func (n Node) Holds(ip net.IP) bool {
+	for _, a := range n.Addresses {
+		if a.Equal(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// ParseNode reads a Node's annotations. A node that has published nothing
+// (yet) holds nothing and cannot take replies.
+func ParseNode(n *coreapi.Node) (Node, error) {
+	out := Node{Name: n.Name}
+	if s := n.Annotations[AnnAddresses]; s != "" {
+		for _, f := range strings.Split(s, ",") {
+			ip := net.ParseIP(strings.TrimSpace(f)).To4()
+			if ip == nil {
+				return out, fmt.Errorf("node %s: %s: %q is not an IPv4 address", n.Name, AnnAddresses, f)
+			}
+			out.Addresses = append(out.Addresses, ip)
+		}
+	}
+	if s := n.Annotations[AnnWeave]; s != "" {
+		f := strings.Split(s, ",")
+		if len(f) != 2 {
+			return out, fmt.Errorf("node %s: %s: want \"<ip>,<mac>\", got %q", n.Name, AnnWeave, s)
+		}
+		ip := net.ParseIP(f[0]).To4()
+		mac, err := net.ParseMAC(f[1])
+		if ip == nil || err != nil {
+			return out, fmt.Errorf("node %s: %s: want \"<ip>,<mac>\", got %q", n.Name, AnnWeave, s)
+		}
+		out.WeaveIP, out.WeaveMAC = ip, mac
+	}
+	return out, nil
+}
+
+// FormatAddresses is the AnnAddresses value for a set of addresses (sorted, so it only changes when the set does).
+func FormatAddresses(ips []net.IP) string {
+	s := make([]string, 0, len(ips))
+	for _, ip := range ips {
+		s = append(s, ip.String())
+	}
+	sort.Strings(s)
+	return strings.Join(s, ",")
+}
+
+// Endpoint is one ready backend of one Service port.
+type Endpoint struct {
+	IP   net.IP
+	Port int32
+}
+
+func (e Endpoint) String() string { return fmt.Sprintf("%s:%d", e.IP, e.Port) }
+
+// Port is one Service port at the held address.
+type Port struct {
+	Protocol  coreapi.Protocol
+	Port      int32
+	Endpoints []Endpoint
+}
+
+// Forward is a Service whose address this node holds.
+type Forward struct {
+	Namespace, Name string
+	Address         net.IP
+	Ports           []Port
+}
+
+// Status is a write to a Service's status.loadBalancer: Address set, or cleared when nil.
+type Status struct {
+	Namespace, Name string
+	Address         net.IP
+}
+
+// Holder is another node that forwards held addresses' connections to pods; Mark identifies its connections here.
+type Holder struct {
+	Node Node
+	Mark uint32 // 1..MaxMarks, in the connmark's top byte
+}
+
+// MaxMarks is how many holders a node can tell apart (the top byte of the mark, 0 meaning none).
+const MaxMarks = 250
+
+// Desired is what one node must do.
+type Desired struct {
+	Forwards []Forward
+	Holders  []Holder
+	Status   []Status
+	Problems []string // what an operator should hear about (logged and evented by the controller)
+}
+
+// eligible reports whether the Service asks for this agent.
+func eligible(s *coreapi.Service) bool {
+	return s.Spec.Type == coreapi.ServiceTypeLoadBalancer && s.Spec.LoadBalancerClass != nil &&
+		*s.Spec.LoadBalancerClass == Class
+}
+
+// shown is the address status.loadBalancer currently shows (nil if none or not a single IP).
+func shown(s *coreapi.Service) net.IP {
+	if len(s.Status.LoadBalancer.Ingress) != 1 {
+		return nil
+	}
+	return net.ParseIP(s.Status.LoadBalancer.Ingress[0].IP).To4()
+}
+
+func hasStatus(s *coreapi.Service) bool { return len(s.Status.LoadBalancer.Ingress) != 0 }
+
+// Compute decides what node `self` does, given every node, every Service and every EndpointSlice.
+func Compute(self string, nodes []Node, services []*coreapi.Service, slices []*discovery.EndpointSlice) Desired {
+	var d Desired
+	sortedNodes := append([]Node(nil), nodes...)
+	sort.Slice(sortedNodes, func(i, j int) bool { return sortedNodes[i].Name < sortedNodes[j].Name })
+	holdersOf := func(ip net.IP) []Node {
+		var hs []Node
+		for _, n := range sortedNodes {
+			if ip != nil && n.Holds(ip) {
+				hs = append(hs, n)
+			}
+		}
+		return hs
+	}
+	// who may write a status that shows `ip`: its single holder, or (nobody holds it) the first node by name
+	writerFor := func(ip net.IP) string {
+		if hs := holdersOf(ip); len(hs) == 1 {
+			return hs[0].Name
+		}
+		if len(sortedNodes) > 0 {
+			return sortedNodes[0].Name
+		}
+		return ""
+	}
+	holding := map[string]bool{} // nodes that forward some address
+
+	svcs := append([]*coreapi.Service(nil), services...)
+	sort.Slice(svcs, func(i, j int) bool { return key(svcs[i]) < key(svcs[j]) })
+	for _, s := range svcs {
+		if !eligible(s) {
+			// loadBalancerClass is immutable and only allowed on type LoadBalancer, so a
+			// Service never leaves the class while keeping a status we wrote
+			continue
+		}
+		want := net.ParseIP(s.Spec.LoadBalancerIP).To4()
+		var holder *Node
+		switch hs := holdersOf(want); {
+		case want == nil:
+			d.Problems = append(d.Problems, fmt.Sprintf("%s: spec.loadBalancerIP must name an IPv4 address a node holds (got %q)", key(s), s.Spec.LoadBalancerIP))
+		case len(hs) == 0:
+			d.Problems = append(d.Problems, fmt.Sprintf("%s: no node holds %s", key(s), want))
+		case len(hs) > 1:
+			names := make([]string, len(hs))
+			for i, h := range hs {
+				names[i] = h.Name
+			}
+			d.Problems = append(d.Problems, fmt.Sprintf("%s: %s is held by several nodes (%s); serving it from none", key(s), want, strings.Join(names, ", ")))
+		default:
+			holder = &hs[0]
+		}
+		if holder == nil {
+			// fail closed: nobody forwards, and the status must not claim an address
+			if hasStatus(s) && writerFor(shown(s)) == self {
+				d.Status = append(d.Status, Status{Namespace: s.Namespace, Name: s.Name})
+			}
+			continue
+		}
+		holding[holder.Name] = true
+		if holder.Name != self {
+			continue
+		}
+		if cur := shown(s); cur == nil || !cur.Equal(want) {
+			d.Status = append(d.Status, Status{Namespace: s.Namespace, Name: s.Name, Address: want})
+		}
+		d.Forwards = append(d.Forwards, Forward{Namespace: s.Namespace, Name: s.Name, Address: want, Ports: ports(s, slices)})
+	}
+
+	var others []Node
+	for _, n := range sortedNodes {
+		if holding[n.Name] && n.Name != self {
+			if n.WeaveIP == nil {
+				d.Problems = append(d.Problems, fmt.Sprintf("node %s holds an address but has not published its weave bridge; replies to its forwards cannot reach it", n.Name))
+				continue
+			}
+			others = append(others, n)
+		}
+	}
+	d.Holders = assignMarks(others)
+	return d
+}
+
+// assignMarks gives each holder a mark from its name, probing past collisions in name order: stable while the set is.
+func assignMarks(nodes []Node) []Holder {
+	taken := map[uint32]bool{}
+	out := make([]Holder, 0, len(nodes))
+	for _, n := range nodes {
+		if len(out) == MaxMarks {
+			break
+		}
+		h := fnv.New32a()
+		h.Write([]byte(n.Name))
+		m := h.Sum32()%MaxMarks + 1
+		for taken[m] {
+			m = m%MaxMarks + 1
+		}
+		taken[m] = true
+		out = append(out, Holder{Node: n, Mark: m})
+	}
+	return out
+}
+
+func key(s *coreapi.Service) string { return s.Namespace + "/" + s.Name }
+
+// ports pairs each Service port with its ready endpoints.
+func ports(s *coreapi.Service, slices []*discovery.EndpointSlice) []Port {
+	var out []Port
+	for _, sp := range s.Spec.Ports {
+		proto := sp.Protocol
+		if proto == "" {
+			proto = coreapi.ProtocolTCP
+		}
+		seen := map[string]bool{}
+		var eps []Endpoint
+		for _, sl := range slices {
+			if sl.Namespace != s.Namespace || sl.Labels[serviceNameLabel] != s.Name || sl.AddressType != discovery.AddressTypeIPv4 {
+				continue
+			}
+			for _, p := range sl.Ports {
+				pname, pproto := "", coreapi.ProtocolTCP
+				if p.Name != nil {
+					pname = *p.Name
+				}
+				if p.Protocol != nil {
+					pproto = *p.Protocol
+				}
+				if pname != sp.Name || pproto != proto || p.Port == nil {
+					continue
+				}
+				for _, ep := range sl.Endpoints {
+					if ep.Conditions.Ready != nil && !*ep.Conditions.Ready {
+						continue
+					}
+					for _, a := range ep.Addresses {
+						ip := net.ParseIP(a).To4()
+						if ip == nil {
+							continue
+						}
+						e := Endpoint{IP: ip, Port: *p.Port}
+						if !seen[e.String()] {
+							seen[e.String()] = true
+							eps = append(eps, e)
+						}
+					}
+				}
+			}
+		}
+		sort.Slice(eps, func(i, j int) bool { return eps[i].String() < eps[j].String() })
+		out = append(out, Port{Protocol: proto, Port: sp.Port, Endpoints: eps})
+	}
+	return out
+}

lb/model_test.go

@@ -0,0 +1,253 @@
+package lb
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	coreapi "k8s.io/api/core/v1"
+	discovery "k8s.io/api/discovery/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func mac(s string) net.HardwareAddr {
+	m, err := net.ParseMAC(s)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}
+
+var (
+	cp = Node{Name: "cp", Addresses: []net.IP{net.ParseIP("86.107.168.24").To4()},
+		WeaveIP: net.ParseIP("10.32.0.1").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:01")}
+	w1 = Node{Name: "w1", Addresses: []net.IP{net.ParseIP("86.107.168.15").To4()},
+		WeaveIP: net.ParseIP("10.40.0.0").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:02")}
+	w2 = Node{Name: "w2", Addresses: []net.IP{net.ParseIP("86.107.168.88").To4()},
+		WeaveIP: net.ParseIP("10.44.0.0").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:03")}
+	nodes = []Node{w2, cp, w1}
+)
+
+func svc(name, ip string, mutate ...func(*coreapi.Service)) *coreapi.Service {
+	class := Class
+	s := &coreapi.Service{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "gitea", Name: name},
+		Spec: coreapi.ServiceSpec{
+			Type:              coreapi.ServiceTypeLoadBalancer,
+			LoadBalancerClass: &class,
+			LoadBalancerIP:    ip,
+			Ports: []coreapi.ServicePort{
+				{Name: "ssh", Protocol: coreapi.ProtocolTCP, Port: 22},
+				{Name: "https", Port: 443}, // protocol defaults to TCP
+			},
+		},
+	}
+	for _, m := range mutate {
+		m(s)
+	}
+	return s
+}
+
+func shows(ip string) func(*coreapi.Service) {
+	return func(s *coreapi.Service) {
+		s.Status.LoadBalancer.Ingress = []coreapi.LoadBalancerIngress{{IP: ip}}
+	}
+}
+
+func ptr[T any](v T) *T { return &v }
+
+func slice(svcName string, eps map[string]bool, ports map[string]int32) *discovery.EndpointSlice {
+	sl := &discovery.EndpointSlice{
+		ObjectMeta:  metav1.ObjectMeta{Namespace: "gitea", Name: svcName + "-x", Labels: map[string]string{serviceNameLabel: svcName}},
+		AddressType: discovery.AddressTypeIPv4,
+	}
+	for ip, ready := range eps {
+		sl.Endpoints = append(sl.Endpoints, discovery.Endpoint{Addresses: []string{ip}, Conditions: discovery.EndpointConditions{Ready: ptr(ready)}})
+	}
+	for name, port := range ports {
+		sl.Ports = append(sl.Ports, discovery.EndpointPort{Name: ptr(name), Protocol: ptr(coreapi.ProtocolTCP), Port: ptr(port)})
+	}
+	return sl
+}
+
+func TestHolderForwardsToReadyEndpointsAndWritesStatus(t *testing.T) {
+	s := svc("gitea", "86.107.168.15")
+	sl := slice("gitea", map[string]bool{"10.40.0.7": true, "10.44.0.3": true, "10.44.0.9": false},
+		map[string]int32{"ssh": 2222, "https": 3000})
+	d := Compute("w1", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
+	if len(d.Problems) != 0 {
+		t.Fatalf("problems: %v", d.Problems)
+	}
+	if len(d.Forwards) != 1 || !d.Forwards[0].Address.Equal(net.ParseIP("86.107.168.15")) {
+		t.Fatalf("forwards: %+v", d.Forwards)
+	}
+	f := d.Forwards[0]
+	if len(f.Ports) != 2 || f.Ports[0].Port != 22 || f.Ports[1].Port != 443 || f.Ports[1].Protocol != coreapi.ProtocolTCP {
+		t.Fatalf("ports: %+v", f.Ports)
+	}
+	if got := f.Ports[0].Endpoints; len(got) != 2 || got[0].String() != "10.40.0.7:2222" || got[1].String() != "10.44.0.3:2222" {
+		t.Fatalf("ssh endpoints (not-ready one must be left out): %v", got)
+	}
+	if got := f.Ports[1].Endpoints; len(got) != 2 || got[0].Port != 3000 {
+		t.Fatalf("https endpoints: %v", got)
+	}
+	if len(d.Status) != 1 || !d.Status[0].Address.Equal(net.ParseIP("86.107.168.15")) {
+		t.Fatalf("status: %+v", d.Status)
+	}
+	if len(d.Holders) != 0 {
+		t.Fatalf("a holder does not steer its own replies: %+v", d.Holders)
+	}
+	// already shown: nothing to write
+	d = Compute("w1", nodes, []*coreapi.Service{svc("gitea", "86.107.168.15", shows("86.107.168.15"))}, nil)
+	if len(d.Status) != 0 {
+		t.Fatalf("status rewritten: %+v", d.Status)
+	}
+}
+
+func TestOtherNodesSteerRepliesToTheHolderAndNeverWrite(t *testing.T) {
+	s := svc("gitea", "86.107.168.15")
+	for _, self := range []string{"cp", "w2"} {
+		d := Compute(self, nodes, []*coreapi.Service{s}, nil)
+		if len(d.Forwards) != 0 || len(d.Status) != 0 {
+			t.Fatalf("%s: only the holder forwards and writes: %+v", self, d)
+		}
+		if len(d.Holders) != 1 || d.Holders[0].Node.Name != "w1" || d.Holders[0].Mark < 1 || d.Holders[0].Mark > MaxMarks {
+			t.Fatalf("%s: holders: %+v", self, d.Holders)
+		}
+	}
+}
+
+func TestUnheldOrContestedAddressFailsClosed(t *testing.T) {
+	// nobody holds it: no forward anywhere; the stale status is cleared by exactly one node (first by name)
+	s := svc("gitea", "86.107.168.99", shows("86.107.168.99"))
+	for _, self := range []string{"cp", "w1", "w2"} {
+		d := Compute(self, nodes, []*coreapi.Service{s}, nil)
+		if len(d.Forwards) != 0 || len(d.Holders) != 0 || len(d.Problems) != 1 || !strings.Contains(d.Problems[0], "no node holds") {
+			t.Fatalf("%s: %+v", self, d)
+		}
+		clears := len(d.Status) == 1 && d.Status[0].Address == nil
+		if clears != (self == "cp") {
+			t.Fatalf("%s: clears=%v", self, clears)
+		}
+	}
+	// two nodes claim it: neither serves it, its holder's view clears the status
+	both := w2
+	both.Addresses = append([]net.IP{net.ParseIP("86.107.168.15").To4()}, w2.Addresses...)
+	d := Compute("w1", []Node{cp, w1, both}, []*coreapi.Service{svc("gitea", "86.107.168.15", shows("86.107.168.15"))}, nil)
+	if len(d.Forwards) != 0 || len(d.Problems) != 1 || !strings.Contains(d.Problems[0], "several nodes") {
+		t.Fatalf("contested: %+v", d)
+	}
+	// a bad request is reported, not guessed at
+	d = Compute("w1", nodes, []*coreapi.Service{svc("gitea", "")}, nil)
+	if len(d.Forwards) != 0 || len(d.Problems) != 1 || !strings.Contains(d.Problems[0], "spec.loadBalancerIP") {
+		t.Fatalf("no address: %+v", d)
+	}
+}
+
+func TestOnlyOptedInLoadBalancersAreServed(t *testing.T) {
+	other := svc("a", "86.107.168.15", func(s *coreapi.Service) { s.Spec.LoadBalancerClass = ptr("example.com/other") })
+	noClass := svc("b", "86.107.168.15", func(s *coreapi.Service) { s.Spec.LoadBalancerClass = nil })
+	clusterIP := svc("c", "86.107.168.15", func(s *coreapi.Service) {
+		s.Spec.Type = coreapi.ServiceTypeClusterIP
+	})
+	d := Compute("w1", nodes, []*coreapi.Service{other, noClass, clusterIP}, nil)
+	if len(d.Forwards)+len(d.Status)+len(d.Holders)+len(d.Problems) != 0 {
+		t.Fatalf("touched a Service that did not opt in: %+v", d)
+	}
+}
+
+func TestParseNodeAndAnnotationsRoundTrip(t *testing.T) {
+	l := Local{Addresses: []net.IP{net.ParseIP("86.107.168.88").To4(), net.ParseIP("86.107.168.15").To4()},
+		WeaveIP: net.ParseIP("10.44.0.0").To4(), WeaveMAC: mac("aa:aa:aa:aa:aa:03")}
+	a := l.Annotations()
+	if a[AnnAddresses] != "86.107.168.15,86.107.168.88" || a[AnnWeave] != "10.44.0.0,aa:aa:aa:aa:aa:03" {
+		t.Fatalf("annotations: %v", a)
+	}
+	n, err := ParseNode(&coreapi.Node{ObjectMeta: metav1.ObjectMeta{Name: "w2", Annotations: a}})
+	if err != nil || !n.Holds(net.ParseIP("86.107.168.15")) || !n.WeaveIP.Equal(l.WeaveIP) || n.WeaveMAC.String() != "aa:aa:aa:aa:aa:03" {
+		t.Fatalf("parsed %+v, %v", n, err)
+	}
+	for _, bad := range []map[string]string{{AnnAddresses: "1.2.3.4,nope"}, {AnnWeave: "10.44.0.0"}, {AnnWeave: "x,aa:aa:aa:aa:aa:03"}} {
+		if _, err := ParseNode(&coreapi.Node{ObjectMeta: metav1.ObjectMeta{Name: "w2", Annotations: bad}}); err == nil {
+			t.Fatalf("accepted %v", bad)
+		}
+	}
+	if n, err := ParseNode(&coreapi.Node{ObjectMeta: metav1.ObjectMeta{Name: "new"}}); err != nil || len(n.Addresses) != 0 || n.WeaveIP != nil {
+		t.Fatalf("a node that published nothing holds nothing: %+v %v", n, err)
+	}
+}
+
+func TestMarksAreDistinctAndStable(t *testing.T) {
+	var ns []Node
+	for _, name := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {
+		ns = append(ns, Node{Name: name})
+	}
+	first := assignMarks(ns)
+	seen := map[uint32]bool{}
+	for _, h := range first {
+		if seen[h.Mark] || h.Mark < 1 || h.Mark > MaxMarks {
+			t.Fatalf("bad or repeated mark: %+v", first)
+		}
+		seen[h.Mark] = true
+	}
+	again := assignMarks(ns)
+	for i := range first {
+		if first[i].Mark != again[i].Mark {
+			t.Fatalf("marks moved between passes")
+		}
+	}
+}
+
+func TestRenderedRules(t *testing.T) {
+	_, podNet, _ := net.ParseCIDR("10.32.0.0/12")
+	s := svc("gitea", "86.107.168.15")
+	sl := slice("gitea", map[string]bool{"10.40.0.7": true, "10.44.0.3": true, "10.44.0.4": true}, map[string]int32{"ssh": 2222, "https": 3000})
+	d := Compute("w1", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
+	r := Render(d, EgressDesired{}, "w1", podNet, []string{"REWEAVE-LBS-DEADBEEF0000"})
+	for _, want := range []string{
+		"*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\n:REWEAVE-LB-CLASS - [0:0]\n",
+		// the client's address survives weave's masquerade; a pod's own connection (hairpin) is masqueraded here
+		"-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -s 10.32.0.0/12 -j MASQUERADE\n",
+		"-A REWEAVE-LB-POST -m conntrack --ctstate DNAT --ctorigdst 86.107.168.15/32 -j ACCEPT\n",
+		":REWEAVE-LBS-DEADBEEF0000 - [0:0]\n", // stale chain emptied
+		"-A REWEAVE-LB -d 86.107.168.15/32 -p tcp -m tcp --dport 22 -m comment --comment \"gitea/gitea\" -j REWEAVE-LBS-",
+		"-p tcp -m statistic --mode random --probability 0.3333333333 -j DNAT --to-destination 10.40.0.7:2222\n",
+		"-p tcp -m statistic --mode random --probability 0.5000000000 -j DNAT --to-destination 10.44.0.3:2222\n",
+		"-p tcp -j DNAT --to-destination 10.44.0.4:2222\n",
+		"COMMIT\n",
+	} {
+		if !strings.Contains(r.Nat, want) {
+			t.Fatalf("nat lacks %q:\n%s", want, r.Nat)
+		}
+	}
+	for _, line := range strings.Split(r.Nat, "\n") {
+		// iptables refuses a DNAT to ip:port without the protocol in the same rule
+		if strings.Contains(line, "-j DNAT") && !strings.Contains(line, " -p tcp ") {
+			t.Fatalf("DNAT without its protocol: %q", line)
+		}
+	}
+	if strings.Count(r.Nat, "MASQUERADE") != 1 || strings.Contains(r.Nat, "SNAT") {
+		t.Fatalf("only a pod's hairpin connection may be masqueraded:\n%s", r.Nat)
+	}
+	if r.Mangle != "*mangle\n:REWEAVE-LB-MARK - [0:0]\n:REWEAVE-LB-MSS - [0:0]\nCOMMIT\n" {
+		t.Fatalf("a holder marks nothing:\n%s", r.Mangle)
+	}
+	d = Compute("w2", nodes, []*coreapi.Service{s}, []*discovery.EndpointSlice{sl})
+	r = Render(d, EgressDesired{}, "w2", podNet, nil)
+	m := d.Holders[0].Mark << 24
+	for _, want := range []string{
+		"-A REWEAVE-LB-MARK -m conntrack --ctdir REPLY -m connmark ! --mark 0x0/0xff000000 -j CONNMARK --restore-mark --nfmask 0xff000000 --ctmask 0xff000000\n",
+		"-A REWEAVE-LB-MARK -i weave -m mac --mac-source aa:aa:aa:aa:aa:02 ! -s 10.32.0.0/12 -d 10.32.0.0/12 -m conntrack --ctstate NEW -m comment --comment \"holder w1\" -j CONNMARK --set-xmark " + fmt.Sprintf("0x%x", m) + "/0xff000000\n",
+	} {
+		if !strings.Contains(r.Mangle, want) {
+			t.Fatalf("mangle lacks %q:\n%s", want, r.Mangle)
+		}
+	}
+	if r.Nat != "*nat\n:REWEAVE-LB - [0:0]\n:REWEAVE-LB-POST - [0:0]\n:REWEAVE-LB-CLASS - [0:0]\nCOMMIT\n" {
+		t.Fatalf("a non-holder forwards nothing:\n%s", r.Nat)
+	}
+	if r.Filter != "*filter\n:REWEAVE-LB-FWD - [0:0]\n:REWEAVE-LB-IN - [0:0]\nCOMMIT\n" {
+		t.Fatalf("no EgressService, no filter rules:\n%s", r.Filter)
+	}
+}

net/portlock.go

@@ -0,0 +1,220 @@
+package net
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netlink/nl"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+// A pod's port is locked to the pod's own addresses: what the pod sends in through the host side of its veth has to
+// carry the pod's MAC as its Ethernet source and be either IPv4 from the pod's address or ARP speaking for exactly
+// that MAC and address (or probing, from 0.0.0.0). Everything else is dropped before the bridge sees it, so a pod can
+// claim neither another pod's identity nor a node's, on its own node or, through weave, on any other.
+//
+// The lock is a set of tc filters on the veth's ingress qdisc. They match the way iproute2 lays out "ether src"
+// (the Ethernet source at -8 and -4 from the network header) and "ip src" (at 12), and they go away with the veth.
+
+const (
+	portLockPrioIPv4  = 1
+	portLockPrioARP   = 2
+	portLockPrioProbe = 3
+	portLockPrioDrop  = 9
+)
+
+var ingressParent = netlink.MakeHandle(0xffff, 0)
+
+// PortLockFilters are the filters that lock the port with the given index to a pod's MAC and IPv4 address.
+func PortLockFilters(index int, mac net.HardwareAddr, ip net.IP) ([]netlink.Filter, error) {
+	if len(mac) != 6 {
+		return nil, fmt.Errorf("port lock needs an Ethernet MAC, got %q", mac)
+	}
+	ip4 := ip.To4()
+	if ip4 == nil {
+		return nil, fmt.Errorf("port lock needs an IPv4 address, got %q", ip)
+	}
+	src := etherSource(mac)
+	arp := func(spa net.IP) []netlink.TcU32Key {
+		keys := append([]netlink.TcU32Key{}, src...)
+		return append(keys,
+			// Ethernet/IPv4 ARP: hardware type 1, protocol 0x0800, address lengths 6 and 4 (any operation)
+			netlink.TcU32Key{Mask: 0xffffffff, Val: 0x00010800, Off: 0},
+			netlink.TcU32Key{Mask: 0xffff0000, Val: 0x06040000, Off: 4},
+			// the sender hardware address (8..13) and the sender protocol address (14..17)
+			netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: 8},
+			netlink.TcU32Key{Mask: 0xffffffff, Val: uint32(binary.BigEndian.Uint16(mac[4:6]))<<16 | uint32(binary.BigEndian.Uint16(spa[0:2])), Off: 12},
+			netlink.TcU32Key{Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(spa[2:4])) << 16, Off: 16},
+		)
+	}
+	ipv4 := append(append([]netlink.TcU32Key{}, src...), netlink.TcU32Key{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(ip4), Off: 12})
+	return []netlink.Filter{
+		portLockU32(index, portLockPrioIPv4, unix.ETH_P_IP, ipv4),
+		portLockU32(index, portLockPrioARP, unix.ETH_P_ARP, arp(ip4)),
+		portLockU32(index, portLockPrioProbe, unix.ETH_P_ARP, arp(net.IPv4zero.To4())),
+		&netlink.MatchAll{
+			FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: portLockPrioDrop, Protocol: unix.ETH_P_ALL},
+			Actions:     []netlink.Action{gact(netlink.TC_ACT_SHOT)},
+		},
+	}, nil
+}
+
+// etherSource matches the Ethernet source address, which sits 8 bytes before the network header.
+func etherSource(mac net.HardwareAddr) []netlink.TcU32Key {
+	return []netlink.TcU32Key{
+		{Mask: 0xffffffff, Val: binary.BigEndian.Uint32(mac[0:4]), Off: -8},
+		{Mask: 0xffff0000, Val: uint32(binary.BigEndian.Uint16(mac[4:6])) << 16, Off: -4},
+	}
+}
+
+func portLockU32(index int, prio uint16, proto uint16, keys []netlink.TcU32Key) *netlink.U32 {
+	// clipped: the vendored netlink serialises cap(Keys) keys, and spare capacity would go out as extra all-zero keys
+	keys = keys[:len(keys):len(keys)]
+	return &netlink.U32{
+		FilterAttrs: netlink.FilterAttrs{LinkIndex: index, Parent: ingressParent, Priority: prio, Protocol: proto},
+		Sel:         &netlink.TcU32Sel{Flags: nl.TC_U32_TERMINAL, Keys: keys},
+		Actions:     []netlink.Action{gact(netlink.TC_ACT_OK)},
+	}
+}
+
+func gact(action netlink.TcAct) netlink.Action {
+	return &netlink.GenericAction{ActionAttrs: netlink.ActionAttrs{Action: action}}
+}
+
+func ingressQdisc(index int) *netlink.Ingress {
+	return &netlink.Ingress{QdiscAttrs: netlink.QdiscAttrs{LinkIndex: index, Handle: ingressParent, Parent: netlink.HANDLE_INGRESS}}
+}
+
+// LockPort locks a pod's port (the host side of its veth) to the pod's MAC and IPv4 address. A port already locked
+// to them is left alone; anything else on its ingress is replaced, and the pod's own traffic is let through before
+// the rest is dropped, so a running pod never loses its own frames.
+func LockPort(link netlink.Link, mac net.HardwareAddr, ip net.IP) error {
+	index := link.Attrs().Index
+	want, err := PortLockFilters(index, mac, ip)
+	if err != nil {
+		return err
+	}
+	if locked, _ := portHasFilters(link, want); locked {
+		return nil
+	}
+	// removing the ingress qdisc takes its filters with it
+	if err := netlink.QdiscDel(ingressQdisc(index)); err != nil && !errors.Is(err, unix.ENOENT) && !errors.Is(err, unix.EINVAL) {
+		return fmt.Errorf("clearing ingress of %s: %w", link.Attrs().Name, err)
+	}
+	if err := netlink.QdiscAdd(ingressQdisc(index)); err != nil {
+		return fmt.Errorf("adding ingress qdisc to %s: %w", link.Attrs().Name, err)
+	}
+	for _, f := range want {
+		if err := netlink.FilterAdd(f); err != nil {
+			return fmt.Errorf("adding port lock filter (prio %d) to %s: %w", f.Attrs().Priority, link.Attrs().Name, err)
+		}
+	}
+	return nil
+}
+
+// LockContainerPort locks the port of a pod attached by the CNI plugin: the host side of the veth whose container
+// end is ifName in ns, to that end's MAC and the pod's address.
+func LockContainerPort(ns netns.NsHandle, ifName string, ip net.IP) error {
+	var mac net.HardwareAddr
+	peer := 0
+	if err := WithNetNSLink(ns, ifName, func(link netlink.Link) error {
+		// a veth's link attribute is its peer's index, in the peer's namespace: the host's
+		mac, peer = link.Attrs().HardwareAddr, link.Attrs().ParentIndex
+		return nil
+	}); err != nil {
+		return err
+	}
+	if peer == 0 {
+		return fmt.Errorf("%s has no peer: not a veth", ifName)
+	}
+	host, err := netlink.LinkByIndex(peer)
+	if err != nil {
+		return fmt.Errorf("host side of %s: %w", ifName, err)
+	}
+	if host.Type() != "veth" || !strings.HasPrefix(host.Attrs().Name, vethPrefix+"pl") {
+		return fmt.Errorf("host side of %s is %s (%s), not a weave pod veth", ifName, host.Attrs().Name, host.Type())
+	}
+	return LockPort(host, mac, ip)
+}
+
+// PortLocked reports whether a port is locked to exactly this MAC and address.
+func PortLocked(link netlink.Link, mac net.HardwareAddr, ip net.IP) (bool, error) {
+	want, err := PortLockFilters(link.Attrs().Index, mac, ip)
+	if err != nil {
+		return false, err
+	}
+	return portHasFilters(link, want)
+}
+
+func portHasFilters(link netlink.Link, want []netlink.Filter) (bool, error) {
+	have, err := netlink.FilterList(link, ingressParent)
+	if err != nil {
+		return false, err
+	}
+	var got []netlink.Filter
+	for _, f := range have {
+		// u32 lists the hash table that holds a filter as a filter of its own, without a selector
+		if u, ok := f.(*netlink.U32); ok && u.Sel == nil {
+			continue
+		}
+		got = append(got, f)
+	}
+	if len(got) != len(want) {
+		return false, nil
+	}
+	for _, w := range want {
+		found := false
+		for _, g := range got {
+			if sameFilter(g, w) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func sameFilter(a, b netlink.Filter) bool {
+	if a.Attrs().Priority != b.Attrs().Priority || a.Attrs().Protocol != b.Attrs().Protocol || a.Type() != b.Type() {
+		return false
+	}
+	switch fa := a.(type) {
+	case *netlink.U32:
+		fb := b.(*netlink.U32)
+		if fa.Sel == nil || fb.Sel == nil || len(fa.Sel.Keys) != len(fb.Sel.Keys) || fa.Sel.Flags&nl.TC_U32_TERMINAL == 0 {
+			return false
+		}
+		for i := range fa.Sel.Keys {
+			ka, kb := fa.Sel.Keys[i], fb.Sel.Keys[i]
+			if ka.Mask != kb.Mask || ka.Val != kb.Val || ka.Off != kb.Off {
+				return false
+			}
+		}
+		return sameActions(fa.Actions, fb.Actions)
+	case *netlink.MatchAll:
+		return sameActions(fa.Actions, b.(*netlink.MatchAll).Actions)
+	}
+	return false
+}
+
+func sameActions(a, b []netlink.Action) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		ga, ok1 := a[i].(*netlink.GenericAction)
+		gb, ok2 := b[i].(*netlink.GenericAction)
+		if !ok1 || !ok2 || ga.Attrs().Action != gb.Attrs().Action {
+			return false
+		}
+	}
+	return true
+}

net/portlock_kernel_test.go

@@ -0,0 +1,228 @@
+package net
+
+import (
+	"bytes"
+	"encoding/binary"
+	"net"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+// TestPortLockInKernel builds a bridge with two pods in throwaway network namespaces, locks pod A's port and sends
+// frames from A: what A may send reaches B, what claims anything but A's own addresses does not. It needs root.
+func TestPortLockInKernel(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("needs root (creates network namespaces)")
+	}
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	orig, err := netns.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer orig.Close()
+	defer netns.Set(orig)
+
+	hub, err := netns.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer hub.Close()
+	podA, _ := netns.New()
+	defer podA.Close()
+	podB, _ := netns.New()
+	defer podB.Close()
+	must(t, netns.Set(hub))
+
+	br := &netlink.Bridge{LinkAttrs: netlink.LinkAttrs{Name: "br0"}}
+	must(t, netlink.LinkAdd(br))
+	must(t, netlink.LinkSetUp(br))
+	ipA, ipB := net.IPv4(10, 99, 0, 1).To4(), net.IPv4(10, 99, 0, 2).To4()
+	hostA, macA := attach(t, hub, podA, br, "vethwepltsta", ipA)
+	_, macB := attach(t, hub, podB, br, "vethwepltstb", ipB)
+
+	// as the CNI plugin does it: from the pod's end of the veth, in the pod's namespace
+	must(t, LockContainerPort(podA, "eth0", ipA))
+	if ok, err := PortLocked(hostA, macA, ipA); err != nil || !ok {
+		dumpIngress(t, hostA)
+		t.Fatalf("port not locked after LockContainerPort: %v %v", ok, err)
+	}
+	// locking again changes nothing (the filters are compared, not rebuilt)
+	must(t, LockPort(hostA, macA, ipA))
+	if ok, _ := PortLocked(hostA, macA, ipA); !ok {
+		t.Fatal("port lost its lock on a repeated LockPort")
+	}
+
+	send := socketIn(t, podA, "eth0")
+	recv := socketIn(t, podB, "eth0")
+	defer unix.Close(send)
+	defer unix.Close(recv)
+	other, _ := net.ParseMAC("02:00:00:00:00:99")
+	spoofIP := net.IPv4(10, 99, 0, 200).To4()
+
+	cases := []struct {
+		name   string
+		frame  []byte
+		passes bool
+	}{
+		{"ipv4 from its own addresses", ipv4Frame(macB, macA, ipA, ipB, "own-ip"), true},
+		{"ipv4 from another MAC", ipv4Frame(macB, other, ipA, ipB, "mac-spoof"), false},
+		{"ipv4 from another address", ipv4Frame(macB, macA, spoofIP, ipB, "ip-spoof"), false},
+		{"arp for itself", arpFrame(macA, macA, ipA, ipB, "own-arp"), true},
+		{"arp probe", arpFrame(macA, macA, net.IPv4zero.To4(), ipB, "probe"), true},
+		{"arp claiming another address", arpFrame(macA, macA, spoofIP, ipB, "arp-ip-spoof"), false},
+		{"arp claiming another MAC", arpFrame(macA, other, ipA, ipB, "arp-sha-spoof"), false},
+		{"arp from another MAC", arpFrame(other, other, ipA, ipB, "arp-eth-spoof"), false},
+		{"ipv6", etherFrame(macB, macA, 0x86dd, []byte("ipv6-frame")), false},
+		{"other ethertype", etherFrame(macB, macA, 0x88b5, []byte("experimental")), false},
+	}
+	for _, c := range cases {
+		drain(recv)
+		if _, err := unix.Write(send, c.frame); err != nil {
+			t.Fatalf("%s: send: %v", c.name, err)
+		}
+		got := sawPayload(recv, c.frame[len(c.frame)-12:], 300*time.Millisecond)
+		if got != c.passes {
+			t.Errorf("%s: reached B = %v, want %v", c.name, got, c.passes)
+		}
+	}
+}
+
+func dumpIngress(t *testing.T, link netlink.Link) {
+	have, _ := netlink.FilterList(link, netlink.MakeHandle(0xffff, 0))
+	for _, f := range have {
+		t.Logf("have %s %+v", f.Type(), f.Attrs())
+		if u, ok := f.(*netlink.U32); ok && u.Sel != nil {
+			t.Logf("  sel flags %x keys %+v actions %+v", u.Sel.Flags, u.Sel.Keys, u.Actions)
+		}
+		if m, ok := f.(*netlink.MatchAll); ok {
+			t.Logf("  actions %+v", m.Actions)
+		}
+	}
+}
+
+func must(t *testing.T, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// attach plugs a pod namespace into the bridge through a veth named host on the hub side, eth0 on the pod side.
+func attach(t *testing.T, hub, pod netns.NsHandle, br netlink.Link, host string, ip net.IP) (netlink.Link, net.HardwareAddr) {
+	t.Helper()
+	must(t, netlink.LinkAdd(&netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: host}, PeerName: "tmp" + host[len(host)-2:]}))
+	h, err := netlink.LinkByName(host)
+	must(t, err)
+	must(t, netlink.LinkSetMaster(h, br))
+	must(t, netlink.LinkSetUp(h))
+	p, err := netlink.LinkByName("tmp" + host[len(host)-2:])
+	must(t, err)
+	must(t, netlink.LinkSetNsFd(p, int(pod)))
+	must(t, netns.Set(pod))
+	p, err = netlink.LinkByName("tmp" + host[len(host)-2:])
+	must(t, err)
+	must(t, netlink.LinkSetName(p, "eth0"))
+	must(t, netlink.AddrAdd(p, &netlink.Addr{IPNet: &net.IPNet{IP: ip, Mask: net.CIDRMask(24, 32)}}))
+	must(t, netlink.LinkSetUp(p))
+	mac := p.Attrs().HardwareAddr
+	must(t, netns.Set(hub))
+	return h, mac
+}
+
+// socketIn opens a raw packet socket on an interface of a namespace (the socket stays in that namespace).
+func socketIn(t *testing.T, ns netns.NsHandle, ifName string) int {
+	t.Helper()
+	cur, _ := netns.Get()
+	defer cur.Close()
+	must(t, netns.Set(ns))
+	defer netns.Set(cur)
+	link, err := netlink.LinkByName(ifName)
+	must(t, err)
+	fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(htons(unix.ETH_P_ALL)))
+	must(t, err)
+	must(t, unix.Bind(fd, &unix.SockaddrLinklayer{Protocol: htons(unix.ETH_P_ALL), Ifindex: link.Attrs().Index}))
+	return fd
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }
+
+func drain(fd int) {
+	buf := make([]byte, 2048)
+	for {
+		if _, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT); err != nil {
+			return
+		}
+	}
+}
+
+func sawPayload(fd int, marker []byte, within time.Duration) bool {
+	buf := make([]byte, 2048)
+	deadline := time.Now().Add(within)
+	for time.Now().Before(deadline) {
+		n, _, err := unix.Recvfrom(fd, buf, unix.MSG_DONTWAIT)
+		if err != nil {
+			time.Sleep(5 * time.Millisecond)
+			continue
+		}
+		if bytes.Contains(buf[:n], marker) {
+			return true
+		}
+	}
+	return false
+}
+
+func etherFrame(dst, src net.HardwareAddr, ethertype uint16, payload []byte) []byte {
+	f := make([]byte, 14, 14+len(payload)+12)
+	copy(f[0:6], dst)
+	copy(f[6:12], src)
+	binary.BigEndian.PutUint16(f[12:14], ethertype)
+	f = append(f, payload...)
+	// a fixed-size tail the receiver looks for, so each case is told apart
+	tail := make([]byte, 12)
+	copy(tail, payload)
+	return append(f, tail...)
+}
+
+func ipv4Frame(dst, src net.HardwareAddr, from, to net.IP, tag string) []byte {
+	body := []byte(tag)
+	h := make([]byte, 20)
+	h[0] = 0x45
+	binary.BigEndian.PutUint16(h[2:4], uint16(20+len(body)+12))
+	h[8] = 64
+	h[9] = 253 // experimental protocol: nothing answers it
+	copy(h[12:16], from)
+	copy(h[16:20], to)
+	binary.BigEndian.PutUint16(h[10:12], checksum(h))
+	return etherFrame(dst, src, unix.ETH_P_IP, append(h, body...))
+}
+
+func arpFrame(ethSrc, sha net.HardwareAddr, spa, tpa net.IP, tag string) []byte {
+	a := make([]byte, 28)
+	binary.BigEndian.PutUint16(a[0:2], 1)
+	binary.BigEndian.PutUint16(a[2:4], unix.ETH_P_IP)
+	a[4], a[5] = 6, 4
+	binary.BigEndian.PutUint16(a[6:8], 1) // request
+	copy(a[8:14], sha)
+	copy(a[14:18], spa)
+	copy(a[24:28], tpa)
+	bcast, _ := net.ParseMAC("ff:ff:ff:ff:ff:ff")
+	return etherFrame(bcast, ethSrc, unix.ETH_P_ARP, append(a, []byte(tag)...))
+}
+
+func checksum(b []byte) uint16 {
+	var s uint32
+	for i := 0; i+1 < len(b); i += 2 {
+		s += uint32(binary.BigEndian.Uint16(b[i:]))
+	}
+	for s>>16 != 0 {
+		s = s&0xffff + s>>16
+	}
+	return ^uint16(s)
+}

net/portlock_test.go

@@ -0,0 +1,115 @@
+package net
+
+import (
+	"net"
+	"testing"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netlink/nl"
+	"golang.org/x/sys/unix"
+)
+
+// The keys iproute2 writes for "u32 match ether src 02:11:22:33:44:55 match ip src 10.32.0.7/32" (tc filter show,
+// iproute2 6.x): the port lock has to match the same bytes.
+func TestPortLockMatchesIproute2Layout(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	fs, err := PortLockFilters(7, mac, net.ParseIP("10.32.0.7"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fs) != 4 {
+		t.Fatalf("want 4 filters, got %d", len(fs))
+	}
+	ipv4 := fs[0].(*netlink.U32)
+	want := []netlink.TcU32Key{
+		{Val: 0x02112233, Mask: 0xffffffff, Off: -8},
+		{Val: 0x44550000, Mask: 0xffff0000, Off: -4},
+		{Val: 0x0a200007, Mask: 0xffffffff, Off: 12},
+	}
+	if ipv4.Protocol != unix.ETH_P_IP || ipv4.Priority != portLockPrioIPv4 || ipv4.Sel.Flags&nl.TC_U32_TERMINAL == 0 {
+		t.Fatalf("ipv4 filter attrs: %+v flags %x", ipv4.FilterAttrs, ipv4.Sel.Flags)
+	}
+	assertKeys(t, "ipv4", ipv4.Sel.Keys, want)
+	for _, f := range fs {
+		if f.Attrs().LinkIndex != 7 || f.Attrs().Parent != netlink.MakeHandle(0xffff, 0) {
+			t.Fatalf("filter not on ingress of link 7: %+v", f.Attrs())
+		}
+	}
+}
+
+func TestPortLockARP(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	fs, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.7"))
+	common := []netlink.TcU32Key{
+		{Val: 0x02112233, Mask: 0xffffffff, Off: -8},
+		{Val: 0x44550000, Mask: 0xffff0000, Off: -4},
+		{Val: 0x00010800, Mask: 0xffffffff, Off: 0},
+		{Val: 0x06040000, Mask: 0xffff0000, Off: 4},
+		{Val: 0x02112233, Mask: 0xffffffff, Off: 8},
+	}
+	own := append(append([]netlink.TcU32Key{}, common...),
+		netlink.TcU32Key{Val: 0x44550a20, Mask: 0xffffffff, Off: 12}, // sha 44:55, spa 10.32
+		netlink.TcU32Key{Val: 0x00070000, Mask: 0xffff0000, Off: 16}, // spa .0.7
+	)
+	probe := append(append([]netlink.TcU32Key{}, common...),
+		netlink.TcU32Key{Val: 0x44550000, Mask: 0xffffffff, Off: 12},
+		netlink.TcU32Key{Val: 0x00000000, Mask: 0xffff0000, Off: 16},
+	)
+	arp, pr := fs[1].(*netlink.U32), fs[2].(*netlink.U32)
+	if arp.Protocol != unix.ETH_P_ARP || pr.Protocol != unix.ETH_P_ARP {
+		t.Fatal("ARP filters must be protocol arp")
+	}
+	assertKeys(t, "arp", arp.Sel.Keys, own)
+	assertKeys(t, "probe", pr.Sel.Keys, probe)
+	drop := fs[3].(*netlink.MatchAll)
+	if drop.Protocol != unix.ETH_P_ALL || drop.Priority != portLockPrioDrop {
+		t.Fatalf("drop filter attrs: %+v", drop.FilterAttrs)
+	}
+	if a := drop.Actions[0].(*netlink.GenericAction); a.Attrs().Action != netlink.TC_ACT_SHOT {
+		t.Fatalf("last filter must drop, has action %v", a.Attrs().Action)
+	}
+	for _, f := range fs[:3] {
+		if a := f.(*netlink.U32).Actions[0].(*netlink.GenericAction); a.Attrs().Action != netlink.TC_ACT_OK {
+			t.Fatalf("prio %d must pass, has action %v", f.Attrs().Priority, a.Attrs().Action)
+		}
+	}
+}
+
+func TestPortLockRefusesWhatItCannotPin(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	if _, err := PortLockFilters(7, mac, net.ParseIP("fd00::1")); err == nil {
+		t.Fatal("an IPv6 address must be refused: the lock pins IPv4 and ARP")
+	}
+	long, _ := net.ParseMAC("02:11:22:33:44:55:66:77")
+	if _, err := PortLockFilters(7, long, net.ParseIP("10.32.0.7")); err == nil {
+		t.Fatal("a non-Ethernet hardware address must be refused")
+	}
+}
+
+func TestSameFilterComparesKeysAndActions(t *testing.T) {
+	mac, _ := net.ParseMAC("02:11:22:33:44:55")
+	a, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.7"))
+	b, _ := PortLockFilters(7, mac, net.ParseIP("10.32.0.8"))
+	if !sameFilter(a[0], a[0]) || sameFilter(a[0], b[0]) {
+		t.Fatal("ipv4 filters for different addresses must differ")
+	}
+	if !sameFilter(a[3], b[3]) {
+		t.Fatal("the drop filters are the same for every pod")
+	}
+	if sameFilter(a[1], a[2]) {
+		t.Fatal("the ARP and probe filters differ in priority and keys")
+	}
+}
+
+func assertKeys(t *testing.T, what string, got, want []netlink.TcU32Key) {
+	t.Helper()
+	if len(got) != len(want) {
+		t.Fatalf("%s: want %d keys, got %d: %+v", what, len(want), len(got), got)
+	}
+	for i := range want {
+		if got[i].Val != want[i].Val || got[i].Mask != want[i].Mask || got[i].Off != want[i].Off {
+			t.Fatalf("%s key %d: want %08x/%08x at %d, got %08x/%08x at %d", what, i,
+				want[i].Val, want[i].Mask, want[i].Off, got[i].Val, got[i].Mask, got[i].Off)
+		}
+	}
+}