Add release v1.9.1

Lowercase the request before comparing it

Fix #2817

.gitignore

@@ -43,12 +43,16 @@ prog/weavewait/weavewait
 prog/weavewait/weavewait_noop
 prog/weavewait/weavewait_nomcast
 prog/weaveutil/weaveutil
+prog/*/Dockerfile.weaveworks
+prog/kube-peers/kube-peers
+prog/weave-kube/kube-peers
+prog/weave-npc/weave-npc
 prog/plugin/plugin
 testing/cover/cover
 testing/runner/runner
 releases
 .*.uptodate
-weave.tar.gz
+weave*.tar.gz
 prog/weaveexec/weave
 prog/weaveexec/weavehosts
 prog/weaveexec/weaveproxy
@@ -57,9 +61,12 @@ prog/weaveexec/weavewait_noop
 prog/weaveexec/weavewait_nomcast
 prog/weaveexec/sigproxy
 prog/weaveexec/weaveutil
-prog/weaveexec/docker*.tgz
+prog/weaveexec/docker*
 Vagrantfile.local
 test/tls/tls
 test/tls/*.pem
 test/coverage
 test/coverage.*
+*qemu-*-static
+terraform.tfstate
+terraform.tfstate.backup

.gitmodules

@@ -79,6 +79,48 @@
 [submodule "vendor/golang.org/x/sys"]
 	path = vendor/golang.org/x/sys
 	url = https://go.googlesource.com/sys
-[submodule "vendor/github.com/weaveworks/docker"]
-	path = vendor/github.com/weaveworks/docker
-	url = https://github.com/weaveworks/docker
+[submodule "vendor/github.com/beorn7/perks"]
+	path = vendor/github.com/beorn7/perks
+	url = https://github.com/beorn7/perks
+[submodule "vendor/github.com/prometheus/procfs"]
+	path = vendor/github.com/prometheus/procfs
+	url = https://github.com/prometheus/procfs
+[submodule "vendor/github.com/prometheus/client_golang"]
+	path = vendor/github.com/prometheus/client_golang
+	url = https://github.com/prometheus/client_golang
+[submodule "vendor/github.com/prometheus/client_model"]
+	path = vendor/github.com/prometheus/client_model
+	url = https://github.com/prometheus/client_model
+[submodule "vendor/github.com/prometheus/common"]
+	path = vendor/github.com/prometheus/common
+	url = https://github.com/prometheus/common
+[submodule "vendor/github.com/golang/protobuf"]
+	path = vendor/github.com/golang/protobuf
+	url = https://github.com/golang/protobuf
+[submodule "vendor/github.com/matttproud/golang_protobuf_extensions"]
+	path = vendor/github.com/matttproud/golang_protobuf_extensions
+	url = https://github.com/matttproud/golang_protobuf_extensions
+[submodule "vendor/k8s.io/client-go"]
+	path = vendor/k8s.io/client-go
+	url = https://github.com/kubernetes/client-go
+[submodule "vendor/github.com/pkg/errors"]
+	path = vendor/github.com/pkg/errors
+	url = https://github.com/pkg/errors
+[submodule "vendor/github.com/spf13/cobra"]
+	path = vendor/github.com/spf13/cobra
+	url = https://github.com/spf13/cobra
+[submodule "vendor/github.com/spf13/plag"]
+	path = vendor/github.com/spf13/plag
+	url = https://github.com/spf13/pflag
+[submodule "vendor/github.com/spf13/pflag"]
+	path = vendor/github.com/spf13/pflag
+	url = https://github.com/spf13/pflag
+[submodule "vendor/github.com/mgutz/ansi"]
+	path = vendor/github.com/mgutz/ansi
+	url = https://github.com/mgutz/ansi
+[submodule "vendor/github.com/estesp/manifest-tool"]
+	path = vendor/github.com/estesp/manifest-tool
+	url = https://github.com/estesp/manifest-tool
+[submodule "vendor/github.com/weaveworks/common"]
+	path = vendor/github.com/weaveworks/common
+	url = https://github.com/weaveworks/common

.lintignore

@@ -0,0 +1,27 @@
+# IMPORTANT: Please only use GLOBs to ignore files and directories.
+
+# Do not lint the below bash scripts in Weave Net, in order to 
+# avoid breaking the build on make lint:
+bin/circle-deploy-issues
+bin/circle-deploy-master
+bin/circle-teardown-pre
+bin/circle-test-teardown
+bin/circle-test-unit
+bin/install-wordepress
+bin/multiweave
+bin/publish-site
+bin/release
+build/build.sh
+prog/weave-kube/launch.sh
+prog/weaveexec/symlink
+test/*_test.sh
+test/600_proxy_docker_py.sh
+test/assert.sh
+test/config.sh
+test/gce.sh
+test/gen_coverage_reports.sh
+test/run_all.sh
+test/sanity_check.sh
+test/setup.sh
+vendor/*.sh
+weave

CHANGELOG.md

@@ -1,3 +1,98 @@
+## Release 1.9.1
+
+# Bug fixes and minor improvements
+* Fix a race condition when the Weave Net container is restarted
+  which could allow a new container to be allocated the same IP
+  address as an existing one #2784,#2787
+* Handle the message type received when a pod has been deleted during
+  Kubernetes api-server fail-over #2772,#2773
+* Make weave-kube work with `dockerd --iptables=false` #2726
+* Ensure we have the right kernel modules loaded for Network Policy in weave-kube #2819
+* Reference-count addresses in Network Policy Controller, to avoid
+  errors when updates come in an unexpected order #2792,#2795
+* Allow the soft connection limit to be raised in weave-kube, so
+  larger clusters can be created #2781
+* WeaveDNS was incorrectly case-sensitive for reverse DNS lookups #2817,#2818
+
+# Build and Testing
+* Scripts to create VMs to run automated tests were rewritten to use
+  Terraform and Ansible, to make it much easier to test with different
+  versions of components such as Docker and Kubernetes #2647,#2694,#2775,#2796
+* Upgrade to latest Weaveworks common build-tools #2780
+* Improve encryption tests #2793
+* Update vishvananda/netlink library to bring in changes we had previously forked #2790
+* Slight change to the build container to avoid permission errors and slow builds #2761,#2802
+
+## Release 1.9.0
+
+Highlights:
+
+* Encryption is now available for Fast Datapath connections, which
+  greatly improves the performance. #1644,#2687
+* We now build images for Intel/AMD 64-bit, ARM and ARM 64-bit. #2713
+* Weave Net Docker images are now labelled with description, vendor, etc. #2712
+* `weave status connections` now shows the MTU #2389,#2663
+* CNI plugin is now a stand-alone binary that does not depend on Docker #2594,#2662
+* Embedded docker client updated to version 1.10.3 #2395
+
+**NOTE:** The move to multi-architecture required that we update the
+  embedded Docker client, and this has the effect that this release of
+  Weave Net will not work with Docker installations older than
+  1.10.
+
+[Full list of changes](https://github.com/weaveworks/weave/issues?q=milestone%3A1.9.0).
+
+## Release 1.8.2
+
+Bug fixes and minor improvements
+
+* Fixed a bug where looping flows were installed which caused high CPU
+  usage #2650, #2674
+* Fixed a bug where Kubernetes master could not contact pods #2673, #2683
+* Fixed a bug where weave-kube was crashing in a loop due to invalid
+  Weave bridge state #2657
+* Fixed a bug where iptables NAT rules were not appended due to
+  "temporary unavailable" iptables error #2679
+* Added a detection of enabling the hairpin mode on the Weave bridge port
+  which caused installation of looping flows #2674
+* Added a detection of overlaps between Weave and the host IP address
+  ranges when launching weave-kube #2669, #2672
+* Added logging of connections blocked by weave-npc #2546, #2573
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.8.2).
+
+## Release 1.8.1
+
+Bug fixes and minor improvements
+
+* Fixed weave-npc crash from Succeeded/Failed pods #2632,#2658
+* Fixed occasional failure to create Weave bridge on node reboot
+  #2617,#2637
+* Fixed a bug where weave-kube would fail to install when run with
+  unrelease snapshot builds #2642
+* Improved conformance to CNI spec by not releasing IP addresses when
+  a container dies #2643
+* Improved troubleshooting of install failure by creating CNI config
+  after Weave Net is up #2570
+* "up to date" shown even when the version check was blocked by
+  firewall #2537,#2565,#2645
+* "Unable to claim" message on re-launching Weave after using CNI
+  #2548,#2577
+* Eliminated spurious IP reclaim operations when IPAM was disabled
+  #2567,#2644
+* Include `jq` tool in our build VM configuration #2656
+
+## Release 1.8.0
+
+Highlights:
+
+* Exposed network policy controller Prometheus metrics
+  weaveworks/weave-npc#23, #2595, #2549
+* Exposed router Prometheus metrics #2535, #2547, #2523, #2579, #2578,
+  #2568, #2560, #2561
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.8.0).
+
 ## Release 1.7.2
 
 Bug fixes and minor improvements

DEPENDENCIES

@@ -0,0 +1,3 @@
+DOCKER_VERSION=1.11.2
+KUBERNETES_VERSION=1.5.2
+KUBERNETES_CNI_VERSION=0.3.0.1

Makefile

@@ -1,7 +1,5 @@
-PUBLISH=publish_weave publish_weaveexec publish_plugin
-
 .DEFAULT: all
-.PHONY: all exes testrunner update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests
+.PHONY: all exes testrunner update tests lint publish-one-arch $(PUBLISH) clean clean-bin prerequisites build run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
@@ -10,43 +8,130 @@ RM=--rm
 RUN_FLAGS=-ti
 COVERAGE=
 
-DOCKERHUB_USER=weaveworks
-WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
+# This specifies the architecture we're building for
+ARCH?=amd64
 
+# We're using QEMU to be able to build Docker images for other architectures
+# from amd64 machines.
+QEMU_VERSION=v2.7.0
+
+# A list of all supported architectures here. Should be named as Go is naming platforms
+# All supported architectures must have an "ifeq" block below that customizes the parameters
+ALL_ARCHITECTURES=amd64 arm arm64
+ML_PLATFORMS=linux/amd64,linux/arm,linux/arm64
+
+ifeq ($(ARCH),amd64)
+# The architecture to use when downloading the docker binary
+	WEAVEEXEC_DOCKER_ARCH?=x86_64
+
+# The name of the alpine baseimage to use as the base for weave images
+	ALPINE_BASEIMAGE?=alpine:3.4
+
+# The extension for the made images
+# Specifying none means for example weaveworks/weave:latest
+	ARCH_EXT?=
+
+# The name of the gcc we're using for compiling C code
+	CC=gcc
+
+# Optional parameters that can be passed when linking C code into the Go binary
+	CGO_LDFLAGS=
+endif
+ifeq ($(ARCH),arm)
+# The architecture to use when downloading the docker binary
+	WEAVEEXEC_DOCKER_ARCH?=armel
+
+# Using the (semi-)official alpine image
+	ALPINE_BASEIMAGE?=armhf/alpine:3.4
+
+# arm images have the -arm suffix, for instance weaveworks/weave-arm:latest
+	ARCH_EXT?=-arm
+
+# The name of the gcc binary
+	CC=arm-linux-gnueabihf-gcc
+
+# The architecture name to use when downloading a prebuilt QEMU binary
+	QEMUARCH=arm
+
+# In the weaveworks/build image; libpcap libraries for arm are placed here
+# Tell the gcc linker to search for libpcap here
+	CGO_LDFLAGS="-L/usr/local/lib/$(CC)"
+endif
+ifeq ($(ARCH),arm64)
+# Use the arm 32-bit docker client variant
+	WEAVEEXEC_DOCKER_ARCH?=armel
+
+# Using the (semi-)official alpine image
+	ALPINE_BASEIMAGE?=aarch64/alpine:3.5
+
+# arm64 images have the -arm64 suffix, for instance weaveworks/weave-arm64:latest
+	ARCH_EXT?=-arm64
+
+# The name of the gcc binary
+	CC=aarch64-linux-gnu-gcc
+
+# The architecture name to use when downloading a prebuilt QEMU binary
+	QEMUARCH=aarch64
+
+# In the weaveworks/build image; libpcap libraries for arm64 are placed here
+# Tell the gcc linker to search for libpcap here
+	CGO_LDFLAGS="-L/usr/local/lib/$(CC)"
+endif
+
+# The name of the user that this Makefile should produce image artifacts for. Can/should be overridden
+DOCKERHUB_USER?=weaveworks
+# The default version that's chosen when pushing the images. Can/should be overridden
+WEAVE_VERSION?=git-$(shell git rev-parse --short=12 HEAD)
+
+# Paths to all relevant binaries that should be compiled
 WEAVER_EXE=prog/weaver/weaver
 WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
 SIGPROXY_EXE=prog/sigproxy/sigproxy
+KUBEPEERS_EXE=prog/kube-peers/kube-peers
+WEAVENPC_EXE=prog/weave-npc/weave-npc
 WEAVEWAIT_EXE=prog/weavewait/weavewait
 WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop
 WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast
 WEAVEUTIL_EXE=prog/weaveutil/weaveutil
 PLUGIN_EXE=prog/plugin/plugin
 RUNNER_EXE=tools/runner/runner
+MANIFEST_TOOL_DIR=vendor/github.com/estesp/manifest-tool
+MANIFEST_TOOL_EXE=$(MANIFEST_TOOL_DIR)/manifest-tool
 TEST_TLS_EXE=test/tls/tls
 
-EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(TEST_TLS_EXE)
+# All binaries together in a list
+EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(RUNNER_EXE) $(TEST_TLS_EXE) $(MANIFEST_TOOL_EXE)
 
+# These stamp files are used to mark the current state of the build; whether an image has been built or not
 BUILD_UPTODATE=.build.uptodate
-WEAVER_UPTODATE=.weaver.uptodate
-WEAVEEXEC_UPTODATE=.weaveexec.uptodate
-PLUGIN_UPTODATE=.plugin.uptodate
+WEAVER_UPTODATE=.weaver$(ARCH_EXT).uptodate
+WEAVEEXEC_UPTODATE=.weaveexec$(ARCH_EXT).uptodate
+PLUGIN_UPTODATE=.plugin$(ARCH_EXT).uptodate
+WEAVEKUBE_UPTODATE=.weavekube$(ARCH_EXT).uptodate
+WEAVENPC_UPTODATE=.weavenpc$(ARCH_EXT).uptodate
 WEAVEDB_UPTODATE=.weavedb.uptodate
 
-IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEDB_UPTODATE)
+IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE)
 
-WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
-WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
-PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin
-BUILD_IMAGE=$(DOCKERHUB_USER)/weavebuild
+# The names of the images. Note that the images for other architectures than amd64 have a suffix in the image name.
+WEAVER_IMAGE=$(DOCKERHUB_USER)/weave$(ARCH_EXT)
+WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec$(ARCH_EXT)
+PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin$(ARCH_EXT)
+WEAVEKUBE_IMAGE=$(DOCKERHUB_USER)/weave-kube$(ARCH_EXT)
+WEAVENPC_IMAGE=$(DOCKERHUB_USER)/weave-npc$(ARCH_EXT)
+BUILD_IMAGE=weaveworks/weavebuild
 WEAVEDB_IMAGE=$(DOCKERHUB_USER)/weavedb
 
-IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(PLUGIN_IMAGE) $(WEAVEDB_IMAGE)
+IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(PLUGIN_IMAGE) $(WEAVEKUBE_IMAGE) $(WEAVENPC_IMAGE) $(WEAVEDB_IMAGE)
 
-WEAVE_EXPORT=weave.tar.gz
+PUBLISH=publish_weave publish_weaveexec publish_plugin publish_weave-kube publish_weave-npc
+PUSH_ML=push_ml_weave push_ml_weaveexec push_ml_plugin push_ml_weave-kube push_ml_weave-npc
 
-WEAVEEXEC_DOCKER_VERSION=1.6.2
-DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
-DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
+WEAVE_EXPORT=weave$(ARCH_EXT).tar.gz
+
+DOCKER_VERSION=1.10.3
+DOCKER_DISTRIB=prog/weaveexec/docker-$(DOCKER_VERSION).tgz
+DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/$(WEAVEEXEC_DOCKER_ARCH)/docker-$(DOCKER_VERSION).tgz
 NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	rm $@; \
 	echo "\nYour go standard library was built without the 'netgo' build tag."; \
@@ -55,7 +140,10 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	echo "    sudo go install -tags netgo std"; \
 	false; \
 }
-BUILD_FLAGS=-i -ldflags "-extldflags \"-static\" -X main.version=$(WEAVE_VERSION)" -tags netgo
+# The flags we are passing to go build. -extldflags -static for making a static binary, 
+# -linkmode external for linking external C libraries into the binary, -X main.version for telling the
+# Go binary which version it is, -tags netgo for enforcing the native Go DNS resolver
+BUILD_FLAGS=-i -ldflags "-linkmode external -extldflags -static -X main.version=$(WEAVE_VERSION)" -tags netgo
 
 PACKAGE_BASE=$(shell go list -e ./)
 
@@ -65,10 +153,14 @@ testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE)
 $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go
 $(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go
 $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go
-$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go
+$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go db/*.go
 $(SIGPROXY_EXE): prog/sigproxy/*.go
+$(KUBEPEERS_EXE): prog/kube-peers/*.go
+$(WEAVENPC_EXE): prog/weave-npc/*.go npc/*.go npc/*/*.go
 $(PLUGIN_EXE): prog/plugin/*.go plugin/*/*.go api/*.go common/*.go common/docker/*.go net/*.go
 $(TEST_TLS_EXE): test/tls/*.go
+$(RUNNER_EXE): tools/runner/*.go
+$(MANIFEST_TOOL_EXE): $(MANIFEST_TOOL_DIR)/*.go
 $(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
 $(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
 $(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go
@@ -77,14 +169,16 @@ lint: tools/.git
 
 ifeq ($(BUILD_IN_CONTAINER),true)
 
+# This make target compiles all binaries inside of the weaveworks/build container
+# It bind-mounts the source into the container and passes all important variables
 exes $(EXES) tests lint: $(BUILD_UPTODATE)
 	git submodule update --init
 	@mkdir -p $(shell pwd)/.pkg
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
 	    -v $(shell pwd):/go/src/github.com/weaveworks/weave \
 		-v $(shell pwd)/.pkg:/go/pkg \
-		-e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \
-		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) $@
+		-e GOARCH=$(ARCH) -e CGO_ENABLED=1 -e GOOS=linux -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW -e DEBUG \
+		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) CC=$(CC) QEMUARCH=$(QEMUARCH) CGO_LDFLAGS=$(CGO_LDFLAGS) $@
 
 else
 
@@ -99,7 +193,7 @@ else
 endif
 	$(NETGO_CHECK)
 
-$(WEAVEUTIL_EXE):
+$(WEAVEUTIL_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE):
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
@@ -113,26 +207,48 @@ $(WEAVEWAIT_NOMCAST_EXE):
 
 # These programs need a separate rule as they fail the netgo check in
 # the main build stanza due to not importing net package
-$(SIGPROXY_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE):
+$(SIGPROXY_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE) $(RUNNER_EXE) $(MANIFEST_TOOL_EXE):
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 
 tests:
-	./tools/test -no-go-get
+	./tools/test -no-go-get -netgo -timeout 8m
 
 lint:
-	./tools/lint -nocomment -notestpackage .
+	./tools/lint -nocomment -notestpackage
 
 endif
 
+# This rule makes sure the build image is up-to-date.
+# It also makes sure the multiarch hooks are reqistered in the kernel so the QEMU emulation works
 $(BUILD_UPTODATE): build/*
 	$(SUDO) docker build -t $(BUILD_IMAGE) build/
+	$(SUDO) docker run --rm --privileged multiarch/qemu-user-static:register --reset
 	touch $@
 
-$(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVER_IMAGE) prog/weaver
+# Creates the Dockerfile.your-user-here file from the template
+# Also replaces all placeholders with real values
+# If the architecture is amd64, it deletes all CROSS_BUILD lines
+# but otherwise, it only removes the "CROSS_BUILD_" placeholder and downloads QEMU
+%/Dockerfile.$(DOCKERHUB_USER): %/Dockerfile.template
+	echo "DOCKERHUB_USER|$(DOCKERHUB_USER)|g;s|ARCH_EXT|$(ARCH_EXT)|g;s|ALPINE_BASEIMAGE|$(ALPINE_BASEIMAGE)|g;s|QEMUARCH|$(QEMUARCH)"
+	sed -e "s|DOCKERHUB_USER|$(DOCKERHUB_USER)|g;s|ARCH_EXT|$(ARCH_EXT)|g;s|ALPINE_BASEIMAGE|$(ALPINE_BASEIMAGE)|g;s|QEMUARCH|$(QEMUARCH)|g" $^ > $@
+ifeq ($(ARCH),amd64)
+# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	sed -i "/CROSS_BUILD_/d" $@
+else
+# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
+# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
+	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(shell dirname $@)
+	cd $(shell dirname $@) && sha256sum -c $(shell pwd)/build/shasums/qemu-$(QEMUARCH)-static.sha256sum
+	sed -i "s/CROSS_BUILD_//g" $@
+endif
+
+# The targets below builds the weave images
+$(WEAVER_UPTODATE): prog/weaver/Dockerfile.$(DOCKERHUB_USER) $(WEAVER_EXE) $(WEAVEEXEC_UPTODATE)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaver/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVER_IMAGE) prog/weaver
 	touch $@
 
-$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE)
+$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE)
 	cp weave prog/weaveexec/weave
 	cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy
 	cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy
@@ -140,49 +256,101 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER
 	cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop
 	cp $(WEAVEWAIT_NOMCAST_EXE) prog/weaveexec/weavewait_nomcast
 	cp $(WEAVEUTIL_EXE) prog/weaveexec/weaveutil
-	cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
+	tar -xf $(DOCKER_DISTRIB) usr/local/bin/docker -O > prog/weaveexec/docker
+	chmod +x prog/weaveexec/docker
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEEXEC_IMAGE) prog/weaveexec
 	touch $@
 
-$(PLUGIN_UPTODATE): prog/plugin/Dockerfile $(PLUGIN_EXE) $(WEAVEEXEC_UPTODATE)
-	$(SUDO) docker build -t $(PLUGIN_IMAGE) prog/plugin
+$(PLUGIN_UPTODATE): prog/plugin/Dockerfile.$(DOCKERHUB_USER) $(PLUGIN_EXE) $(WEAVER_UPTODATE)
+	$(SUDO) docker build -f prog/plugin/Dockerfile.$(DOCKERHUB_USER) -t $(PLUGIN_IMAGE) prog/plugin
+	touch $@
+
+$(WEAVEKUBE_UPTODATE): prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) prog/weave-kube/launch.sh $(KUBEPEERS_EXE) $(WEAVER_UPTODATE)
+	cp $(KUBEPEERS_EXE) prog/weave-kube/
+	$(SUDO) docker build -f prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEKUBE_IMAGE) prog/weave-kube
+	touch $@
+
+$(WEAVENPC_UPTODATE): prog/weave-npc/Dockerfile.$(DOCKERHUB_USER) $(WEAVENPC_EXE) prog/weave-npc/ulogd.conf
+	$(SUDO) docker build -f prog/weave-npc/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVENPC_IMAGE) prog/weave-npc
 	touch $@
 
 $(WEAVEDB_UPTODATE): prog/weavedb/Dockerfile
 	$(SUDO) docker build -t $(WEAVEDB_IMAGE) prog/weavedb
 	touch $@
 
-$(WEAVE_EXPORT): $(IMAGES_UPTODATE)
+$(WEAVE_EXPORT): $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE)
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@
 
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
+	cd $(shell dirname $@) && sha256sum -c $(shell pwd)/build/shasums/docker-tgz-$(WEAVEEXEC_DOCKER_ARCH).sha256sum
 
-tools/.git:
+tools/.git $(MANIFEST_TOOL_DIR)/.git:
 	git submodule update --init
 
-$(RUNNER_EXE): tools/.git
-	make -C tools/runner
+# CODE FOR PUBLISHING THE IMAGES
 
+# This target first runs "make publish" for each architecture
+# Then it pushes the manifest lists
+publish: $(addprefix sub-publish-,$(ALL_ARCHITECTURES)) $(WEAVEDB_UPTODATE)
+	$(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) $(PUSH_ML)
+ifeq ($(PUBLISH_WEAVEDB),true)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/weavedb:latest
+endif
+
+sub-publish-%:
+	$(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) clean-bin
+	$(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) $(IMAGES_UPTODATE)
+	$(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) publish-one-arch
+
+publish-one-arch: $(PUBLISH)
+
+# This rule handles the pushing of the built images
 $(PUBLISH): publish_%: $(IMAGES_UPTODATE)
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+# Tag :latest with the real version
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$*$(ARCH_EXT) $(DOCKERHUB_USER)/$*$(ARCH_EXT):$(WEAVE_VERSION)
+# Push the image with the arch suffix for arm and arm64, and without suffix for amd64
+	$(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) push_$*$(ARCH_EXT)
+
+ifeq ($(ARCH),amd64)
+# If the architecture is amd64, add the -amd64 suffix.
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) $(DOCKERHUB_USER)/$*-amd64:$(WEAVE_VERSION)
+# If the version is latest, tag the -amd64 with latest as well
+ifneq ($(UPDATE_LATEST),false)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$*-amd64:$(WEAVE_VERSION) $(DOCKERHUB_USER)/$*-amd64:latest
+endif
+# Push the image with the -amd64-suffix so BINARY-ARCH-named images exist for all arches, so manifest lists may be made and ARCH is replaceable in scripts
+	$(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) push_$*-amd64
+endif
+
+# This target pushes an image, and if UPDATE_LATEST is anything but false, also updates the latest tag
+# It takes one parameter, the image name
+push_%:
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
 ifneq ($(UPDATE_LATEST),false)
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:latest
 endif
 
-publish: $(PUBLISH)
-ifeq ($(PUBLISH_WEAVEDB),true)
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push   $(DOCKERHUB_USER)/weavedb:latest
+# This target pushes a manifest list; it takes one parameter, the image name.
+# The variable UPDATE_LATEST controls whether it updates the versioned tag and/or the latest tag
+$(PUSH_ML): push_ml_%: $(MANIFEST_TOOL_EXE)
+ifneq ($(UPDATE_LATEST),latest-only)
+	$(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:$(WEAVE_VERSION) --target $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+endif
+ifneq ($(UPDATE_LATEST),false)
+# Push the manifest list to :latest as well
+	$(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:latest --target $(DOCKERHUB_USER)/$*:latest
 endif
 
 clean-bin:
 	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES)
-	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg
+	find prog -type f -name "Dockerfile.*" -not -name "Dockerfile.template" -print | xargs rm -f
+	find prog -type f -name "*qemu-*" -print | xargs rm -f
+	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE) weave*.tar.gz $(DOCKER_DISTRIB) prog/weaveexec/docker .pkg
 
 clean: clean-bin
 	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(BUILD_IMAGE)
-	rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE)
+	rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE) $(MANIFEST_TOOL_EXE)
 
 build:
 	$(SUDO) go clean -i net
@@ -191,3 +359,20 @@ build:
 
 run-smoketests: all testrunner
 	cd test && ./setup.sh && ./run_all.sh
+
+integration-tests: all testrunner
+# Usage:
+#   $ make \
+#     NAME="<prefix used to name VMs and other resources>" \
+#     PROVIDER="<provider among {vagrant|gcp|aws|do}>" \
+#     NUM_HOSTS="<# test machines>" \
+#     PLAYBOOK="<filename>" \
+#     RUNNER_ARGS="<...>" \
+#     TESTS="<...>" \  # Can be set to only run one or a few tests instead of the full test suite.
+#     DOCKER_VERSION=<...> \
+#     KUBERNETES_VERSION=<...> \
+#     KUBERNETES_CNI_VERSION=<...> \
+#     <...> # See also run-integration-test.sh for all variables and individual functions.
+#     integration-tests
+#
+	RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh

README.md

@@ -11,9 +11,11 @@
 [Weaveworks](https://www.weave.works) is the company that delivers the most productive way for developers to connect, observe and control
 Docker containers.
 
-This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, and with nearly 5 million downloads to date, it enables you to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
+This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, and with over 8 million downloads to date, it enables you to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
 
-Other products from Weaveworks include [Weave Scope](https://www.weave.works/products/weave-scope/), a powerful container monitoring tool that automatically maps Docker containers and their interactions, and [Weave Flux](https://www.weave.works/products/weave-flux/), a microservice router that automates the access of containers as services.
+Weaveworks' product [Weave Cloud](https://www.weave.works/solution/cloud/) takes the heavy lifting out of building, integrating and operating open source components, so you don’t have to manage databases, storage and availability.
+
+Weave Cloud is built using these Open Source projects: [Weave Scope](https://www.weave.works/products/weave-scope/), a powerful container monitoring tool that automatically maps Docker containers and their interactions, [Weave Cortex](https://github.com/weaveworks/cortex), a horizontally-scalable version of Prometheus, and [Weave Flux](https://www.weave.works/products/weave-flux/), a continuous deployment tool that works with Kubernetes.
 
 # Weave Net
 
@@ -21,5 +23,20 @@ Weave Net creates a virtual network that connects Docker containers across multi
 
 Services provided by application containers on the Weave network can be exposed to the outside world, regardless of where they are running. Similarly, existing internal systems can be opened to accept connections from application containers irrespective of their location.
 
-* [Install Weave Net](https://www.weave.works/docs/net/latest/installing-weave/)
-* [Find out more about Weave Net](https://www.weave.works/products/weave-net/)
+## <a name="help"></a>Getting help
+
+If you have any questions about, feedback for or a problem with Weave Net:
+
+- Read [the Weave Net docs](https://www.weave.works/docs/net/latest/introducing-weave/).
+- Invite yourself to the <a href="https://weaveworks.github.io/community-slack/" target="_blank"> #weave-community </a> slack channel.
+- Ask a question on the <a href="https://weave-community.slack.com/messages/general/"> #weave-community</a> slack channel.
+- Join the <a href="https://www.meetup.com/pro/Weave/"> Weave User Group </a> and get invited to online talks, hands-on training and meetups in your area.
+- Send an email to <a href="mailto:weave-users@weave.works">weave-users@weave.works</a>
+- <a href="https://github.com/weaveworks/weave/issues/new">File an issue.</a>
+
+Your feedback is always welcome!
+
+##Further Information
+
+* [Installing Weave Net](https://www.weave.works/docs/net/latest/installing-weave/)
+* [Weave Net Product Overview](https://www.weave.works/products/weave-net/)

Vagrantfile

@@ -1,40 +1,36 @@
-VAGRANTFILE_API_VERSION = "2"
-
-require './vagrant-common.rb'
+require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb'))
 
 vm_ip = "172.16.0.3" # arbitrary private IP
 
-pkgs = %w(
-  docker-engine=1.10.2-0~wily
-  aufs-tools
-  build-essential
-  ethtool
-  iputils-arping
-  libpcap-dev
-  git
-  mercurial
-  bc
-)
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.box = "ubuntu/wily64"
+  config.vm.box = VAGRANT_IMAGE
 
   config.vm.network "private_network", ip: vm_ip
   config.vm.provider :virtualbox do |vb|
-    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
-    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
+    vb.memory = 2048
+    configure_nat_dns(vb)
   end
 
+  # Disable default Vagrant shared folder, which we don't need:
   config.vm.synced_folder ".", "/vagrant", disabled: true
+  # Keep Weave Net sources' in sync:
   config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave"
+  # Create a convenience symlink to $HOME/src/github.com/weaveworks/weave
+  config.vm.provision :shell, :inline => 'ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/'
 
-  install_build_deps config.vm, pkgs
-  install_go_toochain config.vm
-  tweak_user_env config.vm
-  tweak_docker_daemon config.vm
+  # Set SSH keys up to be able to run smoke tests straightaway:
+  config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/home/vagrant/src/github.com/weaveworks/weave/test/insecure_private_key"
+  # Grant permissions on sources:
+  config.vm.provision :shell, :inline => 'sudo chown -R vagrant:vagrant ~vagrant/src', :privileged => false
   cleanup config.vm
 
+  config.vm.provision 'ansible' do |ansible|
+    ansible.playbook = 'tools/config_management/setup_weave-net_dev.yml'
+    ansible.extra_vars = {
+      go_version: GO_VERSION
+    }.merge(ansibleize(get_dependencies_version_from_file_and_env()))
+  end
 end
 
 begin

api/ipam.go

@@ -5,6 +5,15 @@ import (
 	"net"
 )
 
+// Special token used in place of a container identifier when:
+// - the caller does not know the container ID, or
+// - the caller is unsure whether IPAM was previously told the container ID.
+//
+// While addresses are typically stored under their container IDs, when this
+// token is used, the address will be stored under its own string, something
+// which should be kept in mind if the entry needs to be removed at any point.
+const NoContainerID string = "_"
+
 func (client *Client) ipamOp(ID string, op string) (*net.IPNet, error) {
 	ip, err := client.httpVerb(op, fmt.Sprintf("/ip/%s", ID), nil)
 	if err != nil {

bin/circle-dependencies-post

@@ -2,42 +2,16 @@
 
 set -e
 
-# We need jq to analyse the PR files JSON, so install that first
-sudo apt-get -q update && sudo apt-get -q install bc jq
-
-echo "TEST_AND_PUBLISH=1" > "$STATE"
-
-if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ] ; then
-    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$') ; then
-        echo "GH_PR=$PR" >> "$STATE"
-        if FILES=$(curl -s https://api.github.com/repos/weaveworks/weave/pulls/$PR/files | jq -r '.[]|.filename') ; then
-            echo "PR touches [$FILES]"
-            if ! echo "$FILES" | grep -v ^site/ ; then
-                echo "PR contains site changes only - skipping tests"
-                echo "TEST_AND_PUBLISH=" >> "$STATE"
-            else
-                echo "PR contains non-site changes - running tests"
-            fi
-        else
-            echo "Unable to get files for PR $PR" >&2
-        fi
-    else
-        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
-    fi
-fi
-
 source "$STATE"
 
-mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    cd $SRCDIR; git submodule update --init
-    if [ "$CIRCLE_NODE_INDEX" = "0" ] ; then
-        cd $SRCDIR/test; ./gce.sh make_template
-    fi
-    cd $SRCDIR/build
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+    cd "$SRCDIR/build"
     ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh
-    touch $SRCDIR/.build.uptodate
+    touch "$SRCDIR/.build.uptodate"
     sudo chown -R ubuntu /usr/local/go
+    cd "$SRCDIR"
+    make testrunner
+    make COVERAGE=true RM= exes all
 fi

bin/circle-dependencies-post-sources

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+echo "TEST_AND_PUBLISH=1" >"$STATE"
+
+# Avoid doing a full build if we can.  Note if the PR doesn't exist
+# when the build is kicked off, we can't run this check so we do a
+# full build.  Subsequent pushes to the same branch will have the desired behaviour
+if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then
+    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then
+        echo "GH_PR=$PR" >>"$STATE"
+        if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then
+            echo "PR touches [$FILES]"
+            if ! echo "$FILES" | grep -v ^site/; then
+                echo "PR contains site changes only - skipping tests"
+                echo "TEST_AND_PUBLISH=" >>"$STATE"
+            else
+                echo "PR contains non-site changes - running tests"
+            fi
+        else
+            echo "Unable to get files for PR $PR" >&2
+        fi
+    else
+        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
+    fi
+fi
+
+source "$STATE"
+
+mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR"
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+fi

bin/circle-deploy-master

@@ -8,7 +8,8 @@ if [ -n "$TEST_AND_PUBLISH" ] ; then
     docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
     cd $SRCDIR
     make clean-bin
-    make publish
+    # CircleCI doesn't let us install the qemu module to do ARM docker builds
+    make publish ALL_ARCHITECTURES=amd64 UPDATE_LATEST=latest-only
 fi
 
 cd $SRCDIR

bin/circle-test-pre

@@ -2,13 +2,29 @@
 
 set -e
 
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_READY_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR
-    make testrunner
-    make COVERAGE=true RM= all
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
 
-    cd $SRCDIR/test
-    ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+
+    # Upload built binaries to testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
+    echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi

bin/circle-test-smoke

@@ -3,10 +3,21 @@
 set -e
 
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR/test
-    eval $(./gce.sh hosts)
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
+
+    # Run integration tests:
     export COVERAGE=true
-    ./run_all.sh
+    export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
+    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
+    ./run-integration-tests.sh test
 fi

bin/circle-test-teardown

@@ -6,6 +6,5 @@ source "$STATE"
 
 if [ -n "$TEST_AND_PUBLISH" ] ; then
     cd $SRCDIR/test
-    ./gce.sh destroy
     test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh)
 fi

bin/circle-test-unit

@@ -6,6 +6,6 @@ source "$STATE"
 
 if [ -n "$TEST_AND_PUBLISH" ] ; then
     cd $SRCDIR
-    make lint
+    make DEBUG=1 lint
     COVERDIR=test/coverage make RM= tests
 fi

bin/provision_test_vms.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
+source "$STATE"
+
+function install_terraform() {
+    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
+}
+
+function install_ansible() {
+    sudo apt-get update || true
+    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible
+    export PATH="$PATH:$HOME/.local/bin"
+}
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+
+    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
+    [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0
+
+    # Provision and configure testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
+fi

bin/release

@@ -89,6 +89,7 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
+    sed -i -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset.yaml
     make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
 
     if make tests; then
@@ -157,6 +158,13 @@ draft() {
         --name "weave" \
         --file "./weave"
 
+    github-release upload \
+        --user $GITHUB_USER \
+        --repo weave \
+        --tag $LATEST_TAG \
+        --name "weave-daemonset.yaml" \
+        --file "./prog/weave-kube/weave-daemonset.yaml"
+
     echo "** Draft $TYPE $RELEASE_NAME $VERSION created at"
     echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG"
 }
@@ -236,6 +244,13 @@ publish() {
             --name "weave" \
             --file "./weave"
 
+        github-release upload \
+            --user $GITHUB_USER \
+            --repo weave \
+            --tag latest_release \
+            --name "weave-daemonset.yaml" \
+            --file "./prog/weave-kube/weave-daemonset.yaml"
+
         echo "** Release $RELEASE_NAME $VERSION published at"
         echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG"
         echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/latest_release"

bin/setup-ci-secrets

@@ -1,57 +0,0 @@
-#!/bin/sh
-
-## This is the shape of unencrypted `do-setup-ci-secrets`
-## 
-## cat > /tmp/boto_gsutil_cert.pem << EOF
-## -----BEGIN PRIVATE KEY-----
-## <...>
-## -----END PRIVATE KEY-----
-## EOF
-## 
-## cat > ~/.boto << EOF
-## [Credentials]
-## gs_service_client_id = <...>
-## gs_service_key_file = /tmp/boto_gsutil_cert.pem
-## 
-## [Boto]
-## https_validate_certificates = True
-## 
-## [GSUtil]
-## content_language = en
-## default_api_version = 2
-## default_project_id = <...>
-## EOF
-
-openssl base64 -d << EOF \
-  | openssl aes-256-cbc -d -K $1 -iv $2 \
-    -out bin/do-setup-ci-secrets
-mOYNzNsAdwva0LbB2YFMY9VgL0SITFu1bPrbOujyLtOrBNjmOYbzR5d/sG+ARMFr
-7NZ5OKcvMbItllz0n7TtNkzkCOMztMC870Wcx3nGJzbBephdcYwos4tBlp5bihTU
-AXgxLUiDT1broYAFKnbE1aZC/XyNxuLTN+nXJWIWeijaXTgvhkEEm3tab+z7ioCb
-tlg71W1BrUuocctAJAuVwH/uTZAaXaJGJj1s2S7F2DrFnG6sFlrp8rnCucPLWKnC
-4IwJFC5saR5gxK6wECffwmwbpeXVOoQ58OgzxoZTEyAr9nhVbaBt5UZz3tfHlx8W
-Rn5jHehACw5kg5eMtGPqMEksJ63LX4h3mfSTLJXJHEmOrzCRpFrigVrsbA408U2u
-8jJTHJmKrGfwBh5ZFzBg6NuV4VplOp+ZquJLgegv2WH/iu/ciopXqB2YDCdx5ZBi
-nHV2Puth7kjS4EMCKUGfeqpesjL9Djb8cygzv/J3ZBmLjNH9dqlSXWy2DcFZA1Iy
-pFTTHkVU3w5sNz3nYOuT2UPQvULBW0FUndDKX6olyVa0mlxPwGBP9N3yGg918/rw
-7rNV/r54lA334F5QMhDrVxar6sl9wfLA+Z+8qemBbNFlLog1uQGhuwWK2nFu1tRM
-9uyvQIq0oq5NHKpogWc7NEuk7CPe1JyfRIcbMVj8+ilmKjUPB2HPC43IDmKt9/56
-rY3IXAcxZ/ClGfGpeNHXqop9T/WhfaowponOZFcdGSUddG0OswqkTE4KIv71mgTF
-jXooMkTQBOG+gBnCe+8bCRdRCG3anxHX74za4dGD58cOJtl6DYWnjWuR91NWnxdT
-1Zr3iUe41txg9hCNY9LHo2QHGLPxe6pmNnSClQ1mcFhuY6NF8RmCX3HAGKJQkxJb
-vpLGbXTXjAIFgIYgLNgKP2RLvCif/LmSWgkvE3ZU9GXtCUbm6xgJodwOqXiI9uuO
-4H4+/vpGHnSFENiclkgMFrLG4yiMu8ArgmnCBzg9E2AoOX5zCrh/2ZRTU8Dk4mt7
-jUd5SKQLsjB9QtB+y3tRO7zY9RDvR6iBOPRsjx6t8EJMCfPb06KchfMJw+VGxvMM
-48jtmepREFnhd/8XDRGnw5bznjRuwohGik8BToltjIviMkkpifKWaLwlaOiDXaHh
-VBFNXnUKvOBwRMU+PZqv4+DP2qff1gSXnGKqA5GIURYrK9P7TqHHNTnmjBdfJLzc
-0x+W1DTiYhLT2vrLrkEpZ99o6qrg+lwvjat6IsJF8xoQEo4pLD/ZRPJ0WzoDyctT
-oWZ5rMsN1TzTvJrBrTAWIJyhIzZcuNwChMrU6qogMr0fd6TSxWhWn0sOJ5y8AaTR
-ej4JHx0qroS/9JPmgFYU61pX3c0ypdef/sSHWG23mcFzLTgwe2d4ty9VakypZy3T
-sXCl6EHVVXlUFWhY4zQCjdZpYJv/rtoKL1BL4fPE2NV5WTjCL2GNvaO5SfHR7pA0
-yQJR/iU+cS2n8UXsWNjyV+/qEn6+QXuyVHGLbxyH8EeqlliAbyHsc2rzVI8Xctzo
-W5DuIqxwHAkNNurhmx27ktIIQ6HhaCp028SWwvLttkbBBgF4v1mmd4u/2bKbKjUd
-dOaetaIUx1hU9OAeaePvVkeTDcc3mlFtPARCS9Fqj16hgJQ43onZyC0H1K2WAxmU
-GsmzjCHlTNquzFTuuQihYGZKfsWIh9z6HEOkvNJ3VEQ=
-EOF
-
-exec sh bin/do-setup-ci-secrets

bin/setup-circleci-secrets

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin
-# openssl base64 < setup-circleci-secrets.orig
-
-openssl base64 -d << EOF \
-  | openssl enc \
-    -out bin/do-setup-circleci-secrets \
-		-d -aes256 -pass pass:$1
-U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+
-c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx
-QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z
-/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R
-Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs
-DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH
-41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ
-LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+
-fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG
-S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi
-wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM
-/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP
-QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi
-vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j
-BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc
-eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F
-jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs
-9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl
-A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R
-MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T
-OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5
-kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ
-lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j
-0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy
-4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv
-cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X
-BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE
-xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl
-WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc
-Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p
-IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g
-K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv
-qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+
-j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt
-LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw
-68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv
-nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+
-42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG
-3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG
-fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s
-w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz
-yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3
-8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu
-02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd
-WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1
-q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc
-BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu
-TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T
-9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N
-tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH
-7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW
-lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF
-gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3
-g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d
-AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW
-EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK
-ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo
-vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo
-nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB
-sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0
-aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL
-/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T
-FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR
-pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ
-9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824
-HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ
-vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I
-Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74
-2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0
-hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi
-Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv
-xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg
-wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V
-NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8
-JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP
-jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t
-a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t
-acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J
-yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg
-e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx
-dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa
-bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc
-C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx
-cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA
-1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK
-8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM
-/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC
-jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6
-Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW
-vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE
-V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK
-QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h
-PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv
-b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr
-9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM
-c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy
-90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf
-aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl
-6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO
-HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb
-EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg=
-EOF
-
-exec sh bin/do-setup-circleci-secrets

bin/wait_for.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# shellcheck disable=SC2155
+wait_for() {
+    local timeout=$1
+    local lock_file=$2
+    for i in $(seq 1 "$timeout"); do
+        if [ -f "$lock_file" ]; then
+            local status="$(cat "$lock_file")"
+            if [ "$status" == "OK" ]; then
+                echo "[$i seconds]: $lock_file found and status: $status." && return
+            else
+                echo "[$i seconds]: $lock_file found and status: $status." && return 1
+            fi
+        fi
+        if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
+        sleep 1
+    done
+    echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
+    return 1
+}

build/Dockerfile

@@ -1,8 +1,84 @@
-FROM golang:1.5.2
-ENV GO15VENDOREXPERIMENT 1
-RUN apt-get update && apt-get install -y libpcap-dev python-requests time file
-RUN go get github.com/fzipp/gocyclo github.com/client9/misspell/cmd/misspell
-RUN go clean -i net && go install -tags netgo std
-RUN go install -race -tags netgo std
+FROM golang:1.7.4
+
+# Support Raspberry Pi 2 and newer
+ENV GOARM 7
+
+# The names of the architectures that we're fetching cross-compilers for from the Ubuntu packages
+ENV DEB_CROSSPLATFORMS \
+	armhf \
+	arm64
+
+# The go platforms we are supporting to cross-compile
+ENV GO_CROSSPLATFORMS \
+	linux/arm \
+	linux/arm64
+
+# The names of the gcc cross-compilers we have installed
+ENV GCC_CROSSCOMPILERS \
+	arm-linux-gnueabihf \
+	aarch64-linux-gnu
+
+# Temporarily add the Ubuntu repositories, because we're going to install gcc cross-compilers from there
+# Install the build-essential and crossbuild-essential-ARCH packages
+RUN echo "deb http://archive.ubuntu.com/ubuntu xenial main universe" > /etc/apt/sources.list.d/cgocrosscompiling.list \
+  && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 3B4FE6ACC0B21F32 \
+  && apt-get update \
+  && apt-get install -y build-essential \
+  && for platform in ${DEB_CROSSPLATFORMS}; do apt-get install -y crossbuild-essential-${platform}; done \
+  && rm /etc/apt/sources.list.d/cgocrosscompiling.list \
+  && apt-get update \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install some required packages
+# libpcap is required because we're linking against its C libraries from the prog/weaver binary
+# flex and bison are required packages for compiling libpcap manually later
+RUN apt-get update \
+	&& apt-get install -y \
+		libpcap-dev \
+		python-requests \
+		time \
+		file \
+		flex \
+		bison
+
+# Install common Go tools
+RUN go get \
+	github.com/golang/lint/golint \
+	github.com/fzipp/gocyclo \
+	gopkg.in/mvdan/sh.v1/cmd/shfmt \
+	github.com/fatih/hclfmt \
+	github.com/client9/misspell/cmd/misspell
+
+# First clean, because we're gonna rebuild the std both with -race enabled and disabled
+# Running go install std twice is intentional and required, as content is written in two different directories
+# (/usr/local/go/pkg/linux_amd64 vs /usr/local/go/pkg/linux_amd64_race)
+# The other architectures do not have support for -race
+RUN go clean -i net \
+	&& go install -tags netgo std \
+	&& go install -race -tags netgo std
+
+# Prebuild std for the other architectures
+RUN for platform in ${GO_CROSSPLATFORMS}; do \
+		GOOS=${platform%/*} GOARCH=${platform##*/} \
+			go install -tags netgo std; done
+
+# Allow full write access to the Go folders for anyone
+RUN chmod -R a+w /usr/local/go
+
+# The libpcap version from Debian packages is 1.6.2, matching that version here although newer versions of libpcap have been released
+# We have to cross-compile the libpcap library for the new architectures, that's what we're doing here
+ENV LIBPCAP_CROSS_VERSION=1.6.2
+RUN curl -sSL http://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.tar.gz | tar -xz \
+	&& cd libpcap-${LIBPCAP_CROSS_VERSION} \
+	&& for crosscompiler in ${GCC_CROSSCOMPILERS}; do \
+		CC=${crosscompiler}-gcc ac_cv_linux_vers=2 ./configure --host=${crosscompiler} --with-pcap=linux \
+		&& make \
+		&& export LIB_DIR="/usr/local/lib/${crosscompiler}-gcc" VER="$(cat ./VERSION)" MAJOR_VER="$(sed 's/\([0-9][0-9]*\)\..*/\1/' ./VERSION)" \
+		&& mkdir -p ${LIB_DIR} \
+		&& cp -f libpcap.a libpcap.so.${VER} ${LIB_DIR} \
+		&& ln -sf libpcap.so.${VER} /usr/local/lib/libpcap.so.${MAJOR_VER} \
+		&& ln -sf libpcap.so.${MAJOR_VER} /usr/local/lib/libpcap.so \
+		&& make clean; done
+
 COPY build.sh /
 ENTRYPOINT ["sh", "/build.sh"]

build/shasums/docker-tgz-armel.sha256sum

@@ -0,0 +1 @@
+8f313a9df867c207ae9ceac6e0935d1d2a0360aafce780dd5d1e0687ee98e0c3 docker-1.10.3.tgz

build/shasums/docker-tgz-x86_64.sha256sum

@@ -0,0 +1 @@
+7541b5490cee1183d911f909c0e91d3d9d57b2b775e40f14a1a7a6de216ca4a8 docker-1.10.3.tgz

build/shasums/qemu-aarch64-static.sha256sum

@@ -0,0 +1 @@
+c02d861383d035ba020b361ef93bf14beb3964a42a6ae645b505b6148493e7ec qemu-aarch64-static

build/shasums/qemu-arm-static.sha256sum

@@ -0,0 +1 @@
+0ff63682b8868620d771aa812d3053dc3c579dc8633ce62d3d7ae7eee83beb06 qemu-arm-static

circle.yml

@@ -2,9 +2,10 @@ general:
   branches:
     ignore:
       - gh\-pages
-      - /pull\/.*/
 
 machine:
+  pre:
+    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
   services:
     - docker
   environment:
@@ -13,17 +14,31 @@ machine:
     PATH: $PATH:$HOME/.local/bin
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
     STATE: /home/ubuntu/state.env
+    # Base name of VMs for integration tests:
+    NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+    TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt
+    TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
+    TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt
+    # Google Cloud Platorm images for integration tests:
+    USE_IMAGE: 1
+    DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0
 
 dependencies:
   cache_directories:
     - "~/docker"
   post:
+    - bin/circle-dependencies-post-sources
+    # Provision & configure testing VMs:
+    - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh":
+        parallel: true
+        background: true
     - bin/circle-dependencies-post
 
 test:
   pre:
     - "$SRCDIR/bin/circle-test-pre":
         parallel: true
+        background: true
   override:
     - "$SRCDIR/bin/circle-test-unit":
         parallel: true
@@ -36,6 +51,10 @@ test:
 
 teardown:
   pre:
+    # Destroy testing VMs:
+    - "cd $SRCDIR/test && ./run-integration-tests.sh destroy":
+        parallel: true
+        background: true
     - "$SRCDIR/bin/circle-teardown-pre":
         parallel: true
 

common/mflagext/listvar.go

@@ -1,31 +0,0 @@
-package mflagext
-
-import (
-	"fmt"
-
-	"github.com/weaveworks/docker/pkg/mflag"
-)
-
-type listOpts struct {
-	value      *[]string
-	hasBeenSet bool
-}
-
-func ListVar(p *[]string, names []string, value []string, usage string) {
-	*p = value
-	mflag.Var(&listOpts{p, false}, names, usage)
-}
-
-func (opts *listOpts) Set(value string) error {
-	if opts.hasBeenSet {
-		(*opts.value) = append((*opts.value), value)
-	} else {
-		(*opts.value) = []string{value}
-		opts.hasBeenSet = true
-	}
-	return nil
-}
-
-func (opts *listOpts) String() string {
-	return fmt.Sprintf("%v", []string(*opts.value))
-}

common/odp/odp.go

@@ -21,8 +21,13 @@ func CreateDatapath(dpname string) (supported bool, err error) {
 	defer dpif.Close()
 
 	dp, err := dpif.CreateDatapath(dpname)
-	if err != nil && !odp.IsDatapathNameAlreadyExistsError(err) {
-		return true, err
+	if err != nil {
+		if !odp.IsDatapathNameAlreadyExistsError(err) {
+			return true, err
+		}
+		if dp, err = dpif.LookupDatapath(dpname); err != nil {
+			return true, err
+		}
 	}
 
 	// Pick an ephemeral port number to use in probing for vxlan

common/signals.go

@@ -1,32 +0,0 @@
-package common
-
-import (
-	"os"
-	"os/signal"
-	"runtime"
-	"syscall"
-)
-
-// A subsystem/server/... that can be stopped or queried about the status with a signal
-type SignalReceiver interface {
-	Stop() error
-}
-
-func SignalHandlerLoop(ss ...SignalReceiver) {
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
-	buf := make([]byte, 1<<20)
-	for {
-		switch <-sigs {
-		case syscall.SIGINT, syscall.SIGTERM:
-			Log.Infof("=== received SIGINT/SIGTERM ===\n*** exiting")
-			for _, subsystem := range ss {
-				subsystem.Stop()
-			}
-			return
-		case syscall.SIGQUIT:
-			stacklen := runtime.Stack(buf, true)
-			Log.Infof("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end", buf[:stacklen])
-		}
-	}
-}

common/utils.go

@@ -1,16 +1,6 @@
 package common
 
-import (
-	"fmt"
-	"net"
-	"os"
-	"strings"
-
-	"github.com/vishvananda/netlink"
-	"github.com/vishvananda/netns"
-
-	weavenet "github.com/weaveworks/weave/net"
-)
+import "strings"
 
 // Assert test is true, panic otherwise
 func Assert(test bool) {
@@ -26,143 +16,3 @@ func ErrorMessages(errors []error) string {
 	}
 	return strings.Join(result, "\n")
 }
-
-type NetDev struct {
-	Name  string
-	MAC   net.HardwareAddr
-	CIDRs []*net.IPNet
-}
-
-// Search the network namespace of a process for interfaces matching a predicate
-// Note that the predicate is called while the goroutine is inside the process' netns
-func FindNetDevs(processID int, match func(link netlink.Link) bool) ([]NetDev, error) {
-	var netDevs []NetDev
-
-	ns, err := netns.GetFromPid(processID)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	defer ns.Close()
-
-	err = weavenet.WithNetNSUnsafe(ns, func() error {
-		return forEachLink(func(link netlink.Link) error {
-			if match(link) {
-				netDev, err := linkToNetDev(link)
-				if err != nil {
-					return err
-				}
-				netDevs = append(netDevs, netDev)
-			}
-			return nil
-		})
-	})
-
-	return netDevs, err
-}
-
-func forEachLink(f func(netlink.Link) error) error {
-	links, err := netlink.LinkList()
-	if err != nil {
-		return err
-	}
-	for _, link := range links {
-		if err := f(link); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func linkToNetDev(link netlink.Link) (NetDev, error) {
-	addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
-	if err != nil {
-		return NetDev{}, err
-	}
-
-	netDev := NetDev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr}
-	for _, addr := range addrs {
-		netDev.CIDRs = append(netDev.CIDRs, addr.IPNet)
-	}
-	return netDev, nil
-}
-
-// ConnectedToBridgePredicate returns a function which is used to query whether
-// a given link is a veth interface which one end is connected to a bridge.
-// The returned function should be called from a container network namespace which
-// the bridge does NOT belong to.
-func ConnectedToBridgePredicate(bridgeName string) (func(link netlink.Link) bool, error) {
-	indexes := make(map[int]struct{})
-
-	// Scan devices in root namespace to find those attached to weave bridge
-	err := weavenet.WithNetNSLinkByPidUnsafe(1, bridgeName,
-		func(br netlink.Link) error {
-			return forEachLink(func(link netlink.Link) error {
-				if link.Attrs().MasterIndex == br.Attrs().Index {
-					peerIndex := link.Attrs().ParentIndex
-					if peerIndex == 0 {
-						// perhaps running on an older kernel where ParentIndex doesn't work.
-						// as fall-back, assume the indexes are consecutive
-						peerIndex = link.Attrs().Index - 1
-					}
-					indexes[peerIndex] = struct{}{}
-				}
-				return nil
-			})
-		})
-	if err != nil {
-		return nil, err
-	}
-
-	return func(link netlink.Link) bool {
-		_, isveth := link.(*netlink.Veth)
-		_, found := indexes[link.Attrs().Index]
-		return isveth && found
-	}, nil
-}
-
-func GetNetDevsWithPredicate(processID int, predicate func(link netlink.Link) bool) ([]NetDev, error) {
-	// Bail out if this process is running in the root namespace
-	nsToplevel, err := netns.GetFromPid(1)
-	if err != nil {
-		return nil, fmt.Errorf("unable to open root namespace: %s", err)
-	}
-	defer nsToplevel.Close()
-	nsContainr, err := netns.GetFromPid(processID)
-	if err != nil {
-		// Unable to find a namespace for this process - just return nothing
-		if os.IsNotExist(err) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err)
-	}
-	defer nsContainr.Close()
-	if nsToplevel.Equal(nsContainr) {
-		return nil, nil
-	}
-
-	return FindNetDevs(processID, predicate)
-}
-
-// Lookup the weave interface of a container
-func GetWeaveNetDevs(processID int) ([]NetDev, error) {
-	p, err := ConnectedToBridgePredicate("weave")
-	if err != nil {
-		return nil, err
-	}
-
-	return GetNetDevsWithPredicate(processID, p)
-}
-
-// Get the weave bridge interface.
-// NB: Should be called from the root network namespace.
-func GetBridgeNetDev(bridgeName string) (NetDev, error) {
-	link, err := netlink.LinkByName(bridgeName)
-	if err != nil {
-		return NetDev{}, err
-	}
-
-	return linkToNetDev(link)
-}

db/boltdb.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/gob"
 	"fmt"
+	"time"
 
 	"github.com/boltdb/bolt"
 )
@@ -23,14 +24,39 @@ var (
 	topBucket          = []byte("top")
 )
 
+const (
+	NameIdent = "peername"
+	FileName  = "data.db"
+)
+
 func NewBoltDB(dbPathname string) (*BoltDB, error) {
 	db, err := bolt.Open(dbPathname, 0660, nil)
 	if err != nil {
 		return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err)
 	}
-	// check the persistence version
-	err = db.Update(func(tx *bolt.Tx) error {
+	err = db.Update(checkVersion(false))
+	return &BoltDB{db: db}, err
+}
+
+func NewBoltDBReadOnly(dbPathname string) (*BoltDB, error) {
+	options := bolt.Options{Timeout: time.Millisecond * 50, ReadOnly: true}
+	db, err := bolt.Open(dbPathname, 0660, &options)
+	if err != nil {
+		return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err)
+	}
+	err = db.View(checkVersion(true))
+	if err != nil {
+		return nil, fmt.Errorf("[boltDB] Cannot use persistence file %s: %s", dbPathname, err)
+	}
+	return &BoltDB{db: db}, nil
+}
+
+func checkVersion(readOnly bool) func(tx *bolt.Tx) error {
+	return func(tx *bolt.Tx) error {
 		if top := tx.Bucket(topBucket); top == nil {
+			if readOnly {
+				return fmt.Errorf("no top bucket")
+			}
 			top, err := tx.CreateBucket(topBucket)
 			if err != nil {
 				return err
@@ -41,13 +67,12 @@ func NewBoltDB(dbPathname string) (*BoltDB, error) {
 		} else {
 			if checkVersion := top.Get(versionIdent); checkVersion != nil {
 				if checkVersion[0] != persistenceVersion[0] {
-					return fmt.Errorf("[boltDB] Cannot use persistence file %s - version %x", dbPathname, checkVersion)
+					return fmt.Errorf("mismatched version %x", checkVersion)
 				}
 			}
 		}
 		return nil
-	})
-	return &BoltDB{db: db}, err
+	}
 }
 
 func (d *BoltDB) Load(ident string, data interface{}) (bool, error) {

docs/fastdp-crypto.md

@@ -0,0 +1,155 @@
+This document describes implementation details of the fast datapath encryption.
+
+# Overview
+
+At the high level, we use the ESP protocol ([RFC 2406][esp]) in the Transport
+mode. Each packet is encrypted with AES in GCM mode ([RFC 4106][aesgcm]), with
+32 byte key and 4 byte salt. This combo provides the following security
+properties:
+
+* Data confidentiality.
+* Data origin authentication.
+* Integrity.
+* Anti-replay.
+* Limited traffic flow confidentiality as fast datapath VXLAN packets are fully
+  encrypted.
+
+## Notation
+
+* `SAin`:  IPsec security association for inbound connections.
+* `SAout`: IPsec security association for outbound connections.
+* `SPout`: IPsec security policy for outbound connections. Used to match
+           outbound flows to SAout.
+
+## SA Key Derivation
+
+For each connection direction, a different AES-GCM key and salt is used.
+The pairs are derived with HKDF ([RFC 5869][hkdf]):
+
+```
+SAin[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceIn, info=localPeerName)
+SAout[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceOut, info=remotePeerName)
+```
+
+Where:
+
+* the mutual `sessionKey` is derived by the [Mesh][mesh] library during
+  the control plane connection establishment between the local peer and
+  the remote peer.
+* `nonceIn` and `nonceOut` are randomly generated 32byte nonces which
+  are exchanged over the encrypted control plane channel.
+
+## SPI
+
+A directional secure connection between two peers is identified with SPI.
+
+The kernel requires the pair of SPI and dst IP to be unique among security
+associations. Thus, to avoid collisions, we generate outbound SPI on a remote
+peer.
+
+## Connection Establishment
+
+```
+Peer A                                                        Peer B
+-----------------------------------------------------------------------------------------------------------------
+
+fastdp.fwd.Confirm():
+    install iptables blocking rules,                          fastdp.fwd.Confirm():
+    nonce_BA = rand(),                                            install iptables blocking rules,
+    {key,salt}_BA = hkdf(sessionKey, nonce_BA, A),                nonce_AB = rand(),
+    spi_BA = allocspi(),                                          {key,salt}_AB = hkdf(sessionKey, nonce_AB, B),
+    create SA_BA(B<-A, spi_BA, key_BA, salt_BA),                  spi_AB = allocspi(),
+    send InitSARemote(spi_BA, nonce_BA). -->                      create SA_AB(A<-B, spi_AB, key_AB, salt_AB),
+                                                              <-- send InitSARemote(spi_AB, nonce_AB).
+
+
+                                                          --> recv InitSARemote(spi_BA, nonce_BA):
+recv InitSARemote(spi_AB, nonce_AB): <--                          {key,salt}_BA = hkdf(sessionKey, nonce_BA, A),
+    {key,salt}_AB = hkdf(sessionKey, nonce_AB, B),                create SA_BA(B<-A, spi_BA, key_BA, salt_BA),
+    create SA_AB(A<-B, spi_AB, key_AB, salt_AB),                  create SP_BA(B<-A, spi_BA),
+    create SP_AB(A<-B, spi_AB),                                   install marking rule.
+    install marking rule.
+```
+
+# Implementation Details
+
+## XFRM
+
+The implementation is based on the kernel IP packet transformation framework
+called XFRM. Unfortunately, docs are barely existing and the complexity of
+the framework is high. The best resource I found is Chapter 10 in
+"Linux Kernel Networking: Implementation and Theory" by Rami Rosen.
+
+## iptables Rules
+
+The kernel VXLAN driver does not set a dst port of a tunnel in the ip flow
+descriptor, thus XFRM policy lookup cannot match a policy (SPout) which includes
+the port. This makes impossible to encrypt only tunneled traffic between
+peers. To work around, we mark such outgoing packets with iptables and set
+the same mark in the policy selector (funnily enough, `iptables_mangle` module
+eventually sets the missing dst port in the flow descriptor). The challenge
+here is to pick a mark that it would not interfere with other networking
+applications before OUTPUT'ing a packet. For example, Kubernetes by default
+uses 1<<14 and 1<<15 marks and we choose 1<<17 (0x20000). Additionally,
+such workaround brings the requirement for at least the 4.2 kernel.
+
+The marking rules are the following:
+
+```
+iptables -t mangle -A OUTPUT -j WEAVE-IPSEC-OUT
+iptables -t mangle -A WEAVE-IPSEC-OUT -s ${LOCAL_PEER_IP} -d ${REMOTE_PEER_IP} \
+         -p udp --dport ${TUNNEL_PORT} -j WEAVE-IPSEC-OUT-MARK
+iptables -t mangle -A WEAVE-IPSEC-OUT-MARK --set-xmark ${MARK} -j MARK
+```
+
+As Linux does not implement [IP Security Levels][ipseclevel], we install
+additional iptables rules to prevent from accidentally sending unencrypted
+traffic between peers which have previously established the secure connection.
+
+For inbound traffic, we mark each ESP packet with the mark and drop non-marked
+tunnel traffic:
+
+```
+iptables -t mangle -A INPUT -j WEAVE-IPSEC-IN
+iptables -t mangle -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \
+         -m esp --espspi ${SPIin} -j WEAVE-IPSEC-IN-MARK
+iptables -t mangle -A WEAVE-IPSEC-IN-MARK --set-xmark ${MARK} -j MARK
+
+iptables -t filter -A INPUT -j WEAVE-IPSEC-IN
+iptables -t filter -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \
+         -p udp --dport ${TUNNEL_PORT} -m mark ! --mark ${MARK} -j DROP
+```
+
+For outbound traffic, we drop marked traffic which does not match any SPout:
+
+```
+iptables -t filter -A OUTPUT ! -p esp -m policy --dir out --pol none \
+         -m mark --mark ${MARK} -j DROP
+```
+
+## ESN
+
+To prevent from cycling SeqNo which makes replay attacks possible, we use
+64-bit extended sequence numbers known as [ESN](esn).
+
+## MTU
+
+In addition to the VXLAN overhead, the MTU calculation should take into account
+the ESP overhead which is 34-37 bytes (encrypted Payload is 4 bytes aligned) and
+consists of:
+
+* 4 bytes (SPI).
+* 4 bytes (Sequence Number).
+* 8 bytes (ESP IV).
+* 1 byte (Pad Length).
+* 1 byte (NextHeader).
+* 16 bytes (ICV).
+* 0-3 bytes (Padding).
+
+
+[esp]:              https://tools.ietf.org/html/rfc2406
+[aesgcm]:           https://tools.ietf.org/html/rfc4106
+[hkdf]:             https://tools.ietf.org/html/rfc5869
+[esn]:              https://tools.ietf.org/html/rfc4304
+[ipseclevel]:       https://tools.ietf.org/html/draft-mcdonald-simple-ipsec-api-01)
+[mesh]:             https://github.com/weaveworks/mesh

docs/fastdp.md

@@ -52,9 +52,9 @@ overlay, which has a more sophisticated dynamic mechanism for coping
 with low path MTUs.
 
 To avoid triggering this fallback in typical deployments, the datapath
-interface is statically configured with an MTU of 1410 bytes allowing
+interface is statically configured with an MTU of 1376 bytes allowing
 it to work with most underlay network provider MTUs, including GCE at
-1460 bytes (the fifty byte difference accommodates the vxlan overhead).
+1460 bytes (the eighty four byte difference accommodates the encrypted vxlan overhead).
 This value can be overridden by setting `WEAVE_MTU` at launch if
 necessary.
 

docs/ipam.md

@@ -65,14 +65,30 @@ peer for space:
   - it will continue to ask for space until it receives some, or its
     copy of the ring tells it all peers are full in that subnet.
 
+### Data persistence
+
+Key IPAM data is saved to disk, in a [BoltDB](https://github.com/boltdb/bolt)
+file, stored within a [data volume container](https://docs.docker.com/engine/userguide/containers/dockervolumes/#creating-and-mounting-a-data-volume-container)
+named `weavedb`.
+
+This file is used to persist data for various Weave Net components; for IPAM it contains:
+
+* the division of the IP allocation range amongst peers, and
+* allocation of addresses to containers on the local peer,
+
+so that it is immediately available when the peer restarts.
+
 ### Claiming an address
 
-If a Weave process is restarted, in most cases it will hear from
-another peer which ranges it used to own, but it needs to know which
-individual IP addresses are assigned to containers in order to avoid
-giving the same address out in subsequent allocation requests. The
-weave script invokes the `claim` command in `populate_ipam` to do
-this.
+If a Weave process is restarted,
+
+- if `weavedb` is present, then it loads persisted IPAM data from there, as
+described, in the previous section;
+- else, it learns from other peers which ranges it used to own.
+
+Weave Net then consults the allocator to know which individual IP addresses
+are assigned to containers, and therefore avoid giving the same address out in
+subsequent allocation requests.
 
 When the Allocator is told to claim an address, there are four
 possibilities:

docs/weavenpc-design.md

@@ -0,0 +1,106 @@
+# Overview
+
+# ipsets
+
+The policy controller maintains a number of ipsets which are
+subsequently referred to by the iptables rules used to effect network
+policy specifications. These ipsets are created, modified and
+destroyed automatically in response to Pod, Namespace and
+NetworkPolicy object updates from the k8s API server:
+
+* A `hash:ip` set per namespace, containing the IP addresses of all
+  pods in that namespace
+* A `list:set` per distinct (across all network policies in all
+  namespaces) namespace selector mentioned in a network policy,
+  containing the names of any of the above hash:ip sets whose
+  corresponding namespace labels match the selector
+* A `hash:ip` set for each distinct (within the scope of the
+  containing network policy's namespace) pod selector mentioned in a
+  network policy, containing the IP addresses of all pods in the
+  namespace whose labels match that selector
+
+IPsets are implemented by the kernel module `xt_set`, without which
+weave-npc will not work.
+
+ipset names are generated deterministically from a string
+representation of the corresponding label selector. Because ipset
+names are limited to 31 characters in length, this is done by taking a
+SHA hash of the selector string and then printing that out as a base
+85 string with a "weave-" prefix e.g.:
+
+    weave-k?Z;25^M}|1s7P3|H9i;*;MhG
+
+Because pod selectors are scoped to a namespace, we need to make sure
+that if the same selector definition is used in different namespaces
+that we maintain distinct ipsets. Consequently, for such selectors the
+namespace name is prepended to the label selector string before
+hashing to avoid clashes.
+
+# iptables chains
+
+The policy controller maintains two iptables chains in response to
+changes to pods, namespaces and network policies. One chain contains
+the ingress rules that implement the network policy specifications,
+and the other is used to bypass the ingress rules for namespaces which
+have an ingress isolation policy of `DefaultAllow`.
+
+## Dynamically maintained `WEAVE-NPC-DEFAULT` chain
+
+The policy controller maintains a rule in this chain for every
+namespace whose ingress isolation policy is `DefaultAllow`. The
+purpose of this rule is simply to ACCEPT any traffic destined for such
+namespaces before it reaches the ingress chain.
+
+```
+iptables -A WEAVE-NPC-DEFAULT -m set --match-set $NSIPSET dst -j ACCEPT
+```
+
+## Dynamically maintained `WEAVE-NPC-INGRESS` chain
+
+For each namespace network policy ingress rule peer/port combination:
+
+```
+iptables -A WEAVE-NPC-INGRESS -p $PROTO [-m set --match-set $SRCSET] -m set --match-set $DSTSET --dport $DPORT -j ACCEPT
+```
+
+## Static `WEAVE-NPC` chain
+
+Static configuration:
+
+```
+iptables -A WEAVE-NPC -m state --state RELATED,ESTABLISHED -j ACCEPT
+iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-DEFAULT
+iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-INGRESS
+```
+
+# Steering traffic into the policy engine
+
+To direct traffic into the policy engine:
+
+iptables -A FORWARD -o weave -m physdev ! --physdev-out vethwe-bridge -j WEAVE-NPC
+iptables -A FORWARD -o weave -m physdev ! --physdev-out vethwe-bridge -j DROP
+
+Note this only affects traffic which egresses the bridge on a physical
+port which is not the Weave Net router - in other words, it is
+destined for an application container veth. The following traffic is
+affected:
+
+* Traffic bridged between local application containers
+* Traffic bridged from the router to a local application container
+* Traffic originating from the internet destined for nodeports - this
+  is routed via the FORWARD chain to a container pod IP after DNAT
+
+The following traffic is NOT affected:
+
+* Traffic bridged from a local application container to the router
+* Traffic originating from processes in the host network namespace
+  (e.g. kubelet health checks)
+* Traffic routed from an application container to the internet
+
+The above mechanism relies on the kernel module `br_netfilter` being
+loaded and enabled via `/proc/sys/net/bridge/bridge-nf-call-iptables`.
+
+See these resources for helpful context:
+
+* http://ebtables.netfilter.org/br_fw_ia/br_fw_ia.html
+* https://commons.wikimedia.org/wiki/File:Netfilter-packet-flow.svg

ipam/allocate.go

@@ -3,6 +3,7 @@ package ipam
 import (
 	"fmt"
 
+	"github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/net/address"
 )
 
@@ -13,9 +14,9 @@ type allocateResult struct {
 
 type allocate struct {
 	resultChan       chan<- allocateResult
-	ident            string
+	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
 	r                address.CIDR // Subnet we are trying to allocate within
-	isContainer      bool
+	isContainer      bool         // true if ident is a container ID
 	hasBeenCancelled func() bool
 }
 
@@ -43,7 +44,7 @@ func (g *allocate) Try(alloc *Allocator) bool {
 	if ok, addr := alloc.space.Allocate(g.r.HostRange()); ok {
 		// If caller hasn't supplied a unique ID, file it under the IP address
 		// which lets the caller then release the address using DELETE /ip/address
-		if g.ident == "_" {
+		if g.ident == api.NoContainerID {
 			g.ident = addr.String()
 		}
 		alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r)

ipam/allocator.go

@@ -76,6 +76,13 @@ type Allocator struct {
 	now               func() time.Time
 }
 
+// PreClaims are IP addresses discovered before we could initialize IPAM
+type PreClaim struct {
+	Ident       string // a container ID, something like "weave:expose", or api.NoContainerID
+	IsContainer bool   // true if Ident is a container ID
+	Cidr        address.CIDR
+}
+
 type Config struct {
 	OurName     mesh.PeerName
 	OurUID      mesh.PeerUID
@@ -83,6 +90,7 @@ type Config struct {
 	Seed        []mesh.PeerName
 	Universe    address.CIDR
 	IsObserver  bool
+	PreClaims   []PreClaim
 	Quorum      func() uint
 	Db          db.DB
 	IsKnownPeer func(name mesh.PeerName) bool
@@ -123,6 +131,12 @@ func NewAllocator(config Config) *Allocator {
 		dead:        make(map[string]time.Time),
 		now:         time.Now,
 	}
+
+	alloc.pendingClaims = make([]operation, len(config.PreClaims))
+	for i, c := range config.PreClaims {
+		alloc.pendingClaims[i] = &claim{ident: c.Ident, cidr: c.Cidr}
+	}
+
 	return alloc
 }
 
@@ -156,6 +170,9 @@ func (alloc *Allocator) Start() {
 	default:
 		alloc.infof("Initialising as observer - awaiting IPAM data from another peer")
 	}
+	if loadedPersistedData { // do any pre-claims right away
+		alloc.tryOps(&alloc.pendingClaims)
+	}
 	actionChan := make(chan func(), mesh.ChannelSize)
 	stopChan := make(chan struct{})
 	alloc.actionChan = actionChan
@@ -322,7 +339,7 @@ func (alloc *Allocator) Claim(ident string, cidr address.CIDR, isContainer, noEr
 // ContainerDied called from the updater interface.  Async.
 func (alloc *Allocator) ContainerDied(ident string) {
 	alloc.actionChan <- func() {
-		if alloc.hasOwned(ident) {
+		if alloc.hasOwnedByContainer(ident) {
 			alloc.debugln("Container", ident, "died; noting to remove later")
 			alloc.dead[ident] = alloc.now()
 		}
@@ -335,7 +352,7 @@ func (alloc *Allocator) ContainerDied(ident string) {
 // ContainerDestroyed called from the updater interface.  Async.
 func (alloc *Allocator) ContainerDestroyed(ident string) {
 	alloc.actionChan <- func() {
-		if alloc.hasOwned(ident) {
+		if alloc.hasOwnedByContainer(ident) {
 			alloc.debugln("Container", ident, "destroyed; removing addresses")
 			alloc.delete(ident)
 			delete(alloc.dead, ident)
@@ -920,13 +937,12 @@ func (alloc *Allocator) reportFreeSpace() {
 // Persistent data
 const (
 	ringIdent  = "ring"
-	nameIdent  = "peername"
 	ownedIdent = "ownedAddresses"
 )
 
 func (alloc *Allocator) persistRing() {
 	// It would be better if these two Save operations happened in the same transaction
-	if err := alloc.db.Save(nameIdent, alloc.ourName); err != nil {
+	if err := alloc.db.Save(db.NameIdent, alloc.ourName); err != nil {
 		alloc.fatalf("Error persisting ring data: %s", err)
 		return
 	}
@@ -938,7 +954,7 @@ func (alloc *Allocator) persistRing() {
 // Returns true if persisted data is to be used, otherwise false
 func (alloc *Allocator) loadPersistedData() bool {
 	var checkPeerName mesh.PeerName
-	nameFound, err := alloc.db.Load(nameIdent, &checkPeerName)
+	nameFound, err := alloc.db.Load(db.NameIdent, &checkPeerName)
 	if err != nil {
 		alloc.fatalf("Error loading persisted peer name: %s", err)
 	}
@@ -996,9 +1012,9 @@ func (alloc *Allocator) persistOwned() {
 
 // Owned addresses
 
-func (alloc *Allocator) hasOwned(ident string) bool {
-	_, b := alloc.owned[ident]
-	return b
+func (alloc *Allocator) hasOwnedByContainer(ident string) bool {
+	d, b := alloc.owned[ident]
+	return b && d.IsContainer
 }
 
 // NB: addr must not be owned by ident already

ipam/allocator_test.go

@@ -168,11 +168,13 @@ func TestAllocatorClaim(t *testing.T) {
 		container1 = "abcdef"
 		container3 = "b01df00d"
 		universe   = "10.0.3.0/24"
-		testAddr1  = "10.0.3.2/24"
+		testAddr1  = "10.0.3.5/24"
 		testAddr2  = "10.0.4.2/24"
+		testPre    = "10.0.3.1/24"
 	)
 
-	allocs, router, subnet := makeNetworkOfAllocators(2, universe)
+	preAddr, _ := address.ParseCIDR(testPre)
+	allocs, router, subnet := makeNetworkOfAllocators(2, universe, []PreClaim{{container1, true, preAddr}})
 	defer stopNetworkOfAllocators(allocs, router)
 	alloc := allocs[1]
 	addr1, _ := address.ParseCIDR(testAddr1)
@@ -184,6 +186,8 @@ func TestAllocatorClaim(t *testing.T) {
 	alloc.Prime()
 	// Do an allocate on the other peer, which we will try to claim later
 	addrx, err := allocs[0].Allocate(container1, subnet, true, returnFalse)
+	// Should not get the address we pre-claimed
+	require.NotEqual(t, addrx, preAddr)
 	router.Flush()
 
 	// Now try the claim again
@@ -201,6 +205,11 @@ func TestAllocatorClaim(t *testing.T) {
 	// claiming the address allocated on the other peer should fail
 	err = alloc.SimplyClaim(container1, address.MakeCIDR(subnet, addrx))
 	require.Error(t, err, "claiming address allocated on other peer should fail")
+	// claiming the pre-claimed address should fail on both peers
+	err = alloc.SimplyClaim(container3, preAddr)
+	require.Error(t, err, "claiming address allocated on other peer should fail")
+	err = allocs[0].SimplyClaim(container3, preAddr)
+	require.Error(t, err, "claiming address allocated on other peer should fail")
 	// Check an address outside of our universe
 	addr2, _ := address.ParseCIDR(testAddr2)
 	err = alloc.SimplyClaim(container1, addr2)

ipam/claim.go

@@ -5,16 +5,17 @@ import (
 
 	"github.com/weaveworks/mesh"
 
+	"github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
 )
 
 type claim struct {
 	resultChan       chan<- error
-	ident            string
-	cidr             address.CIDR
-	isContainer      bool
-	noErrorOnUnknown bool
+	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
+	cidr             address.CIDR // single address being claimed
+	isContainer      bool         // true if ident is a container ID
+	noErrorOnUnknown bool         // if false, error or block if we don't know; if true return ok but keep trying
 	hasBeenCancelled func() bool
 }
 
@@ -34,7 +35,7 @@ func (c *claim) sendResult(result error) {
 
 // Try returns true for success (or failure), false if we need to try again later
 func (c *claim) Try(alloc *Allocator) bool {
-	if c.hasBeenCancelled() {
+	if c.hasBeenCancelled != nil && c.hasBeenCancelled() {
 		c.Cancel()
 		return true
 	}
@@ -77,26 +78,34 @@ func (c *claim) Try(alloc *Allocator) bool {
 		return false
 	}
 
-	if c.ident == "_" { // Special "I don't have a unique ID" identifier
-		c.ident = c.cidr.Addr.String()
-	}
 	// We are the owner, check we haven't given it to another container
-	switch existingIdent := alloc.findOwner(c.cidr.Addr); existingIdent {
-	case "":
+	existingIdent := alloc.findOwner(c.cidr.Addr)
+	switch {
+	case existingIdent == "":
+		// Unused address, we try to claim it:
 		if err := alloc.space.Claim(c.cidr.Addr); err == nil {
 			alloc.debugln("Claimed", c.cidr, "for", c.ident)
-			alloc.addOwned(c.ident, c.cidr, c.isContainer)
+			if c.ident == api.NoContainerID {
+				alloc.addOwned(c.cidr.Addr.String(), c.cidr, c.isContainer)
+			} else {
+				alloc.addOwned(c.ident, c.cidr, c.isContainer)
+			}
 			c.sendResult(nil)
 		} else {
 			c.sendResult(err)
 		}
-	case c.ident:
+	case (existingIdent == c.ident) || (c.ident == api.NoContainerID && existingIdent == c.cidr.Addr.String()):
 		// same identifier is claiming same address; that's OK
 		alloc.debugln("Re-Claimed", c.cidr, "for", c.ident)
 		c.sendResult(nil)
-	case c.cidr.Addr.String():
-		// Address already allocated via "_" name
+	case existingIdent == c.cidr.Addr.String():
+		// Address already allocated via api.NoContainerID name and current ID is a real container ID:
 		c.sendResult(fmt.Errorf("address %s already in use", c.cidr))
+	case c.ident == api.NoContainerID:
+		// We do not know whether this is the same container or another one,
+		// but we also cannot prove otherwise, so we let it reclaim the address:
+		alloc.debugln("Re-Claimed", c.cidr, "for ID", c.ident, "having existing ID as", existingIdent)
+		c.sendResult(nil)
 	default:
 		// Addr already owned by container on this machine
 		c.sendResult(fmt.Errorf("address %s is already owned by %s", c.cidr.String(), existingIdent))

ipam/ring/expvar.go

@@ -1,31 +0,0 @@
-package ring
-
-/* Exported variables for monitoring and management */
-
-import (
-	"expvar"
-	"strconv"
-)
-
-var (
-	expRingSize    = expvar.NewMap("ipam.ringSize")
-	expRingEntries = expvar.NewMap("ipam.ringEntries")
-)
-
-type _int int
-
-func (i _int) String() string {
-	return strconv.Itoa(int(i))
-}
-
-type _uint32 uint32
-
-func (i _uint32) String() string {
-	return strconv.FormatUint(uint64(i), 10)
-}
-
-func (r *Ring) updateExportedVariables() {
-	ringName := r.Start.String()
-	expRingSize.Set(ringName, _uint32(r.End-r.Start))
-	expRingEntries.Set(ringName, _int(len(r.Entries)))
-}

ipam/ring/ring.go

@@ -41,8 +41,8 @@ var (
 	ErrNotSorted       = errors.New("Ring not sorted")
 	ErrTokenRepeated   = errors.New("Token appears twice in ring")
 	ErrTokenOutOfRange = errors.New("Token is out of range")
-	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
-	ErrDifferentRange  = errors.New("Received range differs from ours!")
+	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours")
+	ErrDifferentRange  = errors.New("Received range differs from ours")
 )
 
 func errInconsistentEntry(mine, theirs *entry) error {
@@ -114,7 +114,6 @@ func New(start, end address.Address, peer mesh.PeerName, f OnUpdate) *Ring {
 	common.Assert(start < end)
 
 	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0), onUpdate: f}
-	ring.updateExportedVariables()
 	return ring
 }
 
@@ -151,7 +150,6 @@ func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName)
 	r.assertInvariants()
 	defer r.trackUpdates()()
 	defer r.assertInvariants()
-	defer r.updateExportedVariables()
 
 	// ----------------- Start of Checks -----------------
 
@@ -218,7 +216,6 @@ func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName)
 func (r *Ring) Merge(gossip Ring) (bool, error) {
 	r.assertInvariants()
 	defer r.trackUpdates()()
-	defer r.updateExportedVariables()
 
 	// Don't panic when checking the gossiped in ring.
 	// In this case just return any error found.
@@ -404,7 +401,6 @@ func (r *Ring) ClaimForPeers(peers []mesh.PeerName) {
 
 	defer r.trackUpdates()()
 	defer r.assertInvariants()
-	defer r.updateExportedVariables()
 	defer func() {
 		e := r.Entries[len(r.Entries)-1]
 		common.Assert(address.Add(e.Token, address.Offset(e.Free)) == r.End)
@@ -455,7 +451,6 @@ func (r *Ring) String() string {
 func (r *Ring) ReportFree(freespace map[address.Address]address.Count) (updated bool) {
 	r.assertInvariants()
 	defer r.assertInvariants()
-	defer r.updateExportedVariables()
 
 	common.Assert(!r.Empty())
 	entries := r.Entries
@@ -564,7 +559,6 @@ func (r *Ring) Transfer(from, to mesh.PeerName) []address.Range {
 	defer r.trackUpdates()()
 	defer r.trackUpdatesOfPeer(from)()
 	defer r.assertInvariants()
-	defer r.updateExportedVariables()
 
 	var newRanges []address.Range
 

ipam/ring/ring_test.go

@@ -494,7 +494,7 @@ func TestTransfer(t *testing.T) {
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	ring1.Transfer(peer2name, peer1name)
-	require.Equal(t, []address.Range{{start, middle}, {middle, end}}, ring1.OwnedRanges())
+	require.Equal(t, []address.Range{{Start: start, End: middle}, {Start: middle, End: end}}, ring1.OwnedRanges())
 
 	// Second test is what happens when a token exists at the end of a range but is transferred
 	// - does it get resurrected correctly?
@@ -503,7 +503,7 @@ func TestTransfer(t *testing.T) {
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	ring1.Transfer(peer2name, peer1name)
 	ring1.GrantRangeToHost(dot10, middle, peer2name)
-	require.Equal(t, []address.Range{{start, dot10}, {middle, end}}, ring1.OwnedRanges())
+	require.Equal(t, []address.Range{{Start: start, End: dot10}, {Start: middle, End: end}}, ring1.OwnedRanges())
 }
 
 func TestOwner(t *testing.T) {

ipam/space/space.go

@@ -88,6 +88,14 @@ func (s *Space) Claim(addr address.Address) error {
 	return nil
 }
 
+func (s *Space) NumOwnedAddresses() address.Count {
+	res := address.Count(0)
+	for i := 0; i < len(s.ours); i += 2 {
+		res += address.Length(s.ours[i+1], s.ours[i])
+	}
+	return res
+}
+
 func (s *Space) NumFreeAddresses() address.Count {
 	res := address.Count(0)
 	for i := 0; i < len(s.free); i += 2 {

ipam/space/space_test.go

@@ -98,16 +98,19 @@ func TestSpaceAllocate(t *testing.T) {
 
 	space1 := makeSpace(start, size)
 	require.Equal(t, address.Count(20), space1.NumFreeAddresses())
+	require.Equal(t, address.Count(0), space1.NumOwnedAddresses())
 	space1.assertInvariants()
 
 	_, addr1 := space1.Allocate(address.NewRange(start, size))
 	require.Equal(t, testAddr1, addr1.String(), "address")
 	require.Equal(t, address.Count(19), space1.NumFreeAddresses())
+	require.Equal(t, address.Count(1), space1.NumOwnedAddresses())
 	space1.assertInvariants()
 
 	_, addr2 := space1.Allocate(address.NewRange(start, size))
 	require.False(t, addr2.String() == testAddr1, "address")
 	require.Equal(t, address.Count(18), space1.NumFreeAddresses())
+	require.Equal(t, address.Count(2), space1.NumOwnedAddresses())
 	require.Equal(t, address.Count(13), space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddrx)}))
 	require.Equal(t, address.Count(18), space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddry)}))
 	space1.assertInvariants()

ipam/status.go

@@ -10,6 +10,7 @@ type Status struct {
 	Paxos            *paxos.Status
 	Range            string
 	RangeNumIPs      int
+	ActiveIPs        int
 	DefaultSubnet    string
 	Entries          []EntryStatus
 	PendingClaims    []ClaimStatus
@@ -52,6 +53,7 @@ func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status {
 			paxosStatus,
 			allocator.universe.String(),
 			int(allocator.universe.Size()),
+			int(allocator.space.NumOwnedAddresses()),
 			defaultSubnet.String(),
 			newEntryStatusSlice(allocator),
 			newClaimStatusSlice(allocator),

ipam/testutils_test.go

@@ -128,7 +128,7 @@ type mockDB struct{}
 func (d *mockDB) Load(_ string, _ interface{}) (bool, error) { return false, nil }
 func (d *mockDB) Save(_ string, _ interface{}) error         { return nil }
 
-func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.CIDR) {
+func makeAllocator(name string, cidrStr string, quorum uint, preClaims ...PreClaim) (*Allocator, address.CIDR) {
 	peername, err := mesh.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
@@ -145,6 +145,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres
 		OurNickname: "nick-" + name,
 		Universe:    cidr,
 		IsObserver:  quorum == 0,
+		PreClaims:   preClaims,
 		Quorum:      func() uint { return quorum },
 		Db:          new(mockDB),
 		IsKnownPeer: func(mesh.PeerName) bool { return true },
@@ -213,15 +214,19 @@ func AssertNothingSentErr(t *testing.T, ch <-chan error) {
 	}
 }
 
-func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
+func makeNetworkOfAllocators(size int, cidr string, preClaims ...[]PreClaim) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
 	gossipRouter := gossip.NewTestRouter(0.0)
 	allocs := make([]*Allocator, size)
 	var subnet address.CIDR
 
 	for i := 0; i < size; i++ {
 		var alloc *Allocator
+		preClaim := []PreClaim{}
+		if i < len(preClaims) {
+			preClaim = preClaims[i]
+		}
 		alloc, subnet = makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
-			cidr, uint(size/2+1))
+			cidr, uint(size/2+1), preClaim...)
 		alloc.SetInterfaces(gossipRouter.Connect(alloc.ourName, alloc))
 		alloc.Start()
 		allocs[i] = alloc

nameserver/dns.go

@@ -222,7 +222,7 @@ func (h *handler) handleReverse(w dns.ResponseWriter, req *dns.Msg) {
 		return
 	}
 
-	ipStr := strings.TrimSuffix(req.Question[0].Name, "."+reverseDNSdomain)
+	ipStr := strings.TrimSuffix(strings.ToLower(req.Question[0].Name), "."+reverseDNSdomain)
 	ip, err := address.ParseIP(ipStr)
 	if err != nil {
 		h.nameError(w, req)

nameserver/entry.go

@@ -120,7 +120,7 @@ func (e1 *Entry) tombstone() bool {
 
 func check(es SortableEntries) error {
 	if !sort.IsSorted(es) {
-		return fmt.Errorf("Not sorted!")
+		return fmt.Errorf("Not sorted")
 	}
 	for i := 1; i < es.Len(); i++ {
 		if es.Get(i).equal(es.Get(i - 1)) {

net/bridge.go

@@ -1,6 +1,10 @@
 package net
 
 import (
+	"fmt"
+	"net"
+
+	"github.com/Sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 )
 
@@ -51,3 +55,56 @@ func isDatapath(link netlink.Link) bool {
 		return false
 	}
 }
+
+func DetectHairpin(portIfName string, log *logrus.Logger) error {
+	link, err := netlink.LinkByName(portIfName)
+	if err != nil {
+		return fmt.Errorf("Unable to find link %q: %s", portIfName, err)
+	}
+
+	ch := make(chan netlink.LinkUpdate)
+	// See EnsureInterface for why done channel is not passed
+	if err := netlink.LinkSubscribe(ch, nil); err != nil {
+		return fmt.Errorf("Unable to subscribe to netlink updates: %s", err)
+	}
+
+	pi, err := netlink.LinkGetProtinfo(link)
+	if err != nil {
+		return fmt.Errorf("Unable to get link protinfo %q: %s", portIfName, err)
+	}
+	if pi.Hairpin {
+		return fmt.Errorf("Hairpin mode enabled on %q", portIfName)
+	}
+
+	go func() {
+		for up := range ch {
+			if up.Attrs().Name == portIfName && up.Attrs().Protinfo != nil &&
+				up.Attrs().Protinfo.Hairpin {
+				log.Errorf("Hairpin mode enabled on %q", portIfName)
+			}
+		}
+	}()
+
+	return nil
+}
+
+var ErrBridgeNoIP = fmt.Errorf("Bridge has no IP address")
+
+func FindBridgeIP(bridgeName string, subnet *net.IPNet) (net.IP, error) {
+	netdev, err := GetBridgeNetDev(bridgeName)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get netdev for %q bridge: %s", bridgeName, err)
+	}
+	if len(netdev.CIDRs) == 0 {
+		return nil, ErrBridgeNoIP
+	}
+	if subnet != nil {
+		for _, cidr := range netdev.CIDRs {
+			if subnet.Contains(cidr.IP) {
+				return cidr.IP, nil
+			}
+		}
+	}
+	// No subnet, or none in the required subnet; just return the first one
+	return netdev.CIDRs[0].IP, nil
+}

net/ipsec/ipsec.go

@@ -0,0 +1,562 @@
+// package IPsec provides primitives for establishing IPsec in the fastdp mode.
+package ipsec
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/crypto/hkdf"
+
+	"github.com/weaveworks/mesh"
+)
+
+const (
+	keySize   = 36 // AES-GCM key 32 bytes + 4 bytes salt
+	nonceSize = 32 // HKDF nonce size
+
+	mark    = uint32(0x1) << 17 // iptables marks
+	markStr = "0x20000/0x20000" // update if the above mark changes
+
+	tableMangle  = "mangle"
+	tableFilter  = "filter"
+	chainIn      = "WEAVE-IPSEC-IN"
+	chainInMark  = "WEAVE-IPSEC-IN-MARK"
+	chainOut     = "WEAVE-IPSEC-OUT"
+	chainOutMark = "WEAVE-IPSEC-OUT-MARK"
+)
+
+type SPI uint32
+
+// Used to identify:
+// - directional SPIs,
+// - ipsec establishments.
+type spiID [24]byte
+
+func getSPIId(srcPeer, dstPeer mesh.PeerName, connUID uint64) (id spiID) {
+	binary.BigEndian.PutUint64(id[:], uint64(srcPeer))
+	binary.BigEndian.PutUint64(id[8:], uint64(dstPeer))
+	binary.BigEndian.PutUint64(id[16:], connUID)
+	return
+}
+
+type spiInfo struct {
+	spi      SPI
+	isDirOut bool
+}
+
+// IPSec
+
+type IPSec struct {
+	sync.RWMutex
+	ipt *iptables.IPTables
+	log *logrus.Logger
+
+	spiInfo map[spiID]spiInfo
+	// A reference to spiInfo; spiInfo might be of an expired SPI.
+	spis map[SPI]*spiInfo
+}
+
+func New(log *logrus.Logger) (*IPSec, error) {
+	ipt, err := iptables.New()
+	if err != nil {
+		return nil, errors.Wrap(err, "iptables new")
+	}
+
+	ipsec := &IPSec{
+		ipt:     ipt,
+		log:     log,
+		spiInfo: make(map[spiID]spiInfo),
+		spis:    make(map[SPI]*spiInfo),
+	}
+
+	return ipsec, nil
+}
+
+// InitSALocal initializes inbound ipsec from remotePeer and triggers
+// the initialization on remotePeer.
+func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte, initRemote func([]byte) error) error {
+	// ID of inbound SPI
+	spiID := getSPIId(remotePeer, localPeer, connUID)
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	// Derive SA key
+	nonce, err := genNonce()
+	if err != nil {
+		return errors.Wrap(err, "generate nonce")
+	}
+	key, err := deriveKey(sessionKey[:], nonce, localPeer)
+	if err != nil {
+		return errors.Wrap(err, "derive key")
+	}
+
+	// Allocate SA
+	sa, err := netlink.XfrmStateAllocSpi(xfrmAllocSpiState(remoteIP, localIP))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("ip xfrm state allocspi (in, %s, %s)", remoteIP, localIP))
+	}
+
+	// Use SPI generated by the kernel. The kernel ensures {dstIP, spi} to
+	// be unique.
+	spi := SPI(sa.Spi)
+
+	ipsec.log.Infof("ipsec: InitSALocal: %s -> %s :%d 0x%x", remoteIP, localIP, udpPort, spi)
+
+	// Create SA
+	if sa, err := xfrmState(remoteIP, localIP, spi, false, key); err == nil {
+		if err := netlink.XfrmStateUpdate(sa); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("xfrm state update (in, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+		}
+	} else {
+		return errors.Wrap(err, "new xfrm state (in)")
+	}
+
+	// Install iptables rules
+	if err := ipsec.installDropNonEncrypted(localIP, remoteIP, udpPort, spi); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("install protecting rules (%s, %s, %d, 0x%x)", localIP, remoteIP, udpPort, spi))
+	}
+
+	si := spiInfo{spi: spi, isDirOut: false}
+	ipsec.spiInfo[spiID] = si
+	ipsec.spis[spi] = &si
+
+	// Trigger the initialization on the remote peer
+	msg := &msgInitSARemote{nonce, spi}
+	if err := initRemote(msg.serialize()); err != nil {
+		return errors.Wrap(err, "send InitSARemote")
+	}
+
+	return nil
+}
+
+// InitSARemote initializes outbound ipsec to remotePeer.
+// Triggered by remotePeer.
+func (ipsec *IPSec) InitSARemote(msgInitSARemote []byte, localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte) error {
+	// ID of outbound SPI
+	spiID := getSPIId(localPeer, remotePeer, connUID)
+
+	msg, err := deserializeMsgInitSARemote(msgInitSARemote)
+	if err != nil {
+		return errors.Wrap(err, "deserialize InitSARemote")
+	}
+	spi := msg.spi
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	ipsec.log.Infof("ipsec: InitSARemote: %s -> %s :%d 0x%x", localIP, remoteIP, udpPort, spi)
+
+	// Derive SA key by using the received nonce
+	key, err := deriveKey(sessionKey[:], msg.nonce, remotePeer)
+	if err != nil {
+		return errors.Wrap(err, "derive key")
+	}
+
+	// Create SA
+	if sa, err := xfrmState(localIP, remoteIP, spi, true, key); err == nil {
+		if err := netlink.XfrmStateAdd(sa); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("xfrm state update (out, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+		}
+	} else {
+		return errors.Wrap(err, "new xfrm state (out)")
+	}
+
+	// Create or update SP
+	sp := xfrmPolicy(localIP, remoteIP, spi)
+	if err := netlink.XfrmPolicyUpdate(sp); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("xfrm policy update (%s, %s, 0x%x)", localIP, remoteIP, spi))
+	}
+
+	si := spiInfo{spi: spi, isDirOut: true}
+	ipsec.spiInfo[spiID] = si
+	ipsec.spis[spi] = &si
+
+	return nil
+}
+
+// Destroy destroys any (inbound / outbound) ipsec establishment between the peers.
+func (ipsec *IPSec) Destroy(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int) error {
+	outSPIID := getSPIId(localPeer, remotePeer, connUID)
+	inSPIID := getSPIId(remotePeer, localPeer, connUID)
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	// Destroy inbound
+
+	if inSPIInfo, ok := ipsec.spiInfo[inSPIID]; ok {
+		ipsec.log.Infof("ipsec: destroy: in %s -> %s 0x%x", remoteIP, localIP, inSPIInfo.spi)
+
+		inSPI := inSPIInfo.spi
+
+		inSA := &netlink.XfrmState{
+			Src:   remoteIP,
+			Dst:   localIP,
+			Proto: netlink.XFRM_PROTO_ESP,
+			Spi:   int(inSPI),
+		}
+		if err := netlink.XfrmStateDel(inSA); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm state del (in, %s, %s, 0x%x) failed: %s", inSA.Src, inSA.Dst, inSA.Spi, err)
+		}
+
+		if err := ipsec.removeDropNonEncrypted(localIP, remoteIP, udpPort, inSPI); err != nil {
+			ipsec.log.Warnf("ipsec: remove protecting rules (%s, %s, %d, 0x%x) failed: %s", localIP, remoteIP, udpPort, inSPI, err)
+		}
+
+		delete(ipsec.spiInfo, inSPIID)
+		delete(ipsec.spis, inSPI)
+	}
+
+	// Destroy outbound
+
+	if outSPIInfo, ok := ipsec.spiInfo[outSPIID]; ok {
+		ipsec.log.Infof("ipsec: destroy: out %s -> %s 0x%x", localIP, remoteIP, outSPIInfo.spi)
+
+		if err := netlink.XfrmPolicyDel(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi)); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm policy del (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err)
+		}
+
+		outSA := &netlink.XfrmState{
+			Src:   localIP,
+			Dst:   remoteIP,
+			Proto: netlink.XFRM_PROTO_ESP,
+			Spi:   int(outSPIInfo.spi),
+		}
+		if err := netlink.XfrmStateDel(outSA); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm state del (out, %s, %s, 0x%x) failed: %s", outSA.Src, outSA.Dst, outSA.Spi, err)
+		}
+
+		delete(ipsec.spiInfo, outSPIID)
+		delete(ipsec.spis, outSPIInfo.spi)
+	}
+
+	return nil
+}
+
+// Flush removes all policies/SAs established by us. Also, it removes chains and
+// rules of iptables.
+//
+// If destroy is true, the chains and the rules won't be re-created.
+func (ipsec *IPSec) Flush(destroy bool) error {
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	policies, err := netlink.XfrmPolicyList(syscall.AF_INET)
+	if err != nil {
+		return errors.Wrap(err, "xfrm policy list")
+	}
+	for _, p := range policies {
+		if p.Mark != nil && p.Mark.Value == mark && len(p.Tmpls) != 0 {
+			spi := SPI(p.Tmpls[0].Spi)
+			if err := netlink.XfrmPolicyDel(&p); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("xfrm policy del (%s, %s, 0x%x)", p.Src, p.Dst, spi))
+			}
+		}
+	}
+
+	states, err := netlink.XfrmStateList(syscall.AF_INET)
+	if err != nil {
+		return errors.Wrap(err, "xfrm state list")
+	}
+	for _, s := range states {
+		if _, ok := ipsec.spis[SPI(s.Spi)]; ok {
+			if err := netlink.XfrmStateDel(&s); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("xfrm state list (%s, %s, 0x%x)", s.Src, s.Dst, s.Spi))
+			}
+		}
+	}
+
+	if err := ipsec.resetIPTables(destroy); err != nil {
+		return errors.Wrap(err, "reset ip tables")
+	}
+
+	return nil
+}
+
+// iptables
+
+type chain struct {
+	table string
+	chain string
+}
+type rule struct {
+	table    string
+	chain    string
+	rulespec []string
+	unique   bool
+}
+
+func (ipsec *IPSec) clearChains(chains []chain) error {
+	for _, c := range chains {
+		if err := ipsec.ipt.ClearChain(c.table, c.chain); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables clear chain (%s, %s)", c.table, c.chain))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) deleteChains(chains []chain) error {
+	for _, c := range chains {
+		if err := ipsec.ipt.DeleteChain(c.table, c.chain); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables delete chain (%s, %s)", c.table, c.chain))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) resetRules(rules []rule, destroy bool) error {
+	for _, r := range rules {
+		ok, err := ipsec.ipt.Exists(r.table, r.chain, r.rulespec...)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables exists rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+		}
+		switch {
+		case !destroy && !ok:
+			if err := ipsec.ipt.Append(r.table, r.chain, r.rulespec...); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("iptables append rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+			}
+		case destroy && ok:
+			if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("iptables delete rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+			}
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) resetIPTables(destroy bool) error {
+	chains := []chain{
+		{tableMangle, chainIn},
+		{tableMangle, chainInMark},
+		{tableFilter, chainIn},
+		{tableMangle, chainOut},
+		{tableMangle, chainOutMark},
+	}
+	rules := []rule{
+		{tableMangle, "INPUT", []string{"-j", chainIn}, true},
+		{tableMangle, chainInMark, []string{"-j", "MARK", "--set-xmark", markStr}, true},
+		{tableFilter, "INPUT", []string{"-j", chainIn}, true},
+		{tableMangle, "OUTPUT", []string{"-j", chainOut}, true},
+		{tableMangle, chainOutMark, []string{"-j", "MARK", "--set-xmark", markStr}, true},
+		{tableFilter, "OUTPUT",
+			[]string{
+				"!", "-p", "esp",
+				"-m", "policy", "--dir", "out", "--pol", "none",
+				"-m", "mark", "--mark", markStr,
+				"-j", "DROP"}, true},
+	}
+
+	if err := ipsec.clearChains(chains); err != nil {
+		return err
+	}
+
+	if err := ipsec.resetRules(rules, destroy); err != nil {
+		return err
+	}
+
+	if destroy {
+		if err := ipsec.deleteChains(chains); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ruleMarkInboundESP(srcIP, dstIP net.IP, inSPI SPI) rule {
+	return rule{tableMangle, chainIn,
+		[]string{
+			"-s", dstIP.String(), "-d", srcIP.String(),
+			"-p", "esp",
+			"-m", "esp", "--espspi", "0x" + strconv.FormatUint(uint64(inSPI), 16),
+			"-j", chainInMark,
+		}, true}
+}
+
+func rulesDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) []rule {
+	udpPortStr := strconv.FormatUint(uint64(udpPort), 10)
+	return []rule{
+		ruleMarkInboundESP(srcIP, dstIP, inSPI),
+		{tableFilter, chainIn,
+			[]string{
+				"-s", dstIP.String(), "-d", srcIP.String(),
+				"-p", "udp", "--dport", udpPortStr,
+				"-m", "mark", "!", "--mark", markStr,
+				"-j", "DROP",
+			}, false},
+		{tableMangle, chainOut,
+			[]string{
+				"-s", srcIP.String(), "-d", dstIP.String(),
+				"-p", "udp", "--dport", udpPortStr,
+				"-j", chainOutMark,
+			}, false},
+	}
+}
+
+func (ipsec *IPSec) installDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error {
+	rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI)
+	for _, r := range rules {
+		appendFunc := ipsec.ipt.Append
+		if r.unique {
+			appendFunc = ipsec.ipt.AppendUnique
+		}
+		if err := appendFunc(r.table, r.chain, r.rulespec...); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables append (%s, %s, %s)", r.table, r.chain, r.rulespec))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) removeDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error {
+	rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI)
+	if err := ipsec.resetRules(rules, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ipsec *IPSec) removeDropNonEncryptedInbound(srcIP, dstIP net.IP, inSPI SPI) error {
+	r := ruleMarkInboundESP(srcIP, dstIP, inSPI)
+	if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("iptables delete unique (%s, %s, %s)", r.table, r.chain, r.rulespec))
+	}
+	return nil
+}
+
+// xfrm
+
+func xfrmAllocSpiState(srcIP, dstIP net.IP) *netlink.XfrmState {
+	return &netlink.XfrmState{
+		Src:          srcIP,
+		Dst:          dstIP,
+		Proto:        netlink.XFRM_PROTO_ESP,
+		Mode:         netlink.XFRM_MODE_TRANSPORT,
+		ReplayWindow: 256,
+		ESN:          true,
+	}
+}
+
+func xfrmState(srcIP, dstIP net.IP, spi SPI, isDirOut bool, key []byte) (*netlink.XfrmState, error) {
+	if len(key) != keySize {
+		return nil, fmt.Errorf("key should be %d bytes long", keySize)
+	}
+
+	state := xfrmAllocSpiState(srcIP, dstIP)
+
+	state.Spi = int(spi)
+	state.Aead = &netlink.XfrmStateAlgo{
+		Name:   "rfc4106(gcm(aes))",
+		Key:    key,
+		ICVLen: 128,
+	}
+
+	return state, nil
+}
+
+func xfrmPolicy(srcIP, dstIP net.IP, spi SPI) *netlink.XfrmPolicy {
+	ipMask := []byte{0xff, 0xff, 0xff, 0xff} // /32
+
+	return &netlink.XfrmPolicy{
+		Src:   &net.IPNet{IP: srcIP, Mask: ipMask},
+		Dst:   &net.IPNet{IP: dstIP, Mask: ipMask},
+		Proto: syscall.IPPROTO_UDP,
+		Dir:   netlink.XFRM_DIR_OUT,
+		Mark: &netlink.XfrmMark{
+			Value: mark,
+			Mask:  mark,
+		},
+		Tmpls: []netlink.XfrmPolicyTmpl{
+			{
+				Src:   srcIP,
+				Dst:   dstIP,
+				Proto: netlink.XFRM_PROTO_ESP,
+				Mode:  netlink.XFRM_MODE_TRANSPORT,
+				Spi:   int(spi),
+			},
+		},
+	}
+}
+
+// Key derivation
+
+func genNonce() ([]byte, error) {
+	buf := make([]byte, nonceSize)
+	n, err := rand.Read(buf)
+	if err != nil {
+		return nil, fmt.Errorf("crypto rand failed: %s", err)
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("not enough of random data: %d", n)
+	}
+	return buf, nil
+}
+
+func deriveKey(sessionKey []byte, nonce []byte, peerName mesh.PeerName) ([]byte, error) {
+	key := make([]byte, keySize)
+
+	info := make([]byte, 8)
+	binary.BigEndian.PutUint64(info, uint64(peerName))
+
+	hkdf := hkdf.New(sha256.New, sessionKey, nonce, info)
+
+	n, err := io.ReadFull(hkdf, key)
+	if err != nil {
+		return nil, err
+	}
+	if n != keySize {
+		return nil, fmt.Errorf("derived too short key: %d", n)
+	}
+
+	return key, nil
+}
+
+// Protocol Messages
+
+type msgInitSARemote struct {
+	nonce []byte
+	spi   SPI
+}
+
+func deserializeMsgInitSARemote(b []byte) (*msgInitSARemote, error) {
+	if len(b) == 0 {
+		return nil, fmt.Errorf("empty msg")
+	}
+
+	msg := &msgInitSARemote{}
+	if len(b) != msg.size() {
+		return nil, fmt.Errorf("invalid payload size: %d", len(b))
+	}
+
+	msg.nonce = make([]byte, nonceSize)
+	copy(msg.nonce, b[:nonceSize])
+	b = b[nonceSize:]
+
+	msg.spi = SPI(binary.BigEndian.Uint32(b))
+
+	return msg, nil
+}
+
+func (msg *msgInitSARemote) size() int {
+	return nonceSize + 32 // SPI
+}
+
+func (msg *msgInitSARemote) serialize() []byte {
+	b := make([]byte, msg.size())
+
+	copy(b[:nonceSize], msg.nonce)
+	binary.BigEndian.PutUint32(b[nonceSize:], uint32(msg.spi))
+
+	return b
+}

net/mac.go

@@ -0,0 +1,38 @@
+package net
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"net"
+)
+
+func RandomMAC() (net.HardwareAddr, error) {
+	mac := make([]byte, 6)
+	if _, err := rand.Read(mac); err != nil {
+		return nil, err
+	}
+
+	setUnicastAndLocal(mac)
+
+	return net.HardwareAddr(mac), nil
+}
+
+func MACfromUUID(uuid []byte) net.HardwareAddr {
+	hash := sha256.New()
+	// We salt the input just as a precaution to avoid clashes with
+	// other applications who might have had the bright idea of
+	// generating MACs in the same way.
+	hash.Write([]byte("9oBJ0Jmip-"))
+	hash.Write(uuid)
+	sum := hash.Sum(nil)
+
+	setUnicastAndLocal(sum)
+
+	return net.HardwareAddr(sum[:6])
+}
+
+// In the first byte of the MAC, the 'multicast' bit should be
+// clear and 'locally administered' bit should be set.
+func setUnicastAndLocal(mac []byte) {
+	mac[0] = (mac[0] & 0xFE) | 0x02
+}

net/netdev.go

@@ -0,0 +1,116 @@
+package net
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"strconv"
+
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+)
+
+type Dev struct {
+	Name  string           `json:"Name,omitempty"`
+	MAC   net.HardwareAddr `json:"MAC,omitempty"`
+	CIDRs []*net.IPNet     `json:"CIDRs,omitempty"`
+}
+
+func LinkToNetDev(link netlink.Link) (Dev, error) {
+	addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
+	if err != nil {
+		return Dev{}, err
+	}
+
+	netDev := Dev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr}
+	for _, addr := range addrs {
+		netDev.CIDRs = append(netDev.CIDRs, addr.IPNet)
+	}
+	return netDev, nil
+}
+
+// ConnectedToBridgeVethPeerIds returns peer indexes of veth links connected to
+// the given bridge. The peer index is used to query from a container netns
+// whether the container is connected to the bridge.
+func ConnectedToBridgeVethPeerIds(bridgeName string) ([]int, error) {
+	var ids []int
+
+	br, err := netlink.LinkByName(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+	links, err := netlink.LinkList()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, link := range links {
+		if _, isveth := link.(*netlink.Veth); isveth && link.Attrs().MasterIndex == br.Attrs().Index {
+			peerID := link.Attrs().ParentIndex
+			if peerID == 0 {
+				// perhaps running on an older kernel where ParentIndex doesn't work.
+				// as fall-back, assume the peers are consecutive
+				peerID = link.Attrs().Index - 1
+			}
+			ids = append(ids, peerID)
+		}
+	}
+
+	return ids, nil
+}
+
+// Lookup the weave interface of a container
+func GetWeaveNetDevs(processID int) ([]Dev, error) {
+	peerIDs, err := ConnectedToBridgeVethPeerIds("weave")
+	if err != nil {
+		return nil, err
+	}
+
+	return GetNetDevsByVethPeerIds(processID, peerIDs)
+}
+
+func GetNetDevsByVethPeerIds(processID int, peerIDs []int) ([]Dev, error) {
+	// Bail out if this container is running in the root namespace
+	netnsRoot, err := netns.GetFromPid(1)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open root namespace: %s", err)
+	}
+	defer netnsRoot.Close()
+	netnsContainer, err := netns.GetFromPid(processID)
+	if err != nil {
+		// Unable to find a namespace for this process - just return nothing
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err)
+	}
+	defer netnsContainer.Close()
+	if netnsRoot.Equal(netnsContainer) {
+		return nil, nil
+	}
+
+	var netdevs []Dev
+	peersStr := make([]string, len(peerIDs))
+	for i, id := range peerIDs {
+		peersStr[i] = strconv.Itoa(id)
+	}
+	nds, err := WithNetNSByPid(processID, "list-netdevs", peersStr...)
+	if err != nil {
+		return nil, fmt.Errorf("list-netdevs failed: %s", err)
+	}
+	err = json.Unmarshal(nds, &netdevs)
+
+	return netdevs, err
+}
+
+// Get the weave bridge interface.
+// NB: Should be called from the root network namespace.
+func GetBridgeNetDev(bridgeName string) (Dev, error) {
+	link, err := netlink.LinkByName(bridgeName)
+	if err != nil {
+		return Dev{}, err
+	}
+
+	return LinkToNetDev(link)
+}

net/netns.go

@@ -57,33 +57,25 @@ func WithNetNSLinkUnsafe(ns netns.NsHandle, ifName string, work func(link netlin
 	})
 }
 
-func WithNetNSLinkByPidUnsafe(pid int, ifName string, work func(link netlink.Link) error) error {
-	ns, err := netns.GetFromPid(pid)
-	if err != nil {
-		return err
-	}
-	defer ns.Close()
-
-	return WithNetNSLinkUnsafe(ns, ifName, work)
-}
+var WeaveUtilCmd = "weaveutil"
 
 // A safe version of WithNetNS* which creates a process executing
 // "nsenter --net=<ns-path> weaveutil <cmd> [args]".
-func WithNetNS(nsPath string, cmd string, args ...string) (string, error) {
+func WithNetNS(nsPath string, cmd string, args ...string) ([]byte, error) {
 	var stdout, stderr bytes.Buffer
 
-	args = append([]string{"--net=" + nsPath, "weaveutil", cmd}, args...)
+	args = append([]string{"--net=" + nsPath, WeaveUtilCmd, cmd}, args...)
 	c := exec.Command("nsenter", args...)
 	c.Stdout = &stdout
 	c.Stderr = &stderr
 	if err := c.Run(); err != nil {
-		return "", fmt.Errorf("%s: %s", string(stderr.Bytes()), err)
+		return nil, fmt.Errorf("%s: %s", string(stderr.Bytes()), err)
 	}
 
-	return string(stdout.Bytes()), nil
+	return stdout.Bytes(), nil
 }
 
-func WithNetNSByPid(pid int, cmd string, args ...string) (string, error) {
+func WithNetNSByPid(pid int, cmd string, args ...string) ([]byte, error) {
 	return WithNetNS(NSPathByPid(pid), cmd, args...)
 }
 

net/route.go

@@ -13,7 +13,7 @@ import (
 func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
 	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && overlaps(route.Dst, subnet) {
-			return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
+			return fmt.Errorf("Network %s overlaps with existing route %s on host", subnet, route.Dst)
 		}
 		return nil
 	})
@@ -29,7 +29,7 @@ func overlaps(n1, n2 *net.IPNet) bool {
 func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
 	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && route.Dst.Contains(addr) {
-			return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
+			return fmt.Errorf("Address %s overlaps with existing route %s on host", addr, route.Dst)
 		}
 		return nil
 	})

net/uuid.go

@@ -0,0 +1,84 @@
+package net
+
+import (
+	"errors"
+	"io/ioutil"
+	"net"
+	"os"
+
+	"github.com/weaveworks/mesh"
+	"github.com/weaveworks/weave/db"
+)
+
+func getOldStyleSystemUUID() ([]byte, error) {
+	uuid, err := ioutil.ReadFile("/sys/class/dmi/id/product_uuid")
+	if os.IsNotExist(err) {
+		uuid, err = ioutil.ReadFile("/sys/hypervisor/uuid")
+	}
+	return uuid, err
+}
+
+func getSystemUUID(hostRoot string) ([]byte, error) {
+	uuid, err := getOldStyleSystemUUID()
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	machineid, err := ioutil.ReadFile(hostRoot + "/etc/machine-id")
+	if os.IsNotExist(err) {
+		machineid, err = ioutil.ReadFile(hostRoot + "/var/lib/dbus/machine-id")
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	if len(uuid) == 0 && len(machineid) == 0 {
+		return nil, errors.New("All system IDs are blank")
+	}
+	return append(machineid, uuid...), nil
+}
+
+func getPersistedPeerName(pathName string) (mesh.PeerName, error) {
+	d, err := db.NewBoltDBReadOnly(pathName)
+	if err != nil {
+		return mesh.UnknownPeerName, err
+	}
+	defer d.Close()
+	var peerName mesh.PeerName
+	nameFound, err := d.Load(db.NameIdent, &peerName)
+	if err != nil || !nameFound {
+		return mesh.UnknownPeerName, err
+	}
+	return peerName, nil
+}
+
+// GetSystemPeerName returns an ID derived from concatenated machine-id
+// (either systemd or dbus), the system (aka bios) UUID and the
+// hypervisor UUID.  It is tweaked and formatted to be usable as a mac address
+func GetSystemPeerName(dbPrefix, hostRoot string) (string, error) {
+	// Check if we have a persisted name that matches the old-style ID for this host
+	if oldUUID, err := getOldStyleSystemUUID(); err == nil {
+		pathName := dbPrefix + db.FileName
+		if _, err := os.Stat(pathName); err == nil {
+			persistedPeerName, err := getPersistedPeerName(pathName)
+			if err != nil && !os.IsNotExist(err) {
+				return "", err
+			}
+			if persistedPeerName == mesh.PeerNameFromBin(MACfromUUID(oldUUID)) {
+				return persistedPeerName.String(), nil
+			}
+		}
+	} else if !os.IsNotExist(err) {
+		return "", err
+	}
+	var mac net.HardwareAddr
+	if uuid, err := getSystemUUID(hostRoot); err == nil {
+		mac = MACfromUUID(uuid)
+	} else if !os.IsNotExist(err) {
+		return "", err
+	} else {
+		mac, err = RandomMAC()
+		if err != nil {
+			return "", err
+		}
+	}
+	return mac.String(), nil
+}

net/veth.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/coreos/go-iptables/iptables"
@@ -105,21 +106,26 @@ const (
 	vethPrefix = "v" + VethName // starts with "veth" to suppress UI notifications
 )
 
-func interfaceExistsInNamespace(ns netns.NsHandle, ifName string) bool {
-	err := WithNetNSUnsafe(ns, func() error {
-		_, err := netlink.LinkByName(ifName)
-		return err
-	})
+func interfaceExistsInNamespace(netNSPath string, ifName string) bool {
+	_, err := WithNetNS(netNSPath, "check-iface", ifName)
 	return err == nil
 }
 
-func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
+// NB: This function can be used only by a process that terminates immediately
+//     after calling the function as it changes netns via WithNetNSLinkUnsafe.
+func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
+	ns, err := netns.GetFromPath(netNSPath)
+	if err != nil {
+		return err
+	}
+	defer ns.Close()
+
 	ipt, err := iptables.New()
 	if err != nil {
 		return err
 	}
 
-	if !interfaceExistsInNamespace(ns, ifName) {
+	if !interfaceExistsInNamespace(netNSPath, ifName) {
 		maxIDLen := IFNAMSIZ - 1 - len(vethPrefix+"pl")
 		if len(id) > maxIDLen {
 			id = id[:maxIDLen] // trim passed ID if too long
@@ -129,18 +135,7 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
 			if err := netlink.LinkSetNsFd(veth, int(ns)); err != nil {
 				return fmt.Errorf("failed to move veth to container netns: %s", err)
 			}
-			if err := WithNetNSUnsafe(ns, func() error {
-				if err := netlink.LinkSetName(veth, ifName); err != nil {
-					return err
-				}
-				if err := ConfigureARPCache(ifName); err != nil {
-					return err
-				}
-				if err := ipt.Append("filter", "INPUT", "-i", ifName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil {
-					return err
-				}
-				return nil
-			}); err != nil {
+			if _, err := WithNetNS(netNSPath, "setup-iface", peerName, ifName); err != nil {
 				return fmt.Errorf("error setting up interface: %s", err)
 			}
 			return nil
@@ -201,7 +196,15 @@ func AttachContainer(ns netns.NsHandle, id, ifName, bridgeName string, mtu int,
 	return nil
 }
 
-func DetachContainer(ns netns.NsHandle, id, ifName string, cidrs []*net.IPNet) error {
+// NB: This function can be used only by a process that terminates immediately
+//     after calling the function as it changes netns via WithNetNSLinkUnsafe.
+func DetachContainer(netNSPath, id, ifName string, cidrs []*net.IPNet) error {
+	ns, err := netns.GetFromPath(netNSPath)
+	if err != nil {
+		return err
+	}
+	defer ns.Close()
+
 	ipt, err := iptables.New()
 	if err != nil {
 		return err
@@ -269,3 +272,39 @@ func subnets(addrs []netlink.Addr) map[string]struct{} {
 	}
 	return subnets
 }
+
+func addNatRule(ipt *iptables.IPTables, rulespec ...string) error {
+	// Loop until we get an exit code other than "temporarily unavailable"
+	for {
+		if err := ipt.AppendUnique("nat", "WEAVE", rulespec...); err != nil {
+			if ierr, ok := err.(*iptables.Error); ok {
+				if status, ok := ierr.ExitError.Sys().(syscall.WaitStatus); ok {
+					// (magic exit code 4 found in iptables source code; undocumented)
+					if status.ExitStatus() == 4 {
+						continue
+					}
+				}
+			}
+			return err
+		}
+		return nil
+	}
+}
+
+func ExposeNAT(ipnet net.IPNet) error {
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+	cidr := ipnet.String()
+	if err := addNatRule(ipt, "-s", cidr, "-d", "224.0.0.0/4", "-j", "RETURN"); err != nil {
+		return err
+	}
+	if err := addNatRule(ipt, "-d", cidr, "!", "-s", cidr, "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+	if err := addNatRule(ipt, "-s", cidr, "!", "-d", cidr, "-j", "MASQUERADE"); err != nil {
+		return err
+	}
+	return nil
+}

npc/analyser.go

@@ -0,0 +1,114 @@
+package npc
+
+import (
+	"fmt"
+
+	"k8s.io/client-go/pkg/api"
+	extnapi "k8s.io/client-go/pkg/apis/extensions/v1beta1"
+	"k8s.io/client-go/pkg/util/intstr"
+
+	"github.com/weaveworks/weave/npc/ipset"
+)
+
+func (ns *ns) analysePolicy(policy *extnapi.NetworkPolicy) (
+	rules map[string]*ruleSpec,
+	nsSelectors, podSelectors map[string]*selectorSpec,
+	err error) {
+
+	nsSelectors = make(map[string]*selectorSpec)
+	podSelectors = make(map[string]*selectorSpec)
+	rules = make(map[string]*ruleSpec)
+
+	dstSelector, err := newSelectorSpec(&policy.Spec.PodSelector, ns.name, ipset.HashIP)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	podSelectors[dstSelector.key] = dstSelector
+
+	for _, ingressRule := range policy.Spec.Ingress {
+		if ingressRule.Ports != nil && len(ingressRule.Ports) == 0 {
+			// Ports is empty, this rule matches no ports (no traffic matches).
+			continue
+		}
+
+		if ingressRule.From != nil && len(ingressRule.From) == 0 {
+			// From is empty, this rule matches no sources (no traffic matches).
+			continue
+		}
+
+		if ingressRule.From == nil {
+			// From is not provided, this rule matches all sources (traffic not restricted by source).
+			if ingressRule.Ports == nil {
+				// Ports is not provided, this rule matches all ports (traffic not restricted by port).
+				rule := newRuleSpec(nil, nil, dstSelector, nil)
+				rules[rule.key] = rule
+			} else {
+				// Ports is present and contains at least one item, then this rule allows traffic
+				// only if the traffic matches at least one port in the ports list.
+				withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) {
+					rule := newRuleSpec(&proto, nil, dstSelector, &port)
+					rules[rule.key] = rule
+				})
+			}
+		} else {
+			// From is present and contains at least on item, this rule allows traffic only if the
+			// traffic matches at least one item in the from list.
+			for _, peer := range ingressRule.From {
+				var srcSelector *selectorSpec
+				if peer.PodSelector != nil {
+					srcSelector, err = newSelectorSpec(peer.PodSelector, ns.name, ipset.HashIP)
+					if err != nil {
+						return nil, nil, nil, err
+					}
+					podSelectors[srcSelector.key] = srcSelector
+				}
+				if peer.NamespaceSelector != nil {
+					srcSelector, err = newSelectorSpec(peer.NamespaceSelector, "", ipset.ListSet)
+					if err != nil {
+						return nil, nil, nil, err
+					}
+					nsSelectors[srcSelector.key] = srcSelector
+				}
+
+				if ingressRule.Ports == nil {
+					// Ports is not provided, this rule matches all ports (traffic not restricted by port).
+					rule := newRuleSpec(nil, srcSelector, dstSelector, nil)
+					rules[rule.key] = rule
+				} else {
+					// Ports is present and contains at least one item, then this rule allows traffic
+					// only if the traffic matches at least one port in the ports list.
+					withNormalisedProtoAndPort(ingressRule.Ports, func(proto, port string) {
+						rule := newRuleSpec(&proto, srcSelector, dstSelector, &port)
+						rules[rule.key] = rule
+					})
+				}
+			}
+		}
+	}
+
+	return rules, nsSelectors, podSelectors, nil
+}
+
+func withNormalisedProtoAndPort(npps []extnapi.NetworkPolicyPort, f func(proto, port string)) {
+	for _, npp := range npps {
+		// If no proto is specified, default to TCP
+		proto := string(api.ProtocolTCP)
+		if npp.Protocol != nil {
+			proto = string(*npp.Protocol)
+		}
+
+		// If no port is specified, match any port. Let iptables executable handle
+		// service name resolution
+		port := "0:65535"
+		if npp.Port != nil {
+			switch npp.Port.Type {
+			case intstr.Int:
+				port = fmt.Sprintf("%d", npp.Port.IntVal)
+			case intstr.String:
+				port = npp.Port.StrVal
+			}
+		}
+
+		f(proto, port)
+	}
+}

npc/betatypes.go

@@ -0,0 +1,40 @@
+package npc
+
+// These types are defined in https://github.com/kubernetes/kubernetes/blob/master/docs/proposals/network-policy.md
+// but not in the k8s API yet. Copies are included here for decoding the `net.beta.kubernetes.io/network-policy`
+// annotation specified in the above document.
+
+type IngressIsolationPolicy string
+
+const (
+	// Deny all ingress traffic to pods in this namespace. Ingress means
+	// any incoming traffic to pods, whether that be from other pods within this namespace
+	// or any source outside of this namespace.
+	DefaultDeny IngressIsolationPolicy = "DefaultDeny"
+)
+
+// Standard NamespaceSpec object, modified to include a new
+// NamespaceNetworkPolicy field.
+type NamespaceSpec struct {
+	// This is a pointer so that it can be left undefined.
+	NetworkPolicy *NamespaceNetworkPolicy `json:"networkPolicy,omitempty"`
+}
+
+type NamespaceNetworkPolicy struct {
+	// Ingress configuration for this namespace.  This config is
+	// applied to all pods within this namespace. For now, only
+	// ingress is supported.  This field is optional - if not
+	// defined, then the cluster default for ingress is applied.
+	Ingress *NamespaceIngressPolicy `json:"ingress,omitempty"`
+}
+
+// Configuration for ingress to pods within this namespace.
+// For now, this only supports specifying an isolation policy.
+type NamespaceIngressPolicy struct {
+	// The isolation policy to apply to pods in this namespace.
+	// Currently this field only supports "DefaultDeny", but could
+	// be extended to support other policies in the future.  When set to DefaultDeny,
+	// pods in this namespace are denied ingress traffic by default.  When not defined,
+	// the cluster default ingress isolation policy is applied (currently allow all).
+	Isolation *IngressIsolationPolicy `json:"isolation,omitempty"`
+}

npc/constants.go

@@ -0,0 +1,9 @@
+package npc
+
+const (
+	TableFilter = "filter"
+
+	MainChain    = "WEAVE-NPC"
+	DefaultChain = "WEAVE-NPC-DEFAULT"
+	IngressChain = "WEAVE-NPC-INGRESS"
+)

npc/controller.go

@@ -0,0 +1,173 @@
+package npc
+
+import (
+	"sync"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
+	coreapi "k8s.io/client-go/pkg/api/v1"
+	extnapi "k8s.io/client-go/pkg/apis/extensions/v1beta1"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/npc/ipset"
+)
+
+type NetworkPolicyController interface {
+	AddNamespace(ns *coreapi.Namespace) error
+	UpdateNamespace(oldObj, newObj *coreapi.Namespace) error
+	DeleteNamespace(ns *coreapi.Namespace) error
+
+	AddPod(obj *coreapi.Pod) error
+	UpdatePod(oldObj, newObj *coreapi.Pod) error
+	DeletePod(obj *coreapi.Pod) error
+
+	AddNetworkPolicy(obj *extnapi.NetworkPolicy) error
+	UpdateNetworkPolicy(oldObj, newObj *extnapi.NetworkPolicy) error
+	DeleteNetworkPolicy(obj *extnapi.NetworkPolicy) error
+}
+
+type controller struct {
+	sync.Mutex
+
+	ipt *iptables.IPTables
+	ips ipset.Interface
+
+	nss         map[string]*ns // ns name -> ns struct
+	nsSelectors *selectorSet   // selector string -> nsSelector
+}
+
+func New(ipt *iptables.IPTables, ips ipset.Interface) NetworkPolicyController {
+	c := &controller{
+		ipt: ipt,
+		ips: ips,
+		nss: make(map[string]*ns)}
+
+	c.nsSelectors = newSelectorSet(ips, c.onNewNsSelector)
+
+	return c
+}
+
+func (npc *controller) onNewNsSelector(selector *selector) error {
+	for _, ns := range npc.nss {
+		if ns.namespace != nil {
+			if selector.matches(ns.namespace.ObjectMeta.Labels) {
+				if err := ns.ips.AddEntry(selector.spec.ipsetName, string(ns.allPods.ipsetName)); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (npc *controller) withNS(name string, f func(ns *ns) error) error {
+	ns, found := npc.nss[name]
+	if !found {
+		newNs, err := newNS(name, npc.ipt, npc.ips, npc.nsSelectors)
+		if err != nil {
+			return err
+		}
+		npc.nss[name] = newNs
+		ns = newNs
+	}
+	if err := f(ns); err != nil {
+		return err
+	}
+	if ns.empty() {
+		if err := ns.destroy(); err != nil {
+			return err
+		}
+		delete(npc.nss, name)
+	}
+	return nil
+}
+
+func (npc *controller) AddPod(obj *coreapi.Pod) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Debugf("EVENT AddPod %s", js(obj))
+	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
+		return errors.Wrap(ns.addPod(obj), "add pod")
+	})
+}
+
+func (npc *controller) UpdatePod(oldObj, newObj *coreapi.Pod) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Debugf("EVENT UpdatePod %s %s", js(oldObj), js(newObj))
+	return npc.withNS(oldObj.ObjectMeta.Namespace, func(ns *ns) error {
+		return errors.Wrap(ns.updatePod(oldObj, newObj), "update pod")
+	})
+}
+
+func (npc *controller) DeletePod(obj *coreapi.Pod) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Debugf("EVENT DeletePod %s", js(obj))
+	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
+		return errors.Wrap(ns.deletePod(obj), "delete pod")
+	})
+}
+
+func (npc *controller) AddNetworkPolicy(obj *extnapi.NetworkPolicy) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Infof("EVENT AddNetworkPolicy %s", js(obj))
+	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
+		return errors.Wrap(ns.addNetworkPolicy(obj), "add network policy")
+	})
+}
+
+func (npc *controller) UpdateNetworkPolicy(oldObj, newObj *extnapi.NetworkPolicy) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Infof("EVENT UpdateNetworkPolicy %s %s", js(oldObj), js(newObj))
+	return npc.withNS(oldObj.ObjectMeta.Namespace, func(ns *ns) error {
+		return errors.Wrap(ns.updateNetworkPolicy(oldObj, newObj), "update network policy")
+	})
+}
+
+func (npc *controller) DeleteNetworkPolicy(obj *extnapi.NetworkPolicy) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Infof("EVENT DeleteNetworkPolicy %s", js(obj))
+	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
+		return errors.Wrap(ns.deleteNetworkPolicy(obj), "delete network policy")
+	})
+}
+
+func (npc *controller) AddNamespace(obj *coreapi.Namespace) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Infof("EVENT AddNamespace %s", js(obj))
+	return npc.withNS(obj.ObjectMeta.Name, func(ns *ns) error {
+		return errors.Wrap(ns.addNamespace(obj), "add namespace")
+	})
+}
+
+func (npc *controller) UpdateNamespace(oldObj, newObj *coreapi.Namespace) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Infof("EVENT UpdateNamespace %s %s", js(oldObj), js(newObj))
+	return npc.withNS(oldObj.ObjectMeta.Name, func(ns *ns) error {
+		return errors.Wrap(ns.updateNamespace(oldObj, newObj), "update namespace")
+	})
+}
+
+func (npc *controller) DeleteNamespace(obj *coreapi.Namespace) error {
+	npc.Lock()
+	defer npc.Unlock()
+
+	common.Log.Infof("EVENT DeleteNamespace %s", js(obj))
+	return npc.withNS(obj.ObjectMeta.Name, func(ns *ns) error {
+		return errors.Wrap(ns.deleteNamespace(obj), "delete namespace")
+	})
+}

npc/ipset/ipset.go

@@ -0,0 +1,115 @@
+package ipset
+
+import (
+	"os/exec"
+
+	"github.com/pkg/errors"
+)
+
+type Name string
+
+type Type string
+
+const (
+	ListSet = Type("list:set")
+	HashIP  = Type("hash:ip")
+)
+
+type Interface interface {
+	Create(ipsetName Name, ipsetType Type) error
+	AddEntry(ipsetName Name, entry string) error
+	DelEntry(ipsetName Name, entry string) error
+	Flush(ipsetName Name) error
+	Destroy(ipsetName Name) error
+
+	FlushAll() error
+	DestroyAll() error
+}
+
+type ipset struct {
+	refCount
+}
+
+func New() Interface {
+	return &ipset{refCount: newRefCount()}
+}
+
+func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
+	return doExec("create", string(ipsetName), string(ipsetType))
+}
+
+func (i *ipset) AddEntry(ipsetName Name, entry string) error {
+	if i.inc(ipsetName, entry) > 1 { // already in the set
+		return nil
+	}
+	return doExec("add", string(ipsetName), entry)
+}
+
+func (i *ipset) DelEntry(ipsetName Name, entry string) error {
+	if i.dec(ipsetName, entry) > 0 { // still needed
+		return nil
+	}
+	return doExec("del", string(ipsetName), entry)
+}
+
+func (i *ipset) Flush(ipsetName Name) error {
+	i.removeSet(ipsetName)
+	return doExec("flush", string(ipsetName))
+}
+
+func (i *ipset) FlushAll() error {
+	i.refCount = newRefCount()
+	return doExec("flush")
+}
+
+func (i *ipset) Destroy(ipsetName Name) error {
+	i.removeSet(ipsetName)
+	return doExec("destroy", string(ipsetName))
+}
+
+func (i *ipset) DestroyAll() error {
+	i.refCount = newRefCount()
+	return doExec("destroy")
+}
+
+func doExec(args ...string) error {
+	if output, err := exec.Command("ipset", args...).CombinedOutput(); err != nil {
+		return errors.Wrapf(err, "ipset %v failed: %s", args, output)
+	}
+	return nil
+}
+
+// Reference-counting
+type key struct {
+	ipsetName Name
+	entry     string
+}
+
+// note no locking is required as all operations are serialised in the controller
+type refCount struct {
+	ref map[key]int
+}
+
+func newRefCount() refCount {
+	return refCount{ref: make(map[key]int)}
+}
+
+func (rc *refCount) inc(ipsetName Name, entry string) int {
+	k := key{ipsetName, entry}
+	rc.ref[k]++
+	return rc.ref[k]
+}
+
+func (rc *refCount) dec(ipsetName Name, entry string) int {
+	k := key{ipsetName, entry}
+	rc.ref[k]--
+	return rc.ref[k]
+}
+
+func (rc *refCount) removeSet(ipsetName Name) {
+	for k := range rc.ref {
+		if k.ipsetName == ipsetName {
+			delete(rc.ref, k)
+		}
+	}
+}

npc/json.go

@@ -0,0 +1,10 @@
+package npc
+
+import (
+	"encoding/json"
+)
+
+func js(v interface{}) string {
+	a, _ := json.Marshal(v)
+	return string(a)
+}

npc/metrics/metrics.go

@@ -0,0 +1,111 @@
+package metrics
+
+import (
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/google/gopacket/pcapgo"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/weaveworks/weave/common"
+)
+
+var (
+	blockedConnections = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "weavenpc_blocked_connections_total",
+			Help: "Connection attempts blocked by policy controller.",
+		},
+		[]string{"protocol", "dport"},
+	)
+)
+
+func gatherMetrics() {
+	pipe, err := os.Open("/var/log/ulogd.pcap")
+	if err != nil {
+		common.Log.Fatalf("Failed to open pcap: %v", err)
+	}
+
+	reader, err := pcapgo.NewReader(pipe)
+	if err != nil {
+		common.Log.Fatalf("Failed to read pcap header: %v", err)
+	}
+
+	for {
+		data, _, err := reader.ReadPacketData()
+		if err != nil {
+			common.Log.Fatalf("Failed to read pcap packet: %v", err)
+		}
+
+		packet := gopacket.NewPacket(data, layers.LayerTypeIPv4, gopacket.Default)
+
+		if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
+			tcp, _ := tcpLayer.(*layers.TCP)
+			if tcp.SYN && !tcp.ACK { // Only plain SYN constitutes a NEW TCP connection
+				blockedConnections.With(prometheus.Labels{"protocol": "tcp", "dport": strconv.Itoa(int(tcp.DstPort))}).Inc()
+				common.Log.Warnf("TCP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), tcp.SrcPort, dstIP(packet), tcp.DstPort)
+				continue
+			}
+		}
+
+		if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
+			udp, _ := udpLayer.(*layers.UDP)
+			blockedConnections.With(prometheus.Labels{"protocol": "udp", "dport": strconv.Itoa(int(udp.DstPort))}).Inc()
+			common.Log.Warnf("UDP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), udp.SrcPort, dstIP(packet), udp.DstPort)
+			continue
+		}
+	}
+}
+
+const unknownIP string = "<unknown IP>"
+
+func srcIP(packet gopacket.Packet) string {
+	if layer := packet.Layer(layers.LayerTypeIPv4); layer != nil {
+		if ip, ok := layer.(*layers.IPv4); ok {
+			return ip.SrcIP.String()
+		}
+	}
+	if layer := packet.Layer(layers.LayerTypeIPv6); layer != nil {
+		if ip, ok := layer.(*layers.IPv6); ok {
+			return ip.SrcIP.String()
+		}
+	}
+	return unknownIP
+}
+
+func dstIP(packet gopacket.Packet) string {
+	if layer := packet.Layer(layers.LayerTypeIPv4); layer != nil {
+		if ip, ok := layer.(*layers.IPv4); ok {
+			return ip.DstIP.String()
+		}
+	}
+	if layer := packet.Layer(layers.LayerTypeIPv6); layer != nil {
+		if ip, ok := layer.(*layers.IPv6); ok {
+			return ip.DstIP.String()
+		}
+	}
+	return unknownIP
+}
+
+func Start(addr string) error {
+	if err := prometheus.Register(blockedConnections); err != nil {
+		return err
+	}
+
+	http.Handle("/metrics", promhttp.Handler())
+
+	go func() {
+		common.Log.Infof("Serving /metrics on %s", addr)
+		if err := http.ListenAndServe(addr, nil); err != nil {
+			common.Log.Fatalf("Failed to bind metrics server: %v", err)
+		}
+	}()
+
+	go gatherMetrics()
+
+	return nil
+}

npc/namespace.go

@@ -0,0 +1,338 @@
+package npc
+
+import (
+	"encoding/json"
+
+	"github.com/coreos/go-iptables/iptables"
+	"k8s.io/client-go/pkg/api/unversioned"
+	coreapi "k8s.io/client-go/pkg/api/v1"
+	extnapi "k8s.io/client-go/pkg/apis/extensions/v1beta1"
+	"k8s.io/client-go/pkg/types"
+	"k8s.io/client-go/pkg/util/uuid"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/npc/ipset"
+)
+
+type ns struct {
+	ipt *iptables.IPTables // interface to iptables
+	ips ipset.Interface    // interface to ipset
+
+	name      string                               // k8s Namespace name
+	namespace *coreapi.Namespace                   // k8s Namespace object
+	pods      map[types.UID]*coreapi.Pod           // k8s Pod objects by UID
+	policies  map[types.UID]*extnapi.NetworkPolicy // k8s NetworkPolicy objects by UID
+
+	uid     types.UID     // surrogate UID to own allPods selector
+	allPods *selectorSpec // hash:ip ipset of all pod IPs in this namespace
+
+	nsSelectors  *selectorSet
+	podSelectors *selectorSet
+	rules        *ruleSet
+}
+
+func newNS(name string, ipt *iptables.IPTables, ips ipset.Interface, nsSelectors *selectorSet) (*ns, error) {
+	allPods, err := newSelectorSpec(&unversioned.LabelSelector{}, name, ipset.HashIP)
+	if err != nil {
+		return nil, err
+	}
+
+	ns := &ns{
+		ipt:         ipt,
+		ips:         ips,
+		name:        name,
+		pods:        make(map[types.UID]*coreapi.Pod),
+		policies:    make(map[types.UID]*extnapi.NetworkPolicy),
+		uid:         uuid.NewUUID(),
+		allPods:     allPods,
+		nsSelectors: nsSelectors,
+		rules:       newRuleSet(ipt)}
+
+	ns.podSelectors = newSelectorSet(ips, ns.onNewPodSelector)
+
+	if err := ns.podSelectors.provision(ns.uid, nil, map[string]*selectorSpec{ns.allPods.key: ns.allPods}); err != nil {
+		return nil, err
+	}
+
+	return ns, nil
+}
+
+func (ns *ns) empty() bool {
+	return len(ns.pods) == 0 && len(ns.policies) == 0 && ns.namespace == nil
+}
+
+func (ns *ns) destroy() error {
+	if err := ns.podSelectors.deprovision(ns.uid, map[string]*selectorSpec{ns.allPods.key: ns.allPods}, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ns *ns) onNewPodSelector(selector *selector) error {
+	for _, pod := range ns.pods {
+		if hasIP(pod) {
+			if selector.matches(pod.ObjectMeta.Labels) {
+				if err := selector.addEntry(pod.Status.PodIP); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (ns *ns) addPod(obj *coreapi.Pod) error {
+	ns.pods[obj.ObjectMeta.UID] = obj
+
+	if !hasIP(obj) {
+		return nil
+	}
+
+	return ns.podSelectors.addToMatching(obj.ObjectMeta.Labels, obj.Status.PodIP)
+}
+
+func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
+	delete(ns.pods, oldObj.ObjectMeta.UID)
+	ns.pods[newObj.ObjectMeta.UID] = newObj
+
+	if !hasIP(oldObj) && !hasIP(newObj) {
+		return nil
+	}
+
+	if hasIP(oldObj) && !hasIP(newObj) {
+		return ns.podSelectors.delFromMatching(oldObj.ObjectMeta.Labels, oldObj.Status.PodIP)
+	}
+
+	if !hasIP(oldObj) && hasIP(newObj) {
+		return ns.podSelectors.addToMatching(newObj.ObjectMeta.Labels, newObj.Status.PodIP)
+	}
+
+	if !equals(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) ||
+		oldObj.Status.PodIP != newObj.Status.PodIP {
+
+		for _, ps := range ns.podSelectors.entries {
+			oldMatch := ps.matches(oldObj.ObjectMeta.Labels)
+			newMatch := ps.matches(newObj.ObjectMeta.Labels)
+			if oldMatch == newMatch && oldObj.Status.PodIP == newObj.Status.PodIP {
+				continue
+			}
+			if oldMatch {
+				if err := ps.delEntry(oldObj.Status.PodIP); err != nil {
+					return err
+				}
+			}
+			if newMatch {
+				if err := ps.addEntry(newObj.Status.PodIP); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (ns *ns) deletePod(obj *coreapi.Pod) error {
+	delete(ns.pods, obj.ObjectMeta.UID)
+
+	if !hasIP(obj) {
+		return nil
+	}
+
+	return ns.podSelectors.delFromMatching(obj.ObjectMeta.Labels, obj.Status.PodIP)
+}
+
+func (ns *ns) addNetworkPolicy(obj *extnapi.NetworkPolicy) error {
+	ns.policies[obj.ObjectMeta.UID] = obj
+
+	// Analyse policy, determine which rules and ipsets are required
+	rules, nsSelectors, podSelectors, err := ns.analysePolicy(obj)
+	if err != nil {
+		return err
+	}
+
+	// Provision required resources in dependency order
+	if err := ns.nsSelectors.provision(obj.ObjectMeta.UID, nil, nsSelectors); err != nil {
+		return err
+	}
+	if err := ns.podSelectors.provision(obj.ObjectMeta.UID, nil, podSelectors); err != nil {
+		return err
+	}
+	if err := ns.rules.provision(obj.ObjectMeta.UID, nil, rules); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ns *ns) updateNetworkPolicy(oldObj, newObj *extnapi.NetworkPolicy) error {
+	delete(ns.policies, oldObj.ObjectMeta.UID)
+	ns.policies[newObj.ObjectMeta.UID] = newObj
+
+	// Analyse the old and the new policy so we can determine differences
+	oldRules, oldNsSelectors, oldPodSelectors, err := ns.analysePolicy(oldObj)
+	if err != nil {
+		return err
+	}
+	newRules, newNsSelectors, newPodSelectors, err := ns.analysePolicy(newObj)
+	if err != nil {
+		return err
+	}
+
+	// Deprovision unused and provision newly required resources in dependency order
+	if err := ns.rules.deprovision(oldObj.ObjectMeta.UID, oldRules, newRules); err != nil {
+		return err
+	}
+	if err := ns.nsSelectors.deprovision(oldObj.ObjectMeta.UID, oldNsSelectors, newNsSelectors); err != nil {
+		return err
+	}
+	if err := ns.podSelectors.deprovision(oldObj.ObjectMeta.UID, oldPodSelectors, newPodSelectors); err != nil {
+		return err
+	}
+	if err := ns.nsSelectors.provision(oldObj.ObjectMeta.UID, oldNsSelectors, newNsSelectors); err != nil {
+		return err
+	}
+	if err := ns.podSelectors.provision(oldObj.ObjectMeta.UID, oldPodSelectors, newPodSelectors); err != nil {
+		return err
+	}
+	if err := ns.rules.provision(oldObj.ObjectMeta.UID, oldRules, newRules); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ns *ns) deleteNetworkPolicy(obj *extnapi.NetworkPolicy) error {
+	delete(ns.policies, obj.ObjectMeta.UID)
+
+	// Analyse network policy to free resources
+	rules, nsSelectors, podSelectors, err := ns.analysePolicy(obj)
+	if err != nil {
+		return err
+	}
+
+	// Deprovision unused resources in dependency order
+	if err := ns.rules.deprovision(obj.ObjectMeta.UID, rules, nil); err != nil {
+		return err
+	}
+	if err := ns.nsSelectors.deprovision(obj.ObjectMeta.UID, nsSelectors, nil); err != nil {
+		return err
+	}
+	if err := ns.podSelectors.deprovision(obj.ObjectMeta.UID, podSelectors, nil); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func bypassRule(nsIpsetName ipset.Name) []string {
+	return []string{"-m", "set", "--match-set", string(nsIpsetName), "dst", "-j", "ACCEPT"}
+}
+
+func (ns *ns) ensureBypassRule(nsIpsetName ipset.Name) error {
+	return ns.ipt.Append(TableFilter, DefaultChain, bypassRule(ns.allPods.ipsetName)...)
+}
+
+func (ns *ns) deleteBypassRule(nsIpsetName ipset.Name) error {
+	return ns.ipt.Delete(TableFilter, DefaultChain, bypassRule(ns.allPods.ipsetName)...)
+}
+
+func (ns *ns) addNamespace(obj *coreapi.Namespace) error {
+	ns.namespace = obj
+
+	// Insert a rule to bypass policies if namespace is DefaultAllow
+	if !isDefaultDeny(obj) {
+		return ns.ensureBypassRule(ns.allPods.ipsetName)
+	}
+
+	// Add namespace ipset to matching namespace selectors
+	return ns.nsSelectors.addToMatching(obj.ObjectMeta.Labels, string(ns.allPods.ipsetName))
+}
+
+func (ns *ns) updateNamespace(oldObj, newObj *coreapi.Namespace) error {
+	ns.namespace = newObj
+
+	// Update bypass rule if ingress default has changed
+	oldDefaultDeny := isDefaultDeny(oldObj)
+	newDefaultDeny := isDefaultDeny(newObj)
+
+	if oldDefaultDeny != newDefaultDeny {
+		if oldDefaultDeny {
+			return ns.ensureBypassRule(ns.allPods.ipsetName)
+		}
+		if newDefaultDeny {
+			return ns.deleteBypassRule(ns.allPods.ipsetName)
+		}
+	}
+
+	// Re-evaluate namespace selector membership if labels have changed
+	if !equals(oldObj.ObjectMeta.Labels, newObj.ObjectMeta.Labels) {
+		for _, selector := range ns.nsSelectors.entries {
+			oldMatch := selector.matches(oldObj.ObjectMeta.Labels)
+			newMatch := selector.matches(newObj.ObjectMeta.Labels)
+			if oldMatch == newMatch {
+				continue
+			}
+			if oldMatch {
+				if err := selector.delEntry(string(ns.allPods.ipsetName)); err != nil {
+					return err
+				}
+			}
+			if newMatch {
+				if err := selector.addEntry(string(ns.allPods.ipsetName)); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func (ns *ns) deleteNamespace(obj *coreapi.Namespace) error {
+	ns.namespace = nil
+
+	// Remove bypass rule
+	if !isDefaultDeny(obj) {
+		return ns.deleteBypassRule(ns.allPods.ipsetName)
+	}
+
+	// Remove namespace ipset from any matching namespace selectors
+	return ns.nsSelectors.delFromMatching(obj.ObjectMeta.Labels, string(ns.allPods.ipsetName))
+}
+
+func hasIP(pod *coreapi.Pod) bool {
+	// Ensure pod isn't dead, has an IP address and isn't sharing the host network namespace
+	return pod.Status.Phase != "Succeeded" && pod.Status.Phase != "Failed" &&
+		len(pod.Status.PodIP) > 0 && !pod.Spec.HostNetwork
+}
+
+func equals(a, b map[string]string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for ak, av := range a {
+		if b[ak] != av {
+			return false
+		}
+	}
+	return true
+}
+
+func isDefaultDeny(namespace *coreapi.Namespace) bool {
+	nnpJSON, found := namespace.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"]
+	if !found {
+		return false
+	}
+
+	var nnp NamespaceNetworkPolicy
+	if err := json.Unmarshal([]byte(nnpJSON), &nnp); err != nil {
+		common.Log.Warn("Ignoring network policy annotation: unmarshal failed:", err)
+		// If we can't understand the annotation, behave as if it isn't present
+		return false
+	}
+
+	return nnp.Ingress != nil &&
+		nnp.Ingress.Isolation != nil &&
+		*(nnp.Ingress.Isolation) == DefaultDeny
+}

npc/rule.go

@@ -0,0 +1,78 @@
+package npc
+
+import (
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	"k8s.io/client-go/pkg/types"
+
+	"github.com/weaveworks/weave/common"
+)
+
+type ruleSpec struct {
+	key  string
+	args []string
+}
+
+func newRuleSpec(proto *string, srcHost *selectorSpec, dstHost *selectorSpec, dstPort *string) *ruleSpec {
+	args := []string{}
+	if proto != nil {
+		args = append(args, "-p", *proto)
+	}
+	if srcHost != nil {
+		args = append(args, "-m", "set", "--match-set", string(srcHost.ipsetName), "src")
+	}
+	if dstHost != nil {
+		args = append(args, "-m", "set", "--match-set", string(dstHost.ipsetName), "dst")
+	}
+	if dstPort != nil {
+		args = append(args, "--dport", *dstPort)
+	}
+	args = append(args, "-j", "ACCEPT")
+	key := strings.Join(args, " ")
+
+	return &ruleSpec{key, args}
+}
+
+type ruleSet struct {
+	ipt   *iptables.IPTables
+	users map[string]map[types.UID]struct{}
+}
+
+func newRuleSet(ipt *iptables.IPTables) *ruleSet {
+	return &ruleSet{ipt, make(map[string]map[types.UID]struct{})}
+}
+
+func (rs *ruleSet) deprovision(user types.UID, current, desired map[string]*ruleSpec) error {
+	for key, spec := range current {
+		if _, found := desired[key]; !found {
+			delete(rs.users[key], user)
+			if len(rs.users[key]) == 0 {
+				common.Log.Infof("deleting rule: %v", spec.args)
+				if err := rs.ipt.Delete(TableFilter, IngressChain, spec.args...); err != nil {
+					return err
+				}
+				delete(rs.users, key)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (rs *ruleSet) provision(user types.UID, current, desired map[string]*ruleSpec) error {
+	for key, spec := range desired {
+		if _, found := current[key]; !found {
+			if _, found := rs.users[key]; !found {
+				common.Log.Infof("adding rule: %v", spec.args)
+				if err := rs.ipt.Append(TableFilter, IngressChain, spec.args...); err != nil {
+					return err
+				}
+				rs.users[key] = make(map[types.UID]struct{})
+			}
+			rs.users[key][user] = struct{}{}
+		}
+	}
+
+	return nil
+}

npc/selector.go

@@ -0,0 +1,130 @@
+package npc
+
+import (
+	"k8s.io/client-go/pkg/api/unversioned"
+	"k8s.io/client-go/pkg/labels"
+	"k8s.io/client-go/pkg/types"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/npc/ipset"
+)
+
+type selectorSpec struct {
+	key      string          // string representation (for hash keying/equality comparison)
+	selector labels.Selector // k8s Selector object (for matching)
+
+	ipsetType ipset.Type // type of ipset to provision
+	ipsetName ipset.Name // generated ipset name
+}
+
+func newSelectorSpec(json *unversioned.LabelSelector, nsName string, ipsetType ipset.Type) (*selectorSpec, error) {
+	selector, err := unversioned.LabelSelectorAsSelector(json)
+	if err != nil {
+		return nil, err
+	}
+	key := selector.String()
+	return &selectorSpec{
+		key:      key,
+		selector: selector,
+		// We prefix the selector string with the namespace name when generating
+		// the shortname because you can specify the same selector in multiple
+		// namespaces - we need those to map to distinct ipsets
+		ipsetName: ipset.Name("weave-" + shortName(nsName+":"+key)),
+		ipsetType: ipsetType}, nil
+}
+
+type selector struct {
+	ips  ipset.Interface
+	spec *selectorSpec
+}
+
+func (s *selector) matches(labelMap map[string]string) bool {
+	return s.spec.selector.Matches(labels.Set(labelMap))
+}
+
+func (s *selector) addEntry(entry string) error {
+	common.Log.Infof("adding entry %s to %s", entry, s.spec.ipsetName)
+	return s.ips.AddEntry(s.spec.ipsetName, entry)
+}
+
+func (s *selector) delEntry(entry string) error {
+	common.Log.Infof("deleting entry %s from %s", entry, s.spec.ipsetName)
+	return s.ips.DelEntry(s.spec.ipsetName, entry)
+}
+
+type selectorFn func(selector *selector) error
+
+type selectorSet struct {
+	ips           ipset.Interface
+	onNewSelector selectorFn
+	users         map[string]map[types.UID]struct{} // list of users per selector
+	entries       map[string]*selector
+}
+
+func newSelectorSet(ips ipset.Interface, onNewSelector selectorFn) *selectorSet {
+	return &selectorSet{
+		ips:           ips,
+		onNewSelector: onNewSelector,
+		users:         make(map[string]map[types.UID]struct{}),
+		entries:       make(map[string]*selector)}
+}
+
+func (ss *selectorSet) addToMatching(labelMap map[string]string, entry string) error {
+	for _, s := range ss.entries {
+		if s.matches(labelMap) {
+			if err := s.addEntry(entry); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) delFromMatching(labelMap map[string]string, entry string) error {
+	for _, s := range ss.entries {
+		if s.matches(labelMap) {
+			if err := s.delEntry(entry); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) deprovision(user types.UID, current, desired map[string]*selectorSpec) error {
+	for key, spec := range current {
+		if _, found := desired[key]; !found {
+			delete(ss.users[key], user)
+			if len(ss.users[key]) == 0 {
+				common.Log.Infof("destroying ipset: %#v", spec)
+				if err := ss.ips.Destroy(spec.ipsetName); err != nil {
+					return err
+				}
+				delete(ss.entries, key)
+				delete(ss.users, key)
+			}
+		}
+	}
+	return nil
+}
+
+func (ss *selectorSet) provision(user types.UID, current, desired map[string]*selectorSpec) error {
+	for key, spec := range desired {
+		if _, found := current[key]; !found {
+			if _, found := ss.users[key]; !found {
+				common.Log.Infof("creating ipset: %#v", spec)
+				if err := ss.ips.Create(spec.ipsetName, spec.ipsetType); err != nil {
+					return err
+				}
+				selector := &selector{ss.ips, spec}
+				if err := ss.onNewSelector(selector); err != nil {
+					return err
+				}
+				ss.users[key] = make(map[types.UID]struct{})
+				ss.entries[key] = selector
+			}
+			ss.users[key][user] = struct{}{}
+		}
+	}
+	return nil
+}

npc/shortname.go

@@ -0,0 +1,36 @@
+package npc
+
+import (
+	"crypto/sha1"
+	"math/big"
+)
+
+const (
+	// This array:
+	// * Must include only printable UTF8 characters that are represented with a single byte
+	// * Must be at least of length 85 (`len("weave-") + l(2^160)/l(85)` equals 31, the maximum ipset name length)
+	// * Must not include commas as those are treated specially by `ipset add` when adding a named set to a list:set
+	// * Should not include space for readability
+	// * Should not include single quote or backslash to be nice to shell users
+	ShortNameSymbols = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789(){}[]<>_$%^&*|/?.;:@#~"
+)
+
+// sha1 hash an arbitrary string and represent it using the full range of
+// printable ascii characters
+func shortName(arbitrary string) string {
+	symbols := []byte(ShortNameSymbols)
+	sum := sha1.Sum([]byte(arbitrary))
+	i := big.NewInt(0).SetBytes(sum[:])
+	base := big.NewInt(int64(len(symbols)))
+	zero := big.NewInt(0)
+	result := make([]byte, 0)
+
+	for i.Cmp(zero) > 0 {
+		remainder := new(big.Int).Mod(i, base)
+		i.Sub(i, remainder)
+		i.Div(i, base)
+		result = append(result, symbols[remainder.Int64()])
+	}
+
+	return string(result)
+}

npc/ulogd/ulogd.go

@@ -0,0 +1,34 @@
+package ulogd
+
+import (
+	"io"
+	"os"
+	"os/exec"
+	"syscall"
+
+	"github.com/weaveworks/weave/common"
+)
+
+func waitForExit(cmd *exec.Cmd) {
+	if err := cmd.Wait(); err != nil {
+		common.Log.Fatalf("ulogd terminated: %v", err)
+	}
+	common.Log.Fatal("ulogd terminated normally")
+}
+
+func Start() error {
+	cmd := exec.Command("/usr/sbin/ulogd", "-v")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Pdeathsig: syscall.SIGKILL,
+	}
+	stdout, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	go io.Copy(os.Stdout, stdout)
+	go waitForExit(cmd)
+	return nil
+}

plugin/ipam/driver.go

@@ -72,9 +72,8 @@ func splitPoolID(poolID string) (subnet, iprange *net.IPNet, err error) {
 func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
 	i.logReq("RequestAddress", poolID, address, options)
 	defer func() { i.logRes("RequestAddress", err, ip) }()
-	// If we pass magic string "_" to weave IPAM it stores the address under its own string
 	if poolID == "weavepool" { // old-style
-		ip, err = i.weave.AllocateIP("_")
+		ip, err = i.weave.AllocateIP(api.NoContainerID)
 		return
 	}
 	subnet, iprange, err := splitPoolID(poolID)
@@ -83,12 +82,12 @@ func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]
 	}
 	if address != nil { // try to claim specific address requested
 		ip = &net.IPNet{IP: address, Mask: subnet.Mask}
-		if err = i.weave.ClaimIP("_", ip); err != nil {
+		if err = i.weave.ClaimIP(api.NoContainerID, ip); err != nil {
 			return
 		}
 	} else {
 		// We are lying slightly to IPAM here: the range is not a subnet
-		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
+		if ip, err = i.weave.AllocateIPInSubnet(api.NoContainerID, iprange); err != nil {
 			return
 		}
 		ip.Mask = subnet.Mask // fix up the subnet we lied about

plugin/net/cni.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"syscall"
 
 	"github.com/appc/cni/pkg/ipam"
 	"github.com/appc/cni/pkg/skel"
@@ -12,7 +13,6 @@ import (
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	weaveapi "github.com/weaveworks/weave/api"
-	"github.com/weaveworks/weave/common"
 	weavenet "github.com/weaveworks/weave/net"
 	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
 )
@@ -73,8 +73,8 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 
 	// If config says nothing about routes or gateway, default one will be via the bridge
 	if result.IP4.Routes == nil && result.IP4.Gateway == nil {
-		bridgeIP, err := findBridgeIP(conf.BrName, result.IP4.IP)
-		if err == errBridgeNoIP {
+		bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &result.IP4.IP)
+		if err == weavenet.ErrBridgeNoIP {
 			bridgeArgs := *args
 			bridgeArgs.ContainerID = "weave:expose"
 			bridgeIPResult, err := c.getIP(conf.IPAM.Type, &bridgeArgs)
@@ -84,6 +84,9 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 			if err := assignBridgeIP(conf.BrName, bridgeIPResult.IP4.IP); err != nil {
 				return fmt.Errorf("unable to assign IP address to bridge: %s", err)
 			}
+			if err := weavenet.ExposeNAT(bridgeIPResult.IP4.IP); err != nil {
+				return fmt.Errorf("unable to create NAT rules: %s", err)
+			}
 			bridgeIP = bridgeIPResult.IP4.IP.IP
 		} else if err != nil {
 			return err
@@ -93,7 +96,7 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 
 	ns, err := netns.GetFromPath(args.Netns)
 	if err != nil {
-		return err
+		return fmt.Errorf("error accessing namespace %q: %s", args.Netns, err)
 	}
 	defer ns.Close()
 
@@ -107,7 +110,7 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 		id = fmt.Sprintf("%x", data)
 	}
 
-	if err := weavenet.AttachContainer(ns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil {
+	if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil {
 		return err
 	}
 	if err := weavenet.WithNetNSLinkUnsafe(ns, args.IfName, func(link netlink.Link) error {
@@ -151,50 +154,24 @@ func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
 		return err
 	}
 	if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil {
+		// Treat as non-error if this address is already there
+		// - maybe another copy of this program just added it
+		if err == syscall.Errno(syscall.EEXIST) {
+			return nil
+		}
 		return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err)
 	}
 	return nil
 }
 
-var errBridgeNoIP = fmt.Errorf("Bridge has no IP address")
-
-func findBridgeIP(bridgeName string, subnet net.IPNet) (net.IP, error) {
-	netdev, err := common.GetBridgeNetDev(bridgeName)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get netdev for %q bridge: %s", bridgeName, err)
-	}
-	if len(netdev.CIDRs) == 0 {
-		return nil, errBridgeNoIP
-	}
-	for _, cidr := range netdev.CIDRs {
-		if subnet.Contains(cidr.IP) {
-			return cidr.IP, nil
-		}
-	}
-	// None in the required subnet; just return the first one
-	return netdev.CIDRs[0].IP, nil
-}
-
 func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
 	conf, err := loadNetConf(args.StdinData)
 	if err != nil {
 		return err
 	}
 
-	ns, err := netns.GetFromPath(args.Netns)
-	if err != nil {
-		return err
-	}
-	defer ns.Close()
-	err = weavenet.WithNetNSUnsafe(ns, func() error {
-		link, err := netlink.LinkByName(args.IfName)
-		if err != nil {
-			return err
-		}
-		return netlink.LinkDel(link)
-	})
-	if err != nil {
-		return fmt.Errorf("error removing interface: %s", err)
+	if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil {
+		return fmt.Errorf("error removing interface %q: %s", args.IfName, err)
 	}
 
 	// Default IPAM is Weave's own

prog/kube-peers/main.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"k8s.io/client-go/kubernetes"
+	api "k8s.io/client-go/pkg/api/v1"
+	"k8s.io/client-go/rest"
+)
+
+func getKubePeers() ([]string, error) {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	c, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+	nodeList, err := c.Nodes().List(api.ListOptions{})
+	if err != nil {
+		// Fallback for cases (e.g. from kube-up.sh) where kube-proxy is not running on master
+		config.Host = "http://localhost:8080"
+		log.Print("error contacting APIServer: ", err, "; trying with fallback: ", config.Host)
+		c, err = kubernetes.NewForConfig(config)
+		if err != nil {
+			return nil, err
+		}
+		nodeList, err = c.Nodes().List(api.ListOptions{})
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	addresses := make([]string, 0, len(nodeList.Items))
+	for _, peer := range nodeList.Items {
+		for _, addr := range peer.Status.Addresses {
+			if addr.Type == "InternalIP" {
+				addresses = append(addresses, addr.Address)
+			}
+		}
+	}
+	return addresses, nil
+}
+
+func main() {
+	peers, err := getKubePeers()
+	if err != nil {
+		log.Fatalf("Could not get peers: %v", err)
+	}
+	for _, addr := range peers {
+		fmt.Println(addr)
+	}
+}

prog/plugin/Dockerfile.template

@@ -1,3 +1,3 @@
-FROM weaveworks/weaveexec
+FROM DOCKERHUB_USER/weaveARCH_EXT
 ADD ./plugin /home/weave/
 ENTRYPOINT ["/home/weave/plugin"]

prog/plugin/main.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 	"syscall"
 
-	cni "github.com/appc/cni/pkg/skel"
 	"github.com/docker/libnetwork/ipamapi"
 	weaveapi "github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
@@ -22,15 +21,13 @@ import (
 	"github.com/weaveworks/weave/plugin/skel"
 )
 
-var version = "(unreleased version)"
+var version = "unreleased"
 
 var Log = common.Log
 
 func main() {
 	var (
 		justVersion      bool
-		cniNet           bool
-		cniIpam          bool
 		address          string
 		meshAddress      string
 		logLevel         string
@@ -38,8 +35,6 @@ func main() {
 	)
 
 	flag.BoolVar(&justVersion, "version", false, "print version and exit")
-	flag.BoolVar(&cniNet, "cni-net", false, "act as a CNI network plugin")
-	flag.BoolVar(&cniIpam, "cni-ipam", false, "act as a CNI IPAM plugin")
 	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
 	flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen")
 	flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode")
@@ -56,17 +51,6 @@ func main() {
 
 	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log)
 
-	switch {
-	case cniIpam || strings.HasSuffix(os.Args[0], "weave-ipam"):
-		i := ipamplugin.NewIpam(weave)
-		cni.PluginMain(i.CmdAdd, i.CmdDel)
-		os.Exit(0)
-	case cniNet || strings.HasSuffix(os.Args[0], "weave-net"):
-		n := netplugin.NewCNIPlugin(weave)
-		cni.PluginMain(n.CmdAdd, n.CmdDel)
-		os.Exit(0)
-	}
-
 	// API 1.21 is the first version that supports docker network commands
 	dockerClient, err := docker.NewVersionedClientFromEnv("1.21")
 	if err != nil {

prog/weave-kube/Dockerfile.template

@@ -0,0 +1,3 @@
+FROM DOCKERHUB_USER/weaveARCH_EXT
+ADD ./launch.sh ./kube-peers /home/weave/
+ENTRYPOINT ["/home/weave/launch.sh"]

prog/weave-kube/launch.sh

@@ -0,0 +1,108 @@
+#!/bin/sh
+
+set -e
+
+# Default if not supplied - same as weave net default
+IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12}
+HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
+STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
+HOST_ROOT=${HOST_ROOT:-/host}
+CONN_LIMIT=${CONN_LIMIT:-30}
+
+# Check if the IP range overlaps anything existing on the host
+/usr/bin/weaveutil netcheck $IPALLOC_RANGE weave
+
+# Default for network policy
+EXPECT_NPC=${EXPECT_NPC:-1}
+
+# Ensure we have the required modules for NPC
+if [ "${EXPECT_NPC}" != "0" ]; then
+    modprobe br_netfilter
+    modprobe xt_set
+fi
+
+# kube-proxy requires that bridged traffic passes through netfilter
+if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then
+    echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2
+else
+    if [ "$BRIDGE_NF_ENABLED" != "1" ]; then
+        echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
+    fi
+fi
+
+# Explicitly create the bridge so we can pass --expect-npc
+WEAVE_NPC_OPTS="--expect-npc"
+if [ "${EXPECT_NPC}" = "0" ]; then
+    WEAVE_NPC_OPTS=""
+fi
+
+export HOST_ROOT=/host
+/home/weave/weave --local create-bridge --force $WEAVE_NPC_OPTS
+
+# Kubernetes sets HOSTNAME to the host's hostname
+# when running a pod in host namespace.
+NICKNAME_ARG=""
+if [ -n "$HOSTNAME" ] ; then
+    NICKNAME_ARG="--nickname=$HOSTNAME"
+fi
+
+BRIDGE_OPTIONS="--datapath=datapath"
+if [ "$(/home/weave/weave --local bridge-type)" = "bridge" ] ; then
+    # TODO: Call into weave script to do this
+    if ! ip link show vethwe-pcap >/dev/null 2>&1 ; then
+        ip link add name vethwe-bridge type veth peer name vethwe-pcap
+        ip link set vethwe-bridge up
+        ip link set vethwe-pcap up
+        ip link set vethwe-bridge master weave
+    fi
+    BRIDGE_OPTIONS="--iface=vethwe-pcap"
+fi
+
+if [ -z "$KUBE_PEERS" ]; then
+    if ! KUBE_PEERS=$(/home/weave/kube-peers) || [ -z "$KUBE_PEERS" ]; then
+        echo Failed to get peers >&2
+        exit 1
+    fi
+fi
+
+peer_count() {
+    echo $#
+}
+
+if [ -z "$IPALLOC_INIT" ]; then
+    IPALLOC_INIT="consensus=$(peer_count $KUBE_PEERS)"
+fi
+
+post_start_actions() {
+    # Wait for weave process to become responsive
+    while true ; do
+        curl $HTTP_ADDR/status >/dev/null 2>&1 && break
+        sleep 1
+    done
+
+    # Install CNI plugin binary to typical CNI bin location
+    # with fall-back to CNI directory used by kube-up on GCI OS
+    if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then
+        if mkdir -p $HOST_ROOT/home/kubernetes/bin ; then
+            export WEAVE_CNI_PLUGIN_DIR=$HOST_ROOT/home/kubernetes/bin
+        else
+            echo "Failed to install the Weave CNI plugin" >&2
+            exit 1
+        fi
+    fi
+    mkdir -p $HOST_ROOT/etc/cni/net.d
+    /home/weave/weave --local setup-cni
+
+    # Expose the weave network so host processes can communicate with pods
+    /home/weave/weave --local expose $WEAVE_EXPOSE_IP
+}
+
+post_start_actions &
+
+exec /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \
+     --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \
+     --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \
+     --ipalloc-init $IPALLOC_INIT \
+     --conn-limit=$CONN_LIMIT \
+     "$@" \
+     $KUBE_PEERS

prog/weave-kube/weave-daemonset.yaml

@@ -0,0 +1,80 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: weave-net
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        name: weave-net
+      annotations:
+        scheduler.alpha.kubernetes.io/tolerations: |
+          [
+            {
+              "key": "dedicated",
+              "operator": "Equal",
+              "value": "master",
+              "effect": "NoSchedule"
+            }
+          ]
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+        - name: weave
+          image: weaveworks/weave-kube:latest
+          imagePullPolicy: Always
+          command:
+            - /home/weave/launch.sh
+          livenessProbe:
+            initialDelaySeconds: 30
+            httpGet:
+              host: 127.0.0.1
+              path: /status
+              port: 6784
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: weavedb
+              mountPath: /weavedb
+            - name: cni-bin
+              mountPath: /host/opt
+            - name: cni-bin2
+              mountPath: /host/home
+            - name: cni-conf
+              mountPath: /host/etc
+            - name: dbus
+              mountPath: /host/var/lib/dbus
+            - name: lib-modules
+              mountPath: /lib/modules
+          resources:
+            requests:
+              cpu: 10m
+        - name: weave-npc
+          image: weaveworks/weave-npc:latest
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 10m
+          securityContext:
+            privileged: true
+      restartPolicy: Always
+      volumes:
+        - name: weavedb
+          emptyDir: {}
+        - name: cni-bin
+          hostPath:
+            path: /opt
+        - name: cni-bin2
+          hostPath:
+            path: /home
+        - name: cni-conf
+          hostPath:
+            path: /etc
+        - name: dbus
+          hostPath:
+            path: /var/lib/dbus
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules

prog/weave-npc/Dockerfile.template

@@ -0,0 +1,23 @@
+FROM ALPINE_BASEIMAGE
+
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
+
+LABEL maintainer "Weaveworks Inc <help@weave.works>"
+LABEL works.weave.role=system \
+      org.label-schema.name="Weave Net" \
+      org.label-schema.description="Weave Net creates a virtual network that connects Docker containers across multiple hosts and enables their automatic discovery" \
+      org.label-schema.url="https://weave.works" \
+      org.label-schema.vcs-url="https://github.com/weaveworks/weave" \
+      org.label-schema.vendor="Weaveworks"
+
+RUN apk add --update \
+    iptables \
+    ipset \
+	ulogd \
+  && rm -rf /var/cache/apk/* \
+  && mknod /var/log/ulogd.pcap p
+COPY ./weave-npc /usr/bin/weave-npc
+COPY ./ulogd.conf /etc/ulogd.conf
+ENTRYPOINT ["/usr/bin/weave-npc"]

prog/weave-npc/main.go

@@ -0,0 +1,204 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/kubernetes"
+	coreapi "k8s.io/client-go/pkg/api/v1"
+	extnapi "k8s.io/client-go/pkg/apis/extensions/v1beta1"
+	"k8s.io/client-go/pkg/fields"
+	"k8s.io/client-go/pkg/runtime"
+	"k8s.io/client-go/pkg/util/wait"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/npc"
+	"github.com/weaveworks/weave/npc/ipset"
+	"github.com/weaveworks/weave/npc/metrics"
+	"github.com/weaveworks/weave/npc/ulogd"
+)
+
+var (
+	version     = "unreleased"
+	metricsAddr string
+	logLevel    string
+	allowMcast  bool
+)
+
+func handleError(err error) { common.CheckFatal(err) }
+
+func makeController(getter cache.Getter, resource string,
+	objType runtime.Object, handlers cache.ResourceEventHandlerFuncs) *cache.Controller {
+	listWatch := cache.NewListWatchFromClient(getter, resource, "", fields.Everything())
+	_, controller := cache.NewInformer(listWatch, objType, 0, handlers)
+	return controller
+}
+
+func resetIPTables(ipt *iptables.IPTables) error {
+	// Flush chains first so there are no refs to extant ipsets
+	if err := ipt.ClearChain(npc.TableFilter, npc.IngressChain); err != nil {
+		return err
+	}
+
+	if err := ipt.ClearChain(npc.TableFilter, npc.DefaultChain); err != nil {
+		return err
+	}
+
+	if err := ipt.ClearChain(npc.TableFilter, npc.MainChain); err != nil {
+		return err
+	}
+
+	// Configure main chain static rules
+	if err := ipt.Append(npc.TableFilter, npc.MainChain,
+		"-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil {
+		return err
+	}
+
+	if allowMcast {
+		if err := ipt.Append(npc.TableFilter, npc.MainChain,
+			"-d", "224.0.0.0/4", "-j", "ACCEPT"); err != nil {
+			return err
+		}
+	}
+
+	if err := ipt.Append(npc.TableFilter, npc.MainChain,
+		"-m", "state", "--state", "NEW", "-j", string(npc.DefaultChain)); err != nil {
+		return err
+	}
+
+	if err := ipt.Append(npc.TableFilter, npc.MainChain,
+		"-m", "state", "--state", "NEW", "-j", string(npc.IngressChain)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func resetIPSets(ips ipset.Interface) error {
+	// TODO should restrict ipset operations to the `weave-` prefix:
+
+	if err := ips.FlushAll(); err != nil {
+		return err
+	}
+
+	if err := ips.DestroyAll(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func root(cmd *cobra.Command, args []string) {
+	common.SetLogLevel(logLevel)
+	common.Log.Infof("Starting Weaveworks NPC %s", version)
+
+	if err := metrics.Start(metricsAddr); err != nil {
+		common.Log.Fatalf("Failed to start metrics: %v", err)
+	}
+
+	if err := ulogd.Start(); err != nil {
+		common.Log.Fatalf("Failed to start ulogd: %v", err)
+	}
+
+	config, err := rest.InClusterConfig()
+	handleError(err)
+
+	client, err := kubernetes.NewForConfig(config)
+	handleError(err)
+
+	ipt, err := iptables.New()
+	handleError(err)
+
+	ips := ipset.New()
+
+	handleError(resetIPTables(ipt))
+	handleError(resetIPSets(ips))
+
+	npc := npc.New(ipt, ips)
+
+	nsController := makeController(client.Core().RESTClient(), "namespaces", &coreapi.Namespace{},
+		cache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				handleError(npc.AddNamespace(obj.(*coreapi.Namespace)))
+			},
+			DeleteFunc: func(obj interface{}) {
+				switch obj := obj.(type) {
+				case *coreapi.Namespace:
+					handleError(npc.DeleteNamespace(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeleteNamespace(obj.Obj.(*coreapi.Namespace)))
+				}
+			},
+			UpdateFunc: func(old, new interface{}) {
+				handleError(npc.UpdateNamespace(old.(*coreapi.Namespace), new.(*coreapi.Namespace)))
+			}})
+
+	podController := makeController(client.Core().RESTClient(), "pods", &coreapi.Pod{},
+		cache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				handleError(npc.AddPod(obj.(*coreapi.Pod)))
+			},
+			DeleteFunc: func(obj interface{}) {
+				switch obj := obj.(type) {
+				case *coreapi.Pod:
+					handleError(npc.DeletePod(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeletePod(obj.Obj.(*coreapi.Pod)))
+				}
+			},
+			UpdateFunc: func(old, new interface{}) {
+				handleError(npc.UpdatePod(old.(*coreapi.Pod), new.(*coreapi.Pod)))
+			}})
+
+	npController := makeController(client.Extensions().RESTClient(), "networkpolicies", &extnapi.NetworkPolicy{},
+		cache.ResourceEventHandlerFuncs{
+			AddFunc: func(obj interface{}) {
+				handleError(npc.AddNetworkPolicy(obj.(*extnapi.NetworkPolicy)))
+			},
+			DeleteFunc: func(obj interface{}) {
+				switch obj := obj.(type) {
+				case *extnapi.NetworkPolicy:
+					handleError(npc.DeleteNetworkPolicy(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeleteNetworkPolicy(obj.Obj.(*extnapi.NetworkPolicy)))
+				}
+			},
+			UpdateFunc: func(old, new interface{}) {
+				handleError(npc.UpdateNetworkPolicy(old.(*extnapi.NetworkPolicy), new.(*extnapi.NetworkPolicy)))
+			}})
+
+	go nsController.Run(wait.NeverStop)
+	go podController.Run(wait.NeverStop)
+	go npController.Run(wait.NeverStop)
+
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
+	common.Log.Fatalf("Exiting: %v", <-signals)
+}
+
+func main() {
+	rootCmd := &cobra.Command{
+		Use:   "weave-npc",
+		Short: "Weaveworks Kubernetes Network Policy Controller",
+		Run:   root}
+
+	rootCmd.PersistentFlags().StringVar(&metricsAddr, "metrics-addr", ":6781", "metrics server bind address")
+	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "debug", "logging level (debug, info, warning, error)")
+	rootCmd.PersistentFlags().BoolVar(&allowMcast, "allow-mcast", true, "allow all multicast traffic")
+
+	handleError(rootCmd.Execute())
+}

prog/weave-npc/ulogd.conf

@@ -0,0 +1,13 @@
+[global]
+logfile="/dev/null"
+plugin="/usr/lib/ulogd/ulogd_inppkt_NFLOG.so"
+plugin="/usr/lib/ulogd/ulogd_raw2packet_BASE.so"
+plugin="/usr/lib/ulogd/ulogd_output_PCAP.so"
+stack=log1:NFLOG,base1:BASE,pcap1:PCAP
+
+[log1]
+group=86
+
+[pcap1]
+file="/var/log/ulogd.pcap"
+sync=1

prog/weavedb/Dockerfile

@@ -1,6 +1,6 @@
 # This is a nearly-empty image that we use to create a data-only container for persistence
 FROM scratch
-MAINTAINER Weaveworks Inc <help@weave.works>
+LABEL maintainer "Weaveworks Inc <help@weave.works>"
 ENTRYPOINT ["data-only"]
 # Work round Docker refusing to save an empty image
 COPY Dockerfile /

prog/weaveexec/Dockerfile

@@ -1,24 +0,0 @@
-FROM alpine
-
-MAINTAINER Weaveworks Inc <help@weave.works>
-LABEL works.weave.role=system
-
-WORKDIR /home/weave
-ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
-
-RUN apk add --update \
-    curl \
-    ethtool \
-    iptables \
-    iproute2 \
-    util-linux \
-    conntrack-tools \
-    bind-tools \
-  && rm -rf /var/cache/apk/*
-
-ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/
-ADD ./weaveutil /usr/bin/
-ADD ./weavewait /w/w
-ADD ./weavewait_noop /w-noop/w
-ADD ./weavewait_nomcast /w-nomcast/w
-ADD ./docker.tgz /

prog/weaveexec/Dockerfile.template

@@ -0,0 +1,33 @@
+FROM ALPINE_BASEIMAGE
+
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
+
+LABEL maintainer "Weaveworks Inc <help@weave.works>"
+LABEL works.weave.role=system \
+      org.label-schema.name="Weave Net" \
+      org.label-schema.description="Weave Net creates a virtual network that connects Docker containers across multiple hosts and enables their automatic discovery" \
+      org.label-schema.url="https://weave.works" \
+      org.label-schema.vcs-url="https://github.com/weaveworks/weave" \
+      org.label-schema.vendor="Weaveworks"
+
+WORKDIR /home/weave
+ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
+
+RUN apk add --update \
+    curl \
+    ethtool \
+    iptables \
+    iproute2 \
+    util-linux \
+    conntrack-tools \
+    bind-tools \
+  && rm -rf /var/cache/apk/*
+
+ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/
+ADD ./weaveutil /usr/bin/
+ADD ./weavewait /w/w
+ADD ./weavewait_noop /w-noop/w
+ADD ./weavewait_nomcast /w-nomcast/w
+ADD ./docker /usr/bin/

prog/weaveproxy/main.go

@@ -5,13 +5,14 @@ import (
 	"os"
 	"strings"
 
-	"github.com/weaveworks/docker/pkg/mflag"
+	"github.com/weaveworks/common/mflag"
+	"github.com/weaveworks/common/mflagext"
+	"github.com/weaveworks/common/signals"
 	"github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/common/mflagext"
 	"github.com/weaveworks/weave/proxy"
 )
 
-var version = "(unreleased version)"
+var version = "unreleased"
 
 var Log = common.Log
 
@@ -79,5 +80,5 @@ func main() {
 	p.AttachExistingContainers()
 	go p.Serve(listeners)
 	go p.ListenAndServeStatus("/home/weave/status.sock")
-	common.SignalHandlerLoop()
+	signals.SignalHandlerLoop(common.Log)
 }

prog/weaver/Dockerfile.template

@@ -1,6 +1,4 @@
-FROM scratch
-MAINTAINER Weaveworks Inc <help@weave.works>
-LABEL works.weave.role=system
+FROM DOCKERHUB_USER/weaveexecARCH_EXT
 WORKDIR /home/weave
 ADD ./weaver /home/weave/
 ADD weavedata.db /weavedb/

prog/weaver/checkpoint.go

@@ -10,6 +10,7 @@ import (
 
 var checker *checkpoint.Checker
 var newVersion atomic.Value
+var success atomic.Value
 
 const (
 	updateCheckPeriod = 6 * time.Hour
@@ -17,9 +18,11 @@ const (
 
 func checkForUpdates(dockerVersion string, network string) {
 	newVersion.Store("")
+	success.Store(true)
 
 	handleResponse := func(r *checkpoint.CheckResponse, err error) {
 		if err != nil {
+			success.Store(false)
 			Log.Printf("Error checking version: %v", err)
 			return
 		}

prog/weaver/http.go

@@ -19,16 +19,10 @@ import (
 	weave "github.com/weaveworks/weave/router"
 )
 
+var allConnectionStates = []string{"established", "pending", "retrying", "failed", "connecting"}
+
 var rootTemplate = template.New("root").Funcs(map[string]interface{}{
-	"countDNSEntries": func(entries []nameserver.EntryStatus) int {
-		count := 0
-		for _, entry := range entries {
-			if entry.Tombstone == 0 {
-				count++
-			}
-		}
-		return count
-	},
+	"countDNSEntries": countDNSEntries,
 	"printList": func(list []string) string {
 		if len(list) == 0 {
 			return "none"
@@ -55,27 +49,28 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{
 			s.ips += entry.Size
 		}
 
-		printOwned := func(name string, nickName string, reachable bool, ips uint32) {
-			reachableStr := ""
-			if !reachable {
-				reachableStr = "- unreachable!"
-			}
+		printOwned := func(name string, nickName string, info string, ips uint32) {
 			percentageRanges := float32(ips) * 100.0 / float32(status.RangeNumIPs)
 
 			displayName := name + "(" + nickName + ")"
 			fmt.Fprintf(&buffer, "%-37v %8d IPs (%04.1f%% of total) %s\n",
-				displayName, ips, percentageRanges, reachableStr)
+				displayName, ips, percentageRanges, info)
 		}
 
 		// print the local info first
 		if ourStats := peerStats[router.Name]; ourStats != nil {
-			printOwned(router.Name, ourStats.nickname, true, ourStats.ips)
+			activeStr := fmt.Sprintf("(%d active)", status.ActiveIPs)
+			printOwned(router.Name, ourStats.nickname, activeStr, ourStats.ips)
 		}
 
 		// and then the rest
 		for peer, stats := range peerStats {
 			if peer != router.Name {
-				printOwned(peer, stats.nickname, stats.reachable, stats.ips)
+				reachableStr := ""
+				if !stats.reachable {
+					reachableStr = "- unreachable!"
+				}
+				printOwned(peer, stats.nickname, reachableStr, stats.ips)
 			}
 		}
 
@@ -94,7 +89,7 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{
 		for _, conn := range conns {
 			counts[conn.State]++
 		}
-		return printCounts(counts, []string{"established", "pending", "retrying", "failed", "connecting"})
+		return printCounts(counts, allConnectionStates)
 	},
 	"printPeerConnectionCounts": func(peers []mesh.PeerStatus) string {
 		counts := make(map[string]int)
@@ -118,6 +113,26 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{
 	"trimSuffix": strings.TrimSuffix,
 })
 
+func countDNSEntries(entries []nameserver.EntryStatus) int {
+	count := 0
+	for _, entry := range entries {
+		if entry.Tombstone == 0 {
+			count++
+		}
+	}
+	return count
+}
+
+func countDNSEntriesForPeer(peername string, entries []nameserver.EntryStatus) int {
+	count := 0
+	for _, entry := range entries {
+		if entry.Tombstone == 0 && entry.Origin == peername {
+			count++
+		}
+	}
+	return count
+}
+
 // Print counts in a specified order
 func printCounts(counts map[string]int, keys []string) string {
 	var stringCounts []string
@@ -196,7 +211,7 @@ var targetsTemplate = defTemplate("targetsTemplate", `\
 
 var connectionsTemplate = defTemplate("connectionsTemplate", `\
 {{range .Router.Connections}}\
-{{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} {{printf "%-11v" .State}} {{.Info}}
+{{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} {{printf "%-11v" .State}} {{.Info}} {{range $key,$element := .Attrs}}{{if ne $key "name"}}{{$key}}={{$element}}{{end}}{{end}}
 {{end}}\
 `)
 
@@ -225,6 +240,7 @@ var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}
 
 type VersionCheck struct {
 	Enabled     bool
+	Success     bool
 	NewVersion  string
 	NextCheckAt time.Time
 }
@@ -236,6 +252,7 @@ func versionCheck() *VersionCheck {
 	}
 
 	v.Enabled = true
+	v.Success = success.Load().(bool)
 	v.NewVersion = newVersion.Load().(string)
 	v.NextCheckAt = checker.NextCheckAt()
 
@@ -246,12 +263,12 @@ func (v *VersionCheck) String() string {
 	switch {
 	case !v.Enabled:
 		return "version check update disabled"
+	case !v.Success:
+		return fmt.Sprintf("failed to check latest version - see logs; next check at %s", v.NextCheckAt.Format("2006/01/02 15:04:05"))
 	case v.NewVersion != "":
-		return fmt.Sprintf("version %s available - please upgrade!",
-			v.NewVersion)
+		return fmt.Sprintf("version %s available - please upgrade!", v.NewVersion)
 	default:
-		return fmt.Sprintf("up to date; next check at %s",
-			v.NextCheckAt.Format("2006/01/02 15:04:05"))
+		return fmt.Sprintf("up to date; next check at %s", v.NextCheckAt.Format("2006/01/02 15:04:05"))
 	}
 }
 
@@ -263,6 +280,7 @@ type WeaveStatus struct {
 	DNS          *nameserver.Status         `json:"DNS,omitempty"`
 }
 
+// Read-only functions, suitable for exposing on an unprotected socket
 func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRouter, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
 	status := func() WeaveStatus {
 		return WeaveStatus{

prog/weaver/main.go

@@ -13,7 +13,8 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/pkg/profile"
-	"github.com/weaveworks/docker/pkg/mflag"
+	"github.com/weaveworks/common/mflag"
+	"github.com/weaveworks/common/signals"
 	"github.com/weaveworks/mesh"
 
 	"github.com/weaveworks/weave/common"
@@ -27,7 +28,7 @@ import (
 	weave "github.com/weaveworks/weave/router"
 )
 
-var version = "(unreleased version)"
+var version = "unreleased"
 
 var Log = common.Log
 
@@ -132,6 +133,7 @@ func main() {
 		bufSzMB            int
 		noDiscovery        bool
 		httpAddr           string
+		statusAddr         string
 		ipamConfig         ipamConfig
 		dockerAPI          string
 		peers              []string
@@ -165,6 +167,7 @@ func main() {
 	mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery")
 	mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB")
 	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
+	mflag.StringVar(&statusAddr, []string{"-status-addr"}, "", "address to bind status+metrics interface to (disabled if blank, absolute path indicates unix domain socket)")
 	mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)")
 	mflag.StringVar(&ipamConfig.IPRangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
 	mflag.StringVar(&ipamConfig.IPSubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
@@ -221,10 +224,18 @@ func main() {
 		networkConfig.PacketLogging = nopPacketLogging{}
 	}
 
-	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
+	config.Password = determinePassword(password)
+
+	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil)
 	networkConfig.Bridge = bridge
 
-	name := peerName(routerName, bridge.Interface())
+	if bridge != nil {
+		if err := weavenet.DetectHairpin("vethwe-bridge", Log); err != nil {
+			Log.Errorf("DetectHairpin failed: %s", err)
+		}
+	}
+
+	name := peerName(routerName)
 
 	if nickName == "" {
 		var err error
@@ -232,7 +243,6 @@ func main() {
 		checkFatal(err)
 	}
 
-	config.Password = determinePassword(password)
 	config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr)
 	config.PeerDiscovery = !noDiscovery
 
@@ -240,7 +250,7 @@ func main() {
 		Log.Fatalf("--awsvpc mode is not compatible with the --password option")
 	}
 
-	db, err := db.NewBoltDB(dbPrefix + "data.db")
+	db, err := db.NewBoltDB(dbPrefix + db.FileName)
 	checkFatal(err)
 	defer db.Close()
 
@@ -296,13 +306,16 @@ func main() {
 			}
 			trackerName = "awsvpc"
 		}
-		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, t, isKnownPeer)
-		observeContainers(allocator)
+		var allContainerIDs []string
 		if dockerCli != nil {
-			ids, err := dockerCli.AllContainerIDs()
+			allContainerIDs, err = dockerCli.AllContainerIDs()
 			checkFatal(err)
-			allocator.PruneOwned(ids)
 		}
+		preClaims, err := findExistingAddresses(dockerCli, allContainerIDs, weavenet.WeaveBridgeName)
+		checkFatal(err)
+		allocator, defaultSubnet = createAllocator(router, ipamConfig, preClaims, db, t, isKnownPeer)
+		observeContainers(allocator)
+		allocator.PruneOwned(allContainerIDs)
 	}
 
 	var (
@@ -337,12 +350,23 @@ func main() {
 		}
 		router.HandleHTTP(muxRouter)
 		HandleHTTP(muxRouter, version, router, allocator, defaultSubnet, ns, dnsserver)
+		muxRouter.Methods("GET").Path("/metrics").Handler(metricsHandler(router, allocator, ns, dnsserver))
 		http.Handle("/", common.LoggingHTTPHandler(muxRouter))
 		Log.Println("Listening for HTTP control messages on", httpAddr)
-		go listenAndServeHTTP(httpAddr)
+		go listenAndServeHTTP(httpAddr, nil)
 	}
 
-	common.SignalHandlerLoop(router)
+	if statusAddr != "" {
+		muxRouter := mux.NewRouter()
+		HandleHTTP(muxRouter, version, router, allocator, defaultSubnet, ns, dnsserver)
+		muxRouter.Methods("GET").Path("/metrics").Handler(metricsHandler(router, allocator, ns, dnsserver))
+		statusMux := http.NewServeMux()
+		statusMux.Handle("/", muxRouter)
+		Log.Println("Listening for metrics requests on", statusAddr)
+		go listenAndServeHTTP(statusAddr, statusMux)
+	}
+
+	signals.SignalHandlerLoop(common.Log, router)
 }
 
 func options() map[string]string {
@@ -385,7 +409,7 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
 func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
-func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) {
+func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) {
 	overlay := weave.NewOverlaySwitch()
 	var bridge weave.Bridge
 	var ignoreSleeve bool
@@ -402,7 +426,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st
 	case datapathName != "":
 		iface, err := weavenet.EnsureInterface(datapathName)
 		checkFatal(err)
-		fastdp, err := weave.NewFastDatapath(iface, port)
+		fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption)
 		checkFatal(err)
 		bridge = fastdp.Bridge()
 		overlay.Add("fastdp", fastdp.Overlay())
@@ -424,7 +448,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st
 	return overlay, bridge
 }
 
-func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
+func createAllocator(router *weave.NetworkRouter, config ipamConfig, preClaims []ipam.PreClaim, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
 	ipRange, err := ipam.ParseCIDRSubnet(config.IPRangeCIDR)
 	checkFatal(err)
 	defaultSubnet := ipRange
@@ -443,6 +467,7 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, t
 		Seed:        config.SeedPeerNames,
 		Universe:    ipRange,
 		IsObserver:  config.Observer,
+		PreClaims:   preClaims,
 		Quorum:      func() uint { return determineQuorum(config.PeerCount, router) },
 		Db:          db,
 		IsKnownPeer: isKnownPeer,
@@ -509,10 +534,11 @@ func determinePassword(password string) []byte {
 	return []byte(password)
 }
 
-func peerName(routerName string, iface *net.Interface) mesh.PeerName {
+func peerName(routerName string) mesh.PeerName {
 	if routerName == "" {
-		if iface == nil {
-			Log.Fatal("Either an interface must be specified with --datapath or --iface, or a name with --name")
+		iface, err := net.InterfaceByName(weavenet.WeaveBridgeName)
+		if err != nil {
+			Log.Fatalf("Unable to find bridge %q", weavenet.WeaveBridgeName)
 		}
 		routerName = iface.HardwareAddr.String()
 	}
@@ -555,7 +581,7 @@ func parsePeerNames(s string) ([]mesh.PeerName, error) {
 	return peerNames, nil
 }
 
-func listenAndServeHTTP(httpAddr string) {
+func listenAndServeHTTP(httpAddr string, handler http.Handler) {
 	protocol := "tcp"
 	if strings.HasPrefix(httpAddr, "/") {
 		os.Remove(httpAddr) // in case it's there from last time
@@ -565,7 +591,7 @@ func listenAndServeHTTP(httpAddr string) {
 	if err != nil {
 		Log.Fatal("Unable to create http listener socket: ", err)
 	}
-	err = http.Serve(l, nil)
+	err = http.Serve(l, handler)
 	if err != nil {
 		Log.Fatal("Unable to create http server", err)
 	}

prog/weaver/metrics.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/weaveworks/weave/ipam"
+	"github.com/weaveworks/weave/nameserver"
+	"github.com/weaveworks/weave/net/address"
+	weave "github.com/weaveworks/weave/router"
+)
+
+func metricsHandler(router *weave.NetworkRouter, allocator *ipam.Allocator, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) http.Handler {
+	reg := prometheus.NewRegistry()
+	reg.MustRegister(prometheus.NewProcessCollector(os.Getpid(), ""))
+	reg.MustRegister(newMetrics(router, allocator, ns, dnsserver))
+	return promhttp.HandlerFor(reg, promhttp.HandlerOpts{})
+}
+
+type collector struct {
+	router    *weave.NetworkRouter
+	allocator *ipam.Allocator
+	ns        *nameserver.Nameserver
+	dnsserver *nameserver.DNSServer
+}
+
+type metric struct {
+	*prometheus.Desc
+	Collect func(WeaveStatus, *prometheus.Desc, chan<- prometheus.Metric)
+}
+
+func desc(fqName, help string, variableLabels ...string) *prometheus.Desc {
+	return prometheus.NewDesc(fqName, help, variableLabels, prometheus.Labels{})
+}
+
+func intGauge(desc *prometheus.Desc, val int, labels ...string) prometheus.Metric {
+	return prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, float64(val), labels...)
+}
+func uint64Counter(desc *prometheus.Desc, val uint64, labels ...string) prometheus.Metric {
+	return prometheus.MustNewConstMetric(desc, prometheus.CounterValue, float64(val), labels...)
+}
+
+var metrics = []metric{
+	{desc("weave_connections", "Number of peer-to-peer connections.", "state"),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			counts := make(map[string]int)
+			for _, conn := range s.Router.Connections {
+				counts[conn.State]++
+			}
+			for _, state := range allConnectionStates {
+				ch <- intGauge(desc, counts[state], state)
+			}
+		}},
+	{desc("weave_connection_terminations_total", "Number of peer-to-peer connections terminated."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			ch <- uint64Counter(desc, uint64(s.Router.TerminationCount))
+		}},
+	{desc("weave_ips", "Number of IP addresses.", "state"),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, s.IPAM.ActiveIPs, "local-used")
+			}
+		}},
+	{desc("weave_max_ips", "Size of IP address space used by allocator."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, s.IPAM.RangeNumIPs)
+			}
+		}},
+	{desc("weave_dns_entries", "Number of DNS entries."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.DNS != nil {
+				ch <- intGauge(desc, countDNSEntriesForPeer(s.Router.Name, s.DNS.Entries))
+			}
+		}},
+	{desc("weave_flows", "Number of FastDP flows."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if metrics := fastDPMetrics(s); metrics != nil {
+				ch <- intGauge(desc, metrics.Flows)
+			}
+		}},
+	{desc("weave_ipam_pending_allocates", "Number of pending allocates."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, len(s.IPAM.PendingAllocates))
+			}
+		}},
+	{desc("weave_ipam_pending_claims", "Number of pending claims."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, len(s.IPAM.PendingClaims))
+			}
+		}},
+}
+
+func fastDPMetrics(s WeaveStatus) *weave.FastDPMetrics {
+	if diagMap, ok := s.Router.OverlayDiagnostics.(map[string]interface{}); ok {
+		if diag, ok := diagMap["fastdp"]; ok {
+			if fastDPStats, ok := diag.(weave.FastDPStatus); ok {
+				return fastDPStats.Metrics().(*weave.FastDPMetrics)
+			}
+		}
+	}
+	return nil
+}
+
+func newMetrics(router *weave.NetworkRouter, allocator *ipam.Allocator, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) *collector {
+	return &collector{
+		router:    router,
+		allocator: allocator,
+		ns:        ns,
+		dnsserver: dnsserver,
+	}
+}
+
+func (m *collector) Collect(ch chan<- prometheus.Metric) {
+
+	status := WeaveStatus{"", nil,
+		weave.NewNetworkRouterStatus(m.router),
+		ipam.NewStatus(m.allocator, address.CIDR{}),
+		nameserver.NewStatus(m.ns, m.dnsserver)}
+
+	for _, metric := range metrics {
+		metric.Collect(status, metric.Desc, ch)
+	}
+}
+
+func (m *collector) Describe(ch chan<- *prometheus.Desc) {
+	for _, metric := range metrics {
+		ch <- metric.Desc
+	}
+}

prog/weaver/reclaim.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"net"
+
+	docker "github.com/fsouza/go-dockerclient"
+
+	"github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	weavedocker "github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/ipam"
+	weavenet "github.com/weaveworks/weave/net"
+	"github.com/weaveworks/weave/net/address"
+)
+
+func a(cidr *net.IPNet) address.CIDR {
+	prefixLength, _ := cidr.Mask.Size()
+	return address.CIDR{Addr: address.FromIP4(cidr.IP), PrefixLen: prefixLength}
+}
+
+// Get all the existing Weave IPs at startup, so we can stop IPAM
+// giving out any as duplicates
+func findExistingAddresses(dockerCli *weavedocker.Client, containerIDs []string, bridgeName string) (addrs []ipam.PreClaim, err error) {
+	Log.Infof("Checking for pre-existing addresses on %s bridge", bridgeName)
+	// First get the address for the bridge
+	bridgeNetDev, err := weavenet.GetBridgeNetDev(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+	for _, cidr := range bridgeNetDev.CIDRs {
+		Log.Infof("%s bridge has address %v", bridgeName, cidr)
+		addrs = append(addrs, ipam.PreClaim{Ident: "weave:expose", Cidr: a(cidr)})
+	}
+
+	add := func(cid string, isContainer bool, netDevs []weavenet.Dev) {
+		for _, netDev := range netDevs {
+			for _, cidr := range netDev.CIDRs {
+				Log.Infof("Found address %v for ID %s", cidr, cid)
+				addrs = append(addrs, ipam.PreClaim{Ident: cid, IsContainer: isContainer, Cidr: a(cidr)})
+			}
+		}
+	}
+
+	// Then find all veths connected to the bridge
+	peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Now iterate over all containers to see if they have a network
+	// namespace with an attached interface
+	if dockerCli != nil {
+		for _, cid := range containerIDs {
+			container, err := dockerCli.InspectContainer(cid)
+			if err != nil {
+				if _, ok := err.(*docker.NoSuchContainer); ok {
+					continue
+				}
+				return nil, err
+			}
+			if container.State.Pid != 0 {
+				netDevs, err := weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs)
+				if err != nil {
+					return nil, err
+				}
+				add(cid, true, netDevs)
+			}
+		}
+	} else {
+		// If we don't have a Docker connection, iterate over all processes
+		pids, err := common.AllPids("/proc")
+		if err != nil {
+			return nil, err
+		}
+		for _, pid := range pids {
+			netDevs, err := weavenet.GetNetDevsByVethPeerIds(pid, peerIDs)
+			if err != nil {
+				return nil, err
+			}
+			add(api.NoContainerID, false, netDevs)
+		}
+	}
+	return addrs, nil
+}

prog/weaveutil/addrs.go

@@ -4,9 +4,7 @@ import (
 	"fmt"
 
 	"github.com/fsouza/go-dockerclient"
-	"github.com/vishvananda/netlink"
 
-	"github.com/weaveworks/weave/common"
 	weavenet "github.com/weaveworks/weave/net"
 )
 
@@ -21,7 +19,7 @@ func containerAddrs(args []string) error {
 		return err
 	}
 
-	pred, err := common.ConnectedToBridgePredicate(bridgeName)
+	peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
 	if err != nil {
 		if err == weavenet.ErrLinkNotFound {
 			return nil
@@ -34,11 +32,11 @@ func containerAddrs(args []string) error {
 
 	for _, cid := range args[1:] {
 		if cid == "weave:expose" {
-			netDev, err := common.GetBridgeNetDev(bridgeName)
+			netDev, err := weavenet.GetBridgeNetDev(bridgeName)
 			if err != nil {
 				return err
 			}
-			printNetDevs(cid, []common.NetDev{netDev})
+			printNetDevs(cid, []weavenet.Dev{netDev})
 			continue
 		}
 		if containers[cid], err = client.InspectContainer(cid); err != nil {
@@ -51,11 +49,8 @@ func containerAddrs(args []string) error {
 		containerIDs = append(containerIDs, cid)
 	}
 
-	// NB: Because network namespaces (netns) are changed many times inside the loop,
-	//     it's NOT safe to exec any code depending on the root netns without
-	//     wrapping with WithNetNS*.
 	for _, cid := range containerIDs {
-		netDevs, err := getNetDevs(client, containers[cid], pred)
+		netDevs, err := getNetDevs(client, containers[cid], peerIDs)
 		if err != nil {
 			return err
 		}
@@ -65,14 +60,14 @@ func containerAddrs(args []string) error {
 	return nil
 }
 
-func getNetDevs(c *docker.Client, container *docker.Container, pred func(netlink.Link) bool) ([]common.NetDev, error) {
+func getNetDevs(c *docker.Client, container *docker.Container, peerIDs []int) ([]weavenet.Dev, error) {
 	if container.State.Pid == 0 {
 		return nil, nil
 	}
-	return common.GetNetDevsWithPredicate(container.State.Pid, pred)
+	return weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs)
 }
 
-func printNetDevs(cid string, netDevs []common.NetDev) {
+func printNetDevs(cid string, netDevs []weavenet.Dev) {
 	for _, netDev := range netDevs {
 		fmt.Printf("%12s %s %s", cid, netDev.Name, netDev.MAC.String())
 		for _, cidr := range netDev.CIDRs {
@@ -82,3 +77,16 @@ func printNetDevs(cid string, netDevs []common.NetDev) {
 		fmt.Println()
 	}
 }
+
+func bridgeIP(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("bridge-ip", "<bridgeName>")
+	}
+	bridgeName := args[0]
+	ip, err := weavenet.FindBridgeIP(bridgeName, nil)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%v", ip)
+	return nil
+}

prog/weaveutil/attach.go

@@ -32,14 +32,19 @@ func attach(args []string) error {
 		}
 	}
 
-	pid, nsContainer, err := containerPidAndNs(args[0])
+	pid, err := containerPid(args[0])
 	if err != nil {
 		return err
 	}
+	nsContainer, err := netns.GetFromPid(pid)
+	if err != nil {
+		return fmt.Errorf("unable to open namespace for container %s: %s", args[0], err)
+	}
+
 	if nsHost, err := netns.GetFromPid(1); err != nil {
 		return fmt.Errorf("unable to open host namespace: %s", err)
 	} else if nsHost.Equal(nsContainer) {
-		return fmt.Errorf("Container is running in the host network namespace, and therefore cannot be\nconnected to weave. Perhaps the container was started with --net=host.")
+		return fmt.Errorf("Container is running in the host network namespace, and therefore cannot be\nconnected to weave - perhaps the container was started with --net=host")
 	}
 	mtu, err := strconv.Atoi(args[2])
 	if err != nil && args[3] != "" {
@@ -50,7 +55,7 @@ func attach(args []string) error {
 		return err
 	}
 
-	err = weavenet.AttachContainer(nsContainer, fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn)
+	err = weavenet.AttachContainer(weavenet.NSPathByPid(pid), fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn)
 	// If we detected an error but the container has died, tell the user that instead.
 	if err != nil && !processExists(pid) {
 		err = fmt.Errorf("Container %s died", args[0])
@@ -58,23 +63,19 @@ func attach(args []string) error {
 	return err
 }
 
-func containerPidAndNs(containerID string) (int, netns.NsHandle, error) {
+func containerPid(containerID string) (int, error) {
 	c, err := docker.NewVersionedClientFromEnv("1.18")
 	if err != nil {
-		return 0, 0, fmt.Errorf("unable to connect to docker: %s", err)
+		return 0, fmt.Errorf("unable to connect to docker: %s", err)
 	}
 	container, err := c.InspectContainer(containerID)
 	if err != nil {
-		return 0, 0, fmt.Errorf("unable to inspect container %s: %s", containerID, err)
+		return 0, fmt.Errorf("unable to inspect container %s: %s", containerID, err)
 	}
 	if container.State.Pid == 0 {
-		return 0, 0, fmt.Errorf("container %s not running", containerID)
+		return 0, fmt.Errorf("container %s not running", containerID)
 	}
-	ns, err := netns.GetFromPid(container.State.Pid)
-	if err != nil {
-		return 0, 0, fmt.Errorf("unable to open namespace for container %s: %s", containerID, err)
-	}
-	return container.State.Pid, ns, nil
+	return container.State.Pid, nil
 }
 
 func processExists(pid int) bool {
@@ -99,7 +100,7 @@ func detach(args []string) error {
 		cmdUsage("detach-container", "<container-id> <cidr>...")
 	}
 
-	_, ns, err := containerPidAndNs(args[0])
+	pid, err := containerPid(args[0])
 	if err != nil {
 		return err
 	}
@@ -107,5 +108,5 @@ func detach(args []string) error {
 	if err != nil {
 		return err
 	}
-	return weavenet.DetachContainer(ns, args[0], weavenet.VethName, cidrs)
+	return weavenet.DetachContainer(weavenet.NSPathByPid(pid), args[0], weavenet.VethName, cidrs)
 }

prog/weaveutil/cni.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"os"
+
+	cni "github.com/appc/cni/pkg/skel"
+
+	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
+	netplugin "github.com/weaveworks/weave/plugin/net"
+)
+
+func cniNet(args []string) error {
+	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
+	n := netplugin.NewCNIPlugin(weave)
+	cni.PluginMain(n.CmdAdd, n.CmdDel)
+	return nil
+}
+
+func cniIPAM(args []string) error {
+	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
+	i := ipamplugin.NewIpam(weave)
+	cni.PluginMain(i.CmdAdd, i.CmdDel)
+	return nil
+}

prog/weaveutil/expose.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func exposeNAT(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("expose-nat", "<cidr>...")
+	}
+
+	cidrs, err := parseCIDRs(args)
+	if err != nil {
+		return err
+	}
+
+	for _, cidr := range cidrs {
+		if err := weavenet.ExposeNAT(*cidr); err != nil {
+			return err
+		}
+	}
+	return nil
+}

prog/weaveutil/main.go

@@ -4,6 +4,10 @@ package main
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
+
+	weavenet "github.com/weaveworks/weave/net"
 )
 
 var commands map[string]func([]string) error
@@ -24,10 +28,37 @@ func init() {
 		"attach-container":       attach,
 		"detach-container":       detach,
 		"configure-arp":          configureARP,
+		"check-iface":            checkIface,
+		"del-iface":              delIface,
+		"setup-iface":            setupIface,
+		"list-netdevs":           listNetDevs,
+		"cni-net":                cniNet,
+		"cni-ipam":               cniIPAM,
+		"expose-nat":             exposeNAT,
+		"bridge-ip":              bridgeIP,
+		"unique-id":              uniqueID,
 	}
 }
 
 func main() {
+	// force re-exec of this binary
+	selfPath, err := filepath.EvalSymlinks("/proc/self/exe")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	weavenet.WeaveUtilCmd = selfPath
+
+	// If no args passed, act as CNI plugin based on executable name
+	switch {
+	case len(os.Args) == 1 && strings.HasSuffix(os.Args[0], "weave-ipam"):
+		cniIPAM(os.Args)
+		os.Exit(0)
+	case len(os.Args) == 1 && strings.HasSuffix(os.Args[0], "weave-net"):
+		cniNet(os.Args)
+		os.Exit(0)
+	}
+
 	if len(os.Args) < 2 {
 		usage()
 		os.Exit(1)

prog/weaveutil/netdev.go

@@ -1,14 +1,75 @@
 package main
 
 import (
+	"encoding/json"
+	"fmt"
+	"strconv"
 	"strings"
 
+	"github.com/coreos/go-iptables/iptables"
 	"github.com/j-keck/arping"
 	"github.com/vishvananda/netlink"
 
 	weavenet "github.com/weaveworks/weave/net"
 )
 
+// checkIface returns an error if the given interface cannot be found.
+func checkIface(args []string) error {
+	if len(args) != 1 {
+		cmdUsage("check-iface", "<iface-name>")
+	}
+	ifaceName := args[0]
+
+	if _, err := netlink.LinkByName(ifaceName); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func delIface(args []string) error {
+	if len(args) != 1 {
+		cmdUsage("del-iface", "<iface-name>")
+	}
+	ifName := args[0]
+
+	link, err := netlink.LinkByName(ifName)
+	if err != nil {
+		return err
+	}
+	return netlink.LinkDel(link)
+}
+
+// setupIface renames the given iface and configures its ARP cache settings.
+func setupIface(args []string) error {
+	if len(args) != 2 {
+		cmdUsage("setup-iface", "<iface-name> <new-iface-name>")
+	}
+	ifaceName := args[0]
+	newIfName := args[1]
+
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+
+	link, err := netlink.LinkByName(ifaceName)
+	if err != nil {
+		return err
+	}
+	if err := netlink.LinkSetName(link, newIfName); err != nil {
+		return err
+	}
+	if err := weavenet.ConfigureARPCache(newIfName); err != nil {
+		return err
+	}
+	if err := ipt.Append("filter", "INPUT", "-i", newIfName, "-d", "224.0.0.0/4", "-j", "DROP"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func configureARP(args []string) error {
 	if len(args) != 1 {
 		cmdUsage("configure-arp", "<iface-name-prefix>")
@@ -33,3 +94,47 @@ func configureARP(args []string) error {
 
 	return nil
 }
+
+// listNetDevs outputs network ifaces identified by the given indexes
+// in the format of weavenet.Dev.
+func listNetDevs(args []string) error {
+	if len(args) == 0 {
+		cmdUsage("list-netdevs", "<iface-index>[ <iface-index>]")
+	}
+
+	indexes := make(map[int]struct{})
+	for _, index := range args {
+		if index != "" {
+			id, err := strconv.Atoi(index)
+			if err != nil {
+				return err
+			}
+			indexes[id] = struct{}{}
+		}
+	}
+
+	links, err := netlink.LinkList()
+	if err != nil {
+		return err
+	}
+
+	var netdevs []weavenet.Dev
+
+	for _, link := range links {
+		if _, found := indexes[link.Attrs().Index]; found {
+			netdev, err := weavenet.LinkToNetDev(link)
+			if err != nil {
+				return err
+			}
+			netdevs = append(netdevs, netdev)
+		}
+	}
+
+	nds, err := json.Marshal(netdevs)
+	if err != nil {
+		return err
+	}
+	fmt.Println(string(nds))
+
+	return nil
+}

prog/weaveutil/proc-addrs.go

@@ -13,7 +13,7 @@ func processAddrs(args []string) error {
 	}
 	bridgeName := args[0]
 
-	pred, err := common.ConnectedToBridgePredicate(bridgeName)
+	peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
 	if err != nil {
 		if err == weavenet.ErrLinkNotFound {
 			return nil
@@ -26,11 +26,8 @@ func processAddrs(args []string) error {
 		return err
 	}
 
-	// NB: Because network namespaces (netns) are changed many times inside the loop,
-	//     it's NOT safe to exec any code depending on the root netns without
-	//     wrapping with WithNetNS*.
 	for _, pid := range pids {
-		netDevs, err := common.GetNetDevsWithPredicate(pid, pred)
+		netDevs, err := weavenet.GetNetDevsByVethPeerIds(pid, peerIDs)
 		if err != nil {
 			return err
 		}

prog/weaveutil/unique_id.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func uniqueID(args []string) error {
+	if len(args) != 2 {
+		cmdUsage("unique-id", "<db-prefix> <host-root>")
+	}
+	dbPrefix := args[0]
+	hostRoot := args[1]
+	uid, err := weavenet.GetSystemPeerName(dbPrefix, hostRoot)
+	if err != nil {
+		return err
+	}
+	fmt.Printf(uid)
+	return nil
+}

proxy/malformed_host_override.go

@@ -0,0 +1,122 @@
+// (C) Docker, Inc.
+// Redistributed under the Apache License 2.0
+
+package proxy
+
+import "net"
+
+// MalformedHostHeaderOverride is a wrapper to be able
+// to overcome the 400 Bad request coming from old docker
+// clients that send an invalid Host header.
+type MalformedHostHeaderOverride struct {
+	net.Listener
+}
+
+// MalformedHostHeaderOverrideConn wraps the underlying unix
+// connection and keeps track of the first read from http.Server
+// which just reads the headers.
+type MalformedHostHeaderOverrideConn struct {
+	net.Conn
+	first bool
+}
+
+var closeConnHeader = []byte("\r\nConnection: close\r")
+
+// Read reads the first *read* request from http.Server to inspect
+// the Host header. If the Host starts with / then we're talking to
+// an old docker client which send an invalid Host header. To not
+// error out in http.Server we rewrite the first bytes of the request
+// to sanitize the Host header itself.
+// In case we're not dealing with old docker clients the data is just passed
+// to the server w/o modification.
+func (l *MalformedHostHeaderOverrideConn) Read(b []byte) (n int, err error) {
+	// http.Server uses a 4k buffer
+	if l.first && len(b) == 4096 {
+		// This keeps track of the first read from http.Server which just reads
+		// the headers
+		l.first = false
+		// The first read of the connection by http.Server is done limited to
+		// DefaultMaxHeaderBytes (usually 1 << 20) + 4096.
+		// Here we do the first read which gets us all the http headers to
+		// be inspected and modified below.
+		c, err := l.Conn.Read(b)
+		if err != nil {
+			return c, err
+		}
+
+		var (
+			start, end    int
+			firstLineFeed = -1
+			buf           []byte
+		)
+		for i := 0; i <= c-1-7; i++ {
+			if b[i] == '\n' && firstLineFeed == -1 {
+				firstLineFeed = i
+			}
+			if b[i] != '\n' {
+				continue
+			}
+
+			if b[i+1] == '\r' && b[i+2] == '\n' {
+				return c, nil
+			}
+
+			if b[i+1] != 'H' {
+				continue
+			}
+			if b[i+2] != 'o' {
+				continue
+			}
+			if b[i+3] != 's' {
+				continue
+			}
+			if b[i+4] != 't' {
+				continue
+			}
+			if b[i+5] != ':' {
+				continue
+			}
+			if b[i+6] != ' ' {
+				continue
+			}
+			if b[i+7] != '/' {
+				continue
+			}
+			// ensure clients other than the docker clients do not get this hack
+			if i != firstLineFeed {
+				return c, nil
+			}
+			start = i + 7
+			// now find where the value ends
+			for ii, bbb := range b[start:c] {
+				if bbb == '\n' {
+					end = start + ii
+					break
+				}
+			}
+			buf = make([]byte, 0, c+len(closeConnHeader)-(end-start))
+			// strip the value of the host header and
+			// inject `Connection: close` to ensure we don't reuse this connection
+			buf = append(buf, b[:start]...)
+			buf = append(buf, closeConnHeader...)
+			buf = append(buf, b[end:c]...)
+			copy(b, buf)
+			break
+		}
+		if len(buf) == 0 {
+			return c, nil
+		}
+		return len(buf), nil
+	}
+	return l.Conn.Read(b)
+}
+
+// Accept makes the listener accepts connections and wraps the connection
+// in a MalformedHostHeaderOverrideConn initilizing first to true.
+func (l *MalformedHostHeaderOverride) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return c, err
+	}
+	return &MalformedHostHeaderOverrideConn{c, true}, nil
+}

proxy/proxy.go

@@ -17,7 +17,6 @@ import (
 
 	docker "github.com/fsouza/go-dockerclient"
 	weaveapi "github.com/weaveworks/weave/api"
-	"github.com/weaveworks/weave/common"
 	weavedocker "github.com/weaveworks/weave/common/docker"
 	weavenet "github.com/weaveworks/weave/net"
 	"github.com/weaveworks/weave/net/address"
@@ -149,14 +148,12 @@ func NewProxy(c Config) (*Proxy, error) {
 	p.client = client.Client
 
 	if !p.WithoutDNS {
-		netDev, err := common.GetBridgeNetDev(c.DockerBridge)
+		ip, err := weavenet.FindBridgeIP(c.DockerBridge, nil)
 		if err != nil {
 			return nil, err
 		}
-		if len(netDev.CIDRs) != 1 {
-			return nil, fmt.Errorf("Could not obtain address of %s", c.DockerBridge)
-		}
-		p.dockerBridgeIP = netDev.CIDRs[0].IP.String()
+		p.dockerBridgeIP = ip.String()
+		Log.Infof("Using docker bridge IP for DNS: %v", p.dockerBridgeIP)
 	}
 
 	p.hostnameMatchRegexp, err = regexp.Compile(c.HostnameMatch)
@@ -379,7 +376,7 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 		Log.Fatalf("Invalid protocol format: %q", proto)
 	}
 
-	return listener, fmt.Sprintf("%s://%s", proto, addr), nil
+	return &MalformedHostHeaderOverride{listener}, fmt.Sprintf("%s://%s", proto, addr), nil
 }
 
 // weavedocker.ContainerObserver interface
@@ -625,7 +622,7 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
 	if err := proxy.waitForStartByIdent(containerID); err != nil {
 		return err
 	}
-	netDevs, err := common.GetWeaveNetDevs(pid)
+	netDevs, err := weavenet.GetWeaveNetDevs(pid)
 	if err != nil || len(netDevs) == 0 || len(netDevs[0].CIDRs) == 0 {
 		return err
 	}