Add release v1.9.7

Weave-NPC should get hostname from K8s Downward API.

.gitignore

@@ -52,7 +52,7 @@ testing/cover/cover
 testing/runner/runner
 releases
 .*.uptodate
-weave.tar.gz
+weave*.tar.gz
 prog/weaveexec/weave
 prog/weaveexec/weavehosts
 prog/weaveexec/weaveproxy
@@ -61,9 +61,15 @@ prog/weaveexec/weavewait_noop
 prog/weaveexec/weavewait_nomcast
 prog/weaveexec/sigproxy
 prog/weaveexec/weaveutil
-prog/weaveexec/docker*.tgz
+prog/weaveexec/docker*
 Vagrantfile.local
 test/tls/tls
 test/tls/*.pem
 test/coverage
 test/coverage.*
+*qemu-*-static
+terraform.tfstate
+terraform.tfstate.backup
+
+# Website
+site-build

.gitmodules

@@ -61,9 +61,6 @@
 [submodule "vendor/github.com/hashicorp/go-cleanhttp"]
 	path = vendor/github.com/hashicorp/go-cleanhttp
 	url = http://github.com/hashicorp/go-cleanhttp
-[submodule "vendor/github.com/appc/cni"]
-	path = vendor/github.com/appc/cni
-	url = https://github.com/appc/cni
 [submodule "vendor/github.com/coreos/go-iptables"]
 	path = vendor/github.com/coreos/go-iptables
 	url = https://github.com/coreos/go-iptables
@@ -79,9 +76,6 @@
 [submodule "vendor/golang.org/x/sys"]
 	path = vendor/golang.org/x/sys
 	url = https://go.googlesource.com/sys
-[submodule "vendor/github.com/weaveworks/docker"]
-	path = vendor/github.com/weaveworks/docker
-	url = https://github.com/weaveworks/docker
 [submodule "vendor/github.com/beorn7/perks"]
 	path = vendor/github.com/beorn7/perks
 	url = https://github.com/beorn7/perks
@@ -118,3 +112,15 @@
 [submodule "vendor/github.com/spf13/pflag"]
 	path = vendor/github.com/spf13/pflag
 	url = https://github.com/spf13/pflag
+[submodule "vendor/github.com/mgutz/ansi"]
+	path = vendor/github.com/mgutz/ansi
+	url = https://github.com/mgutz/ansi
+[submodule "vendor/github.com/estesp/manifest-tool"]
+	path = vendor/github.com/estesp/manifest-tool
+	url = https://github.com/estesp/manifest-tool
+[submodule "vendor/github.com/weaveworks/common"]
+	path = vendor/github.com/weaveworks/common
+	url = https://github.com/weaveworks/common
+[submodule "vendor/github.com/containernetworking/cni"]
+	path = vendor/github.com/containernetworking/cni
+	url = https://github.com/containernetworking/cni

.lintignore

@@ -0,0 +1,27 @@
+# IMPORTANT: Please only use GLOBs to ignore files and directories.
+
+# Do not lint the below bash scripts in Weave Net, in order to 
+# avoid breaking the build on make lint:
+bin/circle-deploy-issues
+bin/circle-deploy-master
+bin/circle-teardown-pre
+bin/circle-test-teardown
+bin/circle-test-unit
+bin/install-wordepress
+bin/multiweave
+bin/publish-site
+bin/release
+build/build.sh
+prog/weave-kube/launch.sh
+prog/weaveexec/symlink
+test/*_test.sh
+test/600_proxy_docker_py.sh
+test/assert.sh
+test/config.sh
+test/gce.sh
+test/gen_coverage_reports.sh
+test/run_all.sh
+test/sanity_check.sh
+test/setup.sh
+vendor/*.sh
+weave

.ruby-version

@@ -0,0 +1 @@
+2.3.0

CHANGELOG.md

@@ -1,3 +1,142 @@
+## Release 1.9.7
+
+This is identical to 1.9.6 with one additional bug-fix:
+
+* weave-npc would block everything if `kubectl --hostname-override` was used #2995,#2996
+
+## Release 1.9.6
+
+Bug fixes and minor improvements
+
+* Ensure that Kubernetes pods can contact a service implemented within
+  the same pod, by turning on "hairpin mode". This is required because
+  of a quiet change between Kubernetes 1.5 and 1.6. #2993
+* Network Policy Controller (`weave-npc`) now checks local addresses
+  only, so it doesn't interfere with cross-cluster traffic. It should
+  be more efficient too #2622,#2973,#2979
+* Stop reporting back to Kubernetes any issues encountered when
+  deleting a pod's network interface. This is required because
+  of a quiet change between Kubernetes 1.5 and 1.6. #2921,#2928
+* Fixed an issue whereby `weave-npc` couldn't start because one
+  `ipset` was referring to another one and could not be destroyed #2915,#2949
+* Improved the code which checks whether the kernel supports `ipset` #2934,#2935
+* `weave-npc` now creates ipsets with only valid xml characters in the
+  name #2958,#2959
+
+Build and Testing
+
+* In build container use cross-compilers from debian package
+  repository, so they match other components #2940
+* Pin the version of the linting tool `shfmt` so the set of things it
+  checks is stable #2987
+* Fix lint error in script that runs smoke-tests #2962
+* Moved website publishing from Wordpress to Netlify #2986
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/55?closed=1).
+
+## Release 1.9.5
+
+Bug fixes and minor improvements
+
+* Improve log messages generated if "hairpin" conditions are detected,
+  to make clear which kind is likely to cause problems #2808/#2926
+* Filter out IPv6 peer addresses from Kubernetes; Weave Net currently
+  only supports IPv4 #2904/#2912
+* Fix rare crash during initialization of weave-kube #2893/#2892
+* Include overlay and encryption modes in checkpoint reports, in case
+  this is relevant to a version upgrade #2771/#2907
+
+Build and Testing
+
+* Ensure CI build can run gcloud tools  #2887
+* Prevent kubeadm from upgrading Kubernetes if we are trying to test an older version #2886
+* Upgrade build scripts to support Kubernetes 1.6 #2880
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/54?closed=1).
+
+## Release 1.9.4
+
+Bug fixes and minor improvements
+
+* Support Kubernetes 1.6 by creating a new DaemonSet #2777,#2801
+* Support Kubernetes 1.6 by allowing CNI callers to send a
+  network-delete request for a container that is not running or has
+  never been attached to the network #2850
+* Leave non-weave ipsets alone in Network Policy Controller (e.g. when
+  running Weave Net alonside keepalived-vip) #2751,#2846
+* Fix various small issues revealed by 'staticcheck' tool #2843,#2857
+* Avoid leaving 'defunct' processes when weave-kube container restarts #2836,#2845
+* When using the CNI plugin with a non-standard network configuration
+  file, the weave bridge could get the same IP as a container, if
+  'weave expose' hadn't run at that point #2839,#2856
+
+Build and Testing
+
+* Check that no defunct processes remain after each test #2852
+* Update build and test scripts to work with Kubernetes 1.6 beta #2851
+
+[Full list of changes](https://github.com/weaveworks/weave/milestone/53?closed=1).
+
+## Release 1.9.3
+
+Bug fixes and minor improvements
+
+* Fixed a race condition in Fast Datapath encrypted connections which could lead Weave Net to crash: #2824, #2825.
+
+## Release 1.9.2
+
+Bug fixes and minor improvements
+
+* Fix a weave-kube bug when `br_netfilter` or `xt_set` module is compiled into
+  kernel #2820/#2821
+* Detect the absence of the required `xt_set` kernel module #2821
+
+## Release 1.9.1
+
+Bug fixes and minor improvements
+
+* Fix a race condition when the Weave Net container is restarted
+  which could allow a new container to be allocated the same IP
+  address as an existing one #2784,#2787
+* Handle the message type received when a pod has been deleted during
+  Kubernetes api-server fail-over #2772,#2773
+* Make weave-kube work with `dockerd --iptables=false` #2726
+* Ensure we have the right kernel modules loaded for Network Policy in weave-kube #2819
+* Reference-count addresses in Network Policy Controller, to avoid
+  errors when updates come in an unexpected order #2792,#2795
+* Allow the soft connection limit to be raised in weave-kube, so
+  larger clusters can be created #2781
+* WeaveDNS was incorrectly case-sensitive for reverse DNS lookups #2817,#2818
+
+Build and Testing
+
+* Scripts to create VMs to run automated tests were rewritten to use
+  Terraform and Ansible, to make it much easier to test with different
+  versions of components such as Docker and Kubernetes #2647,#2694,#2775,#2796
+* Upgrade to latest Weaveworks common build-tools #2780
+* Improve encryption tests #2793
+* Update vishvananda/netlink library to bring in changes we had previously forked #2790
+* Slight change to the build container to avoid permission errors and slow builds #2761,#2802
+
+## Release 1.9.0
+
+Highlights:
+
+* Encryption is now available for Fast Datapath connections, which
+  greatly improves the performance. #1644,#2687
+* We now build images for Intel/AMD 64-bit, ARM and ARM 64-bit. #2713
+* Weave Net Docker images are now labelled with description, vendor, etc. #2712
+* `weave status connections` now shows the MTU #2389,#2663
+* CNI plugin is now a stand-alone binary that does not depend on Docker #2594,#2662
+* Embedded docker client updated to version 1.10.3 #2395
+
+**NOTE:** The move to multi-architecture required that we update the
+  embedded Docker client, and this has the effect that this release of
+  Weave Net will not work with Docker installations older than
+  1.10.
+
+[Full list of changes](https://github.com/weaveworks/weave/issues?q=milestone%3A1.9.0).
+
 ## Release 1.8.2
 
 Bug fixes and minor improvements

DEPENDENCIES

@@ -0,0 +1,3 @@
+DOCKER_VERSION=1.11.2
+KUBERNETES_VERSION=1.5.2
+KUBERNETES_CNI_VERSION=0.3.0.1

Makefile

@@ -1,7 +1,5 @@
-PUBLISH=publish_weave publish_weaveexec publish_plugin publish_weave-kube publish_weave-npc
-
 .DEFAULT: all
-.PHONY: all exes testrunner update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests
+.PHONY: all exes testrunner update tests lint publish-one-arch $(PUBLISH) clean clean-bin prerequisites build run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
 SUDO=$(shell docker info >/dev/null 2>&1 || echo "sudo -E")
@@ -10,9 +8,82 @@ RM=--rm
 RUN_FLAGS=-ti
 COVERAGE=
 
-DOCKERHUB_USER=weaveworks
-WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
+# This specifies the architecture we're building for
+ARCH?=amd64
 
+# We're using QEMU to be able to build Docker images for other architectures
+# from amd64 machines.
+QEMU_VERSION=v2.7.0
+
+# A list of all supported architectures here. Should be named as Go is naming platforms
+# All supported architectures must have an "ifeq" block below that customizes the parameters
+ALL_ARCHITECTURES=amd64 arm arm64
+ML_PLATFORMS=linux/amd64,linux/arm,linux/arm64
+
+ifeq ($(ARCH),amd64)
+# The architecture to use when downloading the docker binary
+	WEAVEEXEC_DOCKER_ARCH?=x86_64
+
+# The name of the alpine baseimage to use as the base for weave images
+	ALPINE_BASEIMAGE?=alpine:3.4
+
+# The extension for the made images
+# Specifying none means for example weaveworks/weave:latest
+	ARCH_EXT?=
+
+# The name of the gcc we're using for compiling C code
+	CC=gcc
+
+# Optional parameters that can be passed when linking C code into the Go binary
+	CGO_LDFLAGS=
+endif
+ifeq ($(ARCH),arm)
+# The architecture to use when downloading the docker binary
+	WEAVEEXEC_DOCKER_ARCH?=armel
+
+# Using the (semi-)official alpine image
+	ALPINE_BASEIMAGE?=armhf/alpine:3.4
+
+# arm images have the -arm suffix, for instance weaveworks/weave-arm:latest
+	ARCH_EXT?=-arm
+
+# The name of the gcc binary
+	CC=arm-linux-gnueabihf-gcc
+
+# The architecture name to use when downloading a prebuilt QEMU binary
+	QEMUARCH=arm
+
+# In the weaveworks/build image; libpcap libraries for arm are placed here
+# Tell the gcc linker to search for libpcap here
+	CGO_LDFLAGS="-L/usr/local/lib/$(CC)"
+endif
+ifeq ($(ARCH),arm64)
+# Use the arm 32-bit docker client variant
+	WEAVEEXEC_DOCKER_ARCH?=armel
+
+# Using the (semi-)official alpine image
+	ALPINE_BASEIMAGE?=aarch64/alpine:3.5
+
+# arm64 images have the -arm64 suffix, for instance weaveworks/weave-arm64:latest
+	ARCH_EXT?=-arm64
+
+# The name of the gcc binary
+	CC=aarch64-linux-gnu-gcc
+
+# The architecture name to use when downloading a prebuilt QEMU binary
+	QEMUARCH=aarch64
+
+# In the weaveworks/build image; libpcap libraries for arm64 are placed here
+# Tell the gcc linker to search for libpcap here
+	CGO_LDFLAGS="-L/usr/local/lib/$(CC)"
+endif
+
+# The name of the user that this Makefile should produce image artifacts for. Can/should be overridden
+DOCKERHUB_USER?=weaveworks
+# The default version that's chosen when pushing the images. Can/should be overridden
+WEAVE_VERSION?=git-$(shell git rev-parse --short=12 HEAD)
+
+# Paths to all relevant binaries that should be compiled
 WEAVER_EXE=prog/weaver/weaver
 WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
 SIGPROXY_EXE=prog/sigproxy/sigproxy
@@ -24,35 +95,43 @@ WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast
 WEAVEUTIL_EXE=prog/weaveutil/weaveutil
 PLUGIN_EXE=prog/plugin/plugin
 RUNNER_EXE=tools/runner/runner
+MANIFEST_TOOL_DIR=vendor/github.com/estesp/manifest-tool
+MANIFEST_TOOL_EXE=$(MANIFEST_TOOL_DIR)/manifest-tool
 TEST_TLS_EXE=test/tls/tls
 
-EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(TEST_TLS_EXE)
+# All binaries together in a list
+EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(KUBEPEERS_EXE) $(WEAVENPC_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(RUNNER_EXE) $(TEST_TLS_EXE) $(MANIFEST_TOOL_EXE)
 
+# These stamp files are used to mark the current state of the build; whether an image has been built or not
 BUILD_UPTODATE=.build.uptodate
-WEAVER_UPTODATE=.weaver.uptodate
-WEAVEEXEC_UPTODATE=.weaveexec.uptodate
-PLUGIN_UPTODATE=.plugin.uptodate
-WEAVEKUBE_UPTODATE=.weavekube.uptodate
-WEAVENPC_UPTODATE=.weavenpc.uptodate
+WEAVER_UPTODATE=.weaver$(ARCH_EXT).uptodate
+WEAVEEXEC_UPTODATE=.weaveexec$(ARCH_EXT).uptodate
+PLUGIN_UPTODATE=.plugin$(ARCH_EXT).uptodate
+WEAVEKUBE_UPTODATE=.weavekube$(ARCH_EXT).uptodate
+WEAVENPC_UPTODATE=.weavenpc$(ARCH_EXT).uptodate
 WEAVEDB_UPTODATE=.weavedb.uptodate
 
-IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE) $(WEAVEDB_UPTODATE)
+IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(WEAVEKUBE_UPTODATE) $(WEAVENPC_UPTODATE)
 
-WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
-WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
-PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin
-WEAVEKUBE_IMAGE=$(DOCKERHUB_USER)/weave-kube
-WEAVENPC_IMAGE=$(DOCKERHUB_USER)/weave-npc
-BUILD_IMAGE=$(DOCKERHUB_USER)/weavebuild
+# The names of the images. Note that the images for other architectures than amd64 have a suffix in the image name.
+WEAVER_IMAGE=$(DOCKERHUB_USER)/weave$(ARCH_EXT)
+WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec$(ARCH_EXT)
+PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin$(ARCH_EXT)
+WEAVEKUBE_IMAGE=$(DOCKERHUB_USER)/weave-kube$(ARCH_EXT)
+WEAVENPC_IMAGE=$(DOCKERHUB_USER)/weave-npc$(ARCH_EXT)
+BUILD_IMAGE=weaveworks/weavebuild
 WEAVEDB_IMAGE=$(DOCKERHUB_USER)/weavedb
 
 IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(PLUGIN_IMAGE) $(WEAVEKUBE_IMAGE) $(WEAVENPC_IMAGE) $(WEAVEDB_IMAGE)
 
-WEAVE_EXPORT=weave.tar.gz
+PUBLISH=publish_weave publish_weaveexec publish_plugin publish_weave-kube publish_weave-npc
+PUSH_ML=push_ml_weave push_ml_weaveexec push_ml_plugin push_ml_weave-kube push_ml_weave-npc
 
-WEAVEEXEC_DOCKER_VERSION=1.6.2
-DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
-DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
+WEAVE_EXPORT=weave$(ARCH_EXT).tar.gz
+
+DOCKER_VERSION=1.10.3
+DOCKER_DISTRIB=prog/weaveexec/docker-$(DOCKER_VERSION).tgz
+DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/$(WEAVEEXEC_DOCKER_ARCH)/docker-$(DOCKER_VERSION).tgz
 NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	rm $@; \
 	echo "\nYour go standard library was built without the 'netgo' build tag."; \
@@ -61,6 +140,9 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	echo "    sudo go install -tags netgo std"; \
 	false; \
 }
+# The flags we are passing to go build. -extldflags -static for making a static binary, 
+# -linkmode external for linking external C libraries into the binary, -X main.version for telling the
+# Go binary which version it is, -tags netgo for enforcing the native Go DNS resolver
 BUILD_FLAGS=-i -ldflags "-linkmode external -extldflags -static -X main.version=$(WEAVE_VERSION)" -tags netgo
 
 PACKAGE_BASE=$(shell go list -e ./)
@@ -71,12 +153,14 @@ testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE)
 $(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go
 $(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go
 $(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go
-$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go
+$(WEAVEUTIL_EXE): prog/weaveutil/*.go net/*.go plugin/net/*.go plugin/ipam/*.go db/*.go
 $(SIGPROXY_EXE): prog/sigproxy/*.go
 $(KUBEPEERS_EXE): prog/kube-peers/*.go
 $(WEAVENPC_EXE): prog/weave-npc/*.go npc/*.go npc/*/*.go
 $(PLUGIN_EXE): prog/plugin/*.go plugin/*/*.go api/*.go common/*.go common/docker/*.go net/*.go
 $(TEST_TLS_EXE): test/tls/*.go
+$(RUNNER_EXE): tools/runner/*.go
+$(MANIFEST_TOOL_EXE): $(MANIFEST_TOOL_DIR)/*.go
 $(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
 $(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
 $(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go
@@ -85,14 +169,18 @@ lint: tools/.git
 
 ifeq ($(BUILD_IN_CONTAINER),true)
 
+# This make target compiles all binaries inside of the weaveworks/build container
+# It bind-mounts the source into the container and passes all important variables
 exes $(EXES) tests lint: $(BUILD_UPTODATE)
 	git submodule update --init
+# Containernetworking has another copy of vishvananda/netlink which leads to duplicate definitions
+	-@rm -r vendor/github.com/containernetworking/cni/vendor
 	@mkdir -p $(shell pwd)/.pkg
 	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
 	    -v $(shell pwd):/go/src/github.com/weaveworks/weave \
 		-v $(shell pwd)/.pkg:/go/pkg \
-		-e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \
-		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) $@
+		-e GOARCH=$(ARCH) -e CGO_ENABLED=1 -e GOOS=linux -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW -e DEBUG \
+		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) CC=$(CC) QEMUARCH=$(QEMUARCH) CGO_LDFLAGS=$(CGO_LDFLAGS) $@
 
 else
 
@@ -121,29 +209,48 @@ $(WEAVEWAIT_NOMCAST_EXE):
 
 # These programs need a separate rule as they fail the netgo check in
 # the main build stanza due to not importing net package
-$(SIGPROXY_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE):
+$(SIGPROXY_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE) $(RUNNER_EXE) $(MANIFEST_TOOL_EXE):
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 
 tests:
-	./tools/test -no-go-get
+	./tools/test -no-go-get -netgo -timeout 8m
 
 lint:
-	./tools/lint -nocomment -notestpackage .
+	./tools/lint -nocomment -notestpackage
 
 endif
 
+# This rule makes sure the build image is up-to-date.
+# It also makes sure the multiarch hooks are reqistered in the kernel so the QEMU emulation works
 $(BUILD_UPTODATE): build/*
 	$(SUDO) docker build -t $(BUILD_IMAGE) build/
+	$(SUDO) docker run --rm --privileged multiarch/qemu-user-static:register --reset
 	touch $@
 
+# Creates the Dockerfile.your-user-here file from the template
+# Also replaces all placeholders with real values
+# If the architecture is amd64, it deletes all CROSS_BUILD lines
+# but otherwise, it only removes the "CROSS_BUILD_" placeholder and downloads QEMU
 %/Dockerfile.$(DOCKERHUB_USER): %/Dockerfile.template
-	sed -e "s/DOCKERHUB_USER/$(DOCKERHUB_USER)/" $^ > $@
+	echo "DOCKERHUB_USER|$(DOCKERHUB_USER)|g;s|ARCH_EXT|$(ARCH_EXT)|g;s|ALPINE_BASEIMAGE|$(ALPINE_BASEIMAGE)|g;s|QEMUARCH|$(QEMUARCH)"
+	sed -e "s|DOCKERHUB_USER|$(DOCKERHUB_USER)|g;s|ARCH_EXT|$(ARCH_EXT)|g;s|ALPINE_BASEIMAGE|$(ALPINE_BASEIMAGE)|g;s|QEMUARCH|$(QEMUARCH)|g" $^ > $@
+ifeq ($(ARCH),amd64)
+# When building "normally" for amd64, remove the whole line, it has no part in the amd64 image
+	sed -i "/CROSS_BUILD_/d" $@
+else
+# When cross-building, only the placeholder "CROSS_BUILD_" should be removed
+# Register /usr/bin/qemu-ARCH-static as the handler for ARM binaries in the kernel
+	curl -sSL https://github.com/multiarch/qemu-user-static/releases/download/$(QEMU_VERSION)/x86_64_qemu-$(QEMUARCH)-static.tar.gz | tar -xz -C $(shell dirname $@)
+	cd $(shell dirname $@) && sha256sum -c $(shell pwd)/build/shasums/qemu-$(QEMUARCH)-static.sha256sum
+	sed -i "s/CROSS_BUILD_//g" $@
+endif
 
+# The targets below builds the weave images
 $(WEAVER_UPTODATE): prog/weaver/Dockerfile.$(DOCKERHUB_USER) $(WEAVER_EXE) $(WEAVEEXEC_UPTODATE)
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaver/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVER_IMAGE) prog/weaver
 	touch $@
 
-$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE)
+$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE)
 	cp weave prog/weaveexec/weave
 	cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy
 	cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy
@@ -151,8 +258,9 @@ $(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER
 	cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop
 	cp $(WEAVEWAIT_NOMCAST_EXE) prog/weaveexec/weavewait_nomcast
 	cp $(WEAVEUTIL_EXE) prog/weaveexec/weaveutil
-	cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
+	tar -xf $(DOCKER_DISTRIB) usr/local/bin/docker -O > prog/weaveexec/docker
+	chmod +x prog/weaveexec/docker
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -f prog/weaveexec/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEEXEC_IMAGE) prog/weaveexec
 	touch $@
 
 $(PLUGIN_UPTODATE): prog/plugin/Dockerfile.$(DOCKERHUB_USER) $(PLUGIN_EXE) $(WEAVER_UPTODATE)
@@ -164,45 +272,87 @@ $(WEAVEKUBE_UPTODATE): prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) prog/weave-k
 	$(SUDO) docker build -f prog/weave-kube/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVEKUBE_IMAGE) prog/weave-kube
 	touch $@
 
-$(WEAVENPC_UPTODATE): prog/weave-npc/Dockerfile $(WEAVENPC_EXE) prog/weave-npc/ulogd.conf
-	$(SUDO) docker build -f prog/weave-npc/Dockerfile -t $(WEAVENPC_IMAGE) prog/weave-npc
+$(WEAVENPC_UPTODATE): prog/weave-npc/Dockerfile.$(DOCKERHUB_USER) $(WEAVENPC_EXE) prog/weave-npc/ulogd.conf
+	$(SUDO) docker build -f prog/weave-npc/Dockerfile.$(DOCKERHUB_USER) -t $(WEAVENPC_IMAGE) prog/weave-npc
 	touch $@
 
 $(WEAVEDB_UPTODATE): prog/weavedb/Dockerfile
 	$(SUDO) docker build -t $(WEAVEDB_IMAGE) prog/weavedb
 	touch $@
 
-$(WEAVE_EXPORT): $(IMAGES_UPTODATE)
+$(WEAVE_EXPORT): $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE)
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@
 
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
+	cd $(shell dirname $@) && sha256sum -c $(shell pwd)/build/shasums/docker-tgz-$(WEAVEEXEC_DOCKER_ARCH).sha256sum
 
-tools/.git:
+tools/.git $(MANIFEST_TOOL_DIR)/.git:
 	git submodule update --init
 
-$(RUNNER_EXE): tools/.git
-	GO15VENDOREXPERIMENT=1 make -C tools/runner
+# CODE FOR PUBLISHING THE IMAGES
 
+# This target first runs "make publish" for each architecture
+# Then it pushes the manifest lists
+publish: $(addprefix sub-publish-,$(ALL_ARCHITECTURES)) $(WEAVEDB_UPTODATE)
+	$(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) $(PUSH_ML)
+ifeq ($(PUBLISH_WEAVEDB),true)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/weavedb:latest
+endif
+
+sub-publish-%:
+	$(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) clean-bin
+	$(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) $(IMAGES_UPTODATE)
+	$(MAKE) ARCH=$* DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) publish-one-arch
+
+publish-one-arch: $(PUBLISH)
+
+# This rule handles the pushing of the built images
 $(PUBLISH): publish_%: $(IMAGES_UPTODATE)
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+# Tag :latest with the real version
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$*$(ARCH_EXT) $(DOCKERHUB_USER)/$*$(ARCH_EXT):$(WEAVE_VERSION)
+# Push the image with the arch suffix for arm and arm64, and without suffix for amd64
+	$(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) push_$*$(ARCH_EXT)
+
+ifeq ($(ARCH),amd64)
+# If the architecture is amd64, add the -amd64 suffix.
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION) $(DOCKERHUB_USER)/$*-amd64:$(WEAVE_VERSION)
+# If the version is latest, tag the -amd64 with latest as well
+ifneq ($(UPDATE_LATEST),false)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag  $(DOCKERHUB_USER)/$*-amd64:$(WEAVE_VERSION) $(DOCKERHUB_USER)/$*-amd64:latest
+endif
+# Push the image with the -amd64-suffix so BINARY-ARCH-named images exist for all arches, so manifest lists may be made and ARCH is replaceable in scripts
+	$(MAKE) DOCKER_HOST=$(DOCKER_HOST) DOCKERHUB_USER=$(DOCKERHUB_USER) WEAVE_VERSION=$(WEAVE_VERSION) UPDATE_LATEST=$(UPDATE_LATEST) push_$*-amd64
+endif
+
+# This target pushes an image, and if UPDATE_LATEST is anything but false, also updates the latest tag
+# It takes one parameter, the image name
+push_%:
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
 ifneq ($(UPDATE_LATEST),false)
 	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push $(DOCKERHUB_USER)/$*:latest
 endif
 
-publish: $(PUBLISH)
-ifeq ($(PUBLISH_WEAVEDB),true)
-	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push   $(DOCKERHUB_USER)/weavedb:latest
+# This target pushes a manifest list; it takes one parameter, the image name.
+# The variable UPDATE_LATEST controls whether it updates the versioned tag and/or the latest tag
+$(PUSH_ML): push_ml_%: $(MANIFEST_TOOL_EXE)
+ifneq ($(UPDATE_LATEST),latest-only)
+	$(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:$(WEAVE_VERSION) --target $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+endif
+ifneq ($(UPDATE_LATEST),false)
+# Push the manifest list to :latest as well
+	$(MANIFEST_TOOL_EXE) push from-args --platforms $(ML_PLATFORMS) --template $(DOCKERHUB_USER)/$*-ARCH:latest --target $(DOCKERHUB_USER)/$*:latest
 endif
 
 clean-bin:
 	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES)
-	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg
+	find prog -type f -name "Dockerfile.*" -not -name "Dockerfile.template" -print | xargs rm -f
+	find prog -type f -name "*qemu-*" -print | xargs rm -f
+	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVEDB_UPTODATE) weave*.tar.gz $(DOCKER_DISTRIB) prog/weaveexec/docker .pkg
 
 clean: clean-bin
 	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(BUILD_IMAGE)
-	rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE)
+	rm -rf test/tls/*.pem test/coverage.* test/coverage $(BUILD_UPTODATE) $(MANIFEST_TOOL_EXE)
 
 build:
 	$(SUDO) go clean -i net
@@ -211,3 +361,31 @@ build:
 
 run-smoketests: all testrunner
 	cd test && ./setup.sh && ./run_all.sh
+
+integration-tests: all testrunner
+# Usage:
+#   $ make \
+#     NAME="<prefix used to name VMs and other resources>" \
+#     PROVIDER="<provider among {vagrant|gcp|aws|do}>" \
+#     NUM_HOSTS="<# test machines>" \
+#     PLAYBOOK="<filename>" \
+#     RUNNER_ARGS="<...>" \
+#     TESTS="<...>" \  # Can be set to only run one or a few tests instead of the full test suite.
+#     DOCKER_VERSION=<...> \
+#     KUBERNETES_VERSION=<...> \
+#     KUBERNETES_CNI_VERSION=<...> \
+#     <...> # See also run-integration-test.sh for all variables and individual functions.
+#     integration-tests
+#
+	RUNNER_ARGS="-parallel" ./test/run-integration-tests.sh
+
+# This target is only intended for use in Netlify CI environment for generating preview pages on feature branches and pull requests.
+# We need to obtain website repository (checked out under `site-build`) and place `site` directory into the context (`site-build/_weave_net_docs`).
+# We then run make in `site-build` and Netlify will publish the output (`site-build/_site`).
+netlify-site-preview:
+	@mkdir -p site-build
+	@curl --user $(WEBSITE_GITHUB_USER) --silent 'https://codeload.github.com/weaveworks/website-next/tar.gz/$(WEBSITE_BRANCH)' \
+	  | tar --extract --gunzip --directory site-build --strip 1
+	@cp -r site site-build/_weave_net_docs
+	@$(MAKE) -C site-build netlify_ensure_install
+	@$(MAKE) -C site-build BUILD_ENV=netlify

README.md

@@ -11,9 +11,11 @@
 [Weaveworks](https://www.weave.works) is the company that delivers the most productive way for developers to connect, observe and control
 Docker containers.
 
-This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, and with nearly 5 million downloads to date, it enables you to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
+This repository contains [Weave Net](https://www.weave.works/products/weave-net/), the first product developed by Weaveworks, and with over 8 million downloads to date, it enables you to get started with Docker clusters and portable apps in a fraction of the time compared with other solutions.
 
-Other products from Weaveworks include [Weave Scope](https://www.weave.works/products/weave-scope/), a powerful container monitoring tool that automatically maps Docker containers and their interactions, and [Weave Flux](https://www.weave.works/products/weave-flux/), a microservice router that automates the access of containers as services.
+Weaveworks' product [Weave Cloud](https://www.weave.works/solution/cloud/) takes the heavy lifting out of building, integrating and operating open source components, so you don’t have to manage databases, storage and availability.
+
+Weave Cloud is built using these Open Source projects: [Weave Scope](https://www.weave.works/products/weave-scope/), a powerful container monitoring tool that automatically maps Docker containers and their interactions, [Weave Cortex](https://github.com/weaveworks/cortex), a horizontally-scalable version of Prometheus, and [Weave Flux](https://www.weave.works/products/weave-flux/), a continuous deployment tool that works with Kubernetes.
 
 # Weave Net
 
@@ -21,5 +23,20 @@ Weave Net creates a virtual network that connects Docker containers across multi
 
 Services provided by application containers on the Weave network can be exposed to the outside world, regardless of where they are running. Similarly, existing internal systems can be opened to accept connections from application containers irrespective of their location.
 
-* [Install Weave Net](https://www.weave.works/docs/net/latest/installing-weave/)
-* [Find out more about Weave Net](https://www.weave.works/products/weave-net/)
+## <a name="help"></a>Getting help
+
+If you have any questions about, feedback for or a problem with Weave Net:
+
+- Read [the Weave Net docs](https://www.weave.works/docs/net/latest/introducing-weave/).
+- Invite yourself to the <a href="https://weaveworks.github.io/community-slack/" target="_blank"> #weave-community </a> slack channel.
+- Ask a question on the <a href="https://weave-community.slack.com/messages/general/"> #weave-community</a> slack channel.
+- Join the <a href="https://www.meetup.com/pro/Weave/"> Weave User Group </a> and get invited to online talks, hands-on training and meetups in your area.
+- Send an email to <a href="mailto:weave-users@weave.works">weave-users@weave.works</a>
+- <a href="https://github.com/weaveworks/weave/issues/new">File an issue.</a>
+
+Your feedback is always welcome!
+
+##Further Information
+
+* [Installing Weave Net](https://www.weave.works/docs/net/latest/installing-weave/)
+* [Weave Net Product Overview](https://www.weave.works/products/weave-net/)

Vagrantfile

@@ -1,45 +1,36 @@
-VAGRANTFILE_API_VERSION = "2"
-
-require './vagrant-common.rb'
+require File.expand_path(File.join(File.dirname(__FILE__), 'vagrant-common.rb'))
 
 vm_ip = "172.16.0.3" # arbitrary private IP
 
-pkgs = %w(
-  docker-engine=1.10.2-0~wily
-  aufs-tools
-  build-essential
-  ethtool
-  iputils-arping
-  libpcap-dev
-  git
-  mercurial
-  bc
-  jq
-)
-
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.box = "ubuntu/wily64"
+  config.vm.box = VAGRANT_IMAGE
 
   config.vm.network "private_network", ip: vm_ip
   config.vm.provider :virtualbox do |vb|
     vb.memory = 2048
-    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "off"]
-    vb.customize ["modifyvm", :id, "--natdnsproxy1", "off"]
+    configure_nat_dns(vb)
   end
 
+  # Disable default Vagrant shared folder, which we don't need:
   config.vm.synced_folder ".", "/vagrant", disabled: true
+  # Keep Weave Net sources' in sync:
   config.vm.synced_folder ".", "/home/vagrant/src/github.com/weaveworks/weave"
+  # Create a convenience symlink to $HOME/src/github.com/weaveworks/weave
+  config.vm.provision :shell, :inline => 'ln -sf ~vagrant/src/github.com/weaveworks/weave ~vagrant/'
 
   # Set SSH keys up to be able to run smoke tests straightaway:
   config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/home/vagrant/src/github.com/weaveworks/weave/test/insecure_private_key"
-
-  install_build_deps config.vm, pkgs
-  install_go_toochain config.vm
-  tweak_user_env config.vm
-  tweak_docker_daemon config.vm
+  # Grant permissions on sources:
+  config.vm.provision :shell, :inline => 'sudo chown -R vagrant:vagrant ~vagrant/src', :privileged => false
   cleanup config.vm
 
+  config.vm.provision 'ansible' do |ansible|
+    ansible.playbook = 'tools/config_management/setup_weave-net_dev.yml'
+    ansible.extra_vars = {
+      go_version: GO_VERSION
+    }.merge(ansibleize(get_dependencies_version_from_file_and_env()))
+  end
 end
 
 begin

bin/circle-dependencies-post

@@ -2,39 +2,16 @@
 
 set -e
 
-echo "TEST_AND_PUBLISH=1" > "$STATE"
-
-if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ] ; then
-    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$') ; then
-        echo "GH_PR=$PR" >> "$STATE"
-        if FILES=$(curl -s https://api.github.com/repos/weaveworks/weave/pulls/$PR/files | jq -r '.[]|.filename') ; then
-            echo "PR touches [$FILES]"
-            if ! echo "$FILES" | grep -v ^site/ ; then
-                echo "PR contains site changes only - skipping tests"
-                echo "TEST_AND_PUBLISH=" >> "$STATE"
-            else
-                echo "PR contains non-site changes - running tests"
-            fi
-        else
-            echo "Unable to get files for PR $PR" >&2
-        fi
-    else
-        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
-    fi
-fi
-
 source "$STATE"
 
-mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR
-
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    cd $SRCDIR; git submodule update --init
-    if [ "$CIRCLE_NODE_INDEX" = "0" ] ; then
-        cd $SRCDIR/test; ./gce.sh make_template
-    fi
-    cd $SRCDIR/build
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+    cd "$SRCDIR/build"
     ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh
-    touch $SRCDIR/.build.uptodate
+    touch "$SRCDIR/.build.uptodate"
     sudo chown -R ubuntu /usr/local/go
+    cd "$SRCDIR"
+    make testrunner
+    make COVERAGE=true RM= exes all
 fi

bin/circle-dependencies-post-sources

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+
+echo "TEST_AND_PUBLISH=1" >"$STATE"
+
+# Avoid doing a full build if we can.  Note if the PR doesn't exist
+# when the build is kicked off, we can't run this check so we do a
+# full build.  Subsequent pushes to the same branch will have the desired behaviour
+if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then
+    if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then
+        echo "GH_PR=$PR" >>"$STATE"
+        if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then
+            echo "PR touches [$FILES]"
+            if ! echo "$FILES" | grep -v ^site/; then
+                echo "PR contains site changes only - skipping tests"
+                echo "TEST_AND_PUBLISH=" >>"$STATE"
+            else
+                echo "PR contains non-site changes - running tests"
+            fi
+        else
+            echo "Unable to get files for PR $PR" >&2
+        fi
+    else
+        echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2
+    fi
+fi
+
+source "$STATE"
+
+mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR"
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    cd "$SRCDIR"
+    git submodule update --init
+fi

bin/circle-deploy-issues

@@ -1,43 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o pipefail
-
-source "$STATE"
-
-site_changes() {
-    curl -s https://api.github.com/repos/weaveworks/weave/pulls/$1/files |
-        jq -r '.[]|.status + " " + .filename' |
-        grep -oP '^(added|modified|removed) site/.*(?=\.md$)'
-}
-
-format_comment() {
-    if [ ! -d .git ] || ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ] ; then
-        echo "Could not determine branch" >&2
-        return 1
-    fi
-
-    echo "#### Documentation changes:"
-    while read ACTION PAGE; do
-        echo "$ACTION [${PAGE#site/}]($WP_DEV_URL/docs/net/${BRANCH#issues/}/${PAGE#site/}/)"
-    done | sort
-}
-
-post_comment() {
-    jq -s -R '. | tostring | { body: . }' |
-        curl -s -d @- \
-            -H "Authorization: Bearer $WP_GH_COMMENT_TOKEN" \
-            -H 'Content-Type: application/json' \
-            https://api.github.com/repos/weaveworks/weave/issues/$GH_PR/comments > /dev/null
-}
-
-cd $SRCDIR
-bin/install-wordepress v1.0.0
-bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"
-
-if [ -n "$WP_GH_COMMENT_TOKEN" -a -n "$GH_PR" ] ; then
-    echo ">>> Adding doc changelog to GitHub PR $GH_PR"
-    if CHANGES=$(site_changes $GH_PR) && [ -n "$CHANGES" ] ; then
-        echo "$CHANGES" | format_comment | post_comment
-    fi
-fi

bin/circle-deploy-master

@@ -8,9 +8,6 @@ if [ -n "$TEST_AND_PUBLISH" ] ; then
     docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
     cd $SRCDIR
     make clean-bin
-    make publish
+    # CircleCI doesn't let us install the qemu module to do ARM docker builds
+    make publish ALL_ARCHITECTURES=amd64 UPDATE_LATEST=latest-only
 fi
-
-cd $SRCDIR
-bin/install-wordepress v1.0.0
-bin/publish-site "$WP_DEV_URL" "$WP_DEV_USER" "$WP_DEV_PASSWORD"

bin/circle-deploy-release

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -e
-
-cd $SRCDIR
-bin/install-wordepress v1.0.0
-bin/publish-site "$WP_LIVE_URL" "$WP_LIVE_USER" "$WP_LIVE_PASSWORD"

bin/circle-test-pre

@@ -2,13 +2,29 @@
 
 set -e
 
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_READY_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR
-    make testrunner
-    make COVERAGE=true RM= all
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
 
-    cd $SRCDIR/test
-    ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+
+    # Upload built binaries to testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_READY_LOCK_FILE"
+    echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
 fi

bin/circle-test-smoke

@@ -3,10 +3,21 @@
 set -e
 
 source "$STATE"
+source "$(dirname "$0")/wait_for.sh"
 
-if [ -n "$TEST_AND_PUBLISH" ] ; then
-    cd $SRCDIR/test
-    eval $(./gce.sh hosts)
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    # Provisioning from image typically take 90 seconds, 
+    # but if we are creating a GCP image, it can take more than 10 minutes:
+    wait_for 900 "$TEST_VMS_READY_LOCK_FILE"
+
+    # Run integration tests:
     export COVERAGE=true
-    ./run_all.sh
+    export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log"
+    cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs.
+    ./run-integration-tests.sh test
 fi

bin/circle-test-teardown

@@ -6,6 +6,5 @@ source "$STATE"
 
 if [ -n "$TEST_AND_PUBLISH" ] ; then
     cd $SRCDIR/test
-    ./gce.sh destroy
     test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh)
 fi

bin/circle-test-unit

@@ -6,6 +6,6 @@ source "$STATE"
 
 if [ -n "$TEST_AND_PUBLISH" ] ; then
     cd $SRCDIR
-    make lint
+    make DEBUG=1 lint
     COVERDIR=test/coverage make RM= tests
 fi

bin/install-wordepress

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-if [ $# -ne 1 ] ; then
-    echo USAGE: install-wordepress [COMMIT-ISH] >&2
-    exit 1
-fi
-
-go get github.com/weaveworks/wordepress
-cd /home/ubuntu/src/github.com/weaveworks/wordepress
-git checkout $1
-cd cmd/wordepress
-go get

bin/provision_test_vms.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+
+# Signal failures in lock file, in order to fail fast:
+function signal_failure() {
+    echo "KO" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    exit 1
+}
+trap signal_failure ERR
+
+source "$STATE"
+
+function install_terraform() {
+    curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin
+}
+
+function install_ansible() {
+    sudo apt-get update || true
+    sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev && pip install --user -U cffi && pip install --user ansible
+    export PATH="$PATH:$HOME/.local/bin"
+}
+
+if [ -n "$TEST_AND_PUBLISH" ]; then
+    [ -n "$SECRET_KEY" ] || {
+        echo "Cannot run smoke tests: no secret key"
+        exit 1
+    }
+
+    install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+
+    # Only attempt to create GCP image in first container, wait for it to be created otherwise:
+    [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0
+
+    # Provision and configure testing VMs:
+    cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests.
+    ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1
+    echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE"
+    echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE"
+fi

bin/publish-site

@@ -1,46 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o pipefail
-
-PRODUCT=net
-
-fatal() {
-    echo "$@" >&2
-    exit 1
-}
-
-if [ ! -d .git ] ; then
-    fatal "Current directory is not a git clone"
-fi
-
-if ! BRANCH=$(git symbolic-ref --short HEAD) || [ -z "$BRANCH" ] ; then
-    fatal "Could not determine branch"
-fi
-
-case "$BRANCH" in
-    issues/*)
-        VERSION="${BRANCH#issues/}"
-        TAGS="$VERSION"
-        ;;
-    *)
-        if echo "$BRANCH" | grep -qE '^[0-9]+\.[0-9]+' ; then
-            DESCRIBE=$(git describe --match 'v*')
-            if ! VERSION=$(echo "$DESCRIBE" | grep -oP '(?<=^v)[0-9]+\.[0-9]+\.[0-9]+') ; then
-                fatal "Could not infer latest $BRANCH version from $DESCRIBE"
-            fi
-            TAGS="latest"
-        else
-            VERSION="$BRANCH"
-            TAGS="$VERSION"
-        fi
-        ;;
-esac
-
-for TAG in $TAGS ; do
-    echo ">>> Publishing $PRODUCT $VERSION to $1/docs/$PRODUCT/$TAG"
-    wordepress \
-        --url "$1" --user "$2" --password "$3" \
-        --product "$PRODUCT" --version "$VERSION" --tag "$TAG" \
-        publish site
-done

bin/release

@@ -89,6 +89,8 @@ build() {
     ## Inject the version numbers and build the distributables
     ## (library versions?)
     sed -i "/SCRIPT_VERSION=/ c\SCRIPT_VERSION=\"$VERSION\"" ./weave
+    sed -i -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset.yaml
+    sed -i -e "s/:latest/:$VERSION/" -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset-k8s-1.6.yaml
     make SUDO=$SUDO WEAVE_VERSION=$VERSION DOCKERHUB_USER=$DOCKERHUB_USER
 
     if make tests; then
@@ -157,17 +159,9 @@ draft() {
         --name "weave" \
         --file "./weave"
 
-    github-release release $RELEASE_ARGS \
-        --user $GITHUB_USER \
-        --repo weave-kube \
-        --tag $LATEST_TAG \
-        --name "$RELEASE_NAME $VERSION" \
-        --description "$RELEASE_DESCRIPTION"
-
-    sed -i -e s/:latest/:$VERSION/ -e "/imagePullPolicy: Always/d" ./prog/weave-kube/weave-daemonset.yaml
     github-release upload \
         --user $GITHUB_USER \
-        --repo weave-kube \
+        --repo weave \
         --tag $LATEST_TAG \
         --name "weave-daemonset.yaml" \
         --file "./prog/weave-kube/weave-daemonset.yaml"
@@ -176,8 +170,8 @@ draft() {
         --user $GITHUB_USER \
         --repo weave \
         --tag $LATEST_TAG \
-        --name "weave-daemonset.yaml" \
-        --file "./prog/weave-kube/weave-daemonset.yaml"
+        --name "weave-daemonset-k8s-1.6.yaml" \
+        --file "./prog/weave-kube/weave-daemonset-k8s-1.6.yaml"
 
     echo "** Draft $TYPE $RELEASE_NAME $VERSION created at"
     echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG"
@@ -258,29 +252,9 @@ publish() {
             --name "weave" \
             --file "./weave"
 
-        github-release publish \
-            --user $GITHUB_USER \
-            --repo weave-kube \
-            --tag $LATEST_TAG
-
-        if github-release info --user $GITHUB_USER --repo weave-kube \
-            --tag latest_release >/dev/null 2>&1; then
-            github-release delete \
-                --user $GITHUB_USER \
-                --repo weave-kube \
-                --tag latest_release
-        fi
-
-        github-release release \
-            --user $GITHUB_USER \
-            --repo weave-kube \
-            --tag latest_release \
-            --name "$RELEASE_NAME latest ($VERSION)" \
-            --description "[Release Notes](https://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG)"
-
         github-release upload \
             --user $GITHUB_USER \
-            --repo weave-kube \
+            --repo weave \
             --tag latest_release \
             --name "weave-daemonset.yaml" \
             --file "./prog/weave-kube/weave-daemonset.yaml"
@@ -289,8 +263,8 @@ publish() {
             --user $GITHUB_USER \
             --repo weave \
             --tag latest_release \
-            --name "weave-daemonset.yaml" \
-            --file "./prog/weave-kube/weave-daemonset.yaml"
+            --name "weave-daemonset-k8s-1.6.yaml" \
+            --file "./prog/weave-kube/weave-daemonset-k8s-1.6.yaml"
 
         echo "** Release $RELEASE_NAME $VERSION published at"
         echo -e "\thttps://github.com/$GITHUB_USER/weave/releases/$LATEST_TAG"

bin/setup-ci-secrets

@@ -1,57 +0,0 @@
-#!/bin/sh
-
-## This is the shape of unencrypted `do-setup-ci-secrets`
-## 
-## cat > /tmp/boto_gsutil_cert.pem << EOF
-## -----BEGIN PRIVATE KEY-----
-## <...>
-## -----END PRIVATE KEY-----
-## EOF
-## 
-## cat > ~/.boto << EOF
-## [Credentials]
-## gs_service_client_id = <...>
-## gs_service_key_file = /tmp/boto_gsutil_cert.pem
-## 
-## [Boto]
-## https_validate_certificates = True
-## 
-## [GSUtil]
-## content_language = en
-## default_api_version = 2
-## default_project_id = <...>
-## EOF
-
-openssl base64 -d << EOF \
-  | openssl aes-256-cbc -d -K $1 -iv $2 \
-    -out bin/do-setup-ci-secrets
-mOYNzNsAdwva0LbB2YFMY9VgL0SITFu1bPrbOujyLtOrBNjmOYbzR5d/sG+ARMFr
-7NZ5OKcvMbItllz0n7TtNkzkCOMztMC870Wcx3nGJzbBephdcYwos4tBlp5bihTU
-AXgxLUiDT1broYAFKnbE1aZC/XyNxuLTN+nXJWIWeijaXTgvhkEEm3tab+z7ioCb
-tlg71W1BrUuocctAJAuVwH/uTZAaXaJGJj1s2S7F2DrFnG6sFlrp8rnCucPLWKnC
-4IwJFC5saR5gxK6wECffwmwbpeXVOoQ58OgzxoZTEyAr9nhVbaBt5UZz3tfHlx8W
-Rn5jHehACw5kg5eMtGPqMEksJ63LX4h3mfSTLJXJHEmOrzCRpFrigVrsbA408U2u
-8jJTHJmKrGfwBh5ZFzBg6NuV4VplOp+ZquJLgegv2WH/iu/ciopXqB2YDCdx5ZBi
-nHV2Puth7kjS4EMCKUGfeqpesjL9Djb8cygzv/J3ZBmLjNH9dqlSXWy2DcFZA1Iy
-pFTTHkVU3w5sNz3nYOuT2UPQvULBW0FUndDKX6olyVa0mlxPwGBP9N3yGg918/rw
-7rNV/r54lA334F5QMhDrVxar6sl9wfLA+Z+8qemBbNFlLog1uQGhuwWK2nFu1tRM
-9uyvQIq0oq5NHKpogWc7NEuk7CPe1JyfRIcbMVj8+ilmKjUPB2HPC43IDmKt9/56
-rY3IXAcxZ/ClGfGpeNHXqop9T/WhfaowponOZFcdGSUddG0OswqkTE4KIv71mgTF
-jXooMkTQBOG+gBnCe+8bCRdRCG3anxHX74za4dGD58cOJtl6DYWnjWuR91NWnxdT
-1Zr3iUe41txg9hCNY9LHo2QHGLPxe6pmNnSClQ1mcFhuY6NF8RmCX3HAGKJQkxJb
-vpLGbXTXjAIFgIYgLNgKP2RLvCif/LmSWgkvE3ZU9GXtCUbm6xgJodwOqXiI9uuO
-4H4+/vpGHnSFENiclkgMFrLG4yiMu8ArgmnCBzg9E2AoOX5zCrh/2ZRTU8Dk4mt7
-jUd5SKQLsjB9QtB+y3tRO7zY9RDvR6iBOPRsjx6t8EJMCfPb06KchfMJw+VGxvMM
-48jtmepREFnhd/8XDRGnw5bznjRuwohGik8BToltjIviMkkpifKWaLwlaOiDXaHh
-VBFNXnUKvOBwRMU+PZqv4+DP2qff1gSXnGKqA5GIURYrK9P7TqHHNTnmjBdfJLzc
-0x+W1DTiYhLT2vrLrkEpZ99o6qrg+lwvjat6IsJF8xoQEo4pLD/ZRPJ0WzoDyctT
-oWZ5rMsN1TzTvJrBrTAWIJyhIzZcuNwChMrU6qogMr0fd6TSxWhWn0sOJ5y8AaTR
-ej4JHx0qroS/9JPmgFYU61pX3c0ypdef/sSHWG23mcFzLTgwe2d4ty9VakypZy3T
-sXCl6EHVVXlUFWhY4zQCjdZpYJv/rtoKL1BL4fPE2NV5WTjCL2GNvaO5SfHR7pA0
-yQJR/iU+cS2n8UXsWNjyV+/qEn6+QXuyVHGLbxyH8EeqlliAbyHsc2rzVI8Xctzo
-W5DuIqxwHAkNNurhmx27ktIIQ6HhaCp028SWwvLttkbBBgF4v1mmd4u/2bKbKjUd
-dOaetaIUx1hU9OAeaePvVkeTDcc3mlFtPARCS9Fqj16hgJQ43onZyC0H1K2WAxmU
-GsmzjCHlTNquzFTuuQihYGZKfsWIh9z6HEOkvNJ3VEQ=
-EOF
-
-exec sh bin/do-setup-ci-secrets

bin/setup-circleci-secrets

@@ -1,115 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# openssl enc -in do-setup-circleci-secrets.orig -out setup-circleci-secrets.orig -e -aes256 -pass stdin
-# openssl base64 < setup-circleci-secrets.orig
-
-openssl base64 -d << EOF \
-  | openssl enc \
-    -out bin/do-setup-circleci-secrets \
-		-d -aes256 -pass pass:$1
-U2FsdGVkX1/3QHnDgpZn0srp+GiEQBTlag0CBdsw6Bp6iaIKy6Jy8gFshrsmvl3+
-c+7hn1nqsBVpzVH0S5QA3Ex5nx9rl7/dcGoogsySFcwMmBFcauT0UTMK1bvX0Hqx
-QrshYSndUkww72KA7EsSnjR1BBJWm3AhCDOTIcnO6nQhig1Fc+SBEj5FGS5xTQ5Z
-/8hDsKf54Ms49vOO6QeE5w5ndZ+k7IvVJfq9Evhb7htD20RntDhR86foAwMnLV7R
-Wh9qAqI3gnRq0GmbeRKaHmB7pcLq9f0OdScqM/Tn1D+q0556fqJjBziXpZiGbxVs
-DxEp8zDJ8yNDAi0Wb2NXjoA79SXJ8Q7iTieaST4y2OmXQpY+Vw8RVDXxi6ERyVnH
-41OeMzyLm0qvCoMl32pe1sSG5DRCUiAm7wOtDTVvNDRqmGJYUgwwexyayecJI1wQ
-LlhYLJavUvw3UGDZOld8BaMQhfOKBVCdFrjmA4HuVxmOf3029qkgzOCSZiQVNqs+
-fvM6J05zBuk+GPy8YInvRkzfxum/x00vmUyGjMs/lHDFfrmYDFK3ux41EoVqvWLG
-S5uYxdZJFdR3Xsrs+sMr51XNBQGMmz6KoEEzFYyHs5fZgtdpRdPAuQS1m6FqgZyi
-wSz6+EKsALtmGorEp+CF8JN5YzKJCrKzNkg0fILBqsqmJZGP64lmX69950lRLNGM
-/l3KVR2zH31b4AnnzTcvldVSCKDXjaMw3JA2Ce/MK6XhOWOcpEv4oDt/62lbhUKP
-QUQTPbfpQRvVfBcoIrfAm3ghi/CZc2TDVEuD7DhasLTZD3r7LoK261BWC9O5Wubi
-vlXuFIF+uyrpTDf6EJKE/+XywtArppffMP6RkHu1Iqnd8JtsqPaIZsmMmZbjgb6j
-BX4ZLb5U9R7jYoSgIke4+u6geFcrMQ+Lny4r7MxXaS0Y1UJ6Jo1bwpYw1TBIobZc
-eQ30+xhbSeRC9vEw1NXI6qM2Ai2gIHtAZvEiBkBBXTG5Ld7N/mBAmDIJEzVRxh6F
-jRkB3dzESF6eeTBm61OQepwV3aW6lslMaf26drPGv2W2q+K/WOpQOejfggtHUgVs
-9SWqtn7hHYSpIASe1Ihi14HAWv3kPhQien0zY1DLKJWf4ilbRaICn1QSJjNuKwtl
-A2ZwEw7FFddFcsF4okHPkxEoCre3shq0DEw9UA1gnwEaEF1Q+i9cvLtQg56zK3+R
-MfTzFUv2K1z3j1s5tC6Z6LzL8j682EFOll7MkihqiIWh+po1zcNs6Ln0SkjB3v9T
-OFRYVJis9GqXYO9PUW1mI69udR4nXbKlBl5tWNyAxn9hufJHLSAmbGMwActKLRM5
-kROapKrN1Ra/sHUmhoSSDUK9AXeYKkqPXVMbhly9xC1Kya4hFaFokjRQIMLPmXdQ
-lP46RL3HOq0v1BkXg05poSwDm+BtitZkcIhOzcmeucUqn1whsz39y3VoOhJqQu2j
-0dUcj19Fue1CDZfwzACb9mLZr643h0yCIHQaVT7is/yiN2vEt0S0zynvOA1D+Xgy
-4bFdjqed9DgBShLwF/lh4/hur5iZPpVr9A8rt3Vq0k5A1tdjsZpXP3qv5jLc7lJv
-cAQxdVLEuv8WPwwTV/EQsoH7lpoEdjktw/gu0v5PkUgR/WBxEqJq4liTekViQ+8X
-BAmdm/FGWeIxSSEm0a8I+muiBpIuKjuQoJit1f0k+lLUohoVDqpDDfeYHh1TxXHE
-xSKAKNmFSc2XOr5Zg2mOInxeH7WoxH6iylXKADBgEgAQ4C55DoAPRjFNtGGz2iJl
-WkzhQOfgsDss1VxpPZ4SWDQO+xxuuEnYcCYdJ6Tyi5hAwZ0+RZmXx0g9BCK+5qzc
-Ffei066kCItBcayekwCs0i86r8JvDtsMUUkzEKDQTSWHpP8ypIZoz3P+Skdqcs6p
-IcZHy2fODIsClctRAsHOjZ9BnWSY2rqf1lysqvRfOkNTViTOUVx2hf+LuL+8rT6g
-K4GvS39Q+c9FYSMjCNxv1m37NfHJv2W7AlqVV95gMBEFB97M3388T7PfvmSLsyPv
-qAKDE7genTOycQvEX9oeEACxPo08ai2Mgs/YXCig0g0btZoScoKPelQKtD0HLpZ+
-j7ruaKFy4/vySDMostB3AKSkCE/SkGu/W9oI37n1guSYC/9yJjtTcwZULXAClfwt
-LyHvLAw4+7BdGJqDe9Tf8z4zqVLax+BcBHkW7pbP+pvxC6fhIA4PO8k5HwwAVlRw
-68lG6e3z9+Vst4zrFLewHJvkDRSOTSecs9h9SPOjReTK0ovlDzMHVakfUvqpk9mv
-nCj8SWaMPF70z138AuRuKN7X4exNXPBFnOcseKp/NpT1DyvgoHm/oIO65mq9vvo+
-42uSbyyegYiTdVK4SaXBc2hixkhaTteqdEy6GxfWqKCvHtYuRD2LyUSjZn/0Y6CG
-3wZapylKL1sxfH+b1gogv2wAA8KtZAfiTlMISuymwQr5RbOFma7UVMVecCa7XrNG
-fZwO8R+6++ru0efDzeC/gtLOItjn+yVfsbk2gfu0FikHIV1sogOjnsv3dl6IpH/s
-w6WbuK/yZhyrrmhIIueHF30ES512Peogf/Gxp8U/ZAmqBa7p4qsv93XWDR52IPYz
-yU1MtboLD02J39daBNrYZuO4Yfo/xAvU7Ijg2KIR9HS0bSWNuSFAZaZXlJ2bSag3
-8l0dZMx2fZa0FALxWvY2+nd1tWcH+Wo99RSxX+4TH3guCpe1x/58yKOKL4d1xRpu
-02nGaqerV9bHU6WjCUYTrrZurTp6yqivpSFCALDMhAU5Ihd/Qjz69zg4uZ7iqbNd
-WqIYJ3SAOei2tSfoOJbdh8mohotnMFFbupck3Fv+lXAXYDp0V/sSuK32MSeJxan1
-q3t0eenAh6zoBP0Tj53N8B9YmrgZ1i1AljQqYsAdb8N5Z3vecuzQa946U8H4axEc
-BL5fgmqSfsbj1EjJVUrw3cE9hgqeqAYBBvZsyW5zSAsLq/RJKYtBxDGADqr0TOOu
-TjVqfPObb1u9EAQj15ALBhhPORa2gBAK5gnhJWW5deNgA0UOYoWARFYfATuQvr1T
-9xTVbqzZ0VJR4n7X5JyxJo2tZ44QYVzA0fqX0lt+uRkarrkbZMgplKp4MJTFCp2N
-tLciWS0YSHPbKTm7svw5mROazUvEoj1N3SYrJT+Ai9UgSL1cA4FmQ5Jsi+GOeQbH
-7E4pzjeST4b1aATY9ZZSy4TzK+UNYBcCVqY1dN5Xnk0e5yPMnrkkcYI41s/6oOaW
-lQkp/yrHkvpzQWZa7vF2yWg54j3eX8YWx/TsLvmuJmSQqiIHfu9TXcAIPDispCpF
-gp4ZbYPZ8RDQLw3ROc+bZ9E0J8e+FHwgXEELsMLW9n4d3PTj9pLFej1vmBKa03o3
-g+JViNxb1gUPNmC6ADk4o7TQzBrjV2pmmuCza16kOmq6oAbJPOeCLtVWA5HJTX8d
-AuOzBLO37tWP457dNqoNDaCMik3LCy8cslZjAz02D6VG/QGOAXP9x8bJ3zg9N7vW
-EOn1BMtIeMI4yIxpC6HgotNWDNbUBWremAFWYQRgYQqo9L1SdW4hz6X8FqdlBFvK
-ml7sdUwjEy2tNW+aiZI4cgAlI/VvMWYr5kPfhvaUu1wGYeS1TD1B+DPcjcXpOGWo
-vB1f/uvGc61zZWD0ph9f3S9OnvEko5qV3VhxOnxYUZTrqlM7LVgy/sHrSaPdR7Eo
-nb4QGdi5wEGMxVgQlDGwOmEmS5nCaA0RcfHbkQV4SebRleocLB9qcSwj5JzPICAB
-sn1qeSv45LnGdTzUnCQErw1EzgrK1FXOknujjZ3GX5dM7a6Gag0E6cAxoQeC8Mh0
-aUWolyBo9aSXltn7Vnywoe4X3v96eVH6tUQtN3wt51LBlfSDTWOQDeXLFyW57oYL
-/ErbmJLaSHpANBcC9InI+rxNYePTTba7K8rdPDppmMEmdOXjaL+5aPay2Bay9K8T
-FUyZdoyrbzxFZYT9/YyVJV/ukVcT9ru3HpzLTg9Vgc/CyRfIvCZY/Gzh+pPUbRsR
-pHYDT09BFS+mjszTkcdbceweYXJvknXpajMiISrPUUkb9rwCTmLS4QR4y7I9zAgJ
-9U72mCZB9yhhzS17kyKAANihXvOFrMXCO7QTWZbtSkobMBS2SybDxYP3rM5b6824
-HQaY4MZW5qCndydfz4fa908XbPTDka9Nu3CZfI3fhGiiGDFWViotGEYE4i9BuGZJ
-vmtzII7zeIoHwM4t5fb5mYDd9H+Rkct1yATGEIFEx7cu1fj4Ht2177EzhdkF3Q6I
-Fccn43yGDW06nFvEjnjb+Lz4EvtGp6dDefUhNBohdzd4Rm2o+etKobm0lZ1xOX74
-2gOGBLfXS6PF+msNkWP76mqKE6hG0+R1iPrN06Arti+ZRBhNGJDp1pnOERl7IyB0
-hiLUqae7xVSMVRDeq4O04EUpMXh5ixs3RJXjthYILt7vX+6qskJC3rMuyThrsvMi
-Kk0wErDpzR5U7KFnM2fk4jY5by/0b/jHZ/PWFGnaFrXNOiwK/mV1RIJtg5FCrvMv
-xJN5Qo7o2ZYzUL+IBf5mP5O49FCnz3e++9r4ySY/K57ybhQft1l+cQPWS7xuvVAg
-wZGM9DPl9AhBus0I3V3XzfkcWWHkMynMRcfTnkBHrmKjMopIOggsq05JPalM9b0V
-NfwZngVvTFbQejbGjgQQQ+G7spav/icCIA8Iu8JPHaKZt2IX77L1xMcQOouqMrx8
-JCXnJ7zsuU+8W3GQGm70H2kcen4bf4beA8TSkFmbVsaqDNtGaG+ZopWO8ctZX3CP
-jFgIYgEGi137IWvs1KD5QNFHZI6l0JMzglmvcfnViRPDHLi+EQXAoL/98j/YzA9t
-a9L4xMO0oucovKkqYOS2cdFaQKVIUM9L5ibcbW7Tol4ugMo5ibIsTgjZoyFohG6t
-acg+h2wTTyxtrIapvOOBeAo9V/iQowTAPQf38LKQZ5tApksiCG1qvJerKALTMp1J
-yUHjTCPCPsaiXVVO0mfa9OPtzVvi/icLmlz7umk82mew13djedgMS3/ghq5aEHbg
-e7rtI5snFlm/GQnV6IaCDytb+jYxqQ0DZ9Mb0CHcMpnLyiBO6nzHfadpZYlrxrpx
-dPx9MVqfLavj1UVjKuVj4Mgec9J38IBBRGMJZ4MGhde2hrw/s2jVzG1BzaIzcvFa
-bWiJOkKKRC4BY9DBDYfae5c/obwzkxLmvkbWAVzoFXc+46K1IK6bbC2a4EerOAxc
-C7B3X8BucD8OtiPORwXgjctd9AJ7cXq0EEVGoZZ8jkILnklsK6RXsxrT4gSOGFpx
-cBU7TF4kHpH+o4KGQmjlFA2VYP1tXirjEep57LOHfL+zy+HiSXdsbexf6odkqLYA
-1RR0TzSkmqJYvP0GfHPMvp2xredx1kP934fhTIUcWvErrG3NPd/Dod4BzQye1nNK
-8Y8aFYF6fACcD4SFZ11ecK9DJmB+WXGlsxtO3w1LeEysny2lVkkH6HuMRIzPyXAM
-/sEpwh3hwPJosAh6vGM6JxuL7CXP8hC1oUvK91VR0hxeDcO/LgdcHtDnlNMqt1wC
-jvYt9Hg+3aY5GehBnQPRppnIMaGhiBrRC6HjBg8Umt6UTUr5PXNY2aia5wRcx4M6
-Tj0O0dAt8WWZ9ySgzOoq4SRoaAp6m4E7CpRNMgXH031OWEeHqtt8rsYqAcvcQLuW
-vIwts5hiXZOfgXyR6SBY3ANE1SjDY46na2G4oiG8FI7eAqYgehjHrXqCnGrzjefE
-V1eVcIr7DBYuM8wq7C9x+qdMir6RNV4MrVD+7rO/drivnd3MaqDHlG0LfMZo6GIK
-QZGIy0z1W5lOXm9x3RAz7/Ea11ynkZyPDDxBE5NRcV6JfukdjDrvsaQA/Qk2Wo7h
-PdUZjv17H4syMdap6tUDTc8Cohut6n1UxnDnhaAYEDGR3TgPqI5vEaJ+yoaM9itv
-b2I/K+0uK6GfWwca7PeTtYn92/5etO4czG99xEBFzw7tiEmBTA8KfseZIUe/i3fr
-9LO+rKEGalFB4mVTYyCKPYYx4hOIvjdCox/jEBTFyYUEhgGnZvArgw9XabvII7eM
-c1FmwnN2CM/MHUzM0Kw7cS/oaoRFcG/PqioO/krL2Bd+xThwSH4Zn+Y1UVO7aXNy
-90MOehHSc2nhN984Y2re3ffVsjPhC3KdWxr8jbho/Az65rUkYg64roiO2ZqdqRvf
-aYtX9fCe8N8SzgLTlWZt+l8sjY8HP2k9NoajphV1nG4iC/pX0ZJiVQ+VBRNaGhSl
-6IhM9gM1ZsI86cULJpOTAhNu1fZBv+Mq8IXYD8SqlNVwVtjvshtlzBN4InSGEKiO
-HM7coo7aDuu+Go5AEiIVMbR09jAa0VINsCVScfSbTFhlGRTrZr2DSO/s8WxbV+Mb
-EQAfZmyrLF71Z3MfIukg+ps5iuqFhZ+gJSXXSFwumpg=
-EOF
-
-exec sh bin/do-setup-circleci-secrets

bin/wait_for.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# shellcheck disable=SC2155
+wait_for() {
+    local timeout=$1
+    local lock_file=$2
+    for i in $(seq 1 "$timeout"); do
+        if [ -f "$lock_file" ]; then
+            local status="$(cat "$lock_file")"
+            if [ "$status" == "OK" ]; then
+                echo "[$i seconds]: $lock_file found and status: $status." && return
+            else
+                echo "[$i seconds]: $lock_file found and status: $status." && return 1
+            fi
+        fi
+        if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi
+        sleep 1
+    done
+    echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2
+    return 1
+}

build/Dockerfile

@@ -1,8 +1,87 @@
-FROM golang:1.5.2
-ENV GO15VENDOREXPERIMENT 1
-RUN apt-get update && apt-get install -y libpcap-dev python-requests time file
-RUN go get github.com/fzipp/gocyclo github.com/client9/misspell/cmd/misspell
-RUN go clean -i net && go install -tags netgo std
-RUN go install -race -tags netgo std
+FROM golang:1.7.4
+
+# Support Raspberry Pi 2 and newer
+ENV GOARM 7
+
+# The names of the architectures that we're fetching cross-compilers for from the Ubuntu packages
+ENV DEB_CROSSPLATFORMS \
+	armhf \
+	arm64
+
+# The go platforms we are supporting to cross-compile
+ENV GO_CROSSPLATFORMS \
+	linux/arm \
+	linux/arm64
+
+# The names of the gcc cross-compilers we have installed
+ENV GCC_CROSSCOMPILERS \
+	arm-linux-gnueabihf \
+	aarch64-linux-gnu
+
+# Temporarily add the Ubuntu repositories, because we're going to install gcc cross-compilers from there
+# Install the build-essential and crossbuild-essential-ARCH packages
+RUN echo "deb http://emdebian.org/tools/debian/ jessie main" > /etc/apt/sources.list.d/cgocrosscompiling.list \
+  && curl http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add - \
+  && for platform in ${DEB_CROSSPLATFORMS}; do dpkg --add-architecture $platform; done \
+  && apt-get update \
+  && apt-get install -y build-essential \
+  && for platform in ${DEB_CROSSPLATFORMS}; do apt-get install -y crossbuild-essential-${platform}; done \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Install some required packages
+# libpcap is required because we're linking against its C libraries from the prog/weaver binary
+# flex and bison are required packages for compiling libpcap manually later
+RUN apt-get update \
+	&& apt-get install -y \
+		libpcap-dev \
+		python-requests \
+		time \
+		file \
+		flex \
+		bison
+
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+	chmod +x shfmt && \
+	mv shfmt /usr/bin
+
+# Install common Go tools
+RUN go get \
+	github.com/golang/lint/golint \
+	github.com/fzipp/gocyclo \
+	github.com/fatih/hclfmt \
+	github.com/client9/misspell/cmd/misspell
+
+# First clean, because we're gonna rebuild the std both with -race enabled and disabled
+# Running go install std twice is intentional and required, as content is written in two different directories
+# (/usr/local/go/pkg/linux_amd64 vs /usr/local/go/pkg/linux_amd64_race)
+# The other architectures do not have support for -race
+RUN go clean -i net \
+	&& go install -tags netgo std \
+	&& go install -race -tags netgo std
+
+# Prebuild std for the other architectures
+RUN for platform in ${GO_CROSSPLATFORMS}; do \
+		GOOS=${platform%/*} GOARCH=${platform##*/} \
+			go install -tags netgo std; done
+
+# Allow full write access to the Go folders for anyone
+RUN chmod -R a+w /usr/local/go
+
+# The libpcap version from Debian packages is 1.6.2, matching that version here although newer versions of libpcap have been released
+# We have to cross-compile the libpcap library for the new architectures, that's what we're doing here
+ENV LIBPCAP_CROSS_VERSION=1.6.2
+RUN curl -sSL http://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.tar.gz | tar -xz \
+	&& cd libpcap-${LIBPCAP_CROSS_VERSION} \
+	&& for crosscompiler in ${GCC_CROSSCOMPILERS}; do \
+		CC=${crosscompiler}-gcc ac_cv_linux_vers=2 ./configure --host=${crosscompiler} --with-pcap=linux \
+		&& make \
+		&& export LIB_DIR="/usr/local/lib/${crosscompiler}-gcc" VER="$(cat ./VERSION)" MAJOR_VER="$(sed 's/\([0-9][0-9]*\)\..*/\1/' ./VERSION)" \
+		&& mkdir -p ${LIB_DIR} \
+		&& cp -f libpcap.a libpcap.so.${VER} ${LIB_DIR} \
+		&& ln -sf libpcap.so.${VER} /usr/local/lib/libpcap.so.${MAJOR_VER} \
+		&& ln -sf libpcap.so.${MAJOR_VER} /usr/local/lib/libpcap.so \
+		&& make clean; done
+
 COPY build.sh /
 ENTRYPOINT ["sh", "/build.sh"]

build/shasums/docker-tgz-armel.sha256sum

@@ -0,0 +1 @@
+8f313a9df867c207ae9ceac6e0935d1d2a0360aafce780dd5d1e0687ee98e0c3 docker-1.10.3.tgz

build/shasums/docker-tgz-x86_64.sha256sum

@@ -0,0 +1 @@
+7541b5490cee1183d911f909c0e91d3d9d57b2b775e40f14a1a7a6de216ca4a8 docker-1.10.3.tgz

build/shasums/qemu-aarch64-static.sha256sum

@@ -0,0 +1 @@
+c02d861383d035ba020b361ef93bf14beb3964a42a6ae645b505b6148493e7ec qemu-aarch64-static

build/shasums/qemu-arm-static.sha256sum

@@ -0,0 +1 @@
+0ff63682b8868620d771aa812d3053dc3c579dc8633ce62d3d7ae7eee83beb06 qemu-arm-static

circle.yml

@@ -2,28 +2,43 @@ general:
   branches:
     ignore:
       - gh\-pages
-      - /pull\/.*/
 
 machine:
+  pre:
+    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
   services:
     - docker
   environment:
     GOPATH: /home/ubuntu:$GOPATH
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
-    PATH: $PATH:$HOME/.local/bin
+    PATH: $PATH:$HOME/.local/bin:/opt/google-cloud-sdk/bin
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
     STATE: /home/ubuntu/state.env
+    # Base name of VMs for integration tests:
+    NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX
+    TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt
+    TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt
+    TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt
+    # Google Cloud Platorm images for integration tests:
+    USE_IMAGE: 1
+    DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0
 
 dependencies:
   cache_directories:
     - "~/docker"
   post:
+    - bin/circle-dependencies-post-sources
+    # Provision & configure testing VMs:
+    - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh":
+        parallel: true
+        background: true
     - bin/circle-dependencies-post
 
 test:
   pre:
     - "$SRCDIR/bin/circle-test-pre":
         parallel: true
+        background: true
   override:
     - "$SRCDIR/bin/circle-test-unit":
         parallel: true
@@ -36,22 +51,16 @@ test:
 
 teardown:
   pre:
+    # Destroy testing VMs:
+    - "cd $SRCDIR/test && ./run-integration-tests.sh destroy":
+        parallel: true
+        background: true
     - "$SRCDIR/bin/circle-teardown-pre":
         parallel: true
 
 deployment:
-  release:
-    branch: /[0-9]+\.[0-9]+/
-    owner: weaveworks
-    commands:
-      - "$SRCDIR/bin/circle-deploy-release"
   master:
     branch: master
     owner: weaveworks
     commands:
       - "$SRCDIR/bin/circle-deploy-master"
-  issues:
-    branch: /.*/
-    owner: weaveworks
-    commands:
-      - "$SRCDIR/bin/circle-deploy-issues"

common/mflagext/listvar.go

@@ -1,31 +0,0 @@
-package mflagext
-
-import (
-	"fmt"
-
-	"github.com/weaveworks/docker/pkg/mflag"
-)
-
-type listOpts struct {
-	value      *[]string
-	hasBeenSet bool
-}
-
-func ListVar(p *[]string, names []string, value []string, usage string) {
-	*p = value
-	mflag.Var(&listOpts{p, false}, names, usage)
-}
-
-func (opts *listOpts) Set(value string) error {
-	if opts.hasBeenSet {
-		(*opts.value) = append((*opts.value), value)
-	} else {
-		(*opts.value) = []string{value}
-		opts.hasBeenSet = true
-	}
-	return nil
-}
-
-func (opts *listOpts) String() string {
-	return fmt.Sprintf("%v", []string(*opts.value))
-}

common/odp/odp.go

@@ -21,8 +21,13 @@ func CreateDatapath(dpname string) (supported bool, err error) {
 	defer dpif.Close()
 
 	dp, err := dpif.CreateDatapath(dpname)
-	if err != nil && !odp.IsDatapathNameAlreadyExistsError(err) {
-		return true, err
+	if err != nil {
+		if !odp.IsDatapathNameAlreadyExistsError(err) {
+			return true, err
+		}
+		if dp, err = dpif.LookupDatapath(dpname); err != nil {
+			return true, err
+		}
 	}
 
 	// Pick an ephemeral port number to use in probing for vxlan

common/signals.go

@@ -1,32 +0,0 @@
-package common
-
-import (
-	"os"
-	"os/signal"
-	"runtime"
-	"syscall"
-)
-
-// A subsystem/server/... that can be stopped or queried about the status with a signal
-type SignalReceiver interface {
-	Stop() error
-}
-
-func SignalHandlerLoop(ss ...SignalReceiver) {
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM)
-	buf := make([]byte, 1<<20)
-	for {
-		switch <-sigs {
-		case syscall.SIGINT, syscall.SIGTERM:
-			Log.Infof("=== received SIGINT/SIGTERM ===\n*** exiting")
-			for _, subsystem := range ss {
-				subsystem.Stop()
-			}
-			return
-		case syscall.SIGQUIT:
-			stacklen := runtime.Stack(buf, true)
-			Log.Infof("=== received SIGQUIT ===\n*** goroutine dump...\n%s\n*** end", buf[:stacklen])
-		}
-	}
-}

db/boltdb.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/gob"
 	"fmt"
+	"time"
 
 	"github.com/boltdb/bolt"
 )
@@ -23,14 +24,39 @@ var (
 	topBucket          = []byte("top")
 )
 
+const (
+	NameIdent = "peername"
+	FileName  = "data.db"
+)
+
 func NewBoltDB(dbPathname string) (*BoltDB, error) {
 	db, err := bolt.Open(dbPathname, 0660, nil)
 	if err != nil {
 		return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err)
 	}
-	// check the persistence version
-	err = db.Update(func(tx *bolt.Tx) error {
+	err = db.Update(checkVersion(false))
+	return &BoltDB{db: db}, err
+}
+
+func NewBoltDBReadOnly(dbPathname string) (*BoltDB, error) {
+	options := bolt.Options{Timeout: time.Millisecond * 50, ReadOnly: true}
+	db, err := bolt.Open(dbPathname, 0660, &options)
+	if err != nil {
+		return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err)
+	}
+	err = db.View(checkVersion(true))
+	if err != nil {
+		return nil, fmt.Errorf("[boltDB] Cannot use persistence file %s: %s", dbPathname, err)
+	}
+	return &BoltDB{db: db}, nil
+}
+
+func checkVersion(readOnly bool) func(tx *bolt.Tx) error {
+	return func(tx *bolt.Tx) error {
 		if top := tx.Bucket(topBucket); top == nil {
+			if readOnly {
+				return fmt.Errorf("no top bucket")
+			}
 			top, err := tx.CreateBucket(topBucket)
 			if err != nil {
 				return err
@@ -41,13 +67,12 @@ func NewBoltDB(dbPathname string) (*BoltDB, error) {
 		} else {
 			if checkVersion := top.Get(versionIdent); checkVersion != nil {
 				if checkVersion[0] != persistenceVersion[0] {
-					return fmt.Errorf("[boltDB] Cannot use persistence file %s - version %x", dbPathname, checkVersion)
+					return fmt.Errorf("mismatched version %x", checkVersion)
 				}
 			}
 		}
 		return nil
-	})
-	return &BoltDB{db: db}, err
+	}
 }
 
 func (d *BoltDB) Load(ident string, data interface{}) (bool, error) {

docs/fastdp-crypto.md

@@ -0,0 +1,155 @@
+This document describes implementation details of the fast datapath encryption.
+
+# Overview
+
+At the high level, we use the ESP protocol ([RFC 2406][esp]) in the Transport
+mode. Each packet is encrypted with AES in GCM mode ([RFC 4106][aesgcm]), with
+32 byte key and 4 byte salt. This combo provides the following security
+properties:
+
+* Data confidentiality.
+* Data origin authentication.
+* Integrity.
+* Anti-replay.
+* Limited traffic flow confidentiality as fast datapath VXLAN packets are fully
+  encrypted.
+
+## Notation
+
+* `SAin`:  IPsec security association for inbound connections.
+* `SAout`: IPsec security association for outbound connections.
+* `SPout`: IPsec security policy for outbound connections. Used to match
+           outbound flows to SAout.
+
+## SA Key Derivation
+
+For each connection direction, a different AES-GCM key and salt is used.
+The pairs are derived with HKDF ([RFC 5869][hkdf]):
+
+```
+SAin[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceIn, info=localPeerName)
+SAout[KeyAndSalt] = HKDF(sha256, ikm=sessionKey, salt=nonceOut, info=remotePeerName)
+```
+
+Where:
+
+* the mutual `sessionKey` is derived by the [Mesh][mesh] library during
+  the control plane connection establishment between the local peer and
+  the remote peer.
+* `nonceIn` and `nonceOut` are randomly generated 32byte nonces which
+  are exchanged over the encrypted control plane channel.
+
+## SPI
+
+A directional secure connection between two peers is identified with SPI.
+
+The kernel requires the pair of SPI and dst IP to be unique among security
+associations. Thus, to avoid collisions, we generate outbound SPI on a remote
+peer.
+
+## Connection Establishment
+
+```
+Peer A                                                        Peer B
+-----------------------------------------------------------------------------------------------------------------
+
+fastdp.fwd.Confirm():
+    install iptables blocking rules,                          fastdp.fwd.Confirm():
+    nonce_BA = rand(),                                            install iptables blocking rules,
+    {key,salt}_BA = hkdf(sessionKey, nonce_BA, A),                nonce_AB = rand(),
+    spi_BA = allocspi(),                                          {key,salt}_AB = hkdf(sessionKey, nonce_AB, B),
+    create SA_BA(B<-A, spi_BA, key_BA, salt_BA),                  spi_AB = allocspi(),
+    send InitSARemote(spi_BA, nonce_BA). -->                      create SA_AB(A<-B, spi_AB, key_AB, salt_AB),
+                                                              <-- send InitSARemote(spi_AB, nonce_AB).
+
+
+                                                          --> recv InitSARemote(spi_BA, nonce_BA):
+recv InitSARemote(spi_AB, nonce_AB): <--                          {key,salt}_BA = hkdf(sessionKey, nonce_BA, A),
+    {key,salt}_AB = hkdf(sessionKey, nonce_AB, B),                create SA_BA(B<-A, spi_BA, key_BA, salt_BA),
+    create SA_AB(A<-B, spi_AB, key_AB, salt_AB),                  create SP_BA(B<-A, spi_BA),
+    create SP_AB(A<-B, spi_AB),                                   install marking rule.
+    install marking rule.
+```
+
+# Implementation Details
+
+## XFRM
+
+The implementation is based on the kernel IP packet transformation framework
+called XFRM. Unfortunately, docs are barely existing and the complexity of
+the framework is high. The best resource I found is Chapter 10 in
+"Linux Kernel Networking: Implementation and Theory" by Rami Rosen.
+
+## iptables Rules
+
+The kernel VXLAN driver does not set a dst port of a tunnel in the ip flow
+descriptor, thus XFRM policy lookup cannot match a policy (SPout) which includes
+the port. This makes impossible to encrypt only tunneled traffic between
+peers. To work around, we mark such outgoing packets with iptables and set
+the same mark in the policy selector (funnily enough, `iptables_mangle` module
+eventually sets the missing dst port in the flow descriptor). The challenge
+here is to pick a mark that it would not interfere with other networking
+applications before OUTPUT'ing a packet. For example, Kubernetes by default
+uses 1<<14 and 1<<15 marks and we choose 1<<17 (0x20000). Additionally,
+such workaround brings the requirement for at least the 4.2 kernel.
+
+The marking rules are the following:
+
+```
+iptables -t mangle -A OUTPUT -j WEAVE-IPSEC-OUT
+iptables -t mangle -A WEAVE-IPSEC-OUT -s ${LOCAL_PEER_IP} -d ${REMOTE_PEER_IP} \
+         -p udp --dport ${TUNNEL_PORT} -j WEAVE-IPSEC-OUT-MARK
+iptables -t mangle -A WEAVE-IPSEC-OUT-MARK --set-xmark ${MARK} -j MARK
+```
+
+As Linux does not implement [IP Security Levels][ipseclevel], we install
+additional iptables rules to prevent from accidentally sending unencrypted
+traffic between peers which have previously established the secure connection.
+
+For inbound traffic, we mark each ESP packet with the mark and drop non-marked
+tunnel traffic:
+
+```
+iptables -t mangle -A INPUT -j WEAVE-IPSEC-IN
+iptables -t mangle -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \
+         -m esp --espspi ${SPIin} -j WEAVE-IPSEC-IN-MARK
+iptables -t mangle -A WEAVE-IPSEC-IN-MARK --set-xmark ${MARK} -j MARK
+
+iptables -t filter -A INPUT -j WEAVE-IPSEC-IN
+iptables -t filter -A WEAVE-IPSEC-IN -s ${REMOTE_PEER_IP} -d ${LOCAL_PEER_IP} \
+         -p udp --dport ${TUNNEL_PORT} -m mark ! --mark ${MARK} -j DROP
+```
+
+For outbound traffic, we drop marked traffic which does not match any SPout:
+
+```
+iptables -t filter -A OUTPUT ! -p esp -m policy --dir out --pol none \
+         -m mark --mark ${MARK} -j DROP
+```
+
+## ESN
+
+To prevent from cycling SeqNo which makes replay attacks possible, we use
+64-bit extended sequence numbers known as [ESN](esn).
+
+## MTU
+
+In addition to the VXLAN overhead, the MTU calculation should take into account
+the ESP overhead which is 34-37 bytes (encrypted Payload is 4 bytes aligned) and
+consists of:
+
+* 4 bytes (SPI).
+* 4 bytes (Sequence Number).
+* 8 bytes (ESP IV).
+* 1 byte (Pad Length).
+* 1 byte (NextHeader).
+* 16 bytes (ICV).
+* 0-3 bytes (Padding).
+
+
+[esp]:              https://tools.ietf.org/html/rfc2406
+[aesgcm]:           https://tools.ietf.org/html/rfc4106
+[hkdf]:             https://tools.ietf.org/html/rfc5869
+[esn]:              https://tools.ietf.org/html/rfc4304
+[ipseclevel]:       https://tools.ietf.org/html/draft-mcdonald-simple-ipsec-api-01)
+[mesh]:             https://github.com/weaveworks/mesh

docs/fastdp.md

@@ -52,9 +52,9 @@ overlay, which has a more sophisticated dynamic mechanism for coping
 with low path MTUs.
 
 To avoid triggering this fallback in typical deployments, the datapath
-interface is statically configured with an MTU of 1410 bytes allowing
+interface is statically configured with an MTU of 1376 bytes allowing
 it to work with most underlay network provider MTUs, including GCE at
-1460 bytes (the fifty byte difference accommodates the vxlan overhead).
+1460 bytes (the eighty four byte difference accommodates the encrypted vxlan overhead).
 This value can be overridden by setting `WEAVE_MTU` at launch if
 necessary.
 

docs/ipam.md

@@ -65,14 +65,30 @@ peer for space:
   - it will continue to ask for space until it receives some, or its
     copy of the ring tells it all peers are full in that subnet.
 
+### Data persistence
+
+Key IPAM data is saved to disk, in a [BoltDB](https://github.com/boltdb/bolt)
+file, stored within a [data volume container](https://docs.docker.com/engine/userguide/containers/dockervolumes/#creating-and-mounting-a-data-volume-container)
+named `weavedb`.
+
+This file is used to persist data for various Weave Net components; for IPAM it contains:
+
+* the division of the IP allocation range amongst peers, and
+* allocation of addresses to containers on the local peer,
+
+so that it is immediately available when the peer restarts.
+
 ### Claiming an address
 
-If a Weave process is restarted, in most cases it will hear from
-another peer which ranges it used to own, but it needs to know which
-individual IP addresses are assigned to containers in order to avoid
-giving the same address out in subsequent allocation requests. The
-weave script invokes the `claim` command in `populate_ipam` to do
-this.
+If a Weave process is restarted,
+
+- if `weavedb` is present, then it loads persisted IPAM data from there, as
+described, in the previous section;
+- else, it learns from other peers which ranges it used to own.
+
+Weave Net then consults the allocator to know which individual IP addresses
+are assigned to containers, and therefore avoid giving the same address out in
+subsequent allocation requests.
 
 When the Allocator is told to claim an address, there are four
 possibilities:

docs/release-process.md

@@ -107,15 +107,6 @@ the build script will complain otherwise.
 N.B. if you're testing the release process, push to your fork
 instead!
 
-While the git.io link for weave-kube is still pointing to the
-weave-kube repository, we also need to update the tag there:
-
-    cd ../weave-kube
-    git tag -a -m "Release $TAG" $TAG
-    git tag -af -m "Release $TAG" latest_release $TAG
-    git push -f git@github.com:weaveworks/weave latest_release
-
-
 ### Publish Release & Distributable Artefacts
 
 You can now publish the release and upload the remaining
@@ -147,6 +138,8 @@ Finally, for **Mainline** releases only:
   published to https://www.weave.works
 * Add the new version of `weave-net` to the checkpoint system at
   https://checkpoint-api.weave.works/admin
+* File a PR to update the version of the daemonset at
+  https://github.com/kubernetes/kops/tree/master/upup/models/cloudup/resources/addons/networking.weave
 
 ## Troubleshooting
 

docs/weavenpc-design.md

@@ -19,6 +19,9 @@ NetworkPolicy object updates from the k8s API server:
   network policy, containing the IP addresses of all pods in the
   namespace whose labels match that selector
 
+IPsets are implemented by the kernel module `xt_set`, without which
+weave-npc will not work.
+
 ipset names are generated deterministically from a string
 representation of the corresponding label selector. Because ipset
 names are limited to 31 characters in length, this is done by taking a
@@ -74,8 +77,10 @@ iptables -A WEAVE-NPC -m state --state NEW -j WEAVE-NPC-INGRESS
 
 To direct traffic into the policy engine:
 
+```
 iptables -A FORWARD -o weave -m physdev ! --physdev-out vethwe-bridge -j WEAVE-NPC
 iptables -A FORWARD -o weave -m physdev ! --physdev-out vethwe-bridge -j DROP
+```
 
 Note this only affects traffic which egresses the bridge on a physical
 port which is not the Weave Net router - in other words, it is
@@ -94,6 +99,9 @@ The following traffic is NOT affected:
   (e.g. kubelet health checks)
 * Traffic routed from an application container to the internet
 
+The above mechanism relies on the kernel module `br_netfilter` being
+loaded and enabled via `/proc/sys/net/bridge/bridge-nf-call-iptables`.
+
 See these resources for helpful context:
 
 * http://ebtables.netfilter.org/br_fw_ia/br_fw_ia.html

ipam/allocate.go

@@ -14,9 +14,9 @@ type allocateResult struct {
 
 type allocate struct {
 	resultChan       chan<- allocateResult
-	ident            string
+	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
 	r                address.CIDR // Subnet we are trying to allocate within
-	isContainer      bool
+	isContainer      bool         // true if ident is a container ID
 	hasBeenCancelled func() bool
 }
 

ipam/allocator.go

@@ -76,6 +76,13 @@ type Allocator struct {
 	now               func() time.Time
 }
 
+// PreClaims are IP addresses discovered before we could initialize IPAM
+type PreClaim struct {
+	Ident       string // a container ID, something like "weave:expose", or api.NoContainerID
+	IsContainer bool   // true if Ident is a container ID
+	Cidr        address.CIDR
+}
+
 type Config struct {
 	OurName     mesh.PeerName
 	OurUID      mesh.PeerUID
@@ -83,6 +90,7 @@ type Config struct {
 	Seed        []mesh.PeerName
 	Universe    address.CIDR
 	IsObserver  bool
+	PreClaims   []PreClaim
 	Quorum      func() uint
 	Db          db.DB
 	IsKnownPeer func(name mesh.PeerName) bool
@@ -123,6 +131,12 @@ func NewAllocator(config Config) *Allocator {
 		dead:        make(map[string]time.Time),
 		now:         time.Now,
 	}
+
+	alloc.pendingClaims = make([]operation, len(config.PreClaims))
+	for i, c := range config.PreClaims {
+		alloc.pendingClaims[i] = &claim{ident: c.Ident, cidr: c.Cidr}
+	}
+
 	return alloc
 }
 
@@ -156,6 +170,9 @@ func (alloc *Allocator) Start() {
 	default:
 		alloc.infof("Initialising as observer - awaiting IPAM data from another peer")
 	}
+	if loadedPersistedData { // do any pre-claims right away
+		alloc.tryOps(&alloc.pendingClaims)
+	}
 	actionChan := make(chan func(), mesh.ChannelSize)
 	stopChan := make(chan struct{})
 	alloc.actionChan = actionChan
@@ -193,9 +210,9 @@ func (alloc *Allocator) doOperation(op operation, ops *[]operation) {
 //  already succeeded.  If it is on the queue, we call
 //  cancel on it, allowing callers waiting for the resultChans
 //  to unblock.
-func (alloc *Allocator) cancelOp(op operation, ops *[]operation) {
+func (alloc *Allocator) cancelOp(opToCancel operation, ops *[]operation) {
 	for i, op := range *ops {
-		if op == op {
+		if op == opToCancel {
 			*ops = append((*ops)[:i], (*ops)[i+1:]...)
 			op.Cancel()
 			break
@@ -248,6 +265,10 @@ func (alloc *Allocator) tryPendingOps() {
 	alloc.tryOps(&alloc.pendingAllocates)
 }
 
+func (alloc *Allocator) havePendingOps() bool {
+	return len(alloc.pendingPrimes)+len(alloc.pendingClaims)+len(alloc.pendingAllocates) > 0
+}
+
 func (alloc *Allocator) spaceRequestDenied(sender mesh.PeerName, r address.Range) {
 	for i := 0; i < len(alloc.pendingClaims); {
 		claim := alloc.pendingClaims[i].(*claim)
@@ -679,8 +700,12 @@ func (alloc *Allocator) actorLoop(actionChan <-chan func(), stopChan <-chan stru
 			// Retry things in case messages got lost between here and recipients
 			if alloc.awaitingConsensus {
 				alloc.propose()
-			} else {
-				alloc.tryPendingOps()
+			} else if alloc.havePendingOps() {
+				if alloc.ring.Empty() {
+					alloc.establishRing()
+				} else {
+					alloc.tryPendingOps()
+				}
 			}
 			alloc.removeDeadContainers()
 		}
@@ -920,13 +945,12 @@ func (alloc *Allocator) reportFreeSpace() {
 // Persistent data
 const (
 	ringIdent  = "ring"
-	nameIdent  = "peername"
 	ownedIdent = "ownedAddresses"
 )
 
 func (alloc *Allocator) persistRing() {
 	// It would be better if these two Save operations happened in the same transaction
-	if err := alloc.db.Save(nameIdent, alloc.ourName); err != nil {
+	if err := alloc.db.Save(db.NameIdent, alloc.ourName); err != nil {
 		alloc.fatalf("Error persisting ring data: %s", err)
 		return
 	}
@@ -938,7 +962,7 @@ func (alloc *Allocator) persistRing() {
 // Returns true if persisted data is to be used, otherwise false
 func (alloc *Allocator) loadPersistedData() bool {
 	var checkPeerName mesh.PeerName
-	nameFound, err := alloc.db.Load(nameIdent, &checkPeerName)
+	nameFound, err := alloc.db.Load(db.NameIdent, &checkPeerName)
 	if err != nil {
 		alloc.fatalf("Error loading persisted peer name: %s", err)
 	}

ipam/allocator_test.go

@@ -168,11 +168,13 @@ func TestAllocatorClaim(t *testing.T) {
 		container1 = "abcdef"
 		container3 = "b01df00d"
 		universe   = "10.0.3.0/24"
-		testAddr1  = "10.0.3.2/24"
+		testAddr1  = "10.0.3.5/24"
 		testAddr2  = "10.0.4.2/24"
+		testPre    = "10.0.3.1/24"
 	)
 
-	allocs, router, subnet := makeNetworkOfAllocators(2, universe)
+	preAddr, _ := address.ParseCIDR(testPre)
+	allocs, router, subnet := makeNetworkOfAllocators(2, universe, []PreClaim{{container1, true, preAddr}})
 	defer stopNetworkOfAllocators(allocs, router)
 	alloc := allocs[1]
 	addr1, _ := address.ParseCIDR(testAddr1)
@@ -184,6 +186,9 @@ func TestAllocatorClaim(t *testing.T) {
 	alloc.Prime()
 	// Do an allocate on the other peer, which we will try to claim later
 	addrx, err := allocs[0].Allocate(container1, subnet, true, returnFalse)
+	require.NoError(t, err)
+	// Should not get the address we pre-claimed
+	require.NotEqual(t, addrx, preAddr)
 	router.Flush()
 
 	// Now try the claim again
@@ -201,6 +206,11 @@ func TestAllocatorClaim(t *testing.T) {
 	// claiming the address allocated on the other peer should fail
 	err = alloc.SimplyClaim(container1, address.MakeCIDR(subnet, addrx))
 	require.Error(t, err, "claiming address allocated on other peer should fail")
+	// claiming the pre-claimed address should fail on both peers
+	err = alloc.SimplyClaim(container3, preAddr)
+	require.Error(t, err, "claiming address allocated on other peer should fail")
+	err = allocs[0].SimplyClaim(container3, preAddr)
+	require.Error(t, err, "claiming address allocated on other peer should fail")
 	// Check an address outside of our universe
 	addr2, _ := address.ParseCIDR(testAddr2)
 	err = alloc.SimplyClaim(container1, addr2)

ipam/claim.go

@@ -12,10 +12,10 @@ import (
 
 type claim struct {
 	resultChan       chan<- error
-	ident            string
-	cidr             address.CIDR
-	isContainer      bool
-	noErrorOnUnknown bool
+	ident            string       // a container ID, something like "weave:expose", or api.NoContainerID
+	cidr             address.CIDR // single address being claimed
+	isContainer      bool         // true if ident is a container ID
+	noErrorOnUnknown bool         // if false, error or block if we don't know; if true return ok but keep trying
 	hasBeenCancelled func() bool
 }
 
@@ -35,7 +35,7 @@ func (c *claim) sendResult(result error) {
 
 // Try returns true for success (or failure), false if we need to try again later
 func (c *claim) Try(alloc *Allocator) bool {
-	if c.hasBeenCancelled() {
+	if c.hasBeenCancelled != nil && c.hasBeenCancelled() {
 		c.Cancel()
 		return true
 	}

ipam/ring/ring.go

@@ -41,8 +41,8 @@ var (
 	ErrNotSorted       = errors.New("Ring not sorted")
 	ErrTokenRepeated   = errors.New("Token appears twice in ring")
 	ErrTokenOutOfRange = errors.New("Token is out of range")
-	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
-	ErrDifferentRange  = errors.New("Received range differs from ours!")
+	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours")
+	ErrDifferentRange  = errors.New("Received range differs from ours")
 )
 
 func errInconsistentEntry(mine, theirs *entry) error {

ipam/ring/ring_test.go

@@ -494,7 +494,7 @@ func TestTransfer(t *testing.T) {
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	ring1.Transfer(peer2name, peer1name)
-	require.Equal(t, []address.Range{{start, middle}, {middle, end}}, ring1.OwnedRanges())
+	require.Equal(t, []address.Range{{Start: start, End: middle}, {Start: middle, End: end}}, ring1.OwnedRanges())
 
 	// Second test is what happens when a token exists at the end of a range but is transferred
 	// - does it get resurrected correctly?
@@ -503,7 +503,7 @@ func TestTransfer(t *testing.T) {
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	ring1.Transfer(peer2name, peer1name)
 	ring1.GrantRangeToHost(dot10, middle, peer2name)
-	require.Equal(t, []address.Range{{start, dot10}, {middle, end}}, ring1.OwnedRanges())
+	require.Equal(t, []address.Range{{Start: start, End: dot10}, {Start: middle, End: end}}, ring1.OwnedRanges())
 }
 
 func TestOwner(t *testing.T) {

ipam/space/space_test.go

@@ -52,12 +52,12 @@ func TestLowlevel(t *testing.T) {
 
 	s := New()
 	require.Equal(t, address.Count(0), s.NumFreeAddresses())
-	ok, got := s.Allocate(address.NewRange(0, 1000))
+	ok, _ := s.Allocate(address.NewRange(0, 1000))
 	require.False(t, ok, "allocate in empty space should fail")
 
 	s.Add(100, 100)
 	require.Equal(t, address.Count(100), s.NumFreeAddresses())
-	ok, got = s.Allocate(address.NewRange(0, 1000))
+	ok, got := s.Allocate(address.NewRange(0, 1000))
 	require.True(t, ok && got == 100, "allocate")
 	require.Equal(t, address.Count(99), s.NumFreeAddresses())
 	require.NoError(t, s.Claim(150))
@@ -68,6 +68,7 @@ func TestLowlevel(t *testing.T) {
 	wt.AssertErrorInterface(t, (*error)(nil), s.Free(100), "double free")
 
 	r, ok := s.Donate(address.NewRange(0, 1000))
+	require.True(t, ok, "donate")
 	require.Equal(t, address.NewRange(0xa0, 0x20), r, "donate")
 
 	// test Donate when addresses are scarce
@@ -78,6 +79,7 @@ func TestLowlevel(t *testing.T) {
 	require.NoError(t, s.Claim(0))
 	require.NoError(t, s.Claim(2))
 	r, ok = s.Donate(address.NewRange(0, 1000))
+	require.True(t, ok, "donate")
 	require.Equal(t, address.NewRange(1, 1), r, "donate")
 	r, ok = s.Donate(address.NewRange(0, 1000))
 	require.True(t, !ok, "donate should fail")

ipam/testutils_test.go

@@ -128,7 +128,7 @@ type mockDB struct{}
 func (d *mockDB) Load(_ string, _ interface{}) (bool, error) { return false, nil }
 func (d *mockDB) Save(_ string, _ interface{}) error         { return nil }
 
-func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.CIDR) {
+func makeAllocator(name string, cidrStr string, quorum uint, preClaims ...PreClaim) (*Allocator, address.CIDR) {
 	peername, err := mesh.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
@@ -145,6 +145,7 @@ func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, addres
 		OurNickname: "nick-" + name,
 		Universe:    cidr,
 		IsObserver:  quorum == 0,
+		PreClaims:   preClaims,
 		Quorum:      func() uint { return quorum },
 		Db:          new(mockDB),
 		IsKnownPeer: func(mesh.PeerName) bool { return true },
@@ -213,15 +214,19 @@ func AssertNothingSentErr(t *testing.T, ch <-chan error) {
 	}
 }
 
-func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
+func makeNetworkOfAllocators(size int, cidr string, preClaims ...[]PreClaim) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
 	gossipRouter := gossip.NewTestRouter(0.0)
 	allocs := make([]*Allocator, size)
 	var subnet address.CIDR
 
 	for i := 0; i < size; i++ {
 		var alloc *Allocator
+		preClaim := []PreClaim{}
+		if i < len(preClaims) {
+			preClaim = preClaims[i]
+		}
 		alloc, subnet = makeAllocator(fmt.Sprintf("%02d:00:00:02:00:00", i),
-			cidr, uint(size/2+1))
+			cidr, uint(size/2+1), preClaim...)
 		alloc.SetInterfaces(gossipRouter.Connect(alloc.ourName, alloc))
 		alloc.Start()
 		allocs[i] = alloc

nameserver/dns.go

@@ -222,7 +222,7 @@ func (h *handler) handleReverse(w dns.ResponseWriter, req *dns.Msg) {
 		return
 	}
 
-	ipStr := strings.TrimSuffix(req.Question[0].Name, "."+reverseDNSdomain)
+	ipStr := strings.TrimSuffix(strings.ToLower(req.Question[0].Name), "."+reverseDNSdomain)
 	ip, err := address.ParseIP(ipStr)
 	if err != nil {
 		h.nameError(w, req)

nameserver/entry.go

@@ -120,7 +120,7 @@ func (e1 *Entry) tombstone() bool {
 
 func check(es SortableEntries) error {
 	if !sort.IsSorted(es) {
-		return fmt.Errorf("Not sorted!")
+		return fmt.Errorf("Not sorted")
 	}
 	for i := 1; i < es.Len(); i++ {
 		if es.Get(i).equal(es.Get(i - 1)) {

net/bridge.go

@@ -2,6 +2,7 @@ package net
 
 import (
 	"fmt"
+	"net"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -86,3 +87,24 @@ func DetectHairpin(portIfName string, log *logrus.Logger) error {
 
 	return nil
 }
+
+var ErrBridgeNoIP = fmt.Errorf("Bridge has no IP address")
+
+func FindBridgeIP(bridgeName string, subnet *net.IPNet) (net.IP, error) {
+	netdev, err := GetBridgeNetDev(bridgeName)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get netdev for %q bridge: %s", bridgeName, err)
+	}
+	if len(netdev.CIDRs) == 0 {
+		return nil, ErrBridgeNoIP
+	}
+	if subnet != nil {
+		for _, cidr := range netdev.CIDRs {
+			if subnet.Contains(cidr.IP) {
+				return cidr.IP, nil
+			}
+		}
+	}
+	// No subnet, or none in the required subnet; just return the first one
+	return netdev.CIDRs[0].IP, nil
+}

net/ipsec/ipsec.go

@@ -0,0 +1,562 @@
+// package IPsec provides primitives for establishing IPsec in the fastdp mode.
+package ipsec
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+	"sync"
+	"syscall"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/pkg/errors"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/crypto/hkdf"
+
+	"github.com/weaveworks/mesh"
+)
+
+const (
+	keySize   = 36 // AES-GCM key 32 bytes + 4 bytes salt
+	nonceSize = 32 // HKDF nonce size
+
+	mark    = uint32(0x1) << 17 // iptables marks
+	markStr = "0x20000/0x20000" // update if the above mark changes
+
+	tableMangle  = "mangle"
+	tableFilter  = "filter"
+	chainIn      = "WEAVE-IPSEC-IN"
+	chainInMark  = "WEAVE-IPSEC-IN-MARK"
+	chainOut     = "WEAVE-IPSEC-OUT"
+	chainOutMark = "WEAVE-IPSEC-OUT-MARK"
+)
+
+type SPI uint32
+
+// Used to identify:
+// - directional SPIs,
+// - ipsec establishments.
+type spiID [24]byte
+
+func getSPIId(srcPeer, dstPeer mesh.PeerName, connUID uint64) (id spiID) {
+	binary.BigEndian.PutUint64(id[:], uint64(srcPeer))
+	binary.BigEndian.PutUint64(id[8:], uint64(dstPeer))
+	binary.BigEndian.PutUint64(id[16:], connUID)
+	return
+}
+
+type spiInfo struct {
+	spi      SPI
+	isDirOut bool
+}
+
+// IPSec
+
+type IPSec struct {
+	sync.RWMutex
+	ipt *iptables.IPTables
+	log *logrus.Logger
+
+	spiInfo map[spiID]spiInfo
+	// A reference to spiInfo; spiInfo might be of an expired SPI.
+	spis map[SPI]*spiInfo
+}
+
+func New(log *logrus.Logger) (*IPSec, error) {
+	ipt, err := iptables.New()
+	if err != nil {
+		return nil, errors.Wrap(err, "iptables new")
+	}
+
+	ipsec := &IPSec{
+		ipt:     ipt,
+		log:     log,
+		spiInfo: make(map[spiID]spiInfo),
+		spis:    make(map[SPI]*spiInfo),
+	}
+
+	return ipsec, nil
+}
+
+// InitSALocal initializes inbound ipsec from remotePeer and triggers
+// the initialization on remotePeer.
+func (ipsec *IPSec) InitSALocal(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte, initRemote func([]byte) error) error {
+	// ID of inbound SPI
+	spiID := getSPIId(remotePeer, localPeer, connUID)
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	// Derive SA key
+	nonce, err := genNonce()
+	if err != nil {
+		return errors.Wrap(err, "generate nonce")
+	}
+	key, err := deriveKey(sessionKey[:], nonce, localPeer)
+	if err != nil {
+		return errors.Wrap(err, "derive key")
+	}
+
+	// Allocate SA
+	sa, err := netlink.XfrmStateAllocSpi(xfrmAllocSpiState(remoteIP, localIP))
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("ip xfrm state allocspi (in, %s, %s)", remoteIP, localIP))
+	}
+
+	// Use SPI generated by the kernel. The kernel ensures {dstIP, spi} to
+	// be unique.
+	spi := SPI(sa.Spi)
+
+	ipsec.log.Infof("ipsec: InitSALocal: %s -> %s :%d 0x%x", remoteIP, localIP, udpPort, spi)
+
+	// Create SA
+	if sa, err := xfrmState(remoteIP, localIP, spi, false, key); err == nil {
+		if err := netlink.XfrmStateUpdate(sa); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("xfrm state update (in, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+		}
+	} else {
+		return errors.Wrap(err, "new xfrm state (in)")
+	}
+
+	// Install iptables rules
+	if err := ipsec.installDropNonEncrypted(localIP, remoteIP, udpPort, spi); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("install protecting rules (%s, %s, %d, 0x%x)", localIP, remoteIP, udpPort, spi))
+	}
+
+	si := spiInfo{spi: spi, isDirOut: false}
+	ipsec.spiInfo[spiID] = si
+	ipsec.spis[spi] = &si
+
+	// Trigger the initialization on the remote peer
+	msg := &msgInitSARemote{nonce, spi}
+	if err := initRemote(msg.serialize()); err != nil {
+		return errors.Wrap(err, "send InitSARemote")
+	}
+
+	return nil
+}
+
+// InitSARemote initializes outbound ipsec to remotePeer.
+// Triggered by remotePeer.
+func (ipsec *IPSec) InitSARemote(msgInitSARemote []byte, localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int, sessionKey *[32]byte) error {
+	// ID of outbound SPI
+	spiID := getSPIId(localPeer, remotePeer, connUID)
+
+	msg, err := deserializeMsgInitSARemote(msgInitSARemote)
+	if err != nil {
+		return errors.Wrap(err, "deserialize InitSARemote")
+	}
+	spi := msg.spi
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	ipsec.log.Infof("ipsec: InitSARemote: %s -> %s :%d 0x%x", localIP, remoteIP, udpPort, spi)
+
+	// Derive SA key by using the received nonce
+	key, err := deriveKey(sessionKey[:], msg.nonce, remotePeer)
+	if err != nil {
+		return errors.Wrap(err, "derive key")
+	}
+
+	// Create SA
+	if sa, err := xfrmState(localIP, remoteIP, spi, true, key); err == nil {
+		if err := netlink.XfrmStateAdd(sa); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("xfrm state update (out, %s, %s, 0x%x)", sa.Src, sa.Dst, sa.Spi))
+		}
+	} else {
+		return errors.Wrap(err, "new xfrm state (out)")
+	}
+
+	// Create or update SP
+	sp := xfrmPolicy(localIP, remoteIP, spi)
+	if err := netlink.XfrmPolicyUpdate(sp); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("xfrm policy update (%s, %s, 0x%x)", localIP, remoteIP, spi))
+	}
+
+	si := spiInfo{spi: spi, isDirOut: true}
+	ipsec.spiInfo[spiID] = si
+	ipsec.spis[spi] = &si
+
+	return nil
+}
+
+// Destroy destroys any (inbound / outbound) ipsec establishment between the peers.
+func (ipsec *IPSec) Destroy(localPeer, remotePeer mesh.PeerName, connUID uint64, localIP, remoteIP net.IP, udpPort int) error {
+	outSPIID := getSPIId(localPeer, remotePeer, connUID)
+	inSPIID := getSPIId(remotePeer, localPeer, connUID)
+
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	// Destroy inbound
+
+	if inSPIInfo, ok := ipsec.spiInfo[inSPIID]; ok {
+		ipsec.log.Infof("ipsec: destroy: in %s -> %s 0x%x", remoteIP, localIP, inSPIInfo.spi)
+
+		inSPI := inSPIInfo.spi
+
+		inSA := &netlink.XfrmState{
+			Src:   remoteIP,
+			Dst:   localIP,
+			Proto: netlink.XFRM_PROTO_ESP,
+			Spi:   int(inSPI),
+		}
+		if err := netlink.XfrmStateDel(inSA); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm state del (in, %s, %s, 0x%x) failed: %s", inSA.Src, inSA.Dst, inSA.Spi, err)
+		}
+
+		if err := ipsec.removeDropNonEncrypted(localIP, remoteIP, udpPort, inSPI); err != nil {
+			ipsec.log.Warnf("ipsec: remove protecting rules (%s, %s, %d, 0x%x) failed: %s", localIP, remoteIP, udpPort, inSPI, err)
+		}
+
+		delete(ipsec.spiInfo, inSPIID)
+		delete(ipsec.spis, inSPI)
+	}
+
+	// Destroy outbound
+
+	if outSPIInfo, ok := ipsec.spiInfo[outSPIID]; ok {
+		ipsec.log.Infof("ipsec: destroy: out %s -> %s 0x%x", localIP, remoteIP, outSPIInfo.spi)
+
+		if err := netlink.XfrmPolicyDel(xfrmPolicy(localIP, remoteIP, outSPIInfo.spi)); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm policy del (%s, %s, 0x%x) failed: %s", localIP, remoteIP, outSPIInfo.spi, err)
+		}
+
+		outSA := &netlink.XfrmState{
+			Src:   localIP,
+			Dst:   remoteIP,
+			Proto: netlink.XFRM_PROTO_ESP,
+			Spi:   int(outSPIInfo.spi),
+		}
+		if err := netlink.XfrmStateDel(outSA); err != nil {
+			ipsec.log.Warnf("ipsec: xfrm state del (out, %s, %s, 0x%x) failed: %s", outSA.Src, outSA.Dst, outSA.Spi, err)
+		}
+
+		delete(ipsec.spiInfo, outSPIID)
+		delete(ipsec.spis, outSPIInfo.spi)
+	}
+
+	return nil
+}
+
+// Flush removes all policies/SAs established by us. Also, it removes chains and
+// rules of iptables.
+//
+// If destroy is true, the chains and the rules won't be re-created.
+func (ipsec *IPSec) Flush(destroy bool) error {
+	ipsec.Lock()
+	defer ipsec.Unlock()
+
+	policies, err := netlink.XfrmPolicyList(syscall.AF_INET)
+	if err != nil {
+		return errors.Wrap(err, "xfrm policy list")
+	}
+	for _, p := range policies {
+		if p.Mark != nil && p.Mark.Value == mark && len(p.Tmpls) != 0 {
+			spi := SPI(p.Tmpls[0].Spi)
+			if err := netlink.XfrmPolicyDel(&p); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("xfrm policy del (%s, %s, 0x%x)", p.Src, p.Dst, spi))
+			}
+		}
+	}
+
+	states, err := netlink.XfrmStateList(syscall.AF_INET)
+	if err != nil {
+		return errors.Wrap(err, "xfrm state list")
+	}
+	for _, s := range states {
+		if _, ok := ipsec.spis[SPI(s.Spi)]; ok {
+			if err := netlink.XfrmStateDel(&s); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("xfrm state list (%s, %s, 0x%x)", s.Src, s.Dst, s.Spi))
+			}
+		}
+	}
+
+	if err := ipsec.resetIPTables(destroy); err != nil {
+		return errors.Wrap(err, "reset ip tables")
+	}
+
+	return nil
+}
+
+// iptables
+
+type chain struct {
+	table string
+	chain string
+}
+type rule struct {
+	table    string
+	chain    string
+	rulespec []string
+	unique   bool
+}
+
+func (ipsec *IPSec) clearChains(chains []chain) error {
+	for _, c := range chains {
+		if err := ipsec.ipt.ClearChain(c.table, c.chain); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables clear chain (%s, %s)", c.table, c.chain))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) deleteChains(chains []chain) error {
+	for _, c := range chains {
+		if err := ipsec.ipt.DeleteChain(c.table, c.chain); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables delete chain (%s, %s)", c.table, c.chain))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) resetRules(rules []rule, destroy bool) error {
+	for _, r := range rules {
+		ok, err := ipsec.ipt.Exists(r.table, r.chain, r.rulespec...)
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables exists rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+		}
+		switch {
+		case !destroy && !ok:
+			if err := ipsec.ipt.Append(r.table, r.chain, r.rulespec...); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("iptables append rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+			}
+		case destroy && ok:
+			if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("iptables delete rule (%s, %s, %s)", r.table, r.chain, r.rulespec))
+			}
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) resetIPTables(destroy bool) error {
+	chains := []chain{
+		{tableMangle, chainIn},
+		{tableMangle, chainInMark},
+		{tableFilter, chainIn},
+		{tableMangle, chainOut},
+		{tableMangle, chainOutMark},
+	}
+	rules := []rule{
+		{tableMangle, "INPUT", []string{"-j", chainIn}, true},
+		{tableMangle, chainInMark, []string{"-j", "MARK", "--set-xmark", markStr}, true},
+		{tableFilter, "INPUT", []string{"-j", chainIn}, true},
+		{tableMangle, "OUTPUT", []string{"-j", chainOut}, true},
+		{tableMangle, chainOutMark, []string{"-j", "MARK", "--set-xmark", markStr}, true},
+		{tableFilter, "OUTPUT",
+			[]string{
+				"!", "-p", "esp",
+				"-m", "policy", "--dir", "out", "--pol", "none",
+				"-m", "mark", "--mark", markStr,
+				"-j", "DROP"}, true},
+	}
+
+	if err := ipsec.clearChains(chains); err != nil {
+		return err
+	}
+
+	if err := ipsec.resetRules(rules, destroy); err != nil {
+		return err
+	}
+
+	if destroy {
+		if err := ipsec.deleteChains(chains); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ruleMarkInboundESP(srcIP, dstIP net.IP, inSPI SPI) rule {
+	return rule{tableMangle, chainIn,
+		[]string{
+			"-s", dstIP.String(), "-d", srcIP.String(),
+			"-p", "esp",
+			"-m", "esp", "--espspi", "0x" + strconv.FormatUint(uint64(inSPI), 16),
+			"-j", chainInMark,
+		}, true}
+}
+
+func rulesDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) []rule {
+	udpPortStr := strconv.FormatUint(uint64(udpPort), 10)
+	return []rule{
+		ruleMarkInboundESP(srcIP, dstIP, inSPI),
+		{tableFilter, chainIn,
+			[]string{
+				"-s", dstIP.String(), "-d", srcIP.String(),
+				"-p", "udp", "--dport", udpPortStr,
+				"-m", "mark", "!", "--mark", markStr,
+				"-j", "DROP",
+			}, false},
+		{tableMangle, chainOut,
+			[]string{
+				"-s", srcIP.String(), "-d", dstIP.String(),
+				"-p", "udp", "--dport", udpPortStr,
+				"-j", chainOutMark,
+			}, false},
+	}
+}
+
+func (ipsec *IPSec) installDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error {
+	rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI)
+	for _, r := range rules {
+		appendFunc := ipsec.ipt.Append
+		if r.unique {
+			appendFunc = ipsec.ipt.AppendUnique
+		}
+		if err := appendFunc(r.table, r.chain, r.rulespec...); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("iptables append (%s, %s, %s)", r.table, r.chain, r.rulespec))
+		}
+	}
+	return nil
+}
+
+func (ipsec *IPSec) removeDropNonEncrypted(srcIP, dstIP net.IP, udpPort int, inSPI SPI) error {
+	rules := rulesDropNonEncrypted(srcIP, dstIP, udpPort, inSPI)
+	if err := ipsec.resetRules(rules, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ipsec *IPSec) removeDropNonEncryptedInbound(srcIP, dstIP net.IP, inSPI SPI) error {
+	r := ruleMarkInboundESP(srcIP, dstIP, inSPI)
+	if err := ipsec.ipt.Delete(r.table, r.chain, r.rulespec...); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("iptables delete unique (%s, %s, %s)", r.table, r.chain, r.rulespec))
+	}
+	return nil
+}
+
+// xfrm
+
+func xfrmAllocSpiState(srcIP, dstIP net.IP) *netlink.XfrmState {
+	return &netlink.XfrmState{
+		Src:          srcIP,
+		Dst:          dstIP,
+		Proto:        netlink.XFRM_PROTO_ESP,
+		Mode:         netlink.XFRM_MODE_TRANSPORT,
+		ReplayWindow: 256,
+		ESN:          true,
+	}
+}
+
+func xfrmState(srcIP, dstIP net.IP, spi SPI, isDirOut bool, key []byte) (*netlink.XfrmState, error) {
+	if len(key) != keySize {
+		return nil, fmt.Errorf("key should be %d bytes long", keySize)
+	}
+
+	state := xfrmAllocSpiState(srcIP, dstIP)
+
+	state.Spi = int(spi)
+	state.Aead = &netlink.XfrmStateAlgo{
+		Name:   "rfc4106(gcm(aes))",
+		Key:    key,
+		ICVLen: 128,
+	}
+
+	return state, nil
+}
+
+func xfrmPolicy(srcIP, dstIP net.IP, spi SPI) *netlink.XfrmPolicy {
+	ipMask := []byte{0xff, 0xff, 0xff, 0xff} // /32
+
+	return &netlink.XfrmPolicy{
+		Src:   &net.IPNet{IP: srcIP, Mask: ipMask},
+		Dst:   &net.IPNet{IP: dstIP, Mask: ipMask},
+		Proto: syscall.IPPROTO_UDP,
+		Dir:   netlink.XFRM_DIR_OUT,
+		Mark: &netlink.XfrmMark{
+			Value: mark,
+			Mask:  mark,
+		},
+		Tmpls: []netlink.XfrmPolicyTmpl{
+			{
+				Src:   srcIP,
+				Dst:   dstIP,
+				Proto: netlink.XFRM_PROTO_ESP,
+				Mode:  netlink.XFRM_MODE_TRANSPORT,
+				Spi:   int(spi),
+			},
+		},
+	}
+}
+
+// Key derivation
+
+func genNonce() ([]byte, error) {
+	buf := make([]byte, nonceSize)
+	n, err := rand.Read(buf)
+	if err != nil {
+		return nil, fmt.Errorf("crypto rand failed: %s", err)
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("not enough of random data: %d", n)
+	}
+	return buf, nil
+}
+
+func deriveKey(sessionKey []byte, nonce []byte, peerName mesh.PeerName) ([]byte, error) {
+	key := make([]byte, keySize)
+
+	info := make([]byte, 8)
+	binary.BigEndian.PutUint64(info, uint64(peerName))
+
+	hkdf := hkdf.New(sha256.New, sessionKey, nonce, info)
+
+	n, err := io.ReadFull(hkdf, key)
+	if err != nil {
+		return nil, err
+	}
+	if n != keySize {
+		return nil, fmt.Errorf("derived too short key: %d", n)
+	}
+
+	return key, nil
+}
+
+// Protocol Messages
+
+type msgInitSARemote struct {
+	nonce []byte
+	spi   SPI
+}
+
+func deserializeMsgInitSARemote(b []byte) (*msgInitSARemote, error) {
+	if len(b) == 0 {
+		return nil, fmt.Errorf("empty msg")
+	}
+
+	msg := &msgInitSARemote{}
+	if len(b) != msg.size() {
+		return nil, fmt.Errorf("invalid payload size: %d", len(b))
+	}
+
+	msg.nonce = make([]byte, nonceSize)
+	copy(msg.nonce, b[:nonceSize])
+	b = b[nonceSize:]
+
+	msg.spi = SPI(binary.BigEndian.Uint32(b))
+
+	return msg, nil
+}
+
+func (msg *msgInitSARemote) size() int {
+	return nonceSize + 32 // SPI
+}
+
+func (msg *msgInitSARemote) serialize() []byte {
+	b := make([]byte, msg.size())
+
+	copy(b[:nonceSize], msg.nonce)
+	binary.BigEndian.PutUint32(b[nonceSize:], uint32(msg.spi))
+
+	return b
+}

net/mac.go

@@ -0,0 +1,38 @@
+package net
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"net"
+)
+
+func RandomMAC() (net.HardwareAddr, error) {
+	mac := make([]byte, 6)
+	if _, err := rand.Read(mac); err != nil {
+		return nil, err
+	}
+
+	setUnicastAndLocal(mac)
+
+	return net.HardwareAddr(mac), nil
+}
+
+func MACfromUUID(uuid []byte) net.HardwareAddr {
+	hash := sha256.New()
+	// We salt the input just as a precaution to avoid clashes with
+	// other applications who might have had the bright idea of
+	// generating MACs in the same way.
+	hash.Write([]byte("9oBJ0Jmip-"))
+	hash.Write(uuid)
+	sum := hash.Sum(nil)
+
+	setUnicastAndLocal(sum)
+
+	return net.HardwareAddr(sum[:6])
+}
+
+// In the first byte of the MAC, the 'multicast' bit should be
+// clear and 'locally administered' bit should be set.
+func setUnicastAndLocal(mac []byte) {
+	mac[0] = (mac[0] & 0xFE) | 0x02
+}

net/route.go

@@ -13,7 +13,7 @@ import (
 func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
 	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && overlaps(route.Dst, subnet) {
-			return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
+			return fmt.Errorf("Network %s overlaps with existing route %s on host", subnet, route.Dst)
 		}
 		return nil
 	})
@@ -29,7 +29,7 @@ func overlaps(n1, n2 *net.IPNet) bool {
 func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
 	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && route.Dst.Contains(addr) {
-			return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
+			return fmt.Errorf("Address %s overlaps with existing route %s on host", addr, route.Dst)
 		}
 		return nil
 	})

net/uuid.go

@@ -0,0 +1,84 @@
+package net
+
+import (
+	"errors"
+	"io/ioutil"
+	"net"
+	"os"
+
+	"github.com/weaveworks/mesh"
+	"github.com/weaveworks/weave/db"
+)
+
+func getOldStyleSystemUUID() ([]byte, error) {
+	uuid, err := ioutil.ReadFile("/sys/class/dmi/id/product_uuid")
+	if os.IsNotExist(err) {
+		uuid, err = ioutil.ReadFile("/sys/hypervisor/uuid")
+	}
+	return uuid, err
+}
+
+func getSystemUUID(hostRoot string) ([]byte, error) {
+	uuid, err := getOldStyleSystemUUID()
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	machineid, err := ioutil.ReadFile(hostRoot + "/etc/machine-id")
+	if os.IsNotExist(err) {
+		machineid, err = ioutil.ReadFile(hostRoot + "/var/lib/dbus/machine-id")
+	}
+	if err != nil && !os.IsNotExist(err) {
+		return nil, err
+	}
+	if len(uuid) == 0 && len(machineid) == 0 {
+		return nil, errors.New("All system IDs are blank")
+	}
+	return append(machineid, uuid...), nil
+}
+
+func getPersistedPeerName(pathName string) (mesh.PeerName, error) {
+	d, err := db.NewBoltDBReadOnly(pathName)
+	if err != nil {
+		return mesh.UnknownPeerName, err
+	}
+	defer d.Close()
+	var peerName mesh.PeerName
+	nameFound, err := d.Load(db.NameIdent, &peerName)
+	if err != nil || !nameFound {
+		return mesh.UnknownPeerName, err
+	}
+	return peerName, nil
+}
+
+// GetSystemPeerName returns an ID derived from concatenated machine-id
+// (either systemd or dbus), the system (aka bios) UUID and the
+// hypervisor UUID.  It is tweaked and formatted to be usable as a mac address
+func GetSystemPeerName(dbPrefix, hostRoot string) (string, error) {
+	// Check if we have a persisted name that matches the old-style ID for this host
+	if oldUUID, err := getOldStyleSystemUUID(); err == nil {
+		pathName := dbPrefix + db.FileName
+		if _, err := os.Stat(pathName); err == nil {
+			persistedPeerName, err := getPersistedPeerName(pathName)
+			if err != nil && !os.IsNotExist(err) {
+				return "", err
+			}
+			if persistedPeerName == mesh.PeerNameFromBin(MACfromUUID(oldUUID)) {
+				return persistedPeerName.String(), nil
+			}
+		}
+	} else if !os.IsNotExist(err) {
+		return "", err
+	}
+	var mac net.HardwareAddr
+	if uuid, err := getSystemUUID(hostRoot); err == nil {
+		mac = MACfromUUID(uuid)
+	} else if !os.IsNotExist(err) {
+		return "", err
+	} else {
+		mac, err = RandomMAC()
+		if err != nil {
+			return "", err
+		}
+	}
+	return mac.String(), nil
+}

net/veth.go

@@ -113,7 +113,7 @@ func interfaceExistsInNamespace(netNSPath string, ifName string) bool {
 
 // NB: This function can be used only by a process that terminates immediately
 //     after calling the function as it changes netns via WithNetNSLinkUnsafe.
-func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool) error {
+func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMulticastRoute bool, cidrs []*net.IPNet, keepTXOn bool, hairpinMode bool) error {
 	ns, err := netns.GetFromPath(netNSPath)
 	if err != nil {
 		return err
@@ -131,7 +131,7 @@ func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMult
 			id = id[:maxIDLen] // trim passed ID if too long
 		}
 		name, peerName := vethPrefix+"pl"+id, vethPrefix+"pg"+id
-		_, err := CreateAndAttachVeth(name, peerName, bridgeName, mtu, keepTXOn, func(veth netlink.Link) error {
+		veth, err := CreateAndAttachVeth(name, peerName, bridgeName, mtu, keepTXOn, func(veth netlink.Link) error {
 			if err := netlink.LinkSetNsFd(veth, int(ns)); err != nil {
 				return fmt.Errorf("failed to move veth to container netns: %s", err)
 			}
@@ -143,6 +143,10 @@ func AttachContainer(netNSPath, id, ifName, bridgeName string, mtu int, withMult
 		if err != nil {
 			return err
 		}
+		if err = netlink.LinkSetHairpin(veth, hairpinMode); err != nil {
+			return fmt.Errorf("unable to set hairpin mode to %t for bridge side of veth %s: %s", hairpinMode, name, err)
+		}
+
 	}
 
 	if err := WithNetNSLinkUnsafe(ns, ifName, func(veth netlink.Link) error {

npc/constants.go

@@ -6,4 +6,8 @@ const (
 	MainChain    = "WEAVE-NPC"
 	DefaultChain = "WEAVE-NPC-DEFAULT"
 	IngressChain = "WEAVE-NPC-INGRESS"
+
+	IpsetNamePrefix = "weave-"
+
+	LocalIpset = IpsetNamePrefix + "local-pods"
 )

npc/controller.go

@@ -3,12 +3,12 @@ package npc
 import (
 	"sync"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/pkg/errors"
 	coreapi "k8s.io/client-go/pkg/api/v1"
 	extnapi "k8s.io/client-go/pkg/apis/extensions/v1beta1"
 
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/npc/ipset"
 )
 
@@ -29,6 +29,8 @@ type NetworkPolicyController interface {
 type controller struct {
 	sync.Mutex
 
+	nodeName string // my node name
+
 	ipt *iptables.IPTables
 	ips ipset.Interface
 
@@ -36,11 +38,12 @@ type controller struct {
 	nsSelectors *selectorSet   // selector string -> nsSelector
 }
 
-func New(ipt *iptables.IPTables, ips ipset.Interface) NetworkPolicyController {
+func New(nodeName string, ipt *iptables.IPTables, ips ipset.Interface) NetworkPolicyController {
 	c := &controller{
-		ipt: ipt,
-		ips: ips,
-		nss: make(map[string]*ns)}
+		nodeName: nodeName,
+		ipt:      ipt,
+		ips:      ips,
+		nss:      make(map[string]*ns)}
 
 	c.nsSelectors = newSelectorSet(ips, c.onNewNsSelector)
 
@@ -82,11 +85,22 @@ func (npc *controller) withNS(name string, f func(ns *ns) error) error {
 	return nil
 }
 
+func (npc *controller) checkLocalPod(obj *coreapi.Pod, event string) bool {
+	if obj.Spec.NodeName != npc.nodeName {
+		common.Log.Debugf("%s ignored for pod %s/%s on node %s", event, obj.ObjectMeta.Namespace, obj.ObjectMeta.Name, obj.Spec.NodeName)
+		return false
+	}
+	return true
+}
+
 func (npc *controller) AddPod(obj *coreapi.Pod) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT AddPod %#v", obj)
+	if !npc.checkLocalPod(obj, "AddPod") {
+		return nil
+	}
+	common.Log.Debugf("EVENT AddPod %s", js(obj))
 	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
 		return errors.Wrap(ns.addPod(obj), "add pod")
 	})
@@ -96,7 +110,11 @@ func (npc *controller) UpdatePod(oldObj, newObj *coreapi.Pod) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT UpdatePod %#v %#v", oldObj, newObj)
+	// Pods don't move across nodes so we only check the new version
+	if !npc.checkLocalPod(newObj, "UpdatePod") {
+		return nil
+	}
+	common.Log.Debugf("EVENT UpdatePod %s %s", js(oldObj), js(newObj))
 	return npc.withNS(oldObj.ObjectMeta.Namespace, func(ns *ns) error {
 		return errors.Wrap(ns.updatePod(oldObj, newObj), "update pod")
 	})
@@ -106,7 +124,10 @@ func (npc *controller) DeletePod(obj *coreapi.Pod) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT DeletePod %#v", obj)
+	if !npc.checkLocalPod(obj, "DeletePod") {
+		return nil
+	}
+	common.Log.Debugf("EVENT DeletePod %s", js(obj))
 	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
 		return errors.Wrap(ns.deletePod(obj), "delete pod")
 	})
@@ -116,7 +137,7 @@ func (npc *controller) AddNetworkPolicy(obj *extnapi.NetworkPolicy) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT AddNetworkPolicy %#v", obj)
+	common.Log.Infof("EVENT AddNetworkPolicy %s", js(obj))
 	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
 		return errors.Wrap(ns.addNetworkPolicy(obj), "add network policy")
 	})
@@ -126,7 +147,7 @@ func (npc *controller) UpdateNetworkPolicy(oldObj, newObj *extnapi.NetworkPolicy
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT UpdateNetworkPolicy %#v %#v", oldObj, newObj)
+	common.Log.Infof("EVENT UpdateNetworkPolicy %s %s", js(oldObj), js(newObj))
 	return npc.withNS(oldObj.ObjectMeta.Namespace, func(ns *ns) error {
 		return errors.Wrap(ns.updateNetworkPolicy(oldObj, newObj), "update network policy")
 	})
@@ -136,7 +157,7 @@ func (npc *controller) DeleteNetworkPolicy(obj *extnapi.NetworkPolicy) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT DeleteNetworkPolicy %#v", obj)
+	common.Log.Infof("EVENT DeleteNetworkPolicy %s", js(obj))
 	return npc.withNS(obj.ObjectMeta.Namespace, func(ns *ns) error {
 		return errors.Wrap(ns.deleteNetworkPolicy(obj), "delete network policy")
 	})
@@ -146,7 +167,7 @@ func (npc *controller) AddNamespace(obj *coreapi.Namespace) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT AddNamespace %#v", obj)
+	common.Log.Infof("EVENT AddNamespace %s", js(obj))
 	return npc.withNS(obj.ObjectMeta.Name, func(ns *ns) error {
 		return errors.Wrap(ns.addNamespace(obj), "add namespace")
 	})
@@ -156,7 +177,7 @@ func (npc *controller) UpdateNamespace(oldObj, newObj *coreapi.Namespace) error
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT UpdateNamespace%#v %#v", oldObj, newObj)
+	common.Log.Infof("EVENT UpdateNamespace %s %s", js(oldObj), js(newObj))
 	return npc.withNS(oldObj.ObjectMeta.Name, func(ns *ns) error {
 		return errors.Wrap(ns.updateNamespace(oldObj, newObj), "update namespace")
 	})
@@ -166,7 +187,7 @@ func (npc *controller) DeleteNamespace(obj *coreapi.Namespace) error {
 	npc.Lock()
 	defer npc.Unlock()
 
-	log.Infof("EVENT DeleteNamespace %#v", obj)
+	common.Log.Infof("EVENT DeleteNamespace %s", js(obj))
 	return npc.withNS(obj.ObjectMeta.Name, func(ns *ns) error {
 		return errors.Wrap(ns.deleteNamespace(obj), "delete namespace")
 	})

npc/ipset/ipset.go

@@ -1,7 +1,9 @@
 package ipset
 
 import (
+	"log"
 	"os/exec"
+	"strings"
 
 	"github.com/pkg/errors"
 )
@@ -22,15 +24,19 @@ type Interface interface {
 	Flush(ipsetName Name) error
 	Destroy(ipsetName Name) error
 
+	List(prefix string) ([]Name, error)
+
 	FlushAll() error
 	DestroyAll() error
 }
 
 type ipset struct {
+	refCount
+	*log.Logger
 }
 
-func New() Interface {
-	return &ipset{}
+func New(logger *log.Logger) Interface {
+	return &ipset{refCount: newRefCount(), Logger: logger}
 }
 
 func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
@@ -38,32 +44,97 @@ func (i *ipset) Create(ipsetName Name, ipsetType Type) error {
 }
 
 func (i *ipset) AddEntry(ipsetName Name, entry string) error {
+	i.Logger.Printf("adding entry %s to %s", entry, ipsetName)
+	if i.inc(ipsetName, entry) > 1 { // already in the set
+		return nil
+	}
 	return doExec("add", string(ipsetName), entry)
 }
 
 func (i *ipset) DelEntry(ipsetName Name, entry string) error {
+	i.Logger.Printf("deleting entry %s from %s", entry, ipsetName)
+	if i.dec(ipsetName, entry) > 0 { // still needed
+		return nil
+	}
 	return doExec("del", string(ipsetName), entry)
 }
 
 func (i *ipset) Flush(ipsetName Name) error {
+	i.removeSet(ipsetName)
 	return doExec("flush", string(ipsetName))
 }
 
 func (i *ipset) FlushAll() error {
+	i.refCount = newRefCount()
 	return doExec("flush")
 }
 
 func (i *ipset) Destroy(ipsetName Name) error {
+	i.removeSet(ipsetName)
 	return doExec("destroy", string(ipsetName))
 }
 
 func (i *ipset) DestroyAll() error {
+	i.refCount = newRefCount()
 	return doExec("destroy")
 }
 
+// Fetch a list of all existing sets with a given prefix
+func (i *ipset) List(prefix string) ([]Name, error) {
+	output, err := exec.Command("ipset", "list", "-name", "-output", "plain").Output()
+	if err != nil {
+		return nil, err
+	}
+
+	var selected []Name
+	sets := strings.Split(string(output), "\n")
+	for _, v := range sets {
+		if strings.HasPrefix(v, prefix) {
+			selected = append(selected, Name(v))
+		}
+	}
+
+	return selected, err
+}
+
 func doExec(args ...string) error {
 	if output, err := exec.Command("ipset", args...).CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "ipset %v failed: %s", args, output)
 	}
 	return nil
 }
+
+// Reference-counting
+type key struct {
+	ipsetName Name
+	entry     string
+}
+
+// note no locking is required as all operations are serialised in the controller
+type refCount struct {
+	ref map[key]int
+}
+
+func newRefCount() refCount {
+	return refCount{ref: make(map[key]int)}
+}
+
+func (rc *refCount) inc(ipsetName Name, entry string) int {
+	k := key{ipsetName, entry}
+	rc.ref[k]++
+	return rc.ref[k]
+}
+
+func (rc *refCount) dec(ipsetName Name, entry string) int {
+	k := key{ipsetName, entry}
+	rc.ref[k]--
+	return rc.ref[k]
+}
+
+func (rc *refCount) removeSet(ipsetName Name) {
+	for k := range rc.ref {
+		if k.ipsetName == ipsetName {
+			delete(rc.ref, k)
+		}
+	}
+}

npc/json.go

@@ -0,0 +1,10 @@
+package npc
+
+import (
+	"encoding/json"
+)
+
+func js(v interface{}) string {
+	a, _ := json.Marshal(v)
+	return string(a)
+}

npc/metrics/metrics.go

@@ -1,7 +1,6 @@
 package metrics
 
 import (
-	log "github.com/Sirupsen/logrus"
 	"net/http"
 	"os"
 	"strconv"
@@ -11,6 +10,8 @@ import (
 	"github.com/google/gopacket/pcapgo"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/weaveworks/weave/common"
 )
 
 var (
@@ -26,18 +27,18 @@ var (
 func gatherMetrics() {
 	pipe, err := os.Open("/var/log/ulogd.pcap")
 	if err != nil {
-		log.Fatalf("Failed to open pcap: %v", err)
+		common.Log.Fatalf("Failed to open pcap: %v", err)
 	}
 
 	reader, err := pcapgo.NewReader(pipe)
 	if err != nil {
-		log.Fatalf("Failed to read pcap header: %v", err)
+		common.Log.Fatalf("Failed to read pcap header: %v", err)
 	}
 
 	for {
 		data, _, err := reader.ReadPacketData()
 		if err != nil {
-			log.Fatalf("Failed to read pcap packet: %v", err)
+			common.Log.Fatalf("Failed to read pcap packet: %v", err)
 		}
 
 		packet := gopacket.NewPacket(data, layers.LayerTypeIPv4, gopacket.Default)
@@ -46,7 +47,7 @@ func gatherMetrics() {
 			tcp, _ := tcpLayer.(*layers.TCP)
 			if tcp.SYN && !tcp.ACK { // Only plain SYN constitutes a NEW TCP connection
 				blockedConnections.With(prometheus.Labels{"protocol": "tcp", "dport": strconv.Itoa(int(tcp.DstPort))}).Inc()
-				log.Warnf("TCP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), tcp.SrcPort, dstIP(packet), tcp.DstPort)
+				common.Log.Warnf("TCP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), tcp.SrcPort, dstIP(packet), tcp.DstPort)
 				continue
 			}
 		}
@@ -54,7 +55,7 @@ func gatherMetrics() {
 		if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil {
 			udp, _ := udpLayer.(*layers.UDP)
 			blockedConnections.With(prometheus.Labels{"protocol": "udp", "dport": strconv.Itoa(int(udp.DstPort))}).Inc()
-			log.Warnf("UDP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), udp.SrcPort, dstIP(packet), udp.DstPort)
+			common.Log.Warnf("UDP connection from %v:%d to %v:%d blocked by Weave NPC.", srcIP(packet), udp.SrcPort, dstIP(packet), udp.DstPort)
 			continue
 		}
 	}
@@ -98,9 +99,9 @@ func Start(addr string) error {
 	http.Handle("/metrics", promhttp.Handler())
 
 	go func() {
-		log.Infof("Serving /metrics on %s", addr)
+		common.Log.Infof("Serving /metrics on %s", addr)
 		if err := http.ListenAndServe(addr, nil); err != nil {
-			log.Fatalf("Failed to bind metrics server: %v", err)
+			common.Log.Fatalf("Failed to bind metrics server: %v", err)
 		}
 	}()
 

npc/namespace.go

@@ -3,7 +3,6 @@ package npc
 import (
 	"encoding/json"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/coreos/go-iptables/iptables"
 	"k8s.io/client-go/pkg/api/unversioned"
 	coreapi "k8s.io/client-go/pkg/api/v1"
@@ -11,6 +10,7 @@ import (
 	"k8s.io/client-go/pkg/types"
 	"k8s.io/client-go/pkg/util/uuid"
 
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/npc/ipset"
 )
 
@@ -88,6 +88,7 @@ func (ns *ns) addPod(obj *coreapi.Pod) error {
 		return nil
 	}
 
+	ns.ips.AddEntry(LocalIpset, obj.Status.PodIP)
 	return ns.podSelectors.addToMatching(obj.ObjectMeta.Labels, obj.Status.PodIP)
 }
 
@@ -100,10 +101,12 @@ func (ns *ns) updatePod(oldObj, newObj *coreapi.Pod) error {
 	}
 
 	if hasIP(oldObj) && !hasIP(newObj) {
+		ns.ips.DelEntry(LocalIpset, oldObj.Status.PodIP)
 		return ns.podSelectors.delFromMatching(oldObj.ObjectMeta.Labels, oldObj.Status.PodIP)
 	}
 
 	if !hasIP(oldObj) && hasIP(newObj) {
+		ns.ips.AddEntry(LocalIpset, newObj.Status.PodIP)
 		return ns.podSelectors.addToMatching(newObj.ObjectMeta.Labels, newObj.Status.PodIP)
 	}
 
@@ -139,6 +142,7 @@ func (ns *ns) deletePod(obj *coreapi.Pod) error {
 		return nil
 	}
 
+	ns.ips.DelEntry(LocalIpset, obj.Status.PodIP)
 	return ns.podSelectors.delFromMatching(obj.ObjectMeta.Labels, obj.Status.PodIP)
 }
 
@@ -192,7 +196,7 @@ func (ns *ns) updateNetworkPolicy(oldObj, newObj *extnapi.NetworkPolicy) error {
 	if err := ns.nsSelectors.provision(oldObj.ObjectMeta.UID, oldNsSelectors, newNsSelectors); err != nil {
 		return err
 	}
-	if err := ns.podSelectors.provision(oldObj.ObjectMeta.UID, oldPodSelectors, newNsSelectors); err != nil {
+	if err := ns.podSelectors.provision(oldObj.ObjectMeta.UID, oldPodSelectors, newPodSelectors); err != nil {
 		return err
 	}
 	if err := ns.rules.provision(oldObj.ObjectMeta.UID, oldRules, newRules); err != nil {
@@ -320,14 +324,14 @@ func equals(a, b map[string]string) bool {
 }
 
 func isDefaultDeny(namespace *coreapi.Namespace) bool {
-	nnpJson, found := namespace.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"]
+	nnpJSON, found := namespace.ObjectMeta.Annotations["net.beta.kubernetes.io/network-policy"]
 	if !found {
 		return false
 	}
 
 	var nnp NamespaceNetworkPolicy
-	if err := json.Unmarshal([]byte(nnpJson), &nnp); err != nil {
-		log.Warn("Ignoring network policy annotation: unmarshal failed:", err)
+	if err := json.Unmarshal([]byte(nnpJSON), &nnp); err != nil {
+		common.Log.Warn("Ignoring network policy annotation: unmarshal failed:", err)
 		// If we can't understand the annotation, behave as if it isn't present
 		return false
 	}

npc/rule.go

@@ -3,9 +3,10 @@ package npc
 import (
 	"strings"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/coreos/go-iptables/iptables"
 	"k8s.io/client-go/pkg/types"
+
+	"github.com/weaveworks/weave/common"
 )
 
 type ruleSpec struct {
@@ -47,7 +48,7 @@ func (rs *ruleSet) deprovision(user types.UID, current, desired map[string]*rule
 		if _, found := desired[key]; !found {
 			delete(rs.users[key], user)
 			if len(rs.users[key]) == 0 {
-				log.Infof("deleting rule: %v", spec.args)
+				common.Log.Infof("deleting rule: %v", spec.args)
 				if err := rs.ipt.Delete(TableFilter, IngressChain, spec.args...); err != nil {
 					return err
 				}
@@ -63,7 +64,7 @@ func (rs *ruleSet) provision(user types.UID, current, desired map[string]*ruleSp
 	for key, spec := range desired {
 		if _, found := current[key]; !found {
 			if _, found := rs.users[key]; !found {
-				log.Infof("adding rule: %v", spec.args)
+				common.Log.Infof("adding rule: %v", spec.args)
 				if err := rs.ipt.Append(TableFilter, IngressChain, spec.args...); err != nil {
 					return err
 				}

npc/selector.go

@@ -1,11 +1,11 @@
 package npc
 
 import (
-	log "github.com/Sirupsen/logrus"
 	"k8s.io/client-go/pkg/api/unversioned"
 	"k8s.io/client-go/pkg/labels"
 	"k8s.io/client-go/pkg/types"
 
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/npc/ipset"
 )
 
@@ -29,7 +29,7 @@ func newSelectorSpec(json *unversioned.LabelSelector, nsName string, ipsetType i
 		// We prefix the selector string with the namespace name when generating
 		// the shortname because you can specify the same selector in multiple
 		// namespaces - we need those to map to distinct ipsets
-		ipsetName: ipset.Name("weave-" + shortName(nsName+":"+key)),
+		ipsetName: ipset.Name(IpsetNamePrefix + shortName(nsName+":"+key)),
 		ipsetType: ipsetType}, nil
 }
 
@@ -43,12 +43,10 @@ func (s *selector) matches(labelMap map[string]string) bool {
 }
 
 func (s *selector) addEntry(entry string) error {
-	log.Infof("adding entry %s to %s", entry, s.spec.ipsetName)
 	return s.ips.AddEntry(s.spec.ipsetName, entry)
 }
 
 func (s *selector) delEntry(entry string) error {
-	log.Infof("deleting entry %s from %s", entry, s.spec.ipsetName)
 	return s.ips.DelEntry(s.spec.ipsetName, entry)
 }
 
@@ -96,7 +94,7 @@ func (ss *selectorSet) deprovision(user types.UID, current, desired map[string]*
 		if _, found := desired[key]; !found {
 			delete(ss.users[key], user)
 			if len(ss.users[key]) == 0 {
-				log.Infof("destroying ipset: %#v", spec)
+				common.Log.Infof("destroying ipset: %#v", spec)
 				if err := ss.ips.Destroy(spec.ipsetName); err != nil {
 					return err
 				}
@@ -112,7 +110,7 @@ func (ss *selectorSet) provision(user types.UID, current, desired map[string]*se
 	for key, spec := range desired {
 		if _, found := current[key]; !found {
 			if _, found := ss.users[key]; !found {
-				log.Infof("creating ipset: %#v", spec)
+				common.Log.Infof("creating ipset: %#v", spec)
 				if err := ss.ips.Create(spec.ipsetName, spec.ipsetType); err != nil {
 					return err
 				}

npc/shortname.go

@@ -7,12 +7,13 @@ import (
 
 const (
 	// This array:
-	// * Must include only printable UTF8 characters that are represented with a single byte
+	// * Must include only printable UTF8 characters that are represented with a single byte (http://www.utf8-chartable.de/)
 	// * Must be at least of length 85 (`len("weave-") + l(2^160)/l(85)` equals 31, the maximum ipset name length)
 	// * Must not include commas as those are treated specially by `ipset add` when adding a named set to a list:set
 	// * Should not include space for readability
+	// * Should not include invalid chars for XML `<>&"'`
 	// * Should not include single quote or backslash to be nice to shell users
-	ShortNameSymbols = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789(){}[]<>_$%^&*|/?.;:@#~"
+	ShortNameSymbols = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789(){}[]+=_$%^!*|/?.;:@#~"
 )
 
 // sha1 hash an arbitrary string and represent it using the full range of

npc/ulogd/ulogd.go

@@ -1,21 +1,26 @@
 package ulogd
 
 import (
-	log "github.com/Sirupsen/logrus"
 	"io"
 	"os"
 	"os/exec"
+	"syscall"
+
+	"github.com/weaveworks/weave/common"
 )
 
 func waitForExit(cmd *exec.Cmd) {
 	if err := cmd.Wait(); err != nil {
-		log.Fatalf("ulogd terminated: %v", err)
+		common.Log.Fatalf("ulogd terminated: %v", err)
 	}
-	log.Fatal("ulogd terminated normally")
+	common.Log.Fatal("ulogd terminated normally")
 }
 
 func Start() error {
 	cmd := exec.Command("/usr/sbin/ulogd", "-v")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Pdeathsig: syscall.SIGKILL,
+	}
 	stdout, err := cmd.StderrPipe()
 	if err != nil {
 		return err

plugin/ipam/cni.go

@@ -5,19 +5,24 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/appc/cni/pkg/skel"
-	"github.com/appc/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/types/current"
 )
 
 func (i *Ipam) CmdAdd(args *skel.CmdArgs) error {
+	var conf types.NetConf
+	if err := json.Unmarshal(args.StdinData, &conf); err != nil {
+		return fmt.Errorf("failed to load netconf: %v", err)
+	}
 	result, err := i.Allocate(args)
 	if err != nil {
 		return err
 	}
-	return result.Print()
+	return types.PrintResult(result, conf.CNIVersion)
 }
 
-func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) {
+func (i *Ipam) Allocate(args *skel.CmdArgs) (types.Result, error) {
 	// extract the things we care about
 	conf, err := loadIPAMConf(args.StdinData)
 	if err != nil {
@@ -46,12 +51,13 @@ func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) {
 	if err != nil {
 		return nil, err
 	}
-	result := &types.Result{
-		IP4: &types.IPConfig{
-			IP:      *ipnet,
+	result := &current.Result{
+		IPs: []*current.IPConfig{{
+			Version: "4",
+			Address: *ipnet,
 			Gateway: conf.Gateway,
-			Routes:  conf.Routes,
-		},
+		}},
+		Routes: conf.Routes,
 	}
 	return result, nil
 }
@@ -65,9 +71,9 @@ func (i *Ipam) Release(args *skel.CmdArgs) error {
 }
 
 type ipamConf struct {
-	Subnet  string        `json:"subnet,omitempty"`
-	Gateway net.IP        `json:"gateway,omitempty"`
-	Routes  []types.Route `json:"routes"`
+	Subnet  string         `json:"subnet,omitempty"`
+	Gateway net.IP         `json:"gateway,omitempty"`
+	Routes  []*types.Route `json:"routes"`
 }
 
 type netConf struct {

plugin/net/cni.go

@@ -5,11 +5,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
 	"syscall"
 
-	"github.com/appc/cni/pkg/ipam"
-	"github.com/appc/cni/pkg/skel"
-	"github.com/appc/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/ipam"
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/vishvananda/netlink"
 	"github.com/vishvananda/netns"
 	weaveapi "github.com/weaveworks/weave/api"
@@ -32,7 +34,8 @@ func NewCNIPlugin(weave *weaveapi.Client) *CNIPlugin {
 
 func loadNetConf(bytes []byte) (*NetConf, error) {
 	n := &NetConf{
-		BrName: weavenet.WeaveBridgeName,
+		BrName:      weavenet.WeaveBridgeName,
+		HairpinMode: true,
 	}
 	if err := json.Unmarshal(bytes, n); err != nil {
 		return nil, fmt.Errorf("failed to load netconf: %v", err)
@@ -40,17 +43,26 @@ func loadNetConf(bytes []byte) (*NetConf, error) {
 	return n, nil
 }
 
-func (c *CNIPlugin) getIP(ipamType string, args *skel.CmdArgs) (result *types.Result, err error) {
+func (c *CNIPlugin) getIP(ipamType string, args *skel.CmdArgs) (newResult *current.Result, err error) {
+	var result types.Result
 	// Default IPAM is Weave's own
 	if ipamType == "" {
 		result, err = ipamplugin.NewIpam(c.weave).Allocate(args)
 	} else {
 		result, err = ipam.ExecAdd(ipamType, args.StdinData)
 	}
-	if err == nil && result.IP4 == nil {
+	if err != nil {
+		return nil, err
+	}
+	if result == nil {
+		return nil, fmt.Errorf("Received no usable result from IPAM plugin")
+	}
+	newResult, err = current.NewResultFromResult(result)
+	// Check if ipam returned no results without error
+	if err == nil && len(newResult.IPs) == 0 {
 		return nil, fmt.Errorf("IPAM plugin failed to allocate IP address")
 	}
-	return result, err
+	return newResult, err
 }
 
 func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
@@ -70,28 +82,35 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 	if err != nil {
 		return fmt.Errorf("unable to allocate IP address: %s", err)
 	}
+	// Only expecting one address
+	ip := result.IPs[0]
 
 	// If config says nothing about routes or gateway, default one will be via the bridge
-	if result.IP4.Routes == nil && result.IP4.Gateway == nil {
-		bridgeIP, err := findBridgeIP(conf.BrName, result.IP4.IP)
-		if err == errBridgeNoIP {
+	if len(result.Routes) == 0 && ip.Gateway == nil {
+		bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &ip.Address)
+		if err == weavenet.ErrBridgeNoIP {
 			bridgeArgs := *args
 			bridgeArgs.ContainerID = "weave:expose"
+			// It would be better if libcni let us send just the desired parameters,
+			// but there is a bug: https://github.com/containernetworking/cni/issues/410
+			// so just blank out the one we want to change
+			os.Setenv("CNI_CONTAINERID", bridgeArgs.ContainerID)
 			bridgeIPResult, err := c.getIP(conf.IPAM.Type, &bridgeArgs)
 			if err != nil {
 				return fmt.Errorf("unable to allocate IP address for bridge: %s", err)
 			}
-			if err := assignBridgeIP(conf.BrName, bridgeIPResult.IP4.IP); err != nil {
+			bridgeCIDR := bridgeIPResult.IPs[0].Address
+			if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil {
 				return fmt.Errorf("unable to assign IP address to bridge: %s", err)
 			}
-			if err := weavenet.ExposeNAT(bridgeIPResult.IP4.IP); err != nil {
+			if err := weavenet.ExposeNAT(bridgeCIDR); err != nil {
 				return fmt.Errorf("unable to create NAT rules: %s", err)
 			}
-			bridgeIP = bridgeIPResult.IP4.IP.IP
+			bridgeIP = bridgeCIDR.IP
 		} else if err != nil {
 			return err
 		}
-		result.IP4.Gateway = bridgeIP
+		result.IPs[0].Gateway = bridgeIP
 	}
 
 	ns, err := netns.GetFromPath(args.Netns)
@@ -110,20 +129,20 @@ func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
 		id = fmt.Sprintf("%x", data)
 	}
 
-	if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&result.IP4.IP}, false); err != nil {
+	if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&ip.Address}, false, conf.HairpinMode); err != nil {
 		return err
 	}
 	if err := weavenet.WithNetNSLinkUnsafe(ns, args.IfName, func(link netlink.Link) error {
-		return setupRoutes(link, args.IfName, result.IP4.IP, result.IP4.Gateway, result.IP4.Routes)
+		return setupRoutes(link, args.IfName, ip.Address, ip.Gateway, result.Routes)
 	}); err != nil {
 		return fmt.Errorf("error setting up routes: %s", err)
 	}
 
 	result.DNS = conf.DNS
-	return result.Print()
+	return types.PrintResult(result, conf.CNIVersion)
 }
 
-func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []types.Route) error {
+func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []*types.Route) error {
 	var err error
 	if routes == nil { // If config says nothing about routes, add a default one
 		if !ipnet.Contains(gw) {
@@ -133,7 +152,7 @@ func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, rou
 				return err
 			}
 		}
-		routes = []types.Route{{Dst: zeroNetwork}}
+		routes = []*types.Route{{Dst: zeroNetwork}}
 	}
 	for _, r := range routes {
 		if r.GW != nil {
@@ -164,33 +183,22 @@ func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
 	return nil
 }
 
-var errBridgeNoIP = fmt.Errorf("Bridge has no IP address")
-
-func findBridgeIP(bridgeName string, subnet net.IPNet) (net.IP, error) {
-	netdev, err := weavenet.GetBridgeNetDev(bridgeName)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get netdev for %q bridge: %s", bridgeName, err)
-	}
-	if len(netdev.CIDRs) == 0 {
-		return nil, errBridgeNoIP
-	}
-	for _, cidr := range netdev.CIDRs {
-		if subnet.Contains(cidr.IP) {
-			return cidr.IP, nil
-		}
-	}
-	// None in the required subnet; just return the first one
-	return netdev.CIDRs[0].IP, nil
-}
-
+// As of CNI 0.5 spec:
+//   "Plugins should generally complete a DEL action without error even if some resources are missing"
+// this method should therefore return nil in most, if not all, cases.
 func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
 	conf, err := loadNetConf(args.StdinData)
 	if err != nil {
-		return err
+		logOnStderr(err)
+		return nil
 	}
 
-	if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil {
-		return fmt.Errorf("error removing interface %q: %s", args.IfName, err)
+	// As of CNI 0.3 spec, runtimes can send blank if they just want the address deallocated
+	if args.Netns != "" {
+		if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil {
+			// We log the error and carry on instead of returning nil, as there may still be resources to free in IPAM.
+			logOnStderr(fmt.Errorf("error removing interface %q: %s", args.IfName, err))
+		}
 	}
 
 	// Default IPAM is Weave's own
@@ -199,16 +207,21 @@ func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
 	} else {
 		err = ipam.ExecDel(conf.IPAM.Type, args.StdinData)
 	}
-	if err != nil {
-		return fmt.Errorf("unable to release IP address: %s", err)
-	}
+	logOnStderr(fmt.Errorf("unable to release IP address: %s", err))
 	return nil
 }
 
+// Standard error is inherited from the caller (e.g. Kubelet, when running Weave Net's CNI plugin
+// under Kubernetes) hence provided errors should eventually appear in the caller's log files.
+func logOnStderr(err error) {
+	fmt.Fprintln(os.Stderr, "weave-cni:", err)
+}
+
 type NetConf struct {
 	types.NetConf
-	BrName string `json:"bridge"`
-	IsGW   bool   `json:"isGateway"`
-	IPMasq bool   `json:"ipMasq"`
-	MTU    int    `json:"mtu"`
+	BrName      string `json:"bridge"`
+	IsGW        bool   `json:"isGateway"`
+	IPMasq      bool   `json:"ipMasq"`
+	MTU         int    `json:"mtu"`
+	HairpinMode bool   `json:"hairpinMode"`
 }

prog/kube-peers/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"log"
+	"net"
 
 	"k8s.io/client-go/kubernetes"
 	api "k8s.io/client-go/pkg/api/v1"
@@ -35,10 +36,25 @@ func getKubePeers() ([]string, error) {
 	}
 	addresses := make([]string, 0, len(nodeList.Items))
 	for _, peer := range nodeList.Items {
+		var internalIP, externalIP string
 		for _, addr := range peer.Status.Addresses {
-			if addr.Type == "InternalIP" {
-				addresses = append(addresses, addr.Address)
+			// Check it's a valid ipv4 address
+			ip := net.ParseIP(addr.Address)
+			if ip == nil || ip.To4() == nil {
+				continue
 			}
+			if addr.Type == "InternalIP" {
+				internalIP = ip.To4().String()
+			} else if addr.Type == "ExternalIP" {
+				externalIP = ip.To4().String()
+			}
+		}
+
+		// Fallback for cases where a Node has an ExternalIP but no InternalIP
+		if internalIP != "" {
+			addresses = append(addresses, internalIP)
+		} else if externalIP != "" {
+			addresses = append(addresses, externalIP)
 		}
 	}
 	return addresses, nil

prog/plugin/Dockerfile.template

@@ -1,3 +1,3 @@
-FROM DOCKERHUB_USER/weave
+FROM DOCKERHUB_USER/weaveARCH_EXT
 ADD ./plugin /home/weave/
 ENTRYPOINT ["/home/weave/plugin"]

prog/plugin/main.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 	"syscall"
 
-	cni "github.com/appc/cni/pkg/skel"
 	"github.com/docker/libnetwork/ipamapi"
 	weaveapi "github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
@@ -29,8 +28,6 @@ var Log = common.Log
 func main() {
 	var (
 		justVersion      bool
-		cniNet           bool
-		cniIpam          bool
 		address          string
 		meshAddress      string
 		logLevel         string
@@ -38,8 +35,6 @@ func main() {
 	)
 
 	flag.BoolVar(&justVersion, "version", false, "print version and exit")
-	flag.BoolVar(&cniNet, "cni-net", false, "act as a CNI network plugin")
-	flag.BoolVar(&cniIpam, "cni-ipam", false, "act as a CNI IPAM plugin")
 	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
 	flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen")
 	flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode")
@@ -56,17 +51,6 @@ func main() {
 
 	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log)
 
-	switch {
-	case cniIpam || strings.HasSuffix(os.Args[0], "weave-ipam"):
-		i := ipamplugin.NewIpam(weave)
-		cni.PluginMain(i.CmdAdd, i.CmdDel)
-		os.Exit(0)
-	case cniNet || strings.HasSuffix(os.Args[0], "weave-net"):
-		n := netplugin.NewCNIPlugin(weave)
-		cni.PluginMain(n.CmdAdd, n.CmdDel)
-		os.Exit(0)
-	}
-
 	// API 1.21 is the first version that supports docker network commands
 	dockerClient, err := docker.NewVersionedClientFromEnv("1.21")
 	if err != nil {
@@ -110,7 +94,7 @@ func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddre
 	}
 	go serveStatus(statusListener)
 	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM)
+	signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
 
 	select {
 	case sig := <-sigChan:

prog/weave-kube/Dockerfile.template

@@ -1,3 +1,3 @@
-FROM DOCKERHUB_USER/weave
+FROM DOCKERHUB_USER/weaveARCH_EXT
 ADD ./launch.sh ./kube-peers /home/weave/
 ENTRYPOINT ["/home/weave/launch.sh"]

prog/weave-kube/launch.sh

@@ -2,10 +2,34 @@
 
 set -e
 
+modprobe_safe() {
+    modprobe $1 || echo "Ignore the error if \"$1\" is built-in in the kernel" >&2
+}
+
+# Check whether xt_set actually exists
+xt_set_exists() {
+    # Clean everything up in advance, in case there's leftovers
+    iptables -F WEAVE-KUBE-TEST 2>/dev/null || true
+    iptables -X WEAVE-KUBE-TEST 2>/dev/null || true
+    ipset destroy weave-kube-test 2>/dev/null || true
+
+    ipset create weave-kube-test hash:ip
+    iptables -t filter -N WEAVE-KUBE-TEST
+    if ! iptables -A WEAVE-KUBE-TEST -m set --match-set weave-kube-test src -j DROP; then
+        NOT_EXIST=1
+    fi
+    iptables -F WEAVE-KUBE-TEST
+    iptables -X WEAVE-KUBE-TEST
+    ipset destroy weave-kube-test
+    [ -z "$NOT_EXIST" ] || (echo "\"xt_set\" does not exist" >&2 && return 1)
+}
+
 # Default if not supplied - same as weave net default
 IPALLOC_RANGE=${IPALLOC_RANGE:-10.32.0.0/12}
 HTTP_ADDR=${WEAVE_HTTP_ADDR:-127.0.0.1:6784}
 STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
+HOST_ROOT=${HOST_ROOT:-/host}
+CONN_LIMIT=${CONN_LIMIT:-30}
 
 # Check if the IP range overlaps anything existing on the host
 /usr/bin/weaveutil netcheck $IPALLOC_RANGE weave
@@ -13,66 +37,29 @@ STATUS_ADDR=${WEAVE_STATUS_ADDR:-0.0.0.0:6782}
 # Default for network policy
 EXPECT_NPC=${EXPECT_NPC:-1}
 
+# Ensure we have the required modules for NPC
+if [ "${EXPECT_NPC}" != "0" ]; then
+    modprobe_safe br_netfilter
+    modprobe_safe xt_set
+    xt_set_exists
+fi
+
 # kube-proxy requires that bridged traffic passes through netfilter
-if [ ! -f /proc/sys/net/bridge/bridge-nf-call-iptables ] ; then
-    echo /proc/sys/net/bridge/bridge-nf-call-iptables not found >&2
-    exit 1
-fi
-
-echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
-
-SOURCE_BINARY=/usr/bin/weaveutil
-VERSION=$(/home/weave/weaver $EXTRA_ARGS --version | sed -e 's/weave router //')
-PLUGIN="weave-plugin-$VERSION"
-
-install_cni_plugin() {
-    mkdir -p $1 || return 1
-    if [ ! -f "$1/$PLUGIN" ]; then
-        cp "$SOURCE_BINARY" "$1/$PLUGIN"
-    fi
-}
-
-upgrade_cni_plugin_symlink() {
-    # Remove potential temporary symlink from previous failed upgrade:
-    rm -f $1/$2.tmp
-    # Atomically create a symlink to the plugin:
-    ln -s "$1/$PLUGIN" $1/$2.tmp && mv -f $1/$2.tmp $1/$2
-}
-
-upgrade_cni_plugin() {
-    # Check if weave-net and weave-ipam are (legacy) copies of the plugin, and
-    # if so remove these so symlinks can be used instead from now onwards.
-    if [ -f $1/weave-net  -a ! -L $1/weave-net  ];  then rm $1/weave-net;   fi
-    if [ -f $1/weave-ipam -a ! -L $1/weave-ipam ];  then rm $1/weave-ipam;  fi
-
-    # Create two symlinks to the plugin, as it has a different 
-    # behaviour depending on its name:
-    if [ "$(readlink -f $1/weave-net)" != "$1/$PLUGIN" ]; then
-        upgrade_cni_plugin_symlink $1 weave-net
-    fi
-    if [ "$(readlink -f $1/weave-ipam)" != "$1/$PLUGIN" ]; then
-        upgrade_cni_plugin_symlink $1 weave-ipam
-    fi
-}
-
-# Install CNI plugin binary to typical CNI bin location
-# with fall-back to CNI directory used by kube-up on GCI OS.
-if install_cni_plugin /opt/cni/bin ; then
-    upgrade_cni_plugin /opt/cni/bin
-elif install_cni_plugin /host_home/kubernetes/bin ; then
-    upgrade_cni_plugin /host_home/kubernetes/bin
+if ! BRIDGE_NF_ENABLED=$(cat /proc/sys/net/bridge/bridge-nf-call-iptables); then
+    echo "Cannot detect bridge-nf support - network policy and iptables mode kubeproxy may not work reliably" >&2
 else
-    echo "Failed to install the Weave CNI plugin" >&2
-    exit 1
+    if [ "$BRIDGE_NF_ENABLED" != "1" ]; then
+        echo 1 > /proc/sys/net/bridge/bridge-nf-call-iptables
+    fi
 fi
 
-# Need to create bridge before running weaver so we can use the peer address
-# (because of https://github.com/weaveworks/weave/issues/2480)
+# Explicitly create the bridge so we can pass --expect-npc
 WEAVE_NPC_OPTS="--expect-npc"
 if [ "${EXPECT_NPC}" = "0" ]; then
     WEAVE_NPC_OPTS=""
 fi
 
+export HOST_ROOT=/host
 /home/weave/weave --local create-bridge --force $WEAVE_NPC_OPTS
 
 # Kubernetes sets HOSTNAME to the host's hostname
@@ -109,53 +96,36 @@ if [ -z "$IPALLOC_INIT" ]; then
     IPALLOC_INIT="consensus=$(peer_count $KUBE_PEERS)"
 fi
 
+post_start_actions() {
+    # Wait for weave process to become responsive
+    while true ; do
+        curl $HTTP_ADDR/status >/dev/null 2>&1 && break
+        sleep 1
+    done
+
+    # Install CNI plugin binary to typical CNI bin location
+    # with fall-back to CNI directory used by kube-up on GCI OS
+    if ! mkdir -p $HOST_ROOT/opt/cni/bin ; then
+        if mkdir -p $HOST_ROOT/home/kubernetes/bin ; then
+            export WEAVE_CNI_PLUGIN_DIR=$HOST_ROOT/home/kubernetes/bin
+        else
+            echo "Failed to install the Weave CNI plugin" >&2
+            exit 1
+        fi
+    fi
+    mkdir -p $HOST_ROOT/etc/cni/net.d
+    /home/weave/weave --local setup-cni
+
+    # Expose the weave network so host processes can communicate with pods
+    /home/weave/weave --local expose $WEAVE_EXPOSE_IP
+}
+
+post_start_actions &
+
 /home/weave/weaver $EXTRA_ARGS --port=6783 $BRIDGE_OPTIONS \
      --http-addr=$HTTP_ADDR --status-addr=$STATUS_ADDR --docker-api='' --no-dns \
      --ipalloc-range=$IPALLOC_RANGE $NICKNAME_ARG \
      --ipalloc-init $IPALLOC_INIT \
-     --name=$(cat /sys/class/net/weave/address) "$@" \
-     $KUBE_PEERS &
-WEAVE_PID=$!
-
-# Wait for weave process to become responsive
-while true ; do
-    curl $HTTP_ADDR/status >/dev/null 2>&1 && break
-    if ! kill -0 $WEAVE_PID >/dev/null 2>&1 ; then
-        echo Weave process has died >&2
-        exit 1
-    fi
-    sleep 1
-done
-
-reclaim_ips() {
-    ID=$1
-    shift
-    for CIDR in "$@" ; do
-        curl -s -S -X PUT "$HTTP_ADDR/ip/$ID/$CIDR" || true
-    done
-}
-
-# Tell the newly-started weave about existing weave bridge IPs
-/usr/bin/weaveutil container-addrs weave weave:expose | while read ID IFACE MAC IPS; do
-    reclaim_ips "weave:expose" $IPS
-done
-# Tell weave about existing weave process IPs
-/usr/bin/weaveutil process-addrs weave | while read ID IFACE MAC IPS; do
-    reclaim_ips "_" $IPS
-done
-
-# Create CNI config, if not already there
-if [ ! -f /etc/cni/net.d/10-weave.conf ] ; then
-    mkdir -p /etc/cni/net.d
-    cat > /etc/cni/net.d/10-weave.conf <<EOF
-{
-    "name": "weave",
-    "type": "weave-net"
-}
-EOF
-fi
-
-# Expose the weave network so host processes can communicate with pods
-/home/weave/weave --local expose $WEAVE_EXPOSE_IP
-
-wait $WEAVE_PID
+     --conn-limit=$CONN_LIMIT \
+     "$@" \
+     $KUBE_PEERS

prog/weave-kube/weave-daemonset-k8s-1.6.yaml

@@ -0,0 +1,126 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: weave-net
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - namespaces
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - extensions
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: weave-net
+  namespace: kube-system
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: weave-net
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: weave-net
+subjects:
+- kind: ServiceAccount
+  name: weave-net
+  namespace: kube-system
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: weave-net
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        name: weave-net
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+        - name: weave
+          image: weaveworks/weave-kube:latest
+          imagePullPolicy: Always
+          command:
+            - /home/weave/launch.sh
+          livenessProbe:
+            initialDelaySeconds: 30
+            httpGet:
+              host: 127.0.0.1
+              path: /status
+              port: 6784
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: weavedb
+              mountPath: /weavedb
+            - name: cni-bin
+              mountPath: /host/opt
+            - name: cni-bin2
+              mountPath: /host/home
+            - name: cni-conf
+              mountPath: /host/etc
+            - name: dbus
+              mountPath: /host/var/lib/dbus
+            - name: lib-modules
+              mountPath: /lib/modules
+          resources:
+            requests:
+              cpu: 10m
+        - name: weave-npc
+          image: weaveworks/weave-npc:latest
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 10m
+          securityContext:
+            privileged: true
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
+      restartPolicy: Always
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      serviceAccountName: weave-net
+      securityContext:
+        seLinuxOptions:
+          type: spc_t
+      volumes:
+        - name: weavedb
+          emptyDir: {}
+        - name: cni-bin
+          hostPath:
+            path: /opt
+        - name: cni-bin2
+          hostPath:
+            path: /home
+        - name: cni-conf
+          hostPath:
+            path: /etc
+        - name: dbus
+          hostPath:
+            path: /var/lib/dbus
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules

prog/weave-kube/weave-daemonset.yaml

@@ -39,11 +39,15 @@ spec:
             - name: weavedb
               mountPath: /weavedb
             - name: cni-bin
-              mountPath: /opt
+              mountPath: /host/opt
             - name: cni-bin2
-              mountPath: /host_home
+              mountPath: /host/home
             - name: cni-conf
-              mountPath: /etc
+              mountPath: /host/etc
+            - name: dbus
+              mountPath: /host/var/lib/dbus
+            - name: lib-modules
+              mountPath: /lib/modules
           resources:
             requests:
               cpu: 10m
@@ -55,6 +59,12 @@ spec:
               cpu: 10m
           securityContext:
             privileged: true
+          env:
+          - name: HOSTNAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
       restartPolicy: Always
       volumes:
         - name: weavedb
@@ -68,3 +78,9 @@ spec:
         - name: cni-conf
           hostPath:
             path: /etc
+        - name: dbus
+          hostPath:
+            path: /var/lib/dbus
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules

prog/weave-npc/Dockerfile

@@ -1,10 +0,0 @@
-FROM alpine
-RUN apk add --update \
-    iptables \
-    ipset \
-	ulogd \
-  && rm -rf /var/cache/apk/* \
-  && mknod /var/log/ulogd.pcap p
-COPY ./weave-npc /usr/bin/weave-npc
-COPY ./ulogd.conf /etc/ulogd.conf
-ENTRYPOINT ["/usr/bin/weave-npc"]

prog/weave-npc/Dockerfile.template

@@ -0,0 +1,23 @@
+FROM ALPINE_BASEIMAGE
+
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
+
+LABEL maintainer "Weaveworks Inc <help@weave.works>"
+LABEL works.weave.role=system \
+      org.label-schema.name="Weave Net" \
+      org.label-schema.description="Weave Net creates a virtual network that connects Docker containers across multiple hosts and enables their automatic discovery" \
+      org.label-schema.url="https://weave.works" \
+      org.label-schema.vcs-url="https://github.com/weaveworks/weave" \
+      org.label-schema.vendor="Weaveworks"
+
+RUN apk add --update \
+    iptables \
+    ipset \
+	ulogd \
+  && rm -rf /var/cache/apk/* \
+  && mknod /var/log/ulogd.pcap p
+COPY ./weave-npc /usr/bin/weave-npc
+COPY ./ulogd.conf /etc/ulogd.conf
+ENTRYPOINT ["/usr/bin/weave-npc"]

prog/weave-npc/main.go

@@ -5,7 +5,6 @@ import (
 	"os/signal"
 	"syscall"
 
-	log "github.com/Sirupsen/logrus"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/kubernetes"
@@ -17,6 +16,7 @@ import (
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/npc"
 	"github.com/weaveworks/weave/npc/ipset"
 	"github.com/weaveworks/weave/npc/metrics"
@@ -26,13 +26,12 @@ import (
 var (
 	version     = "unreleased"
 	metricsAddr string
+	logLevel    string
+	allowMcast  bool
+	nodeName    string
 )
 
-func handleError(err error) {
-	if err != nil {
-		log.Fatal(err)
-	}
-}
+func handleError(err error) { common.CheckFatal(err) }
 
 func makeController(getter cache.Getter, resource string,
 	objType runtime.Object, handlers cache.ResourceEventHandlerFuncs) *cache.Controller {
@@ -55,12 +54,54 @@ func resetIPTables(ipt *iptables.IPTables) error {
 		return err
 	}
 
+	return nil
+}
+
+func resetIPSets(ips ipset.Interface) error {
+	// Remove ipsets prefixed `weave-` only
+
+	sets, err := ips.List(npc.IpsetNamePrefix)
+	if err != nil {
+		common.Log.Errorf("Failed to retrieve list of ipsets")
+		return err
+	}
+
+	common.Log.Debugf("Got list of ipsets: %v", sets)
+
+	// Must remove references to ipsets by other ipsets before they're destroyed
+	for _, s := range sets {
+		common.Log.Debugf("Flushing ipset '%s'", string(s))
+		if err := ips.Flush(s); err != nil {
+			common.Log.Errorf("Failed to flush ipset '%s'", string(s))
+			return err
+		}
+	}
+
+	for _, s := range sets {
+		common.Log.Debugf("Destroying ipset '%s'", string(s))
+		if err := ips.Destroy(s); err != nil {
+			common.Log.Errorf("Failed to destroy ipset '%s'", string(s))
+			return err
+		}
+	}
+
+	return nil
+}
+
+func createBaseRules(ipt *iptables.IPTables, ips ipset.Interface) error {
 	// Configure main chain static rules
 	if err := ipt.Append(npc.TableFilter, npc.MainChain,
 		"-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT"); err != nil {
 		return err
 	}
 
+	if allowMcast {
+		if err := ipt.Append(npc.TableFilter, npc.MainChain,
+			"-d", "224.0.0.0/4", "-j", "ACCEPT"); err != nil {
+			return err
+		}
+	}
+
 	if err := ipt.Append(npc.TableFilter, npc.MainChain,
 		"-m", "state", "--state", "NEW", "-j", string(npc.DefaultChain)); err != nil {
 		return err
@@ -71,17 +112,13 @@ func resetIPTables(ipt *iptables.IPTables) error {
 		return err
 	}
 
-	return nil
-}
-
-func resetIPSets(ips ipset.Interface) error {
-	// TODO should restrict ipset operations to the `weave-` prefix:
-
-	if err := ips.FlushAll(); err != nil {
+	// If the destination address is not any of the local pods, let it through
+	if err := ips.Create(npc.LocalIpset, ipset.HashIP); err != nil {
 		return err
 	}
-
-	if err := ips.DestroyAll(); err != nil {
+	if err := ipt.Append(npc.TableFilter, npc.MainChain,
+		"-m", "set", "--match-set", npc.LocalIpset, "src",
+		"-m", "set", "!", "--match-set", npc.LocalIpset, "dst", "-j", "ACCEPT"); err != nil {
 		return err
 	}
 
@@ -89,37 +126,40 @@ func resetIPSets(ips ipset.Interface) error {
 }
 
 func root(cmd *cobra.Command, args []string) {
-	log.Infof("Starting Weaveworks NPC %s", version)
+	common.SetLogLevel(logLevel)
+	if nodeName == "" {
+		// HOSTNAME is set by Kubernetes for pods in the host network namespace
+		nodeName = os.Getenv("HOSTNAME")
+	}
+	if nodeName == "" {
+		common.Log.Fatalf("Must set node name via --node-name or $HOSTNAME")
+	}
+	common.Log.Infof("Starting Weaveworks NPC %s; node name %q", version, nodeName)
 
 	if err := metrics.Start(metricsAddr); err != nil {
-		log.Fatalf("Failed to start metrics: %v", err)
+		common.Log.Fatalf("Failed to start metrics: %v", err)
 	}
 
 	if err := ulogd.Start(); err != nil {
-		log.Fatalf("Failed to start ulogd: %v", err)
+		common.Log.Fatalf("Failed to start ulogd: %v", err)
 	}
 
 	config, err := rest.InClusterConfig()
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleError(err)
 
 	client, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleError(err)
 
 	ipt, err := iptables.New()
-	if err != nil {
-		log.Fatal(err)
-	}
+	handleError(err)
 
-	ips := ipset.New()
+	ips := ipset.New(common.LogLogger())
 
 	handleError(resetIPTables(ipt))
 	handleError(resetIPSets(ips))
+	handleError(createBaseRules(ipt, ips))
 
-	npc := npc.New(ipt, ips)
+	npc := npc.New(nodeName, ipt, ips)
 
 	nsController := makeController(client.Core().RESTClient(), "namespaces", &coreapi.Namespace{},
 		cache.ResourceEventHandlerFuncs{
@@ -127,7 +167,15 @@ func root(cmd *cobra.Command, args []string) {
 				handleError(npc.AddNamespace(obj.(*coreapi.Namespace)))
 			},
 			DeleteFunc: func(obj interface{}) {
-				handleError(npc.DeleteNamespace(obj.(*coreapi.Namespace)))
+				switch obj := obj.(type) {
+				case *coreapi.Namespace:
+					handleError(npc.DeleteNamespace(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeleteNamespace(obj.Obj.(*coreapi.Namespace)))
+				}
 			},
 			UpdateFunc: func(old, new interface{}) {
 				handleError(npc.UpdateNamespace(old.(*coreapi.Namespace), new.(*coreapi.Namespace)))
@@ -139,7 +187,15 @@ func root(cmd *cobra.Command, args []string) {
 				handleError(npc.AddPod(obj.(*coreapi.Pod)))
 			},
 			DeleteFunc: func(obj interface{}) {
-				handleError(npc.DeletePod(obj.(*coreapi.Pod)))
+				switch obj := obj.(type) {
+				case *coreapi.Pod:
+					handleError(npc.DeletePod(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeletePod(obj.Obj.(*coreapi.Pod)))
+				}
 			},
 			UpdateFunc: func(old, new interface{}) {
 				handleError(npc.UpdatePod(old.(*coreapi.Pod), new.(*coreapi.Pod)))
@@ -151,7 +207,15 @@ func root(cmd *cobra.Command, args []string) {
 				handleError(npc.AddNetworkPolicy(obj.(*extnapi.NetworkPolicy)))
 			},
 			DeleteFunc: func(obj interface{}) {
-				handleError(npc.DeleteNetworkPolicy(obj.(*extnapi.NetworkPolicy)))
+				switch obj := obj.(type) {
+				case *extnapi.NetworkPolicy:
+					handleError(npc.DeleteNetworkPolicy(obj))
+				case cache.DeletedFinalStateUnknown:
+					// We know this object has gone away, but its final state is no longer
+					// available from the API server. Instead we use the last copy of it
+					// that we have, which is good enough for our cleanup.
+					handleError(npc.DeleteNetworkPolicy(obj.Obj.(*extnapi.NetworkPolicy)))
+				}
 			},
 			UpdateFunc: func(old, new interface{}) {
 				handleError(npc.UpdateNetworkPolicy(old.(*extnapi.NetworkPolicy), new.(*extnapi.NetworkPolicy)))
@@ -163,7 +227,7 @@ func root(cmd *cobra.Command, args []string) {
 
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
-	log.Fatalf("Exiting: %v", <-signals)
+	common.Log.Fatalf("Exiting: %v", <-signals)
 }
 
 func main() {
@@ -173,8 +237,9 @@ func main() {
 		Run:   root}
 
 	rootCmd.PersistentFlags().StringVar(&metricsAddr, "metrics-addr", ":6781", "metrics server bind address")
+	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "debug", "logging level (debug, info, warning, error)")
+	rootCmd.PersistentFlags().BoolVar(&allowMcast, "allow-mcast", true, "allow all multicast traffic")
+	rootCmd.PersistentFlags().StringVar(&nodeName, "node-name", "", "only generate rules that apply to this node")
 
-	if err := rootCmd.Execute(); err != nil {
-		log.Fatal(err)
-	}
+	handleError(rootCmd.Execute())
 }

prog/weavedb/Dockerfile

@@ -1,6 +1,6 @@
 # This is a nearly-empty image that we use to create a data-only container for persistence
 FROM scratch
-MAINTAINER Weaveworks Inc <help@weave.works>
+LABEL maintainer "Weaveworks Inc <help@weave.works>"
 ENTRYPOINT ["data-only"]
 # Work round Docker refusing to save an empty image
 COPY Dockerfile /

prog/weaveexec/Dockerfile

@@ -1,24 +0,0 @@
-FROM alpine
-
-MAINTAINER Weaveworks Inc <help@weave.works>
-LABEL works.weave.role=system
-
-WORKDIR /home/weave
-ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
-
-RUN apk add --update \
-    curl \
-    ethtool \
-    iptables \
-    iproute2 \
-    util-linux \
-    conntrack-tools \
-    bind-tools \
-  && rm -rf /var/cache/apk/*
-
-ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/
-ADD ./weaveutil /usr/bin/
-ADD ./weavewait /w/w
-ADD ./weavewait_noop /w-noop/w
-ADD ./weavewait_nomcast /w-nomcast/w
-ADD ./docker.tgz /

prog/weaveexec/Dockerfile.template

@@ -0,0 +1,34 @@
+FROM ALPINE_BASEIMAGE
+
+# If we're building for another architecture than amd64, the CROSS_BUILD_ placeholder is removed so e.g. CROSS_BUILD_COPY turns into COPY
+# If we're building normally, for amd64, CROSS_BUILD lines are removed
+CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/
+
+LABEL maintainer "Weaveworks Inc <help@weave.works>"
+LABEL works.weave.role=system \
+      org.label-schema.name="Weave Net" \
+      org.label-schema.description="Weave Net creates a virtual network that connects Docker containers across multiple hosts and enables their automatic discovery" \
+      org.label-schema.url="https://weave.works" \
+      org.label-schema.vcs-url="https://github.com/weaveworks/weave" \
+      org.label-schema.vendor="Weaveworks"
+
+WORKDIR /home/weave
+ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
+
+RUN apk add --update \
+    curl \
+    ethtool \
+    iptables \
+    ipset \
+    iproute2 \
+    util-linux \
+    conntrack-tools \
+    bind-tools \
+  && rm -rf /var/cache/apk/*
+
+ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/
+ADD ./weaveutil /usr/bin/
+ADD ./weavewait /w/w
+ADD ./weavewait_noop /w-noop/w
+ADD ./weavewait_nomcast /w-nomcast/w
+ADD ./docker /usr/bin/

prog/weaveproxy/main.go

@@ -5,9 +5,10 @@ import (
 	"os"
 	"strings"
 
-	"github.com/weaveworks/docker/pkg/mflag"
+	"github.com/weaveworks/common/mflag"
+	"github.com/weaveworks/common/mflagext"
+	"github.com/weaveworks/common/signals"
 	"github.com/weaveworks/weave/common"
-	"github.com/weaveworks/weave/common/mflagext"
 	"github.com/weaveworks/weave/proxy"
 )
 
@@ -79,5 +80,5 @@ func main() {
 	p.AttachExistingContainers()
 	go p.Serve(listeners)
 	go p.ListenAndServeStatus("/home/weave/status.sock")
-	common.SignalHandlerLoop()
+	signals.SignalHandlerLoop(common.Log)
 }

prog/weaver/Dockerfile.template

@@ -1,6 +1,4 @@
-FROM DOCKERHUB_USER/weaveexec
-MAINTAINER Weaveworks Inc <help@weave.works>
-LABEL works.weave.role=system
+FROM DOCKERHUB_USER/weaveexecARCH_EXT
 WORKDIR /home/weave
 ADD ./weaver /home/weave/
 ADD weavedata.db /weavedb/

prog/weaver/checkpoint.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/weaveworks/go-checkpoint"
+	weave "github.com/weaveworks/weave/router"
 )
 
 var checker *checkpoint.Checker
@@ -16,7 +17,7 @@ const (
 	updateCheckPeriod = 6 * time.Hour
 )
 
-func checkForUpdates(dockerVersion string, network string) {
+func checkForUpdates(dockerVersion string, router *weave.NetworkRouter) {
 	newVersion.Store("")
 	success.Store(true)
 
@@ -51,9 +52,6 @@ func checkForUpdates(dockerVersion string, network string) {
 		"docker-version": dockerVersion,
 		"kernel-version": kernelVersion,
 	}
-	if network != "" {
-		flags["network"] = network
-	}
 
 	// Start background version checking
 	params := checkpoint.CheckParams{
@@ -61,6 +59,21 @@ func checkForUpdates(dockerVersion string, network string) {
 		Version:       version,
 		SignatureFile: "",
 		Flags:         flags,
+		ExtraFlags:    func() []checkpoint.Flag { return checkpointFlags(router) },
 	}
 	checker = checkpoint.CheckInterval(&params, updateCheckPeriod, handleResponse)
 }
+
+func checkpointFlags(router *weave.NetworkRouter) []checkpoint.Flag {
+	flags := []checkpoint.Flag{}
+	status := weave.NewNetworkRouterStatus(router)
+	for _, conn := range status.Connections {
+		if connectionName, ok := conn.Attrs["name"].(string); ok {
+			if _, encrypted := conn.Attrs["encrypted"]; encrypted {
+				connectionName = connectionName + " encrypted"
+			}
+			flags = append(flags, checkpoint.Flag{Key: "network", Value: connectionName})
+		}
+	}
+	return flags
+}

prog/weaver/http.go

@@ -211,7 +211,7 @@ var targetsTemplate = defTemplate("targetsTemplate", `\
 
 var connectionsTemplate = defTemplate("connectionsTemplate", `\
 {{range .Router.Connections}}\
-{{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} {{printf "%-11v" .State}} {{.Info}}
+{{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} {{printf "%-11v" .State}} {{.Info}} {{range $key,$element := .Attrs}}{{if ne $key "name"}}{{$key}}={{$element}}{{end}}{{end}}
 {{end}}\
 `)
 

prog/weaver/main.go

@@ -13,7 +13,8 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/pkg/profile"
-	"github.com/weaveworks/docker/pkg/mflag"
+	"github.com/weaveworks/common/mflag"
+	"github.com/weaveworks/common/signals"
 	"github.com/weaveworks/mesh"
 
 	"github.com/weaveworks/weave/common"
@@ -223,16 +224,19 @@ func main() {
 		networkConfig.PacketLogging = nopPacketLogging{}
 	}
 
-	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB)
+	config.Password = determinePassword(password)
+
+	overlay, bridge := createOverlay(datapathName, ifaceName, isAWSVPC, config.Host, config.Port, bufSzMB, config.Password != nil)
 	networkConfig.Bridge = bridge
 
 	if bridge != nil {
 		if err := weavenet.DetectHairpin("vethwe-bridge", Log); err != nil {
-			Log.Errorf("DetectHairpin failed: %s", err)
+			Log.Errorf("Setting may cause connectivity issues : %s", err)
+			Log.Infof("Hairpin mode may have been enabled by other software on this machine")
 		}
 	}
 
-	name := peerName(routerName, bridge.Interface())
+	name := peerName(routerName)
 
 	if nickName == "" {
 		var err error
@@ -240,7 +244,6 @@ func main() {
 		checkFatal(err)
 	}
 
-	config.Password = determinePassword(password)
 	config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr)
 	config.PeerDiscovery = !noDiscovery
 
@@ -248,11 +251,12 @@ func main() {
 		Log.Fatalf("--awsvpc mode is not compatible with the --password option")
 	}
 
-	db, err := db.NewBoltDB(dbPrefix + "data.db")
+	db, err := db.NewBoltDB(dbPrefix + db.FileName)
 	checkFatal(err)
 	defer db.Close()
 
-	router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
+	router, err := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
+	checkFatal(err)
 	Log.Println("Our name is", router.Ourself)
 
 	if peers, err = router.InitialPeers(resume, peers); err != nil {
@@ -272,11 +276,7 @@ func main() {
 		dockerVersion = dockerCli.DockerVersion()
 	}
 
-	network := ""
-	if isAWSVPC {
-		network = "awsvpc"
-	}
-	checkForUpdates(dockerVersion, network)
+	checkForUpdates(dockerVersion, router)
 
 	observeContainers := func(o docker.ContainerObserver) {
 		if dockerCli != nil {
@@ -304,13 +304,16 @@ func main() {
 			}
 			trackerName = "awsvpc"
 		}
-		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, t, isKnownPeer)
-		observeContainers(allocator)
+		var allContainerIDs []string
 		if dockerCli != nil {
-			ids, err := dockerCli.AllContainerIDs()
+			allContainerIDs, err = dockerCli.AllContainerIDs()
 			checkFatal(err)
-			allocator.PruneOwned(ids)
 		}
+		preClaims, err := findExistingAddresses(dockerCli, allContainerIDs, weavenet.WeaveBridgeName)
+		checkFatal(err)
+		allocator, defaultSubnet = createAllocator(router, ipamConfig, preClaims, db, t, isKnownPeer)
+		observeContainers(allocator)
+		allocator.PruneOwned(allContainerIDs)
 	}
 
 	var (
@@ -361,7 +364,7 @@ func main() {
 		go listenAndServeHTTP(statusAddr, statusMux)
 	}
 
-	common.SignalHandlerLoop(router)
+	signals.SignalHandlerLoop(common.Log, router)
 }
 
 func options() map[string]string {
@@ -404,7 +407,7 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
 func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
-func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) {
+func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host string, port int, bufSzMB int, enableEncryption bool) (weave.NetworkOverlay, weave.Bridge) {
 	overlay := weave.NewOverlaySwitch()
 	var bridge weave.Bridge
 	var ignoreSleeve bool
@@ -421,7 +424,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st
 	case datapathName != "":
 		iface, err := weavenet.EnsureInterface(datapathName)
 		checkFatal(err)
-		fastdp, err := weave.NewFastDatapath(iface, port)
+		fastdp, err := weave.NewFastDatapath(iface, port, enableEncryption)
 		checkFatal(err)
 		bridge = fastdp.Bridge()
 		overlay.Add("fastdp", fastdp.Overlay())
@@ -443,7 +446,7 @@ func createOverlay(datapathName string, ifaceName string, isAWSVPC bool, host st
 	return overlay, bridge
 }
 
-func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
+func createAllocator(router *weave.NetworkRouter, config ipamConfig, preClaims []ipam.PreClaim, db db.DB, track tracker.LocalRangeTracker, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
 	ipRange, err := ipam.ParseCIDRSubnet(config.IPRangeCIDR)
 	checkFatal(err)
 	defaultSubnet := ipRange
@@ -462,6 +465,7 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, t
 		Seed:        config.SeedPeerNames,
 		Universe:    ipRange,
 		IsObserver:  config.Observer,
+		PreClaims:   preClaims,
 		Quorum:      func() uint { return determineQuorum(config.PeerCount, router) },
 		Db:          db,
 		IsKnownPeer: isKnownPeer,
@@ -470,7 +474,9 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, t
 
 	allocator := ipam.NewAllocator(c)
 
-	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
+	gossip, err := router.NewGossip("IPallocation", allocator)
+	checkFatal(err)
+	allocator.SetInterfaces(gossip)
 	allocator.Start()
 	router.Peers.OnGC(func(peer *mesh.Peer) { allocator.PeerGone(peer.Name) })
 
@@ -480,7 +486,9 @@ func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, t
 func createDNSServer(config dnsConfig, router *mesh.Router, isKnownPeer func(mesh.PeerName) bool) (*nameserver.Nameserver, *nameserver.DNSServer) {
 	ns := nameserver.New(router.Ourself.Peer.Name, config.Domain, isKnownPeer)
 	router.Peers.OnGC(func(peer *mesh.Peer) { ns.PeerGone(peer.Name) })
-	ns.SetGossip(router.NewGossip("nameserver", ns))
+	gossip, err := router.NewGossip("nameserver", ns)
+	checkFatal(err)
+	ns.SetGossip(gossip)
 	upstream := nameserver.NewUpstream(config.ResolvConf, config.EffectiveListenAddress)
 	dnsserver, err := nameserver.NewDNSServer(ns, config.Domain, config.ListenAddress,
 		upstream, uint32(config.TTL), config.ClientTimeout)
@@ -528,10 +536,11 @@ func determinePassword(password string) []byte {
 	return []byte(password)
 }
 
-func peerName(routerName string, iface *net.Interface) mesh.PeerName {
+func peerName(routerName string) mesh.PeerName {
 	if routerName == "" {
-		if iface == nil {
-			Log.Fatal("Either an interface must be specified with --datapath or --iface, or a name with --name")
+		iface, err := net.InterfaceByName(weavenet.WeaveBridgeName)
+		if err != nil {
+			Log.Fatalf("Unable to find bridge %q", weavenet.WeaveBridgeName)
 		}
 		routerName = iface.HardwareAddr.String()
 	}

prog/weaver/metrics.go

@@ -43,7 +43,7 @@ func uint64Counter(desc *prometheus.Desc, val uint64, labels ...string) promethe
 	return prometheus.MustNewConstMetric(desc, prometheus.CounterValue, float64(val), labels...)
 }
 
-var metrics []metric = []metric{
+var metrics = []metric{
 	{desc("weave_connections", "Number of peer-to-peer connections.", "state"),
 		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
 			counts := make(map[string]int)
@@ -82,6 +82,18 @@ var metrics []metric = []metric{
 				ch <- intGauge(desc, metrics.Flows)
 			}
 		}},
+	{desc("weave_ipam_pending_allocates", "Number of pending allocates."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, len(s.IPAM.PendingAllocates))
+			}
+		}},
+	{desc("weave_ipam_pending_claims", "Number of pending claims."),
+		func(s WeaveStatus, desc *prometheus.Desc, ch chan<- prometheus.Metric) {
+			if s.IPAM != nil {
+				ch <- intGauge(desc, len(s.IPAM.PendingClaims))
+			}
+		}},
 }
 
 func fastDPMetrics(s WeaveStatus) *weave.FastDPMetrics {

prog/weaver/reclaim.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"net"
+
+	docker "github.com/fsouza/go-dockerclient"
+
+	"github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	weavedocker "github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/ipam"
+	weavenet "github.com/weaveworks/weave/net"
+	"github.com/weaveworks/weave/net/address"
+)
+
+func a(cidr *net.IPNet) address.CIDR {
+	prefixLength, _ := cidr.Mask.Size()
+	return address.CIDR{Addr: address.FromIP4(cidr.IP), PrefixLen: prefixLength}
+}
+
+// Get all the existing Weave IPs at startup, so we can stop IPAM
+// giving out any as duplicates
+func findExistingAddresses(dockerCli *weavedocker.Client, containerIDs []string, bridgeName string) (addrs []ipam.PreClaim, err error) {
+	Log.Infof("Checking for pre-existing addresses on %s bridge", bridgeName)
+	// First get the address for the bridge
+	bridgeNetDev, err := weavenet.GetBridgeNetDev(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+	for _, cidr := range bridgeNetDev.CIDRs {
+		Log.Infof("%s bridge has address %v", bridgeName, cidr)
+		addrs = append(addrs, ipam.PreClaim{Ident: "weave:expose", Cidr: a(cidr)})
+	}
+
+	add := func(cid string, isContainer bool, netDevs []weavenet.Dev) {
+		for _, netDev := range netDevs {
+			for _, cidr := range netDev.CIDRs {
+				Log.Infof("Found address %v for ID %s", cidr, cid)
+				addrs = append(addrs, ipam.PreClaim{Ident: cid, IsContainer: isContainer, Cidr: a(cidr)})
+			}
+		}
+	}
+
+	// Then find all veths connected to the bridge
+	peerIDs, err := weavenet.ConnectedToBridgeVethPeerIds(bridgeName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Now iterate over all containers to see if they have a network
+	// namespace with an attached interface
+	if dockerCli != nil {
+		for _, cid := range containerIDs {
+			container, err := dockerCli.InspectContainer(cid)
+			if err != nil {
+				if _, ok := err.(*docker.NoSuchContainer); ok {
+					continue
+				}
+				return nil, err
+			}
+			if container.State.Pid != 0 {
+				netDevs, err := weavenet.GetNetDevsByVethPeerIds(container.State.Pid, peerIDs)
+				if err != nil {
+					return nil, err
+				}
+				add(cid, true, netDevs)
+			}
+		}
+	} else {
+		// If we don't have a Docker connection, iterate over all processes
+		pids, err := common.AllPids("/proc")
+		if err != nil {
+			return nil, err
+		}
+		for _, pid := range pids {
+			netDevs, err := weavenet.GetNetDevsByVethPeerIds(pid, peerIDs)
+			if err != nil {
+				return nil, err
+			}
+			add(api.NoContainerID, false, netDevs)
+		}
+	}
+	return addrs, nil
+}

prog/weaveutil/addrs.go

@@ -77,3 +77,16 @@ func printNetDevs(cid string, netDevs []weavenet.Dev) {
 		fmt.Println()
 	}
 }
+
+func bridgeIP(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("bridge-ip", "<bridgeName>")
+	}
+	bridgeName := args[0]
+	ip, err := weavenet.FindBridgeIP(bridgeName, nil)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%v", ip)
+	return nil
+}

prog/weaveutil/attach.go

@@ -14,11 +14,12 @@ import (
 
 func attach(args []string) error {
 	if len(args) < 4 {
-		cmdUsage("attach-container", "[--no-multicast-route] [--keep-tx-on] <container-id> <bridge-name> <mtu> <cidr>...")
+		cmdUsage("attach-container", "[--no-multicast-route] [--keep-tx-on] [--hairpin-mode=true|false] <container-id> <bridge-name> <mtu> <cidr>...")
 	}
 
 	keepTXOn := false
 	withMulticastRoute := true
+	hairpinMode := true
 	for i := 0; i < len(args); {
 		switch args[i] {
 		case "--no-multicast-route":
@@ -27,6 +28,9 @@ func attach(args []string) error {
 		case "--keep-tx-on":
 			keepTXOn = true
 			args = append(args[:i], args[i+1:]...)
+		case "--hairpin-mode=false":
+			hairpinMode = false
+			args = append(args[:i], args[i+1:]...)
 		default:
 			i++
 		}
@@ -44,7 +48,7 @@ func attach(args []string) error {
 	if nsHost, err := netns.GetFromPid(1); err != nil {
 		return fmt.Errorf("unable to open host namespace: %s", err)
 	} else if nsHost.Equal(nsContainer) {
-		return fmt.Errorf("Container is running in the host network namespace, and therefore cannot be\nconnected to weave. Perhaps the container was started with --net=host.")
+		return fmt.Errorf("Container is running in the host network namespace, and therefore cannot be\nconnected to weave - perhaps the container was started with --net=host")
 	}
 	mtu, err := strconv.Atoi(args[2])
 	if err != nil && args[3] != "" {
@@ -55,7 +59,7 @@ func attach(args []string) error {
 		return err
 	}
 
-	err = weavenet.AttachContainer(weavenet.NSPathByPid(pid), fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn)
+	err = weavenet.AttachContainer(weavenet.NSPathByPid(pid), fmt.Sprint(pid), weavenet.VethName, args[1], mtu, withMulticastRoute, cidrs, keepTXOn, hairpinMode)
 	// If we detected an error but the container has died, tell the user that instead.
 	if err != nil && !processExists(pid) {
 		err = fmt.Errorf("Container %s died", args[0])

prog/weaveutil/cni.go

@@ -3,7 +3,8 @@ package main
 import (
 	"os"
 
-	cni "github.com/appc/cni/pkg/skel"
+	cni "github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/version"
 
 	weaveapi "github.com/weaveworks/weave/api"
 	"github.com/weaveworks/weave/common"
@@ -14,13 +15,13 @@ import (
 func cniNet(args []string) error {
 	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
 	n := netplugin.NewCNIPlugin(weave)
-	cni.PluginMain(n.CmdAdd, n.CmdDel)
+	cni.PluginMain(n.CmdAdd, n.CmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0"))
 	return nil
 }
 
 func cniIPAM(args []string) error {
 	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), common.Log)
 	i := ipamplugin.NewIpam(weave)
-	cni.PluginMain(i.CmdAdd, i.CmdDel)
+	cni.PluginMain(i.CmdAdd, i.CmdDel, version.PluginSupports("0.1.0", "0.2.0", "0.3.0"))
 	return nil
 }

prog/weaveutil/docker_tls_args.go

@@ -45,7 +45,6 @@ func dockerTLSArgs(args []string) error {
 			switch {
 			case arg == "-d" || arg == "daemon":
 				isDaemon = true
-				break
 			case arg == "--tls", arg == "--tlsverify":
 				tlsArgs = append(tlsArgs, arg)
 			case strings.HasPrefix(arg, "--tls"):

prog/weaveutil/main.go

@@ -35,6 +35,8 @@ func init() {
 		"cni-net":                cniNet,
 		"cni-ipam":               cniIPAM,
 		"expose-nat":             exposeNAT,
+		"bridge-ip":              bridgeIP,
+		"unique-id":              uniqueID,
 	}
 }
 

prog/weaveutil/unique_id.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func uniqueID(args []string) error {
+	if len(args) != 2 {
+		cmdUsage("unique-id", "<db-prefix> <host-root>")
+	}
+	dbPrefix := args[0]
+	hostRoot := args[1]
+	uid, err := weavenet.GetSystemPeerName(dbPrefix, hostRoot)
+	if err != nil {
+		return err
+	}
+	fmt.Print(uid)
+	return nil
+}

proxy/malformed_host_override.go

@@ -0,0 +1,122 @@
+// (C) Docker, Inc.
+// Redistributed under the Apache License 2.0
+
+package proxy
+
+import "net"
+
+// MalformedHostHeaderOverride is a wrapper to be able
+// to overcome the 400 Bad request coming from old docker
+// clients that send an invalid Host header.
+type MalformedHostHeaderOverride struct {
+	net.Listener
+}
+
+// MalformedHostHeaderOverrideConn wraps the underlying unix
+// connection and keeps track of the first read from http.Server
+// which just reads the headers.
+type MalformedHostHeaderOverrideConn struct {
+	net.Conn
+	first bool
+}
+
+var closeConnHeader = []byte("\r\nConnection: close\r")
+
+// Read reads the first *read* request from http.Server to inspect
+// the Host header. If the Host starts with / then we're talking to
+// an old docker client which send an invalid Host header. To not
+// error out in http.Server we rewrite the first bytes of the request
+// to sanitize the Host header itself.
+// In case we're not dealing with old docker clients the data is just passed
+// to the server w/o modification.
+func (l *MalformedHostHeaderOverrideConn) Read(b []byte) (n int, err error) {
+	// http.Server uses a 4k buffer
+	if l.first && len(b) == 4096 {
+		// This keeps track of the first read from http.Server which just reads
+		// the headers
+		l.first = false
+		// The first read of the connection by http.Server is done limited to
+		// DefaultMaxHeaderBytes (usually 1 << 20) + 4096.
+		// Here we do the first read which gets us all the http headers to
+		// be inspected and modified below.
+		c, err := l.Conn.Read(b)
+		if err != nil {
+			return c, err
+		}
+
+		var (
+			start, end    int
+			firstLineFeed = -1
+			buf           []byte
+		)
+		for i := 0; i <= c-1-7; i++ {
+			if b[i] == '\n' && firstLineFeed == -1 {
+				firstLineFeed = i
+			}
+			if b[i] != '\n' {
+				continue
+			}
+
+			if b[i+1] == '\r' && b[i+2] == '\n' {
+				return c, nil
+			}
+
+			if b[i+1] != 'H' {
+				continue
+			}
+			if b[i+2] != 'o' {
+				continue
+			}
+			if b[i+3] != 's' {
+				continue
+			}
+			if b[i+4] != 't' {
+				continue
+			}
+			if b[i+5] != ':' {
+				continue
+			}
+			if b[i+6] != ' ' {
+				continue
+			}
+			if b[i+7] != '/' {
+				continue
+			}
+			// ensure clients other than the docker clients do not get this hack
+			if i != firstLineFeed {
+				return c, nil
+			}
+			start = i + 7
+			// now find where the value ends
+			for ii, bbb := range b[start:c] {
+				if bbb == '\n' {
+					end = start + ii
+					break
+				}
+			}
+			buf = make([]byte, 0, c+len(closeConnHeader)-(end-start))
+			// strip the value of the host header and
+			// inject `Connection: close` to ensure we don't reuse this connection
+			buf = append(buf, b[:start]...)
+			buf = append(buf, closeConnHeader...)
+			buf = append(buf, b[end:c]...)
+			copy(b, buf)
+			break
+		}
+		if len(buf) == 0 {
+			return c, nil
+		}
+		return len(buf), nil
+	}
+	return l.Conn.Read(b)
+}
+
+// Accept makes the listener accepts connections and wraps the connection
+// in a MalformedHostHeaderOverrideConn initilizing first to true.
+func (l *MalformedHostHeaderOverride) Accept() (net.Conn, error) {
+	c, err := l.Listener.Accept()
+	if err != nil {
+		return c, err
+	}
+	return &MalformedHostHeaderOverrideConn{c, true}, nil
+}

proxy/proxy.go

@@ -148,14 +148,12 @@ func NewProxy(c Config) (*Proxy, error) {
 	p.client = client.Client
 
 	if !p.WithoutDNS {
-		netDev, err := weavenet.GetBridgeNetDev(c.DockerBridge)
+		ip, err := weavenet.FindBridgeIP(c.DockerBridge, nil)
 		if err != nil {
 			return nil, err
 		}
-		if len(netDev.CIDRs) != 1 {
-			return nil, fmt.Errorf("Could not obtain address of %s", c.DockerBridge)
-		}
-		p.dockerBridgeIP = netDev.CIDRs[0].IP.String()
+		p.dockerBridgeIP = ip.String()
+		Log.Infof("Using docker bridge IP for DNS: %v", p.dockerBridgeIP)
 	}
 
 	p.hostnameMatchRegexp, err = regexp.Compile(c.HostnameMatch)
@@ -378,7 +376,7 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 		Log.Fatalf("Invalid protocol format: %q", proto)
 	}
 
-	return listener, fmt.Sprintf("%s://%s", proto, addr), nil
+	return &MalformedHostHeaderOverride{listener}, fmt.Sprintf("%s://%s", proto, addr), nil
 }
 
 // weavedocker.ContainerObserver interface

router/awsvpc.go

@@ -32,8 +32,8 @@ func (conn *AWSVPCConnection) Stop() {}
 func (conn *AWSVPCConnection) ControlMessage(tag byte, msg []byte) {
 }
 
-func (conn *AWSVPCConnection) DisplayName() string {
-	return "awsvpc"
+func (conn *AWSVPCConnection) Attrs() map[string]interface{} {
+	return map[string]interface{}{"name": "awsvpc"}
 }
 
 // OverlayForwarder
@@ -64,6 +64,8 @@ func (vpc AWSVPC) Diagnostics() interface{} {
 	return nil
 }
 
+func (vpc AWSVPC) Stop() {}
+
 // NetworkOverlay
 
 func (vpc AWSVPC) InvalidateRoutes() {}

router/fastdp.go

@@ -9,9 +9,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/vishvananda/netlink"
 	"github.com/weaveworks/go-odp/odp"
 	"github.com/weaveworks/mesh"
+
+	"github.com/weaveworks/weave/net/ipsec"
 )
 
 // The virtual bridge accepts packets from ODP vports and the router
@@ -39,6 +42,7 @@ type FastDatapath struct {
 	localPeer        *mesh.Peer
 	peers            *mesh.Peers
 	overlayConsumer  OverlayConsumer
+	ipsec            *ipsec.IPSec
 
 	// Bridge state: How to send to the given bridge port
 	sendToPort map[bridgePortID]bridgeSender
@@ -53,6 +57,7 @@ type FastDatapath struct {
 	vxlanUDPPorts    map[int]odp.VportID
 	vxlanVportIDs    map[odp.VportID]struct{}
 	mainVxlanVportID odp.VportID
+	mainVxlanUDPPort int
 
 	// A singleton pool for the occasions when we need to decode
 	// the packet.
@@ -62,7 +67,9 @@ type FastDatapath struct {
 	forwarders map[mesh.PeerName]*fastDatapathForwarder
 }
 
-func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
+func NewFastDatapath(iface *net.Interface, port int, encryptionEnabled bool) (*FastDatapath, error) {
+	var ipSec *ipsec.IPSec
+
 	dpif, err := odp.NewDpif()
 	if err != nil {
 		return nil, err
@@ -80,11 +87,22 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 		return nil, err
 	}
 
+	if encryptionEnabled {
+		var err error
+		if ipSec, err = ipsec.New(log); err != nil {
+			return nil, errors.Wrap(err, "ipsec new")
+		}
+		if err := ipSec.Flush(false); err != nil {
+			return nil, errors.Wrap(err, "ipsec flush")
+		}
+	}
+
 	fastdp := &FastDatapath{
 		iface:         iface,
 		dpif:          dpif,
 		dp:            dp,
 		missHandlers:  make(map[odp.VportID]missHandler),
+		ipsec:         ipSec,
 		sendToPort:    nil,
 		sendToMAC:     make(map[MAC]bridgeSender),
 		seenMACs:      make(map[MAC]struct{}),
@@ -113,7 +131,8 @@ func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 	// numbers to be independent, but working out how to specify
 	// them on the connecting side.  So we can wait to find out if
 	// anyone wants that.
-	fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(port+1, 5, time.Millisecond*10)
+	fastdp.mainVxlanUDPPort = port + 1
+	fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(fastdp.mainVxlanUDPPort, 5, time.Millisecond*10)
 	if err != nil {
 		return nil, err
 	}
@@ -315,6 +334,14 @@ func (fastdp fastDatapathOverlay) InvalidateShortIDs() {
 	checkWarn(fastdp.deleteFlows())
 }
 
+func (fastdp fastDatapathOverlay) Stop() {
+	if fastdp.ipsec != nil {
+		if err := fastdp.ipsec.Flush(true); err != nil {
+			log.Errorf("ipsec flush failed: %s", err)
+		}
+	}
+}
+
 func (fastDatapathOverlay) AddFeaturesTo(features map[string]string) {
 	// Nothing needed.  Fast datapath support is indicated through
 	// OverlaySwitch.
@@ -542,12 +569,16 @@ type fastDatapathForwarder struct {
 	connUID        uint64
 	vxlanVportID   odp.VportID
 
+	sessionKey                 *[32]byte
+	isEncrypted                bool
+	isOutboundIPSecEstablished bool
+
 	lock              sync.RWMutex
 	confirmed         bool
 	remoteAddr        *net.UDPAddr
 	heartbeatInterval time.Duration
-	heartbeatTimer    *time.Timer
-	heartbeatTimeout  *time.Timer
+	heartbeatTimer    *time.Timer // for sending
+	heartbeatTimeout  *time.Timer // for receiving
 	ackedHeartbeat    bool
 	stopChan          chan struct{}
 	stopped           bool
@@ -557,27 +588,25 @@ type fastDatapathForwarder struct {
 }
 
 func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
-	if params.SessionKey != nil {
-		// No encryption support in fastdp
-		return nil, fmt.Errorf("encryption not supported")
-	}
-
 	vxlanVportID := fastdp.mainVxlanVportID
-	var remoteAddr *net.UDPAddr
+	vxlanUDPPort := fastdp.mainVxlanUDPPort
 
+	remoteAddr := makeUDPAddr(params.RemoteAddr)
 	if params.Outbound {
-		remoteAddr = makeUDPAddr(params.RemoteAddr)
+		var err error
 		// The provided address contains the main weave port
 		// number to connect to.  We need to derive the vxlan
 		// port number from that.
 		vxlanRemoteAddr := *remoteAddr
 		vxlanRemoteAddr.Port++
 		remoteAddr = &vxlanRemoteAddr
-		var err error
-		vxlanVportID, err = fastdp.getVxlanVportID(remoteAddr.Port)
+		vxlanUDPPort = remoteAddr.Port
+		vxlanVportID, err = fastdp.getVxlanVportID(vxlanUDPPort)
 		if err != nil {
 			return nil, err
 		}
+	} else {
+		remoteAddr.Port = vxlanUDPPort
 	}
 
 	localIP, err := ipv4Bytes(params.LocalAddr.IP)
@@ -592,6 +621,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio
 		sendControlMsg: params.SendControlMessage,
 		connUID:        params.ConnUID,
 		vxlanVportID:   vxlanVportID,
+		sessionKey:     params.SessionKey,
 
 		remoteAddr:        remoteAddr,
 		heartbeatInterval: FastHeartbeat,
@@ -601,7 +631,7 @@ func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectio
 		errorChan:       make(chan error, 1),
 	}
 
-	return fwd, err
+	return fwd, nil
 }
 
 func ipv4Bytes(ip net.IP) (res [4]byte, err error) {
@@ -626,11 +656,30 @@ func (fwd *fastDatapathForwarder) Confirm() {
 		log.Fatal(fwd.logPrefix(), "already confirmed")
 	}
 
+	if fwd.fastdp.ipsec != nil && fwd.sessionKey != nil {
+		fwd.isEncrypted = true
+		log.Info("Setting up IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer)
+		err := fwd.fastdp.ipsec.InitSALocal(
+			fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
+			net.IP(fwd.localIP[:]), fwd.remoteAddr.IP,
+			fwd.remoteAddr.Port,
+			fwd.sessionKey,
+			func(msg []byte) error {
+				return fwd.sendControlMsg(FastDatapathCryptoInitSARemote, msg)
+			},
+		)
+		if err != nil {
+			log.Error(fwd.logPrefix(), "ipsec init SA local failed: ", err)
+			fwd.handleError(err)
+			return
+		}
+	}
+
 	log.Debug(fwd.logPrefix(), "confirmed")
 	fwd.fastdp.addForwarder(fwd.remotePeer.Name, fwd)
 	fwd.confirmed = true
 
-	if fwd.remoteAddr != nil {
+	if fwd.remoteAddr != nil && (!fwd.isEncrypted || fwd.isOutboundIPSecEstablished) {
 		// have the goroutine send a heartbeat straight away
 		fwd.heartbeatTimer = time.NewTimer(0)
 	} else {
@@ -719,6 +768,7 @@ func (fwd *fastDatapathForwarder) sendHeartbeat() {
 
 const (
 	FastDatapathHeartbeatAck = iota
+	FastDatapathCryptoInitSARemote
 )
 
 func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte, sender *net.UDPAddr) {
@@ -768,14 +818,16 @@ func (fwd *fastDatapathForwarder) ControlMessage(tag byte, msg []byte) {
 	switch tag {
 	case FastDatapathHeartbeatAck:
 		fwd.handleHeartbeatAck()
+	case FastDatapathCryptoInitSARemote:
+		fwd.handleCryptoInitSARemote(msg)
 
 	default:
 		log.Info(fwd.logPrefix(), "Ignoring unknown control message: ", tag)
 	}
 }
 
-func (fwd *fastDatapathForwarder) DisplayName() string {
-	return "fastdp"
+func (fwd *fastDatapathForwarder) Attrs() map[string]interface{} {
+	return map[string]interface{}{"name": "fastdp", "mtu": fwd.fastdp.iface.MTU}
 }
 
 func (fwd *fastDatapathForwarder) handleHeartbeatAck() {
@@ -790,6 +842,28 @@ func (fwd *fastDatapathForwarder) handleHeartbeatAck() {
 	}
 }
 
+func (fwd *fastDatapathForwarder) handleCryptoInitSARemote(msg []byte) {
+	log.Info(fwd.logPrefix(), "IPSec init SA remote")
+	err := fwd.fastdp.ipsec.InitSARemote(
+		msg,
+		fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
+		net.IP(fwd.localIP[:]), fwd.remoteAddr.IP, fwd.remoteAddr.Port,
+		fwd.sessionKey,
+	)
+	if err != nil {
+		log.Warning(fwd.logPrefix(), "IPSec init SA remote failed: ", err)
+		fwd.handleError(err)
+		return
+	}
+
+	// FastDatapathCryptoInitSARemote can be received before Confirm'ing
+	// connection, thus before InitSALocal.
+	if fwd.confirmed && !fwd.isOutboundIPSecEstablished {
+		fwd.isOutboundIPSecEstablished = true
+		fwd.heartbeatTimer.Reset(0)
+	}
+}
+
 func (fwd *fastDatapathForwarder) Forward(key ForwardPacketKey) FlowOp {
 	if !key.SrcPeer.HasShortID || !key.DstPeer.HasShortID {
 		return nil
@@ -839,6 +913,18 @@ func (fwd *fastDatapathForwarder) Stop() {
 	defer fwd.lock.Unlock()
 	fwd.sendControlMsg = func(byte, []byte) error { return nil }
 
+	if fwd.isEncrypted {
+		localIP := net.IP(fwd.localIP[:])
+		log.Info("Destroying IPsec between ", fwd.fastdp.localPeer, " and ", fwd.remotePeer)
+		err := fwd.fastdp.ipsec.Destroy(
+			fwd.fastdp.localPeer.Name, fwd.remotePeer.Name, fwd.connUID,
+			localIP, fwd.remoteAddr.IP, fwd.remoteAddr.Port,
+		)
+		if err != nil {
+			log.Errorf("ipsec destroy failed: %s", err)
+		}
+	}
+
 	// stop the heartbeat goroutine
 	if !fwd.stopped {
 		fwd.stopped = true
@@ -1138,7 +1224,7 @@ func (fastdp *FastDatapath) send(fops FlowOp, frame []byte, lock *fastDatapathLo
 	}
 
 	if fastdp.isHairpinFlow(&flow) {
-		log.Error("Vetoed installation of hairpin flow ", flow)
+		log.Warning("Vetoed installation of hairpin flow ", flow)
 		return
 	}
 

router/network_router.go

@@ -51,7 +51,7 @@ type NetworkRouter struct {
 	db   db.DB
 }
 
-func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) *NetworkRouter {
+func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh.PeerName, nickName string, overlay NetworkOverlay, db db.DB) (*NetworkRouter, error) {
 	if overlay == nil {
 		overlay = NullNetworkOverlay{}
 	}
@@ -59,7 +59,11 @@ func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh
 		networkConfig.Bridge = NullBridge{}
 	}
 
-	router := &NetworkRouter{Router: mesh.NewRouter(config, name, nickName, overlay, common.LogLogger()), NetworkConfig: networkConfig, db: db}
+	meshRouter, err := mesh.NewRouter(config, name, nickName, overlay, common.LogLogger())
+	if err != nil {
+		return nil, err
+	}
+	router := &NetworkRouter{Router: meshRouter, NetworkConfig: networkConfig, db: db}
 	router.Peers.OnInvalidateShortIDs(overlay.InvalidateShortIDs)
 	router.Routes.OnChange(overlay.InvalidateRoutes)
 	router.Macs = NewMacCache(macMaxAge,
@@ -67,7 +71,7 @@ func NewNetworkRouter(config mesh.Config, networkConfig NetworkConfig, name mesh
 			log.Println("Expired MAC", mac, "at", peer)
 		})
 	router.Peers.OnGC(func(peer *mesh.Peer) { router.Macs.Delete(peer) })
-	return router
+	return router, nil
 }
 
 // Start listening for TCP connections, locally captured packets, and

router/overlay_switch.go

@@ -50,6 +50,12 @@ func (osw *OverlaySwitch) Diagnostics() interface{} {
 	return diagnostics
 }
 
+func (osw *OverlaySwitch) Stop() {
+	for _, overlay := range osw.overlays {
+		overlay.Stop()
+	}
+}
+
 func (osw *OverlaySwitch) InvalidateRoutes() {
 	for _, overlay := range osw.overlays {
 		overlay.InvalidateRoutes()
@@ -424,7 +430,7 @@ func (fwd *overlaySwitchForwarder) ControlMessage(tag byte, msg []byte) {
 	}
 }
 
-func (fwd *overlaySwitchForwarder) DisplayName() string {
+func (fwd *overlaySwitchForwarder) Attrs() map[string]interface{} {
 	var best OverlayForwarder
 
 	fwd.lock.Lock()
@@ -434,8 +440,8 @@ func (fwd *overlaySwitchForwarder) DisplayName() string {
 	fwd.lock.Unlock()
 
 	if best != nil {
-		return best.DisplayName()
+		return best.Attrs()
 	}
 
-	return "none"
+	return nil
 }