Merge pull request #2188 from weaveworks/doc-improvements

Minor improvements to docs

.gitignore

@@ -25,6 +25,7 @@ _testmain.go
 *.prof
 profile.cov
 coverage.html
+.pkg
 
 # Emacs backup files
 *~
@@ -40,25 +41,25 @@ prog/weaveproxy/weaveproxy
 prog/sigproxy/sigproxy
 prog/weavewait/weavewait
 prog/weavewait/weavewait_noop
-prog/netcheck/netcheck
-prog/docker_tls_args/docker_tls_args
+prog/weavewait/weavewait_nomcast
+prog/weaveutil/weaveutil
+prog/plugin/plugin
 testing/cover/cover
 testing/runner/runner
 releases
-.weaver.uptodate
-.weavedns.uptodate
-.weaveexec.uptodate
-weave.tar
+.*.uptodate
+weave.tar.gz
 prog/weaveexec/weave
 prog/weaveexec/weavehosts
 prog/weaveexec/weaveproxy
 prog/weaveexec/weavewait
 prog/weaveexec/weavewait_noop
+prog/weaveexec/weavewait_nomcast
 prog/weaveexec/sigproxy
-prog/weaveexec/netcheck
-prog/weaveexec/docker_tls_args
+prog/weaveexec/weaveutil
 prog/weaveexec/docker*.tgz
 Vagrantfile.local
+test/tls/tls
 test/tls/*.pem
 test/coverage
 test/coverage.*

.gitmodules

@@ -1,3 +1,72 @@
 [submodule "tools"]
 	path = tools
 	url = https://github.com/weaveworks/build-tools
+[submodule "vendor/github.com/vishvananda/netlink"]
+	path = vendor/github.com/vishvananda/netlink
+	url = https://github.com/vishvananda/netlink.git
+[submodule "vendor/github.com/davecheney/profile"]
+	path = vendor/github.com/davecheney/profile
+	url = https://github.com/davecheney/profile.git
+[submodule "vendor/github.com/gorilla/mux"]
+	path = vendor/github.com/gorilla/mux
+	url = https://github.com/gorilla/mux.git
+[submodule "vendor/github.com/gorilla/context"]
+	path = vendor/github.com/gorilla/context
+	url = https://github.com/gorilla/context.git
+[submodule "vendor/github.com/google/gopacket"]
+	path = vendor/github.com/google/gopacket
+	url = https://github.com/google/gopacket.git
+[submodule "vendor/github.com/weaveworks/go-odp"]
+	path = vendor/github.com/weaveworks/go-odp
+	url = https://github.com/weaveworks/go-odp.git
+[submodule "vendor/github.com/opencontainers/runc"]
+	path = vendor/github.com/opencontainers/runc
+	url = https://github.com/opencontainers/runc.git
+[submodule "vendor/github.com/fsouza/go-dockerclient"]
+	path = vendor/github.com/fsouza/go-dockerclient
+	url = https://github.com/fsouza/go-dockerclient.git
+[submodule "vendor/github.com/docker/libnetwork"]
+	path = vendor/github.com/docker/libnetwork
+	url = https://github.com/docker/libnetwork.git
+[submodule "vendor/github.com/docker/docker"]
+	path = vendor/github.com/docker/docker
+	url = https://github.com/docker/docker.git
+[submodule "vendor/github.com/Sirupsen/logrus"]
+	path = vendor/github.com/Sirupsen/logrus
+	url = https://github.com/Sirupsen/logrus.git
+[submodule "vendor/github.com/andybalholm/go-bit"]
+	path = vendor/github.com/andybalholm/go-bit
+	url = https://github.com/andybalholm/go-bit.git
+[submodule "vendor/github.com/miekg/dns"]
+	path = vendor/github.com/miekg/dns
+	url = https://github.com/miekg/dns.git
+[submodule "vendor/github.com/stretchr/testify"]
+	path = vendor/github.com/stretchr/testify
+	url = https://github.com/stretchr/testify.git
+[submodule "vendor/golang.org/x/crypto"]
+	path = vendor/golang.org/x/crypto
+	url = https://github.com/golang/crypto.git
+[submodule "vendor/github.com/docker/machine"]
+	path = vendor/github.com/docker/machine
+	url = https://github.com/docker/machine.git
+[submodule "vendor/github.com/weaveworks/mesh"]
+	path = vendor/github.com/weaveworks/mesh
+	url = https://github.com/weaveworks/mesh
+[submodule "vendor/github.com/vishvananda/netns"]
+	path = vendor/github.com/vishvananda/netns
+	url = https://github.com/vishvananda/netns.git
+[submodule "vendor/github.com/boltdb/bolt"]
+	path = vendor/github.com/boltdb/bolt
+	url = https://github.com/boltdb/bolt
+[submodule "vendor/github.com/weaveworks/go-checkpoint"]
+	path = vendor/github.com/weaveworks/go-checkpoint
+	url = https://github.com/weaveworks/go-checkpoint
+[submodule "vendor/github.com/hashicorp/go-cleanhttp"]
+	path = vendor/github.com/hashicorp/go-cleanhttp
+	url = http://github.com/hashicorp/go-cleanhttp
+[submodule "vendor/github.com/appc/cni"]
+	path = vendor/github.com/appc/cni
+	url = https://github.com/appc/cni
+[submodule "vendor/github.com/coreos/go-iptables"]
+	path = vendor/github.com/coreos/go-iptables
+	url = https://github.com/coreos/go-iptables

.travis.yml

@@ -1,28 +0,0 @@
-language: go
-sudo: false
-go:
-  - 1.4
-
-env:
-  - PATH="${PATH}:${HOME}/.local/bin"
-
-addons:
-  apt:
-    packages:
-    - libpcap-dev
-
-install:
-  - go clean -i net
-  - go install -tags netgo std
-  - go get github.com/golang/lint/golint
-  - pip install --user --upgrade gsutil
-  - bin/setup-ci-secrets $encrypted_5ba036e89377_key $encrypted_5ba036e89377_iv
-  - make travis
-  - go get ./...
-
-script:
-  - make tests
-
-after_success:
-  - go get github.com/mattn/goveralls
-  - test "${TRAVIS_PULL_REQUEST}" = "false" && site/build-and-publish.sh

CHANGELOG.md

@@ -1,3 +1,162 @@
+## Release 1.5.0
+
+## Highlights
+
+- A new [Container Network Interface](https://github.com/appc/cni#cni---the-container-network-interface) plugin.
+- This release is much more robust against unscheduled restarts,
+  because it persists key data to disk.
+- New configuration options that are useful when you create or
+  auto-scale larger networks.
+- Weave now periodically checks for updates (can be disabled)
+
+Plus many bug fixes and minor enhancements. More details below and in
+the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.5.0).
+
+## Release 1.4.6
+
+Fixes some issues encountered by our users.
+
+* Restarting a peer could leave stale entries in WeaveDNS
+  #1867/#2023/#2081
+* Weave proxy occasionally failing to attach any new containers
+  #2016/#2049
+
+Other fixes:
+
+* Resolved a crash when a restarting peer re-connected to a peer that
+  had not received the latest IPAM data #2083/#2092
+* Make IP address space available immediately after a dead peer is
+  removed #2068
+
+## Release 1.4.5
+
+Higher performance for multicast and broadcast traffic when using
+Weave's Fast Datapath
+
+* The flow rule to deliver broadcast and multicast packets in-kernel
+  was not created correctly, hence every such packet caused a
+  context-switch to the software router #2003/#2008
+
+Other fixes:
+
+* Remove DNS entries for containers that are being restarted by Docker
+  but are not live yet #1977/#1989
+* Don't let one failing allocation attempt prevent others from
+  succeeding; they could be in different subnets which are more
+  available #1996/#2001
+* Don't complain on second router launch on kernels that lack support
+  for Fast Datapath #1929/#1983
+* Fix build broken by change in libnetwork IPAM API #1984/#1985
+
+## Release 1.4.4
+
+Fixing a rather serious issue that slipped through our preparations
+for Docker 1.10:
+
+* Restarting Docker or rebooting your machine while the Weave Net
+  plugin is running causes Docker 1.10.0 to fail to start #1959/#1963
+
+Also one other small fix:
+
+* Avoid a hang when trying to use plugin and proxy at the same time
+  #1956/#1961
+
+## Release 1.4.3
+
+Preparing for Docker 1.10, plus some bug-fixes.
+
+* Avoid hang in Docker v1.10 on `docker volume ls` after `weave stop`,
+  `weave stop-plugin` or `weave reset` #1934/#1936
+* Fix "unexpected EOF" from Docker 1.10 on `docker exec` with Weave
+  proxy `--rewrite-inspect` enabled #1911/#1917
+* Avoid losing DNS entries and potentially double-allocating IP
+  Addresses allocated via plugin, on router restart; also extend
+  `weave ps` to show IP addresses allocated via plugin #1745/#1921
+* Stop creating lots of copies of `weavewait` program in Docker
+  volumes #1757/#1935
+* Prevent container starting prematurely when proxy in
+`--no-multicast-route` mode #1942/#1943
+* Log error message from plugin rather than crashing when weave not
+  running #1906/#1918
+* Warn, don't error, if unable to remove plugin network in 'weave
+  stop', to avoid breaking the usual upgrade or config change process
+  #1900/#1919
+* Don't crash if network conditions suggest only very small packets
+  will get through #1905/#1926
+* Cope with unexpected errors during route traversal when starting
+  container via proxy #1909/#1910/#1932
+
+## Release 1.4.2
+
+Bug-fixes and minor improvements.
+
+* A race condition in weavewait that would occasionally hang
+  containers at startup #1882/#1884
+* Having the plugin auto-restart prevents successful `weave launch` on
+  reboot #1869
+* Work round weave router failure on CoreOS 4.3 caused by kernel bug
+  #1854
+* `weave launch` would exit with error code on docker <1.9 #1851
+* Running `eval $(weave env)` multiple times would break `eval $(weave
+  env --restore)` #1824/#1825
+* Don't complain in `weave stop` about "Plugin is not running" when
+  plugin is not enabled #1840/#1841
+* `weave --local launch` would fail if utility program
+  `docker_tls_args` could not be found #1844
+* Improved error reporting when TLS arg detection fails #1843
+* Improve error reporting when docker isn't running #1845
+* Add `--trusted-subnets` usage to `weave` script #1842
+* weave run can hang under rare combinations of options #1858
+
+## Release 1.4.1
+
+This is a bug-fix release to cover a few issues that came up since the
+release of 1.4.0.
+
+* Weave would fail to launch when `$DOCKER_HOST` was set to a TCP socket secured with TLS #1820/#1822
+* Weave would fail to stop when run against Docker pre-1.9 #1815/#1817
+* Issue a warning instead of failing on `--with-dns` option to proxy, which was removed #1810/#1812
+* Make `weave version` show the plugin version #1797/#1813
+* Make `weave launch` show when the a container is restarting #1778/#1814
+* Make `weave launch` fail if the plugin is running, for consistency with router and proxy. #1818/#1819
+
+More details in the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.4.1).
+
+## Release 1.4.0
+
+**Highlights**
+
+* The Docker Network plugin can now operate without a cluster store, so it is now run by default.
+* You can now use the fast datapath over trusted links and Weave encryption over untrusted links, in the same network.
+
+Plus many bug fixes and minor enhancements. More details below and in
+the [change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.4.0).
+
+## Release 1.3.1
+
+**Highlights**
+
+* The minimum Docker version has been increased to 1.6 due to the
+  [upcoming deprecation of Dockerhub access](https://blog.docker.com/2015/10/docker-hub-deprecation-1-5/)
+  for old clients. From December 7th onwards previous versions of the
+  `weave` script will fail to pull down images from the hub; if you
+  are unable to upgrade to 1.3.1 immediately you can work around this
+  by running `weave --local setup` in conjunction with a compatible
+  Docker client installation
+* Docker networking plugin now works with older kernels and allows you
+  to configure the MTU
+
+More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.3.1).
+
+## Release 1.3.0
+
+**Highlights**
+
+This release includes a Docker Plugin, so you have the option to use
+Weave Net that way. More details in the
+[change log](https://github.com/weaveworks/weave/issues?q=milestone%3A1.3.0).
+
 ## Release 1.2.1
 
 This release contains a number of bug fixes and minor

LICENSE

@@ -176,7 +176,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2014-2015 Weaveworks Ltd.
+   Copyright 2014-2016 Weaveworks Ltd.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,10 +1,14 @@
-PUBLISH=publish_weave publish_weaveexec
+PUBLISH=publish_weave publish_weaveexec publish_plugin
 
 .DEFAULT: all
-.PHONY: all update tests lint publish $(PUBLISH) clean clean-bin prerequisites build travis run-smoketests
+.PHONY: all exes testrunner update tests lint publish $(PUBLISH) clean clean-bin prerequisites build run-smoketests
 
 # If you can use docker without being root, you can do "make SUDO="
-SUDO=sudo
+SUDO=sudo -E
+BUILD_IN_CONTAINER=true
+RM=--rm
+RUN_FLAGS=-ti
+COVERAGE=
 
 DOCKERHUB_USER=weaveworks
 WEAVE_VERSION=git-$(shell git rev-parse --short=12 HEAD)
@@ -14,25 +18,33 @@ WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy
 SIGPROXY_EXE=prog/sigproxy/sigproxy
 WEAVEWAIT_EXE=prog/weavewait/weavewait
 WEAVEWAIT_NOOP_EXE=prog/weavewait/weavewait_noop
-NETCHECK_EXE=prog/netcheck/netcheck
-DOCKERTLSARGS_EXE=prog/docker_tls_args/docker_tls_args
-RUNNER_EXE=testing/runner/runner
+WEAVEWAIT_NOMCAST_EXE=prog/weavewait/weavewait_nomcast
+WEAVEUTIL_EXE=prog/weaveutil/weaveutil
+PLUGIN_EXE=prog/plugin/plugin
+RUNNER_EXE=tools/runner/runner
+TEST_TLS_EXE=test/tls/tls
 
-EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE) $(RUNNER_EXE)
+EXES=$(WEAVER_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE) $(PLUGIN_EXE) $(TEST_TLS_EXE)
 
+BUILD_UPTODATE=.build.uptodate
 WEAVER_UPTODATE=.weaver.uptodate
 WEAVEEXEC_UPTODATE=.weaveexec.uptodate
+PLUGIN_UPTODATE=.plugin.uptodate
+WEAVEDB_UPTODATE=.weavedb.uptodate
 
-IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE)
+IMAGES_UPTODATE=$(WEAVER_UPTODATE) $(WEAVEEXEC_UPTODATE) $(PLUGIN_UPTODATE) $(BUILD_UPTODATE) $(WEAVEDB_UPTODATE)
 
 WEAVER_IMAGE=$(DOCKERHUB_USER)/weave
 WEAVEEXEC_IMAGE=$(DOCKERHUB_USER)/weaveexec
+PLUGIN_IMAGE=$(DOCKERHUB_USER)/plugin
+BUILD_IMAGE=$(DOCKERHUB_USER)/weavebuild
+WEAVEDB_IMAGE=$(DOCKERHUB_USER)/weavedb
 
-IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE)
+IMAGES=$(WEAVER_IMAGE) $(WEAVEEXEC_IMAGE) $(PLUGIN_IMAGE) $(WEAVEDB_IMAGE)
 
-WEAVE_EXPORT=weave.tar
+WEAVE_EXPORT=weave.tar.gz
 
-WEAVEEXEC_DOCKER_VERSION=1.3.1
+WEAVEEXEC_DOCKER_VERSION=1.6.2
 DOCKER_DISTRIB=prog/weaveexec/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(WEAVEEXEC_DOCKER_VERSION).tgz
 NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
@@ -43,96 +55,128 @@ NETGO_CHECK=@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	echo "    sudo go install -tags netgo std"; \
 	false; \
 }
-BUILD_FLAGS=-ldflags "-extldflags \"-static\" -X main.version $(WEAVE_VERSION)" -tags netgo
+BUILD_FLAGS=-i -ldflags "-extldflags \"-static\" -X main.version=$(WEAVE_VERSION)" -tags netgo
 
 PACKAGE_BASE=$(shell go list -e ./)
 
-all: $(WEAVE_EXPORT) $(RUNNER_EXE)
+all: $(WEAVE_EXPORT)
+exes: $(EXES)
+testrunner: $(RUNNER_EXE) $(TEST_TLS_EXE)
 
-travis: $(EXES)
+$(EXES): $(BUILD_UPTODATE)
+$(WEAVER_EXE) $(WEAVEPROXY_EXE) $(WEAVEUTIL_EXE): common/*.go common/*/*.go net/*.go net/*/*.go
+$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go db/*.go nameserver/*.go prog/weaver/*.go
+$(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/*.go
+$(WEAVEUTIL_EXE): prog/weaveutil/*.go
+$(SIGPROXY_EXE): prog/sigproxy/*.go
+$(PLUGIN_EXE): prog/plugin/*.go plugin/*/*.go api/*.go common/*.go common/docker/*.go
+$(TEST_TLS_EXE): test/tls/*.go
+$(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
+$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
+$(WEAVEWAIT_NOMCAST_EXE): prog/weavewait/*.go net/*.go
+tests: $(BUILD_UPTODATE) tools/.git
+lint: $(BUILD_UPTODATE) tools/.git
 
-update:
-	go get -u -f -v -tags netgo $(addprefix ./,$(dir $(EXES)))
+ifeq ($(BUILD_IN_CONTAINER),true)
 
-$(WEAVER_EXE) $(WEAVEPROXY_EXE): common/*.go common/*/*.go net/*.go
+$(EXES) tests lint:
+	git submodule update --init
+	@mkdir -p $(shell pwd)/.pkg
+	$(SUDO) docker run $(RM) $(RUN_FLAGS) \
+	    -v $(shell pwd):/go/src/github.com/weaveworks/weave \
+		-v $(shell pwd)/.pkg:/go/pkg \
+		-e GOARCH -e GOOS -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX -e COVERDIR -e SLOW \
+		$(BUILD_IMAGE) COVERAGE=$(COVERAGE) WEAVE_VERSION=$(WEAVE_VERSION) $@
+
+else
+
+$(WEAVER_EXE) $(WEAVEPROXY_EXE) $(PLUGIN_EXE):
 ifeq ($(COVERAGE),true)
-	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | paste -s -d,))
-	go get -t -tags netgo ./$(@D)
+	go build -v $(BUILD_FLAGS) -o $@ ./$(@D)
+	$(eval COVERAGE_MODULES := $(shell (go list ./$(@D); go list -f '{{join .Deps "\n"}}' ./$(@D) | grep "^$(PACKAGE_BASE)/") | grep -v "^$(PACKAGE_BASE)/vendor/" | paste -s -d,))
 	go test -c -o ./$@ $(BUILD_FLAGS) -v -covermode=atomic -coverpkg $(COVERAGE_MODULES) ./$(@D)/
 else
-	go get -tags netgo ./$(@D)
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 endif
 	$(NETGO_CHECK)
 
-$(NETCHECK_EXE): common/*.go common/*/*.go net/*.go
-	go get -tags netgo ./$(@D)
+$(WEAVEUTIL_EXE):
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
-$(WEAVER_EXE): router/*.go ipam/*.go ipam/*/*.go nameserver/*.go prog/weaver/*.go
-$(WEAVEPROXY_EXE): proxy/*.go prog/weaveproxy/main.go
-$(NETCHECK_EXE): prog/netcheck/netcheck.go
+$(WEAVEWAIT_EXE):
+	go build $(BUILD_FLAGS) -tags "netgo iface mcast" -o $@ ./$(@D)
+	$(NETGO_CHECK)
 
-# Sigproxy and weavewait need separate rules as they fail the netgo check in
+$(WEAVEWAIT_NOMCAST_EXE):
+	go build $(BUILD_FLAGS) -tags "netgo iface" -o $@ ./$(@D)
+	$(NETGO_CHECK)
+
+# These programs need a separate rule as they fail the netgo check in
 # the main build stanza due to not importing net package
-$(SIGPROXY_EXE): prog/sigproxy/main.go
-$(WEAVEWAIT_EXE): prog/weavewait/*.go net/*.go
-$(RUNNER_EXE): testing/runner/runner.go
-$(DOCKERTLSARGS_EXE): prog/docker_tls_args/*.go
-
-$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(RUNNER_EXE) $(DOCKERTLSARGS_EXE):
-	go get -tags netgo ./$(@D)
+$(SIGPROXY_EXE) $(TEST_TLS_EXE) $(WEAVEWAIT_NOOP_EXE):
 	go build $(BUILD_FLAGS) -o $@ ./$(@D)
 
-$(WEAVEWAIT_NOOP_EXE): prog/weavewait/*.go
-	go get -tags netgo ./$(@D)
-	go build $(BUILD_FLAGS) -tags noop -o $@ ./$(@D)
+tests:
+	./tools/test -no-go-get
 
-$(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
-	$(SUDO) docker build -t $(WEAVER_IMAGE) prog/weaver
+lint:
+	./tools/lint -nocomment -notestpackage .
+
+endif
+
+$(BUILD_UPTODATE): build/*
+	$(SUDO) docker build -t $(BUILD_IMAGE) build/
 	touch $@
 
-$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(NETCHECK_EXE) $(DOCKERTLSARGS_EXE)
+$(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVER_IMAGE) prog/weaver
+	touch $@
+
+$(WEAVEEXEC_UPTODATE): prog/weaveexec/Dockerfile prog/weaveexec/symlink $(DOCKER_DISTRIB) weave $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(WEAVEWAIT_NOOP_EXE) $(WEAVEWAIT_NOMCAST_EXE) $(WEAVEUTIL_EXE)
 	cp weave prog/weaveexec/weave
 	cp $(SIGPROXY_EXE) prog/weaveexec/sigproxy
 	cp $(WEAVEPROXY_EXE) prog/weaveexec/weaveproxy
 	cp $(WEAVEWAIT_EXE) prog/weaveexec/weavewait
 	cp $(WEAVEWAIT_NOOP_EXE) prog/weaveexec/weavewait_noop
-	cp $(NETCHECK_EXE) prog/weaveexec/netcheck
-	cp $(DOCKERTLSARGS_EXE) prog/weaveexec/docker_tls_args
+	cp $(WEAVEWAIT_NOMCAST_EXE) prog/weaveexec/weavewait_nomcast
+	cp $(WEAVEUTIL_EXE) prog/weaveexec/weaveutil
 	cp $(DOCKER_DISTRIB) prog/weaveexec/docker.tgz
-	$(SUDO) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker build -t $(WEAVEEXEC_IMAGE) prog/weaveexec
+	touch $@
+
+$(PLUGIN_UPTODATE): prog/plugin/Dockerfile $(PLUGIN_EXE) $(WEAVEEXEC_UPTODATE)
+	$(SUDO) docker build -t $(PLUGIN_IMAGE) prog/plugin
+	touch $@
+
+$(WEAVEDB_UPTODATE): prog/weavedb/Dockerfile
+	$(SUDO) docker build -t $(WEAVEDB_IMAGE) prog/weavedb
 	touch $@
 
 $(WEAVE_EXPORT): $(IMAGES_UPTODATE)
-	$(SUDO) docker save $(addsuffix :latest,$(IMAGES)) > $@
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker save $(addsuffix :latest,$(IMAGES)) | gzip > $@
 
 $(DOCKER_DISTRIB):
 	curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL)
 
-tests: tools/.git
-	tools/test
-
-lint: tools/.git
-	tools/lint -nocomment -notestpackage .
-
 tools/.git:
 	git submodule update --init
 
+$(RUNNER_EXE): tools/.git
+	make -C tools/runner
+
 $(PUBLISH): publish_%: $(IMAGES_UPTODATE)
-	$(SUDO) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
-	$(SUDO) docker push   $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker tag -f $(DOCKERHUB_USER)/$* $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push   $(DOCKERHUB_USER)/$*:$(WEAVE_VERSION)
 ifneq ($(UPDATE_LATEST),false)
-	$(SUDO) docker push   $(DOCKERHUB_USER)/$*:latest
+	$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker push   $(DOCKERHUB_USER)/$*:latest
 endif
 
 publish: $(PUBLISH)
 
 clean-bin:
-	-$(SUDO) docker rmi $(IMAGES)
-	go clean -r $(addprefix ./,$(dir $(EXES)))
-	rm -f $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT)
+	-$(SUDO) DOCKER_HOST=$(DOCKER_HOST) docker rmi $(IMAGES) $(BUILD_IMAGE)
+	rm -rf $(EXES) $(IMAGES_UPTODATE) $(WEAVE_EXPORT) .pkg
 
 clean: clean-bin
 	rm -rf test/tls/*.pem test/coverage.* test/coverage
@@ -142,5 +186,5 @@ build:
 	$(SUDO) go install -tags netgo std
 	$(MAKE)
 
-run-smoketests: all
+run-smoketests: all testrunner
 	cd test && ./setup.sh && ./run_all.sh

NOTICE

@@ -1,3 +1,3 @@
 Weave
-Copyright 2014-2015 Weaveworks Ltd.
+Copyright 2014-2016 Weaveworks Ltd.
 This product includes software developed at Weaveworks Ltd.

README.md

@@ -1,19 +1,21 @@
-# Weave - weaving containers into applications
+# Weave Net - Weaving Containers into Applications
 
 [![Build Status](https://travis-ci.org/weaveworks/weave.svg?branch=master)](https://travis-ci.org/weaveworks/weave) [![Integration Tests](https://circleci.com/gh/weaveworks/weave/tree/master.svg?style=shield&circle-token=4933c7dabb3d0383e62117565cb9d16df7b1a811)](https://circleci.com/gh/weaveworks/weave) [![Coverage Status](https://coveralls.io/repos/weaveworks/weave/badge.svg)](https://coveralls.io/r/weaveworks/weave)
 
 # About Weaveworks
 
-[Weaveworks](http://weave.works) is the company that develops Weave -
-the most productive way for developers to connect, observe and control
-Docker containers. To learn about our products, including getting
+[Weaveworks](http://weave.works) is the company that delivers the most productive way for developers to connect, observe and control
+Docker containers. The first product developed by Weaveworks, with nearly 5 million downloads to date, is Weave Net, enabling users to get started with Docker clusters and portable apps in a fraction of the time compared to other solutions. In addition, Weaveworks offers Weave Scope, a powerful container monitoring tool that automatically maps Docker containers and their interactions, as well as Weave Flux, a microservice router that automates the access of containers as services.
+ 
+
+# Introduction
+This document is intended to cover some of the more technical aspects of Weave Net. To learn about our products, including getting
 started tutorials, visit our [website](http://weave.works) and
-[documentation](http://docs.weave.works) or continue to read about
-some of the more technical aspects of Weave in this readme.
+[documentation](http://docs.weave.works).
 
-# Weave
+# Weave Net
 
-Weave creates a virtual network that connects Docker containers
+Weave Net creates a virtual network that connects Docker containers
 deployed across multiple hosts and enables their automatic discovery.
 
 ![Weave Virtual Network](/docs/virtual-network.png?raw=true "Weave Virtual Network")
@@ -26,41 +28,47 @@ regardless of where those containers are running. Similarly, existing
 internal systems can be exposed to application containers irrespective
 of their location.
 
-![Weave Deployment](/docs/deployment.png?raw=true "Weave Deployment")
+![Weave Net Deployment](/docs/deployment.png?raw=true "Weave Net Deployment")
 
-Weave can traverse firewalls and operate in partially connected
+Weave Net can traverse firewalls and operate in partially connected
 networks. Traffic can be encrypted, allowing hosts to be connected
 across an untrusted network.
 
-With weave you can easily construct applications consisting of
+With Weave Net you can easily construct applications consisting of
 multiple containers, running anywhere.
 
-Weave works alongside Docker's existing (single host) networking
+Weave Net works alongside Docker's existing (single host) networking
 capabilities, so these can continue to be used by containers.
 
 ## Installation
 
-Ensure you are running Linux (kernel 3.8 or later) and have Docker
-(version 1.3.1 or later) installed. Then install weave with
+To get started, you should be running Linux (kernel 3.8 or later) and have Docker
+(version 1.6.0 or later) installed. Then install Weave Net with
 
     sudo curl -L git.io/weave -o /usr/local/bin/weave
     sudo chmod a+x /usr/local/bin/weave
 
-For usage on OSX (with Docker Machine) you first need to make sure
-that a VM is running and configured, as shown in [the Docker Machine
-documentation](https://docs.docker.com/installation/mac/#from-your-shell).
-Then you can launch weave directly from the OSX host.
+For usage on OSX (with Docker Machine) you first need to
+[make sure that a VM is running and configured](https://docs.docker.com/installation/mac/#from-your-shell).
+Then you can launch Weave Net directly from the OSX host.
 
-CoreOS users see [here](https://github.com/fintanr/weave-gs/blob/master/coreos-simple/user-data) for an example of installing weave using cloud-config.
+For installing Weave Net on other platforms, follow the [integration guides](http://weave.works/product/integrations/).
 
-Amazon ECS users see
-[here](https://github.com/weaveworks/guides/blob/master/aws-ecs/LATESTAMIs.md)
-for the latest Weave AMIs and
-[here](http://weave.works/guides/service-discovery-with-weave-aws-ecs.html) to get started with Weave on ECS.
-
-Weave respects the environment variable `DOCKER_HOST`, so you can run
+Weave Net respects the environment variable `DOCKER_HOST`, so you can run
 it locally to control a weave network on a remote host.
 
+Weave Net
+[periodically contacts Weaveworks servers for available versions](https://github.com/weaveworks/go-checkpoint).
+New versions are announced in the log and in
+[the status summary](http://docs.weave.works/weave/latest_release/troubleshooting.html#weave-status).
+To disable this check, run:
+
+```
+export CHECKPOINT_DISABLE=1
+```
+
+before launching Weave Net.
+
 ## Quick Start Screencast
 
 <a href="https://youtu.be/kihQCCT1ykE" alt="Click to watch the screencast" target="_blank">
@@ -73,7 +81,7 @@ Say you have docker running on two hosts, accessible to each other as
 `$HOST1` and `$HOST2`, and want to deploy an application consisting of
 two containers, one on each host.
 
-On $HOST1 we run:
+On $HOST1 you would run:
 
     host1$ weave launch
     host1$ eval $(weave env)
@@ -89,40 +97,40 @@ On $HOST1 we run:
 > `sudo`, since some commands modify environment entries and hence
 > they all need to be executed from the same shell.
 
-The first line runs weave. The second line configures our environment
+The first line runs Weave Net. The second line configures your environment
 so that containers launched via the docker command line are
-automatically attached to the weave network. Finally, we run our
+automatically attached to the weave network. Finally, you run your
 application container.
 
-That's it! If our application consists of more than one container on
-this host we simply launch them with `docker run` as appropriate.
+That's it! If your application consists of more than one container on
+this host, you can simply launch them with `docker run` as appropriate.
 
-Next we repeat similar steps on `$HOST2`...
+Next you'll repeat similar steps on `$HOST2`...
 
     host2$ weave launch $HOST1
     host2$ eval $(weave env)
     host2$ docker run --name a2 -ti ubuntu
 
 The only difference, apart from the name of the application container,
-is that we tell our weave that it should peer with the weave on
+is that weave is being told that it should peer with the weave on
 `$HOST1` (specified as the IP address or hostname, and optional
 `:port`, by which `$HOST2` can reach it). NB: if there is a firewall
 between `$HOST1` and `$HOST2`, you must permit traffic to the weave
 control port (TCP 6783) and data ports (UDP 6783/6784).
 
-Note that we could instead have told the weave on `$HOST1` to connect to
+Note that you could instead have told the Weave Net instance on `$HOST1` to connect to
 `$HOST2`, or told both about each other. Order does not matter here;
-weave automatically (re)connects to peers when they become
-available. Also, we can tell weave to connect to multiple peers by
-supplying multiple addresses, separated by spaces. And we can
+Weave Net automatically (re)connects to peers when they become
+available. You can instruct Weave Net to connect to multiple peers by
+supplying multiple addresses, separated by spaces; you can also
 [add peers dynamically](http://docs.weave.works/weave/latest_release/features.html#dynamic-topologies).
 
-Weave must be started once per host. The relevant container images are
+Weave Net must be started once per host. The relevant container images are
 pulled down on demand, but if you wish you can preload them by running
 `weave setup` - this is particularly useful for automated deployments,
 and ensures that there are no delays during later operations.
 
-Now that we've got everything set up, let's see whether our containers
+Now that you've got everything set up, let's confirm that your containers
 can talk to each other...
 
 In the container started on `$HOST1`...
@@ -141,7 +149,7 @@ Similarly, in the container started on `$HOST2`...
     1 packets transmitted, 1 received, 0% packet loss, time 0ms
     rtt min/avg/max/mdev = 0.366/0.366/0.366/0.000 ms
 
-So there we have it, two containers on separate hosts happily talking
+So there you have it, two containers on separate hosts happily talking
 to each other.
 
 ## Find out more
@@ -155,20 +163,19 @@ to each other.
 
 ## Contact Us
 
-Found a bug, want to suggest a feature, or have a question?  You can
-[File an issue](https://github.com/weaveworks/weave/issues).  And
-there is a [Weave Users Google
-Group](https://groups.google.com/a/weave.works/forum/#!forum/weave-users),
-which you can post to by email at weave-users@weave.works.  Or you can
-contact Weaveworks directly at help@weave.works. When reporting a
-bug, please include which version of weave you are running, as shown
-by `weave version`.
+Found a bug, want to suggest a feature, or have a question?  Please
+[file an issue](https://github.com/weaveworks/weave/issues), or post
+to the
+[Weave Users Google Group](https://groups.google.com/a/weave.works/forum/#!forum/weave-users),
+which you can email at weave-users@weave.works. When reporting a bug, please
+include which version of Weave Net you are running, as shown by `weave
+version`.
 
-Follow on Twitter:
+Follow us on Twitter:
 [@weaveworks](https://twitter.com/weaveworks).
 
-Read the Weave blog:
-[Weaveblog](http://weaveblog.com/).
+Check out our blog for the latest product and ecosystem news:
+[Weaveworks Blog](https://www.weave.works/blog/).
 
 IRC:
 [#weavenetwork](https://botbot.me/freenode/weavenetwork/)

Vagrantfile

@@ -5,7 +5,7 @@ require './vagrant-common.rb'
 vm_ip = "172.16.0.3" # arbitrary private IP
 
 pkgs = %w(
-  docker-engine
+  docker-engine=1.10.2-0~wily
   aufs-tools
   build-essential
   ethtool
@@ -18,8 +18,7 @@ pkgs = %w(
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
-  config.vm.box = "ubuntu/ubuntu-15.04-amd64"
-  config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/vivid/current/vivid-server-cloudimg-amd64-vagrant-disk1.box"
+  config.vm.box = "ubuntu/wily64"
 
   config.vm.network "private_network", ip: vm_ip
   config.vm.provider :virtualbox do |vb|

api/dns.go

@@ -0,0 +1,22 @@
+package api
+
+import (
+	"fmt"
+	"net/url"
+)
+
+func (client *Client) DNSDomain() (string, error) {
+	return client.httpVerb("GET", "/domain", nil)
+}
+
+func (client *Client) RegisterWithDNS(ID string, fqdn string, ip string) error {
+	data := url.Values{}
+	data.Add("fqdn", fqdn)
+	_, err := client.httpVerb("PUT", fmt.Sprintf("/name/%s/%s", ID, ip), data)
+	return err
+}
+
+func (client *Client) DeregisterWithDNS(ID string, ip string) error {
+	_, err := client.httpVerb("DELETE", fmt.Sprintf("/name/%s/%s", ID, ip), nil)
+	return err
+}

api/ipam.go

@@ -0,0 +1,63 @@
+package api
+
+import (
+	"fmt"
+	"net"
+)
+
+func (client *Client) ipamOp(ID string, op string) (*net.IPNet, error) {
+	ip, err := client.httpVerb(op, fmt.Sprintf("/ip/%s", ID), nil)
+	if err != nil {
+		return nil, err
+	}
+	return parseIP(ip)
+}
+
+// returns an IP for the ID given, allocating a fresh one if necessary
+func (client *Client) AllocateIP(ID string) (*net.IPNet, error) {
+	return client.ipamOp(ID, "POST")
+}
+
+func (client *Client) AllocateIPInSubnet(ID string, subnet *net.IPNet) (*net.IPNet, error) {
+	ip, err := client.httpVerb("POST", fmt.Sprintf("/ip/%s/%s", ID, subnet), nil)
+	if err != nil {
+		return nil, err
+	}
+	return parseIP(ip)
+}
+
+// returns an IP for the ID given, or nil if one has not been
+// allocated
+func (client *Client) LookupIP(ID string) (*net.IPNet, error) {
+	return client.ipamOp(ID, "GET")
+}
+
+// Claim a specific IP on behalf of the ID
+func (client *Client) ClaimIP(ID string, cidr *net.IPNet) error {
+	_, err := client.httpVerb("PUT", fmt.Sprintf("/ip/%s/%s", ID, cidr), nil)
+	return err
+}
+
+// release all IPs owned by an ID
+func (client *Client) ReleaseIPsFor(ID string) error {
+	_, err := client.httpVerb("DELETE", fmt.Sprintf("/ip/%s", ID), nil)
+	return err
+}
+
+func (client *Client) DefaultSubnet() (*net.IPNet, error) {
+	cidr, err := client.httpVerb("GET", fmt.Sprintf("/ipinfo/defaultsubnet"), nil)
+	if err != nil {
+		return nil, err
+	}
+	_, ipnet, err := net.ParseCIDR(cidr)
+	return ipnet, err
+}
+
+func parseIP(body string) (*net.IPNet, error) {
+	ip, ipnet, err := net.ParseCIDR(string(body))
+	if err != nil {
+		return nil, err
+	}
+	ipnet.IP = ip
+	return ipnet, nil
+}

api/weaveapi.go

@@ -0,0 +1,82 @@
+package api
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+const (
+	WeaveHTTPHost = "127.0.0.1"
+	WeaveHTTPPort = 6784
+)
+
+type Client struct {
+	baseURL string
+	log     Logger
+}
+
+func (client *Client) httpVerb(verb string, url string, values url.Values) (string, error) {
+	url = client.baseURL + url
+	client.log.Debugf("weave %s to %s with %v", verb, url, values)
+	var body io.Reader
+	if values != nil {
+		body = strings.NewReader(values.Encode())
+	}
+	req, err := http.NewRequest(verb, url, body)
+	if err != nil {
+		return "", err
+	}
+	if values != nil {
+		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	rbody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent {
+		return string(rbody), nil
+	}
+	return "", errors.New(resp.Status + ": " + string(rbody))
+}
+
+func NewClient(addr string, log Logger) *Client {
+	host := WeaveHTTPHost
+	port := fmt.Sprintf("%d", WeaveHTTPPort)
+	switch parts := strings.Split(addr, ":"); len(parts) {
+	case 0:
+	case 1:
+		if parts[0] != "" {
+			host = parts[0]
+		}
+	case 2:
+		if parts[0] != "" {
+			host = parts[0]
+		}
+		if parts[1] != "" {
+			port = parts[1]
+		}
+	default:
+		return &Client{baseURL: fmt.Sprintf("http://%s", addr), log: log}
+	}
+	return &Client{baseURL: fmt.Sprintf("http://%s:%s", host, port), log: log}
+}
+
+func (client *Client) Connect(remote string) error {
+	_, err := client.httpVerb("POST", "/connect", url.Values{"peer": {remote}})
+	return err
+}
+
+type Logger interface {
+	Infof(string, ...interface{})
+	Debugf(string, ...interface{})
+}

bin/multiweave

@@ -14,7 +14,8 @@ set -e
 START=${FIRST_WEAVE:-1}
 COUNT=${NUM_WEAVES:-28}
 FINISH=$((START+COUNT-1))
-PORT=6783
+
+SUBNET=192.168.7
 
 weavedir=$(dirname $0)/..
 
@@ -26,29 +27,42 @@ random_mac() {
     od -txC -An -N6 /dev/urandom | ( read a b c d e f && printf "%02x:$b:$c:$d:$e:$f" $((0x$a & ~1 | 2)) )
 }
 
+weave() {
+    i=$1
+    shift
+    WEAVE_CONTAINER_NAME=weave$i WEAVE_HTTP_ADDR=$SUBNET.$i:6784 $weavedir/weave "$@"
+}
+
 case "$1" in
     launch)
         shift 1
+        echo $(date -R) launching
+        ip link show multiweave >/dev/null 2>&1 || sudo ip link add name multiweave type bridge
         for i in $(seq $START $FINISH); do
-            WEAVE_CONTAINER_NAME=weave$i WEAVE_PORT=$((5000+i)) $weavedir/weave launch-router --iface '' --name $(random_mac) --nickname weave$i --port $PORT --no-dns "$@"
+            IP=$SUBNET.$i
+            sudo ip addr add $IP/24 dev multiweave
+            weave $i launch-router --datapath '' --iface '' --no-dns --host $IP \
+                --name $(random_mac) --nickname weave$i "$@"
         done
-        if [ $START -ne 1 ] ; then
-            status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$((START-1)))
-            PREV="${status#true }"
-        fi
+        echo $(date -R) connecting
+        [ $START -gt 1 ] || START=2
         for i in $(seq $START $FINISH); do
-            if [ -n "$PREV" ] ; then
-                echo connecting weave$i to $PREV
-                WEAVE_CONTAINER_NAME=weave$i $weavedir/weave connect $PREV
-            fi
-            status=$(docker inspect --format='{{.State.Running}} {{.NetworkSettings.IPAddress}}' weave$i)
-            PREV="${status#true }"
+            TO=$SUBNET.$((i-1))
+            echo connecting weave$i to $TO
+            weave $i connect $TO
         done
+        echo $(date -R) done
         ;;
     stop)
-        for i in $(seq $START $FINISH); do
+        for i in $(seq $FINISH -1 $START); do
+            IP=$SUBNET.$i
             echo Stopping weave$i
-            WEAVE_CONTAINER_NAME=weave$i $weavedir/weave stop-router
+            weave $i stop-router
+            sudo ip addr del $IP/24 dev multiweave || true
         done
+        [ -z "$(ip -4 -o addr show multiweave)" ] && sudo ip link del multiweave
+        ;;
+    *)
+            weave "$@"
         ;;
 esac

build/Dockerfile

@@ -1,24 +1,8 @@
-FROM ubuntu
-
-RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9
-RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
-RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests
-
-# When doing a build in a container, "apt-get update" happens twice,
-# which can be a very significant overhead for incremental builds.
-# And it's unnecessary, because if any of the prerequisites do change,
-# the right thing to do is to rebuild the container image.  Rather
-# than suppressing the "apt-get update" commands explicitly, we clear
-# out sources.list so that 'apt-get update' doesn't do anything.
-RUN echo >/etc/apt/sources.list
-
-ENV GO_VERSION 1.4.2
-RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xz
-ENV PATH /usr/local/go/bin:$PATH
-
-# So that sudo does not trample on PATH
-RUN sed -ie '/secure_path/d' /etc/sudoers
-
+FROM golang:1.5.2
+ENV GO15VENDOREXPERIMENT 1
+RUN apt-get update && apt-get install -y libpcap-dev python-requests time file
+RUN go get github.com/golang/lint/golint github.com/fzipp/gocyclo github.com/client9/misspell/cmd/misspell
+RUN go clean -i net && go install -tags netgo std
+RUN go install -race -tags netgo std
 COPY build.sh /
 ENTRYPOINT ["sh", "/build.sh"]

build/build.sh

@@ -1,45 +1,28 @@
 #!/bin/sh
-set -e
 
-GOPATH=/home/go
-export GOPATH
+set -eu
 
 WEAVE_SRC=$GOPATH/src/github.com/weaveworks/weave
 
-if [ $# -eq 0 -o "$1" = "tests" ] ; then
-    # No arguments.  Expect that the weave repo will be bind-mounted
-    # into $GOPATH
-    if ! [ -e $WEAVE_SRC ] ; then
-        cat 2>&1 <<EOF
-No container arguments supplied, and nothing at ${WEAVE_SRC}.  Please
-either bind-mount the golang workspace containing weave with the
+if ! [ -e $WEAVE_SRC ] ; then
+    cat 2>&1 <<EOF
+Nothing at ${WEAVE_SRC}.  Please bind-mount the weave repo with the
 docker run -v option, e.g.:
 
-    $ docker run -v <host gopath>:${GOPATH} \\
-          -v /var/run/docker.sock:/var/run/docker.sock weaveworks/weave-build
-
-Or supply git clone arguments to retrieve it, e.g.:
-
-    $ docker run -v /var/run/docker.sock:/var/run/docker.sock \\
-          weaveworks/weave-build https://github.com/weaveworks/weave.git
+    $ cd weave
+	$ docker run -v $(pwd):/go/src/github.com/weaveworks/weave \\
+		weaveworks/weave-build
 EOF
-        exit 1
-    fi
-
-    # If we run make directly, any files created on the bind mount
-    # will have awkward ownership.  So we switch to a user with the
-    # same user and group IDs as source directory.  We have to set a
-    # few things up so that sudo works without complaining later on.
-    uid=$(stat --format="%u" $WEAVE_SRC)
-    gid=$(stat --format="%g" $WEAVE_SRC)
-    echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
-    echo "weave:*:::::::" >>/etc/shadow
-    echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
-
-    su weave -c "PATH=$PATH make -C $WEAVE_SRC build"
-else
-    # There are arguments to pass to git-clone
-    mkdir -p ${WEAVE_SRC%/*}
-    git clone "$@" $WEAVE_SRC
-    make -C $WEAVE_SRC build
 fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" $WEAVE_SRC)
+gid=$(stat --format="%g" $WEAVE_SRC)
+echo "weave:x:$uid:$gid::$WEAVE_SRC:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $WEAVE_SRC BUILD_IN_CONTAINER=false $*"

circle.yml

@@ -12,58 +12,43 @@ machine:
     SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave
     PATH: $PATH:$HOME/.local/bin:$HOME/google-cloud-sdk/bin/
     CLOUDSDK_CORE_DISABLE_PROMPTS: 1
-    WEAVE_TEST: $HOME/docker/weave-test.tar
 
 dependencies:
   cache_directories:
     - "~/docker"
-  override:
-    - if [[ -e "$WEAVE_TEST" ]]; then
-        docker load -i $WEAVE_TEST;
-      else
-        docker build -t weaveworks/weave-build build;
-        docker build -t weaveworks/weave-test test/image;
-        mkdir -p $(dirname "$WEAVE_TEST");
-        docker save weaveworks/weave-test >$WEAVE_TEST;
-      fi
   post:
-    - sudo apt-get update
-    - sudo apt-get install bc jq
-    - pip install requests
+    - "sudo apt-get update && sudo apt-get install bc jq"
     - curl https://sdk.cloud.google.com | bash
     - bin/setup-circleci-secrets "$SECRET_PASSWORD"
-    - mkdir -p $(dirname $SRCDIR)
-    - cp -r $(pwd)/ $SRCDIR
+    - "mkdir -p $(dirname $SRCDIR) && cp -r $(pwd)/ $SRCDIR"
+    - "cd $SRCDIR; git submodule update --init"
     - cd $SRCDIR/test; ./gce.sh make_template:
         parallel: false
+    - "cd $SRCDIR/build; ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh && touch $SRCDIR/.build.uptodate"
 
 test:
   override:
     - cd $SRCDIR; make lint:
         parallel: true
-    - docker run -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-test:
+    - cd $SRCDIR; COVERDIR=test/coverage make RM= tests:
         parallel: true
-    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build:
+    - cd $SRCDIR; make COVERAGE=true RM= all:
         parallel: true
-    - cd $SRCDIR/test; ./gce.sh setup:
+    - cd $SRCDIR; make testrunner:
         parallel: true
-    - cd $SRCDIR/test; eval $(./gce.sh hosts); ./setup.sh:
+    - cd $SRCDIR/test; ./gce.sh setup && eval $(./gce.sh hosts) && ./setup.sh:
         parallel: true
     - cd $SRCDIR/test; eval $(./gce.sh hosts); export COVERAGE=true; ./run_all.sh:
         parallel: true
         timeout: 300
-    - cd $SRCDIR; make clean-bin:
-        parallel: true
-    - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go weaveworks/weave-build:
-        parallel: true
   post:
     - cd $SRCDIR/test; ./gce.sh destroy:
         parallel: true
+    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh):
+        parallel: true
 
 teardown:
   pre:
-    - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh):
-        parallel: true
     - test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true):
         parallel: true
     - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS):
@@ -75,4 +60,5 @@ deployment:
     owner: weaveworks
     commands:
       - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+      - cd $SRCDIR; make clean-bin
       - cd $SRCDIR; make publish

common/arp.go

@@ -0,0 +1,43 @@
+package common
+
+import "fmt"
+import "io"
+import "os"
+
+// Configure the ARP cache parameters for the given interface.  This
+// makes containers react more quickly to a change in the MAC address
+// associated with an IP address.
+func ConfigureARPCache(name string) error {
+	if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/base_reachable_time", name), "5"); err != nil {
+		return err
+	}
+	if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/delay_first_probe_time", name), "2"); err != nil {
+		return err
+	}
+	if err := sysctl(fmt.Sprintf("net/ipv4/neigh/%s/ucast_solicit", name), "1"); err != nil {
+		return err
+	}
+	return nil
+}
+
+func sysctl(variable, value string) error {
+	f, err := os.OpenFile(fmt.Sprintf("/proc/sys/%s", variable), os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+
+	n, err := io.WriteString(f, value)
+	if err != nil {
+		return err
+	}
+	if n < len(value) {
+		return io.ErrShortWrite
+	}
+
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

common/docker/client.go

@@ -1,82 +1,211 @@
 package docker
 
 import (
-	"github.com/fsouza/go-dockerclient"
-	. "github.com/weaveworks/weave/common"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	docker "github.com/fsouza/go-dockerclient"
+
+	"github.com/weaveworks/weave/common"
+)
+
+const (
+	InitialInterval = 1 * time.Second
+	MaxInterval     = 20 * time.Second
 )
 
 // An observer for container events
 type ContainerObserver interface {
 	ContainerStarted(ident string)
 	ContainerDied(ident string)
+	ContainerDestroyed(ident string)
 }
 
 type Client struct {
 	*docker.Client
 }
 
+type syncPair struct {
+	stop chan struct{}
+	done chan struct{}
+}
+
+type pendingStarts map[string]*syncPair
+
 // NewClient creates a new Docker client and checks we can talk to Docker
 func NewClient(apiPath string) (*Client, error) {
+	if apiPath != "" && !strings.Contains(apiPath, "://") {
+		apiPath = "tcp://" + apiPath
+	}
 	dc, err := docker.NewClient(apiPath)
 	if err != nil {
 		return nil, err
 	}
 	client := &Client{dc}
 
-	return client, client.checkWorking(apiPath)
+	return client, client.checkWorking()
 }
 
 func NewVersionedClient(apiPath string, apiVersionString string) (*Client, error) {
+	if !strings.Contains(apiPath, "://") {
+		apiPath = "tcp://" + apiPath
+	}
 	dc, err := docker.NewVersionedClient(apiPath, apiVersionString)
 	if err != nil {
 		return nil, err
 	}
 	client := &Client{dc}
 
-	return client, client.checkWorking(apiPath)
+	return client, client.checkWorking()
 }
 
-func (c *Client) checkWorking(apiPath string) error {
+func NewVersionedClientFromEnv(apiVersionString string) (*Client, error) {
+	dc, err := docker.NewVersionedClientFromEnv(apiVersionString)
+	if err != nil {
+		return nil, err
+	}
+	client := &Client{dc}
+
+	return client, client.checkWorking()
+}
+
+func (c *Client) checkWorking() error {
+	_, err := c.Version()
+	return err
+}
+
+func (c *Client) Info() string {
 	env, err := c.Version()
 	if err != nil {
-		return err
+		return fmt.Sprintf("Docker API error: %s", err)
 	}
-	Log.Infof("[docker] Using Docker API on %s: %v", apiPath, env)
-	return nil
+	return fmt.Sprintf("Docker API on %s: %v", c.Endpoint(), env)
 }
 
 // AddObserver adds an observer for docker events
 func (c *Client) AddObserver(ob ContainerObserver) error {
-	events := make(chan *docker.APIEvents)
-	if err := c.AddEventListener(events); err != nil {
-		Log.Errorf("[docker] Unable to add listener to Docker API: %s", err)
-		return err
-	}
-
 	go func() {
-		for event := range events {
-			switch event.Status {
-			case "start":
-				id := event.ID
-				ob.ContainerStarted(id)
-			case "die":
-				id := event.ID
-				ob.ContainerDied(id)
+		pending := make(pendingStarts)
+		retryInterval := InitialInterval
+		for {
+			events := make(chan *docker.APIEvents)
+			if err := c.AddEventListener(events); err != nil {
+				c.errorf("Unable to add listener to Docker API: %s - retrying in %ds", err, retryInterval/time.Second)
+			} else {
+				start := time.Now()
+				for event := range events {
+					switch event.Status {
+					case "start":
+						pending.finish(event.ID)
+						pending.start(event.ID, c, ob)
+					case "die":
+						pending.finish(event.ID)
+						ob.ContainerDied(event.ID)
+					case "destroy":
+						pending.finish(event.ID)
+						ob.ContainerDestroyed(event.ID)
+					}
+				}
+				if time.Since(start) > retryInterval {
+					retryInterval = InitialInterval
+				}
+				c.errorf("Event listener channel closed - retrying subscription in %ds", retryInterval/time.Second)
+			}
+			time.Sleep(retryInterval)
+			retryInterval = retryInterval * 3 / 2
+			if retryInterval > MaxInterval {
+				retryInterval = MaxInterval
 			}
 		}
 	}()
 	return nil
 }
 
+// Docker sends a 'start' event before it has attempted to start the
+// container.  Delay notifying the observer until the container has a
+// pid, or we are told to stop when a 'die' event arrives.
+//
+// Note we always deliver the event, even if the container seems to
+// have gone away.
+func (pending pendingStarts) start(id string, c *Client, ob ContainerObserver) {
+	sync := syncPair{make(chan struct{}), make(chan struct{})}
+	pending[id] = &sync
+	go func() {
+		defer close(sync.done)
+		defer ob.ContainerStarted(id)
+		for {
+			if container, err := c.InspectContainer(id); err != nil || container.State.Pid != 0 {
+				return
+			}
+			select {
+			case <-sync.stop:
+				return
+			case <-time.After(200 * time.Millisecond):
+			}
+		}
+	}()
+}
+
+func (pending pendingStarts) finish(id string) {
+	if sync, found := pending[id]; found {
+		close(sync.stop)
+		<-sync.done
+		delete(pending, id)
+	}
+}
+
+func (c *Client) AllContainerIDs() ([]string, error) {
+	all, err := c.ListContainers(docker.ListContainersOptions{All: true})
+	if err != nil {
+		return nil, err
+	}
+	var ids []string
+	for _, c := range all {
+		ids = append(ids, c.ID)
+	}
+	return ids, nil
+}
+
 // IsContainerNotRunning returns true if we have checked with Docker that the ID is not running
 func (c *Client) IsContainerNotRunning(idStr string) bool {
 	container, err := c.InspectContainer(idStr)
 	if err == nil {
-		return !container.State.Running
+		return !container.State.Running || container.State.Restarting
 	}
 	if _, notThere := err.(*docker.NoSuchContainer); notThere {
 		return true
 	}
-	Log.Errorf("[docker] Could not check container status: %s", err)
+	c.errorf("Could not check container status: %s", err)
 	return false
 }
+
+// This is intended to find an IP address that we can reach the container on;
+// if it is on the Docker bridge network then that address; if on the host network
+// then localhost
+func (c *Client) GetContainerIP(nameOrID string) (string, error) {
+	info, err := c.InspectContainer(nameOrID)
+	if err != nil {
+		return "", err
+	}
+	if info.NetworkSettings.Networks != nil {
+		if bridgeNetwork, ok := info.NetworkSettings.Networks["bridge"]; ok {
+			return bridgeNetwork.IPAddress, nil
+		} else if _, ok := info.NetworkSettings.Networks["host"]; ok {
+			return "127.0.0.1", nil
+		}
+	} else if info.HostConfig.NetworkMode == "host" {
+		return "127.0.0.1", nil
+	}
+	if info.NetworkSettings.IPAddress == "" {
+		return "", errors.New("No IP address found for container " + nameOrID)
+	}
+	return info.NetworkSettings.IPAddress, nil
+}
+
+// logging
+
+func (c *Client) errorf(fmt string, args ...interface{}) {
+	common.Log.Errorf("[docker] "+fmt, args...)
+}

common/logging.go

@@ -3,6 +3,7 @@ package common
 import (
 	"bytes"
 	"fmt"
+	"log"
 	"strings"
 
 	"github.com/Sirupsen/logrus"
@@ -52,3 +53,35 @@ func SetLogLevel(levelname string) {
 	}
 	Log.Level = level
 }
+
+func CheckFatal(e error) {
+	if e != nil {
+		Log.Fatal(e)
+	}
+}
+
+func CheckWarn(e error) {
+	if e != nil {
+		Log.Warnln(e)
+	}
+}
+
+// Adaptor type to go from Logrus to something expecting a standard library log.Logger
+type logLogger struct {
+	logger *logrus.Logger
+}
+
+// io.Writer interface
+func (l logLogger) Write(p []byte) (n int, err error) {
+	ln := len(p)
+	// Strip a trailing newline, because logrus formatter adds one
+	if ln > 0 && p[ln-1] == '\n' {
+		p = p[:ln-1]
+	}
+	l.logger.Info(string(p))
+	return ln, nil
+}
+
+func LogLogger() *log.Logger {
+	return log.New(logLogger{Log}, "", 0)
+}

common/odp/odp.go

@@ -1,4 +1,4 @@
-package router
+package odp
 
 import (
 	"fmt"
@@ -10,49 +10,45 @@ import (
 
 // ODP admin functionality
 
-func CreateDatapath(dpname string) (err error, supported bool) {
+func CreateDatapath(dpname string) (supported bool, err error) {
 	dpif, err := odp.NewDpif()
 	if err != nil {
 		if odp.IsKernelLacksODPError(err) {
-			return nil, false
+			return false, nil
 		}
-
-		return err, true
+		return true, err
 	}
-
 	defer dpif.Close()
 
 	dp, err := dpif.CreateDatapath(dpname)
 	if err != nil && !odp.IsDatapathNameAlreadyExistsError(err) {
-		return err, true
+		return true, err
 	}
 
 	// Pick an ephemeral port number to use in probing for vxlan
 	// support.
 	udpconn, err := net.ListenUDP("udp4", nil)
 	if err != nil {
-		return err, true
+		return true, err
 	}
+	defer udpconn.Close()
 
 	// we leave the UDP socket open, so creating a vxlan vport on
 	// the same port number should fail.  But that's fine: It's
 	// still sufficient to probe for support.
 	portno := uint16(udpconn.LocalAddr().(*net.UDPAddr).Port)
-	vpid, err := dp.CreateVport(odp.NewVxlanVportSpec(
-		fmt.Sprintf("vxlan-%d", portno), portno))
+	vpid, err := dp.CreateVport(odp.NewVxlanVportSpec(fmt.Sprintf("vxlan-%d", portno), portno))
 	if nlerr, ok := err.(odp.NetlinkError); ok {
 		if syscall.Errno(nlerr) == syscall.EAFNOSUPPORT {
 			dp.Delete()
-			return fmt.Errorf("kernel does not have Open vSwitch VXLAN support"), false
+			return false, fmt.Errorf("kernel does not have Open vSwitch VXLAN support")
 		}
 	}
-
 	if err == nil {
 		dp.DeleteVport(vpid)
 	}
 
-	udpconn.Close()
-	return nil, true
+	return true, nil
 }
 
 func DeleteDatapath(dpname string) error {
@@ -60,7 +56,6 @@ func DeleteDatapath(dpname string) error {
 	if err != nil {
 		return err
 	}
-
 	defer dpif.Close()
 
 	dp, err := dpif.LookupDatapath(dpname)
@@ -68,7 +63,6 @@ func DeleteDatapath(dpname string) error {
 		if odp.IsNoSuchDatapathError(err) {
 			return nil
 		}
-
 		return err
 	}
 
@@ -80,7 +74,6 @@ func AddDatapathInterface(dpname string, ifname string) error {
 	if err != nil {
 		return err
 	}
-
 	defer dpif.Close()
 
 	dp, err := dpif.LookupDatapath(dpname)

common/utils.go

@@ -1,6 +1,12 @@
 package common
 
 import (
+	"fmt"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	"net"
+	"os"
+	"runtime"
 	"strings"
 )
 
@@ -18,3 +24,132 @@ func ErrorMessages(errors []error) string {
 	}
 	return strings.Join(result, "\n")
 }
+
+func WithNetNS(ns netns.NsHandle, work func() error) error {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+
+	oldNs, err := netns.Get()
+	if err == nil {
+		defer oldNs.Close()
+
+		err = netns.Set(ns)
+		if err == nil {
+			defer netns.Set(oldNs)
+
+			err = work()
+		}
+	}
+
+	return err
+}
+
+type NetDev struct {
+	MAC   net.HardwareAddr
+	CIDRs []*net.IPNet
+}
+
+// Search the network namespace of a process for interfaces matching a predicate
+func FindNetDevs(processID int, match func(link netlink.Link) bool) ([]NetDev, error) {
+	var netDevs []NetDev
+
+	ns, err := netns.GetFromPid(processID)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer ns.Close()
+
+	err = WithNetNS(ns, func() error {
+		return forEachLink(func(link netlink.Link) error {
+			if match(link) {
+				netDev, err := linkToNetDev(link)
+				if err != nil {
+					return err
+				}
+				netDevs = append(netDevs, *netDev)
+			}
+			return nil
+		})
+	})
+
+	return netDevs, err
+}
+
+func forEachLink(f func(netlink.Link) error) error {
+	links, err := netlink.LinkList()
+	if err != nil {
+		return err
+	}
+	for _, link := range links {
+		if err := f(link); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func linkToNetDev(link netlink.Link) (*NetDev, error) {
+	addrs, err := netlink.AddrList(link, netlink.FAMILY_V4)
+	if err != nil {
+		return nil, err
+	}
+
+	netDev := &NetDev{MAC: link.Attrs().HardwareAddr}
+	for _, addr := range addrs {
+		netDev.CIDRs = append(netDev.CIDRs, addr.IPNet)
+	}
+	return netDev, nil
+}
+
+// Lookup the weave interface of a container
+func GetWeaveNetDevs(processID int) ([]NetDev, error) {
+	// Bail out if this container is running in the root namespace
+	nsToplevel, err := netns.GetFromPid(1)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open root namespace: %s", err)
+	}
+	nsContainr, err := netns.GetFromPid(processID)
+	if err != nil {
+		return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err)
+	}
+	if nsToplevel.Equal(nsContainr) {
+		return nil, nil
+	}
+
+	weaveBridge, err := netlink.LinkByName("weave")
+	if err != nil {
+		return nil, fmt.Errorf("Cannot find weave bridge: %s", err)
+	}
+	// Scan devices in root namespace to find those attached to weave bridge
+	indexes := make(map[int]struct{})
+	err = forEachLink(func(link netlink.Link) error {
+		if link.Attrs().MasterIndex == weaveBridge.Attrs().Index {
+			peerIndex := link.Attrs().ParentIndex
+			if peerIndex == 0 {
+				// perhaps running on an older kernel where ParentIndex doesn't work.
+				// as fall-back, assume the indexes are consecutive
+				peerIndex = link.Attrs().Index - 1
+			}
+			indexes[peerIndex] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return FindNetDevs(processID, func(link netlink.Link) bool {
+		_, isveth := link.(*netlink.Veth)
+		_, found := indexes[link.Attrs().Index]
+		return isveth && found
+	})
+}
+
+// Get the weave bridge interface
+func GetBridgeNetDev(bridgeName string) ([]NetDev, error) {
+	return FindNetDevs(1, func(link netlink.Link) bool {
+		return link.Attrs().Name == bridgeName
+	})
+}

db/boltdb.go

@@ -0,0 +1,84 @@
+package db
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+
+	"github.com/boltdb/bolt"
+)
+
+type DB interface {
+	Load(string, interface{}) (bool, error)
+	Save(string, interface{}) error
+}
+
+type BoltDB struct {
+	db *bolt.DB
+}
+
+var (
+	versionIdent       = []byte("version")
+	persistenceVersion = []byte{1, 0} // major.minor
+	topBucket          = []byte("top")
+)
+
+func NewBoltDB(dbPathname string) (*BoltDB, error) {
+	db, err := bolt.Open(dbPathname, 0660, nil)
+	if err != nil {
+		return nil, fmt.Errorf("[boltDB] Unable to open %s: %s", dbPathname, err)
+	}
+	// check the persistence version
+	err = db.Update(func(tx *bolt.Tx) error {
+		if top := tx.Bucket(topBucket); top == nil {
+			top, err := tx.CreateBucket(topBucket)
+			if err != nil {
+				return err
+			}
+			if err := top.Put(versionIdent, persistenceVersion); err != nil {
+				return err
+			}
+		} else {
+			if checkVersion := top.Get(versionIdent); checkVersion != nil {
+				if checkVersion[0] != persistenceVersion[0] {
+					return fmt.Errorf("[boltDB] Cannot use persistence file %s - version %x", dbPathname, checkVersion)
+				}
+			}
+		}
+		return nil
+	})
+	return &BoltDB{db: db}, err
+}
+
+func (d *BoltDB) Load(ident string, data interface{}) (bool, error) {
+	found := true
+	err := d.db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket(topBucket)
+		v := b.Get([]byte(ident))
+		if v == nil {
+			found = false
+			return nil
+		}
+		reader := bytes.NewReader(v)
+		decoder := gob.NewDecoder(reader)
+		err := decoder.Decode(data)
+		return err
+	})
+	return found, err
+}
+
+func (d *BoltDB) Save(ident string, data interface{}) error {
+	return d.db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket(topBucket)
+		buf := new(bytes.Buffer)
+		enc := gob.NewEncoder(buf)
+		if err := enc.Encode(data); err != nil {
+			return err
+		}
+		return b.Put([]byte(ident), buf.Bytes())
+	})
+}
+
+func (d *BoltDB) Close() error {
+	return d.db.Close()
+}

docs/fastdp.md

@@ -0,0 +1,170 @@
+# Overlay Method Selection
+
+Weave automatically selects the best working overlay implementation
+when connecting to a new peer. It does this by initiating both
+forwarders in parallel, eventually settling on the most preferred
+established forwarder at which point the other forwarder is shut down,
+assuming it had not failed already. In the event that the remaining
+forwarder fails at a later time (e.g. because of a heartbeat timeout)
+the control plane TCP connection will recycle allowing the selection
+algorithm to run again. By default fast datapath is preferred over
+Sleeve.
+
+# Local Bridging
+
+Unlike the weave bridge netdev used by Sleeve, the OVS datapath has no
+inherent bridging capability. Consequently fastdp implements an
+ethernet bridge in addition to the vxlan overlay, maintaining its own
+forwarding database by learning MAC addresses and dispatching
+broadcast, unicast & multicast traffic accordingly.
+
+# Short Peer IDs
+
+The router relies on having access to the source and destination peer
+when deciding how to forward packets. When the Sleeve overlay is in
+use this information is conveyed directly within the encapsulation by
+including the names of the peers in question, a solution not
+accommodated directly by the vxlan wire format. Fortunately vxlan does
+have a twenty four bit segment ID field in the header we can use to
+encode this data - the challenge is to identify peers uniquely with a
+twelve bit identifier instead of the seventeen bytes used by Sleeve.
+In practice this is achieved by peers adopting a random 'short ID' and
+resolving ownership collisions of same via the existing gossip
+mechanism.
+
+# Heartbeats via vxlan
+
+Heartbeats are implemented by sending an encapsulated ethernet frame
+with source and destination MAC addresses of `00:00:00:00:00:00` and a
+payload consisting of the connection UID and total ethernet frame
+length. The vxlan vport miss handler on the receiving side detects the
+all-zero addresses and acknowledges the heartbeat via the TCP control
+channel after validating the connection UID and frame length against
+their expected values.
+
+# PMTU Discovery
+
+The length of the heartbeat ethernet frame is set to the MTU of the
+datapath interface. In the event the vxlan packet is dropped or
+truncated, the heartbeat will not be acknowledged; this lack of
+acknowledgement will cause the peers to fall back to the Sleeve
+overlay, which has a more sophisticated dynamic mechanism for coping
+with low path MTUs.
+
+To avoid triggering this fallback in typical deployments, the datapath
+interface is statically configured with an MTU of 1410 bytes allowing
+it to work with most underlay network provider MTUs, including GCE at
+1460 bytes (the fifty byte difference accommodates the vxlan overhead).
+This value can be overridden by setting `WEAVE_MTU` at launch if
+necessary.
+
+# Virtual Ports
+
+There are three kinds of virtual port associated with the weave
+datapath:
+
+* `internal` - one of these is created automatically, named after the
+  datapath. It corresponds to the network interface of the same name
+  that appears on the host when the datapath is created; it is the
+  ingress/egress port for `weave expose`
+* `netdev` - one for each application container, corresponding to the
+  host end of the veth pair
+* `vxlan` - one for each UDP port on which we are listening for vxlan
+  encapsulated packets. Typically there is only one of these, but
+  there may be more in a network in which peers do not all use the same
+  port configuration
+
+# Misses and Flow Creation
+
+As mentioned above, the datapath has no inherent behaviour - any
+ingressing packet which does not match a flow is passed to a userspace
+miss handler. The miss handler is then responsible for a) instructing
+the datapath to take actions for the packet that caused the miss (for
+example by copying it to one or more ports) and optionally b)
+installing flow rules which will allow the datapath to act on similar
+packets in future without invoking the miss handler.
+
+At a high level the fast datapath implementation can be viewed as a
+set of miss handlers that determine what actions the OVS datapath
+should take based on router topology information, together with some
+additional machinery that manages the expiry of resulting flows when
+that state changes.
+
+When a miss handler is invoked, it has two pieces of context: a byte
+array containing the packet that triggered the miss, and a set of
+'flow keys' that have been extracted from the packet by the kernel
+module. The following flow keys are of interest to weave:
+
+* `InPort` - the identifier of the ingress virtual port
+* `Ethernet` - the source and destination MAC addresses of the
+  ethernet frame
+* `Tunnel` - tunnel identifier (see section on short peer IDs above),
+  and source/dest IPv4 addresses. Only present for ingress via a vxlan
+  vport
+
+Crucially, the router must use this information alone to determine
+which actions to take - this allows the specification of a flow which
+matches these keys and instructs the kernel to take action
+automatically in future without further userspace involvement.
+
+Two actions are of interest:
+
+* `Output` - output the packet to the specified vport
+* `SetTunnel` - update the effective vxlan tunnel parameters. Only
+  required prior to output from a vxlan vport
+
+It is possible to have multiple actions in a flow, so the router can
+for example create a single rule that copies broadcast traffic to all
+'local' vports (save the ingress vport obviously) as well as to vxlan
+vports for onward peers as dictated by the routing topology.
+
+Every flow specified by the router has the following characteristics:
+
+* An `InPort` key matching the ingress vport
+* A `Tunnel` key if the ingress vport is of type vxlan
+* An `Ethernet` key matching the source and destination MAC
+* A list of `Output` (for internal and netdev vports) and
+  `SetTunnel`+`Output` (for vxlan vports) actions
+
+Under certain circumstances the miss handler will instruct the
+datapath to execute a set of actions for a packet without creating a
+corresponding flow:
+
+* When broadcasting packets to discover the path to unknown
+  destination MACs. This condition is transient in the overwhelming
+  majority of cases and so the benefit of creating a flow is
+  outweighed by the need to invalidate it once the MAC is learned
+* When the packet also needs to be forwarded to one or more peers via
+  Sleeve. In this case the router needs to handle all subsequent
+  matching packets as there is no OVS flow action to do it without
+  userspace involvement
+
+# Flow Invalidation
+
+Once a flow has been created for a particular combination of keys, the
+miss handler will never again be invoked for matching packets. It is
+therefore extremely important that we detect events which invalidate
+existing flow actions:
+
+* Addition of netdev vports to the datapath
+* Route invalidation (topology change)
+* Short peer ID collision
+
+The response in each case is the same - delete all flows from the
+datapath, allowing them to be recreated taking into account the
+updated state.
+
+In addition to these event based invalidations there is an expiry
+process that executes every five minutes. This process enumerates all
+flows in the datapath, removing any which have not been used since the
+last check; this cleans up:
+
+* Flows referring to netdev vports which have been removed
+* Flows created by forwarders which have been stopped
+* Flows related to MAC addresses which have not communicated recently
+
+Flows keyed on tunnel IPv4 address do not need to be cleared when a
+peer appears to change IP address due to NAT; this will cause a miss
+resulting in a new flow, and the old flow will expire naturally via
+the timer mechanism.
+

docs/ipam.md

@@ -210,11 +210,3 @@ reached the peer that does the `rmpeer`, then we get an inconsistent
 ring.  We could make it safe, up to partitions, by inquiring of every
 live peer what its view of the dead peer is, and making sure we use
 the latest info.)
-
-## Limitations
-
-Nothing is persisted. If one peer restarts it should receive gossip
-from other peers showing what it used to own, and in that case it can
-recover quite well. If, however, there are no peers left alive, or
-this peer cannot establish network communication with those that are,
-then it cannot recover.

docs/release-process.md

@@ -7,6 +7,7 @@
   github-release](https://help.github.com/articles/creating-an-access-token-for-command-line-use/);
 set and export `$GITHUB_TOKEN` with this value
 * Update all dependencies with `make update`
+* Ensure you are still on the right branch, since Go may switch to `master` as part of `make update`
 
 ## Release Types
 
@@ -63,9 +64,10 @@ This has the following effects:
 ## Draft Phase
 ### Push Version Tag Upstream
 
-First you must push your version tag upstream, so that an associated
-GitHub release may be created:
+First you must push your branch and version tag upstream, so that an
+associated GitHub release may be created:
 
+    git push git@github.com:weaveworks/weave
     git push git@github.com:weaveworks/weave $TAG
 
 N.B. if you're testing the release process, push to your fork
@@ -128,6 +130,12 @@ Finally, for **Mainline** releases only:
 
 * Images tagged `latest` are updated on DockerHub
 
+### Finish up
+
+* If not on master, merge branch into master and push to GitHub.
+* Merge the branch you are on into `latest_release_doc_updates` and push to GitHub.
+* Close the [milestone](https://github.com/weaveworks/weave/milestones) in GitHub and create the next milestone
+* Update the `#weavenetwork` topic heading on freenode (requires 'chanops' permission)
 
 ## Troubleshooting
 

docs/weavedns-design.md

@@ -9,7 +9,7 @@ The model is that each host has a service that is notified of
 hostnames and weave addresses for containers on the host.  Like IPAM,
 this service is embedded within the router.  It binds to
 the host bridge to answer DNS queries from local containers; for
-anything it can't answer, it uses the infomation in the host's
+anything it can't answer, it uses the information in the host's
 /etc/resolv.conf to query an 'fallback' server.
 
 The service is comprised of a DNS server, which answers all DNS queries
@@ -33,7 +33,7 @@ are re-broadcast around the cluster.
 The DNS server also listens to the Docker event stream, and removes
 entries for containers when they die.  Entries removed in this way are
 tombstoned, and the tombstone lazily broadcast around the cluster.
-After a short timeout the tombstones are independantly removed from
+After a short timeout the tombstones are independently removed from
 each host.
 
 

ipam/allocate.go

@@ -14,7 +14,8 @@ type allocateResult struct {
 type allocate struct {
 	resultChan       chan<- allocateResult
 	ident            string
-	r                address.Range // Range we are trying to allocate within
+	r                address.CIDR // Subnet we are trying to allocate within
+	isContainer      bool
 	hasBeenCancelled func() bool
 }
 
@@ -25,29 +26,36 @@ func (g *allocate) Try(alloc *Allocator) bool {
 		return true
 	}
 
-	if addr, found := alloc.lookupOwned(g.ident, g.r); found {
-		g.resultChan <- allocateResult{addr, nil}
+	if addrs := alloc.ownedInRange(g.ident, g.r.Range()); len(addrs) > 0 {
+		// If we had heard that this container died, resurrect it
+		delete(alloc.dead, g.ident) // delete is no-op if key not in map
+		g.resultChan <- allocateResult{addrs[0].Addr, nil}
 		return true
 	}
 
-	if !alloc.universe.Overlaps(g.r) {
-		g.resultChan <- allocateResult{0, fmt.Errorf("range %s out of bounds: %s", g.r, alloc.universe)}
+	if !alloc.universe.Overlaps(g.r.Range()) {
+		g.resultChan <- allocateResult{err: fmt.Errorf("range %s out of bounds: %s", g.r, alloc.universe)}
 		return true
 	}
 
 	alloc.establishRing()
 
-	if ok, addr := alloc.space.Allocate(g.r); ok {
+	if ok, addr := alloc.space.Allocate(g.r.HostRange()); ok {
+		// If caller hasn't supplied a unique ID, file it under the IP address
+		// which lets the caller then release the address using DELETE /ip/address
+		if g.ident == "_" {
+			g.ident = addr.String()
+		}
 		alloc.debugln("Allocated", addr, "for", g.ident, "in", g.r)
-		alloc.addOwned(g.ident, addr)
+		alloc.addOwned(g.ident, address.MakeCIDR(g.r, addr), g.isContainer)
 		g.resultChan <- allocateResult{addr, nil}
 		return true
 	}
 
 	// out of space
-	donors := alloc.ring.ChoosePeersToAskForSpace(g.r.Start, g.r.End)
+	donors := alloc.ring.ChoosePeersToAskForSpace(g.r.Addr, g.r.Range().End)
 	for _, donor := range donors {
-		if err := alloc.sendSpaceRequest(donor, g.r); err != nil {
+		if err := alloc.sendSpaceRequest(donor, g.r.Range()); err != nil {
 			alloc.debugln("Problem asking peer", donor, "for space:", err)
 		} else {
 			alloc.debugln("Decided to ask peer", donor, "for space in range", g.r)
@@ -59,7 +67,7 @@ func (g *allocate) Try(alloc *Allocator) bool {
 }
 
 func (g *allocate) Cancel() {
-	g.resultChan <- allocateResult{0, &errorCancelled{"Allocate", g.ident}}
+	g.resultChan <- allocateResult{err: &errorCancelled{"Allocate", g.ident}}
 }
 
 func (g *allocate) ForContainer(ident string) bool {

ipam/allocator.go

@@ -7,12 +7,14 @@ import (
 	"sort"
 	"time"
 
+	"github.com/weaveworks/mesh"
+
 	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/db"
 	"github.com/weaveworks/weave/ipam/paxos"
 	"github.com/weaveworks/weave/ipam/ring"
 	"github.com/weaveworks/weave/ipam/space"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 // Kinds of message we can unicast to other peers
@@ -21,8 +23,9 @@ const (
 	msgRingUpdate
 	msgSpaceRequestDenied
 
-	paxosInterval = time.Second * 5
-	MinSubnetSize = 4 // first and last addresses are excluded, so 2 would be too small
+	tickInterval         = time.Second * 5
+	MinSubnetSize        = 4 // first and last addresses are excluded, so 2 would be too small
+	containerDiedTimeout = time.Second * 30
 )
 
 // operation represents something which Allocator wants to do, but
@@ -38,52 +41,94 @@ type operation interface {
 	ForContainer(ident string) bool
 }
 
+// This type is persisted hence all fields exported
+type ownedData struct {
+	IsContainer bool
+	Cidrs       []address.CIDR
+}
+
 // Allocator brings together Ring and space.Set, and does the
 // necessary plumbing.  Runs as a single-threaded Actor, so no locks
 // are used around data structures.
 type Allocator struct {
-	actionChan       chan<- func()
-	ourName          router.PeerName
-	universe         address.Range                // superset of all ranges
-	ring             *ring.Ring                   // information on ranges owned by all peers
-	space            space.Space                  // more detail on ranges owned by us
-	owned            map[string][]address.Address // who owns what addresses, indexed by container-ID
-	nicknames        map[router.PeerName]string   // so we can map nicknames for rmpeer
-	pendingAllocates []operation                  // held until we get some free space
-	pendingClaims    []operation                  // held until we know who owns the space
-	gossip           router.Gossip                // our link to the outside world for sending messages
-	paxos            *paxos.Node
-	paxosTicker      *time.Ticker
-	shuttingDown     bool // to avoid doing any requests while trying to shut down
-	isKnownPeer      func(router.PeerName) bool
-	now              func() time.Time
+	actionChan        chan<- func()
+	stopChan          chan<- struct{}
+	ourName           mesh.PeerName
+	seed              []mesh.PeerName          // optional user supplied ring seed
+	universe          address.Range            // superset of all ranges
+	ring              *ring.Ring               // information on ranges owned by all peers
+	space             space.Space              // more detail on ranges owned by us
+	owned             map[string]ownedData     // who owns what addresses, indexed by container-ID
+	nicknames         map[mesh.PeerName]string // so we can map nicknames for rmpeer
+	pendingAllocates  []operation              // held until we get some free space
+	pendingClaims     []operation              // held until we know who owns the space
+	pendingPrimes     []operation              // held while our ring is empty
+	dead              map[string]time.Time     // containers we heard were dead, and when
+	db                db.DB                    // persistence
+	gossip            mesh.Gossip              // our link to the outside world for sending messages
+	paxos             paxos.Participant
+	awaitingConsensus bool
+	ticker            *time.Ticker
+	shuttingDown      bool // to avoid doing any requests while trying to shut down
+	isKnownPeer       func(mesh.PeerName) bool
+	quorum            func() uint
+	now               func() time.Time
+}
+
+type Config struct {
+	OurName     mesh.PeerName
+	OurUID      mesh.PeerUID
+	OurNickname string
+	Seed        []mesh.PeerName
+	Universe    address.Range
+	IsObserver  bool
+	Quorum      func() uint
+	Db          db.DB
+	IsKnownPeer func(name mesh.PeerName) bool
 }
 
 // NewAllocator creates and initialises a new Allocator
-func NewAllocator(ourName router.PeerName, ourUID router.PeerUID, ourNickname string, universe address.Range, quorum uint, isKnownPeer func(name router.PeerName) bool) *Allocator {
+func NewAllocator(config Config) *Allocator {
+	var participant paxos.Participant
+	if config.IsObserver {
+		participant = paxos.NewObserver()
+	} else {
+		participant = paxos.NewNode(config.OurName, config.OurUID, 0)
+	}
 	return &Allocator{
-		ourName:     ourName,
-		universe:    universe,
-		ring:        ring.New(universe.Start, universe.End, ourName),
-		owned:       make(map[string][]address.Address),
-		paxos:       paxos.NewNode(ourName, ourUID, quorum),
-		nicknames:   map[router.PeerName]string{ourName: ourNickname},
-		isKnownPeer: isKnownPeer,
+		ourName:     config.OurName,
+		seed:        config.Seed,
+		universe:    config.Universe,
+		ring:        ring.New(config.Universe.Start, config.Universe.End, config.OurName),
+		owned:       make(map[string]ownedData),
+		db:          config.Db,
+		paxos:       participant,
+		nicknames:   map[mesh.PeerName]string{config.OurName: config.OurNickname},
+		isKnownPeer: config.IsKnownPeer,
+		quorum:      config.Quorum,
+		dead:        make(map[string]time.Time),
 		now:         time.Now,
 	}
 }
 
 // Start runs the allocator goroutine
 func (alloc *Allocator) Start() {
-	actionChan := make(chan func(), router.ChannelSize)
+	alloc.loadPersistedData()
+	actionChan := make(chan func(), mesh.ChannelSize)
+	stopChan := make(chan struct{})
 	alloc.actionChan = actionChan
-	go alloc.actorLoop(actionChan)
+	alloc.stopChan = stopChan
+	alloc.ticker = time.NewTicker(tickInterval)
+	go alloc.actorLoop(actionChan, stopChan)
 }
 
 // Stop makes the actor routine exit, for test purposes ONLY because any
 // calls after this is processed will hang. Async.
 func (alloc *Allocator) Stop() {
-	alloc.actionChan <- nil
+	select {
+	case alloc.stopChan <- struct{}{}:
+	default:
+	}
 }
 
 // Operation life cycle
@@ -140,35 +185,31 @@ func (alloc *Allocator) cancelOpsFor(ops *[]operation, ident string) bool {
 	return found
 }
 
-// Try all pending operations
-func (alloc *Allocator) tryPendingOps() {
-	// The slightly different semantics requires us to operate on 'claims' and
-	// 'allocates' separately:
-	// Claims must be tried before Allocates
-	for i := 0; i < len(alloc.pendingClaims); {
-		op := alloc.pendingClaims[i]
+// Try all operations in a queue
+func (alloc *Allocator) tryOps(ops *[]operation) {
+	for i := 0; i < len(*ops); {
+		op := (*ops)[i]
 		if !op.Try(alloc) {
 			i++
 			continue
 		}
-		alloc.pendingClaims = append(alloc.pendingClaims[:i], alloc.pendingClaims[i+1:]...)
-	}
-
-	// When the first Allocate fails, bail - no need to
-	// send too many begs for space.
-	for i := 0; i < len(alloc.pendingAllocates); {
-		op := alloc.pendingAllocates[i]
-		if !op.Try(alloc) {
-			break
-		}
-		alloc.pendingAllocates = append(alloc.pendingAllocates[:i], alloc.pendingAllocates[i+1:]...)
+		*ops = append((*ops)[:i], (*ops)[i+1:]...)
 	}
 }
 
-func (alloc *Allocator) spaceRequestDenied(sender router.PeerName, r address.Range) {
+// Try all pending operations
+func (alloc *Allocator) tryPendingOps() {
+	// Unblock pending primes first
+	alloc.tryOps(&alloc.pendingPrimes)
+	// Process existing claims before servicing new allocations
+	alloc.tryOps(&alloc.pendingClaims)
+	alloc.tryOps(&alloc.pendingAllocates)
+}
+
+func (alloc *Allocator) spaceRequestDenied(sender mesh.PeerName, r address.Range) {
 	for i := 0; i < len(alloc.pendingClaims); {
 		claim := alloc.pendingClaims[i].(*claim)
-		if r.Contains(claim.addr) {
+		if r.Contains(claim.cidr.Addr) {
 			claim.deniedBy(alloc, sender)
 			alloc.pendingClaims = append(alloc.pendingClaims[:i], alloc.pendingClaims[i+1:]...)
 			continue
@@ -188,87 +229,134 @@ func (e *errorCancelled) Error() string {
 
 // Actor client API
 
+// Prime (Sync) - wait for consensus
+func (alloc *Allocator) Prime() {
+	resultChan := make(chan struct{})
+	op := &prime{resultChan: resultChan}
+	alloc.doOperation(op, &alloc.pendingPrimes)
+	<-resultChan
+}
+
 // Allocate (Sync) - get new IP address for container with given name in range
 // if there isn't any space in that range we block indefinitely
-func (alloc *Allocator) Allocate(ident string, r address.Range, hasBeenCancelled func() bool) (address.Address, error) {
+func (alloc *Allocator) Allocate(ident string, r address.CIDR, isContainer bool, hasBeenCancelled func() bool) (address.Address, error) {
 	resultChan := make(chan allocateResult)
-	op := &allocate{resultChan: resultChan, ident: ident, r: r, hasBeenCancelled: hasBeenCancelled}
+	op := &allocate{
+		resultChan:       resultChan,
+		ident:            ident,
+		r:                r,
+		isContainer:      isContainer,
+		hasBeenCancelled: hasBeenCancelled,
+	}
 	alloc.doOperation(op, &alloc.pendingAllocates)
 	result := <-resultChan
 	return result.addr, result.err
 }
 
-// Lookup (Sync) - get existing IP address for container with given name in range
-func (alloc *Allocator) Lookup(ident string, r address.Range) (address.Address, error) {
-	resultChan := make(chan allocateResult)
+// Lookup (Sync) - get existing IP addresses for container with given name in range
+func (alloc *Allocator) Lookup(ident string, r address.Range) ([]address.CIDR, error) {
+	resultChan := make(chan []address.CIDR)
 	alloc.actionChan <- func() {
-		if addr, found := alloc.lookupOwned(ident, r); found {
-			resultChan <- allocateResult{addr: addr}
-			return
-		}
-		resultChan <- allocateResult{err: fmt.Errorf("lookup: no address found for %s in range %s", ident, r)}
+		resultChan <- alloc.ownedInRange(ident, r)
 	}
-	result := <-resultChan
-	return result.addr, result.err
+	return <-resultChan, nil
 }
 
 // Claim an address that we think we should own (Sync)
-func (alloc *Allocator) Claim(ident string, addr address.Address, noErrorOnUnknown bool) error {
+func (alloc *Allocator) Claim(ident string, cidr address.CIDR, isContainer, noErrorOnUnknown bool) error {
 	resultChan := make(chan error)
-	op := &claim{resultChan: resultChan, ident: ident, addr: addr, noErrorOnUnknown: noErrorOnUnknown}
+	op := &claim{
+		resultChan:       resultChan,
+		ident:            ident,
+		cidr:             cidr,
+		isContainer:      isContainer,
+		noErrorOnUnknown: noErrorOnUnknown,
+	}
 	alloc.doOperation(op, &alloc.pendingClaims)
 	return <-resultChan
 }
 
-// ContainerDied is provided to satisfy the updater interface; does a 'Delete' underneath.  Sync.
+// ContainerDied called from the updater interface.  Async.
 func (alloc *Allocator) ContainerDied(ident string) {
-	if err := alloc.Delete(ident); err == nil {
-		alloc.debugln("Container", ident, "died; released addresses")
+	alloc.actionChan <- func() {
+		if alloc.hasOwned(ident) {
+			alloc.debugln("Container", ident, "died; noting to remove later")
+			alloc.dead[ident] = alloc.now()
+		}
+		// Also remove any pending ops
+		alloc.cancelOpsFor(&alloc.pendingAllocates, ident)
+		alloc.cancelOpsFor(&alloc.pendingClaims, ident)
 	}
 }
 
-func (alloc *Allocator) ContainerStarted(ident string) {}
+// ContainerDestroyed called from the updater interface.  Async.
+func (alloc *Allocator) ContainerDestroyed(ident string) {
+	alloc.actionChan <- func() {
+		if alloc.hasOwned(ident) {
+			alloc.debugln("Container", ident, "destroyed; removing addresses")
+			alloc.delete(ident)
+			delete(alloc.dead, ident)
+		}
+	}
+}
+
+func (alloc *Allocator) removeDeadContainers() {
+	cutoff := alloc.now().Add(-containerDiedTimeout)
+	for ident, timeOfDeath := range alloc.dead {
+		if timeOfDeath.Before(cutoff) {
+			if err := alloc.delete(ident); err == nil {
+				alloc.debugln("Removed addresses for container", ident)
+			}
+			delete(alloc.dead, ident)
+		}
+	}
+}
+
+func (alloc *Allocator) ContainerStarted(ident string) {
+	alloc.actionChan <- func() {
+		delete(alloc.dead, ident) // delete is no-op if key not in map
+	}
+}
+
+func (alloc *Allocator) AllContainerIDs(ids []string) {
+	alloc.actionChan <- func() {
+		idmap := make(map[string]struct{}, len(ids))
+		for _, id := range ids {
+			idmap[id] = struct{}{}
+		}
+		alloc.syncOwned(idmap)
+	}
+}
 
 // Delete (Sync) - release all IP addresses for container with given name
 func (alloc *Allocator) Delete(ident string) error {
 	errChan := make(chan error)
 	alloc.actionChan <- func() {
-		addrs, found := alloc.owned[ident]
-		for _, addr := range addrs {
-			alloc.space.Free(addr)
-		}
-		delete(alloc.owned, ident)
-
-		// Also remove any pending ops
-		found = alloc.cancelOpsFor(&alloc.pendingAllocates, ident) || found
-		found = alloc.cancelOpsFor(&alloc.pendingClaims, ident) || found
-
-		if !found {
-			errChan <- fmt.Errorf("Delete: no addresses for %s", ident)
-			return
-		}
-		errChan <- nil
+		errChan <- alloc.delete(ident)
 	}
 	return <-errChan
 }
 
+func (alloc *Allocator) delete(ident string) error {
+	cidrs := alloc.removeAllOwned(ident)
+	if len(cidrs) == 0 {
+		return fmt.Errorf("Delete: no addresses for %s", ident)
+	}
+	for _, cidr := range cidrs {
+		alloc.space.Free(cidr.Addr)
+	}
+	return nil
+}
+
 // Free (Sync) - release single IP address for container
 func (alloc *Allocator) Free(ident string, addrToFree address.Address) error {
 	errChan := make(chan error)
 	alloc.actionChan <- func() {
-		addrs := alloc.owned[ident]
-		for i, ownedAddr := range addrs {
-			if ownedAddr == addrToFree {
-				alloc.debugln("Freed", addrToFree, "for", ident)
-				if len(addrs) == 1 {
-					delete(alloc.owned, ident)
-				} else {
-					alloc.owned[ident] = append(addrs[:i], addrs[i+1:]...)
-				}
-				alloc.space.Free(addrToFree)
-				errChan <- nil
-				return
-			}
+		if alloc.removeOwned(ident, addrToFree) {
+			alloc.debugln("Freed", addrToFree, "for", ident)
+			alloc.space.Free(addrToFree)
+			errChan <- nil
+			return
 		}
 
 		errChan <- fmt.Errorf("Free: address %s not found for %s", addrToFree, ident)
@@ -276,30 +364,30 @@ func (alloc *Allocator) Free(ident string, addrToFree address.Address) error {
 	return <-errChan
 }
 
-func (alloc *Allocator) pickPeerFromNicknames(isValid func(router.PeerName) bool) router.PeerName {
+func (alloc *Allocator) pickPeerFromNicknames(isValid func(mesh.PeerName) bool) mesh.PeerName {
 	for name := range alloc.nicknames {
 		if name != alloc.ourName && isValid(name) {
 			return name
 		}
 	}
-	return router.UnknownPeerName
+	return mesh.UnknownPeerName
 }
 
-func (alloc *Allocator) pickPeerForTransfer() router.PeerName {
+func (alloc *Allocator) pickPeerForTransfer() mesh.PeerName {
 	// first try alive peers that actively participate in IPAM (i.e. have entries)
-	if heir := alloc.ring.PickPeerForTransfer(alloc.isKnownPeer); heir != router.UnknownPeerName {
+	if heir := alloc.ring.PickPeerForTransfer(alloc.isKnownPeer); heir != mesh.UnknownPeerName {
 		return heir
 	}
 	// next try alive peers that have IPAM enabled but have no entries
-	if heir := alloc.pickPeerFromNicknames(alloc.isKnownPeer); heir != router.UnknownPeerName {
+	if heir := alloc.pickPeerFromNicknames(alloc.isKnownPeer); heir != mesh.UnknownPeerName {
 		return heir
 	}
 	// next try disappeared peers that still have entries
-	t := func(router.PeerName) bool { return true }
-	if heir := alloc.ring.PickPeerForTransfer(t); heir != router.UnknownPeerName {
+	t := func(mesh.PeerName) bool { return true }
+	if heir := alloc.ring.PickPeerForTransfer(t); heir != mesh.UnknownPeerName {
 		return heir
 	}
-	// finally, disappeared peers that that passively participated in IPAM
+	// finally, disappeared peers that passively participated in IPAM
 	return alloc.pickPeerFromNicknames(t)
 }
 
@@ -311,7 +399,8 @@ func (alloc *Allocator) Shutdown() {
 		alloc.shuttingDown = true
 		alloc.cancelOps(&alloc.pendingClaims)
 		alloc.cancelOps(&alloc.pendingAllocates)
-		if heir := alloc.pickPeerForTransfer(); heir != router.UnknownPeerName {
+		alloc.cancelOps(&alloc.pendingPrimes)
+		if heir := alloc.pickPeerForTransfer(); heir != mesh.UnknownPeerName {
 			alloc.ring.Transfer(alloc.ourName, heir)
 			alloc.space.Clear()
 			alloc.gossip.GossipBroadcast(alloc.Gossip())
@@ -322,26 +411,40 @@ func (alloc *Allocator) Shutdown() {
 	<-doneChan
 }
 
-// AdminTakeoverRanges (Sync) - take over the ranges owned by a given peer.
-// Only done on adminstrator command.
-func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
-	resultChan := make(chan error)
+// AdminTakeoverRanges (Sync) - take over the ranges owned by a given
+// peer, and return how much space was transferred in the process.
+// Only done on administrator command.
+func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) address.Count {
+	resultChan := make(chan address.Count)
 	alloc.actionChan <- func() {
 		peername, err := alloc.lookupPeername(peerNameOrNickname)
 		if err != nil {
-			resultChan <- fmt.Errorf("Cannot find peer '%s'", peerNameOrNickname)
+			alloc.warnf("attempt to take over range from unknown peer '%s'", peerNameOrNickname)
+			resultChan <- address.Count(0)
 			return
 		}
 
 		alloc.debugln("AdminTakeoverRanges:", peername)
 		if peername == alloc.ourName {
-			resultChan <- fmt.Errorf("Cannot take over ranges from yourself!")
+			alloc.warnf("attempt to take over range from ourself")
+			resultChan <- address.Count(0)
 			return
 		}
 
-		newRanges, err := alloc.ring.Transfer(peername, alloc.ourName)
+		newRanges := alloc.ring.Transfer(peername, alloc.ourName)
+
+		if len(newRanges) == 0 {
+			resultChan <- address.Count(0)
+			return
+		}
+
+		before := alloc.space.NumFreeAddresses()
 		alloc.space.AddRanges(newRanges)
-		resultChan <- err
+		after := alloc.space.NumFreeAddresses()
+
+		alloc.gossip.GossipBroadcast(alloc.Gossip())
+
+		resultChan <- after - before
 	}
 	return <-resultChan
 }
@@ -350,14 +453,14 @@ func (alloc *Allocator) AdminTakeoverRanges(peerNameOrNickname string) error {
 // call into the router for this because we are interested in peers
 // that have gone away but are still in the ring, which is why we
 // maintain our own nicknames map.
-func (alloc *Allocator) lookupPeername(name string) (router.PeerName, error) {
+func (alloc *Allocator) lookupPeername(name string) (mesh.PeerName, error) {
 	for peername, nickname := range alloc.nicknames {
 		if nickname == name {
 			return peername, nil
 		}
 	}
 
-	return router.PeerNameFromString(name)
+	return mesh.PeerNameFromString(name)
 }
 
 // Restrict the peers in "nicknames" to those in the ring plus peers known to the router
@@ -370,7 +473,7 @@ func (alloc *Allocator) pruneNicknames() {
 	}
 }
 
-func (alloc *Allocator) annotatePeernames(names []router.PeerName) []string {
+func (alloc *Allocator) annotatePeernames(names []mesh.PeerName) []string {
 	var res []string
 	for _, name := range names {
 		if nickname, found := alloc.nicknames[name]; found {
@@ -388,7 +491,7 @@ func decodeRange(msg []byte) (r address.Range, err error) {
 }
 
 // OnGossipUnicast (Sync)
-func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) error {
+func (alloc *Allocator) OnGossipUnicast(sender mesh.PeerName, msg []byte) error {
 	alloc.debugln("OnGossipUnicast from", sender, ": ", len(msg), "bytes")
 	resultChan := make(chan error)
 	alloc.actionChan <- func() {
@@ -414,7 +517,7 @@ func (alloc *Allocator) OnGossipUnicast(sender router.PeerName, msg []byte) erro
 }
 
 // OnGossipBroadcast (Sync)
-func (alloc *Allocator) OnGossipBroadcast(sender router.PeerName, msg []byte) (router.GossipData, error) {
+func (alloc *Allocator) OnGossipBroadcast(sender mesh.PeerName, msg []byte) (mesh.GossipData, error) {
 	alloc.debugln("OnGossipBroadcast from", sender, ":", len(msg), "bytes")
 	resultChan := make(chan error)
 	alloc.actionChan <- func() {
@@ -425,9 +528,9 @@ func (alloc *Allocator) OnGossipBroadcast(sender router.PeerName, msg []byte) (r
 
 type gossipState struct {
 	// We send a timstamp along with the information to be
-	// gossipped in order to detect skewed clocks
+	// gossipped for backwards-compatibility (previously to detect skewed clocks)
 	Now       int64
-	Nicknames map[router.PeerName]string
+	Nicknames map[mesh.PeerName]string
 
 	Paxos paxos.GossipState
 	Ring  *ring.Ring
@@ -440,6 +543,9 @@ func (alloc *Allocator) encode() []byte {
 	}
 
 	// We're only interested in Paxos until we have a Ring.
+	// Non-electing participants (e.g. observers) return
+	// a nil gossip state in order to provoke a unicast ring
+	// update from electing peers who have reached consensus.
 	if alloc.ring.Empty() {
 		data.Paxos = alloc.paxos.GossipState()
 	} else {
@@ -463,11 +569,11 @@ func (alloc *Allocator) Encode() []byte {
 }
 
 // OnGossip (Sync)
-func (alloc *Allocator) OnGossip(msg []byte) (router.GossipData, error) {
+func (alloc *Allocator) OnGossip(msg []byte) (mesh.GossipData, error) {
 	alloc.debugln("Allocator.OnGossip:", len(msg), "bytes")
 	resultChan := make(chan error)
 	alloc.actionChan <- func() {
-		resultChan <- alloc.update(router.UnknownPeerName, msg)
+		resultChan <- alloc.update(mesh.UnknownPeerName, msg)
 	}
 	return nil, <-resultChan // for now, we never propagate updates. TBD
 }
@@ -478,8 +584,8 @@ type ipamGossipData struct {
 	alloc *Allocator
 }
 
-func (d *ipamGossipData) Merge(other router.GossipData) {
-	// no-op
+func (d *ipamGossipData) Merge(other mesh.GossipData) mesh.GossipData {
+	return d // no-op
 }
 
 func (d *ipamGossipData) Encode() [][]byte {
@@ -488,32 +594,31 @@ func (d *ipamGossipData) Encode() [][]byte {
 
 // Gossip returns a GossipData implementation, which in this case always
 // returns the latest ring state (and does nothing on merge)
-func (alloc *Allocator) Gossip() router.GossipData {
+func (alloc *Allocator) Gossip() mesh.GossipData {
 	return &ipamGossipData{alloc}
 }
 
 // SetInterfaces gives the allocator two interfaces for talking to the outside world
-func (alloc *Allocator) SetInterfaces(gossip router.Gossip) {
+func (alloc *Allocator) SetInterfaces(gossip mesh.Gossip) {
 	alloc.gossip = gossip
 }
 
 // ACTOR server
 
-func (alloc *Allocator) actorLoop(actionChan <-chan func()) {
+func (alloc *Allocator) actorLoop(actionChan <-chan func(), stopChan <-chan struct{}) {
+	defer alloc.ticker.Stop()
 	for {
-		var tickChan <-chan time.Time
-		if alloc.paxosTicker != nil {
-			tickChan = alloc.paxosTicker.C
-		}
-
 		select {
 		case action := <-actionChan:
-			if action == nil {
-				return
-			}
 			action()
-		case <-tickChan:
-			alloc.propose()
+		case <-stopChan:
+			return
+		case <-alloc.ticker.C:
+			if alloc.awaitingConsensus {
+				alloc.propose()
+			}
+			alloc.removeDeadContainers()
+			alloc.tryPendingOps()
 		}
 
 		alloc.assertInvariants()
@@ -525,22 +630,21 @@ func (alloc *Allocator) actorLoop(actionChan <-chan func()) {
 
 // Ensure we are making progress towards an established ring
 func (alloc *Allocator) establishRing() {
-	if !alloc.ring.Empty() || alloc.paxosTicker != nil {
+	if !alloc.ring.Empty() || alloc.awaitingConsensus {
 		return
 	}
 
+	alloc.awaitingConsensus = true
+	alloc.paxos.SetQuorum(alloc.quorum())
 	alloc.propose()
 	if ok, cons := alloc.paxos.Consensus(); ok {
 		// If the quorum was 1, then proposing immediately
 		// leads to consensus
 		alloc.createRing(cons.Value)
-	} else {
-		// re-try until we get consensus
-		alloc.paxosTicker = time.NewTicker(paxosInterval)
 	}
 }
 
-func (alloc *Allocator) createRing(peers []router.PeerName) {
+func (alloc *Allocator) createRing(peers []mesh.PeerName) {
 	alloc.debugln("Paxos consensus:", peers)
 	alloc.ring.ClaimForPeers(normalizeConsensus(peers))
 	alloc.gossip.GossipBroadcast(alloc.Gossip())
@@ -549,21 +653,18 @@ func (alloc *Allocator) createRing(peers []router.PeerName) {
 
 func (alloc *Allocator) ringUpdated() {
 	// When we have a ring, we don't need paxos any more
-	if alloc.paxos != nil {
+	if alloc.awaitingConsensus {
+		alloc.awaitingConsensus = false
 		alloc.paxos = nil
-
-		if alloc.paxosTicker != nil {
-			alloc.paxosTicker.Stop()
-			alloc.paxosTicker = nil
-		}
 	}
 
+	alloc.persistRing()
 	alloc.space.UpdateRanges(alloc.ring.OwnedRanges())
 	alloc.tryPendingOps()
 }
 
 // For compatibility with sort.Interface
-type peerNames []router.PeerName
+type peerNames []mesh.PeerName
 
 func (a peerNames) Len() int           { return len(a) }
 func (a peerNames) Less(i, j int) bool { return a[i] < a[j] }
@@ -572,7 +673,7 @@ func (a peerNames) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 // When we get a consensus from Paxos, the peer names are not in a
 // defined order and may contain duplicates.  This function sorts them
 // and de-dupes.
-func normalizeConsensus(consensus []router.PeerName) []router.PeerName {
+func normalizeConsensus(consensus []mesh.PeerName) []mesh.PeerName {
 	if len(consensus) == 0 {
 		return nil
 	}
@@ -607,22 +708,22 @@ func encodeRange(r address.Range) []byte {
 	return buf.Bytes()
 }
 
-func (alloc *Allocator) sendSpaceRequest(dest router.PeerName, r address.Range) error {
-	msg := router.Concat([]byte{msgSpaceRequest}, encodeRange(r))
+func (alloc *Allocator) sendSpaceRequest(dest mesh.PeerName, r address.Range) error {
+	msg := append([]byte{msgSpaceRequest}, encodeRange(r)...)
 	return alloc.gossip.GossipUnicast(dest, msg)
 }
 
-func (alloc *Allocator) sendSpaceRequestDenied(dest router.PeerName, r address.Range) error {
-	msg := router.Concat([]byte{msgSpaceRequestDenied}, encodeRange(r))
+func (alloc *Allocator) sendSpaceRequestDenied(dest mesh.PeerName, r address.Range) error {
+	msg := append([]byte{msgSpaceRequestDenied}, encodeRange(r)...)
 	return alloc.gossip.GossipUnicast(dest, msg)
 }
 
-func (alloc *Allocator) sendRingUpdate(dest router.PeerName) {
-	msg := router.Concat([]byte{msgRingUpdate}, alloc.encode())
+func (alloc *Allocator) sendRingUpdate(dest mesh.PeerName) {
+	msg := append([]byte{msgRingUpdate}, alloc.encode()...)
 	alloc.gossip.GossipUnicast(dest, msg)
 }
 
-func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
+func (alloc *Allocator) update(sender mesh.PeerName, msg []byte) error {
 	reader := bytes.NewReader(msg)
 	decoder := gob.NewDecoder(reader)
 	var data gossipState
@@ -632,21 +733,21 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 		return err
 	}
 
-	deltat := time.Unix(data.Now, 0).Sub(alloc.now())
-	if deltat > time.Hour || -deltat > time.Hour {
-		return fmt.Errorf("clock skew of %v detected, ignoring update", deltat)
-	}
-
 	// Merge nicknames
 	for peer, nickname := range data.Nicknames {
 		alloc.nicknames[peer] = nickname
 	}
 
-	// only one of Ring and Paxos should be present.  And we
-	// shouldn't get updates for a empty Ring. But tolerate
-	// them just in case.
-	if data.Ring != nil {
+	switch {
+	// If someone sent us a ring, merge it into ours. Note this will move us
+	// out of the awaiting-consensus state if we didn't have a ring already.
+	case data.Ring != nil:
 		switch err = alloc.ring.Merge(*data.Ring); err {
+		case nil:
+			if !alloc.ring.Empty() {
+				alloc.pruneNicknames()
+				alloc.ringUpdated()
+			}
 		case ring.ErrDifferentSeeds:
 			return fmt.Errorf("IP allocation was seeded by different peers (received: %v, ours: %v)",
 				alloc.annotatePeernames(data.Ring.Seeds), alloc.annotatePeernames(alloc.ring.Seeds))
@@ -654,39 +755,52 @@ func (alloc *Allocator) update(sender router.PeerName, msg []byte) error {
 			return fmt.Errorf("Incompatible IP allocation ranges (received: %s, ours: %s)",
 				data.Ring.Range().AsCIDRString(), alloc.ring.Range().AsCIDRString())
 		default:
-			if err == nil && !alloc.ring.Empty() {
-				alloc.pruneNicknames()
-				alloc.ringUpdated()
-			}
 			return err
 		}
-	}
 
-	if data.Paxos != nil {
-		if alloc.ring.Empty() {
-			if alloc.paxos.Update(data.Paxos) {
-				if alloc.paxos.Think() {
-					// If something important changed, broadcast
-					alloc.gossip.GossipBroadcast(alloc.Gossip())
-				}
-
-				if ok, cons := alloc.paxos.Consensus(); ok {
-					alloc.createRing(cons.Value)
-				}
-			}
-		} else if sender != router.UnknownPeerName {
-			// Sender is trying to initialize a ring, but we have one
-			// already - send it straight back
+	// If we reach this point we know the sender is either an elector
+	// broadcasting a paxos proposal to form a ring or a non-elector
+	// broadcasting a ring request. If we have a ring already we can just send
+	// it back regardless.
+	case !alloc.ring.Empty():
+		if sender != mesh.UnknownPeerName {
 			alloc.sendRingUpdate(sender)
 		}
+
+	// Otherwise, we need to react according to whether or not we received a
+	// paxos proposal.
+	case data.Paxos != nil:
+		// Process the proposal (this is a no-op if we're an observer)
+		if alloc.paxos.Update(data.Paxos) {
+			if alloc.paxos.Think() {
+				// If something important changed, broadcast
+				alloc.gossip.GossipBroadcast(alloc.Gossip())
+			}
+
+			if ok, cons := alloc.paxos.Consensus(); ok {
+				alloc.createRing(cons.Value)
+			}
+		}
+
+	// No paxos proposal present, so sender is a non-elector. We don't have a
+	// ring to send, so attempt to establish one on their behalf. NB we only do
+	// this:
+	//
+	// * On an explicit broadcast request triggered by a remote allocation attempt
+	//   (if we did so on periodic gossip we would force consensus unnecessarily)
+	// * If we are an elector (to avoid a broadcast storm of ring request messages)
+	default:
+		if alloc.paxos.IsElector() && sender != mesh.UnknownPeerName {
+			alloc.establishRing()
+		}
 	}
 
 	return nil
 }
 
-func (alloc *Allocator) donateSpace(r address.Range, to router.PeerName) {
+func (alloc *Allocator) donateSpace(r address.Range, to mesh.PeerName) {
 	// No matter what we do, we'll send a unicast gossip
-	// of our ring back to tha chap who asked for space.
+	// of our ring back to the chap who asked for space.
 	// This serves to both tell him of any space we might
 	// have given him, or tell him where he might find some
 	// more.
@@ -705,7 +819,7 @@ func (alloc *Allocator) donateSpace(r address.Range, to router.PeerName) {
 	}
 	alloc.debugln("Giving range", chunk, "to", to)
 	alloc.ring.GrantRangeToHost(chunk.Start, chunk.End, to)
-	alloc.sendRingUpdate(to)
+	alloc.persistRing()
 }
 
 func (alloc *Allocator) assertInvariants() {
@@ -731,33 +845,139 @@ func (alloc *Allocator) reportFreeSpace() {
 		return
 	}
 
-	freespace := make(map[address.Address]address.Offset)
+	freespace := make(map[address.Address]address.Count)
 	for _, r := range ranges {
 		freespace[r.Start] = alloc.space.NumFreeAddressesInRange(r)
 	}
 	alloc.ring.ReportFree(freespace)
 }
 
-// Owned addresses
+// Persistent data
+const (
+	ringIdent  = "ring"
+	nameIdent  = "peername"
+	ownedIdent = "ownedAddresses"
+)
 
-// NB: addr must not be owned by ident already
-func (alloc *Allocator) addOwned(ident string, addr address.Address) {
-	alloc.owned[ident] = append(alloc.owned[ident], addr)
+func (alloc *Allocator) persistRing() {
+	// It would be better if these two Save operations happened in the same transaction
+	if err := alloc.db.Save(nameIdent, alloc.ourName); err != nil {
+		alloc.fatalf("Error persisting ring data: %s", err)
+		return
+	}
+	if err := alloc.db.Save(ringIdent, alloc.ring); err != nil {
+		alloc.fatalf("Error persisting ring data: %s", err)
+	}
 }
 
-func (alloc *Allocator) lookupOwned(ident string, r address.Range) (address.Address, bool) {
-	for _, addr := range alloc.owned[ident] {
-		if r.Contains(addr) {
-			return addr, true
+func (alloc *Allocator) loadPersistedData() {
+	var checkPeerName mesh.PeerName
+	nameFound, err := alloc.db.Load(nameIdent, &checkPeerName)
+	if err != nil {
+		alloc.fatalf("Error loading persisted peer name: %s", err)
+	}
+	ringFound, err := alloc.db.Load(ringIdent, &alloc.ring)
+	if err != nil {
+		alloc.fatalf("Error loading persisted IPAM data: %s", err)
+	}
+	ownedFound, err := alloc.db.Load(ownedIdent, &alloc.owned)
+	if err != nil {
+		alloc.fatalf("Error loading persisted address data: %s", err)
+	}
+
+	if nameFound {
+		if checkPeerName == alloc.ourName {
+			if ringFound {
+				if len(alloc.seed) != 0 {
+					alloc.infof("Found persisted IPAM data, ignoring supplied IPAM seed")
+				}
+				alloc.space.UpdateRanges(alloc.ring.OwnedRanges())
+			}
+			if ownedFound {
+				for _, d := range alloc.owned {
+					for _, cidr := range d.Cidrs {
+						alloc.space.Claim(cidr.Addr)
+					}
+				}
+			}
+			return
+		}
+		alloc.infof("Deleting persisted data for peername %s", checkPeerName)
+		alloc.persistRing()
+		alloc.persistOwned()
+	}
+
+	if len(alloc.seed) != 0 {
+		alloc.infof("Initialising with supplied IPAM seed")
+		alloc.createRing(alloc.seed)
+	} else if alloc.paxos.IsElector() {
+		alloc.infof("Initialising via deferred consensus")
+	} else {
+		alloc.infof("Initialising as observer - awaiting IPAM data from another peer")
+	}
+
+}
+
+func (alloc *Allocator) persistOwned() {
+	if err := alloc.db.Save(ownedIdent, alloc.owned); err != nil {
+		alloc.fatalf("Error persisting address data: %s", err)
+	}
+}
+
+// Owned addresses
+
+func (alloc *Allocator) hasOwned(ident string) bool {
+	_, b := alloc.owned[ident]
+	return b
+}
+
+// NB: addr must not be owned by ident already
+func (alloc *Allocator) addOwned(ident string, cidr address.CIDR, isContainer bool) {
+	d := alloc.owned[ident]
+	d.IsContainer = isContainer
+	d.Cidrs = append(d.Cidrs, cidr)
+	alloc.owned[ident] = d
+	alloc.persistOwned()
+}
+
+func (alloc *Allocator) removeAllOwned(ident string) []address.CIDR {
+	a := alloc.owned[ident]
+	delete(alloc.owned, ident)
+	alloc.persistOwned()
+	return a.Cidrs
+}
+
+func (alloc *Allocator) removeOwned(ident string, addrToFree address.Address) bool {
+	d := alloc.owned[ident]
+	for i, ownedCidr := range d.Cidrs {
+		if ownedCidr.Addr == addrToFree {
+			if len(d.Cidrs) == 1 {
+				delete(alloc.owned, ident)
+			} else {
+				d.Cidrs = append(d.Cidrs[:i], d.Cidrs[i+1:]...)
+				alloc.owned[ident] = d
+			}
+			alloc.persistOwned()
+			return true
 		}
 	}
-	return 0, false
+	return false
+}
+
+func (alloc *Allocator) ownedInRange(ident string, r address.Range) []address.CIDR {
+	var c []address.CIDR
+	for _, cidr := range alloc.owned[ident].Cidrs {
+		if r.Contains(cidr.Addr) {
+			c = append(c, cidr)
+		}
+	}
+	return c
 }
 
 func (alloc *Allocator) findOwner(addr address.Address) string {
-	for ident, addrs := range alloc.owned {
-		for _, candidate := range addrs {
-			if candidate == addr {
+	for ident, d := range alloc.owned {
+		for _, candidate := range d.Cidrs {
+			if candidate.Addr == addr {
 				return ident
 			}
 		}
@@ -765,14 +985,44 @@ func (alloc *Allocator) findOwner(addr address.Address) string {
 	return ""
 }
 
+// For each ID in the 'owned' map, remove the entry if it isn't in the map
+func (alloc *Allocator) syncOwned(ids map[string]struct{}) {
+	changed := false
+	for ident, d := range alloc.owned {
+		if !d.IsContainer {
+			continue
+		}
+		if _, found := ids[ident]; !found {
+			for _, cidr := range d.Cidrs {
+				alloc.space.Free(cidr.Addr)
+			}
+			alloc.debugf("Deleting old entry %s: %v", ident, d.Cidrs)
+			delete(alloc.owned, ident)
+			changed = true
+		}
+	}
+	if changed {
+		alloc.persistOwned()
+	}
+}
+
 // Logging
 
+func (alloc *Allocator) fatalf(fmt string, args ...interface{}) {
+	alloc.logf(common.Log.Fatalf, fmt, args...)
+}
+func (alloc *Allocator) warnf(fmt string, args ...interface{}) {
+	alloc.logf(common.Log.Warnf, fmt, args...)
+}
 func (alloc *Allocator) infof(fmt string, args ...interface{}) {
-	common.Log.Infof("[allocator %s] "+fmt, append([]interface{}{alloc.ourName}, args...)...)
+	alloc.logf(common.Log.Infof, fmt, args...)
+}
+func (alloc *Allocator) debugf(fmt string, args ...interface{}) {
+	alloc.logf(common.Log.Debugf, fmt, args...)
+}
+func (alloc *Allocator) logf(f func(string, ...interface{}), fmt string, args ...interface{}) {
+	f("[allocator %s] "+fmt, append([]interface{}{alloc.ourName}, args...)...)
 }
 func (alloc *Allocator) debugln(args ...interface{}) {
 	common.Log.Debugln(append([]interface{}{fmt.Sprintf("[allocator %s]:", alloc.ourName)}, args...)...)
 }
-func (alloc *Allocator) debugf(fmt string, args ...interface{}) {
-	common.Log.Debugf("[allocator %s] "+fmt, append([]interface{}{alloc.ourName}, args...)...)
-}

ipam/allocator_test.go

@@ -37,40 +37,54 @@ func TestAllocFree(t *testing.T) {
 
 	alloc, subnet := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", universe, 1)
 	defer alloc.Stop()
-	_, cidr1, _ := address.ParseCIDR(subnet1)
-	_, cidr2, _ := address.ParseCIDR(subnet2)
+	cidr1, _ := address.ParseCIDR(subnet1)
+	cidr2, _ := address.ParseCIDR(subnet2)
 
 	alloc.claimRingForTesting()
-	addr1, err := alloc.Allocate(container1, cidr1.HostRange(), returnFalse)
+	addr1, err := alloc.Allocate(container1, cidr1, true, returnFalse)
 	require.NoError(t, err)
 	require.Equal(t, testAddr1, addr1.String(), "address")
 
-	addr2, err := alloc.Allocate(container1, cidr2.HostRange(), returnFalse)
+	addr2, err := alloc.Allocate(container1, cidr2, true, returnFalse)
 	require.NoError(t, err)
 	require.Equal(t, testAddr2, addr2.String(), "address")
 
+	addrs, err := alloc.Lookup(container1, subnet.Range())
+	require.NoError(t, err)
+	require.Equal(t, []address.CIDR{address.MakeCIDR(cidr1, addr1), address.MakeCIDR(cidr2, addr2)}, addrs)
+
 	// Ask for another address for a different container and check it's different
-	addr1b, _ := alloc.Allocate(container2, cidr1.HostRange(), returnFalse)
+	addr1b, _ := alloc.Allocate(container2, cidr1, true, returnFalse)
 	if addr1b.String() == testAddr1 {
 		t.Fatalf("Expected different address but got %s", addr1b.String())
 	}
 
 	// Ask for the first container again and we should get the same addresses again
-	addr1a, _ := alloc.Allocate(container1, cidr1.HostRange(), returnFalse)
+	addr1a, _ := alloc.Allocate(container1, cidr1, true, returnFalse)
 	require.Equal(t, testAddr1, addr1a.String(), "address")
-	addr2a, _ := alloc.Allocate(container1, cidr2.HostRange(), returnFalse)
+	addr2a, _ := alloc.Allocate(container1, cidr2, true, returnFalse)
 	require.Equal(t, testAddr2, addr2a.String(), "address")
 
 	// Now delete the first container, and we should get its addresses back
 	require.NoError(t, alloc.Delete(container1))
-	addr3, _ := alloc.Allocate(container3, cidr1.HostRange(), returnFalse)
+	addr3, _ := alloc.Allocate(container3, cidr1, true, returnFalse)
 	require.Equal(t, testAddr1, addr3.String(), "address")
-	addr4, _ := alloc.Allocate(container3, cidr2.HostRange(), returnFalse)
+	addr4, _ := alloc.Allocate(container3, cidr2, true, returnFalse)
 	require.Equal(t, testAddr2, addr4.String(), "address")
 
 	alloc.ContainerDied(container2)
+
+	// Resurrect
+	addr1c, err := alloc.Allocate(container2, cidr1, true, returnFalse)
+	require.NoError(t, err)
+	require.Equal(t, addr1b, addr1c, "address")
+
 	alloc.ContainerDied(container3)
-	require.Equal(t, address.Offset(spaceSize), alloc.NumFreeAddresses(subnet))
+	alloc.Encode() // sync up
+	// Move the clock forward and clear out the dead container
+	alloc.actionChan <- func() { alloc.now = func() time.Time { return time.Now().Add(containerDiedTimeout * 2) } }
+	alloc.actionChan <- func() { alloc.removeDeadContainers() }
+	require.Equal(t, address.Count(spaceSize+1), alloc.NumFreeAddresses(subnet.Range()))
 }
 
 func TestBootstrap(t *testing.T) {
@@ -95,7 +109,7 @@ func TestBootstrap(t *testing.T) {
 	ExpectBroadcastMessage(alloc1, nil) // alloc1 will try to form consensus
 	done := make(chan bool)
 	go func() {
-		alloc1.Allocate("somecontainer", subnet, returnFalse)
+		alloc1.Allocate("somecontainer", subnet, true, returnFalse)
 		done <- true
 	}()
 	time.Sleep(100 * time.Millisecond)
@@ -103,6 +117,7 @@ func TestBootstrap(t *testing.T) {
 
 	CheckAllExpectedMessagesSent(alloc1, alloc2)
 
+	// Check that tryPendingOps doesn't cause anything to happen here
 	alloc1.actionChan <- func() { alloc1.tryPendingOps() }
 	AssertNothingSent(t, done)
 
@@ -115,14 +130,24 @@ func TestBootstrap(t *testing.T) {
 	ExpectBroadcastMessage(alloc1, nil)
 	alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode())
 
-	// both nodes will get consensus now so initialize the ring
-	ExpectBroadcastMessage(alloc2, nil)
+	CheckAllExpectedMessagesSent(alloc1, alloc2)
+
+	// at this point, both nodes have seen each other, but don't have a valid proposal
+	// so there is another exchange of messages
 	ExpectBroadcastMessage(alloc2, nil)
 	alloc2.OnGossipBroadcast(alloc1.ourName, alloc1.Encode())
 
+	// we have consensus now so alloc1 will initialize the ring
+	ExpectBroadcastMessage(alloc1, nil)
+	alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode())
+
+	CheckAllExpectedMessagesSent(alloc1, alloc2)
+
+	// alloc2 receives the ring and does not reply
+	alloc2.OnGossipBroadcast(alloc1.ourName, alloc1.Encode())
+
 	CheckAllExpectedMessagesSent(alloc1, alloc2)
 
-	alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode())
 	// now alloc1 should have space
 
 	AssertSent(t, done)
@@ -135,42 +160,41 @@ func TestAllocatorClaim(t *testing.T) {
 		container1 = "abcdef"
 		container3 = "b01df00d"
 		universe   = "10.0.3.0/24"
-		testAddr1  = "10.0.3.2"
-		testAddr2  = "10.0.4.2"
+		testAddr1  = "10.0.3.2/24"
+		testAddr2  = "10.0.4.2/24"
 	)
 
-	allocs, _, subnet := makeNetworkOfAllocators(2, universe)
+	allocs, router, subnet := makeNetworkOfAllocators(2, universe)
+	defer stopNetworkOfAllocators(allocs, router)
 	alloc := allocs[1]
-	defer alloc.Stop()
-	addr1, _ := address.ParseIP(testAddr1)
+	addr1, _ := address.ParseCIDR(testAddr1)
 
 	// First claim should trigger "dunno, I'm going to wait"
-	err := alloc.Claim(container3, addr1, true)
+	err := alloc.Claim(container3, addr1, true, true)
 	require.NoError(t, err)
 
-	// Do one allocate to ensure paxos is all done
-	alloc.Allocate("unused", subnet, returnFalse)
+	alloc.Prime()
 	// Do an allocate on the other peer, which we will try to claim later
-	addrx, err := allocs[0].Allocate(container1, subnet, returnFalse)
+	addrx, err := allocs[0].Allocate(container1, subnet, true, returnFalse)
 
 	// Now try the claim again
-	err = alloc.Claim(container3, addr1, true)
+	err = alloc.Claim(container3, addr1, true, true)
 	require.NoError(t, err)
 	// Check we get this address back if we try an allocate
-	addr3, _ := alloc.Allocate(container3, subnet, returnFalse)
-	require.Equal(t, testAddr1, addr3.String(), "address")
+	addr3, _ := alloc.Allocate(container3, subnet, true, returnFalse)
+	require.Equal(t, testAddr1, address.MakeCIDR(subnet, addr3).String(), "address")
 	// one more claim should still work
-	err = alloc.Claim(container3, addr1, true)
+	err = alloc.Claim(container3, addr1, true, true)
 	require.NoError(t, err)
 	// claim for a different container should fail
-	err = alloc.Claim(container1, addr1, true)
+	err = alloc.Claim(container1, addr1, true, true)
 	require.Error(t, err)
 	// claiming the address allocated on the other peer should fail
-	err = alloc.Claim(container1, addrx, true)
+	err = alloc.Claim(container1, address.MakeCIDR(subnet, addrx), true, true)
 	require.Error(t, err, "claiming address allocated on other peer should fail")
 	// Check an address outside of our universe
-	addr2, _ := address.ParseIP(testAddr2)
-	err = alloc.Claim(container1, addr2, true)
+	addr2, _ := address.ParseCIDR(testAddr2)
+	err = alloc.Claim(container1, addr2, true, true)
 	require.NoError(t, err)
 }
 
@@ -187,16 +211,13 @@ func (alloc *Allocator) pause() func() {
 }
 
 func TestCancel(t *testing.T) {
-	const (
-		CIDR = "10.0.1.7/26"
-	)
-
+	const cidr = "10.0.1.7/26"
 	router := gossip.NewTestRouter(0.0)
 
-	alloc1, subnet := makeAllocator("01:00:00:02:00:00", CIDR, 2)
+	alloc1, subnet := makeAllocator("01:00:00:02:00:00", cidr, 2)
 	alloc1.SetInterfaces(router.Connect(alloc1.ourName, alloc1))
 
-	alloc2, _ := makeAllocator("02:00:00:02:00:00", CIDR, 2)
+	alloc2, _ := makeAllocator("02:00:00:02:00:00", cidr, 2)
 	alloc2.SetInterfaces(router.Connect(alloc2.ourName, alloc2))
 	alloc1.claimRingForTesting(alloc1, alloc2)
 	alloc2.claimRingForTesting(alloc1, alloc2)
@@ -208,9 +229,9 @@ func TestCancel(t *testing.T) {
 	alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode())
 
 	// Get some IPs, so each allocator has some space
-	res1, _ := alloc1.Allocate("foo", subnet, returnFalse)
+	res1, _ := alloc1.Allocate("foo", subnet, true, returnFalse)
 	common.Log.Debugf("res1 = %s", res1.String())
-	res2, _ := alloc2.Allocate("bar", subnet, returnFalse)
+	res2, _ := alloc2.Allocate("bar", subnet, true, returnFalse)
 	common.Log.Debugf("res2 = %s", res2.String())
 	if res1 == res2 {
 		require.FailNow(t, "Error: got same ips!")
@@ -221,13 +242,13 @@ func TestCancel(t *testing.T) {
 	unpause := alloc2.pause()
 
 	// Use up all the IPs that alloc1 owns, so the allocation after this will prompt a request to alloc2
-	for i := 0; alloc1.NumFreeAddresses(subnet) > 0; i++ {
-		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, returnFalse)
+	for i := 0; alloc1.NumFreeAddresses(subnet.HostRange()) > 0; i++ {
+		alloc1.Allocate(fmt.Sprintf("tmp%d", i), subnet, true, returnFalse)
 	}
 	cancelChan := make(chan bool, 1)
 	doneChan := make(chan bool)
 	go func() {
-		_, ok := alloc1.Allocate("baz", subnet,
+		_, ok := alloc1.Allocate("baz", subnet, true,
 			func() bool {
 				select {
 				case <-cancelChan:
@@ -251,18 +272,18 @@ func TestCancel(t *testing.T) {
 
 func TestCancelOnDied(t *testing.T) {
 	const (
-		CIDR       = "10.0.1.7/26"
+		cidr       = "10.0.1.7/26"
 		container1 = "abcdef"
 	)
 
 	router := gossip.NewTestRouter(0.0)
-	alloc1, subnet := makeAllocator("01:00:00:02:00:00", CIDR, 2)
+	alloc1, subnet := makeAllocator("01:00:00:02:00:00", cidr, 2)
 	alloc1.SetInterfaces(router.Connect(alloc1.ourName, alloc1))
 	alloc1.Start()
 
 	doneChan := make(chan bool)
 	f := func() {
-		_, ok := alloc1.Allocate(container1, subnet, returnFalse)
+		_, ok := alloc1.Allocate(container1, subnet, true, returnFalse)
 		doneChan <- ok == nil
 	}
 
@@ -293,62 +314,65 @@ func TestGossipShutdown(t *testing.T) {
 	defer alloc.Stop()
 
 	alloc.claimRingForTesting()
-	alloc.Allocate(container1, subnet, returnFalse)
+	alloc.Allocate(container1, subnet, true, returnFalse)
 
 	alloc.Shutdown()
 
-	_, err := alloc.Allocate(container2, subnet, returnFalse) // trying to allocate after shutdown should fail
+	_, err := alloc.Allocate(container2, subnet, true, returnFalse) // trying to allocate after shutdown should fail
 	require.False(t, err == nil, "no address")
 
 	CheckAllExpectedMessagesSent(alloc)
 }
 
 func TestTransfer(t *testing.T) {
-	const (
-		cidr = "10.0.1.7/22"
-	)
+	const cidr = "10.0.1.7/22"
 	allocs, router, subnet := makeNetworkOfAllocators(3, cidr)
-	alloc1 := allocs[0]
-	alloc2 := allocs[1]
-	alloc3 := allocs[2] // This will be 'master' and get the first range
+	defer stopNetworkOfAllocators(allocs, router)
+	alloc0 := allocs[0]
+	alloc1 := allocs[1]
+	alloc2 := allocs[2]
 
-	_, err := alloc2.Allocate("foo", subnet, returnFalse)
+	_, err := alloc1.Allocate("foo", subnet, true, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 
-	_, err = alloc3.Allocate("bar", subnet, returnFalse)
+	_, err = alloc2.Allocate("bar", subnet, true, returnFalse)
 	require.True(t, err == nil, "Failed to get address")
 
+	// simulation of periodic gossip
 	alloc2.gossip.GossipBroadcast(alloc2.Gossip())
 	router.Flush()
-	alloc2.gossip.GossipBroadcast(alloc3.Gossip())
+	alloc1.gossip.GossipBroadcast(alloc1.Gossip())
 	router.Flush()
+
+	free1 := alloc1.NumFreeAddresses(subnet.Range())
+	free2 := alloc2.NumFreeAddresses(subnet.Range())
+
+	router.RemovePeer(alloc1.ourName)
 	router.RemovePeer(alloc2.ourName)
-	router.RemovePeer(alloc3.ourName)
-	alloc2.Stop()
-	alloc3.Stop()
-	router.Flush()
-	require.NoError(t, alloc1.AdminTakeoverRanges(alloc2.ourName.String()))
-	require.NoError(t, alloc1.AdminTakeoverRanges(alloc3.ourName.String()))
-	router.Flush()
-
-	require.Equal(t, address.Offset(1022), alloc1.NumFreeAddresses(subnet))
-
-	_, err = alloc1.Allocate("foo", subnet, returnFalse)
-	require.True(t, err == nil, "Failed to get address")
 	alloc1.Stop()
+	alloc2.Stop()
+	router.Flush()
+
+	require.Equal(t, free1+1, alloc0.AdminTakeoverRanges(alloc1.ourName.String()))
+	require.Equal(t, free2+1, alloc0.AdminTakeoverRanges(alloc2.ourName.String()))
+	router.Flush()
+
+	require.Equal(t, address.Count(1024), alloc0.NumFreeAddresses(subnet.Range()))
+
+	_, err = alloc0.Allocate("foo", subnet, true, returnFalse)
+	require.True(t, err == nil, "Failed to get address")
+	alloc0.Stop()
 }
 
 func TestFakeRouterSimple(t *testing.T) {
-	const (
-		cidr = "10.0.1.7/22"
-	)
-	allocs, _, subnet := makeNetworkOfAllocators(2, cidr)
-	defer stopNetworkOfAllocators(allocs)
+	const cidr = "10.0.1.7/22"
+	allocs, router, subnet := makeNetworkOfAllocators(2, cidr)
+	defer stopNetworkOfAllocators(allocs, router)
 
 	alloc1 := allocs[0]
 	//alloc2 := allocs[1]
 
-	_, err := alloc1.Allocate("foo", subnet, returnFalse)
+	_, err := alloc1.Allocate("foo", subnet, true, returnFalse)
 	require.NoError(t, err, "Failed to get address")
 }
 
@@ -361,8 +385,8 @@ func TestAllocatorFuzz(t *testing.T) {
 		concurrency  = 5
 		cidr         = "10.0.1.7/22"
 	)
-	allocs, _, subnet := makeNetworkOfAllocators(nodes, cidr)
-	defer stopNetworkOfAllocators(allocs)
+	allocs, router, subnet := makeNetworkOfAllocators(nodes, cidr)
+	defer stopNetworkOfAllocators(allocs, router)
 
 	// Test state
 	// For each IP issued we store the allocator
@@ -429,7 +453,7 @@ func TestAllocatorFuzz(t *testing.T) {
 		allocIndex := rand.Int31n(nodes)
 		alloc := allocs[allocIndex]
 		//common.Log.Infof("Allocate: asking allocator %d", allocIndex)
-		addr, err := alloc.Allocate(name, subnet, returnFalse)
+		addr, err := alloc.Allocate(name, subnet, true, returnFalse)
 
 		if err != nil {
 			panic(fmt.Sprintf("Could not allocate addr"))
@@ -486,7 +510,7 @@ func TestAllocatorFuzz(t *testing.T) {
 
 		//common.Log.Infof("Asking for %s (%s) on allocator %d again", res.name, addr, res.alloc)
 
-		newAddr, _ := alloc.Allocate(res.name, subnet, returnFalse)
+		newAddr, _ := alloc.Allocate(res.name, subnet, true, returnFalse)
 		oldAddr, _ := address.ParseIP(addr)
 		if newAddr != oldAddr {
 			panic(fmt.Sprintf("Got different address for repeat request for %s: %s != %s", res.name, newAddr, oldAddr))
@@ -506,8 +530,8 @@ func TestAllocatorFuzz(t *testing.T) {
 		allocIndex := rand.Int31n(nodes)
 		addressIndex := rand.Int31n(int32(subnet.Size()))
 		alloc := allocs[allocIndex]
-		addr := address.Add(subnet.Start, address.Offset(addressIndex))
-		err := alloc.Claim(name, addr, true)
+		addr := address.Add(subnet.Addr, address.Offset(addressIndex))
+		err := alloc.Claim(name, address.MakeCIDR(subnet, addr), true, true)
 		if err == nil {
 			noteAllocation(allocIndex, name, addr)
 		}
@@ -563,14 +587,36 @@ func TestAllocatorFuzz(t *testing.T) {
 	})
 }
 
-func TestGossipSkew(t *testing.T) {
-	alloc1, _ := makeAllocatorWithMockGossip(t, "01:00:00:01:00:00", "10.0.1.0/22", 2)
-	defer alloc1.Stop()
-	alloc2, _ := makeAllocatorWithMockGossip(t, "02:00:00:02:00:00", "10.0.1.0/22", 2)
-	alloc2.now = func() time.Time { return time.Now().Add(time.Hour * 2) }
-	defer alloc2.Stop()
+func TestSpaceRequest(t *testing.T) {
+	const (
+		container1 = "cont-1"
+		container2 = "cont-2"
+		universe   = "10.32.0.0/12"
+	)
+	allocs, router, subnet := makeNetworkOfAllocators(1, universe)
+	defer stopNetworkOfAllocators(allocs, router)
+	alloc1 := allocs[0]
 
-	if _, err := alloc1.OnGossipBroadcast(alloc2.ourName, alloc2.Encode()); err == nil {
-		t.Fail()
-	}
+	addr, err := alloc1.Allocate(container1, subnet, true, returnFalse)
+	require.Nil(t, err, "")
+	// free it again so the donation splits the range neatly
+	err = alloc1.Free(container1, addr)
+	require.Nil(t, err, "")
+	require.Equal(t, cidrRanges(universe), alloc1.OwnedRanges(), "")
+
+	// Start a new peer
+	alloc2, _ := makeAllocator("02:00:00:02:00:00", universe, 2)
+	alloc2.SetInterfaces(router.Connect(alloc2.ourName, alloc2))
+	alloc2.Start()
+	defer alloc2.Stop()
+	alloc2.Allocate(container2, subnet, true, returnFalse)
+
+	// Test whether the universe has been split into two equal halves (GH #2009)
+	require.Equal(t, cidrRanges("10.32.0.0/13"), alloc1.OwnedRanges(), "")
+	require.Equal(t, cidrRanges("10.40.0.0/13"), alloc2.OwnedRanges(), "")
+}
+
+func cidrRanges(s string) []address.Range {
+	c, _ := address.ParseCIDR(s)
+	return []address.Range{c.Range()}
 }

ipam/claim.go

@@ -3,15 +3,17 @@ package ipam
 import (
 	"fmt"
 
+	"github.com/weaveworks/mesh"
+
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 type claim struct {
 	resultChan       chan<- error
 	ident            string
-	addr             address.Address
+	cidr             address.CIDR
+	isContainer      bool
 	noErrorOnUnknown bool
 }
 
@@ -31,33 +33,37 @@ func (c *claim) sendResult(result error) {
 
 // Try returns true for success (or failure), false if we need to try again later
 func (c *claim) Try(alloc *Allocator) bool {
-	if !alloc.ring.Contains(c.addr) {
+	if !alloc.ring.Contains(c.cidr.Addr) {
 		// Address not within our universe; assume user knows what they are doing
-		alloc.infof("Ignored address %s claimed by %s - not in our universe", c.addr, c.ident)
+		alloc.infof("Address %s claimed by %s - not in our range", c.cidr, c.ident)
+		alloc.addOwned(c.ident, c.cidr, c.isContainer)
 		c.sendResult(nil)
 		return true
 	}
 
 	alloc.establishRing()
 
-	switch owner := alloc.ring.Owner(c.addr); owner {
+	// If we had heard that this container died, resurrect it
+	delete(alloc.dead, c.ident) // (delete is no-op if key not in map)
+
+	switch owner := alloc.ring.Owner(c.cidr.Addr); owner {
 	case alloc.ourName:
 		// success
-	case router.UnknownPeerName:
+	case mesh.UnknownPeerName:
 		// If our ring doesn't know, it must be empty.
 		if c.noErrorOnUnknown {
-			alloc.infof("Claim %s for %s: address allocator still initializing; will try later.", c.addr, c.ident)
+			alloc.infof("Claim %s for %s: address allocator still initializing; will try later.", c.cidr, c.ident)
 			c.sendResult(nil)
 		} else {
-			c.sendResult(fmt.Errorf("%s is in the range %s, but the allocator is not initialized yet", c.addr, alloc.universe.AsCIDRString()))
+			c.sendResult(fmt.Errorf("%s is in the range %s, but the allocator is not initialized yet", c.cidr, alloc.universe.AsCIDRString()))
 		}
 		return false
 	default:
-		alloc.debugf("requesting address %s from other peer %s", c.addr, owner)
-		err := alloc.sendSpaceRequest(owner, address.NewRange(c.addr, 1))
+		alloc.debugf("requesting address %s from other peer %s", c.cidr, owner)
+		err := alloc.sendSpaceRequest(owner, address.NewRange(c.cidr.Addr, 1))
 		if err != nil { // can't speak to owner right now
 			if c.noErrorOnUnknown {
-				alloc.infof("Claim %s for %s: %s; will try later.", c.addr, c.ident, err)
+				alloc.infof("Claim %s for %s: %s; will try later.", c.cidr, c.ident, err)
 				c.sendResult(nil)
 			} else { // just tell the user they can't do this.
 				c.deniedBy(alloc, owner)
@@ -67,31 +73,38 @@ func (c *claim) Try(alloc *Allocator) bool {
 	}
 
 	// We are the owner, check we haven't given it to another container
-	switch existingIdent := alloc.findOwner(c.addr); existingIdent {
+	switch existingIdent := alloc.findOwner(c.cidr.Addr); existingIdent {
 	case "":
-		if err := alloc.space.Claim(c.addr); err == nil {
-			alloc.debugln("Claimed", c.addr, "for", c.ident)
-			alloc.addOwned(c.ident, c.addr)
+		if err := alloc.space.Claim(c.cidr.Addr); err == nil {
+			alloc.debugln("Claimed", c.cidr, "for", c.ident)
+			if c.ident == "_" { // Special "I don't have a unique ID" identifier
+				c.ident = c.cidr.String()
+			}
+			alloc.addOwned(c.ident, c.cidr, c.isContainer)
 			c.sendResult(nil)
 		} else {
 			c.sendResult(err)
 		}
 	case c.ident:
 		// same identifier is claiming same address; that's OK
+		alloc.debugln("Re-Claimed", c.cidr, "for", c.ident)
 		c.sendResult(nil)
+	case c.cidr.String():
+		// Address already allocated via "_" name
+		c.sendResult(fmt.Errorf("address %s already in use", c.cidr))
 	default:
 		// Addr already owned by container on this machine
-		c.sendResult(fmt.Errorf("address %s is already owned by %s", c.addr.String(), existingIdent))
+		c.sendResult(fmt.Errorf("address %s is already owned by %s", c.cidr.String(), existingIdent))
 	}
 	return true
 }
 
-func (c *claim) deniedBy(alloc *Allocator, owner router.PeerName) {
+func (c *claim) deniedBy(alloc *Allocator, owner mesh.PeerName) {
 	name, found := alloc.nicknames[owner]
 	if found {
 		name = " (" + name + ")"
 	}
-	c.sendResult(fmt.Errorf("address %s is owned by other peer %s%s", c.addr.String(), owner, name))
+	c.sendResult(fmt.Errorf("address %s is owned by other peer %s%s", c.cidr.String(), owner, name))
 }
 
 func (c *claim) Cancel() {

ipam/http.go

@@ -16,22 +16,31 @@ func badRequest(w http.ResponseWriter, err error) {
 	common.Log.Warningln("[allocator]:", err.Error())
 }
 
-func parseCIDR(w http.ResponseWriter, cidrStr string) (address.CIDR, bool) {
-	subnetAddr, cidr, err := address.ParseCIDR(cidrStr)
+func parseCIDR(w http.ResponseWriter, cidrStr string, net bool) (address.CIDR, bool) {
+	cidr, err := address.ParseCIDR(cidrStr)
 	if err != nil {
 		badRequest(w, err)
 		return address.CIDR{}, false
 	}
-	if cidr.Start != subnetAddr {
+	if net && !cidr.IsSubnet() {
 		badRequest(w, fmt.Errorf("Invalid subnet %s - bits after network prefix are not all zero", cidrStr))
 		return address.CIDR{}, false
 	}
 	return cidr, true
 }
 
+func writeAddresses(w http.ResponseWriter, cidrs []address.CIDR) {
+	for i, cidr := range cidrs {
+		fmt.Fprint(w, cidr)
+		if i < len(cidrs)-1 {
+			w.Write([]byte{' '})
+		}
+	}
+}
+
 func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.ResponseWriter, ident string, checkAlive bool, subnet address.CIDR) {
 	closedChan := w.(http.CloseNotifier).CloseNotify()
-	addr, err := alloc.Allocate(ident, subnet.HostRange(),
+	addr, err := alloc.Allocate(ident, subnet, checkAlive,
 		func() bool {
 			select {
 			case <-closedChan:
@@ -57,46 +66,51 @@ func (alloc *Allocator) handleHTTPAllocate(dockerCli *docker.Client, w http.Resp
 
 // HandleHTTP wires up ipams HTTP endpoints to the provided mux.
 func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CIDR, dockerCli *docker.Client) {
-	router.Methods("PUT").Path("/ip/{id}/{ip}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		vars := mux.Vars(r)
-		ident := vars["id"]
-		ipStr := vars["ip"]
-		noErrorOnUnknown := r.FormValue("noErrorOnUnknown") == "true"
-		if ip, err := address.ParseIP(ipStr); err != nil {
-			badRequest(w, err)
-			return
-		} else if err := alloc.Claim(ident, ip, noErrorOnUnknown); err != nil {
-			badRequest(w, fmt.Errorf("Unable to claim: %s", err))
-			return
-		}
+	router.Methods("GET").Path("/ipinfo/defaultsubnet").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "%s", defaultSubnet)
+	})
 
-		w.WriteHeader(204)
+	router.Methods("PUT").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+		if cidr, ok := parseCIDR(w, vars["ip"]+"/"+vars["prefixlen"], false); ok {
+			noErrorOnUnknown := r.FormValue("noErrorOnUnknown") == "true"
+			checkAlive := r.FormValue("check-alive") == "true"
+			if err := alloc.Claim(vars["id"], cidr, checkAlive, noErrorOnUnknown); err != nil {
+				badRequest(w, fmt.Errorf("Unable to claim: %s", err))
+				return
+			}
+			w.WriteHeader(204)
+		}
+	})
+
+	router.Methods("GET").Path("/ring").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		alloc.Prime()
 	})
 
 	router.Methods("GET").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
-		if subnet, ok := parseCIDR(w, vars["ip"]+"/"+vars["prefixlen"]); ok {
-			addr, err := alloc.Lookup(vars["id"], subnet.HostRange())
+		if subnet, ok := parseCIDR(w, vars["ip"]+"/"+vars["prefixlen"], true); ok {
+			cidrs, err := alloc.Lookup(vars["id"], subnet.HostRange())
 			if err != nil {
 				http.NotFound(w, r)
 				return
 			}
-			fmt.Fprintf(w, "%s/%d", addr, subnet.PrefixLen)
+			writeAddresses(w, cidrs)
 		}
 	})
 
 	router.Methods("GET").Path("/ip/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		addr, err := alloc.Lookup(mux.Vars(r)["id"], defaultSubnet.HostRange())
+		addrs, err := alloc.Lookup(mux.Vars(r)["id"], defaultSubnet.HostRange())
 		if err != nil {
 			http.NotFound(w, r)
 			return
 		}
-		fmt.Fprintf(w, "%s/%d", addr, defaultSubnet.PrefixLen)
+		writeAddresses(w, addrs)
 	})
 
 	router.Methods("POST").Path("/ip/{id}/{ip}/{prefixlen}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		vars := mux.Vars(r)
-		if subnet, ok := parseCIDR(w, vars["ip"]+"/"+vars["prefixlen"]); ok {
+		if subnet, ok := parseCIDR(w, vars["ip"]+"/"+vars["prefixlen"], true); ok {
 			alloc.handleHTTPAllocate(dockerCli, w, vars["id"], r.FormValue("check-alive") == "true", subnet)
 		}
 	})
@@ -138,11 +152,7 @@ func (alloc *Allocator) HandleHTTP(router *mux.Router, defaultSubnet address.CID
 
 	router.Methods("DELETE").Path("/peer/{id}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ident := mux.Vars(r)["id"]
-		if err := alloc.AdminTakeoverRanges(ident); err != nil {
-			badRequest(w, err)
-			return
-		}
-
-		w.WriteHeader(204)
+		transferred := alloc.AdminTakeoverRanges(ident)
+		fmt.Fprintf(w, "%d IPs taken over from %s\n", transferred, ident)
 	})
 }

ipam/http_test.go

@@ -14,7 +14,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
-	wt "github.com/weaveworks/weave/testing"
 )
 
 func HTTPPost(t *testing.T, url string) string {
@@ -81,7 +80,7 @@ func TestHttp(t *testing.T) {
 	)
 
 	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", universe, 1)
-	_, cidr, _ := address.ParseCIDR(universe)
+	cidr, _ := address.ParseCIDR(universe)
 	port := listenHTTP(alloc, cidr)
 	alloc.claimRingForTesting()
 
@@ -123,7 +122,7 @@ func TestBadHttp(t *testing.T) {
 
 	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
 	defer alloc.Stop()
-	_, cidr, _ := address.ParseCIDR(testCIDR1)
+	cidr, _ := address.ParseCIDR(testCIDR1)
 	port := listenHTTP(alloc, cidr)
 
 	alloc.claimRingForTesting()
@@ -145,40 +144,30 @@ func TestBadHttp(t *testing.T) {
 }
 
 func TestHTTPCancel(t *testing.T) {
-	wt.RunWithTimeout(t, 2*time.Second, func() {
-		impTestHTTPCancel(t)
-	})
-}
-
-func impTestHTTPCancel(t *testing.T) {
 	var (
 		containerID = "deadbeef"
 		testCIDR1   = "10.0.3.0/29"
 	)
 
-	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 1)
+	// Say quorum=2, so the allocate won't go ahead
+	alloc, _ := makeAllocatorWithMockGossip(t, "08:00:27:01:c3:9a", testCIDR1, 2)
 	defer alloc.Stop()
-	alloc.claimRingForTesting()
-	_, cidr, _ := address.ParseCIDR(testCIDR1)
+	ExpectBroadcastMessage(alloc, nil) // trying to form consensus
+	cidr, _ := address.ParseCIDR(testCIDR1)
 	port := listenHTTP(alloc, cidr)
 
-	// Stop the alloc so nothing actually works
-	unpause := alloc.pause()
-
 	// Ask the http server for a new address
-	done := make(chan *http.Response)
 	req, _ := http.NewRequest("POST", allocURL(port, testCIDR1, containerID), nil)
+	// On another goroutine, wait for a bit then cancel the request
 	go func() {
-		res, _ := http.DefaultClient.Do(req)
-		done <- res
+		time.Sleep(100 * time.Millisecond)
+		common.Log.Debug("Cancelling allocate")
+		http.DefaultTransport.(*http.Transport).CancelRequest(req)
 	}()
 
-	time.Sleep(100 * time.Millisecond)
-	fmt.Println("Cancelling allocate")
-	http.DefaultTransport.(*http.Transport).CancelRequest(req)
-	unpause()
-	res := <-done
+	res, _ := http.DefaultClient.Do(req)
 	if res != nil {
-		require.FailNow(t, "Error: Allocate returned non-nil")
+		body, _ := ioutil.ReadAll(res.Body)
+		require.FailNow(t, "Error: Allocate returned non-nil", string(body))
 	}
 }

ipam/paxos/participant.go

@@ -0,0 +1,44 @@
+package paxos
+
+type Participant interface {
+	GossipState() GossipState
+	Update(from GossipState) bool
+	SetQuorum(uint)
+	Propose()
+	Think() bool
+	Consensus() (bool, AcceptedValue)
+	IsElector() bool
+}
+
+type Observer struct {
+}
+
+func NewObserver() Participant {
+	return &Observer{}
+}
+
+func (observer *Observer) GossipState() GossipState {
+	return nil
+}
+
+func (observer *Observer) Update(from GossipState) bool {
+	return false
+}
+
+func (observer *Observer) Propose() {
+}
+
+func (observer *Observer) SetQuorum(uint) {
+}
+
+func (observer *Observer) Think() bool {
+	return false
+}
+
+func (observer *Observer) Consensus() (bool, AcceptedValue) {
+	return false, AcceptedValue{}
+}
+
+func (observer *Observer) IsElector() bool {
+	return false
+}

ipam/paxos/paxos.go

@@ -1,7 +1,7 @@
 package paxos
 
 import (
-	"github.com/weaveworks/weave/router"
+	"github.com/weaveworks/mesh"
 )
 
 // The node identifier.  The use of the UID here is important: Paxos
@@ -9,8 +9,8 @@ import (
 // node does not restart and lose its Paxos state but claim to have
 // the same ID.
 type NodeID struct {
-	Name router.PeerName
-	UID  router.PeerUID
+	Name mesh.PeerName
+	UID  mesh.PeerUID
 }
 
 // note all fields exported in structs so we can Gob them
@@ -39,7 +39,7 @@ func (a ProposalID) valid() bool {
 }
 
 // For seeding IPAM, the value we want consensus on is a set of peer names
-type Value []router.PeerName
+type Value []mesh.PeerName
 
 // An AcceptedValue is a Value plus the proposal which originated that
 // Value.  The origin is not essential, but makes comparing
@@ -72,7 +72,7 @@ type Node struct {
 	knows  GossipState
 }
 
-func NewNode(name router.PeerName, uid router.PeerUID, quorum uint) *Node {
+func NewNode(name mesh.PeerName, uid mesh.PeerUID, quorum uint) *Node {
 	return &Node{
 		id:     NodeID{name, uid},
 		quorum: quorum,
@@ -80,6 +80,10 @@ func NewNode(name router.PeerName, uid router.PeerUID, quorum uint) *Node {
 	}
 }
 
+func (node *Node) SetQuorum(quorum uint) {
+	node.quorum = quorum
+}
+
 func (node *Node) GossipState() GossipState {
 	return node.knows
 }
@@ -124,6 +128,9 @@ func max(a uint, b uint) uint {
 // simply a matter of gossipping a new proposal that supersedes all
 // others.
 func (node *Node) Propose() {
+	if node.quorum == 0 {
+		panic("Paxos node.Propose() called with no quorum set")
+	}
 	// Find the highest round number around
 	round := uint(0)
 
@@ -223,7 +230,7 @@ func (node *Node) Think() bool {
 // about.  This is not necessarily all peer names, but it is at least
 // a quorum, and so good enough for seeding the ring.
 func (node *Node) pickValue() Value {
-	val := make([]router.PeerName, len(node.knows))
+	val := make([]mesh.PeerName, len(node.knows))
 	i := 0
 	for id := range node.knows {
 		val[i] = id.Name
@@ -234,6 +241,9 @@ func (node *Node) pickValue() Value {
 
 // Has a consensus been reached, based on the known claims of other nodes?
 func (node *Node) Consensus() (bool, AcceptedValue) {
+	if node.quorum == 0 {
+		return false, AcceptedValue{}
+	}
 	counts := map[ProposalID]uint{}
 
 	for _, claims := range node.knows {
@@ -250,14 +260,16 @@ func (node *Node) Consensus() (bool, AcceptedValue) {
 	return false, AcceptedValue{}
 }
 
+func (node *Node) IsElector() bool {
+	return true
+}
+
 type Status struct {
+	Elector    bool
 	KnownNodes int
 	Quorum     uint
 }
 
 func NewStatus(node *Node) *Status {
-	if node == nil {
-		return nil
-	}
-	return &Status{len(node.knows), node.quorum}
+	return &Status{true, len(node.knows), node.quorum}
 }

ipam/paxos/paxos_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/weaveworks/weave/router"
+	"github.com/weaveworks/mesh"
 )
 
 type TestNode struct {
@@ -97,8 +97,8 @@ func makeRandomModel(params *TestParams, r *rand.Rand, t *testing.T) *Model {
 	}
 
 	for i := range m.nodes {
-		m.nodes[i].Node = NewNode(router.PeerName(i/2+1),
-			router.PeerUID(r.Int63()), m.quorum)
+		m.nodes[i].Node = NewNode(mesh.PeerName(i/2+1),
+			mesh.PeerUID(r.Int63()), m.quorum)
 		m.nodes[i].Propose()
 	}
 
@@ -172,8 +172,8 @@ func (m *Model) isolateNode(node *TestNode) {
 
 // Restart a node
 func (m *Model) restart(node *TestNode) {
-	node.Node = NewNode(router.PeerName(m.nextID),
-		router.PeerUID(m.r.Int63()), m.quorum)
+	node.Node = NewNode(mesh.PeerName(m.nextID),
+		mesh.PeerUID(m.r.Int63()), m.quorum)
 	m.nextID++
 	node.Propose()
 
@@ -300,14 +300,14 @@ func (m *Model) validate() {
 	for i := range m.nodes {
 		ok, val := m.nodes[i].Consensus()
 		if !ok {
-			//m.dump()
+			m.dump()
 			m.t.Fatal("Node doesn't know about consensus")
 		}
 
 		firstConsensus := m.nodes[i].firstConsensus
 		if firstConsensus.Origin.valid() &&
 			firstConsensus.Origin != val.Origin {
-			//m.dump()
+			m.dump()
 			m.t.Fatal("Consensus mismatch")
 		}
 

ipam/prime.go

@@ -0,0 +1,24 @@
+package ipam
+
+type prime struct {
+	resultChan chan<- struct{}
+}
+
+func (c *prime) Try(alloc *Allocator) bool {
+	if !alloc.ring.Empty() {
+		close(c.resultChan)
+		return true
+	}
+
+	alloc.establishRing()
+
+	return false
+}
+
+func (c *prime) Cancel() {
+	close(c.resultChan)
+}
+
+func (c *prime) ForContainer(ident string) bool {
+	return false
+}

ipam/ring/entry.go

@@ -3,17 +3,19 @@ package ring
 import (
 	"sort"
 
+	"github.com/weaveworks/mesh"
+
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 // Entry represents entries around the ring
 type entry struct {
 	Token   address.Address // The start of this range
-	Peer    router.PeerName // Who owns this range
+	Peer    mesh.PeerName   // Who owns this range
 	Version uint32          // Version of this range
 	Free    address.Offset  // Number of free IPs in this range
+	// Note: Free should perhaps be an address.Count, but we can't change the wire protocol
 }
 
 func (e *entry) Equal(e2 *entry) bool {
@@ -21,7 +23,7 @@ func (e *entry) Equal(e2 *entry) bool {
 		e.Version == e2.Version
 }
 
-func (e *entry) update(peername router.PeerName, free address.Offset) {
+func (e *entry) update(peername mesh.PeerName, free address.Offset) {
 	e.Peer = peername
 	e.Version++
 	e.Free = free

ipam/ring/ring.go

@@ -12,17 +12,18 @@ import (
 	"sort"
 	"time"
 
+	"github.com/weaveworks/mesh"
+
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 // Ring represents the ring itself
 type Ring struct {
-	Start, End address.Address   // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
-	Peer       router.PeerName   // name of peer owning this ring instance
-	Entries    entries           // list of entries sorted by token
-	Seeds      []router.PeerName // peers with which the ring was seeded
+	Start, End address.Address // [min, max) tokens in this ring.  Due to wrapping, min == max (effectively)
+	Peer       mesh.PeerName   // name of peer owning this ring instance
+	Entries    entries         // list of entries sorted by token
+	Seeds      []mesh.PeerName // peers with which the ring was seeded
 }
 
 func (r *Ring) assertInvariants() {
@@ -39,40 +40,51 @@ var (
 	ErrTokenOutOfRange = errors.New("Token is out of range")
 	ErrDifferentSeeds  = errors.New("Received ring was seeded differently from ours!")
 	ErrDifferentRange  = errors.New("Received range differs from ours!")
-	ErrNewerVersion    = errors.New("Received new version for entry I own!")
-	ErrInvalidEntry    = errors.New("Received invalid state update!")
-	ErrEntryInMyRange  = errors.New("Received new entry in my range!")
 	ErrNotFound        = errors.New("No entries for peer found")
 )
 
+func errInconsistentEntry(mine, theirs *entry) error {
+	return fmt.Errorf("Inconsistent entries for %s: owned by %s but incoming message says %s", mine.Token, mine.Peer, theirs.Peer)
+}
+func errEntryInMyRange(theirs *entry) error {
+	return fmt.Errorf("Peer %s says it owns the IP range from %s, which I think I own", theirs.Peer, theirs.Token)
+}
+func errNewerVersion(mine, theirs *entry) error {
+	return fmt.Errorf("Received update for IP range I own at %s v%d: incoming message says owner %s v%d", mine.Token, mine.Version, theirs.Peer, theirs.Version)
+}
+
 func (r *Ring) checkInvariants() error {
-	if !sort.IsSorted(r.Entries) {
+	return r.checkEntries(r.Entries)
+}
+
+func (r *Ring) checkEntries(entries entries) error {
+	if !sort.IsSorted(entries) {
 		return ErrNotSorted
 	}
 
 	// Check no token appears twice
 	// We know it's sorted...
-	for i := 1; i < len(r.Entries); i++ {
-		if r.Entries[i-1].Token == r.Entries[i].Token {
+	for i := 1; i < len(entries); i++ {
+		if entries[i-1].Token == entries[i].Token {
 			return ErrTokenRepeated
 		}
 	}
 
-	if len(r.Entries) == 0 {
+	if len(entries) == 0 {
 		return nil
 	}
 
 	// Check tokens are in range
-	if r.Entries.entry(0).Token < r.Start {
+	if entries.entry(0).Token < r.Start {
 		return ErrTokenOutOfRange
 	}
-	if r.Entries.entry(-1).Token >= r.End {
+	if entries.entry(-1).Token >= r.End {
 		return ErrTokenOutOfRange
 	}
 
 	// Check all the freespaces are in range
-	for i, entry := range r.Entries {
-		next := r.Entries.entry(i + 1)
+	for i, entry := range entries {
+		next := entries.entry(i + 1)
 		distance := r.distance(entry.Token, next.Token)
 
 		if entry.Free > distance {
@@ -84,7 +96,7 @@ func (r *Ring) checkInvariants() error {
 }
 
 // New creates an empty ring belonging to peer.
-func New(start, end address.Address, peer router.PeerName) *Ring {
+func New(start, end address.Address, peer mesh.PeerName) *Ring {
 	common.Assert(start < end)
 
 	ring := &Ring{Start: start, End: end, Peer: peer, Entries: make([]*entry, 0)}
@@ -111,7 +123,7 @@ func (r *Ring) distance(start, end address.Address) address.Offset {
 // Preconditions:
 // - start < end
 // - [start, end) must be owned by the calling peer
-func (r *Ring) GrantRangeToHost(start, end address.Address, peer router.PeerName) {
+func (r *Ring) GrantRangeToHost(start, end address.Address, peer mesh.PeerName) {
 	//fmt.Printf("%s GrantRangeToHost [%v,%v) -> %s\n", r.Peer, start, end, peer)
 
 	r.assertInvariants()
@@ -181,7 +193,6 @@ func (r *Ring) GrantRangeToHost(start, end address.Address, peer router.PeerName
 // Merge the given ring into this ring and return any new ranges added
 func (r *Ring) Merge(gossip Ring) error {
 	r.assertInvariants()
-	defer r.assertInvariants()
 	defer r.updateExportedVariables()
 
 	// Don't panic when checking the gossiped in ring.
@@ -210,7 +221,7 @@ func (r *Ring) Merge(gossip Ring) error {
 	addToResult := func(e entry) { result = append(result, &e) }
 
 	var mine, theirs *entry
-	var previousOwner *router.PeerName
+	var previousOwner *mesh.PeerName
 	// i is index into r.Entries; j is index into gossip.Entries
 	var i, j int
 	for i < len(r.Entries) && j < len(gossip.Entries) {
@@ -223,7 +234,7 @@ func (r *Ring) Merge(gossip Ring) error {
 		case mine.Token > theirs.Token:
 			// insert, checking that a range owned by us hasn't been split
 			if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
-				return ErrEntryInMyRange
+				return errEntryInMyRange(theirs)
 			}
 			addToResult(*theirs)
 			previousOwner = nil
@@ -233,14 +244,13 @@ func (r *Ring) Merge(gossip Ring) error {
 			switch {
 			case mine.Version >= theirs.Version:
 				if mine.Version == theirs.Version && !mine.Equal(theirs) {
-					common.Log.Debugf("Error merging entries at %s - %v != %v", mine.Token, mine, theirs)
-					return ErrInvalidEntry
+					return errInconsistentEntry(mine, theirs)
 				}
 				addToResult(*mine)
 				previousOwner = &mine.Peer
 			case mine.Version < theirs.Version:
 				if mine.Peer == r.Peer { // We shouldn't receive updates to our own tokens
-					return ErrNewerVersion
+					return errNewerVersion(mine, theirs)
 				}
 				addToResult(*theirs)
 				previousOwner = nil
@@ -261,12 +271,16 @@ func (r *Ring) Merge(gossip Ring) error {
 	for ; j < len(gossip.Entries); j++ {
 		theirs = gossip.Entries[j]
 		if previousOwner != nil && *previousOwner == r.Peer && theirs.Peer != r.Peer {
-			return ErrEntryInMyRange
+			return errEntryInMyRange(theirs)
 		}
 		addToResult(*theirs)
 		previousOwner = nil
 	}
 
+	if err := r.checkEntries(result); err != nil {
+		return fmt.Errorf("Merge of incoming data causes: %s", err)
+	}
+
 	if len(r.Seeds) == 0 {
 		r.Seeds = gossip.Seeds
 	}
@@ -316,9 +330,28 @@ func (r *Ring) OwnedRanges() (result []address.Range) {
 	return r.splitRangesOverZero(result)
 }
 
+// For printing status
+type RangeInfo struct {
+	Peer mesh.PeerName
+	address.Range
+	Version uint32
+}
+
+func (r *Ring) AllRangeInfo() (result []RangeInfo) {
+	for i, entry := range r.Entries {
+		nextEntry := r.Entries.entry(i + 1)
+		ranges := []address.Range{{Start: entry.Token, End: nextEntry.Token}}
+		ranges = r.splitRangesOverZero(ranges)
+		for _, r := range ranges {
+			result = append(result, RangeInfo{entry.Peer, r, entry.Version})
+		}
+	}
+	return
+}
+
 // ClaimForPeers claims the entire ring for the array of peers passed
 // in.  Only works for empty rings.
-func (r *Ring) ClaimForPeers(peers []router.PeerName) {
+func (r *Ring) ClaimForPeers(peers []mesh.PeerName) {
 	common.Assert(r.Empty())
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
@@ -336,12 +369,7 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 			}
 		}
 
-		if e, found := r.Entries.get(pos); found {
-			e.update(peer, share)
-		} else {
-			r.Entries.insert(entry{Token: pos, Peer: peer, Free: share})
-		}
-
+		r.Entries.insert(entry{Token: pos, Peer: peer, Free: share})
 		pos += address.Address(share)
 	}
 
@@ -350,7 +378,7 @@ func (r *Ring) ClaimForPeers(peers []router.PeerName) {
 	r.Seeds = peers
 }
 
-func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
+func (r *Ring) FprintWithNicknames(w io.Writer, m map[mesh.PeerName]string) {
 	for _, entry := range r.Entries {
 		nickname, found := m[entry.Peer]
 		if found {
@@ -365,14 +393,14 @@ func (r *Ring) FprintWithNicknames(w io.Writer, m map[router.PeerName]string) {
 func (r *Ring) String() string {
 	var buffer bytes.Buffer
 	fmt.Fprintf(&buffer, "Ring [%s, %s)", r.Start, r.End)
-	r.FprintWithNicknames(&buffer, make(map[router.PeerName]string))
+	r.FprintWithNicknames(&buffer, make(map[mesh.PeerName]string))
 	return buffer.String()
 }
 
 // ReportFree is used by the allocator to tell the ring how many free
 // ips are in a given range, so that ChoosePeersToAskForSpace can make
 // more intelligent decisions.
-func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
+func (r *Ring) ReportFree(freespace map[address.Address]address.Count) {
 	r.assertInvariants()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
@@ -402,20 +430,20 @@ func (r *Ring) ReportFree(freespace map[address.Address]address.Offset) {
 		// Check we're not reporting more space than the range
 		entry, next := entries.entry(i), entries.entry(i+1)
 		maxSize := r.distance(entry.Token, next.Token)
-		common.Assert(free <= maxSize)
+		common.Assert(free <= address.Count(maxSize))
 
-		if entries[i].Free == free {
+		if address.Count(entries[i].Free) == free {
 			return
 		}
 
-		entries[i].Free = free
+		entries[i].Free = address.Offset(free)
 		entries[i].Version++
 	}
 }
 
 type weightedPeer struct {
 	weight   float64
-	peername router.PeerName
+	peername mesh.PeerName
 }
 type weightedPeers []weightedPeer
 
@@ -426,11 +454,8 @@ func (ws weightedPeers) Swap(i, j int)      { ws[i], ws[j] = ws[j], ws[i] }
 
 // ChoosePeersToAskForSpace returns all peers we can ask for space in
 // the range [start, end), in weighted-random order.  Assumes start<end.
-func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.PeerName {
-	var (
-		sum               address.Offset
-		totalSpacePerPeer = make(map[router.PeerName]address.Offset) // Compute total free space per peer
-	)
+func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []mesh.PeerName {
+	totalSpacePerPeer := make(map[mesh.PeerName]address.Offset)
 
 	// iterate through tokens
 	for i, entry := range r.Entries {
@@ -452,7 +477,6 @@ func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.Pee
 		}
 
 		totalSpacePerPeer[entry.Peer] += entry.Free
-		sum += entry.Free
 	}
 
 	// Compute weighted random numbers, then sort.
@@ -463,46 +487,40 @@ func (r *Ring) ChoosePeersToAskForSpace(start, end address.Address) []router.Pee
 		ws = append(ws, weightedPeer{weight: float64(space) * rand.Float64(), peername: peername})
 	}
 	sort.Sort(ws)
-	result := make([]router.PeerName, len(ws))
+	result := make([]mesh.PeerName, len(ws))
 	for i, wp := range ws {
 		result[i] = wp.peername
 	}
 	return result
 }
 
-func (r *Ring) PickPeerForTransfer(isValid func(router.PeerName) bool) router.PeerName {
+func (r *Ring) PickPeerForTransfer(isValid func(mesh.PeerName) bool) mesh.PeerName {
 	for _, entry := range r.Entries {
 		if entry.Peer != r.Peer && isValid(entry.Peer) {
 			return entry.Peer
 		}
 	}
-	return router.UnknownPeerName
+	return mesh.UnknownPeerName
 }
 
 // Transfer will mark all entries associated with 'from' peer as owned by 'to' peer
 // and return ranges indicating the new space we picked up
-func (r *Ring) Transfer(from, to router.PeerName) ([]address.Range, error) {
+func (r *Ring) Transfer(from, to mesh.PeerName) []address.Range {
 	r.assertInvariants()
 	defer r.assertInvariants()
 	defer r.updateExportedVariables()
 
 	var newRanges []address.Range
-	found := false
 
 	for i, entry := range r.Entries {
 		if entry.Peer == from {
-			found = true
 			entry.Peer = to
 			entry.Version++
 			newRanges = append(newRanges, address.Range{Start: entry.Token, End: r.Entries.entry(i + 1).Token})
 		}
 	}
 
-	if !found {
-		return nil, ErrNotFound
-	}
-
-	return r.splitRangesOverZero(newRanges), nil
+	return r.splitRangesOverZero(newRanges)
 }
 
 // Contains returns true if addr is in this ring
@@ -511,13 +529,13 @@ func (r *Ring) Contains(addr address.Address) bool {
 }
 
 // Owner returns the peername which owns the range containing addr
-func (r *Ring) Owner(token address.Address) router.PeerName {
+func (r *Ring) Owner(token address.Address) mesh.PeerName {
 	common.Assert(r.Start <= token && token < r.End)
 
 	r.assertInvariants()
 	// There can be no owners on an empty ring
 	if r.Empty() {
-		return router.UnknownPeerName
+		return mesh.UnknownPeerName
 	}
 
 	// Look for the right-most entry, less than or equal to token
@@ -530,8 +548,8 @@ func (r *Ring) Owner(token address.Address) router.PeerName {
 }
 
 // Get the set of PeerNames mentioned in the ring
-func (r *Ring) PeerNames() map[router.PeerName]struct{} {
-	res := make(map[router.PeerName]struct{})
+func (r *Ring) PeerNames() map[mesh.PeerName]struct{} {
+	res := make(map[mesh.PeerName]struct{})
 
 	for _, entry := range r.Entries {
 		res[entry.Peer] = struct{}{}

ipam/ring/ring_test.go

@@ -8,15 +8,16 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"github.com/weaveworks/mesh"
+
 	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 var (
-	peer1name, _ = router.PeerNameFromString("01:00:00:00:02:00")
-	peer2name, _ = router.PeerNameFromString("02:00:00:00:02:00")
-	peer3name, _ = router.PeerNameFromString("03:00:00:00:02:00")
+	peer1name, _ = mesh.PeerNameFromString("01:00:00:00:02:00")
+	peer2name, _ = mesh.PeerNameFromString("02:00:00:00:02:00")
+	peer3name, _ = mesh.PeerNameFromString("03:00:00:00:02:00")
 
 	start, end    = ParseIP("10.0.0.0"), ParseIP("10.0.0.255")
 	dot10, dot245 = ParseIP("10.0.0.10"), ParseIP("10.0.0.245")
@@ -59,13 +60,19 @@ func TestInsert(t *testing.T) {
 
 	ring.Entries.entry(0).Free = 0
 	ring.Entries.insert(entry{Token: dot245, Peer: peer1name})
-	ring2 := New(start, end, peer1name)
-	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 0}, {Token: dot245, Peer: peer1name}}
-	require.Equal(t, ring2, ring)
+	check := []RangeInfo{
+		{Peer: peer1name, Range: address.Range{Start: start, End: dot245}},
+		{Peer: peer1name, Range: address.Range{Start: dot245, End: end}},
+	}
+	require.Equal(t, check, ring.AllRangeInfo())
 
 	ring.Entries.insert(entry{Token: dot10, Peer: peer1name})
-	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Free: 0}, {Token: dot10, Peer: peer1name}, {Token: dot245, Peer: peer1name}}
-	require.Equal(t, ring2, ring)
+	check2 := []RangeInfo{
+		{Peer: peer1name, Range: address.Range{Start: start, End: dot10}},
+		{Peer: peer1name, Range: address.Range{Start: dot10, End: dot245}},
+		{Peer: peer1name, Range: address.Range{Start: dot245, End: end}},
+	}
+	require.Equal(t, check2, ring.AllRangeInfo())
 }
 
 func TestBetween(t *testing.T) {
@@ -218,17 +225,18 @@ func TestMergeErrors(t *testing.T) {
 	ring2 = New(start, end, peer2name)
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: start, Peer: peer1name, Version: 1}}
-	require.True(t, ring1.Merge(*ring2) == ErrNewerVersion, "Expected ErrNewerVersion")
+	fmt.Println(ring1.Merge(*ring2))
+	require.Error(t, ring1.Merge(*ring2), "Expected error")
 
 	// Cannot Merge two entries with same version but different hosts
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: start, Peer: peer2name}}
-	require.True(t, ring1.Merge(*ring2) == ErrInvalidEntry, "Expected ErrInvalidEntry")
+	require.Error(t, ring1.Merge(*ring2), "Expected error")
 
 	// Cannot Merge an entry into a range I own
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name}}
 	ring2.Entries = []*entry{{Token: middle, Peer: peer2name}}
-	require.True(t, ring1.Merge(*ring2) == ErrEntryInMyRange, "Expected ErrEntryInMyRange")
+	require.Error(t, ring1.Merge(*ring2), "Expected error")
 }
 
 func TestMergeMore(t *testing.T) {
@@ -351,7 +359,7 @@ func TestGossip(t *testing.T) {
 	assertRing(ring2, []*entry{{Token: start, Peer: peer1name, Free: 255}})
 }
 
-func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address, expected int) []router.PeerName {
+func assertPeersWithSpace(t *testing.T, ring *Ring, start, end address.Address, expected int) []mesh.PeerName {
 	peers := ring.ChoosePeersToAskForSpace(start, end)
 	require.Equal(t, expected, len(peers))
 	return peers
@@ -366,7 +374,7 @@ func TestFindFree(t *testing.T) {
 	assertPeersWithSpace(t, ring1, start, end, 0)
 
 	// We shouldn't return outselves
-	ring1.ReportFree(map[address.Address]address.Offset{start: 10})
+	ring1.ReportFree(map[address.Address]address.Count{start: 10})
 	assertPeersWithSpace(t, ring1, start, end, 0)
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer1name, Free: 1},
@@ -375,7 +383,7 @@ func TestFindFree(t *testing.T) {
 	ring1.assertInvariants()
 
 	// We should return others
-	var peers []router.PeerName
+	var peers []mesh.PeerName
 
 	ring1.Entries = []*entry{{Token: start, Peer: peer2name, Free: 1}}
 	peers = assertPeersWithSpace(t, ring1, start, end, 1)
@@ -401,7 +409,7 @@ func TestReportFree(t *testing.T) {
 	ring1.GrantRangeToHost(middle, end, peer2name)
 	require.NoError(t, ring2.Merge(*ring1))
 
-	freespace := make(map[address.Address]address.Offset)
+	freespace := make(map[address.Address]address.Count)
 	for _, r := range ring2.OwnedRanges() {
 		freespace[r.Start] = 0
 	}
@@ -493,7 +501,7 @@ func TestOwner(t *testing.T) {
 	require.True(t, ring1.Contains(start), "start should be in ring")
 	require.False(t, ring1.Contains(end), "end should not be in ring")
 
-	require.Equal(t, router.UnknownPeerName, ring1.Owner(start))
+	require.Equal(t, mesh.UnknownPeerName, ring1.Owner(start))
 
 	ring1.ClaimItAll()
 	ring1.GrantRangeToHost(middle, end, peer2name)
@@ -505,6 +513,29 @@ func TestOwner(t *testing.T) {
 
 }
 
+func makePeerName(i int) mesh.PeerName {
+	if i >= 10000 {
+		panic("makePeerName: invalid value")
+	}
+	peer, _ := mesh.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:ff", i/100, i%100))
+	return peer
+}
+
+func TestClaimForPeers(t *testing.T) {
+	const numPeers = 12
+	// Different end to usual so we get a number of addresses that a)
+	// is smaller than the max number of peers, and b) is divisible by
+	// some number of peers. This maximises coverage of edge cases.
+	end := dot10
+	peers := make([]mesh.PeerName, numPeers)
+	// Test for a range of peer counts
+	for i := 0; i < numPeers; i++ {
+		peers[i] = makePeerName(i)
+		ring := New(start, end, peers[0])
+		ring.ClaimForPeers(peers[:i+1])
+	}
+}
+
 type addressSlice []address.Address
 
 func (s addressSlice) Len() int           { return len(s) }
@@ -517,10 +548,9 @@ func TestFuzzRing(t *testing.T) {
 		iterations = 1000
 	)
 
-	peers := make([]router.PeerName, numPeers)
+	peers := make([]mesh.PeerName, numPeers)
 	for i := 0; i < numPeers; i++ {
-		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:00:02:00", i))
-		peers[i] = peer
+		peers[i] = makePeerName(i)
 	}
 
 	// Make a valid, random ring
@@ -601,13 +631,13 @@ func TestFuzzRingHard(t *testing.T) {
 	var (
 		numPeers   = 100
 		iterations = 3000
-		peers      []router.PeerName
+		peers      []mesh.PeerName
 		rings      []*Ring
 		nextPeerID = 0
 	)
 
 	addPeer := func() {
-		peer, _ := router.PeerNameFromString(fmt.Sprintf("%02d:%02d:00:00:00:00", nextPeerID/10, nextPeerID%10))
+		peer := makePeerName(nextPeerID)
 		common.Log.Debugf("%s: Adding peer", peer)
 		nextPeerID++
 		peers = append(peers, peer)
@@ -620,7 +650,7 @@ func TestFuzzRingHard(t *testing.T) {
 
 	rings[0].ClaimItAll()
 
-	randomPeer := func(exclude int) (int, router.PeerName, *Ring) {
+	randomPeer := func(exclude int) (int, mesh.PeerName, *Ring) {
 		var peerIndex int
 		if exclude >= 0 {
 			peerIndex = rand.Intn(len(peers) - 1)
@@ -729,7 +759,7 @@ func TestFuzzRingHard(t *testing.T) {
 }
 
 func (r *Ring) ClaimItAll() {
-	r.ClaimForPeers([]router.PeerName{r.Peer})
+	r.ClaimForPeers([]mesh.PeerName{r.Peer})
 }
 
 func (es entries) String() string {

ipam/space/space.go

@@ -31,8 +31,8 @@ func (s *Space) Add(start address.Address, size address.Offset) {
 
 // Clear removes all spaces from this space set.  Used during node shutdown.
 func (s *Space) Clear() {
-	s.free = s.free[:0]
 	s.ours = s.ours[:0]
+	s.free = s.free[:0]
 }
 
 // Walk down the free list calling f() on the in-range portions, until
@@ -88,8 +88,16 @@ func (s *Space) Claim(addr address.Address) error {
 	return nil
 }
 
-func (s *Space) NumFreeAddressesInRange(r address.Range) address.Offset {
-	res := address.Offset(0)
+func (s *Space) NumFreeAddresses() address.Count {
+	res := address.Count(0)
+	for i := 0; i < len(s.free); i += 2 {
+		res += address.Length(s.free[i+1], s.free[i])
+	}
+	return res
+}
+
+func (s *Space) NumFreeAddressesInRange(r address.Range) address.Count {
+	res := address.Count(0)
 	s.walkFree(r, func(chunk address.Range) bool {
 		res += chunk.Size()
 		return false
@@ -111,7 +119,7 @@ func (s *Space) Free(addr address.Address) error {
 }
 
 func (s *Space) biggestFreeRange(r address.Range) (biggest address.Range) {
-	biggestSize := address.Offset(0)
+	biggestSize := address.Count(0)
 	s.walkFree(r, func(chunk address.Range) bool {
 		if size := chunk.Size(); size >= biggestSize {
 			biggest = chunk
@@ -131,7 +139,7 @@ func (s *Space) Donate(r address.Range) (address.Range, bool) {
 
 	// Donate half of that biggest free range. Note size/2 rounds down, so
 	// the resulting donation size rounds up, and in particular can't be empty.
-	biggest.Start = address.Add(biggest.Start, biggest.Size()/2)
+	biggest.Start = address.Add(biggest.Start, address.Offset(biggest.Size()/2))
 
 	s.ours = subtract(s.ours, biggest.Start, biggest.End)
 	s.free = subtract(s.free, biggest.Start, biggest.End)

ipam/space/space_test.go

@@ -19,16 +19,6 @@ func ip(s string) address.Address {
 	return addr
 }
 
-// Helper function to avoid 'NumFreeAddressesInRange(start, end)'
-// dozens of times in tests
-func (s *Space) NumFreeAddresses() address.Offset {
-	res := address.Offset(0)
-	for i := 0; i < len(s.free); i += 2 {
-		res += address.Subtract(s.free[i+1], s.free[i])
-	}
-	return res
-}
-
 func TestLowlevel(t *testing.T) {
 	a := []address.Address{}
 	a = add(a, 100, 200)
@@ -61,19 +51,19 @@ func TestLowlevel(t *testing.T) {
 	require.Equal(t, []address.Address{}, a)
 
 	s := New()
-	require.Equal(t, address.Offset(0), s.NumFreeAddresses())
+	require.Equal(t, address.Count(0), s.NumFreeAddresses())
 	ok, got := s.Allocate(address.NewRange(0, 1000))
 	require.False(t, ok, "allocate in empty space should fail")
 
 	s.Add(100, 100)
-	require.Equal(t, address.Offset(100), s.NumFreeAddresses())
+	require.Equal(t, address.Count(100), s.NumFreeAddresses())
 	ok, got = s.Allocate(address.NewRange(0, 1000))
 	require.True(t, ok && got == 100, "allocate")
-	require.Equal(t, address.Offset(99), s.NumFreeAddresses())
+	require.Equal(t, address.Count(99), s.NumFreeAddresses())
 	require.NoError(t, s.Claim(150))
-	require.Equal(t, address.Offset(98), s.NumFreeAddresses())
+	require.Equal(t, address.Count(98), s.NumFreeAddresses())
 	require.NoError(t, s.Free(100))
-	require.Equal(t, address.Offset(99), s.NumFreeAddresses())
+	require.Equal(t, address.Count(99), s.NumFreeAddresses())
 	wt.AssertErrorInterface(t, (*error)(nil), s.Free(0), "free not allocated")
 	wt.AssertErrorInterface(t, (*error)(nil), s.Free(100), "double free")
 
@@ -107,19 +97,19 @@ func TestSpaceAllocate(t *testing.T) {
 	)
 
 	space1 := makeSpace(start, size)
-	require.Equal(t, address.Offset(20), space1.NumFreeAddresses())
+	require.Equal(t, address.Count(20), space1.NumFreeAddresses())
 	space1.assertInvariants()
 
 	_, addr1 := space1.Allocate(address.NewRange(start, size))
 	require.Equal(t, testAddr1, addr1.String(), "address")
-	require.Equal(t, address.Offset(19), space1.NumFreeAddresses())
+	require.Equal(t, address.Count(19), space1.NumFreeAddresses())
 	space1.assertInvariants()
 
 	_, addr2 := space1.Allocate(address.NewRange(start, size))
 	require.False(t, addr2.String() == testAddr1, "address")
-	require.Equal(t, address.Offset(18), space1.NumFreeAddresses())
-	require.Equal(t, address.Offset(13), space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddrx)}))
-	require.Equal(t, address.Offset(18), space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddry)}))
+	require.Equal(t, address.Count(18), space1.NumFreeAddresses())
+	require.Equal(t, address.Count(13), space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddrx)}))
+	require.Equal(t, address.Count(18), space1.NumFreeAddressesInRange(address.Range{Start: ip(testAddr1), End: ip(testAddry)}))
 	space1.assertInvariants()
 
 	space1.Free(addr2)
@@ -155,7 +145,7 @@ func TestSpaceFree(t *testing.T) {
 	// Check we are full
 	ok, _ := space.Allocate(entireRange)
 	require.True(t, !ok, "Should have failed to get address")
-	r, ok = space.Donate(entireRange)
+	r, _ = space.Donate(entireRange)
 	require.True(t, r.Size() == 0, "Wrong space")
 
 	// Free in the middle
@@ -172,7 +162,7 @@ func TestSpaceFree(t *testing.T) {
 	require.NoError(t, space.Free(ip("10.0.3.22")))
 	require.NoError(t, space.Free(ip("10.0.3.21")))
 
-	require.Equal(t, address.Offset(4), space.NumFreeAddresses())
+	require.Equal(t, address.Count(4), space.NumFreeAddresses())
 
 	// Now get the biggest free space; should be 3.21
 	r = space.biggestFreeRange(entireRange)
@@ -183,7 +173,7 @@ func TestSpaceFree(t *testing.T) {
 	require.NoError(t, space.Free(ip("10.0.3.11")))
 	require.NoError(t, space.Free(ip("10.0.3.10")))
 
-	require.Equal(t, address.Offset(7), space.NumFreeAddresses())
+	require.Equal(t, address.Count(7), space.NumFreeAddresses())
 
 	// Now get the biggest free space; should be 3.21
 	r = space.biggestFreeRange(entireRange)
@@ -210,8 +200,8 @@ func TestDonateSimple(t *testing.T) {
 	numGivenUp := r.Size()
 	require.True(t, ok, "Donate result")
 	require.Equal(t, "10.0.1.24", r.Start.String(), "Invalid start")
-	require.Equal(t, address.Offset(size/2), numGivenUp)
-	require.Equal(t, address.Offset(size/2), ps1.NumFreeAddresses())
+	require.Equal(t, address.Count(size/2), numGivenUp)
+	require.Equal(t, address.Count(size/2), ps1.NumFreeAddresses())
 
 	// Now check we can give the rest up.
 	count := 0 // count to avoid infinite loop
@@ -222,8 +212,8 @@ func TestDonateSimple(t *testing.T) {
 		}
 		numGivenUp += r.Size()
 	}
-	require.Equal(t, address.Offset(0), ps1.NumFreeAddresses())
-	require.Equal(t, address.Offset(size), numGivenUp)
+	require.Equal(t, address.Count(0), ps1.NumFreeAddresses())
+	require.Equal(t, address.Count(size), numGivenUp)
 }
 
 func TestDonateHard(t *testing.T) {
@@ -240,7 +230,7 @@ func TestDonateHard(t *testing.T) {
 		require.True(t, ok, "Failed to get IP!")
 	}
 
-	require.Equal(t, address.Offset(0), spaceset.NumFreeAddresses())
+	require.Equal(t, address.Count(0), spaceset.NumFreeAddresses())
 
 	// Now free all but the last address
 	// this will force us to split the free list
@@ -252,8 +242,8 @@ func TestDonateHard(t *testing.T) {
 	newRange, ok := spaceset.Donate(address.NewRange(start, size))
 	require.True(t, ok, "GiveUpSpace result")
 	require.Equal(t, ip("10.0.1.23"), newRange.Start)
-	require.Equal(t, address.Offset(24), newRange.Size())
-	require.Equal(t, address.Offset(23), spaceset.NumFreeAddresses())
+	require.Equal(t, address.Count(24), newRange.Size())
+	require.Equal(t, address.Count(23), spaceset.NumFreeAddresses())
 
 	//Space set should now have 2 spaces
 	expected := New()

ipam/status.go

@@ -1,6 +1,7 @@
 package ipam
 
 import (
+	"fmt"
 	"github.com/weaveworks/weave/ipam/paxos"
 	"github.com/weaveworks/weave/net/address"
 )
@@ -8,6 +9,7 @@ import (
 type Status struct {
 	Paxos            *paxos.Status
 	Range            string
+	RangeNumIPs      int
 	DefaultSubnet    string
 	Entries          []EntryStatus
 	PendingClaims    []ClaimStatus
@@ -15,14 +17,17 @@ type Status struct {
 }
 
 type EntryStatus struct {
-	Token   string
-	Peer    string
-	Version uint32
+	Token       string
+	Size        uint32
+	Peer        string
+	Nickname    string
+	IsKnownPeer bool
+	Version     uint32
 }
 
 type ClaimStatus struct {
-	Ident   string
-	Address address.Address
+	Ident string
+	CIDR  address.CIDR
 }
 
 func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status {
@@ -31,8 +36,14 @@ func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status {
 	}
 
 	var paxosStatus *paxos.Status
-	if allocator.paxosTicker != nil {
-		paxosStatus = paxos.NewStatus(allocator.paxos)
+	if allocator.awaitingConsensus && allocator.paxos != nil {
+		if allocator.paxos.IsElector() {
+			if node, ok := allocator.paxos.(*paxos.Node); ok {
+				paxosStatus = paxos.NewStatus(node)
+			}
+		} else {
+			paxosStatus = &paxos.Status{Elector: false}
+		}
 	}
 
 	resultChan := make(chan *Status)
@@ -40,6 +51,7 @@ func NewStatus(allocator *Allocator, defaultSubnet address.CIDR) *Status {
 		resultChan <- &Status{
 			paxosStatus,
 			allocator.universe.String(),
+			int(allocator.universe.Size()),
 			defaultSubnet.String(),
 			newEntryStatusSlice(allocator),
 			newClaimStatusSlice(allocator),
@@ -56,8 +68,15 @@ func newEntryStatusSlice(allocator *Allocator) []EntryStatus {
 		return slice
 	}
 
-	for _, entry := range allocator.ring.Entries {
-		slice = append(slice, EntryStatus{entry.Token.String(), entry.Peer.String(), entry.Version})
+	for _, r := range allocator.ring.AllRangeInfo() {
+		slice = append(slice, EntryStatus{
+			Token:       r.Start.String(),
+			Size:        uint32(r.Size()),
+			Peer:        r.Peer.String(),
+			Nickname:    allocator.nicknames[r.Peer],
+			IsKnownPeer: allocator.isKnownPeer(r.Peer),
+			Version:     r.Version,
+		})
 	}
 
 	return slice
@@ -67,7 +86,7 @@ func newClaimStatusSlice(allocator *Allocator) []ClaimStatus {
 	var slice []ClaimStatus
 	for _, op := range allocator.pendingClaims {
 		claim := op.(*claim)
-		slice = append(slice, ClaimStatus{claim.ident, claim.addr})
+		slice = append(slice, ClaimStatus{claim.ident, claim.cidr})
 	}
 	return slice
 }
@@ -76,7 +95,7 @@ func newAllocateIdentSlice(allocator *Allocator) []string {
 	var slice []string
 	for _, op := range allocator.pendingAllocates {
 		allocate := op.(*allocate)
-		slice = append(slice, allocate.ident)
+		slice = append(slice, fmt.Sprintf("%s %s", allocate.ident, allocate.r.String()))
 	}
 	return slice
 }

ipam/testutils_test.go

@@ -7,14 +7,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/weaveworks/mesh"
+
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 	"github.com/weaveworks/weave/testing/gossip"
 )
 
 type mockMessage struct {
-	dst     router.PeerName
+	dst     mesh.PeerName
 	msgType byte
 	buf     []byte
 }
@@ -48,13 +49,13 @@ func (m *mockGossipComms) String() string {
 // that the contents of messages are never re-ordered.  Which, for instance,
 // requires they are not based off iterating through a map.
 
-func (m *mockGossipComms) GossipBroadcast(update router.GossipData) error {
+func (m *mockGossipComms) GossipBroadcast(update mesh.GossipData) {
 	m.Lock()
 	defer m.Unlock()
 	buf := []byte{}
 	if len(m.messages) == 0 {
 		require.FailNow(m, fmt.Sprintf("%s: Gossip broadcast message unexpected: \n%x", m.name, buf))
-	} else if msg := m.messages[0]; msg.dst != router.UnknownPeerName {
+	} else if msg := m.messages[0]; msg.dst != mesh.UnknownPeerName {
 		require.FailNow(m, fmt.Sprintf("%s: Expected Gossip message to %s but got broadcast", m.name, msg.dst))
 	} else if msg.buf != nil && !equalByteBuffer(msg.buf, buf) {
 		require.FailNow(m, fmt.Sprintf("%s: Gossip message not sent as expected: \nwant: %x\ngot : %x", m.name, msg.buf, buf))
@@ -62,7 +63,6 @@ func (m *mockGossipComms) GossipBroadcast(update router.GossipData) error {
 		// Swallow this message
 		m.messages = m.messages[1:]
 	}
-	return nil
 }
 
 func equalByteBuffer(a, b []byte) bool {
@@ -77,12 +77,12 @@ func equalByteBuffer(a, b []byte) bool {
 	return true
 }
 
-func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte) error {
+func (m *mockGossipComms) GossipUnicast(dstPeerName mesh.PeerName, buf []byte) error {
 	m.Lock()
 	defer m.Unlock()
 	if len(m.messages) == 0 {
 		require.FailNow(m, fmt.Sprintf("%s: Gossip message to %s unexpected: \n%s", m.name, dstPeerName, buf))
-	} else if msg := m.messages[0]; msg.dst == router.UnknownPeerName {
+	} else if msg := m.messages[0]; msg.dst == mesh.UnknownPeerName {
 		require.FailNow(m, fmt.Sprintf("%s: Expected Gossip broadcast message but got dest %s", m.name, dstPeerName))
 	} else if msg.dst != dstPeerName {
 		require.FailNow(m, fmt.Sprintf("%s: Expected Gossip message to %s but got dest %s", m.name, msg.dst, dstPeerName))
@@ -99,7 +99,7 @@ func (m *mockGossipComms) GossipUnicast(dstPeerName router.PeerName, buf []byte)
 
 func ExpectMessage(alloc *Allocator, dst string, msgType byte, buf []byte) {
 	m := alloc.gossip.(*mockGossipComms)
-	dstPeerName, _ := router.PeerNameFromString(dst)
+	dstPeerName, _ := mesh.PeerNameFromString(dst)
 	m.Lock()
 	m.messages = append(m.messages, mockMessage{dstPeerName, msgType, buf})
 	m.Unlock()
@@ -108,7 +108,7 @@ func ExpectMessage(alloc *Allocator, dst string, msgType byte, buf []byte) {
 func ExpectBroadcastMessage(alloc *Allocator, buf []byte) {
 	m := alloc.gossip.(*mockGossipComms)
 	m.Lock()
-	m.messages = append(m.messages, mockMessage{router.UnknownPeerName, 0, buf})
+	m.messages = append(m.messages, mockMessage{mesh.UnknownPeerName, 0, buf})
 	m.Unlock()
 }
 
@@ -123,24 +123,35 @@ func CheckAllExpectedMessagesSent(allocs ...*Allocator) {
 	}
 }
 
-func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.Range) {
-	peername, err := router.PeerNameFromString(name)
+type mockDB struct{}
+
+func (d *mockDB) Load(_ string, _ interface{}) (bool, error) { return false, nil }
+func (d *mockDB) Save(_ string, _ interface{}) error         { return nil }
+
+func makeAllocator(name string, cidrStr string, quorum uint) (*Allocator, address.CIDR) {
+	peername, err := mesh.PeerNameFromString(name)
 	if err != nil {
 		panic(err)
 	}
 
-	_, cidr, err := address.ParseCIDR(cidrStr)
+	cidr, err := address.ParseCIDR(cidrStr)
 	if err != nil {
 		panic(err)
 	}
 
-	alloc := NewAllocator(peername, router.PeerUID(rand.Int63()),
-		"nick-"+name, cidr.Range(), quorum, func(router.PeerName) bool { return true })
-
-	return alloc, cidr.HostRange()
+	return NewAllocator(Config{
+		OurName:     peername,
+		OurUID:      mesh.PeerUID(rand.Int63()),
+		OurNickname: "nick-" + name,
+		Universe:    cidr.Range(),
+		IsObserver:  quorum == 0,
+		Quorum:      func() uint { return quorum },
+		Db:          new(mockDB),
+		IsKnownPeer: func(mesh.PeerName) bool { return true },
+	}), cidr
 }
 
-func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) (*Allocator, address.Range) {
+func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string, quorum uint) (*Allocator, address.CIDR) {
 	alloc, subnet := makeAllocator(name, universeCIDR, quorum)
 	gossip := &mockGossipComms{T: t, name: name}
 	alloc.SetInterfaces(gossip)
@@ -149,7 +160,7 @@ func makeAllocatorWithMockGossip(t *testing.T, name string, universeCIDR string,
 }
 
 func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
-	peers := []router.PeerName{alloc.ourName}
+	peers := []mesh.PeerName{alloc.ourName}
 	for _, alloc2 := range allocs {
 		peers = append(peers, alloc2.ourName)
 	}
@@ -157,14 +168,22 @@ func (alloc *Allocator) claimRingForTesting(allocs ...*Allocator) {
 	alloc.space.AddRanges(alloc.ring.OwnedRanges())
 }
 
-func (alloc *Allocator) NumFreeAddresses(r address.Range) address.Offset {
-	resultChan := make(chan address.Offset)
+func (alloc *Allocator) NumFreeAddresses(r address.Range) address.Count {
+	resultChan := make(chan address.Count)
 	alloc.actionChan <- func() {
 		resultChan <- alloc.space.NumFreeAddressesInRange(r)
 	}
 	return <-resultChan
 }
 
+func (alloc *Allocator) OwnedRanges() (result []address.Range) {
+	resultChan := make(chan []address.Range)
+	alloc.actionChan <- func() {
+		resultChan <- alloc.ring.OwnedRanges()
+	}
+	return <-resultChan
+}
+
 // Check whether or not something was sent on a channel
 func AssertSent(t *testing.T, ch <-chan bool) {
 	timeout := time.After(10 * time.Second)
@@ -194,10 +213,10 @@ func AssertNothingSentErr(t *testing.T, ch <-chan error) {
 	}
 }
 
-func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, *gossip.TestRouter, address.Range) {
+func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, *gossip.TestRouter, address.CIDR) {
 	gossipRouter := gossip.NewTestRouter(0.0)
 	allocs := make([]*Allocator, size)
-	var subnet address.Range
+	var subnet address.CIDR
 
 	for i := 0; i < size; i++ {
 		var alloc *Allocator
@@ -213,7 +232,10 @@ func makeNetworkOfAllocators(size int, cidr string) ([]*Allocator, *gossip.TestR
 	return allocs, gossipRouter, subnet
 }
 
-func stopNetworkOfAllocators(allocs []*Allocator) {
+func stopNetworkOfAllocators(allocs []*Allocator, gossipRouter *gossip.TestRouter) {
+	// NB: We must stop the router first since ipam gossip makes
+	// synchronous calls into the allocator.
+	gossipRouter.Stop()
 	for _, alloc := range allocs {
 		alloc.Stop()
 	}

nameserver/dns_test.go

@@ -11,14 +11,14 @@ import (
 	"github.com/miekg/dns"
 	"github.com/stretchr/testify/require"
 
+	"github.com/weaveworks/mesh"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 func startServer(t *testing.T, upstream *dns.ClientConfig) (*DNSServer, *Nameserver, int, int) {
-	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
+	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, "")
+	nameserver := New(peername, "", func(mesh.PeerName) bool { return true })
 	dnsserver, err := NewDNSServer(nameserver, "weave.local.", "0.0.0.0:0", "", 30, 5*time.Second)
 	require.Nil(t, err)
 	udpPort := dnsserver.servers[0].PacketConn.LocalAddr().(*net.UDPAddr).Port
@@ -39,7 +39,7 @@ func TestTruncation(t *testing.T) {
 	addrs := []address.Address{}
 	for i := address.Address(0); i < 100; i++ {
 		addrs = append(addrs, i)
-		nameserver.AddEntry("foo.weave.local.", "", router.UnknownPeerName, i)
+		nameserver.AddEntry("foo.weave.local.", "", mesh.UnknownPeerName, i)
 	}
 
 	doRequest := func(client *dns.Client, request *dns.Msg, port int, expectedErr error) *dns.Msg {

nameserver/entry.go

@@ -8,15 +8,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/weaveworks/mesh"
+
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 var now = func() int64 { return time.Now().Unix() }
 
 type Entry struct {
 	ContainerID string
-	Origin      router.PeerName
+	Origin      mesh.PeerName
 	Addr        address.Address
 	Hostname    string // as supplied
 	lHostname   string // lowercased (not exported, so not encoded by gob)
@@ -93,6 +94,9 @@ func (e1 *Entry) merge(e2 *Entry) bool {
 		e1.Version = e2.Version
 		e1.Tombstone = e2.Tombstone
 		return true
+	} else if e2.Version == e1.Version && e2.Tombstone > e1.Tombstone {
+		e1.Tombstone = e2.Tombstone
+		return true
 	}
 	return false
 }
@@ -105,6 +109,15 @@ func (e1 *Entry) addLowercase() {
 	e1.lHostname = strings.ToLower(e1.Hostname)
 }
 
+func (e1 *Entry) tombstone() bool {
+	if e1.Tombstone > 0 {
+		return false
+	}
+	e1.Tombstone = now()
+	e1.Version++
+	return true
+}
+
 func check(es SortableEntries) error {
 	if !sort.IsSorted(es) {
 		return fmt.Errorf("Not sorted!")
@@ -128,7 +141,7 @@ func (es *Entries) checkAndPanic() *Entries {
 	return es
 }
 
-func (es *Entries) add(hostname, containerid string, origin router.PeerName, addr address.Address) Entry {
+func (es *Entries) add(hostname, containerid string, origin mesh.PeerName, addr address.Address) Entry {
 	defer es.checkAndPanic().checkAndPanic()
 
 	entry := Entry{Hostname: hostname, lHostname: strings.ToLower(hostname),
@@ -176,14 +189,12 @@ func (es *Entries) merge(incoming Entries) Entries {
 }
 
 // f returning true means keep the entry.
-func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) Entries {
+func (es *Entries) tombstone(ourname mesh.PeerName, f func(*Entry) bool) Entries {
 	defer es.checkAndPanic().checkAndPanic()
 
 	tombstoned := Entries{}
 	for i, e := range *es {
-		if f(&e) && e.Origin == ourname {
-			e.Version++
-			e.Tombstone = now()
+		if f(&e) && e.Origin == ourname && e.tombstone() {
 			(*es)[i] = e
 			tombstoned = append(tombstoned, e)
 		}
@@ -191,6 +202,7 @@ func (es *Entries) tombstone(ourname router.PeerName, f func(*Entry) bool) Entri
 	return tombstoned
 }
 
+// note f() may only modify entries such that they remain in order defined by less()
 func (es *Entries) filter(f func(*Entry) bool) {
 	defer es.checkAndPanic().checkAndPanic()
 
@@ -205,6 +217,16 @@ func (es *Entries) filter(f func(*Entry) bool) {
 	*es = (*es)[:i]
 }
 
+func (es Entries) findEqual(e *Entry) (*Entry, bool) {
+	i := sort.Search(len(es), func(i int) bool {
+		return !es[i].insensitiveLess(e)
+	})
+	if i < len(es) && es[i].equal(*e) {
+		return &es[i], true
+	}
+	return nil, false
+}
+
 func (es Entries) lookup(hostname string) Entries {
 	es.checkAndPanic()
 
@@ -245,12 +267,14 @@ type GossipData struct {
 	Entries
 }
 
-func (g *GossipData) Merge(o router.GossipData) {
+func (g *GossipData) Merge(o mesh.GossipData) mesh.GossipData {
 	other := o.(*GossipData)
-	g.Entries.merge(other.Entries)
-	if g.Timestamp < other.Timestamp {
-		g.Timestamp = other.Timestamp
+	gossip := g.copy()
+	gossip.Entries.merge(other.Entries)
+	if gossip.Timestamp < other.Timestamp {
+		gossip.Timestamp = other.Timestamp
 	}
+	return gossip
 }
 
 func (g *GossipData) Decode(msg []byte) error {
@@ -264,9 +288,7 @@ func (g *GossipData) Decode(msg []byte) error {
 }
 
 func (g *GossipData) Encode() [][]byte {
-	// Make a copy so we can sort: all outgoing data is sent in case-sensitive order
-	g2 := GossipData{Timestamp: g.Timestamp, Entries: make(Entries, len(g.Entries))}
-	copy(g2.Entries, g.Entries)
+	g2 := g.copy()
 	sort.Sort(CaseSensitive(g2.Entries))
 	buf := &bytes.Buffer{}
 	if err := gob.NewEncoder(buf).Encode(g2); err != nil {
@@ -274,3 +296,9 @@ func (g *GossipData) Encode() [][]byte {
 	}
 	return [][]byte{buf.Bytes()}
 }
+
+func (g *GossipData) copy() *GossipData {
+	g2 := &GossipData{Timestamp: g.Timestamp, Entries: make(Entries, len(g.Entries))}
+	copy(g2.Entries, g.Entries)
+	return g2
+}

nameserver/entry_test.go

@@ -4,9 +4,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"github.com/weaveworks/mesh"
 
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 func l(es Entries) Entries {
@@ -28,21 +28,21 @@ func TestAdd(t *testing.T) {
 	now = func() int64 { return 1234 }
 
 	entries := Entries{}
-	entries.add("A", "", router.UnknownPeerName, address.Address(0))
+	entries.add("A", "", mesh.UnknownPeerName, address.Address(0))
 	expected := l(Entries{
-		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0)},
+		Entry{Hostname: "A", Origin: mesh.UnknownPeerName, Addr: address.Address(0)},
 	})
 	require.Equal(t, entries, expected)
 
-	entries.tombstone(router.UnknownPeerName, func(e *Entry) bool { return e.Hostname == "A" })
+	entries.tombstone(mesh.UnknownPeerName, func(e *Entry) bool { return e.Hostname == "A" })
 	expected = l(Entries{
-		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 1, Tombstone: 1234},
+		Entry{Hostname: "A", Origin: mesh.UnknownPeerName, Addr: address.Address(0), Version: 1, Tombstone: 1234},
 	})
 	require.Equal(t, entries, expected)
 
-	entries.add("A", "", router.UnknownPeerName, address.Address(0))
+	entries.add("A", "", mesh.UnknownPeerName, address.Address(0))
 	expected = l(Entries{
-		Entry{Hostname: "A", Origin: router.UnknownPeerName, Addr: address.Address(0), Version: 2},
+		Entry{Hostname: "A", Origin: mesh.UnknownPeerName, Addr: address.Address(0), Version: 2},
 	})
 	require.Equal(t, entries, expected)
 }
@@ -78,7 +78,7 @@ func TestTombstone(t *testing.T) {
 
 	es := makeEntries("AB")
 
-	es.tombstone(router.UnknownPeerName, func(e *Entry) bool {
+	es.tombstone(mesh.UnknownPeerName, func(e *Entry) bool {
 		return e.Hostname == "B"
 	})
 	expected := l(Entries{
@@ -86,6 +86,21 @@ func TestTombstone(t *testing.T) {
 		Entry{Hostname: "B", Version: 1, Tombstone: 1234},
 	})
 	require.Equal(t, expected, es)
+
+	// Now try a merge including two entries which differ only in tombstone
+	e2 := make(Entries, len(es))
+	copy(e2, es)
+	e2[1].Tombstone = 5555
+
+	diff := es.merge(e2)
+
+	expected2 := l(Entries{
+		Entry{Hostname: "A"},
+		Entry{Hostname: "B", Version: 1, Tombstone: 5555},
+	})
+	require.Equal(t, expected2, es)
+	expectedDiff := l(Entries{Entry{Hostname: "B", Version: 1, Tombstone: 5555}})
+	require.Equal(t, expectedDiff, diff)
 }
 
 func TestDelete(t *testing.T) {
@@ -117,7 +132,7 @@ func TestGossipDataMerge(t *testing.T) {
 	g1 := GossipData{Entries: makeEntries("AcDf")}
 	g2 := GossipData{Entries: makeEntries("BEf")}
 
-	g1.Merge(&g2)
+	g3 := g1.Merge(&g2).(*GossipData)
 
-	require.Equal(t, GossipData{Entries: makeEntries("ABcDEf")}, g1)
+	require.Equal(t, GossipData{Entries: makeEntries("ABcDEf")}, *g3)
 }

nameserver/http.go

@@ -40,16 +40,11 @@ func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) {
 			return
 		}
 
-		if err := n.AddEntry(hostname, container, n.ourName, ip); err != nil {
-			n.badRequest(w, fmt.Errorf("Unable to add entry: %v", err))
-			return
-		}
+		n.AddEntry(hostname, container, n.ourName, ip)
 
 		if r.FormValue("check-alive") == "true" && dockerCli != nil && dockerCli.IsContainerNotRunning(container) {
 			n.infof("container '%s' is not running: removing", container)
-			if err := n.Delete(hostname, container, ipStr, ip); err != nil {
-				n.infof("failed to remove: %v", err)
-			}
+			n.Delete(hostname, container, ipStr, ip)
 		}
 
 		w.WriteHeader(204)
@@ -79,10 +74,7 @@ func (n *Nameserver) HandleHTTP(router *mux.Router, dockerCli *docker.Client) {
 			ipStr = "*"
 		}
 
-		if err := n.Delete(hostname, container, ipStr, ip); err != nil {
-			n.badRequest(w, fmt.Errorf("Unable to delete entries: %v", err))
-			return
-		}
+		n.Delete(hostname, container, ipStr, ip)
 		w.WriteHeader(204)
 	}
 	router.Methods("DELETE").Path("/name/{container}/{ip}").HandlerFunc(deleteHandler)

nameserver/nameserver.go

@@ -6,10 +6,10 @@ import (
 	"time"
 
 	"github.com/miekg/dns"
+	"github.com/weaveworks/mesh"
 
-	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
 )
 
 const (
@@ -34,28 +34,24 @@ const (
 // - Update is O(n) for now
 type Nameserver struct {
 	sync.RWMutex
-	ourName router.PeerName
-	domain  string
-	gossip  router.Gossip
-	entries Entries
-	peers   *router.Peers
-	quit    chan struct{}
+	ourName     mesh.PeerName
+	domain      string
+	gossip      mesh.Gossip
+	entries     Entries
+	isKnownPeer func(mesh.PeerName) bool
+	quit        chan struct{}
 }
 
-func New(ourName router.PeerName, peers *router.Peers, domain string) *Nameserver {
-	ns := &Nameserver{
-		ourName: ourName,
-		domain:  dns.Fqdn(domain),
-		peers:   peers,
-		quit:    make(chan struct{}),
+func New(ourName mesh.PeerName, domain string, isKnownPeer func(mesh.PeerName) bool) *Nameserver {
+	return &Nameserver{
+		ourName:     ourName,
+		domain:      dns.Fqdn(domain),
+		isKnownPeer: isKnownPeer,
+		quit:        make(chan struct{}),
 	}
-	if peers != nil {
-		peers.OnGC(ns.PeerGone)
-	}
-	return ns
 }
 
-func (n *Nameserver) SetGossip(gossip router.Gossip) {
+func (n *Nameserver) SetGossip(gossip mesh.Gossip) {
 	n.gossip = gossip
 }
 
@@ -77,22 +73,22 @@ func (n *Nameserver) Stop() {
 	n.quit <- struct{}{}
 }
 
-func (n *Nameserver) broadcastEntries(es ...Entry) error {
-	if n.gossip != nil {
-		return n.gossip.GossipBroadcast(&GossipData{
-			Entries:   Entries(es),
-			Timestamp: now(),
-		})
+func (n *Nameserver) broadcastEntries(es ...Entry) {
+	if n.gossip == nil || len(es) == 0 {
+		return
 	}
-	return nil
+	n.gossip.GossipBroadcast(&GossipData{
+		Entries:   Entries(es),
+		Timestamp: now(),
+	})
 }
 
-func (n *Nameserver) AddEntry(hostname, containerid string, origin router.PeerName, addr address.Address) error {
-	n.infof("adding entry %s -> %s", hostname, addr.String())
+func (n *Nameserver) AddEntry(hostname, containerid string, origin mesh.PeerName, addr address.Address) {
 	n.Lock()
+	n.infof("adding entry for %s: %s -> %s", containerid, hostname, addr.String())
 	entry := n.entries.add(hostname, containerid, origin, addr)
 	n.Unlock()
-	return n.broadcastEntries(entry)
+	n.broadcastEntries(entry)
 }
 
 func (n *Nameserver) Lookup(hostname string) []address.Address {
@@ -125,7 +121,8 @@ func (n *Nameserver) ReverseLookup(ip address.Address) (string, error) {
 	return match.Hostname, nil
 }
 
-func (n *Nameserver) ContainerStarted(ident string) {}
+func (n *Nameserver) ContainerStarted(ident string)   {}
+func (n *Nameserver) ContainerDestroyed(ident string) {}
 
 func (n *Nameserver) ContainerDied(ident string) {
 	n.Lock()
@@ -137,23 +134,19 @@ func (n *Nameserver) ContainerDied(ident string) {
 		return false
 	})
 	n.Unlock()
-	if len(entries) > 0 {
-		if err := n.broadcastEntries(entries...); err != nil {
-			n.errorf("failed to broadcast container %s death: %v", ident, err)
-		}
-	}
+	n.broadcastEntries(entries...)
 }
 
-func (n *Nameserver) PeerGone(peer *router.Peer) {
+func (n *Nameserver) PeerGone(peer mesh.PeerName) {
 	n.infof("peer %s gone", peer.String())
 	n.Lock()
 	defer n.Unlock()
 	n.entries.filter(func(e *Entry) bool {
-		return e.Origin != peer.Name
+		return e.Origin != peer
 	})
 }
 
-func (n *Nameserver) Delete(hostname, containerid, ipStr string, ip address.Address) error {
+func (n *Nameserver) Delete(hostname, containerid, ipStr string, ip address.Address) {
 	n.Lock()
 	n.infof("tombstoning hostname=%s, container=%s, ip=%s", hostname, containerid, ipStr)
 	entries := n.entries.tombstone(n.ourName, func(e *Entry) bool {
@@ -173,7 +166,7 @@ func (n *Nameserver) Delete(hostname, containerid, ipStr string, ip address.Addr
 		return true
 	})
 	n.Unlock()
-	return n.broadcastEntries(entries...)
+	n.broadcastEntries(entries...)
 }
 
 func (n *Nameserver) deleteTombstones() {
@@ -185,7 +178,7 @@ func (n *Nameserver) deleteTombstones() {
 	})
 }
 
-func (n *Nameserver) Gossip() router.GossipData {
+func (n *Nameserver) Gossip() mesh.GossipData {
 	n.RLock()
 	defer n.RUnlock()
 	gossip := &GossipData{
@@ -196,11 +189,11 @@ func (n *Nameserver) Gossip() router.GossipData {
 	return gossip
 }
 
-func (n *Nameserver) OnGossipUnicast(sender router.PeerName, msg []byte) error {
+func (n *Nameserver) OnGossipUnicast(sender mesh.PeerName, msg []byte) error {
 	return nil
 }
 
-func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.GossipData, error) {
+func (n *Nameserver) receiveGossip(msg []byte) (mesh.GossipData, mesh.GossipData, error) {
 	var gossip GossipData
 	if err := gossip.Decode(msg); err != nil {
 		return nil, nil, err
@@ -209,16 +202,43 @@ func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.Gossip
 		return nil, nil, fmt.Errorf("clock skew of %d detected", delta)
 	}
 
-	n.Lock()
-	defer n.Unlock()
+	// Filter to remove entries from unknown peers, done before we take
+	// the nameserver lock, so we don't have to worry what isKnownPeer locks.
+	gossip.Entries.filter(func(e *Entry) bool {
+		return n.isKnownPeer(e.Origin)
+	})
 
-	if n.peers != nil {
-		gossip.Entries.filter(func(e *Entry) bool {
-			return n.peers.Fetch(e.Origin) != nil
-		})
-	}
+	var overriddenEntries []Entry
+	n.Lock()
+
+	// Check entries claiming to originate from us against our current data
+	gossip.Entries.filter(func(e *Entry) bool {
+		if e.Origin == n.ourName {
+			if ourEntry, ok := n.entries.findEqual(e); ok {
+				if ourEntry.Version < e.Version ||
+					(ourEntry.Version == e.Version && ourEntry.Tombstone != e.Tombstone) {
+					// Take our version of the data, but make the version higher than the incoming
+					nextVersion := e.Version + 1
+					*e = *ourEntry
+					e.Version = nextVersion
+					overriddenEntries = append(overriddenEntries, *e)
+				}
+			} else { // We have no entry matching the one that came in with us as Origin
+				if e.tombstone() {
+					overriddenEntries = append(overriddenEntries, *e)
+				}
+			}
+		}
+		return true
+	})
 
 	newEntries := n.entries.merge(gossip.Entries)
+	n.Unlock() // unlock before attempting to broadcast
+
+	// Note that all overriddenEntries have been merged into our entries, either
+	// because we forced the version higher or because they were missing before.
+	n.broadcastEntries(overriddenEntries...)
+
 	if len(newEntries) > 0 {
 		return &GossipData{Entries: newEntries, Timestamp: now()}, &gossip, nil
 	}
@@ -227,24 +247,29 @@ func (n *Nameserver) receiveGossip(msg []byte) (router.GossipData, router.Gossip
 
 // merge received data into state and return "everything new I've
 // just learnt", or nil if nothing in the received data was new
-func (n *Nameserver) OnGossip(msg []byte) (router.GossipData, error) {
+func (n *Nameserver) OnGossip(msg []byte) (mesh.GossipData, error) {
 	newEntries, _, err := n.receiveGossip(msg)
 	return newEntries, err
 }
 
 // merge received data into state and return a representation of
 // the received data, for further propagation
-func (n *Nameserver) OnGossipBroadcast(_ router.PeerName, msg []byte) (router.GossipData, error) {
+func (n *Nameserver) OnGossipBroadcast(_ mesh.PeerName, msg []byte) (mesh.GossipData, error) {
 	_, entries, err := n.receiveGossip(msg)
 	return entries, err
 }
 
+// Logging
+
 func (n *Nameserver) infof(fmt string, args ...interface{}) {
-	Log.Infof("[nameserver %s] "+fmt, append([]interface{}{n.ourName}, args...)...)
+	n.logf(common.Log.Infof, fmt, args...)
 }
 func (n *Nameserver) debugf(fmt string, args ...interface{}) {
-	Log.Debugf("[nameserver %s] "+fmt, append([]interface{}{n.ourName}, args...)...)
+	n.logf(common.Log.Debugf, fmt, args...)
 }
 func (n *Nameserver) errorf(fmt string, args ...interface{}) {
-	Log.Errorf("[nameserver %s] "+fmt, append([]interface{}{n.ourName}, args...)...)
+	n.logf(common.Log.Errorf, fmt, args...)
+}
+func (n *Nameserver) logf(f func(string, ...interface{}), fmt string, args ...interface{}) {
+	f("[nameserver %s] "+fmt, append([]interface{}{n.ourName}, args...)...)
 }

nameserver/nameserver_test.go

@@ -10,20 +10,23 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"github.com/weaveworks/mesh"
 
 	"github.com/weaveworks/weave/net/address"
-	"github.com/weaveworks/weave/router"
-	wt "github.com/weaveworks/weave/testing"
 	"github.com/weaveworks/weave/testing/gossip"
 )
 
+func makeNameserver(name mesh.PeerName) *Nameserver {
+	return New(name, "", func(mesh.PeerName) bool { return true })
+}
+
 func makeNetwork(size int) ([]*Nameserver, *gossip.TestRouter) {
 	gossipRouter := gossip.NewTestRouter(0.0)
 	nameservers := make([]*Nameserver, size)
 
 	for i := 0; i < size; i++ {
-		name, _ := router.PeerNameFromString(fmt.Sprintf("%02d:00:00:02:00:00", i))
-		nameserver := New(name, nil, "")
+		name, _ := mesh.PeerNameFromString(fmt.Sprintf("%02d:00:00:02:00:00", i))
+		nameserver := makeNameserver(name)
 		nameserver.SetGossip(gossipRouter.Connect(nameserver.ourName, nameserver))
 		nameserver.Start()
 		nameservers[i] = nameserver
@@ -40,7 +43,7 @@ func stopNetwork(nameservers []*Nameserver, grouter *gossip.TestRouter) {
 }
 
 type pair struct {
-	origin router.PeerName
+	origin mesh.PeerName
 	addr   address.Address
 }
 
@@ -72,12 +75,6 @@ func (m mapping) Addrs() []address.Address {
 }
 
 func TestNameservers(t *testing.T) {
-	wt.RunWithTimeout(t, 2*time.Minute, func() {
-		testNameservers(t)
-	})
-}
-
-func testNameservers(t *testing.T) {
 	//common.SetLogLevel("debug")
 
 	lookupTimeout := 10 // ms
@@ -87,7 +84,7 @@ func testNameservers(t *testing.T) {
 	badNameservers := nameservers[25:]
 	// This subset will remain well-connected, and we will deal mainly with them
 	nameservers = nameservers[:25]
-	nameserversByName := map[router.PeerName]*Nameserver{}
+	nameserversByName := map[mesh.PeerName]*Nameserver{}
 	for _, n := range nameservers {
 		nameserversByName[n.ourName] = n
 	}
@@ -120,7 +117,7 @@ func testNameservers(t *testing.T) {
 		mapping := mapping{hostname, []pair{{nameserver.ourName, addr}}}
 		mappings = append(mappings, mapping)
 
-		require.Nil(t, nameserver.AddEntry(hostname, "", nameserver.ourName, addr))
+		nameserver.AddEntry(hostname, "", nameserver.ourName, addr)
 		check(nameserver, mapping)
 	}
 
@@ -135,14 +132,14 @@ func testNameservers(t *testing.T) {
 		mapping.addrs = append(mapping.addrs, pair{nameserver.ourName, addr})
 		mappings[i] = mapping
 
-		require.Nil(t, nameserver.AddEntry(mapping.hostname, "", nameserver.ourName, addr))
+		nameserver.AddEntry(mapping.hostname, "", nameserver.ourName, addr)
 		check(nameserver, mapping)
 	}
 
 	loseConnection := func() {
 		nameserver1 := badNameservers[rand.Intn(len(badNameservers))]
 		nameserver2 := nameservers[rand.Intn(len(nameservers))]
-		nameserver1.PeerGone(&router.Peer{Name: nameserver2.ourName})
+		nameserver1.PeerGone(nameserver2.ourName)
 	}
 
 	deleteMapping := func() {
@@ -160,7 +157,7 @@ func testNameservers(t *testing.T) {
 		mappings[i] = mapping
 		nameserver := nameserversByName[pair.origin]
 
-		require.Nil(t, nameserver.Delete(mapping.hostname, "*", pair.addr.String(), pair.addr))
+		nameserver.Delete(mapping.hostname, "*", pair.addr.String(), pair.addr)
 		check(nameserver, mapping)
 	}
 
@@ -222,22 +219,20 @@ func testNameservers(t *testing.T) {
 }
 
 func TestContainerAndPeerDeath(t *testing.T) {
-	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
+	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, "")
+	nameserver := makeNameserver(peername)
 
-	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
-	require.Nil(t, err)
+	nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname"))
 
 	nameserver.ContainerDied("containerid")
 	require.Equal(t, []address.Address{}, nameserver.Lookup("hostname"))
 
-	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
-	require.Nil(t, err)
+	nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname"))
 
-	nameserver.PeerGone(&router.Peer{Name: peername})
+	nameserver.PeerGone(peername)
 	require.Equal(t, []address.Address{}, nameserver.Lookup("hostname"))
 }
 
@@ -246,19 +241,17 @@ func TestTombstoneDeletion(t *testing.T) {
 	defer func() { now = oldNow }()
 	now = func() int64 { return 1234 }
 
-	peername, err := router.PeerNameFromString("00:00:00:02:00:00")
+	peername, err := mesh.PeerNameFromString("00:00:00:02:00:00")
 	require.Nil(t, err)
-	nameserver := New(peername, nil, "")
+	nameserver := makeNameserver(peername)
 
-	err = nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
-	require.Nil(t, err)
+	nameserver.AddEntry("hostname", "containerid", peername, address.Address(0))
 	require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname"))
 
 	nameserver.deleteTombstones()
 	require.Equal(t, []address.Address{0}, nameserver.Lookup("hostname"))
 
-	err = nameserver.Delete("hostname", "containerid", "", address.Address(0))
-	require.Nil(t, err)
+	nameserver.Delete("hostname", "containerid", "", address.Address(0))
 	require.Equal(t, []address.Address{}, nameserver.Lookup("hostname"))
 	require.Equal(t, l(Entries{Entry{
 		ContainerID: "containerid",

nameserver/status.go

@@ -1,10 +1,11 @@
 package nameserver
 
 type Status struct {
-	Domain  string
-	Address string
-	TTL     uint32
-	Entries []EntryStatus
+	Domain   string
+	Upstream []string
+	Address  string
+	TTL      uint32
+	Entries  []EntryStatus
 }
 
 type EntryStatus struct {
@@ -37,6 +38,7 @@ func NewStatus(ns *Nameserver, dnsServer *DNSServer) *Status {
 
 	return &Status{
 		dnsServer.domain,
+		dnsServer.upstream.Servers,
 		dnsServer.address,
 		dnsServer.ttl,
 		entryStatusSlice}

net/address/address.go

@@ -10,6 +10,7 @@ import (
 // Using 32-bit integer to represent IPv4 address
 type Address uint32
 type Offset uint32
+type Count uint32
 
 type Range struct {
 	Start, End Address // [Start, End); Start <= End
@@ -18,8 +19,8 @@ type Range struct {
 func NewRange(start Address, size Offset) Range {
 	return Range{Start: start, End: Add(start, size)}
 }
-func (r Range) Size() Offset               { return Subtract(r.End, r.Start) }
-func (r Range) String() string             { return fmt.Sprintf("[%s-%s)", r.Start, r.End) }
+func (r Range) Size() Count                { return Length(r.End, r.Start) }
+func (r Range) String() string             { return fmt.Sprintf("%s-%s", r.Start, r.End-1) }
 func (r Range) Overlaps(or Range) bool     { return !(r.Start >= or.End || r.End <= or.Start) }
 func (r Range) Contains(addr Address) bool { return addr >= r.Start && addr < r.End }
 
@@ -31,11 +32,15 @@ func (r Range) AsCIDRString() string {
 		}
 		prefixLen--
 	}
-	return CIDR{Start: r.Start, PrefixLen: prefixLen}.String()
+	return CIDR{Addr: r.Start, PrefixLen: prefixLen}.String()
+}
+
+func MakeCIDR(subnet CIDR, addr Address) CIDR {
+	return CIDR{Addr: addr, PrefixLen: subnet.PrefixLen}
 }
 
 type CIDR struct {
-	Start     Address
+	Addr      Address
 	PrefixLen int
 }
 
@@ -46,29 +51,34 @@ func ParseIP(s string) (Address, error) {
 	return 0, &net.ParseError{Type: "IP Address", Text: s}
 }
 
-func ParseCIDR(s string) (Address, CIDR, error) {
+func ParseCIDR(s string) (CIDR, error) {
 	if ip, ipnet, err := net.ParseCIDR(s); err != nil {
-		return 0, CIDR{}, err
+		return CIDR{}, err
 	} else if ipnet.IP.To4() == nil {
-		return 0, CIDR{}, &net.ParseError{Type: "Non-IPv4 address not supported", Text: s}
+		return CIDR{}, &net.ParseError{Type: "Non-IPv4 address not supported", Text: s}
 	} else {
 		prefixLen, _ := ipnet.Mask.Size()
-		return FromIP4(ip), CIDR{Start: FromIP4(ipnet.IP), PrefixLen: prefixLen}, nil
+		return CIDR{Addr: FromIP4(ip), PrefixLen: prefixLen}, nil
 	}
 }
 
+func (cidr CIDR) IsSubnet() bool {
+	mask := cidr.Size() - 1
+	return Offset(cidr.Addr)&mask == 0
+}
+
 func (cidr CIDR) Size() Offset { return 1 << uint(32-cidr.PrefixLen) }
 
 func (cidr CIDR) Range() Range {
-	return NewRange(cidr.Start, cidr.Size())
+	return NewRange(cidr.Addr, cidr.Size())
 }
 func (cidr CIDR) HostRange() Range {
 	// Respect RFC1122 exclusions of first and last addresses
-	return NewRange(cidr.Start+1, cidr.Size()-2)
+	return NewRange(cidr.Addr+1, cidr.Size()-2)
 }
 
 func (cidr CIDR) String() string {
-	return fmt.Sprintf("%s/%d", cidr.Start.String(), cidr.PrefixLen)
+	return fmt.Sprintf("%s/%d", cidr.Addr.String(), cidr.PrefixLen)
 }
 
 // FromIP4 converts an ipv4 address to our integer address type
@@ -107,6 +117,11 @@ func Subtract(a, b Address) Offset {
 	return Offset(a - b)
 }
 
+func Length(a, b Address) Count {
+	common.Assert(a >= b)
+	return Count(a - b)
+}
+
 func Min(a, b Offset) Offset {
 	if a > b {
 		return b

net/if.go

@@ -5,28 +5,35 @@ import (
 	"net"
 	"syscall"
 
-	"github.com/vishvananda/netlink/nl"
+	"github.com/vishvananda/netlink"
 )
 
 // Wait for an interface to come up.
 func EnsureInterface(ifaceName string) (*net.Interface, error) {
-	s, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK)
-	if err != nil {
-		return nil, err
-	}
-	defer s.Close()
-	iface, err := ensureInterface(s, ifaceName)
+	iface, err := ensureInterface(ifaceName)
 	if err != nil {
 		return nil, err
 	}
 	return iface, err
 }
 
-func ensureInterface(s *nl.NetlinkSocket, ifaceName string) (*net.Interface, error) {
+func ensureInterface(ifaceName string) (*net.Interface, error) {
+	ch := make(chan netlink.LinkUpdate)
+	// NB: We do not supply (and eventually close) a 'done' channel
+	// here since that can cause incorrect file descriptor
+	// re-usage. See https://github.com/weaveworks/weave/issues/2120
+	if err := netlink.LinkSubscribe(ch, nil); err != nil {
+		return nil, err
+	}
+	// check for currently-existing interface after subscribing, to avoid race
 	if iface, err := findInterface(ifaceName); err == nil {
 		return iface, nil
 	}
-	netlinkReceiveUntil(s, matchIfName(ifaceName))
+	for update := range ch {
+		if ifaceName == update.Link.Attrs().Name && update.IfInfomsg.Flags&syscall.IFF_UP != 0 {
+			break
+		}
+	}
 	iface, err := findInterface(ifaceName)
 	return iface, err
 }
@@ -35,21 +42,35 @@ func ensureInterface(s *nl.NetlinkSocket, ifaceName string) (*net.Interface, err
 // This matches the behaviour in 'weave attach', which is the only context in which
 // we expect this to be called.  If you change one, change the other to match.
 func EnsureInterfaceAndMcastRoute(ifaceName string) (*net.Interface, error) {
-	s, err := nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_LINK, syscall.RTNLGRP_IPV4_ROUTE)
+	iface, err := ensureInterface(ifaceName)
 	if err != nil {
 		return nil, err
 	}
-	defer s.Close()
-	iface, err := ensureInterface(s, ifaceName)
-	if err != nil {
+	ch := make(chan netlink.RouteUpdate)
+	if err := netlink.RouteSubscribe(ch, nil); err != nil {
 		return nil, err
 	}
 	dest := net.IPv4(224, 0, 0, 0)
-	if CheckRouteExists(ifaceName, dest) {
-		return iface, err
+	check := func(route netlink.Route) bool {
+		return route.Dst != nil && route.Dst.IP.Equal(dest)
 	}
-	netlinkReceiveUntil(s, matchRoute(ifaceName, dest))
-	return iface, err
+	// check for currently-existing route after subscribing, to avoid race
+	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
+	if err != nil {
+		return nil, err
+	}
+	for _, route := range routes {
+		if check(route) {
+			return iface, nil
+		}
+	}
+	for update := range ch {
+		if check(update.Route) {
+			return iface, nil
+		}
+	}
+	// should never get here
+	return iface, nil
 }
 
 func findInterface(ifaceName string) (iface *net.Interface, err error) {
@@ -61,40 +82,3 @@ func findInterface(ifaceName string) (iface *net.Interface, err error) {
 	}
 	return
 }
-
-func netlinkReceiveUntil(s *nl.NetlinkSocket, f func(syscall.NetlinkMessage) (bool, error)) error {
-	for {
-		msgs, err := s.Receive()
-		if err != nil {
-			return err
-		}
-		for _, m := range msgs {
-			if done, err := f(m); done {
-				return err
-			}
-		}
-	}
-}
-
-func matchIfName(ifaceName string) func(m syscall.NetlinkMessage) (bool, error) {
-	return func(m syscall.NetlinkMessage) (bool, error) {
-		switch m.Header.Type {
-		case syscall.RTM_NEWLINK: // receive this type for link 'up'
-			ifmsg := nl.DeserializeIfInfomsg(m.Data)
-			attrs, err := syscall.ParseNetlinkRouteAttr(&m)
-			if err != nil {
-				return true, err
-			}
-			name := ""
-			for _, attr := range attrs {
-				if attr.Attr.Type == syscall.IFA_LABEL {
-					name = string(attr.Value[:len(attr.Value)-1])
-				}
-			}
-			if ifaceName == name && ifmsg.Flags&syscall.IFF_UP != 0 {
-				return true, nil
-			}
-		}
-		return false, nil
-	}
-}

net/listen.go

@@ -0,0 +1,18 @@
+package net
+
+import (
+	"fmt"
+	"net"
+	"os"
+)
+
+func ListenUnixSocket(pathname string) (net.Listener, error) {
+	if err := os.Remove(pathname); err != nil && !os.IsNotExist(err) {
+		return nil, fmt.Errorf("Error removing existing unix socket: %s", err)
+	}
+	listener, err := net.Listen("unix", pathname)
+	if err != nil {
+		return nil, fmt.Errorf("ListenUnixSocket failed: %s", err)
+	}
+	return listener, nil
+}

net/route.go

@@ -3,16 +3,14 @@ package net
 import (
 	"fmt"
 	"net"
-	"syscall"
 
 	"github.com/vishvananda/netlink"
-	"github.com/vishvananda/netlink/nl"
 )
 
 // A network is considered free if it does not overlap any existing
 // routes on this host. This is the same approach taken by Docker.
 func CheckNetworkFree(subnet *net.IPNet, ignoreIfaceNames map[string]struct{}) error {
-	return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
+	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && overlaps(route.Dst, subnet) {
 			return fmt.Errorf("Network %s overlaps with existing route %s on host.", subnet, route.Dst)
 		}
@@ -28,7 +26,7 @@ func overlaps(n1, n2 *net.IPNet) bool {
 // For a specific address, we only care if it is actually *inside* an
 // existing route, because weave-local traffic never hits IP routing.
 func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) error {
-	return forEachRoute(ignoreIfaceNames, func(name string, route netlink.Route) error {
+	return forEachRoute(ignoreIfaceNames, func(route netlink.Route) error {
 		if route.Dst != nil && route.Dst.Contains(addr) {
 			return fmt.Errorf("Address %s overlaps with existing route %s on host.", addr, route.Dst)
 		}
@@ -36,80 +34,24 @@ func CheckAddressOverlap(addr net.IP, ignoreIfaceNames map[string]struct{}) erro
 	})
 }
 
-func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(name string, r netlink.Route) error) error {
+func forEachRoute(ignoreIfaceNames map[string]struct{}, check func(r netlink.Route) error) error {
+	ignoreIfaceIndices := make(map[int]struct{})
+	for ifaceName := range ignoreIfaceNames {
+		if iface, err := net.InterfaceByName(ifaceName); err == nil {
+			ignoreIfaceIndices[iface.Index] = struct{}{}
+		}
+	}
 	routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
 	if err != nil {
 		return err
 	}
 	for _, route := range routes {
-		link, err := netlink.LinkByIndex(route.LinkIndex)
-		if err == nil {
-			if _, found := ignoreIfaceNames[link.Attrs().Name]; found {
-				continue
-			}
+		if _, found := ignoreIfaceIndices[route.LinkIndex]; found {
+			continue
 		}
-		if err := check(link.Attrs().Name, route); err != nil {
+		if err := check(route); err != nil {
 			return err
 		}
 	}
 	return nil
 }
-
-func CheckRouteExists(ifaceName string, dest net.IP) bool {
-	found := false
-	forEachRoute(map[string]struct{}{}, func(name string, route netlink.Route) error {
-		if name == ifaceName && route.Dst.IP.Equal(dest) {
-			found = true
-		}
-		return nil
-	})
-	return found
-}
-
-func matchRoute(ifaceName string, dest net.IP) func(m syscall.NetlinkMessage) (bool, error) {
-	return func(m syscall.NetlinkMessage) (bool, error) {
-		switch m.Header.Type {
-		case syscall.RTM_NEWROUTE:
-			route, err := deserializeRoute(m.Data)
-			if err != nil {
-				return true, err
-			}
-			link, _ := netlink.LinkByIndex(route.LinkIndex)
-			if link.Attrs().Name == ifaceName && route.Dst.IP.Equal(dest) {
-				return true, nil
-			}
-		}
-		return false, nil
-	}
-}
-
-// This code proposed for addition to the vishvananda/netlink library
-// deserializeRoute decodes a binary netlink message into a Route struct
-func deserializeRoute(m []byte) (netlink.Route, error) {
-	route := netlink.Route{}
-	msg := nl.DeserializeRtMsg(m)
-	attrs, err := nl.ParseRouteAttr(m[msg.Len():])
-	if err != nil {
-		return route, err
-	}
-	route.Scope = netlink.Scope(msg.Scope)
-
-	native := nl.NativeEndian()
-	for _, attr := range attrs {
-		switch attr.Attr.Type {
-		case syscall.RTA_GATEWAY:
-			route.Gw = net.IP(attr.Value)
-		case syscall.RTA_PREFSRC:
-			route.Src = net.IP(attr.Value)
-		case syscall.RTA_DST:
-			route.Dst = &net.IPNet{
-				IP:   attr.Value,
-				Mask: net.CIDRMask(int(msg.Dst_len), 8*len(attr.Value)),
-			}
-		case syscall.RTA_OIF:
-			routeIndex := int(native.Uint32(attr.Value[0:4]))
-			route.LinkIndex = routeIndex
-		}
-	}
-	return route, nil
-}

plugin/README.md

@@ -0,0 +1,14 @@
+# Weave network driver extension for Docker and Kubernetes
+
+This directory implements both a 
+[Docker plugin](http://docs.docker.com/engine/extend/plugin_api/) to
+integrate [Weave Net](http://weave.works/net/) with Docker, and a 
+[Container Network Interface (CNI) plugin](https://github.com/appc/cni#cni---the-container-network-interface),
+to integrate Weave Net with [Kubernetes](http://kubernetes.io/).
+
+The Docker plugin runs automatically when you `weave launch`, provided your
+Docker daemon is version 1.9 or newer.
+
+More detail on the Docker plugin [here](http://docs.weave.works/weave/latest_release/plugin.html).
+
+More detail on the CNI plugin [here](https://github.com/weaveworks/weave/blob/master/site/plugin/cni-plugin-how-to.md).

plugin/ipam/cni.go

@@ -0,0 +1,80 @@
+package ipamplugin
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+
+	"github.com/appc/cni/pkg/skel"
+	"github.com/appc/cni/pkg/types"
+)
+
+func (i *Ipam) CmdAdd(args *skel.CmdArgs) error {
+	result, err := i.Allocate(args)
+	if err != nil {
+		return err
+	}
+	return result.Print()
+}
+
+func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) {
+	// extract the things we care about
+	conf, err := loadIPAMConf(args.StdinData)
+	if err != nil {
+		return nil, err
+	}
+	if conf == nil {
+		conf = &ipamConf{}
+	}
+	containerID := args.ContainerID
+	if containerID == "" {
+		return nil, fmt.Errorf("Weave CNI Allocate: blank container name")
+	}
+	var ipnet *net.IPNet
+
+	if conf.Subnet == "" {
+		ipnet, err = i.weave.AllocateIP(containerID)
+	} else {
+		var subnet *net.IPNet
+		subnet, err = types.ParseCIDR(conf.Subnet)
+		if err != nil {
+			return nil, fmt.Errorf("subnet given in config, but not parseable: %s", err)
+		}
+		ipnet, err = i.weave.AllocateIPInSubnet(containerID, subnet)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	result := &types.Result{
+		IP4: &types.IPConfig{
+			IP:      *ipnet,
+			Gateway: conf.Gateway,
+			Routes:  conf.Routes,
+		},
+	}
+	return result, nil
+}
+
+func (i *Ipam) CmdDel(args *skel.CmdArgs) error {
+	return i.Release(args)
+}
+
+func (i *Ipam) Release(args *skel.CmdArgs) error {
+	return i.weave.ReleaseIPsFor(args.ContainerID)
+}
+
+type ipamConf struct {
+	Subnet  string        `json:"subnet,omitempty"`
+	Gateway net.IP        `json:"gateway,omitempty"`
+	Routes  []types.Route `json:"routes"`
+}
+
+type netConf struct {
+	IPAM *ipamConf `json:"ipam"`
+}
+
+func loadIPAMConf(stdinData []byte) (*ipamConf, error) {
+	var conf netConf
+	return conf.IPAM, json.Unmarshal(stdinData, &conf)
+}

plugin/ipam/driver.go

@@ -0,0 +1,119 @@
+package ipamplugin
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/docker/libnetwork/discoverapi"
+	"github.com/docker/libnetwork/netlabel"
+	"github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+)
+
+type Ipam struct {
+	weave *api.Client
+}
+
+func NewIpam(weave *api.Client) *Ipam {
+	return &Ipam{weave: weave}
+}
+
+func (i *Ipam) GetDefaultAddressSpaces() (string, string, error) {
+	i.logReq("GetDefaultAddressSpaces")
+	return "weavelocal", "weaveglobal", nil
+}
+
+func (i *Ipam) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (poolname string, subnet *net.IPNet, data map[string]string, err error) {
+	i.logReq("RequestPool", addressSpace, pool, subPool, options)
+	defer func() { i.logRes("RequestPool", err, poolname, subnet, data) }()
+	if pool == "" {
+		subnet, err = i.weave.DefaultSubnet()
+	} else {
+		_, subnet, err = net.ParseCIDR(pool)
+	}
+	if err != nil {
+		return
+	}
+	iprange := subnet
+	if subPool != "" {
+		if _, iprange, err = net.ParseCIDR(subPool); err != nil {
+			return
+		}
+	}
+	// Cunningly-constructed pool "name" which gives us what we need later
+	poolname = strings.Join([]string{"weave", subnet.String(), iprange.String()}, "-")
+	// Pass back a fake "gateway address"; we don't actually use it,
+	// so just give the network address.
+	data = map[string]string{netlabel.Gateway: subnet.String()}
+	return
+}
+
+func (i *Ipam) ReleasePool(poolID string) error {
+	i.logReq("ReleasePool", poolID)
+	return nil
+}
+
+func (i *Ipam) RequestAddress(poolID string, address net.IP, options map[string]string) (ip *net.IPNet, _ map[string]string, err error) {
+	i.logReq("RequestAddress", poolID, address, options)
+	defer func() { i.logRes("RequestAddress", err, ip) }()
+	// If we pass magic string "_" to weave IPAM it stores the address under its own string
+	if poolID == "weavepool" { // old-style
+		ip, err = i.weave.AllocateIP("_")
+		return
+	}
+	parts := strings.Split(poolID, "-")
+	if len(parts) != 3 || parts[0] != "weave" {
+		err = fmt.Errorf("Unrecognized pool ID: %s", poolID)
+		return
+	}
+	var subnet, iprange *net.IPNet
+	if _, subnet, err = net.ParseCIDR(parts[1]); err != nil {
+		return
+	}
+	if address != nil { // try to claim specific address requested
+		ip = &net.IPNet{IP: address, Mask: subnet.Mask}
+		if err = i.weave.ClaimIP("_", ip); err != nil {
+			return
+		}
+	} else {
+		if _, iprange, err = net.ParseCIDR(parts[2]); err != nil {
+			return
+		}
+		// We are lying slightly to IPAM here: the range is not a subnet
+		if ip, err = i.weave.AllocateIPInSubnet("_", iprange); err != nil {
+			return
+		}
+		ip.Mask = subnet.Mask // fix up the subnet we lied about
+	}
+	return
+}
+
+func (i *Ipam) ReleaseAddress(poolID string, address net.IP) error {
+	i.logReq("ReleaseAddress", poolID, address)
+	return i.weave.ReleaseIPsFor(address.String())
+}
+
+// Functions required by ipamapi "contract" but not actually used.
+
+func (i *Ipam) DiscoverNew(discoverapi.DiscoveryType, interface{}) error {
+	return nil
+}
+
+func (i *Ipam) DiscoverDelete(discoverapi.DiscoveryType, interface{}) error {
+	return nil
+}
+
+// logging
+
+func (i *Ipam) logReq(fun string, args ...interface{}) {
+	common.Log.Infoln(append([]interface{}{fmt.Sprintf("[ipam] %s", fun)}, args...)...)
+}
+
+func (i *Ipam) logRes(fun string, err error, args ...interface{}) {
+	if err == nil {
+		common.Log.Debugln(append([]interface{}{fmt.Sprintf("[ipam] %s result", fun)}, args...)...)
+		return
+	}
+	common.Log.Errorf("[ipam] %s: %s", fun, err)
+}

plugin/net/cni.go

@@ -0,0 +1,240 @@
+package plugin
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+
+	"github.com/appc/cni/pkg/ipam"
+	"github.com/appc/cni/pkg/skel"
+	"github.com/appc/cni/pkg/types"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
+)
+
+var (
+	zeroNetwork = net.IPNet{IP: net.IPv4zero, Mask: net.IPv4Mask(0, 0, 0, 0)}
+	mask32      = net.IPv4Mask(0xff, 0xff, 0xff, 0xff)
+)
+
+type CNIPlugin struct {
+	weave *weaveapi.Client
+}
+
+func NewCNIPlugin(weave *weaveapi.Client) *CNIPlugin {
+	return &CNIPlugin{weave: weave}
+}
+
+func loadNetConf(bytes []byte) (*NetConf, error) {
+	n := &NetConf{
+		BrName: "weave",
+	}
+	if err := json.Unmarshal(bytes, n); err != nil {
+		return nil, fmt.Errorf("failed to load netconf: %v", err)
+	}
+	return n, nil
+}
+
+func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
+	conf, err := loadNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	if conf.IsGW {
+		return fmt.Errorf("Gateway functionality not supported")
+	}
+	if conf.IPMasq {
+		return fmt.Errorf("IP Masquerading functionality not supported")
+	}
+
+	ns, err := netns.GetFromPath(args.Netns)
+	if err != nil {
+		return err
+	}
+	defer ns.Close()
+
+	id := args.ContainerID
+	if len(id) < 5 {
+		data := make([]byte, 5)
+		_, err := rand.Reader.Read(data)
+		if err != nil {
+			return err
+		}
+		id = fmt.Sprintf("%x", data)
+	}
+
+	local, err := createAndAttach(id, conf.BrName, conf.MTU)
+	if err != nil {
+		return err
+	}
+
+	cleanup := func(err error) error {
+		netlink.LinkDel(local)
+		return err
+	}
+	guest, err := netlink.LinkByName(local.PeerName)
+	if err != nil {
+		return cleanup(err)
+	}
+	if err = netlink.LinkSetNsFd(guest, int(ns)); err != nil {
+		return cleanup(fmt.Errorf("failed to move veth to container netns: %s", err))
+	}
+
+	if err := netlink.LinkSetUp(local); err != nil {
+		return cleanup(fmt.Errorf("unable to bring veth up: %s", err))
+	}
+
+	var result *types.Result
+	// Default IPAM is Weave's own
+	if conf.IPAM.Type == "" {
+		result, err = ipamplugin.NewIpam(c.weave).Allocate(args)
+	} else {
+		result, err = ipam.ExecAdd(conf.IPAM.Type, args.StdinData)
+	}
+	if err != nil {
+		return cleanup(fmt.Errorf("unable to allocate IP address: %s", err))
+	}
+	if result.IP4 == nil {
+		return cleanup(fmt.Errorf("IPAM plugin failed to allocate IP address"))
+	}
+
+	// If config says nothing about routes or gateway, default one will be via the bridge
+	if result.IP4.Routes == nil && result.IP4.Gateway == nil {
+		bridgeIP, err := findBridgeIP(conf.BrName, result.IP4.IP)
+		if err != nil {
+			return cleanup(err)
+		}
+		result.IP4.Gateway = bridgeIP
+	}
+
+	err = common.WithNetNS(ns, func() error {
+		return setupGuestIP4(local.PeerName, args.IfName, result.IP4.IP, result.IP4.Gateway, result.IP4.Routes)
+	})
+	if err != nil {
+		return cleanup(fmt.Errorf("error setting up interface: %s", err))
+	}
+
+	result.DNS = conf.DNS
+	return result.Print()
+}
+
+func setupGuestIP4(origName, name string, ipnet net.IPNet, gw net.IP, routes []types.Route) error {
+	guest, err := netlink.LinkByName(origName)
+	if err != nil {
+		return err
+	}
+	if err = netlink.LinkSetName(guest, name); err != nil {
+		return err
+	}
+	if err = netlink.LinkSetUp(guest); err != nil {
+		return err
+	}
+	if err = common.ConfigureARPCache(name); err != nil {
+		return err
+	}
+	if routes == nil { // If config says nothing about routes, add a default one
+		if !ipnet.Contains(gw) {
+			// The bridge IP is not on the same subnet; add a specific route to it
+			gw32 := &net.IPNet{IP: gw, Mask: mask32}
+			if err = addRoute(guest, netlink.SCOPE_LINK, gw32, nil); err != nil {
+				return err
+			}
+		}
+		routes = []types.Route{{Dst: zeroNetwork}}
+	}
+	if err = netlink.AddrAdd(guest, &netlink.Addr{IPNet: &ipnet}); err != nil {
+		return fmt.Errorf("failed to add IP address to %q: %v", name, err)
+	}
+	for _, r := range routes {
+		if r.GW != nil {
+			err = addRoute(guest, netlink.SCOPE_UNIVERSE, &r.Dst, r.GW)
+		} else {
+			err = addRoute(guest, netlink.SCOPE_UNIVERSE, &r.Dst, gw)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, name, err)
+		}
+	}
+	return nil
+}
+
+func addRoute(link netlink.Link, scope netlink.Scope, dst *net.IPNet, gw net.IP) error {
+	err := netlink.RouteAdd(&netlink.Route{
+		LinkIndex: link.Attrs().Index,
+		Scope:     scope,
+		Dst:       dst,
+		Gw:        gw,
+	})
+	if os.IsExist(err) { // squash duplicate route errors
+		err = nil
+	}
+	return err
+}
+
+func findBridgeIP(bridgeName string, subnet net.IPNet) (net.IP, error) {
+	netdevs, err := common.GetBridgeNetDev(bridgeName)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get netdev for %q bridge: %s", bridgeName, err)
+	}
+	if len(netdevs) == 0 {
+		return nil, fmt.Errorf("Could not find %q bridge", bridgeName)
+	}
+	if len(netdevs[0].CIDRs) == 0 {
+		return nil, fmt.Errorf("Bridge %q has no IP addresses", bridgeName)
+	}
+	for _, cidr := range netdevs[0].CIDRs {
+		if subnet.Contains(cidr.IP) {
+			return cidr.IP, nil
+		}
+	}
+	// None in the required subnet; just return the first one
+	return netdevs[0].CIDRs[0].IP, nil
+}
+
+func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
+	conf, err := loadNetConf(args.StdinData)
+	if err != nil {
+		return err
+	}
+
+	ns, err := netns.GetFromPath(args.Netns)
+	if err != nil {
+		return err
+	}
+	defer ns.Close()
+	err = common.WithNetNS(ns, func() error {
+		link, err := netlink.LinkByName(args.IfName)
+		if err != nil {
+			return err
+		}
+		return netlink.LinkDel(link)
+	})
+	if err != nil {
+		return fmt.Errorf("error removing interface: %s", err)
+	}
+
+	// Default IPAM is Weave's own
+	if conf.IPAM.Type == "" {
+		err = ipamplugin.NewIpam(c.weave).Release(args)
+	} else {
+		err = ipam.ExecDel(conf.IPAM.Type, args.StdinData)
+	}
+	if err != nil {
+		return fmt.Errorf("unable to release IP address: %s", err)
+	}
+	return nil
+}
+
+type NetConf struct {
+	types.NetConf
+	BrName string `json:"bridge"`
+	IsGW   bool   `json:"isGateway"`
+	IPMasq bool   `json:"ipMasq"`
+	MTU    int    `json:"mtu"`
+}

plugin/net/driver.go

@@ -0,0 +1,226 @@
+package plugin
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/docker/libnetwork/drivers/remote/api"
+	"github.com/docker/libnetwork/types"
+
+	"github.com/vishvananda/netlink"
+	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/common/odp"
+	"github.com/weaveworks/weave/plugin/skel"
+)
+
+const (
+	WeaveBridge = "weave"
+)
+
+type driver struct {
+	scope            string
+	noMulticastRoute bool
+	sync.RWMutex
+	endpoints map[string]struct{}
+}
+
+func New(client *docker.Client, weave *weaveapi.Client, scope string, noMulticastRoute bool) (skel.Driver, error) {
+	driver := &driver{
+		noMulticastRoute: noMulticastRoute,
+		scope:            scope,
+		endpoints:        make(map[string]struct{}),
+	}
+
+	_, err := NewWatcher(client, weave, driver)
+	if err != nil {
+		return nil, err
+	}
+	return driver, nil
+}
+
+// === protocol handlers
+
+func (driver *driver) GetCapabilities() (*api.GetCapabilityResponse, error) {
+	driver.logReq("GetCapabilities", nil, "")
+	var caps = &api.GetCapabilityResponse{
+		Scope: driver.scope,
+	}
+	driver.logRes("GetCapabilities", caps)
+	return caps, nil
+}
+
+func (driver *driver) CreateNetwork(create *api.CreateNetworkRequest) error {
+	driver.logReq("CreateNetwork", create, create.NetworkID)
+	return nil
+}
+
+func (driver *driver) DeleteNetwork(delete *api.DeleteNetworkRequest) error {
+	driver.logReq("DeleteNetwork", delete, delete.NetworkID)
+	return nil
+}
+
+func (driver *driver) CreateEndpoint(create *api.CreateEndpointRequest) (*api.CreateEndpointResponse, error) {
+	driver.logReq("CreateEndpoint", create, create.EndpointID)
+	endID := create.EndpointID
+
+	if create.Interface == nil {
+		return nil, driver.error("CreateEndpoint", "Not supported: creating an interface from within CreateEndpoint")
+	}
+	driver.Lock()
+	driver.endpoints[endID] = struct{}{}
+	driver.Unlock()
+	resp := &api.CreateEndpointResponse{}
+
+	driver.logRes("CreateEndpoint", resp)
+	return resp, nil
+}
+
+func (driver *driver) DeleteEndpoint(deleteReq *api.DeleteEndpointRequest) error {
+	driver.logReq("DeleteEndpoint", deleteReq, deleteReq.EndpointID)
+	driver.Lock()
+	delete(driver.endpoints, deleteReq.EndpointID)
+	driver.Unlock()
+	return nil
+}
+
+func (driver *driver) HasEndpoint(endpointID string) bool {
+	driver.Lock()
+	_, found := driver.endpoints[endpointID]
+	driver.Unlock()
+	return found
+}
+
+func (driver *driver) EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error) {
+	driver.logReq("EndpointInfo", req, req.EndpointID)
+	return &api.EndpointInfoResponse{Value: map[string]interface{}{}}, nil
+}
+
+func (driver *driver) JoinEndpoint(j *api.JoinRequest) (*api.JoinResponse, error) {
+	driver.logReq("JoinEndpoint", j, fmt.Sprintf("%s:%s to %s", j.NetworkID, j.EndpointID, j.SandboxKey))
+
+	local, err := createAndAttach(j.EndpointID, WeaveBridge, 0)
+	if err != nil {
+		return nil, driver.error("JoinEndpoint", "%s", err)
+	}
+
+	if err := netlink.LinkSetUp(local); err != nil {
+		return nil, driver.error("JoinEndpoint", "unable to bring up veth %s-%s: %s", local.Name, local.PeerName, err)
+	}
+
+	ifname := &api.InterfaceName{
+		SrcName:   local.PeerName,
+		DstPrefix: "ethwe",
+	}
+
+	response := &api.JoinResponse{
+		InterfaceName: ifname,
+	}
+	if !driver.noMulticastRoute {
+		multicastRoute := api.StaticRoute{
+			Destination: "224.0.0.0/4",
+			RouteType:   types.CONNECTED,
+		}
+		response.StaticRoutes = append(response.StaticRoutes, multicastRoute)
+	}
+	driver.logRes("JoinEndpoint", response)
+	return response, nil
+}
+
+// create and attach local name to the Weave bridge
+func createAndAttach(id, bridgeName string, mtu int) (*netlink.Veth, error) {
+	maybeBridge, err := netlink.LinkByName(bridgeName)
+	if err != nil {
+		return nil, fmt.Errorf(`bridge "%s" not present; did you launch weave?`, bridgeName)
+	}
+
+	local := vethPair(id[:5])
+	if mtu == 0 {
+		local.Attrs().MTU = maybeBridge.Attrs().MTU
+	} else {
+		local.Attrs().MTU = mtu
+	}
+	if err := netlink.LinkAdd(local); err != nil {
+		return nil, fmt.Errorf(`could not create veth pair %s-%s: %s`, local.Name, local.PeerName, err)
+	}
+
+	switch maybeBridge.(type) {
+	case *netlink.Bridge:
+		if err := netlink.LinkSetMasterByIndex(local, maybeBridge.Attrs().Index); err != nil {
+			return nil, fmt.Errorf(`unable to set master of %s: %s`, local.Name, err)
+		}
+	case *netlink.GenericLink:
+		if maybeBridge.Type() != "openvswitch" {
+			return nil, fmt.Errorf(`device "%s" is of type "%s"`, bridgeName, maybeBridge.Type())
+		}
+		if err := odp.AddDatapathInterface(bridgeName, local.Name); err != nil {
+			return nil, fmt.Errorf(`failed to attach %s to device "%s": %s`, local.Name, bridgeName, err)
+		}
+	case *netlink.Device:
+		// Assume it's our openvswitch device, and the kernel has not been updated to report the kind.
+		if err := odp.AddDatapathInterface(bridgeName, local.Name); err != nil {
+			return nil, fmt.Errorf(`failed to attach %s to device "%s": %s`, local.Name, bridgeName, err)
+		}
+	default:
+		return nil, fmt.Errorf(`device "%s" is not a bridge`, bridgeName)
+	}
+	return local, nil
+}
+
+func (driver *driver) LeaveEndpoint(leave *api.LeaveRequest) error {
+	driver.logReq("LeaveEndpoint", leave, fmt.Sprintf("%s:%s", leave.NetworkID, leave.EndpointID))
+
+	local := vethPair(leave.EndpointID[:5])
+	if err := netlink.LinkDel(local); err != nil {
+		driver.warn("LeaveEndpoint", "unable to delete veth: %s", err)
+	}
+	return nil
+}
+
+func (driver *driver) DiscoverNew(disco *api.DiscoveryNotification) error {
+	driver.logReq("DiscoverNew", disco, "")
+	return nil
+}
+
+func (driver *driver) DiscoverDelete(disco *api.DiscoveryNotification) error {
+	driver.logReq("DiscoverDelete", disco, "")
+	return nil
+}
+
+// ===
+
+func vethPair(suffix string) *netlink.Veth {
+	return &netlink.Veth{
+		LinkAttrs: netlink.LinkAttrs{Name: "vethwl" + suffix},
+		PeerName:  "vethwg" + suffix,
+	}
+}
+
+// logging
+
+func (driver *driver) logReq(fun string, req interface{}, short string) {
+	driver.log(common.Log.Debugf, " %+v", fun, req)
+	common.Log.Infof("[net] %s %s", fun, short)
+}
+
+func (driver *driver) logRes(fun string, res interface{}) {
+	driver.log(common.Log.Debugf, " %+v", fun, res)
+}
+
+func (driver *driver) warn(fun string, format string, a ...interface{}) {
+	driver.log(common.Log.Warnf, ": "+format, fun, a...)
+}
+
+func (driver *driver) debug(fun string, format string, a ...interface{}) {
+	driver.log(common.Log.Debugf, ": "+format, fun, a...)
+}
+
+func (driver *driver) error(fun string, format string, a ...interface{}) error {
+	driver.log(common.Log.Errorf, ": "+format, fun, a...)
+	return fmt.Errorf(format, a...)
+}
+
+func (driver *driver) log(f func(string, ...interface{}), format string, fun string, a ...interface{}) {
+	f("[net] %s"+format, append([]interface{}{fun}, a...)...)
+}

plugin/net/watcher.go

@@ -0,0 +1,51 @@
+package plugin
+
+import (
+	"fmt"
+
+	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common/docker"
+)
+
+const (
+	WeaveDomain = "weave.local"
+)
+
+type watcher struct {
+	client *docker.Client
+	weave  *weaveapi.Client
+	driver *driver
+}
+
+type Watcher interface {
+}
+
+func NewWatcher(client *docker.Client, weave *weaveapi.Client, driver *driver) (Watcher, error) {
+	w := &watcher{client: client, weave: weave, driver: driver}
+	return w, client.AddObserver(w)
+}
+
+func (w *watcher) ContainerStarted(id string) {
+	w.driver.debug("ContainerStarted", "%s", id)
+	info, err := w.client.InspectContainer(id)
+	if err != nil {
+		w.driver.warn("ContainerStarted", "error inspecting container %s: %s", id, err)
+		return
+	}
+	// check that it's on our network, via the endpointID
+	for _, net := range info.NetworkSettings.Networks {
+		if w.driver.HasEndpoint(net.EndpointID) {
+			fqdn := fmt.Sprintf("%s.%s", info.Config.Hostname, info.Config.Domainname)
+			if err := w.weave.RegisterWithDNS(id, fqdn, net.IPAddress); err != nil {
+				w.driver.warn("ContainerStarted", "unable to register %s with weaveDNS: %s", id, err)
+			}
+		}
+	}
+}
+
+func (w *watcher) ContainerDied(id string) {
+	// don't need to do this as WeaveDNS removes names on container died anyway
+	// (note by the time we get this event we can't see the EndpointID)
+}
+
+func (w *watcher) ContainerDestroyed(id string) {}

plugin/skel/listener.go

@@ -0,0 +1,299 @@
+package skel
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+
+	"github.com/gorilla/mux"
+
+	"github.com/docker/libnetwork/drivers/remote/api"
+	"github.com/docker/libnetwork/ipamapi"
+	iapi "github.com/docker/libnetwork/ipams/remote/api"
+)
+
+const (
+	networkReceiver = "NetworkDriver"
+	ipamReceiver    = ipamapi.PluginEndpointType
+)
+
+type Driver interface {
+	GetCapabilities() (*api.GetCapabilityResponse, error)
+	CreateNetwork(create *api.CreateNetworkRequest) error
+	DeleteNetwork(delete *api.DeleteNetworkRequest) error
+	CreateEndpoint(create *api.CreateEndpointRequest) (*api.CreateEndpointResponse, error)
+	DeleteEndpoint(delete *api.DeleteEndpointRequest) error
+	EndpointInfo(req *api.EndpointInfoRequest) (*api.EndpointInfoResponse, error)
+	JoinEndpoint(j *api.JoinRequest) (response *api.JoinResponse, error error)
+	LeaveEndpoint(leave *api.LeaveRequest) error
+	DiscoverNew(discover *api.DiscoveryNotification) error
+	DiscoverDelete(delete *api.DiscoveryNotification) error
+}
+
+type listener struct {
+	d Driver
+	i ipamapi.Ipam
+}
+
+func Listen(socket net.Listener, driver Driver, ipamDriver ipamapi.Ipam) error {
+	router := mux.NewRouter()
+	router.NotFoundHandler = http.HandlerFunc(notFound)
+
+	listener := &listener{driver, ipamDriver}
+
+	router.Methods("POST").Path("/Plugin.Activate").HandlerFunc(listener.handshake)
+
+	handleMethod := func(receiver, method string, h http.HandlerFunc) {
+		router.Methods("POST").Path(fmt.Sprintf("/%s.%s", receiver, method)).HandlerFunc(h)
+	}
+
+	handleMethod(networkReceiver, "GetCapabilities", listener.getCapabilities)
+
+	if driver != nil {
+		handleMethod(networkReceiver, "CreateNetwork", listener.createNetwork)
+		handleMethod(networkReceiver, "DeleteNetwork", listener.deleteNetwork)
+		handleMethod(networkReceiver, "CreateEndpoint", listener.createEndpoint)
+		handleMethod(networkReceiver, "DeleteEndpoint", listener.deleteEndpoint)
+		handleMethod(networkReceiver, "EndpointOperInfo", listener.infoEndpoint)
+		handleMethod(networkReceiver, "Join", listener.joinEndpoint)
+		handleMethod(networkReceiver, "Leave", listener.leaveEndpoint)
+	}
+
+	if ipamDriver != nil {
+		handleMethod(ipamReceiver, "GetDefaultAddressSpaces", listener.getDefaultAddressSpaces)
+		handleMethod(ipamReceiver, "RequestPool", listener.requestPool)
+		handleMethod(ipamReceiver, "ReleasePool", listener.releasePool)
+		handleMethod(ipamReceiver, "RequestAddress", listener.requestAddress)
+		handleMethod(ipamReceiver, "ReleaseAddress", listener.releaseAddress)
+	}
+
+	return http.Serve(socket, router)
+}
+
+func decode(w http.ResponseWriter, r *http.Request, v interface{}) error {
+	err := json.NewDecoder(r.Body).Decode(v)
+	if err != nil {
+		sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest)
+	}
+	return err
+}
+
+// === protocol handlers
+
+type handshakeResp struct {
+	Implements []string
+}
+
+func (listener *listener) handshake(w http.ResponseWriter, r *http.Request) {
+	var resp handshakeResp
+	if listener.d != nil {
+		resp.Implements = append(resp.Implements, "NetworkDriver")
+	}
+	if listener.i != nil {
+		resp.Implements = append(resp.Implements, "IpamDriver")
+	}
+	err := json.NewEncoder(w).Encode(&resp)
+	if err != nil {
+		sendError(w, "encode error", http.StatusInternalServerError)
+		return
+	}
+}
+
+func (listener *listener) getCapabilities(w http.ResponseWriter, r *http.Request) {
+	caps, err := listener.d.GetCapabilities()
+	objectOrErrorResponse(w, caps, err)
+}
+
+func (listener *listener) createNetwork(w http.ResponseWriter, r *http.Request) {
+	var create api.CreateNetworkRequest
+	err := json.NewDecoder(r.Body).Decode(&create)
+	if err != nil {
+		sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest)
+		return
+	}
+	emptyOrErrorResponse(w, listener.d.CreateNetwork(&create))
+}
+
+func (listener *listener) deleteNetwork(w http.ResponseWriter, r *http.Request) {
+	var delete api.DeleteNetworkRequest
+	if err := json.NewDecoder(r.Body).Decode(&delete); err != nil {
+		sendError(w, "Unable to decode JSON payload: "+err.Error(), http.StatusBadRequest)
+		return
+	}
+	emptyOrErrorResponse(w, listener.d.DeleteNetwork(&delete))
+}
+
+func (listener *listener) createEndpoint(w http.ResponseWriter, r *http.Request) {
+	var create api.CreateEndpointRequest
+	if err := json.NewDecoder(r.Body).Decode(&create); err != nil {
+		sendError(w, "unable to decode JSON payload: "+err.Error(), http.StatusBadRequest)
+		return
+	}
+	res, err := listener.d.CreateEndpoint(&create)
+	objectOrErrorResponse(w, res, err)
+}
+
+func (listener *listener) deleteEndpoint(w http.ResponseWriter, r *http.Request) {
+	var delete api.DeleteEndpointRequest
+	if err := json.NewDecoder(r.Body).Decode(&delete); err != nil {
+		sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest)
+		return
+	}
+	emptyOrErrorResponse(w, listener.d.DeleteEndpoint(&delete))
+}
+
+func (listener *listener) infoEndpoint(w http.ResponseWriter, r *http.Request) {
+	var req api.EndpointInfoRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest)
+		return
+	}
+	info, err := listener.d.EndpointInfo(&req)
+	objectOrErrorResponse(w, info, err)
+}
+
+func (listener *listener) joinEndpoint(w http.ResponseWriter, r *http.Request) {
+	var join api.JoinRequest
+	if err := json.NewDecoder(r.Body).Decode(&join); err != nil {
+		sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest)
+		return
+	}
+	res, err := listener.d.JoinEndpoint(&join)
+	objectOrErrorResponse(w, res, err)
+}
+
+func (listener *listener) leaveEndpoint(w http.ResponseWriter, r *http.Request) {
+	var l api.LeaveRequest
+	if err := json.NewDecoder(r.Body).Decode(&l); err != nil {
+		sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest)
+		return
+	}
+	emptyOrErrorResponse(w, listener.d.LeaveEndpoint(&l))
+}
+
+func (listener *listener) discoverNew(w http.ResponseWriter, r *http.Request) {
+	var disco api.DiscoveryNotification
+	if err := json.NewDecoder(r.Body).Decode(&disco); err != nil {
+		sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest)
+		return
+	}
+	emptyOrErrorResponse(w, listener.d.DiscoverNew(&disco))
+}
+
+func (listener *listener) discoverDelete(w http.ResponseWriter, r *http.Request) {
+	var disco api.DiscoveryNotification
+	if err := json.NewDecoder(r.Body).Decode(&disco); err != nil {
+		sendError(w, "Could not decode JSON encode payload", http.StatusBadRequest)
+		return
+	}
+	emptyOrErrorResponse(w, listener.d.DiscoverDelete(&disco))
+}
+
+// ===
+
+func (listener *listener) getDefaultAddressSpaces(w http.ResponseWriter, r *http.Request) {
+	local, global, err := listener.i.GetDefaultAddressSpaces()
+	response := &iapi.GetAddressSpacesResponse{
+		LocalDefaultAddressSpace:  local,
+		GlobalDefaultAddressSpace: global,
+	}
+	objectOrErrorResponse(w, response, err)
+}
+
+func (listener *listener) requestPool(w http.ResponseWriter, r *http.Request) {
+	var rq iapi.RequestPoolRequest
+	if err := decode(w, r, &rq); err != nil {
+		return
+	}
+	poolID, pool, data, err := listener.i.RequestPool(rq.AddressSpace, rq.Pool, rq.SubPool, rq.Options, rq.V6)
+	if err != nil {
+		errorResponse(w, err.Error())
+		return
+	}
+	response := &iapi.RequestPoolResponse{
+		PoolID: poolID,
+		Pool:   pool.String(),
+		Data:   data,
+	}
+	objectResponse(w, response)
+}
+
+func (listener *listener) releasePool(w http.ResponseWriter, r *http.Request) {
+	var rq iapi.ReleasePoolRequest
+	if err := decode(w, r, &rq); err != nil {
+		return
+	}
+	err := listener.i.ReleasePool(rq.PoolID)
+	emptyOrErrorResponse(w, err)
+}
+
+func (listener *listener) requestAddress(w http.ResponseWriter, r *http.Request) {
+	var rq iapi.RequestAddressRequest
+	if err := decode(w, r, &rq); err != nil {
+		return
+	}
+	address, data, err := listener.i.RequestAddress(rq.PoolID, net.ParseIP(rq.Address), rq.Options)
+	if err != nil {
+		errorResponse(w, err.Error())
+		return
+	}
+	response := &iapi.RequestAddressResponse{
+		Address: address.String(),
+		Data:    data,
+	}
+	objectResponse(w, response)
+}
+
+func (listener *listener) releaseAddress(w http.ResponseWriter, r *http.Request) {
+	var rq iapi.ReleaseAddressRequest
+	if err := decode(w, r, &rq); err != nil {
+		return
+	}
+	err := listener.i.ReleaseAddress(rq.PoolID, net.ParseIP(rq.Address))
+	emptyOrErrorResponse(w, err)
+}
+
+// ===
+
+func notFound(w http.ResponseWriter, r *http.Request) {
+	http.NotFound(w, r)
+}
+
+func sendError(w http.ResponseWriter, msg string, code int) {
+	http.Error(w, msg, code)
+}
+
+func errorResponse(w http.ResponseWriter, fmtString string, item ...interface{}) {
+	w.WriteHeader(http.StatusInternalServerError)
+	json.NewEncoder(w).Encode(map[string]string{
+		"Err": fmt.Sprintf(fmtString, item...),
+	})
+}
+
+func objectResponse(w http.ResponseWriter, obj interface{}) {
+	if err := json.NewEncoder(w).Encode(obj); err != nil {
+		sendError(w, "Could not JSON encode response", http.StatusInternalServerError)
+		return
+	}
+}
+
+func emptyResponse(w http.ResponseWriter) {
+	json.NewEncoder(w).Encode(map[string]string{})
+}
+
+func objectOrErrorResponse(w http.ResponseWriter, obj interface{}, err error) {
+	if err != nil {
+		errorResponse(w, err.Error())
+		return
+	}
+	objectResponse(w, obj)
+}
+
+func emptyOrErrorResponse(w http.ResponseWriter, err error) {
+	if err != nil {
+		errorResponse(w, err.Error())
+		return
+	}
+	emptyResponse(w)
+}

prog/netcheck/netcheck.go

@@ -1,54 +0,0 @@
-/* netcheck: check whether a given network or address overlaps with any existing routes */
-package main
-
-import (
-	"fmt"
-	"net"
-	"os"
-
-	"github.com/docker/docker/pkg/mflag"
-	weavenet "github.com/weaveworks/weave/net"
-)
-
-func fatal(err error) {
-	fmt.Fprintln(os.Stderr, err)
-	os.Exit(1)
-}
-
-type stringList map[string]struct{}
-
-func (ss *stringList) Set(value string) error {
-	(*ss)[value] = struct{}{}
-	return nil
-}
-
-func (ss *stringList) String() string {
-	return fmt.Sprintf("%v", *ss)
-}
-
-func main() {
-	ignoreIfaceNames := make(stringList)
-
-	mflag.Var(&ignoreIfaceNames, []string{"-ignore-iface"}, "name of interface to ignore)")
-	mflag.Parse()
-
-	if len(mflag.Args()) < 1 {
-		fmt.Fprintln(os.Stderr, "usage: netcheck [--ignore-iface <iface-name>] network-cidr")
-		os.Exit(1)
-	}
-
-	cidrStr := mflag.Args()[0]
-	addr, ipnet, err := net.ParseCIDR(cidrStr)
-	if err != nil {
-		fatal(err)
-	}
-	if ipnet.IP.Equal(addr) {
-		err = weavenet.CheckNetworkFree(ipnet, ignoreIfaceNames)
-	} else {
-		err = weavenet.CheckAddressOverlap(addr, ignoreIfaceNames)
-	}
-	if err != nil {
-		fatal(err)
-	}
-	os.Exit(0)
-}

prog/plugin/Dockerfile

@@ -0,0 +1,3 @@
+FROM weaveworks/weaveexec
+ADD ./plugin /home/weave/
+ENTRYPOINT ["/home/weave/plugin"]

prog/plugin/main.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+
+	cni "github.com/appc/cni/pkg/skel"
+	"github.com/docker/libnetwork/ipamapi"
+	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common/docker"
+	weavenet "github.com/weaveworks/weave/net"
+	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
+	netplugin "github.com/weaveworks/weave/plugin/net"
+	"github.com/weaveworks/weave/plugin/skel"
+)
+
+var version = "(unreleased version)"
+
+var Log = common.Log
+
+func main() {
+	var (
+		justVersion      bool
+		cniNet           bool
+		cniIpam          bool
+		address          string
+		meshAddress      string
+		logLevel         string
+		noMulticastRoute bool
+	)
+
+	flag.BoolVar(&justVersion, "version", false, "print version and exit")
+	flag.BoolVar(&cniNet, "cni-net", false, "act as a CNI network plugin")
+	flag.BoolVar(&cniIpam, "cni-ipam", false, "act as a CNI IPAM plugin")
+	flag.StringVar(&logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
+	flag.StringVar(&address, "socket", "/run/docker/plugins/weave.sock", "socket on which to listen")
+	flag.StringVar(&meshAddress, "meshsocket", "/run/docker/plugins/weavemesh.sock", "socket on which to listen in mesh mode")
+	flag.BoolVar(&noMulticastRoute, "no-multicast-route", false, "do not add a multicast route to network endpoints")
+
+	flag.Parse()
+
+	if justVersion {
+		fmt.Printf("weave plugin %s\n", version)
+		os.Exit(0)
+	}
+
+	common.SetLogLevel(logLevel)
+
+	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log)
+
+	switch {
+	case cniIpam || strings.HasSuffix(os.Args[0], "weave-ipam"):
+		i := ipamplugin.NewIpam(weave)
+		cni.PluginMain(i.CmdAdd, i.CmdDel)
+		os.Exit(0)
+	case cniNet || strings.HasSuffix(os.Args[0], "weave-net"):
+		n := netplugin.NewCNIPlugin(weave)
+		cni.PluginMain(n.CmdAdd, n.CmdDel)
+		os.Exit(0)
+	}
+
+	// API 1.21 is the first version that supports docker network commands
+	dockerClient, err := docker.NewVersionedClientFromEnv("1.21")
+	if err != nil {
+		Log.Fatalf("unable to connect to docker: %s", err)
+	}
+
+	Log.Println("Weave plugin", version, "Command line options:", os.Args[1:])
+	Log.Info(dockerClient.Info())
+
+	err = run(dockerClient, weave, address, meshAddress, noMulticastRoute)
+	if err != nil {
+		Log.Fatal(err)
+	}
+}
+
+func run(dockerClient *docker.Client, weave *weaveapi.Client, address, meshAddress string, noMulticastRoute bool) error {
+	endChan := make(chan error, 1)
+	if address != "" {
+		globalListener, err := listenAndServe(dockerClient, weave, address, noMulticastRoute, endChan, "global", false)
+		if err != nil {
+			return err
+		}
+		defer os.Remove(address)
+		defer globalListener.Close()
+	}
+	if meshAddress != "" {
+		meshListener, err := listenAndServe(dockerClient, weave, meshAddress, noMulticastRoute, endChan, "local", true)
+		if err != nil {
+			return err
+		}
+		defer os.Remove(meshAddress)
+		defer meshListener.Close()
+	}
+
+	statusListener, err := weavenet.ListenUnixSocket("/home/weave/status.sock")
+	if err != nil {
+		return err
+	}
+	go serveStatus(statusListener)
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM)
+
+	select {
+	case sig := <-sigChan:
+		Log.Debugf("Caught signal %s; shutting down", sig)
+		return nil
+	case err := <-endChan:
+		return err
+	}
+}
+
+func listenAndServe(dockerClient *docker.Client, weave *weaveapi.Client, address string, noMulticastRoute bool, endChan chan<- error, scope string, withIpam bool) (net.Listener, error) {
+	d, err := netplugin.New(dockerClient, weave, scope, noMulticastRoute)
+	if err != nil {
+		return nil, err
+	}
+
+	var i ipamapi.Ipam
+	if withIpam {
+		i = ipamplugin.NewIpam(weave)
+	}
+
+	listener, err := weavenet.ListenUnixSocket(address)
+	if err != nil {
+		return nil, err
+	}
+	Log.Printf("Listening on %s for %s scope", address, scope)
+
+	go func() {
+		endChan <- skel.Listen(listener, d, i)
+	}()
+
+	return listener, nil
+}
+
+func serveStatus(listener net.Listener) {
+	server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "ok")
+	})}
+	if err := server.Serve(listener); err != nil {
+		Log.Fatalf("ListenAndServeStatus failed: %s", err)
+	}
+}

prog/plugin/main_test.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"testing"
+
+	weavetest "github.com/weaveworks/weave/testing"
+)
+
+func TestMain(t *testing.T) {
+	weavetest.TrimTestArgs()
+	main()
+}

prog/weavedb/Dockerfile

@@ -0,0 +1,6 @@
+# This is a nearly-empty image that we use to create a data-only container for persistence
+FROM scratch
+MAINTAINER Weaveworks Inc <help@weave.works>
+ENTRYPOINT ["data-only"]
+# Work round Docker refusing to save an empty image
+COPY Dockerfile /

prog/weaveexec/Dockerfile

@@ -4,8 +4,6 @@ MAINTAINER Weaveworks Inc <help@weave.works>
 LABEL works.weave.role=system
 
 WORKDIR /home/weave
-VOLUME /w
-VOLUME /w-noop
 ENTRYPOINT ["/home/weave/sigproxy", "/home/weave/weave"]
 
 RUN apk add --update \
@@ -18,8 +16,9 @@ RUN apk add --update \
     bind-tools \
   && rm -rf /var/cache/apk/*
 
-ADD ./weave ./sigproxy ./weaveproxy /home/weave/
-ADD ./netcheck ./docker_tls_args /usr/bin/
+ADD ./weave ./sigproxy ./weaveproxy ./symlink /home/weave/
+ADD ./weaveutil /usr/bin/
 ADD ./weavewait /w/w
 ADD ./weavewait_noop /w-noop/w
+ADD ./weavewait_nomcast /w-nomcast/w
 ADD ./docker.tgz /

prog/weaveexec/symlink

@@ -0,0 +1,18 @@
+#!/bin/sh
+# This lets us force-symlink a bunch of stuff all at once, so we can minimize
+# running containers.
+
+set -e
+
+[ -z "$WEAVE_DEBUG" ] || set -x
+
+target="$1"
+user="$(stat -c '%u:%g' "$target")"
+mode="$(stat -c '%a' "$target")"
+shift 1
+
+for link in "$@" ; do
+  ln -sfT "$target" "$link"
+done
+chown -h "$user" "$@"
+chmod "$mode" "$@"

prog/weaveproxy/main.go

@@ -6,20 +6,28 @@ import (
 	"strings"
 
 	"github.com/docker/docker/pkg/mflag"
-	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/mflagext"
 	"github.com/weaveworks/weave/proxy"
 )
 
-var (
-	version = "(unreleased version)"
-)
+var version = "(unreleased version)"
+
+var Log = common.Log
+
+func getenv(key, defaultVal string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return defaultVal
+}
 
 func main() {
 	var (
 		justVersion bool
 		logLevel    = "info"
-		c           = proxy.Config{ListenAddrs: []string{}}
+		c           proxy.Config
+		withDNS     bool
 	)
 
 	c.Version = version
@@ -35,11 +43,12 @@ func main() {
 	mflag.BoolVar(&c.NoRewriteHosts, []string{"-no-rewrite-hosts"}, false, "do not automatically rewrite /etc/hosts. Use if you need the docker IP to remain in /etc/hosts")
 	mflag.StringVar(&c.TLSConfig.CACert, []string{"#tlscacert", "-tlscacert"}, "", "Trust certs signed only by this CA")
 	mflag.StringVar(&c.TLSConfig.Cert, []string{"#tlscert", "-tlscert"}, "", "Path to TLS certificate file")
-	mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tls-verify")
+	mflag.BoolVar(&c.TLSConfig.Enabled, []string{"#tls", "-tls"}, false, "Use TLS; implied by --tlsverify")
 	mflag.StringVar(&c.TLSConfig.Key, []string{"#tlskey", "-tlskey"}, "", "Path to TLS key file")
 	mflag.BoolVar(&c.TLSConfig.Verify, []string{"#tlsverify", "-tlsverify"}, false, "Use TLS and verify the remote")
-	mflag.BoolVar(&c.WithDNS, []string{"-with-dns", "w"}, false, "instruct created containers to always use weaveDNS as their nameserver")
+	mflag.BoolVar(&withDNS, []string{"#-with-dns", "#w"}, false, "option removed")
 	mflag.BoolVar(&c.WithoutDNS, []string{"-without-dns"}, false, "instruct created containers to never use weaveDNS as their nameserver")
+	mflag.BoolVar(&c.NoMulticastRoute, []string{"-no-multicast-route"}, false, "do not add a multicast route via the weave interface when attaching containers")
 	mflag.Parse()
 
 	if justVersion {
@@ -47,23 +56,28 @@ func main() {
 		os.Exit(0)
 	}
 
-	if c.WithDNS && c.WithoutDNS {
-		Log.Fatalf("Cannot use both '--with-dns' and '--without-dns' flags")
-	}
-
-	SetLogLevel(logLevel)
+	common.SetLogLevel(logLevel)
 
 	Log.Infoln("weave proxy", version)
 	Log.Infoln("Command line arguments:", strings.Join(os.Args[1:], " "))
 
+	if withDNS {
+		Log.Warning("--with-dns option has been removed; DNS is on by default")
+	}
+
+	c.Image = getenv("EXEC_IMAGE", "weaveworks/weaveexec")
+	c.DockerBridge = getenv("DOCKER_BRIDGE", "docker0")
+	c.DockerHost = getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
+
 	p, err := proxy.NewProxy(c)
 	if err != nil {
 		Log.Fatalf("Could not start proxy: %s", err)
 	}
+	defer p.Stop()
 
 	listeners := p.Listen()
 	p.AttachExistingContainers()
 	go p.Serve(listeners)
 	go p.ListenAndServeStatus("/home/weave/status.sock")
-	SignalHandlerLoop()
+	common.SignalHandlerLoop()
 }

prog/weaver/Dockerfile

@@ -3,4 +3,6 @@ MAINTAINER Weaveworks Inc <help@weave.works>
 LABEL works.weave.role=system
 WORKDIR /home/weave
 ADD ./weaver /home/weave/
+ADD weavedata.db /weavedb/
+COPY ca-certificates.crt /etc/ssl/certs/
 ENTRYPOINT ["/home/weave/weaver"]

prog/weaver/http.go

@@ -1,17 +1,22 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"strings"
+	"text/template"
+	"time"
+
 	"github.com/gorilla/mux"
-	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/mesh"
+
+	"github.com/weaveworks/go-checkpoint"
 	"github.com/weaveworks/weave/ipam"
 	"github.com/weaveworks/weave/nameserver"
 	"github.com/weaveworks/weave/net/address"
 	weave "github.com/weaveworks/weave/router"
-	"net/http"
-	"strings"
-	"text/template"
 )
 
 var rootTemplate = template.New("root").Funcs(map[string]interface{}{
@@ -24,14 +29,74 @@ var rootTemplate = template.New("root").Funcs(map[string]interface{}{
 		}
 		return count
 	},
-	"printConnectionCounts": func(conns []weave.LocalConnectionStatus) string {
+	"printList": func(list []string) string {
+		if len(list) == 0 {
+			return "none"
+		}
+		return strings.Join(list, ", ")
+	},
+	"printIPAMRanges": func(router weave.NetworkRouterStatus, status ipam.Status) string {
+		var buffer bytes.Buffer
+
+		type stats struct {
+			ips       uint32
+			nickname  string
+			reachable bool
+		}
+
+		peerStats := make(map[string]*stats)
+
+		for _, entry := range status.Entries {
+			s, found := peerStats[entry.Peer]
+			if !found {
+				s = &stats{nickname: entry.Nickname, reachable: entry.IsKnownPeer}
+				peerStats[entry.Peer] = s
+			}
+			s.ips += entry.Size
+		}
+
+		printOwned := func(name string, nickName string, reachable bool, ips uint32) {
+			reachableStr := ""
+			if !reachable {
+				reachableStr = "- unreachable!"
+			}
+			percentageRanges := float32(ips) * 100.0 / float32(status.RangeNumIPs)
+
+			displayName := name + "(" + nickName + ")"
+			fmt.Fprintf(&buffer, "%-37v %8d IPs (%04.1f%% of total) %s\n",
+				displayName, ips, percentageRanges, reachableStr)
+		}
+
+		// print the local info first
+		if ourStats := peerStats[router.Name]; ourStats != nil {
+			printOwned(router.Name, ourStats.nickname, true, ourStats.ips)
+		}
+
+		// and then the rest
+		for peer, stats := range peerStats {
+			if peer != router.Name {
+				printOwned(peer, stats.nickname, stats.reachable, stats.ips)
+			}
+		}
+
+		return buffer.String()
+	},
+	"allIPAMOwnersUnreachable": func(status ipam.Status) bool {
+		for _, entry := range status.Entries {
+			if entry.Size > 0 && entry.IsKnownPeer {
+				return false
+			}
+		}
+		return true
+	},
+	"printConnectionCounts": func(conns []mesh.LocalConnectionStatus) string {
 		counts := make(map[string]int)
 		for _, conn := range conns {
 			counts[conn.State]++
 		}
 		return printCounts(counts, []string{"established", "pending", "retrying", "failed", "connecting"})
 	},
-	"printPeerConnectionCounts": func(peers []weave.PeerStatus) string {
+	"printPeerConnectionCounts": func(peers []mesh.PeerStatus) string {
 		counts := make(map[string]int)
 		for _, peer := range peers {
 			for _, conn := range peer.Connections {
@@ -75,40 +140,52 @@ func defTemplate(name string, text string) *template.Template {
 }
 
 var statusTemplate = defTemplate("status", `\
-       Version: {{.Version}}
+        Version: {{.Version}} ({{.VersionCheck}})
 
-       Service: router
-      Protocol: {{.Router.Protocol}} \
+        Service: router
+       Protocol: {{.Router.Protocol}} \
 {{if eq .Router.ProtocolMinVersion .Router.ProtocolMaxVersion}}\
 {{.Router.ProtocolMaxVersion}}\
 {{else}}\
 {{.Router.ProtocolMinVersion}}..{{.Router.ProtocolMaxVersion}}\
 {{end}}
-          Name: {{.Router.Name}}({{.Router.NickName}})
-    Encryption: {{printState .Router.Encryption}}
- PeerDiscovery: {{printState .Router.PeerDiscovery}}
-       Targets: {{len .Router.Targets}}
-   Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}}
-         Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}}
+           Name: {{.Router.Name}}({{.Router.NickName}})
+     Encryption: {{printState .Router.Encryption}}
+  PeerDiscovery: {{printState .Router.PeerDiscovery}}
+        Targets: {{len .Router.Targets}}
+    Connections: {{len .Router.Connections}}{{with printConnectionCounts .Router.Connections}} ({{.}}){{end}}
+          Peers: {{len .Router.Peers}}{{with printPeerConnectionCounts .Router.Peers}} (with {{.}} connections){{end}}
+ TrustedSubnets: {{printList .Router.TrustedSubnets}}
 {{if .IPAM}}\
 
-       Service: ipam
+        Service: ipam
 {{if .IPAM.Entries}}\
-     Consensus: achieved
-{{else if .IPAM.Paxos}}\
-     Consensus: waiting (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
+{{if allIPAMOwnersUnreachable .IPAM}}\
+         Status: all IP ranges owned by unreachable peers - use 'rmpeer' if they are dead
+{{else if len .IPAM.PendingAllocates}}\
+         Status: waiting for IP range grant from peers
 {{else}}\
-     Consensus: deferred
+         Status: ready
 {{end}}\
-         Range: {{.IPAM.Range}}
- DefaultSubnet: {{.IPAM.DefaultSubnet}}
+{{else if .IPAM.Paxos}}\
+{{if .IPAM.Paxos.Elector}}\
+         Status: awaiting consensus (quorum: {{.IPAM.Paxos.Quorum}}, known: {{.IPAM.Paxos.KnownNodes}})
+{{else}}\
+         Status: priming
+{{end}}\
+{{else}}\
+         Status: idle
+{{end}}\
+          Range: {{.IPAM.Range}}
+  DefaultSubnet: {{.IPAM.DefaultSubnet}}
 {{end}}\
 {{if .DNS}}\
 
-       Service: dns
-        Domain: {{.DNS.Domain}}
-           TTL: {{.DNS.TTL}}
-       Entries: {{countDNSEntries .DNS.Entries}}
+        Service: dns
+         Domain: {{.DNS.Domain}}
+       Upstream: {{printList .DNS.Upstream}}
+            TTL: {{.DNS.TTL}}
+        Entries: {{countDNSEntries .DNS.Entries}}
 {{end}}\
 `)
 
@@ -128,7 +205,7 @@ var peersTemplate = defTemplate("peers", `\
 {{.Name}}({{.NickName}})
 {{range .Connections}}\
    {{if .Outbound}}->{{else}}<-{{end}} {{printf "%-21v" .Address}} \
-{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-32v" $nameNickName}} \
+{{$nameNickName := printf "%v(%v)" .Name .NickName}}{{printf "%-37v" $nameNickName}} \
 {{if .Established}}established{{else}}pending{{end}}
 {{end}}\
 {{end}}\
@@ -144,18 +221,54 @@ var dnsEntriesTemplate = defTemplate("dnsEntries", `\
 {{end}}\
 `)
 
-type WeaveStatus struct {
-	Version string
-	Router  *weave.Status      `json:"Router,omitempty"`
-	IPAM    *ipam.Status       `json:"IPAM,omitempty"`
-	DNS     *nameserver.Status `json:"DNS,omitempty"`
+var ipamTemplate = defTemplate("ipamTemplate", `{{printIPAMRanges .Router .IPAM}}`)
+
+type VersionCheck struct {
+	Enabled     bool
+	NewVersion  string
+	NextCheckAt time.Time
 }
 
-func HandleHTTP(muxRouter *mux.Router, version string, router *weave.Router, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
+func versionCheck() *VersionCheck {
+	v := &VersionCheck{}
+	if checkpoint.IsCheckDisabled() {
+		return v
+	}
+
+	v.Enabled = true
+	v.NewVersion = newVersion.Load().(string)
+	v.NextCheckAt = checker.NextCheckAt()
+
+	return v
+}
+
+func (v *VersionCheck) String() string {
+	switch {
+	case !v.Enabled:
+		return "version check update disabled"
+	case v.NewVersion != "":
+		return fmt.Sprintf("version %s available - please upgrade!",
+			v.NewVersion)
+	default:
+		return fmt.Sprintf("up to date; next check at %s",
+			v.NextCheckAt.Format("2006/01/02 15:04:05"))
+	}
+}
+
+type WeaveStatus struct {
+	Version      string
+	VersionCheck *VersionCheck              `json:"VersionCheck,omitempty"`
+	Router       *weave.NetworkRouterStatus `json:"Router,omitempty"`
+	IPAM         *ipam.Status               `json:"IPAM,omitempty"`
+	DNS          *nameserver.Status         `json:"DNS,omitempty"`
+}
+
+func HandleHTTP(muxRouter *mux.Router, version string, router *weave.NetworkRouter, allocator *ipam.Allocator, defaultSubnet address.CIDR, ns *nameserver.Nameserver, dnsserver *nameserver.DNSServer) {
 	status := func() WeaveStatus {
 		return WeaveStatus{
 			version,
-			weave.NewStatus(router),
+			versionCheck(),
+			weave.NewNetworkRouterStatus(router),
 			ipam.NewStatus(allocator, defaultSubnet),
 			nameserver.NewStatus(ns, dnsserver)}
 	}
@@ -173,7 +286,13 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.Router, all
 
 	muxRouter.Methods("GET").Path("/report").Queries("format", "{format}").HandlerFunc(
 		func(w http.ResponseWriter, r *http.Request) {
-			formatTemplate, err := template.New("format").Parse(mux.Vars(r)["format"])
+			funcs := template.FuncMap{
+				"json": func(v interface{}) string {
+					a, _ := json.Marshal(v)
+					return string(a)
+				},
+			}
+			formatTemplate, err := template.New("format").Funcs(funcs).Parse(mux.Vars(r)["format"])
 			if err != nil {
 				http.Error(w, err.Error(), http.StatusBadRequest)
 				return
@@ -199,5 +318,5 @@ func HandleHTTP(muxRouter *mux.Router, version string, router *weave.Router, all
 	defHandler("/status/connections", connectionsTemplate)
 	defHandler("/status/peers", peersTemplate)
 	defHandler("/status/dns", dnsEntriesTemplate)
-
+	defHandler("/status/ipam", ipamTemplate)
 }

prog/weaver/main.go

@@ -7,15 +7,20 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"runtime"
+	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/davecheney/profile"
 	"github.com/docker/docker/pkg/mflag"
 	"github.com/gorilla/mux"
+	"github.com/weaveworks/mesh"
 
-	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/go-checkpoint"
+	"github.com/weaveworks/weave/common"
 	"github.com/weaveworks/weave/common/docker"
+	"github.com/weaveworks/weave/db"
 	"github.com/weaveworks/weave/ipam"
 	"github.com/weaveworks/weave/nameserver"
 	weavenet "github.com/weaveworks/weave/net"
@@ -24,6 +29,113 @@ import (
 )
 
 var version = "(unreleased version)"
+var checker *checkpoint.Checker
+var newVersion atomic.Value
+
+var Log = common.Log
+
+type ipamConfig struct {
+	IPRangeCIDR   string
+	IPSubnetCIDR  string
+	PeerCount     int
+	Mode          string
+	Observer      bool
+	SeedPeerNames []mesh.PeerName
+}
+
+type dnsConfig struct {
+	Domain                 string
+	ListenAddress          string
+	TTL                    int
+	ClientTimeout          time.Duration
+	EffectiveListenAddress string
+}
+
+const (
+	updateCheckPeriod = 6 * time.Hour
+)
+
+func checkForUpdates() {
+	newVersion.Store("")
+
+	handleResponse := func(r *checkpoint.CheckResponse, err error) {
+		if err != nil {
+			Log.Printf("Error checking version: %v", err)
+			return
+		}
+		if r.Outdated {
+			newVersion.Store(r.CurrentVersion)
+			Log.Printf("Weave version %s is available; please update at %s",
+				r.CurrentVersion, r.CurrentDownloadURL)
+		}
+	}
+
+	// Start background version checking
+	params := checkpoint.CheckParams{
+		Product:       "weave-net",
+		Version:       version,
+		SignatureFile: "",
+	}
+	checker = checkpoint.CheckInterval(&params, updateCheckPeriod, handleResponse)
+}
+
+func (c *ipamConfig) Enabled() bool {
+	var (
+		hasPeerCount = c.PeerCount > 0
+		hasMode      = len(c.Mode) > 0
+		hasRange     = c.IPRangeCIDR != ""
+		hasSubnet    = c.IPSubnetCIDR != ""
+	)
+	switch {
+	case !(hasPeerCount || hasMode || hasRange || hasSubnet):
+		return false
+	case !hasRange && hasSubnet:
+		Log.Fatal("--ipalloc-default-subnet specified without --ipalloc-range.")
+	case !hasRange:
+		Log.Fatal("--ipalloc-init or --init-peer-count specified without --ipalloc-range.")
+	case hasMode && hasPeerCount:
+		Log.Fatal("At most one of --ipalloc-init or --init-peer-count may be specified.")
+	}
+	if hasMode {
+		if err := c.parseMode(); err != nil {
+			Log.Fatalf("Unable to parse --ipalloc-init: %s", err)
+		}
+	}
+	return true
+}
+
+func (c *ipamConfig) parseMode() error {
+	modeAndParam := strings.SplitN(c.Mode, "=", 2)
+
+	switch modeAndParam[0] {
+	case "consensus":
+		if len(modeAndParam) == 2 {
+			peerCount, err := strconv.Atoi(modeAndParam[1])
+			if err != nil {
+				return fmt.Errorf("bad consensus parameter: %s", err)
+			}
+			c.PeerCount = peerCount
+		}
+	case "seed":
+		if len(modeAndParam) != 2 {
+			return fmt.Errorf("seed mode requires peer name list")
+		}
+		seedPeerNames, err := parsePeerNames(modeAndParam[1])
+		if err != nil {
+			return fmt.Errorf("bad seed parameter: %s", err)
+		}
+		c.SeedPeerNames = seedPeerNames
+	case "observer":
+		if len(modeAndParam) != 1 {
+			return fmt.Errorf("observer mode takes no parameter")
+		}
+		c.Observer = true
+	default:
+		return fmt.Errorf("unknown mode: %s", modeAndParam[0])
+	}
+
+	return nil
+}
 
 func main() {
 	procs := runtime.NumCPU()
@@ -35,46 +147,40 @@ func main() {
 	runtime.GOMAXPROCS(procs)
 
 	var (
-		// flags that cause immediate exit
-		justVersion          bool
-		createDatapath       bool
-		deleteDatapath       bool
-		addDatapathInterface string
+		justVersion        bool
+		config             mesh.Config
+		networkConfig      weave.NetworkConfig
+		protocolMinVersion int
+		ifaceName          string
+		routerName         string
+		nickName           string
+		password           string
+		pktdebug           bool
+		logLevel           string
+		prof               string
+		bufSzMB            int
+		noDiscovery        bool
+		httpAddr           string
+		ipamConfig         ipamConfig
+		dockerAPI          string
+		peers              []string
+		noDNS              bool
+		dnsConfig          dnsConfig
+		datapathName       string
+		trustedSubnetStr   string
+		dbPrefix           string
 
-		config                    weave.Config
-		protocolMinVersion        int
-		ifaceName                 string
-		routerName                string
-		nickName                  string
-		password                  string
-		pktdebug                  bool
-		logLevel                  string
-		prof                      string
-		bufSzMB                   int
-		noDiscovery               bool
-		httpAddr                  string
-		iprangeCIDR               string
-		ipsubnetCIDR              string
-		peerCount                 int
-		dockerAPI                 string
-		peers                     []string
-		noDNS                     bool
-		dnsDomain                 string
-		dnsListenAddress          string
-		dnsTTL                    int
-		dnsClientTimeout          time.Duration
-		dnsEffectiveListenAddress string
-		iface                     *net.Interface
-		datapathName              string
+		defaultDockerHost = "unix:///var/run/docker.sock"
 	)
 
-	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
-	mflag.BoolVar(&createDatapath, []string{"-create-datapath"}, false, "create ODP datapath and exit")
-	mflag.BoolVar(&deleteDatapath, []string{"-delete-datapath"}, false, "delete ODP datapath and exit")
-	mflag.StringVar(&addDatapathInterface, []string{"-add-datapath-iface"}, "", "add a network interface to the ODP datapath and exit")
+	if val := os.Getenv("DOCKER_HOST"); val != "" {
+		defaultDockerHost = val
+	}
 
-	mflag.IntVar(&config.Port, []string{"#port", "-port"}, weave.Port, "router port")
-	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, weave.ProtocolMinVersion, "minimum weave protocol version")
+	mflag.BoolVar(&justVersion, []string{"#version", "-version"}, false, "print version and exit")
+	mflag.StringVar(&config.Host, []string{"-host"}, "", "router host")
+	mflag.IntVar(&config.Port, []string{"#port", "-port"}, mesh.Port, "router port")
+	mflag.IntVar(&protocolMinVersion, []string{"-min-protocol-version"}, mesh.ProtocolMinVersion, "minimum weave protocol version")
 	mflag.StringVar(&ifaceName, []string{"#iface", "-iface"}, "", "name of interface to capture/inject from (disabled if blank)")
 	mflag.StringVar(&routerName, []string{"#name", "-name"}, "", "name of router (defaults to MAC of interface)")
 	mflag.StringVar(&nickName, []string{"#nickname", "-nickname"}, "", "nickname of peer (defaults to hostname)")
@@ -85,18 +191,21 @@ func main() {
 	mflag.IntVar(&config.ConnLimit, []string{"#connlimit", "#-connlimit", "-conn-limit"}, 30, "connection limit (0 for unlimited)")
 	mflag.BoolVar(&noDiscovery, []string{"#nodiscovery", "#-nodiscovery", "-no-discovery"}, false, "disable peer discovery")
 	mflag.IntVar(&bufSzMB, []string{"#bufsz", "-bufsz"}, 8, "capture buffer size in MB")
-	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, fmt.Sprintf(":%d", weave.HTTPPort), "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
-	mflag.StringVar(&iprangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
-	mflag.StringVar(&ipsubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
-	mflag.IntVar(&peerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)")
-	mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, "", "Docker API endpoint, e.g. unix:///var/run/docker.sock")
+	mflag.StringVar(&httpAddr, []string{"#httpaddr", "#-httpaddr", "-http-addr"}, "", "address to bind HTTP interface to (disabled if blank, absolute path indicates unix domain socket)")
+	mflag.StringVar(&ipamConfig.Mode, []string{"-ipalloc-init"}, "", "allocator initialisation strategy (consensus, seed or observer)")
+	mflag.StringVar(&ipamConfig.IPRangeCIDR, []string{"#iprange", "#-iprange", "-ipalloc-range"}, "", "IP address range reserved for automatic allocation, in CIDR notation")
+	mflag.StringVar(&ipamConfig.IPSubnetCIDR, []string{"#ipsubnet", "#-ipsubnet", "-ipalloc-default-subnet"}, "", "subnet to allocate within by default, in CIDR notation")
+	mflag.IntVar(&ipamConfig.PeerCount, []string{"#initpeercount", "#-initpeercount", "-init-peer-count"}, 0, "number of peers in network (for IP address allocation)")
+	mflag.StringVar(&dockerAPI, []string{"#api", "#-api", "-docker-api"}, defaultDockerHost, "Docker API endpoint")
 	mflag.BoolVar(&noDNS, []string{"-no-dns"}, false, "disable DNS server")
-	mflag.StringVar(&dnsDomain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for")
-	mflag.StringVar(&dnsListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests")
-	mflag.IntVar(&dnsTTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
-	mflag.DurationVar(&dnsClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
-	mflag.StringVar(&dnsEffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
+	mflag.StringVar(&dnsConfig.Domain, []string{"-dns-domain"}, nameserver.DefaultDomain, "local domain to server requests for")
+	mflag.StringVar(&dnsConfig.ListenAddress, []string{"-dns-listen-address"}, nameserver.DefaultListenAddress, "address to listen on for DNS requests")
+	mflag.IntVar(&dnsConfig.TTL, []string{"-dns-ttl"}, nameserver.DefaultTTL, "TTL for DNS request from our domain")
+	mflag.DurationVar(&dnsConfig.ClientTimeout, []string{"-dns-fallback-timeout"}, nameserver.DefaultClientTimeout, "timeout for fallback DNS requests")
+	mflag.StringVar(&dnsConfig.EffectiveListenAddress, []string{"-dns-effective-listen-address"}, "", "address DNS will actually be listening, after Docker port mapping")
 	mflag.StringVar(&datapathName, []string{"-datapath"}, "", "ODP datapath name")
+	mflag.StringVar(&trustedSubnetStr, []string{"-trusted-subnets"}, "", "comma-separated list of trusted subnets in CIDR notation")
+	mflag.StringVar(&dbPrefix, []string{"-db-prefix"}, "/weavedb/weave", "pathname/prefix of filename to store data")
 
 	// crude way of detecting that we probably have been started in a
 	// container, with `weave launch` --> suppress misleading paths in
@@ -110,115 +219,16 @@ func main() {
 
 	peers = mflag.Args()
 
-	SetLogLevel(logLevel)
+	common.SetLogLevel(logLevel)
 
-	switch {
-	case justVersion:
+	if justVersion {
 		fmt.Printf("weave router %s\n", version)
 		os.Exit(0)
-
-	case createDatapath:
-		err, odp_supported := weave.CreateDatapath(datapathName)
-		if !odp_supported {
-			if err != nil {
-				Log.Error(err)
-			}
-
-			// When the kernel lacks ODP support, exit
-			// with a special status to distinguish it for
-			// the weave script.
-			os.Exit(17)
-		}
-
-		checkFatal(err)
-		os.Exit(0)
-
-	case deleteDatapath:
-		checkFatal(weave.DeleteDatapath(datapathName))
-		os.Exit(0)
-
-	case addDatapathInterface != "":
-		checkFatal(weave.AddDatapathInterface(datapathName, addDatapathInterface))
-		os.Exit(0)
 	}
 
 	Log.Println("Command line options:", options())
-	Log.Println("Command line peers:", peers)
 
-	if protocolMinVersion < weave.ProtocolMinVersion || protocolMinVersion > weave.ProtocolMaxVersion {
-		Log.Fatalf("--min-protocol-version must be in range [%d,%d]", weave.ProtocolMinVersion, weave.ProtocolMaxVersion)
-	}
-	config.ProtocolMinVersion = byte(protocolMinVersion)
-
-	var fastDPOverlay weave.Overlay
-	if datapathName != "" {
-		// A datapath name implies that "Bridge" and "Overlay"
-		// packet handling use fast datapath, although other
-		// options can override that below.  Even if both
-		// things are overridden, we might need bridging on
-		// the datapath.
-		fastdp, err := weave.NewFastDatapath(weave.FastDatapathConfig{
-			DatapathName: datapathName,
-			Port:         config.Port,
-		})
-
-		checkFatal(err)
-		config.Bridge = fastdp.Bridge()
-		fastDPOverlay = fastdp.Overlay()
-	}
-
-	if ifaceName != "" {
-		// -iface can coexist with -datapath, because
-		// pcap-based packet capture is a bit more efficient
-		// than capture via ODP misses, even when using an
-		// ODP-based bridge.  So when using weave encyption,
-		// it's preferable to use -iface.
-		var err error
-		iface, err = weavenet.EnsureInterface(ifaceName)
-		checkFatal(err)
-
-		// bufsz flag is in MB
-		config.Bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024)
-		checkFatal(err)
-	}
-
-	if password == "" {
-		password = os.Getenv("WEAVE_PASSWORD")
-	}
-
-	if password == "" {
-		Log.Println("Communication between peers is unencrypted.")
-	} else {
-		config.Password = []byte(password)
-		Log.Println("Communication between peers is encrypted.")
-
-		// fastdp doesn't support encryption
-		fastDPOverlay = nil
-	}
-
-	overlays := weave.NewOverlaySwitch()
-	if fastDPOverlay != nil {
-		overlays.Add("fastdp", fastDPOverlay)
-	}
-	sleeve := weave.NewSleeveOverlay(config.Port)
-	overlays.Add("sleeve", sleeve)
-	overlays.SetCompatOverlay(sleeve)
-	config.Overlay = overlays
-
-	if routerName == "" {
-		if iface == nil {
-			Log.Fatal("Either an interface must be specified with --iface or a name with -name")
-		}
-		routerName = iface.HardwareAddr.String()
-	}
-
-	name, err := weave.PeerNameFromUserInput(routerName)
-	checkFatal(err)
-
-	if nickName == "" {
-		nickName, err = os.Hostname()
-		checkFatal(err)
-	}
+	checkForUpdates()
 
 	if prof != "" {
 		p := *profile.CPUProfile
@@ -227,39 +237,74 @@ func main() {
 		defer profile.Start(&p).Stop()
 	}
 
-	config.PeerDiscovery = !noDiscovery
+	if protocolMinVersion < mesh.ProtocolMinVersion || protocolMinVersion > mesh.ProtocolMaxVersion {
+		Log.Fatalf("--min-protocol-version must be in range [%d,%d]", mesh.ProtocolMinVersion, mesh.ProtocolMaxVersion)
+	}
+	config.ProtocolMinVersion = byte(protocolMinVersion)
 
 	if pktdebug {
-		config.PacketLogging = packetLogging{}
+		networkConfig.PacketLogging = packetLogging{}
 	} else {
-		config.PacketLogging = nopPacketLogging{}
+		networkConfig.PacketLogging = nopPacketLogging{}
 	}
 
-	router := weave.NewRouter(config, name, nickName)
+	overlay, bridge := createOverlay(datapathName, ifaceName, config.Host, config.Port, bufSzMB)
+	networkConfig.Bridge = bridge
+
+	name := peerName(routerName, bridge.Interface())
+
+	if nickName == "" {
+		var err error
+		nickName, err = os.Hostname()
+		checkFatal(err)
+	}
+
+	config.Password = determinePassword(password)
+	config.TrustedSubnets = parseTrustedSubnets(trustedSubnetStr)
+	config.PeerDiscovery = !noDiscovery
+
+	db, err := db.NewBoltDB(dbPrefix + "data.db")
+	checkFatal(err)
+	defer db.Close()
+
+	router := weave.NewNetworkRouter(config, networkConfig, name, nickName, overlay, db)
 	Log.Println("Our name is", router.Ourself)
 
+	if peers, err = router.InitialPeers(peers); err != nil {
+		Log.Fatal("Unable to get initial peer set: ", err)
+	}
+
 	var dockerCli *docker.Client
 	if dockerAPI != "" {
 		dc, err := docker.NewClient(dockerAPI)
 		if err != nil {
 			Log.Fatal("Unable to start docker client: ", err)
+		} else {
+			Log.Info(dc.Info())
 		}
 		dockerCli = dc
 	}
 	observeContainers := func(o docker.ContainerObserver) {
 		if dockerCli != nil {
-			if err = dockerCli.AddObserver(o); err != nil {
+			if err := dockerCli.AddObserver(o); err != nil {
 				Log.Fatal("Unable to start watcher", err)
 			}
 		}
 	}
-	var allocator *ipam.Allocator
-	var defaultSubnet address.CIDR
-	if iprangeCIDR != "" {
-		allocator, defaultSubnet = createAllocator(router, iprangeCIDR, ipsubnetCIDR, determineQuorum(peerCount, peers))
+	isKnownPeer := func(name mesh.PeerName) bool {
+		return router.Peers.Fetch(name) != nil
+	}
+
+	var (
+		allocator     *ipam.Allocator
+		defaultSubnet address.CIDR
+	)
+	if ipamConfig.Enabled() {
+		allocator, defaultSubnet = createAllocator(router, ipamConfig, db, isKnownPeer)
 		observeContainers(allocator)
-	} else if peerCount > 0 {
-		Log.Fatal("--init-peer-count flag specified without --ipalloc-range")
+		ids, err := dockerCli.AllContainerIDs()
+		checkFatal(err)
+		allocator.AllContainerIDs(ids)
 	}
 
 	var (
@@ -267,28 +312,17 @@ func main() {
 		dnsserver *nameserver.DNSServer
 	)
 	if !noDNS {
-		ns = nameserver.New(router.Ourself.Peer.Name, router.Peers, dnsDomain)
-		ns.SetGossip(router.NewGossip("nameserver", ns))
+		ns, dnsserver = createDNSServer(dnsConfig, router.Router, isKnownPeer)
 		observeContainers(ns)
 		ns.Start()
 		defer ns.Stop()
-		dnsserver, err = nameserver.NewDNSServer(ns, dnsDomain, dnsListenAddress,
-			dnsEffectiveListenAddress, uint32(dnsTTL), dnsClientTimeout)
-		if err != nil {
-			Log.Fatal("Unable to start dns server: ", err)
-		}
-		listenAddr := dnsListenAddress
-		if dnsEffectiveListenAddress != "" {
-			listenAddr = dnsEffectiveListenAddress
-		}
-		Log.Println("Listening for DNS queries on", listenAddr)
 		dnsserver.ActivateAndServe()
 		defer dnsserver.Stop()
 	}
 
 	router.Start()
-	if errors := router.ConnectionMaker.InitiateConnections(peers, false); len(errors) > 0 {
-		Log.Fatal(ErrorMessages(errors))
+	if errors := router.InitiateConnections(peers, false); len(errors) > 0 {
+		Log.Fatal(common.ErrorMessages(errors))
 	}
 
 	// The weave script always waits for a status call to succeed,
@@ -309,7 +343,7 @@ func main() {
 		go listenAndServeHTTP(httpAddr, muxRouter)
 	}
 
-	SignalHandlerLoop(router)
+	common.SignalHandlerLoop(router)
 }
 
 func options() map[string]string {
@@ -352,27 +386,69 @@ func (nopPacketLogging) LogPacket(string, weave.PacketKey) {
 func (nopPacketLogging) LogForwardPacket(string, weave.ForwardPacketKey) {
 }
 
+func createOverlay(datapathName string, ifaceName string, host string, port int, bufSzMB int) (weave.NetworkOverlay, weave.Bridge) {
+	overlay := weave.NewOverlaySwitch()
+	var bridge weave.Bridge
+	switch {
+	case datapathName != "" && ifaceName != "":
+		Log.Fatal("At most one of --datapath and --iface must be specified.")
+	case datapathName != "":
+		iface, err := weavenet.EnsureInterface(datapathName)
+		checkFatal(err)
+		fastdp, err := weave.NewFastDatapath(iface, port)
+		checkFatal(err)
+		bridge = fastdp.Bridge()
+		overlay.Add("fastdp", fastdp.Overlay())
+	case ifaceName != "":
+		iface, err := weavenet.EnsureInterface(ifaceName)
+		checkFatal(err)
+		bridge, err = weave.NewPcap(iface, bufSzMB*1024*1024) // bufsz flag is in MB
+		checkFatal(err)
+	default:
+		bridge = weave.NullBridge{}
+	}
+	sleeve := weave.NewSleeveOverlay(host, port)
+	overlay.Add("sleeve", sleeve)
+	overlay.SetCompatOverlay(sleeve)
+	return overlay, bridge
+}
+
 func parseAndCheckCIDR(cidrStr string) address.CIDR {
-	_, cidr, err := address.ParseCIDR(cidrStr)
+	cidr, err := address.ParseCIDR(cidrStr)
 	checkFatal(err)
 
+	if !cidr.IsSubnet() {
+		Log.Fatalf("Invalid allocation range %s - bits after network prefix are not all zero", cidrStr)
+	}
 	if cidr.Size() < ipam.MinSubnetSize {
 		Log.Fatalf("Allocation range smaller than minimum size %d: %s", ipam.MinSubnetSize, cidrStr)
 	}
 	return cidr
 }
 
-func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr string, quorum uint) (*ipam.Allocator, address.CIDR) {
-	ipRange := parseAndCheckCIDR(ipRangeStr)
+func createAllocator(router *weave.NetworkRouter, config ipamConfig, db db.DB, isKnownPeer func(mesh.PeerName) bool) (*ipam.Allocator, address.CIDR) {
+	ipRange := parseAndCheckCIDR(config.IPRangeCIDR)
 	defaultSubnet := ipRange
-	if defaultSubnetStr != "" {
-		defaultSubnet = parseAndCheckCIDR(defaultSubnetStr)
+	if config.IPSubnetCIDR != "" {
+		defaultSubnet = parseAndCheckCIDR(config.IPSubnetCIDR)
 		if !ipRange.Range().Overlaps(defaultSubnet.Range()) {
 			Log.Fatalf("IP address allocation default subnet %s does not overlap with allocation range %s", defaultSubnet, ipRange)
 		}
 	}
-	isKnownPeer := func(name weave.PeerName) bool { return router.Peers.Fetch(name) != nil }
-	allocator := ipam.NewAllocator(router.Ourself.Peer.Name, router.Ourself.Peer.UID, router.Ourself.Peer.NickName, ipRange.Range(), quorum, isKnownPeer)
+
+	c := ipam.Config{
+		OurName:     router.Ourself.Peer.Name,
+		OurUID:      router.Ourself.Peer.UID,
+		OurNickname: router.Ourself.Peer.NickName,
+		Seed:        config.SeedPeerNames,
+		Universe:    ipRange.Range(),
+		IsObserver:  config.Observer,
+		Quorum:      func() uint { return determineQuorum(config.PeerCount, router) },
+		Db:          db,
+		IsKnownPeer: isKnownPeer,
+	}
+
+	allocator := ipam.NewAllocator(c)
 
 	allocator.SetInterfaces(router.NewGossip("IPallocation", allocator))
 	allocator.Start()
@@ -380,13 +456,31 @@ func createAllocator(router *weave.Router, ipRangeStr string, defaultSubnetStr s
 	return allocator, defaultSubnet
 }
 
-// Pick a quorum size heuristically based on the number of peer
-// addresses passed.
-func determineQuorum(initPeerCountFlag int, peers []string) uint {
+func createDNSServer(config dnsConfig, router *mesh.Router, isKnownPeer func(mesh.PeerName) bool) (*nameserver.Nameserver, *nameserver.DNSServer) {
+	ns := nameserver.New(router.Ourself.Peer.Name, config.Domain, isKnownPeer)
+	router.Peers.OnGC(func(peer *mesh.Peer) { ns.PeerGone(peer.Name) })
+	ns.SetGossip(router.NewGossip("nameserver", ns))
+	dnsserver, err := nameserver.NewDNSServer(ns, config.Domain, config.ListenAddress,
+		config.EffectiveListenAddress, uint32(config.TTL), config.ClientTimeout)
+	if err != nil {
+		Log.Fatal("Unable to start dns server: ", err)
+	}
+	listenAddr := config.ListenAddress
+	if config.EffectiveListenAddress != "" {
+		listenAddr = config.EffectiveListenAddress
+	}
+	Log.Println("Listening for DNS queries on", listenAddr)
+	return ns, dnsserver
+}
+
+// Pick a quorum size based on the number of peer addresses.
+func determineQuorum(initPeerCountFlag int, router *weave.NetworkRouter) uint {
 	if initPeerCountFlag > 0 {
 		return uint(initPeerCountFlag/2 + 1)
 	}
 
+	peers := router.ConnectionMaker.Targets(true)
+
 	// Guess a suitable quorum size based on the list of peer
 	// addresses.  The peer list might or might not contain an
 	// address for this peer, so the conservative assumption is
@@ -400,6 +494,64 @@ func determineQuorum(initPeerCountFlag int, peers []string) uint {
 	return quorum
 }
 
+func determinePassword(password string) []byte {
+	if password == "" {
+		password = os.Getenv("WEAVE_PASSWORD")
+	}
+	if password == "" {
+		Log.Println("Communication between peers is unencrypted.")
+		return nil
+	}
+	Log.Println("Communication between peers via untrusted networks is encrypted.")
+	return []byte(password)
+}
+
+func peerName(routerName string, iface *net.Interface) mesh.PeerName {
+	if routerName == "" {
+		if iface == nil {
+			Log.Fatal("Either an interface must be specified with --datapath or --iface, or a name with --name")
+		}
+		routerName = iface.HardwareAddr.String()
+	}
+	name, err := mesh.PeerNameFromUserInput(routerName)
+	checkFatal(err)
+	return name
+}
+
+func parseTrustedSubnets(trustedSubnetStr string) []*net.IPNet {
+	trustedSubnets := []*net.IPNet{}
+	if trustedSubnetStr == "" {
+		return trustedSubnets
+	}
+
+	for _, subnetStr := range strings.Split(trustedSubnetStr, ",") {
+		_, subnet, err := net.ParseCIDR(subnetStr)
+		if err != nil {
+			Log.Fatal("Unable to parse trusted subnets: ", err)
+		}
+		trustedSubnets = append(trustedSubnets, subnet)
+	}
+
+	return trustedSubnets
+}
+
+func parsePeerNames(s string) ([]mesh.PeerName, error) {
+	peerNames := []mesh.PeerName{}
+	if s == "" {
+		return peerNames, nil
+	}
+
+	for _, peerNameStr := range strings.Split(s, ",") {
+		peerName, err := mesh.PeerNameFromUserInput(peerNameStr)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing peer names: %s", err)
+		}
+		peerNames = append(peerNames, peerName)
+	}
+
+	return peerNames, nil
+}
+
 func listenAndServeHTTP(httpAddr string, muxRouter *mux.Router) {
 	protocol := "tcp"
 	if strings.HasPrefix(httpAddr, "/") {

prog/weaveutil/addrs.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"fmt"
+	"github.com/fsouza/go-dockerclient"
+	"github.com/weaveworks/weave/common"
+)
+
+func containerAddrs(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("container-addrs", "<bridgeName> [containerID ...]")
+	}
+	bridgeName := args[0]
+
+	c, err := docker.NewVersionedClientFromEnv("1.18")
+	if err != nil {
+		return err
+	}
+
+	for _, containerID := range args[1:] {
+		netDevs, err := getNetDevs(bridgeName, c, containerID)
+		if err != nil {
+			if _, ok := err.(*docker.NoSuchContainer); ok {
+				continue
+			}
+			return err
+		}
+
+		for _, netDev := range netDevs {
+			fmt.Printf("%12s %s", containerID, netDev.MAC.String())
+			for _, cidr := range netDev.CIDRs {
+				prefixLength, _ := cidr.Mask.Size()
+				fmt.Printf(" %v/%v", cidr.IP, prefixLength)
+			}
+			fmt.Println()
+		}
+	}
+
+	return nil
+}
+
+func getNetDevs(bridgeName string, c *docker.Client, containerID string) ([]common.NetDev, error) {
+	if containerID == "weave:expose" {
+		return common.GetBridgeNetDev(bridgeName)
+	}
+
+	container, err := c.InspectContainer(containerID)
+	if err != nil {
+		return nil, err
+	}
+
+	if container.State.Pid == 0 {
+		return nil, nil
+	}
+
+	return common.GetWeaveNetDevs(container.State.Pid)
+}

prog/weaveutil/datapath.go

@@ -0,0 +1,39 @@
+// various fastdp operations
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/weaveworks/weave/common/odp"
+)
+
+func createDatapath(args []string) error {
+	if len(args) != 1 {
+		cmdUsage("create-datapath", "<datapath>")
+	}
+	odpSupported, err := odp.CreateDatapath(args[0])
+	if !odpSupported {
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+		}
+		// When the kernel lacks ODP support, exit with a special
+		// status to distinguish it for the weave script.
+		os.Exit(17)
+	}
+	return err
+}
+
+func deleteDatapath(args []string) error {
+	if len(args) != 1 {
+		cmdUsage("delete-datapath", "<datapath>")
+	}
+	return odp.DeleteDatapath(args[0])
+}
+
+func addDatapathInterface(args []string) error {
+	if len(args) != 2 {
+		cmdUsage("add-datapath-interface", "<datapath> <interface>")
+	}
+	return odp.AddDatapathInterface(args[0], args[1])
+}

prog/weaveutil/docker_tls_args.go

@@ -1,26 +1,24 @@
 // docker_tls_args: find the docker daemon's tls args
-// This reimplements pgrep, because pgrep will only look in /proc, but not in
-// $PROCFS.
 package main
 
 import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 )
 
-func main() {
-	procRoot := os.Getenv("PROCFS")
-	if procRoot == "" {
-		procRoot = "/proc"
+func dockerTLSArgs(args []string) error {
+	if len(args) > 0 {
+		cmdUsage("docker-tls-args", "")
 	}
-
+	procRoot := "/proc"
 	dirEntries, err := ioutil.ReadDir(procRoot)
-	checkErr(err)
+	if err != nil {
+		return err
+	}
 
 	for _, dirEntry := range dirEntries {
 		dirName := dirEntry.Name()
@@ -63,15 +61,8 @@ func main() {
 		}
 
 		fmt.Println(strings.Join(tlsArgs, " "))
-		os.Exit(0)
+		return nil
 	}
 
-	os.Exit(1)
-}
-
-func checkErr(err error) {
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err)
-		os.Exit(1)
-	}
+	return fmt.Errorf("cannot locate running docker daemon")
 }

prog/weaveutil/main.go

@@ -0,0 +1,62 @@
+/* weaveutil: collection of operations required by weave script */
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+var commands map[string]func([]string) error
+
+func init() {
+	commands = map[string]func([]string) error{
+		"help":                   help,
+		"netcheck":               netcheck,
+		"docker-tls-args":        dockerTLSArgs,
+		"create-datapath":        createDatapath,
+		"delete-datapath":        deleteDatapath,
+		"add-datapath-interface": addDatapathInterface,
+		"create-plugin-network":  createPluginNetwork,
+		"remove-plugin-network":  removePluginNetwork,
+		"container-addrs":        containerAddrs,
+	}
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		usage()
+		os.Exit(1)
+	}
+	cmd, found := commands[os.Args[1]]
+	if !found {
+		usage()
+		os.Exit(1)
+	}
+	if err := cmd(os.Args[2:]); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func help(args []string) error {
+	if len(args) > 0 {
+		cmdUsage("help", "")
+	}
+	usage()
+	return nil
+}
+
+func usage() {
+	fmt.Fprintln(os.Stderr, "usage: weaveutil <command> <arg>...")
+	fmt.Fprintln(os.Stderr)
+	fmt.Fprintln(os.Stderr, "where <command> is one of:")
+	fmt.Fprintln(os.Stderr)
+	for cmd := range commands {
+		fmt.Fprintln(os.Stderr, cmd)
+	}
+}
+
+func cmdUsage(cmd string, usage string) {
+	fmt.Fprintf(os.Stderr, "usage: weaveutil %s %s\n", cmd, usage)
+	os.Exit(1)
+}

prog/weaveutil/netcheck.go

@@ -0,0 +1,28 @@
+/* netcheck: check whether a given network or address overlaps with any existing routes */
+package main
+
+import (
+	"net"
+
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func netcheck(args []string) error {
+	if len(args) < 1 {
+		cmdUsage("netcheck", "<cidr> [<interface-to-ignore> ...]")
+	}
+	addr, ipnet, err := net.ParseCIDR(args[0])
+	if err != nil {
+		return err
+	}
+	ignoreIfaceNames := make(map[string]struct{})
+	for _, ifName := range args[1:] {
+		ignoreIfaceNames[ifName] = struct{}{}
+	}
+	if ipnet.IP.Equal(addr) {
+		err = weavenet.CheckNetworkFree(ipnet, ignoreIfaceNames)
+	} else {
+		err = weavenet.CheckAddressOverlap(addr, ignoreIfaceNames)
+	}
+	return err
+}

prog/weaveutil/plugin_network.go

@@ -0,0 +1,67 @@
+/* various weave docker network plugin operations */
+package main
+
+import (
+	"fmt"
+
+	"github.com/fsouza/go-dockerclient"
+)
+
+func createPluginNetwork(args []string) error {
+	if len(args) != 2 {
+		cmdUsage("create-plugin-network", "<network-name> <driver-name>")
+	}
+	networkName := args[0]
+	driverName := args[1]
+	d, err := newDockerClient()
+	if err != nil {
+		return err
+	}
+	_, err = d.CreateNetwork(
+		docker.CreateNetworkOptions{
+			Name:           networkName,
+			CheckDuplicate: true,
+			Driver:         driverName,
+			IPAM:           docker.IPAMOptions{Driver: driverName},
+		})
+	if err != docker.ErrNetworkAlreadyExists && err != nil {
+		// Despite appearances to the contrary, CreateNetwork does
+		// sometimes(always?) *not* return ErrNetworkAlreadyExists
+		// when the network already exists. Hence we need to check for
+		// this explicitly.
+		if _, err2 := d.NetworkInfo(networkName); err2 != nil {
+			return fmt.Errorf("unable to create network: %s", err)
+		}
+	}
+	return nil
+}
+
+func removePluginNetwork(args []string) error {
+	if len(args) != 1 {
+		cmdUsage("remove-plugin-network", "<network-name>")
+	}
+	networkName := args[0]
+	d, err := newDockerClient()
+	if err != nil {
+		return err
+	}
+	err = d.RemoveNetwork(networkName)
+	if _, ok := err.(*docker.NoSuchNetwork); !ok && err != nil {
+		return fmt.Errorf("unable to remove network: %s", err)
+	}
+	return nil
+}
+
+func newDockerClient() (*docker.Client, error) {
+	// API 1.21 is the first version that supports docker network
+	// commands
+	c, err := docker.NewVersionedClientFromEnv("1.21")
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to docker: %s", err)
+	}
+	_, err = c.Version()
+	if err != nil {
+		return nil, fmt.Errorf("unable to connect to docker: %s", err)
+	}
+	return c, nil
+}

prog/weavewait/check_network.go

@@ -1,12 +1,7 @@
-// +build !noop
+// +build !iface,!mcast
 
 package main
 
-import (
-	weavenet "github.com/weaveworks/weave/net"
-)
-
 func checkNetwork() error {
-	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
-	return err
+	return nil
 }

prog/weavewait/check_network_iface.go

@@ -0,0 +1,12 @@
+// +build iface,!mcast
+
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func checkNetwork() error {
+	_, err := weavenet.EnsureInterface("ethwe")
+	return err
+}

prog/weavewait/check_network_iface_mcast.go

@@ -0,0 +1,12 @@
+// +build iface,mcast
+
+package main
+
+import (
+	weavenet "github.com/weaveworks/weave/net"
+)
+
+func checkNetwork() error {
+	_, err := weavenet.EnsureInterfaceAndMcastRoute("ethwe")
+	return err
+}

prog/weavewait/noop.go

@@ -1,7 +0,0 @@
-// +build noop
-
-package main
-
-func checkNetwork() error {
-	return nil
-}

proxy/common.go

@@ -3,10 +3,8 @@ package proxy
 import (
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"os"
 	"os/exec"
@@ -14,7 +12,7 @@ import (
 	"strings"
 
 	"github.com/fsouza/go-dockerclient"
-	. "github.com/weaveworks/weave/common"
+	"github.com/weaveworks/weave/common"
 )
 
 var (
@@ -22,6 +20,8 @@ var (
 	weaveWaitEntrypoint = []string{"/w/w"}
 	weaveEntrypoint     = "/home/weave/weaver"
 	weaveContainerName  = "/weave"
+
+	Log = common.Log
 )
 
 func callWeave(args ...string) ([]byte, []byte, error) {
@@ -29,7 +29,6 @@ func callWeave(args ...string) ([]byte, []byte, error) {
 	cmd := exec.Command("./weave", args...)
 	cmd.Env = []string{
 		"PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
-		"PROCFS=/hostproc",
 	}
 
 	propagateEnv := func(key string) {
@@ -38,15 +37,18 @@ func callWeave(args ...string) ([]byte, []byte, error) {
 		}
 	}
 
-	propagateEnv("DOCKER_BRIDGE")
+	propagateEnv("DOCKER_HOST")
 
-	// Propogage WEAVE_DEBUG, to make debugging easier.
+	// Propagate WEAVE_DEBUG, to make debugging easier.
 	propagateEnv("WEAVE_DEBUG")
 
 	// This prevents the code coverage contortions in our
 	// integration test suite breaking things.
 	propagateEnv("COVERAGE")
 
+	// In case the router control endpoint address is non-standard.
+	propagateEnv("WEAVE_HTTP_ADDR")
+
 	Log.Debug("Calling weave args: ", args, "env: ", cmd.Env)
 	var stdout, stderr bytes.Buffer
 	cmd.Stdout = &stdout
@@ -127,35 +129,6 @@ func inspectContainerInPath(client *docker.Client, path string) (*docker.Contain
 	return container, err
 }
 
-func weaveContainerIPs(containerID string) (mac string, ips []net.IP, nets []*net.IPNet, err error) {
-	stdout, stderr, err := callWeave("ps", containerID)
-	if err != nil || len(stderr) > 0 {
-		err = errors.New(string(stderr))
-		return
-	}
-	if len(stdout) <= 0 {
-		return
-	}
-
-	fields := strings.Fields(string(stdout))
-	if len(fields) < 2 {
-		return
-	}
-	mac = fields[1]
-
-	var ip net.IP
-	var ipnet *net.IPNet
-	for _, cidr := range fields[2:] {
-		ip, ipnet, err = net.ParseCIDR(cidr)
-		if err != nil {
-			return
-		}
-		ips = append(ips, ip)
-		nets = append(nets, ipnet)
-	}
-	return
-}
-
 func addVolume(hostConfig jsonObject, source, target, mode string) error {
 	configBinds, err := hostConfig.StringArray("Binds")
 	if err != nil {

proxy/create_container_interceptor.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/fsouza/go-dockerclient"
-	. "github.com/weaveworks/weave/common"
 )
 
 const MaxDockerHostname = 64
@@ -53,8 +52,14 @@ func (i *createContainerInterceptor) InterceptRequest(r *http.Request) error {
 		Log.Infof("Leaving container alone because %s", err)
 	} else {
 		Log.Infof("Creating container with WEAVE_CIDR \"%s\"", strings.Join(cidrs, " "))
-		if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
-			return err
+		if i.proxy.NoMulticastRoute {
+			if err := addVolume(hostConfig, i.proxy.weaveWaitNomcastVolume, "/w", "ro"); err != nil {
+				return err
+			}
+		} else {
+			if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
+				return err
+			}
 		}
 		if err := i.setWeaveWaitEntrypoint(container); err != nil {
 			return err

proxy/create_exec_interceptor.go

@@ -3,8 +3,6 @@ package proxy
 import (
 	"net/http"
 	"strings"
-
-	. "github.com/weaveworks/weave/common"
 )
 
 type createExecInterceptor struct{ proxy *Proxy }

proxy/inspect_container_interceptor.go

@@ -2,8 +2,6 @@ package proxy
 
 import (
 	"net/http"
-
-	. "github.com/weaveworks/weave/common"
 )
 
 type inspectContainerInterceptor struct{ proxy *Proxy }

proxy/inspect_exec_interceptor.go

@@ -2,8 +2,6 @@ package proxy
 
 import (
 	"net/http"
-
-	. "github.com/weaveworks/weave/common"
 )
 
 type inspectExecInterceptor struct{ proxy *Proxy }
@@ -22,6 +20,10 @@ func (i *inspectExecInterceptor) InterceptResponse(r *http.Response) error {
 		return err
 	}
 
+	if _, ok := exec["Container"]; !ok {
+		return nil
+	}
+
 	container, err := exec.Object("Container")
 	if err != nil {
 		return err

proxy/json.go

@@ -1,6 +1,7 @@
 package proxy
 
 import (
+	"encoding/json"
 	"fmt"
 )
 
@@ -45,6 +46,25 @@ func (j jsonObject) String(key string) (string, error) {
 	return result, nil
 }
 
+func (j jsonObject) Int(key string) (int, error) {
+	iface, ok := j[key]
+	if !ok || iface == nil {
+		return 0, nil
+	}
+
+	result, ok := iface.(json.Number)
+	if !ok {
+		return 0, &UnmarshalWrongTypeError{key, "json.Number", iface}
+	}
+
+	i64, err := result.Int64()
+	if err != nil {
+		return 0, err
+	}
+
+	return int(i64), nil
+}
+
 func (j jsonObject) StringArray(key string) ([]string, error) {
 	iface, ok := j[key]
 	if !ok || iface == nil {

proxy/proxy.go

@@ -1,31 +1,37 @@
 package proxy
 
 import (
+	"bytes"
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
 	"syscall"
+	"time"
 
 	docker "github.com/fsouza/go-dockerclient"
-	. "github.com/weaveworks/weave/common"
+	weaveapi "github.com/weaveworks/weave/api"
+	"github.com/weaveworks/weave/common"
 	weavedocker "github.com/weaveworks/weave/common/docker"
-	"github.com/weaveworks/weave/nameserver"
-	"github.com/weaveworks/weave/router"
+	weavenet "github.com/weaveworks/weave/net"
 )
 
 const (
 	defaultCaFile   = "ca.pem"
 	defaultKeyFile  = "key.pem"
 	defaultCertFile = "cert.pem"
-	dockerSock      = "/var/run/docker.sock"
-	dockerSockUnix  = "unix://" + dockerSock
+
+	weaveSock     = "/var/run/weave/weave.sock"
+	weaveSockUnix = "unix://" + weaveSock
+
+	initialInterval = 2 * time.Second
+	maxInterval     = 1 * time.Minute
 )
 
 var (
@@ -47,14 +53,17 @@ type Config struct {
 	HostnameFromLabel   string
 	HostnameMatch       string
 	HostnameReplacement string
+	Image               string
 	ListenAddrs         []string
 	RewriteInspect      bool
 	NoDefaultIPAM       bool
 	NoRewriteHosts      bool
 	TLSConfig           TLSConfig
 	Version             string
-	WithDNS             bool
 	WithoutDNS          bool
+	NoMulticastRoute    bool
+	DockerBridge        string
+	DockerHost          string
 }
 
 type wait struct {
@@ -66,39 +75,87 @@ type wait struct {
 type Proxy struct {
 	sync.Mutex
 	Config
-	client              *docker.Client
-	dockerBridgeIP      string
-	hostnameMatchRegexp *regexp.Regexp
-	weaveWaitVolume     string
-	weaveWaitNoopVolume string
-	normalisedAddrs     []string
-	waiters             map[*http.Request]*wait
+	client                 *docker.Client
+	dockerBridgeIP         string
+	hostnameMatchRegexp    *regexp.Regexp
+	weaveWaitVolume        string
+	weaveWaitNoopVolume    string
+	weaveWaitNomcastVolume string
+	normalisedAddrs        []string
+	waiters                map[*http.Request]*wait
+	attachJobs             map[string]*attachJob
+	quit                   chan struct{}
+}
+
+type attachJob struct {
+	id          string
+	tryInterval time.Duration // retry delay on next failure
+	timer       *time.Timer
+}
+
+func (proxy *Proxy) attachWithRetry(id string) {
+	proxy.Lock()
+	defer proxy.Unlock()
+	if j, ok := proxy.attachJobs[id]; ok {
+		j.timer.Reset(time.Duration(0))
+		return
+	}
+
+	j := &attachJob{id: id, tryInterval: initialInterval}
+	j.timer = time.AfterFunc(time.Duration(0), func() {
+		if err := proxy.attach(id, false); err != nil {
+			// The delay at the nth retry is a random value in the range
+			// [i-i/2,i+i/2], where i = initialInterval * 1.5^(n-1).
+			j.timer.Reset(j.tryInterval)
+			j.tryInterval = j.tryInterval * 3 / 2
+			if j.tryInterval > maxInterval {
+				j.tryInterval = maxInterval
+			}
+			return
+		}
+		proxy.notifyWaiters(id, nil)
+	})
+	proxy.attachJobs[id] = j
+}
+
+func (j attachJob) Stop() {
+	j.timer.Stop()
 }
 
 func NewProxy(c Config) (*Proxy, error) {
-	p := &Proxy{Config: c, waiters: make(map[*http.Request]*wait)}
+	p := &Proxy{
+		Config:     c,
+		waiters:    make(map[*http.Request]*wait),
+		attachJobs: make(map[string]*attachJob),
+		quit:       make(chan struct{}),
+	}
 
 	if err := p.TLSConfig.LoadCerts(); err != nil {
 		Log.Fatalf("Could not configure tls for proxy: %s", err)
 	}
 
-	// We pin the protocol version to 1.15 (which corresponds to
-	// Docker 1.3.x; the earliest version supported by weave) in order
+	// We pin the protocol version to 1.18 (which corresponds to
+	// Docker 1.6.x; the earliest version supported by weave) in order
 	// to insulate ourselves from breaking changes to the API, as
 	// happened in 1.20 (Docker 1.8.0) when the presentation of
 	// volumes changed in `inspect`.
-	client, err := weavedocker.NewVersionedClient(dockerSockUnix, "1.15")
+	client, err := weavedocker.NewVersionedClient(c.DockerHost, "1.18")
 	if err != nil {
 		return nil, err
 	}
+	Log.Info(client.Info())
+
 	p.client = client.Client
 
 	if !p.WithoutDNS {
-		dockerBridgeIP, stderr, err := callWeave("docker-bridge-ip")
+		netDevs, err := common.GetBridgeNetDev(c.DockerBridge)
 		if err != nil {
-			return nil, fmt.Errorf(string(stderr))
+			return nil, err
 		}
-		p.dockerBridgeIP = string(dockerBridgeIP)
+		if len(netDevs) != 1 || len(netDevs[0].CIDRs) != 1 {
+			return nil, fmt.Errorf("Could not obtain address of %s", c.DockerBridge)
+		}
+		p.dockerBridgeIP = netDevs[0].CIDRs[0].IP.String()
 	}
 
 	p.hostnameMatchRegexp, err = regexp.Compile(c.HostnameMatch)
@@ -119,21 +176,21 @@ func NewProxy(c Config) (*Proxy, error) {
 func (proxy *Proxy) AttachExistingContainers() {
 	containers, _ := proxy.client.ListContainers(docker.ListContainersOptions{})
 	for _, c := range containers {
-		container, err := proxy.client.InspectContainer(c.ID)
-		if err != nil {
-			if _, ok := err.(*docker.NoSuchContainer); !ok {
-				Log.Warningf("unable to attach existing container %s since inspecting it failed: %v", c.ID, err)
-			}
-			continue
-		}
-		if containerShouldAttach(container) && (container.State.Running || container.State.Paused) {
-			proxy.attach(container, false)
-		}
+		proxy.attachWithRetry(c.ID)
 	}
 }
 
 func (proxy *Proxy) Dial() (net.Conn, error) {
-	return net.Dial("unix", dockerSock)
+	proto := "tcp"
+	addr := proxy.Config.DockerHost
+	switch {
+	case strings.HasPrefix(addr, "unix://"):
+		proto = "unix"
+		addr = strings.TrimPrefix(addr, "unix://")
+	case strings.HasPrefix(addr, "tcp://"):
+		addr = strings.TrimPrefix(addr, "tcp://")
+	}
+	return net.Dial(proto, addr)
 }
 
 func (proxy *Proxy) findWeaveWaitVolumes() error {
@@ -141,7 +198,10 @@ func (proxy *Proxy) findWeaveWaitVolumes() error {
 	if proxy.weaveWaitVolume, err = proxy.findVolume("/w"); err != nil {
 		return err
 	}
-	proxy.weaveWaitNoopVolume, err = proxy.findVolume("/w-noop")
+	if proxy.weaveWaitNoopVolume, err = proxy.findVolume("/w-noop"); err != nil {
+		return err
+	}
+	proxy.weaveWaitNomcastVolume, err = proxy.findVolume("/w-nomcast")
 	return err
 }
 
@@ -187,7 +247,12 @@ func (proxy *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func (proxy *Proxy) Listen() []net.Listener {
 	listeners := []net.Listener{}
 	proxy.normalisedAddrs = []string{}
+	unixAddrs := []string{}
 	for _, addr := range proxy.ListenAddrs {
+		if strings.HasPrefix(addr, "unix://") || strings.HasPrefix(addr, "/") {
+			unixAddrs = append(unixAddrs, addr)
+			continue
+		}
 		listener, normalisedAddr, err := proxy.listen(addr)
 		if err != nil {
 			Log.Fatalf("Cannot listen on %s: %s", addr, err)
@@ -196,6 +261,20 @@ func (proxy *Proxy) Listen() []net.Listener {
 		proxy.normalisedAddrs = append(proxy.normalisedAddrs, normalisedAddr)
 	}
 
+	if len(unixAddrs) > 0 {
+		listener, _, err := proxy.listen(weaveSockUnix)
+		if err != nil {
+			Log.Fatalf("Cannot listen on %s: %s", weaveSockUnix, err)
+		}
+		listeners = append(listeners, listener)
+
+		if err := proxy.symlink(unixAddrs); err != nil {
+			Log.Fatalf("Cannot listen on unix sockets: %s", err)
+		}
+
+		proxy.normalisedAddrs = append(proxy.normalisedAddrs, weaveSockUnix)
+	}
+
 	for _, addr := range proxy.normalisedAddrs {
 		Log.Infoln("proxy listening on", addr)
 	}
@@ -218,10 +297,7 @@ func (proxy *Proxy) Serve(listeners []net.Listener) {
 }
 
 func (proxy *Proxy) ListenAndServeStatus(socket string) {
-	if err := os.Remove(socket); err != nil && !os.IsNotExist(err) {
-		Log.Fatalf("Error removing existing status socket: %s", err)
-	}
-	listener, err := net.Listen("unix", socket)
+	listener, err := weavenet.ListenUnixSocket(socket)
 	if err != nil {
 		Log.Fatalf("ListenAndServeStatus failed: %s", err)
 	}
@@ -292,8 +368,10 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 		if err != nil {
 			return nil, "", err
 		}
-		if err = copyOwnerAndPermissions(dockerSock, addr); err != nil {
-			return nil, "", err
+		if strings.HasPrefix(proxy.Config.DockerHost, "unix://") {
+			if err = copyOwnerAndPermissions(strings.TrimPrefix(proxy.Config.DockerHost, "unix://"), addr); err != nil {
+				return nil, "", err
+			}
 		}
 
 	default:
@@ -305,18 +383,7 @@ func (proxy *Proxy) listen(protoAndAddr string) (net.Listener, string, error) {
 
 // weavedocker.ContainerObserver interface
 func (proxy *Proxy) ContainerStarted(ident string) {
-	container, err := proxy.client.InspectContainer(ident)
-	if err == nil {
-		if containerShouldAttach(container) {
-			err = proxy.attach(container, true)
-		} else if containerIsWeaveRouter(container) {
-			err = proxy.attachRouter(container)
-		}
-	}
-	if _, ok := err.(*docker.NoSuchContainer); err != nil && !ok {
-		Log.Warningf("unable to attach new container %s since inspecting it failed: %v", ident, err)
-	}
-	proxy.notifyWaiters(ident, err)
+	proxy.notifyWaiters(ident, proxy.attach(ident, true))
 }
 
 func containerShouldAttach(container *docker.Container) bool {
@@ -342,6 +409,10 @@ func (proxy *Proxy) removeWait(r *http.Request) {
 
 func (proxy *Proxy) notifyWaiters(ident string, err error) {
 	proxy.Lock()
+	if j, ok := proxy.attachJobs[ident]; ok {
+		j.Stop()
+		delete(proxy.attachJobs, ident)
+	}
 	for _, wait := range proxy.waiters {
 		if ident == wait.ident && !wait.done {
 			wait.ch <- err
@@ -385,13 +456,30 @@ func (proxy *Proxy) waitForStartByIdent(ident string) error {
 	return nil
 }
 
-func (proxy *Proxy) ContainerDied(ident string) {
-}
+func (proxy *Proxy) ContainerDied(ident string)      {}
+func (proxy *Proxy) ContainerDestroyed(ident string) {}
+
+// Check if this container needs to be attached, if so then attach it,
+// and return nil on success or not needed.
+func (proxy *Proxy) attach(containerID string, orDie bool) error {
+	container, err := proxy.client.InspectContainer(containerID)
+	if err != nil {
+		if _, ok := err.(*docker.NoSuchContainer); !ok {
+			Log.Warningf("unable to attach existing container %s since inspecting it failed: %v", containerID, err)
+		}
+		return nil
+	}
+	if containerIsWeaveRouter(container) {
+		Log.Infof("Attaching weave router container: %s", container.ID)
+		return callWeaveAttach(container, []string{"attach-router"})
+	}
+	if !containerShouldAttach(container) || !container.State.Running {
+		return nil
+	}
 
-func (proxy *Proxy) attach(container *docker.Container, orDie bool) error {
 	cidrs, err := proxy.weaveCIDRs(container.HostConfig.NetworkMode, container.Config.Env)
 	if err != nil {
-		Log.Infof("Leaving container %s alone because %s", container.ID, err)
+		Log.Infof("Leaving container %s alone because %s", containerID, err)
 		return nil
 	}
 	Log.Infof("Attaching container %s with WEAVE_CIDR \"%s\" to weave network", container.ID, strings.Join(cidrs, " "))
@@ -406,23 +494,17 @@ func (proxy *Proxy) attach(container *docker.Container, orDie bool) error {
 			}
 		}
 	}
+	if proxy.NoMulticastRoute {
+		args = append(args, "--no-multicast-route")
+	}
 	if orDie {
 		args = append(args, "--or-die")
 	}
 	args = append(args, container.ID)
-	if _, stderr, err := callWeave(args...); err != nil {
-		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
-		return errors.New(string(stderr))
-	} else if len(stderr) > 0 {
-		Log.Warningf("Attaching container %s to weave network: %s", container.ID, string(stderr))
-	}
-
-	return nil
+	return callWeaveAttach(container, args)
 }
 
-func (proxy *Proxy) attachRouter(container *docker.Container) error {
-	Log.Infof("Attaching weave router container: %s", container.ID)
-	args := []string{"attach-router"}
+func callWeaveAttach(container *docker.Container, args []string) error {
 	if _, stderr, err := callWeave(args...); err != nil {
 		Log.Warningf("Attaching container %s to weave network failed: %s", container.ID, string(stderr))
 		return errors.New(string(stderr))
@@ -472,37 +554,13 @@ func (proxy *Proxy) setWeaveDNS(hostConfig jsonObject, hostname, dnsDomain strin
 	return nil
 }
 
-func (proxy *Proxy) getDNSDomain() (domain string) {
+func (proxy *Proxy) getDNSDomain() string {
 	if proxy.WithoutDNS {
 		return ""
 	}
-
-	if proxy.WithDNS {
-		domain = nameserver.DefaultDomain
-	}
-
-	weaveContainer, err := proxy.client.InspectContainer("weave")
-	var weaveIP string
-	if err == nil && weaveContainer.NetworkSettings != nil {
-		weaveIP = weaveContainer.NetworkSettings.IPAddress
-	}
-	if weaveIP == "" {
-		weaveIP = "127.0.0.1"
-	}
-
-	url := fmt.Sprintf("http://%s:%d/domain", weaveIP, router.HTTPPort)
-	resp, err := http.Get(url)
-	if err != nil || resp.StatusCode != http.StatusOK {
-		return
-	}
-	defer resp.Body.Close()
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return
-	}
-
-	return string(b)
+	weave := weaveapi.NewClient(os.Getenv("WEAVE_HTTP_ADDR"), Log)
+	domain, _ := weave.DNSDomain()
+	return domain
 }
 
 func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
@@ -511,11 +569,21 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
 		return err
 	}
 
+	state, err := container.Object("State")
+	if err != nil {
+		return err
+	}
+
+	pid, err := state.Int("Pid")
+	if err != nil {
+		return err
+	}
+
 	if err := proxy.waitForStartByIdent(containerID); err != nil {
 		return err
 	}
-	mac, ips, nets, err := weaveContainerIPs(containerID)
-	if err != nil || len(ips) == 0 {
+	netDevs, err := common.GetWeaveNetDevs(pid)
+	if err != nil || len(netDevs) == 0 || len(netDevs[0].CIDRs) == 0 {
 		return err
 	}
 
@@ -523,8 +591,90 @@ func (proxy *Proxy) updateContainerNetworkSettings(container jsonObject) error {
 	if err != nil {
 		return err
 	}
-	networkSettings["MacAddress"] = mac
-	networkSettings["IPAddress"] = ips[0].String()
-	networkSettings["IPPrefixLen"], _ = nets[0].Mask.Size()
+	networkSettings["MacAddress"] = netDevs[0].MAC.String()
+	networkSettings["IPAddress"] = netDevs[0].CIDRs[0].IP.String()
+	networkSettings["IPPrefixLen"], _ = netDevs[0].CIDRs[0].Mask.Size()
 	return nil
 }
+
+func (proxy *Proxy) symlink(unixAddrs []string) (err error) {
+	var container *docker.Container
+	binds := []string{"/var/run/weave:/var/run/weave"}
+	froms := []string{}
+	for _, addr := range unixAddrs {
+		if addr == weaveSockUnix {
+			continue
+		}
+		from := strings.TrimPrefix(addr, "unix://")
+		dir := filepath.Dir(from)
+		binds = append(binds, dir+":"+filepath.Join("/host", dir))
+		froms = append(froms, filepath.Join("/host", from))
+		proxy.normalisedAddrs = append(proxy.normalisedAddrs, addr)
+	}
+	if len(froms) == 0 {
+		return
+	}
+
+	env := []string{
+		"PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+	}
+
+	if val := os.Getenv("WEAVE_DEBUG"); val != "" {
+		env = append(env, fmt.Sprintf("%s=%s", "WEAVE_DEBUG", val))
+	}
+
+	container, err = proxy.client.CreateContainer(docker.CreateContainerOptions{
+		Config: &docker.Config{
+			Image:      proxy.Image,
+			Entrypoint: []string{"/home/weave/symlink", weaveSock},
+			Cmd:        froms,
+			Env:        env,
+		},
+		HostConfig: &docker.HostConfig{Binds: binds},
+	})
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		err2 := proxy.client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})
+		if err == nil {
+			err = err2
+		}
+	}()
+
+	err = proxy.client.StartContainer(container.ID, nil)
+	if err != nil {
+		return
+	}
+
+	var buf bytes.Buffer
+	err = proxy.client.AttachToContainer(docker.AttachToContainerOptions{
+		Container:   container.ID,
+		ErrorStream: &buf,
+		Logs:        true,
+		Stderr:      true,
+	})
+	if err != nil {
+		return
+	}
+
+	var rc int
+	rc, err = proxy.client.WaitContainer(container.ID)
+	if err != nil {
+		return
+	}
+	if rc != 0 {
+		err = errors.New(buf.String())
+	}
+	return
+}
+
+func (proxy *Proxy) Stop() {
+	close(proxy.quit)
+	proxy.Lock()
+	defer proxy.Unlock()
+	for _, j := range proxy.attachJobs {
+		j.Stop()
+	}
+}

proxy/proxy_intercept.go

@@ -11,7 +11,6 @@ import (
 	"sync"
 
 	"github.com/fsouza/go-dockerclient"
-	. "github.com/weaveworks/weave/common"
 )
 
 func (proxy *Proxy) Intercept(i interceptor, w http.ResponseWriter, r *http.Request) {

proxy/start_container_interceptor.go

@@ -15,7 +15,7 @@ func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error {
 
 	// If the client has sent some JSON which might be a HostConfig, add our
 	// parameters back into it, otherwise Docker will consider them overwritten
-	if containerShouldAttach(container) && r.Header.Get("Content-Type") == "application/json" {
+	if containerShouldAttach(container) && r.Header.Get("Content-Type") == "application/json" && r.ContentLength > 0 {
 		params := map[string]interface{}{}
 		if err := unmarshalRequestBody(r, &params); err != nil {
 			return err
@@ -41,8 +41,14 @@ func (i *startContainerInterceptor) InterceptRequest(r *http.Request) error {
 					return err
 				}
 			} else {
-				if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
-					return err
+				if i.proxy.NoMulticastRoute {
+					if err := addVolume(hostConfig, i.proxy.weaveWaitNomcastVolume, "/w", "ro"); err != nil {
+						return err
+					}
+				} else {
+					if err := addVolume(hostConfig, i.proxy.weaveWaitVolume, "/w", "ro"); err != nil {
+						return err
+					}
 				}
 				if dnsDomain := i.proxy.getDNSDomain(); dnsDomain != "" {
 					if err := i.proxy.setWeaveDNS(hostConfig, container.Config.Hostname, dnsDomain); err != nil {

router/bridge.go

@@ -1,5 +1,9 @@
 package router
 
+import (
+	"net"
+)
+
 // Interface to packet handling on the local virtual bridge
 type Bridge interface {
 	// Inject a packet to be delivered locally
@@ -9,6 +13,7 @@ type Bridge interface {
 	// should not be included.
 	StartConsumingPackets(BridgeConsumer) error
 
+	Interface() *net.Interface
 	String() string
 	Stats() map[string]int
 }
@@ -26,6 +31,10 @@ func (NullBridge) StartConsumingPackets(BridgeConsumer) error {
 	return nil
 }
 
+func (NullBridge) Interface() *net.Interface {
+	return nil
+}
+
 func (NullBridge) String() string {
 	return "<no bridge networking>"
 }

router/connection.go

@@ -1,471 +0,0 @@
-package router
-
-import (
-	"fmt"
-	"net"
-	"strconv"
-	"sync"
-	"time"
-)
-
-type Connection interface {
-	Local() *Peer
-	Remote() *Peer
-	RemoteTCPAddr() string
-	Outbound() bool
-	Established() bool
-	BreakTie(Connection) ConnectionTieBreak
-	Shutdown(error)
-	Log(args ...interface{})
-}
-
-type ConnectionTieBreak int
-
-const (
-	TieBreakWon ConnectionTieBreak = iota
-	TieBreakLost
-	TieBreakTied
-)
-
-var ErrConnectToSelf = fmt.Errorf("Cannot connect to ourself")
-
-type RemoteConnection struct {
-	local         *Peer
-	remote        *Peer
-	remoteTCPAddr string
-	outbound      bool
-	established   bool
-}
-
-type LocalConnection struct {
-	sync.RWMutex
-	RemoteConnection
-	TCPConn       *net.TCPConn
-	version       byte
-	tcpSender     TCPSender
-	remoteUDPAddr *net.UDPAddr
-	SessionKey    *[32]byte
-	heartbeatTCP  *time.Ticker
-	Router        *Router
-	uid           uint64
-	actionChan    chan<- ConnectionAction
-	finished      <-chan struct{} // closed to signal that actorLoop has finished
-	forwarder     OverlayForwarder
-}
-
-type ConnectionAction func() error
-
-func NewRemoteConnection(from, to *Peer, tcpAddr string, outbound bool, established bool) *RemoteConnection {
-	return &RemoteConnection{
-		local:         from,
-		remote:        to,
-		remoteTCPAddr: tcpAddr,
-		outbound:      outbound,
-		established:   established,
-	}
-}
-
-func (conn *RemoteConnection) Local() *Peer                           { return conn.local }
-func (conn *RemoteConnection) Remote() *Peer                          { return conn.remote }
-func (conn *RemoteConnection) RemoteTCPAddr() string                  { return conn.remoteTCPAddr }
-func (conn *RemoteConnection) Outbound() bool                         { return conn.outbound }
-func (conn *RemoteConnection) Established() bool                      { return conn.established }
-func (conn *RemoteConnection) BreakTie(Connection) ConnectionTieBreak { return TieBreakTied }
-func (conn *RemoteConnection) Shutdown(error)                         {}
-
-func (conn *RemoteConnection) Log(args ...interface{}) {
-	log.Println(append(append([]interface{}{}, fmt.Sprintf("->[%s|%s]:", conn.remoteTCPAddr, conn.remote)), args...)...)
-}
-
-func (conn *RemoteConnection) ErrorLog(args ...interface{}) {
-	log.Errorln(append(append([]interface{}{}, fmt.Sprintf("->[%s|%s]:", conn.remoteTCPAddr, conn.remote)), args...)...)
-}
-
-// Does not return anything. If the connection is successful, it will
-// end up in the local peer's connections map.
-func StartLocalConnection(connRemote *RemoteConnection, tcpConn *net.TCPConn, udpAddr *net.UDPAddr, router *Router, acceptNewPeer bool) {
-	if connRemote.local != router.Ourself.Peer {
-		log.Fatal("Attempt to create local connection from a peer which is not ourself")
-	}
-	// NB, we're taking a copy of connRemote here.
-	actionChan := make(chan ConnectionAction, ChannelSize)
-	finished := make(chan struct{})
-	conn := &LocalConnection{
-		RemoteConnection: *connRemote,
-		Router:           router,
-		TCPConn:          tcpConn,
-		remoteUDPAddr:    udpAddr,
-		uid:              randUint64(),
-		actionChan:       actionChan,
-		finished:         finished}
-	go conn.run(actionChan, finished, acceptNewPeer)
-}
-
-func (conn *LocalConnection) BreakTie(dupConn Connection) ConnectionTieBreak {
-	dupConnLocal := dupConn.(*LocalConnection)
-	// conn.uid is used as the tie breaker here, in the knowledge that
-	// both sides will make the same decision.
-	if conn.uid < dupConnLocal.uid {
-		return TieBreakWon
-	} else if dupConnLocal.uid < conn.uid {
-		return TieBreakLost
-	} else {
-		return TieBreakTied
-	}
-}
-
-func (conn *LocalConnection) Established() bool {
-	return conn.established
-}
-
-// Send directly, not via the Actor.  If it goes via the Actor we can
-// get a deadlock where LocalConnection is blocked talking to
-// LocalPeer and LocalPeer is blocked trying send a ProtocolMsg via
-// LocalConnection, and the channels are full in both directions so
-// nothing can proceed.
-func (conn *LocalConnection) SendProtocolMsg(m ProtocolMsg) {
-	if err := conn.sendProtocolMsg(m); err != nil {
-		conn.Shutdown(err)
-	}
-}
-
-// ACTOR methods
-
-// NB: The conn.* fields are only written by the connection actor
-// process, which is the caller of the ConnectionAction funs. Hence we
-// do not need locks for reading, and only need write locks for fields
-// read by other processes.
-
-// Async
-func (conn *LocalConnection) Shutdown(err error) {
-	// err should always be a real error, even if only io.EOF
-	if err == nil {
-		panic("nil error")
-	}
-
-	// Run on its own goroutine in case the channel is backed up
-	go func() { conn.sendAction(func() error { return err }) }()
-}
-
-// Send an actor request to the actorLoop, but don't block if
-// actorLoop has exited - see http://blog.golang.org/pipelines for
-// pattern
-func (conn *LocalConnection) sendAction(action ConnectionAction) {
-	select {
-	case conn.actionChan <- action:
-	case <-conn.finished:
-	}
-}
-
-// ACTOR server
-
-func (conn *LocalConnection) run(actionChan <-chan ConnectionAction, finished chan<- struct{}, acceptNewPeer bool) {
-	var err error // important to use this var and not create another one with 'err :='
-	defer func() { conn.shutdown(err) }()
-	defer close(finished)
-
-	conn.TCPConn.SetLinger(0)
-	intro, err := ProtocolIntroParams{
-		MinVersion: conn.Router.ProtocolMinVersion,
-		MaxVersion: ProtocolMaxVersion,
-		Features:   conn.makeFeatures(),
-		Conn:       conn.TCPConn,
-		Password:   conn.Router.Password,
-		Outbound:   conn.outbound,
-	}.DoIntro()
-	if err != nil {
-		return
-	}
-
-	conn.SessionKey = intro.SessionKey
-	conn.tcpSender = intro.Sender
-	conn.version = intro.Version
-
-	remote, err := conn.parseFeatures(intro.Features)
-	if err != nil {
-		return
-	}
-
-	if err = conn.registerRemote(remote, acceptNewPeer); err != nil {
-		return
-	}
-
-	conn.Log("connection ready; using protocol version", conn.version)
-
-	params := ForwarderParams{
-		RemotePeer:         conn.remote,
-		LocalIP:            conn.TCPConn.LocalAddr().(*net.TCPAddr).IP,
-		RemoteAddr:         conn.remoteUDPAddr,
-		ConnUID:            conn.uid,
-		Crypto:             conn.forwarderCrypto(),
-		SendControlMessage: conn.sendOverlayControlMessage,
-		Features:           intro.Features,
-	}
-	if conn.forwarder, err = conn.Router.Overlay.MakeForwarder(params); err != nil {
-		return
-	}
-
-	// As soon as we do AddConnection, the new connection becomes
-	// visible to the packet routing logic.  So AddConnection must
-	// come after MakeForwarder
-	if err = conn.Router.Ourself.AddConnection(conn); err != nil {
-		return
-	}
-	conn.Router.ConnectionMaker.ConnectionCreated(conn)
-
-	// Forwarder confirmation comes after AddConnection, because
-	// only after that completes do we know the connection is
-	// valid: in particular that it is not a duplicate connection
-	// to the same peer. Sending heartbeats on a duplicate
-	// connection can trip up crypto at the other end, since the
-	// associated UDP packets may get decoded by the other
-	// connection. It is also generally wasteful to engage in any
-	// interaction with the remote on a connection that turns out
-	// to be invalid.
-	conn.forwarder.Confirm()
-
-	// receiveTCP must follow also AddConnection. In the absence
-	// of any indirect connectivity to the remote peer, the first
-	// we hear about it (and any peers reachable from it) is
-	// through topology gossip it sends us on the connection. We
-	// must ensure that the connection has been added to Ourself
-	// prior to processing any such gossip, otherwise we risk
-	// immediately gc'ing part of that newly received portion of
-	// the topology (though not the remote peer itself, since that
-	// will have a positive ref count), leaving behind dangling
-	// references to peers. Hence we must invoke AddConnection,
-	// which is *synchronous*, first.
-	conn.heartbeatTCP = time.NewTicker(TCPHeartbeat)
-	go conn.receiveTCP(intro.Receiver)
-
-	// AddConnection must precede actorLoop. More precisely, it
-	// must precede shutdown, since that invokes DeleteConnection
-	// and is invoked on termination of this entire
-	// function. Essentially this boils down to a prohibition on
-	// running AddConnection in a separate goroutine, at least not
-	// without some synchronisation. Which in turn requires the
-	// launching of the receiveTCP goroutine to precede actorLoop.
-	err = conn.actorLoop(actionChan)
-}
-
-func (conn *LocalConnection) makeFeatures() map[string]string {
-	features := map[string]string{
-		"PeerNameFlavour": PeerNameFlavour,
-		"Name":            conn.local.Name.String(),
-		"NickName":        conn.local.NickName,
-		"ShortID":         fmt.Sprint(conn.local.ShortID),
-		"UID":             fmt.Sprint(conn.local.UID),
-		"ConnID":          fmt.Sprint(conn.uid),
-	}
-	conn.Router.Overlay.AddFeaturesTo(features)
-	return features
-}
-
-type features map[string]string
-
-func (features features) MustHave(keys []string) error {
-	for _, key := range keys {
-		if _, ok := features[key]; !ok {
-			return fmt.Errorf("Field %s is missing", key)
-		}
-	}
-	return nil
-}
-
-func (features features) Get(key string) string {
-	return features[key]
-}
-
-func (conn *LocalConnection) parseFeatures(features features) (*Peer, error) {
-	if err := features.MustHave([]string{"PeerNameFlavour", "Name", "NickName", "UID", "ConnID"}); err != nil {
-		return nil, err
-	}
-
-	remotePeerNameFlavour := features.Get("PeerNameFlavour")
-	if remotePeerNameFlavour != PeerNameFlavour {
-		return nil, fmt.Errorf("Peer name flavour mismatch (ours: '%s', theirs: '%s')", PeerNameFlavour, remotePeerNameFlavour)
-	}
-
-	name, err := PeerNameFromString(features.Get("Name"))
-	if err != nil {
-		return nil, err
-	}
-
-	nickName := features.Get("NickName")
-
-	var shortID uint64
-	var hasShortID bool
-	if shortIDStr, present := features["ShortID"]; present {
-		hasShortID = true
-		shortID, err = strconv.ParseUint(shortIDStr, 10,
-			PeerShortIDBits)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	uid, err := ParsePeerUID(features.Get("UID"))
-	if err != nil {
-		return nil, err
-	}
-
-	remoteConnID, err := strconv.ParseUint(features.Get("ConnID"), 10, 64)
-	if err != nil {
-		return nil, err
-	}
-
-	conn.uid ^= remoteConnID
-	peer := NewPeer(name, nickName, uid, 0, PeerShortID(shortID))
-	peer.HasShortID = hasShortID
-	return peer, nil
-}
-
-func (conn *LocalConnection) registerRemote(remote *Peer, acceptNewPeer bool) error {
-	if acceptNewPeer {
-		conn.remote = conn.Router.Peers.FetchWithDefault(remote)
-	} else {
-		conn.remote = conn.Router.Peers.FetchAndAddRef(remote.Name)
-		if conn.remote == nil {
-			return fmt.Errorf("Found unknown remote name: %s at %s", remote.Name, conn.remoteTCPAddr)
-		}
-	}
-
-	if conn.remote.UID != remote.UID {
-		return fmt.Errorf("Connection appears to be with different version of a peer we already know of")
-	}
-
-	if conn.remote == conn.local {
-		return ErrConnectToSelf
-	}
-
-	return nil
-}
-
-func (conn *LocalConnection) actorLoop(actionChan <-chan ConnectionAction) (err error) {
-	fwdErrorChan := conn.forwarder.ErrorChannel()
-	fwdEstablishedChan := conn.forwarder.EstablishedChannel()
-
-	for err == nil {
-		select {
-		case action := <-actionChan:
-			err = action()
-
-		case <-conn.heartbeatTCP.C:
-			err = conn.sendSimpleProtocolMsg(ProtocolHeartbeat)
-
-		case <-fwdEstablishedChan:
-			conn.established = true
-			fwdEstablishedChan = nil
-			conn.Router.Ourself.ConnectionEstablished(conn)
-
-		case err = <-fwdErrorChan:
-		}
-	}
-	return
-}
-
-func (conn *LocalConnection) shutdown(err error) {
-	if conn.remote == nil {
-		log.Errorf("->[%s] connection shutting down due to error during handshake: %v", conn.remoteTCPAddr, err)
-	} else {
-		conn.ErrorLog("connection shutting down due to error:", err)
-	}
-
-	if conn.TCPConn != nil {
-		checkWarn(conn.TCPConn.Close())
-	}
-
-	if conn.remote != nil {
-		conn.Router.Peers.Dereference(conn.remote)
-		conn.Router.Ourself.DeleteConnection(conn)
-	}
-
-	stopTicker(conn.heartbeatTCP)
-
-	if conn.forwarder != nil {
-		conn.forwarder.Stop()
-	}
-
-	conn.Router.ConnectionMaker.ConnectionTerminated(conn, err)
-}
-
-func (conn *LocalConnection) forwarderCrypto() *OverlayCrypto {
-	if !conn.Router.UsingPassword() {
-		return nil
-	}
-
-	name := conn.local.NameByte
-	return &OverlayCrypto{
-		Dec:   NewNaClDecryptor(conn.SessionKey, conn.outbound),
-		Enc:   NewNaClEncryptor(name, conn.SessionKey, conn.outbound, false),
-		EncDF: NewNaClEncryptor(name, conn.SessionKey, conn.outbound, true),
-	}
-}
-
-func (conn *LocalConnection) sendOverlayControlMessage(tag byte, msg []byte) error {
-	return conn.sendProtocolMsg(ProtocolMsg{ProtocolTag(tag), msg})
-}
-
-// Helpers
-
-func (conn *LocalConnection) sendSimpleProtocolMsg(tag ProtocolTag) error {
-	return conn.sendProtocolMsg(ProtocolMsg{tag: tag})
-}
-
-func (conn *LocalConnection) sendProtocolMsg(m ProtocolMsg) error {
-	return conn.tcpSender.Send(Concat([]byte{byte(m.tag)}, m.msg))
-}
-
-func (conn *LocalConnection) receiveTCP(receiver TCPReceiver) {
-	var err error
-	for {
-		conn.extendReadDeadline()
-
-		var msg []byte
-		if msg, err = receiver.Receive(); err != nil {
-			break
-		}
-		if len(msg) < 1 {
-			conn.Log("ignoring blank msg")
-			continue
-		}
-		if err = conn.handleProtocolMsg(ProtocolTag(msg[0]), msg[1:]); err != nil {
-			break
-		}
-	}
-	conn.Shutdown(err)
-}
-
-func (conn *LocalConnection) handleProtocolMsg(tag ProtocolTag, payload []byte) error {
-	switch tag {
-	case ProtocolHeartbeat:
-	case ProtocolConnectionEstablished, ProtocolFragmentationReceived, ProtocolPMTUVerified, ProtocolOverlayControlMsg:
-		conn.forwarder.ControlMessage(byte(tag), payload)
-	case ProtocolGossipUnicast, ProtocolGossipBroadcast, ProtocolGossip:
-		return conn.Router.handleGossip(tag, payload)
-	default:
-		conn.Log("ignoring unknown protocol tag:", tag)
-	}
-	return nil
-}
-
-func (conn *LocalConnection) extendReadDeadline() {
-	conn.TCPConn.SetReadDeadline(time.Now().Add(TCPHeartbeat * 2))
-}
-
-func (conn *LocalConnection) Forward(key ForwardPacketKey) FlowOp {
-	return conn.forwarder.Forward(key)
-}
-
-func tickerChan(ticker *time.Ticker) <-chan time.Time {
-	if ticker != nil {
-		return ticker.C
-	}
-	return nil
-}
-
-func stopTicker(ticker *time.Ticker) {
-	if ticker != nil {
-		ticker.Stop()
-	}
-}

router/connection_maker.go

@@ -1,312 +0,0 @@
-package router
-
-import (
-	"fmt"
-	"math/rand"
-	"net"
-	"time"
-)
-
-const (
-	InitialInterval = 2 * time.Second
-	MaxInterval     = 6 * time.Minute
-	ResetAfter      = 1 * time.Minute
-)
-
-type peerAddrs map[string]*net.TCPAddr
-
-type ConnectionMaker struct {
-	ourself     *LocalPeer
-	peers       *Peers
-	port        int
-	discovery   bool
-	targets     map[string]*Target
-	connections map[Connection]struct{}
-	directPeers peerAddrs
-	actionChan  chan<- ConnectionMakerAction
-}
-
-type TargetState int
-
-const (
-	TargetWaiting    TargetState = iota // we are waiting to connect there
-	TargetAttempting                    // we are attempting to connect there
-	TargetConnected                     // we are connected to there
-)
-
-// Information about an address where we may find a peer
-type Target struct {
-	state       TargetState
-	lastError   error         // reason for disconnection last time
-	tryAfter    time.Time     // next time to try this address
-	tryInterval time.Duration // retry delay on next failure
-}
-
-type ConnectionMakerAction func() bool
-
-func NewConnectionMaker(ourself *LocalPeer, peers *Peers, port int, discovery bool) *ConnectionMaker {
-	actionChan := make(chan ConnectionMakerAction, ChannelSize)
-	cm := &ConnectionMaker{
-		ourself:     ourself,
-		peers:       peers,
-		port:        port,
-		discovery:   discovery,
-		directPeers: peerAddrs{},
-		targets:     make(map[string]*Target),
-		connections: make(map[Connection]struct{}),
-		actionChan:  actionChan}
-	go cm.queryLoop(actionChan)
-	return cm
-}
-
-func (cm *ConnectionMaker) InitiateConnections(peers []string, replace bool) []error {
-	errors := []error{}
-	addrs := peerAddrs{}
-	for _, peer := range peers {
-		host, port, err := net.SplitHostPort(peer)
-		if err != nil {
-			host = peer
-			port = "0" // we use that as an indication that "no port was supplied"
-		}
-		if addr, err := net.ResolveTCPAddr("tcp4", fmt.Sprintf("%s:%s", host, port)); err != nil {
-			errors = append(errors, err)
-		} else {
-			addrs[peer] = addr
-		}
-	}
-	cm.actionChan <- func() bool {
-		if replace {
-			cm.directPeers = peerAddrs{}
-		}
-		for peer, addr := range addrs {
-			cm.directPeers[peer] = addr
-			// curtail any existing reconnect interval
-			if target, found := cm.targets[addr.String()]; found {
-				target.nextTryNow()
-			}
-		}
-		return true
-	}
-	return errors
-}
-
-func (cm *ConnectionMaker) ForgetConnections(peers []string) {
-	cm.actionChan <- func() bool {
-		for _, peer := range peers {
-			delete(cm.directPeers, peer)
-		}
-		return false
-	}
-}
-
-func (cm *ConnectionMaker) ConnectionAborted(address string, err error) {
-	cm.actionChan <- func() bool {
-		target := cm.targets[address]
-		target.state = TargetWaiting
-		target.lastError = err
-		target.nextTryLater()
-		return true
-	}
-}
-
-func (cm *ConnectionMaker) ConnectionCreated(conn Connection) {
-	cm.actionChan <- func() bool {
-		cm.connections[conn] = void
-		if conn.Outbound() {
-			target := cm.targets[conn.RemoteTCPAddr()]
-			target.state = TargetConnected
-		}
-		return false
-	}
-}
-
-func (cm *ConnectionMaker) ConnectionTerminated(conn Connection, err error) {
-	cm.actionChan <- func() bool {
-		delete(cm.connections, conn)
-		if conn.Outbound() {
-			target := cm.targets[conn.RemoteTCPAddr()]
-			target.state = TargetWaiting
-			target.lastError = err
-			switch {
-			case err == ErrConnectToSelf:
-				target.nextTryNever()
-			case time.Now().After(target.tryAfter.Add(ResetAfter)):
-				target.nextTryNow()
-			default:
-				target.nextTryLater()
-			}
-		}
-		return true
-	}
-}
-
-func (cm *ConnectionMaker) Refresh() {
-	cm.actionChan <- func() bool { return true }
-}
-
-func (cm *ConnectionMaker) queryLoop(actionChan <-chan ConnectionMakerAction) {
-	timer := time.NewTimer(MaxDuration)
-	run := func() { timer.Reset(cm.checkStateAndAttemptConnections()) }
-	for {
-		select {
-		case action := <-actionChan:
-			if action() {
-				run()
-			}
-		case <-timer.C:
-			run()
-		}
-	}
-}
-
-func (cm *ConnectionMaker) checkStateAndAttemptConnections() time.Duration {
-	var (
-		validTarget  = make(map[string]struct{})
-		directTarget = make(map[string]struct{})
-	)
-	ourConnectedPeers, ourConnectedTargets, ourInboundIPs := cm.ourConnections()
-
-	addTarget := func(address string) {
-		if _, connected := ourConnectedTargets[address]; connected {
-			return
-		}
-		validTarget[address] = void
-		if _, found := cm.targets[address]; found {
-			return
-		}
-		target := &Target{state: TargetWaiting}
-		target.nextTryNow()
-		cm.targets[address] = target
-	}
-
-	// Add direct targets that are not connected
-	for _, addr := range cm.directPeers {
-		completeAddr := *addr
-		attempt := true
-		if completeAddr.Port == 0 {
-			completeAddr.Port = cm.port
-			// If a peer was specified w/o a port, then we do not
-			// attempt to connect to it if we have any inbound
-			// connections from that IP.
-			if _, connected := ourInboundIPs[completeAddr.IP.String()]; connected {
-				attempt = false
-			}
-		}
-		address := completeAddr.String()
-		directTarget[address] = void
-		if attempt {
-			addTarget(address)
-		}
-	}
-
-	// Add targets for peers that someone else is connected to, but we
-	// aren't
-	if cm.discovery {
-		cm.addPeerTargets(ourConnectedPeers, addTarget)
-	}
-
-	return cm.connectToTargets(validTarget, directTarget)
-}
-
-func (cm *ConnectionMaker) ourConnections() (PeerNameSet, map[string]struct{}, map[string]struct{}) {
-	var (
-		ourConnectedPeers   = make(PeerNameSet)
-		ourConnectedTargets = make(map[string]struct{})
-		ourInboundIPs       = make(map[string]struct{})
-	)
-	for conn := range cm.connections {
-		address := conn.RemoteTCPAddr()
-		ourConnectedPeers[conn.Remote().Name] = void
-		ourConnectedTargets[address] = void
-		if conn.Outbound() {
-			continue
-		}
-		if ip, _, err := net.SplitHostPort(address); err == nil { // should always succeed
-			ourInboundIPs[ip] = void
-		}
-	}
-	return ourConnectedPeers, ourConnectedTargets, ourInboundIPs
-}
-
-func (cm *ConnectionMaker) addPeerTargets(ourConnectedPeers PeerNameSet, addTarget func(string)) {
-	cm.peers.ForEach(func(peer *Peer) {
-		if peer == cm.ourself.Peer {
-			return
-		}
-		// Modifying peer.connections requires a write lock on Peers,
-		// and since we are holding a read lock (due to the ForEach),
-		// access without locking the peer is safe.
-		for otherPeer, conn := range peer.connections {
-			if otherPeer == cm.ourself.Name {
-				continue
-			}
-			if _, connected := ourConnectedPeers[otherPeer]; connected {
-				continue
-			}
-			address := conn.RemoteTCPAddr()
-			if conn.Outbound() {
-				addTarget(address)
-			} else if ip, _, err := net.SplitHostPort(address); err == nil {
-				// There is no point connecting to the (likely
-				// ephemeral) remote port of an inbound connection
-				// that some peer has. Let's try to connect on the
-				// weave port instead.
-				addTarget(fmt.Sprintf("%s:%d", ip, cm.port))
-			}
-		}
-	})
-}
-
-func (cm *ConnectionMaker) connectToTargets(validTarget map[string]struct{}, directTarget map[string]struct{}) time.Duration {
-	now := time.Now() // make sure we catch items just added
-	after := MaxDuration
-	for address, target := range cm.targets {
-		if target.state != TargetWaiting {
-			continue
-		}
-		if _, valid := validTarget[address]; !valid {
-			delete(cm.targets, address)
-			continue
-		}
-		if target.tryAfter.IsZero() {
-			continue
-		}
-		switch duration := target.tryAfter.Sub(now); {
-		case duration <= 0:
-			target.state = TargetAttempting
-			_, isCmdLineTarget := directTarget[address]
-			go cm.attemptConnection(address, isCmdLineTarget)
-		case duration < after:
-			after = duration
-		}
-	}
-	return after
-}
-
-func (cm *ConnectionMaker) attemptConnection(address string, acceptNewPeer bool) {
-	log.Printf("->[%s] attempting connection", address)
-	if err := cm.ourself.CreateConnection(address, acceptNewPeer); err != nil {
-		log.Errorf("->[%s] error during connection attempt: %v", address, err)
-		cm.ConnectionAborted(address, err)
-	}
-}
-
-func (t *Target) nextTryNever() {
-	t.tryAfter = time.Time{}
-	t.tryInterval = MaxInterval
-}
-
-func (t *Target) nextTryNow() {
-	t.tryAfter = time.Now()
-	t.tryInterval = InitialInterval
-}
-
-// The delay at the nth retry is a random value in the range
-// [i-i/2,i+i/2], where i = InitialInterval * 1.5^(n-1).
-func (t *Target) nextTryLater() {
-	t.tryAfter = time.Now().Add(t.tryInterval/2 + time.Duration(rand.Int63n(int64(t.tryInterval))))
-	t.tryInterval = t.tryInterval * 3 / 2
-	if t.tryInterval > MaxInterval {
-		t.tryInterval = MaxInterval
-	}
-}

router/consts.go

@@ -1,21 +0,0 @@
-package router
-
-import (
-	"math"
-	"time"
-)
-
-const (
-	Port                = 6783
-	HTTPPort            = Port + 1
-	MaxUDPPacketSize    = 65535
-	ChannelSize         = 16
-	TCPHeartbeat        = 30 * time.Second
-	GossipInterval      = 30 * time.Second
-	MaxDuration         = time.Duration(math.MaxInt64)
-	MaxTCPMsgSize       = 10 * 1024 * 1024
-	FastHeartbeat       = 500 * time.Millisecond
-	SlowHeartbeat       = 10 * time.Second
-	MaxMissedHeartbeats = 6
-	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
-)

router/fastdp.go

@@ -6,9 +6,11 @@ import (
 	"fmt"
 	"net"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/weaveworks/go-odp/odp"
+	"github.com/weaveworks/mesh"
 )
 
 // The virtual bridge accepts packets from ODP vports and the router
@@ -26,22 +28,15 @@ type bridgeSender func(key PacketKey, lock *fastDatapathLock) FlowOp
 type missHandler func(fks odp.FlowKeys, lock *fastDatapathLock) FlowOp
 
 type FastDatapath struct {
-	dpname string
-
-	// The mtu from the datapath netdev (which should match the
-	// mtus on all the veths hooked up to the datapath).  We
-	// validate that we are able to support that mtu.
-	mtu int
-
-	// The lock guards the FastDatapath state, and also
-	// synchronizes use of the dpif
-	lock             sync.Mutex
+	lock             sync.Mutex // guards state and synchronises use of dpif
+	iface            *net.Interface
 	dpif             *odp.Dpif
 	dp               odp.DatapathHandle
 	deleteFlowsCount uint64
+	missCount        uint64
 	missHandlers     map[odp.VportID]missHandler
-	localPeer        *Peer
-	peers            *Peers
+	localPeer        *mesh.Peer
+	peers            *mesh.Peers
 	overlayConsumer  OverlayConsumer
 
 	// Bridge state: How to send to the given bridge port
@@ -62,17 +57,10 @@ type FastDatapath struct {
 	dec *EthernetDecoder
 
 	// forwarders by remote peer
-	forwarders map[PeerName]*fastDatapathForwarder
+	forwarders map[mesh.PeerName]*fastDatapathForwarder
 }
 
-type FastDatapathConfig struct {
-	DatapathName        string
-	Port                int
-	ExpireFlowsInterval time.Duration
-	ExpireMACsInterval  time.Duration
-}
-
-func NewFastDatapath(config FastDatapathConfig) (*FastDatapath, error) {
+func NewFastDatapath(iface *net.Interface, port int) (*FastDatapath, error) {
 	dpif, err := odp.NewDpif()
 	if err != nil {
 		return nil, err
@@ -85,19 +73,13 @@ func NewFastDatapath(config FastDatapathConfig) (*FastDatapath, error) {
 		}
 	}()
 
-	dp, err := dpif.LookupDatapath(config.DatapathName)
-	if err != nil {
-		return nil, err
-	}
-
-	iface, err := net.InterfaceByName(config.DatapathName)
+	dp, err := dpif.LookupDatapath(iface.Name)
 	if err != nil {
 		return nil, err
 	}
 
 	fastdp := &FastDatapath{
-		dpname:        config.DatapathName,
-		mtu:           iface.MTU,
+		iface:         iface,
 		dpif:          dpif,
 		dp:            dp,
 		missHandlers:  make(map[odp.VportID]missHandler),
@@ -105,9 +87,13 @@ func NewFastDatapath(config FastDatapathConfig) (*FastDatapath, error) {
 		sendToMAC:     make(map[MAC]bridgeSender),
 		seenMACs:      make(map[MAC]struct{}),
 		vxlanVportIDs: make(map[int]odp.VportID),
-		forwarders:    make(map[PeerName]*fastDatapathForwarder),
+		forwarders:    make(map[mesh.PeerName]*fastDatapathForwarder),
 	}
 
+	// This delete happens asynchronously in the kernel, meaning that
+	// we can sometimes fail to recreate the vxlan vport with EADDRINUSE -
+	// consequently we retry a small number of times in
+	// getVxlanVportIDHarder() to compensate.
 	if err := fastdp.deleteVxlanVports(); err != nil {
 		return nil, err
 	}
@@ -124,7 +110,7 @@ func NewFastDatapath(config FastDatapathConfig) (*FastDatapath, error) {
 	// numbers to be independent, but working out how to specify
 	// them on the connecting side.  So we can wait to find out if
 	// anyone wants that.
-	fastdp.mainVxlanVportID, err = fastdp.getVxlanVportID(config.Port + 1)
+	fastdp.mainVxlanVportID, err = fastdp.getVxlanVportIDHarder(port+1, 5, time.Millisecond*10)
 	if err != nil {
 		return nil, err
 	}
@@ -211,12 +197,21 @@ func (fastdp *FastDatapath) Bridge() Bridge {
 	return fastDatapathBridge{fastdp}
 }
 
-func (fastdp fastDatapathBridge) String() string {
-	return fmt.Sprint(fastdp.dpname, " (via ODP)")
+func (fastdp fastDatapathBridge) Interface() *net.Interface {
+	return fastdp.iface
 }
 
-func (fastDatapathBridge) Stats() map[string]int {
-	return nil
+func (fastdp fastDatapathBridge) String() string {
+	return fmt.Sprint(fastdp.iface.Name, " (via ODP)")
+}
+
+func (fastdp fastDatapathBridge) Stats() map[string]int {
+	lock := fastdp.startLock()
+	defer lock.unlock()
+
+	return map[string]int{
+		"FlowMisses": int(fastdp.missCount),
+	}
 }
 
 var routerBridgePortID = bridgePortID{router: true}
@@ -248,8 +243,7 @@ func (fastdp fastDatapathBridge) InjectPacket(key PacketKey) FlowOp {
 
 // Ethernet bridge implementation
 
-func (fastdp *FastDatapath) bridge(ingress bridgePortID,
-	key PacketKey, lock *fastDatapathLock) FlowOp {
+func (fastdp *FastDatapath) bridge(ingress bridgePortID, key PacketKey, lock *fastDatapathLock) FlowOp {
 	lock.relock()
 	if fastdp.sendToMAC[key.SrcMAC] == nil {
 		// Learn the source MAC
@@ -260,8 +254,7 @@ func (fastdp *FastDatapath) bridge(ingress bridgePortID,
 	// If we know about the destination MAC, deliver it to the
 	// associated port.
 	if sender := fastdp.sendToMAC[key.DstMAC]; sender != nil {
-		return NewMultiFlowOp(false, odpEthernetFlowKey(key),
-			sender(key, lock))
+		return NewMultiFlowOp(false, odpEthernetFlowKey(key), sender(key, lock))
 	}
 
 	// Otherwise, it might be a real broadcast, or it might
@@ -299,7 +292,7 @@ type fastDatapathOverlay struct {
 	*FastDatapath
 }
 
-func (fastdp *FastDatapath) Overlay() Overlay {
+func (fastdp *FastDatapath) Overlay() NetworkOverlay {
 	return fastDatapathOverlay{fastdp}
 }
 
@@ -345,8 +338,7 @@ func (flowStatus *FlowStatus) MarshalJSON() ([]byte, error) {
 		actions = append(actions, fmt.Sprint(action))
 	}
 
-	return json.Marshal(&jsonFlowStatus{flowKeys, actions,
-		flowStatus.Packets, flowStatus.Bytes, flowStatus.Used})
+	return json.Marshal(&jsonFlowStatus{flowKeys, actions, flowStatus.Packets, flowStatus.Bytes, flowStatus.Used})
 }
 
 type VportStatus odp.Vport
@@ -358,8 +350,7 @@ func (vport *VportStatus) MarshalJSON() ([]byte, error) {
 		TypeName string
 	}
 
-	return json.Marshal(&jsonVportStatus{vport.ID, vport.Spec.Name(),
-		vport.Spec.TypeName()})
+	return json.Marshal(&jsonVportStatus{vport.ID, vport.Spec.Name(), vport.Spec.TypeName()})
 }
 
 func (fastdp fastDatapathOverlay) Diagnostics() interface{} {
@@ -367,18 +358,14 @@ func (fastdp fastDatapathOverlay) Diagnostics() interface{} {
 	defer lock.unlock()
 
 	vports, err := fastdp.dp.EnumerateVports()
-	if err != nil {
-		log.Warn(err)
-	}
+	checkWarn(err)
 	vportStatuses := make([]VportStatus, 0, len(vports))
 	for _, vport := range vports {
 		vportStatuses = append(vportStatuses, VportStatus(vport))
 	}
 
 	flows, err := fastdp.dp.EnumerateFlows()
-	if err != nil {
-		log.Warn(err)
-	}
+	checkWarn(err)
 	flowStatuses := make([]FlowStatus, 0, len(flows))
 	for _, flow := range flows {
 		flowStatuses = append(flowStatuses, FlowStatus(flow))
@@ -393,8 +380,7 @@ func (fastdp fastDatapathOverlay) Diagnostics() interface{} {
 	}
 }
 
-func (fastdp fastDatapathOverlay) StartConsumingPackets(localPeer *Peer,
-	peers *Peers, consumer OverlayConsumer) error {
+func (fastdp fastDatapathOverlay) StartConsumingPackets(localPeer *mesh.Peer, peers *mesh.Peers, consumer OverlayConsumer) error {
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()
 
@@ -408,6 +394,20 @@ func (fastdp fastDatapathOverlay) StartConsumingPackets(localPeer *Peer,
 	return nil
 }
 
+func (fastdp *FastDatapath) getVxlanVportIDHarder(udpPort int, retries int, duration time.Duration) (odp.VportID, error) {
+	var vxlanVportID odp.VportID
+	var err error
+	for try := 0; try < retries; try++ {
+		vxlanVportID, err = fastdp.getVxlanVportID(udpPort)
+		if err == nil || err != odp.NetlinkError(syscall.EADDRINUSE) {
+			return vxlanVportID, err
+		}
+		log.Warning("Address already in use creating vxlan vport ", udpPort, " - retrying")
+		time.Sleep(duration)
+	}
+	return 0, err
+}
+
 func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()
@@ -417,8 +417,7 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 	}
 
 	vxlanVportID, err := fastdp.dp.CreateVport(
-		odp.NewVxlanVportSpec(fmt.Sprintf("vxlan-%d", udpPort),
-			uint16(udpPort)))
+		odp.NewVxlanVportSpec(fmt.Sprintf("vxlan-%d", udpPort), uint16(udpPort)))
 	if err != nil {
 		return 0, err
 	}
@@ -464,29 +463,27 @@ func (fastdp *FastDatapath) getVxlanVportID(udpPort int) (odp.VportID, error) {
 		tunnelFlowKey.SetIpv4Src(tunKey.Ipv4Src)
 		tunnelFlowKey.SetIpv4Dst(tunKey.Ipv4Dst)
 
-		return NewMultiFlowOp(false, odpFlowKey(tunnelFlowKey),
-			consumer(key))
+		return NewMultiFlowOp(false, odpFlowKey(tunnelFlowKey), consumer(key))
 	}
 
 	return vxlanVportID, nil
 }
 
-func (fastdp *FastDatapath) extractPeers(tunnelID [8]byte) (*Peer, *Peer) {
+func (fastdp *FastDatapath) extractPeers(tunnelID [8]byte) (*mesh.Peer, *mesh.Peer) {
 	vni := binary.BigEndian.Uint64(tunnelID[:])
-	srcPeer := fastdp.peers.FetchByShortID(PeerShortID(vni & 0xfff))
-	dstPeer := fastdp.peers.FetchByShortID(PeerShortID((vni >> 12) & 0xfff))
+	srcPeer := fastdp.peers.FetchByShortID(mesh.PeerShortID(vni & 0xfff))
+	dstPeer := fastdp.peers.FetchByShortID(mesh.PeerShortID((vni >> 12) & 0xfff))
 	return srcPeer, dstPeer
 }
 
 type vxlanSpecialPacketFlowOp struct {
 	NonDiscardingFlowOp
 	fastdp  *FastDatapath
-	srcPeer *Peer
+	srcPeer *mesh.Peer
 	sender  *net.UDPAddr
 }
 
-func (op vxlanSpecialPacketFlowOp) Process(frame []byte, dec *EthernetDecoder,
-	broadcast bool) {
+func (op vxlanSpecialPacketFlowOp) Process(frame []byte, dec *EthernetDecoder, broadcast bool) {
 	op.fastdp.lock.Lock()
 	fwd := op.fastdp.forwarders[op.srcPeer.Name]
 	op.fastdp.lock.Unlock()
@@ -498,7 +495,7 @@ func (op vxlanSpecialPacketFlowOp) Process(frame []byte, dec *EthernetDecoder,
 
 type fastDatapathForwarder struct {
 	fastdp         *FastDatapath
-	remotePeer     *Peer
+	remotePeer     *mesh.Peer
 	localIP        [4]byte
 	sendControlMsg func(byte, []byte) error
 	connUID        uint64
@@ -518,24 +515,23 @@ type fastDatapathForwarder struct {
 	errorChan       chan error
 }
 
-func (fastdp fastDatapathOverlay) MakeForwarder(
-	params ForwarderParams) (OverlayForwarder, error) {
-	if params.Crypto != nil {
-		// No encryption suport in fastdp.  The weaver main.go
-		// is responsible for ensuring this doesn't happen.
-		log.Fatal("Attempt to use FastDatapath with encryption")
+func (fastdp fastDatapathOverlay) PrepareConnection(params mesh.OverlayConnectionParams) (mesh.OverlayConnection, error) {
+	if params.SessionKey != nil {
+		// No encryption support in fastdp
+		return nil, fmt.Errorf("encryption not supported")
 	}
 
 	vxlanVportID := fastdp.mainVxlanVportID
-	remoteAddr := params.RemoteAddr
-	if remoteAddr != nil {
+	var remoteAddr *net.UDPAddr
+
+	if params.Outbound {
+		remoteAddr = makeUDPAddr(params.RemoteAddr)
 		// The provided address contains the main weave port
 		// number to connect to.  We need to derive the vxlan
 		// port number from that.
-		vxlanRemoteAddr := *params.RemoteAddr
+		vxlanRemoteAddr := *remoteAddr
 		vxlanRemoteAddr.Port++
 		remoteAddr = &vxlanRemoteAddr
-
 		var err error
 		vxlanVportID, err = fastdp.getVxlanVportID(remoteAddr.Port)
 		if err != nil {
@@ -543,7 +539,7 @@ func (fastdp fastDatapathOverlay) MakeForwarder(
 		}
 	}
 
-	localIP, err := ipv4Bytes(params.LocalIP)
+	localIP, err := ipv4Bytes(params.LocalAddr.IP)
 	if err != nil {
 		return nil, err
 	}
@@ -662,7 +658,7 @@ func (fwd *fastDatapathForwarder) sendHeartbeat() {
 
 	// the heartbeat payload consists of the 64-bit connection uid
 	// followed by the 16-bit packet size.
-	buf := make([]byte, EthernetOverhead+fwd.fastdp.mtu)
+	buf := make([]byte, EthernetOverhead+fwd.fastdp.iface.MTU)
 	binary.BigEndian.PutUint64(buf[EthernetOverhead:], fwd.connUID)
 	binary.BigEndian.PutUint16(buf[EthernetOverhead+8:], uint16(len(buf)))
 
@@ -674,15 +670,17 @@ func (fwd *fastDatapathForwarder) sendHeartbeat() {
 		DstPeer:   fwd.remotePeer,
 	}
 	fwd.lock.RUnlock()
-	fwd.Forward(pk).Process(buf, dec, false)
+
+	if fop := fwd.Forward(pk); fop != nil {
+		fop.Process(buf, dec, false)
+	}
 }
 
 const (
 	FastDatapathHeartbeatAck = iota
 )
 
-func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte,
-	sender *net.UDPAddr) {
+func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte, sender *net.UDPAddr) {
 	fwd.lock.Lock()
 	defer fwd.lock.Unlock()
 
@@ -706,8 +704,7 @@ func (fwd *fastDatapathForwarder) handleVxlanSpecialPacket(frame []byte,
 			fwd.heartbeatTimer.Reset(0)
 		}
 	} else if !udpAddrsEqual(fwd.remoteAddr, sender) {
-		log.Info(fwd.logPrefix(),
-			"Peer IP address changed to ", sender)
+		log.Info(fwd.logPrefix(), "Peer IP address changed to ", sender)
 		fwd.remoteAddr = sender
 	}
 
@@ -732,8 +729,7 @@ func (fwd *fastDatapathForwarder) ControlMessage(tag byte, msg []byte) {
 		fwd.handleHeartbeatAck()
 
 	default:
-		log.Info(fwd.logPrefix(),
-			"Ignoring unknown control message: ", tag)
+		log.Info(fwd.logPrefix(), "Ignoring unknown control message: ", tag)
 	}
 }
 
@@ -762,10 +758,10 @@ func (fwd *fastDatapathForwarder) Forward(key ForwardPacketKey) FlowOp {
 	defer fwd.lock.RUnlock()
 
 	if fwd.remoteAddr == nil {
-		// Returning nil would discard the packet, but also
-		// result in a flow rule, which we would have to
-		// invalidate when we learn the remote IP.  So for
-		// now, just prevent flows.
+		// Returning a DiscardingFlowOp would discard the
+		// packet, but also result in a flow rule, which we
+		// would have to invalidate when we learn the remote
+		// IP.  So for now, just prevent flows.
 		return vetoFlowCreationFlowOp{}
 	}
 
@@ -809,8 +805,7 @@ func (fwd *fastDatapathForwarder) Stop() {
 	}
 }
 
-func (fastdp *FastDatapath) addForwarder(peer PeerName,
-	fwd *fastDatapathForwarder) {
+func (fastdp *FastDatapath) addForwarder(peer mesh.PeerName, fwd *fastDatapathForwarder) {
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()
 
@@ -819,8 +814,7 @@ func (fastdp *FastDatapath) addForwarder(peer PeerName,
 	fastdp.forwarders[peer] = fwd
 }
 
-func (fastdp *FastDatapath) removeForwarder(peer PeerName,
-	fwd *fastDatapathForwarder) {
+func (fastdp *FastDatapath) removeForwarder(peer mesh.PeerName, fwd *fastDatapathForwarder) {
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()
 	if fastdp.forwarders[peer] == fwd {
@@ -899,9 +893,7 @@ func (fastdp *FastDatapath) expireFlows() {
 	defer lock.unlock()
 
 	flows, err := fastdp.dp.EnumerateFlows()
-	if err != nil {
-		log.Warn(err)
-	}
+	checkWarn(err)
 
 	for _, flow := range flows {
 		if flow.Used == 0 {
@@ -922,8 +914,7 @@ func (fastdp *FastDatapath) expireFlows() {
 // maintain its MAC->peer table.  We do this by querying the router
 // without an actual packet being involved.  Maybe it's
 // worth devising a more unified approach in the future.
-func (fastdp *FastDatapath) touchFlow(fks odp.FlowKeys,
-	lock *fastDatapathLock) {
+func (fastdp *FastDatapath) touchFlow(fks odp.FlowKeys, lock *fastDatapathLock) {
 	// All the flows we create should have an ingress key, but we
 	// check here just in case we encounter one from somewhere
 	// else.
@@ -953,6 +944,8 @@ func (fastdp *FastDatapath) Miss(packet []byte, fks odp.FlowKeys) error {
 	lock := fastdp.startLock()
 	defer lock.unlock()
 
+	fastdp.missCount++
+
 	handler := fastdp.getMissHandler(ingress)
 	if handler == nil {
 		return nil
@@ -963,8 +956,7 @@ func (fastdp *FastDatapath) Miss(packet []byte, fks odp.FlowKeys) error {
 	// delivery to a local netdev based on the dest MAC),
 	// including the ingress in every flow makes things simpler
 	// in touchFlow.
-	mfop := NewMultiFlowOp(false, handler(fks, &lock),
-		odpFlowKey(odp.NewInPortFlowKey(ingress)))
+	mfop := NewMultiFlowOp(false, handler(fks, &lock), odpFlowKey(odp.NewInPortFlowKey(ingress)))
 	fastdp.send(mfop, packet, &lock)
 	return nil
 }
@@ -1032,8 +1024,7 @@ func (fastdp *FastDatapath) makeBridgeVport(vport odp.Vport) {
 
 	// Packets coming from the netdev are processed by the bridge
 	fastdp.missHandlers[vportID] = func(flowKeys odp.FlowKeys, lock *fastDatapathLock) FlowOp {
-		return fastdp.bridge(bridgePortID{vport: vportID},
-			flowKeysToPacketKey(flowKeys), lock)
+		return fastdp.bridge(bridgePortID{vport: vportID}, flowKeysToPacketKey(flowKeys), lock)
 	}
 }
 
@@ -1044,8 +1035,7 @@ func flowKeysToPacketKey(fks odp.FlowKeys) PacketKey {
 
 // The sendToPort map is read-only, so this method does the copy in
 // order to add an entry.
-func (fastdp *FastDatapath) addSendToPort(portID bridgePortID,
-	sender bridgeSender) {
+func (fastdp *FastDatapath) addSendToPort(portID bridgePortID, sender bridgeSender) {
 	sendToPort := map[bridgePortID]bridgeSender{portID: sender}
 	for id, sender := range fastdp.sendToPort {
 		sendToPort[id] = sender
@@ -1064,8 +1054,7 @@ func (fastdp *FastDatapath) deleteSendToPort(portID bridgePortID) {
 }
 
 // Send a packet, creating a corresponding ODP flow rule if possible
-func (fastdp *FastDatapath) send(fops FlowOp, frame []byte,
-	lock *fastDatapathLock) {
+func (fastdp *FastDatapath) send(fops FlowOp, frame []byte, lock *fastDatapathLock) {
 	// Gather the actions from actionFlowOps, execute any others
 	var dec *EthernetDecoder
 	flow := odp.NewFlowSpec()
@@ -1080,6 +1069,10 @@ func (fastdp *FastDatapath) send(fops FlowOp, frame []byte,
 		case vetoFlowCreationFlowOp:
 			createFlow = false
 		default:
+			if xfop.Discards() {
+				continue
+			}
+
 			// A foreign FlowOp (e.g. a sleeve forwarding
 			// FlowOp), so send the packet through the
 			// FlowOp interface, decoding the packet
@@ -1157,8 +1150,7 @@ func (fop odpActionsFlowOp) updateFlowSpec(flow *odp.FlowSpec) {
 	flow.AddActions(fop.actions)
 }
 
-func (fop odpActionsFlowOp) Process(frame []byte, dec *EthernetDecoder,
-	broadcast bool) {
+func (fop odpActionsFlowOp) Process(frame []byte, dec *EthernetDecoder, broadcast bool) {
 	fastdp := fop.fastdp
 	fastdp.lock.Lock()
 	defer fastdp.lock.Unlock()

router/flow.go

@@ -1,6 +1,10 @@
 package router
 
-import "net"
+import (
+	"net"
+
+	"github.com/weaveworks/mesh"
+)
 
 // Just enough flow machinery for the weave router
 
@@ -16,8 +20,8 @@ type PacketKey struct {
 }
 
 type ForwardPacketKey struct {
-	SrcPeer *Peer
-	DstPeer *Peer
+	SrcPeer *mesh.Peer
+	DstPeer *mesh.Peer
 	PacketKey
 }
 
@@ -64,8 +68,7 @@ func (mfop *MultiFlowOp) Add(op FlowOp) {
 	mfop.ops = append(mfop.ops, op)
 }
 
-func (mfop *MultiFlowOp) Process(frame []byte, dec *EthernetDecoder,
-	broadcast bool) {
+func (mfop *MultiFlowOp) Process(frame []byte, dec *EthernetDecoder, broadcast bool) {
 	for _, op := range mfop.ops {
 		op.Process(frame, dec, mfop.broadcast)
 	}
@@ -81,15 +84,12 @@ func (mfop *MultiFlowOp) Discards() bool {
 	return true
 }
 
+// Flatten out a FlowOp to eliminate any MultiFlowOps
 func FlattenFlowOp(fop FlowOp) []FlowOp {
 	return collectFlowOps(nil, fop)
 }
 
 func collectFlowOps(into []FlowOp, fop FlowOp) []FlowOp {
-	if fop == nil {
-		return into
-	}
-
 	if mfop, ok := fop.(*MultiFlowOp); ok {
 		for _, op := range mfop.ops {
 			into = collectFlowOps(into, op)

router/frame_crypto.go

@@ -1,32 +1,13 @@
 package router
 
 import (
-	"crypto/rand"
-	"crypto/sha256"
 	"encoding/binary"
-	"encoding/gob"
 	"fmt"
-	"io"
-	"sync"
 
 	"github.com/andybalholm/go-bit"
-	"golang.org/x/crypto/nacl/box"
 	"golang.org/x/crypto/nacl/secretbox"
 )
 
-func GenerateKeyPair() (publicKey, privateKey *[32]byte, err error) {
-	return box.GenerateKey(rand.Reader)
-}
-
-func FormSessionKey(remotePublicKey, localPrivateKey *[32]byte, secretKey []byte) *[32]byte {
-	var sharedKey [32]byte
-	box.Precompute(&sharedKey, remotePublicKey, localPrivateKey)
-	sharedKeySlice := sharedKey[:]
-	sharedKeySlice = append(sharedKeySlice, secretKey...)
-	sessionKey := sha256.Sum256(sharedKeySlice)
-	return &sessionKey
-}
-
 // Frame Encryptors
 
 type Encryptor interface {
@@ -180,6 +161,10 @@ type PacketDecodingError struct {
 	Desc string
 }
 
+func (pde PacketDecodingError) Error() string {
+	return fmt.Sprint("Failed to decode packet: ", pde.Desc)
+}
+
 func NewNonDecryptor() *NonDecryptor {
 	return &NonDecryptor{}
 }
@@ -297,154 +282,3 @@ func (di *NaClDecryptorInstance) advanceState(seqNo uint64) (int, *bit.Set) {
 		return offset, di.usedOffsets
 	}
 }
-
-// TCP Senders/Receivers
-
-// The lowest 64 bits of the nonce contain the message sequence
-// number. The top most bit indicates the connection polarity at the
-// sender - '1' for outbound; the next indicates protocol type - '1'
-// for TCP. The remaining 126 bits are zero. The polarity is needed so
-// that the two ends of a connection do not use the same nonces; the
-// protocol type so that the TCP and UDP sender nonces are disjoint.
-// This is a requirement of the NaCl Security Model; see
-// http://nacl.cr.yp.to/box.html.
-type TCPCryptoState struct {
-	sessionKey *[32]byte
-	nonce      [24]byte
-	seqNo      uint64
-}
-
-func NewTCPCryptoState(sessionKey *[32]byte, outbound bool) *TCPCryptoState {
-	s := &TCPCryptoState{sessionKey: sessionKey}
-	if outbound {
-		s.nonce[0] |= (1 << 7)
-	}
-	// Ensure that TCP and UDP sequences are disjoint
-	s.nonce[0] |= (1 << 6)
-	return s
-}
-
-func (s *TCPCryptoState) advance() {
-	s.seqNo++
-	binary.BigEndian.PutUint64(s.nonce[16:24], s.seqNo)
-}
-
-type TCPSender interface {
-	Send([]byte) error
-}
-
-type GobTCPSender struct {
-	encoder *gob.Encoder
-}
-
-type LengthPrefixTCPSender struct {
-	writer io.Writer
-}
-
-type EncryptedTCPSender struct {
-	sync.RWMutex
-	sender TCPSender
-	state  *TCPCryptoState
-}
-
-func NewGobTCPSender(encoder *gob.Encoder) *GobTCPSender {
-	return &GobTCPSender{encoder: encoder}
-}
-
-func (sender *GobTCPSender) Send(msg []byte) error {
-	return sender.encoder.Encode(msg)
-}
-
-func NewLengthPrefixTCPSender(writer io.Writer) *LengthPrefixTCPSender {
-	return &LengthPrefixTCPSender{writer: writer}
-}
-
-func (sender *LengthPrefixTCPSender) Send(msg []byte) error {
-	l := len(msg)
-	if l > MaxTCPMsgSize {
-		return fmt.Errorf("outgoing message exceeds maximum size: %d > %d", l, MaxTCPMsgSize)
-	}
-	// We copy the message so we can send it in a single Write
-	// operation, thus making this thread-safe without locking.
-	prefixedMsg := make([]byte, 4+l)
-	binary.BigEndian.PutUint32(prefixedMsg, uint32(l))
-	copy(prefixedMsg[4:], msg)
-	_, err := sender.writer.Write(prefixedMsg)
-	return err
-}
-
-func NewEncryptedTCPSender(sender TCPSender, sessionKey *[32]byte, outbound bool) *EncryptedTCPSender {
-	return &EncryptedTCPSender{sender: sender, state: NewTCPCryptoState(sessionKey, outbound)}
-}
-
-func (sender *EncryptedTCPSender) Send(msg []byte) error {
-	sender.Lock()
-	defer sender.Unlock()
-	encodedMsg := secretbox.Seal(nil, msg, &sender.state.nonce, sender.state.sessionKey)
-	sender.state.advance()
-	return sender.sender.Send(encodedMsg)
-}
-
-type TCPReceiver interface {
-	Receive() ([]byte, error)
-}
-
-type GobTCPReceiver struct {
-	decoder *gob.Decoder
-}
-
-type LengthPrefixTCPReceiver struct {
-	reader io.Reader
-}
-
-type EncryptedTCPReceiver struct {
-	receiver TCPReceiver
-	state    *TCPCryptoState
-}
-
-func NewGobTCPReceiver(decoder *gob.Decoder) *GobTCPReceiver {
-	return &GobTCPReceiver{decoder: decoder}
-}
-
-func (receiver *GobTCPReceiver) Receive() ([]byte, error) {
-	var msg []byte
-	err := receiver.decoder.Decode(&msg)
-	return msg, err
-}
-
-func NewLengthPrefixTCPReceiver(reader io.Reader) *LengthPrefixTCPReceiver {
-	return &LengthPrefixTCPReceiver{reader: reader}
-}
-
-func (receiver *LengthPrefixTCPReceiver) Receive() ([]byte, error) {
-	lenPrefix := make([]byte, 4)
-	if _, err := io.ReadFull(receiver.reader, lenPrefix); err != nil {
-		return nil, err
-	}
-	l := binary.BigEndian.Uint32(lenPrefix)
-	if l > MaxTCPMsgSize {
-		return nil, fmt.Errorf("incoming message exceeds maximum size: %d > %d", l, MaxTCPMsgSize)
-	}
-	msg := make([]byte, l)
-	_, err := io.ReadFull(receiver.reader, msg)
-	return msg, err
-}
-
-func NewEncryptedTCPReceiver(receiver TCPReceiver, sessionKey *[32]byte, outbound bool) *EncryptedTCPReceiver {
-	return &EncryptedTCPReceiver{receiver: receiver, state: NewTCPCryptoState(sessionKey, !outbound)}
-}
-
-func (receiver *EncryptedTCPReceiver) Receive() ([]byte, error) {
-	msg, err := receiver.receiver.Receive()
-	if err != nil {
-		return nil, err
-	}
-
-	decodedMsg, success := secretbox.Open(nil, msg, &receiver.state.nonce, receiver.state.sessionKey)
-	if !success {
-		return nil, fmt.Errorf("Unable to decrypt TCP msg")
-	}
-
-	receiver.state.advance()
-	return decodedMsg, nil
-}